From d1db4d6d723dd7d72a12d7b17c99f9a8fa7973ff Mon Sep 17 00:00:00 2001 From: Gianluca Mardente Date: Mon, 20 Jul 2026 08:39:10 +0200 Subject: [PATCH] (feat) outdated helm charts Sveltos can now tell you when a Helm chart deployed through a ClusterProfile or Profile has a newer version available upstream. This is detection only; nothing about what gets deployed changes. Once a chart has been successfully deployed to a cluster, Sveltos now also remembers exactly which chart, which repository, and which credentials were actually used for that specific deployment. That information used to only exist transiently at deploy time. Now it's kept around per cluster so it doesn't need to be re-derived later, which matters because chart names, versions, and repository locations can be templated and resolve differently per cluster. A background process, running only on the main instance (not on sharded instances, since it needs visibility into every cluster to avoid duplicate work), periodically walks every currently deployed chart, groups together deployments that are actually the same chart from the same source, and checks each one just once against its upstream repository or registry to see what versions are actually published. For each deployment it then knows two things: whether a newer version exists at all, and whether a newer patch release exists within the same minor version line. Any repository or registry that can't be reached is skipped and logged rather than stopping the whole check, and each check has a timeout so one slow or unresponsive source can't hold up the rest. The result is recorded directly on the same per-cluster object that already tracks that chart's deployment status, so it's visible per cluster rather than as one blended answer across a whole fleet. A metric exposes which deployments are currently behind, and it's kept accurate throughout each check. Two additional small metrics expose the health of the checker itsel: when it last finished a run, and how many checks failed. So a broken or stuck checker is visible rather than silently indistinguishable from "everything is up to date." The check interval is configurable and can be turned off entirely. --- api/v1beta1/clustersummary_types.go | 37 ++ api/v1beta1/zz_generated.deepcopy.go | 24 +- cmd/main.go | 68 ++-- ...ig.projectsveltos.io_clustersummaries.yaml | 45 +++ controllers/clustersummary_predicates_test.go | 65 +++ controllers/handlers_helm.go | 86 +++- controllers/helmchart_outdated_check.go | 373 ++++++++++++++++++ controllers/helmchart_outdated_check_test.go | 236 +++++++++++ controllers/helmchart_semver.go | 69 ++++ controllers/helmchart_semver_test.go | 127 ++++++ controllers/helmchart_version_source.go | 225 +++++++++++ controllers/metrics.go | 107 ++++- manifest/manifest.yaml | 45 +++ 13 files changed, 1475 insertions(+), 32 deletions(-) create mode 100644 controllers/helmchart_outdated_check.go create mode 100644 controllers/helmchart_outdated_check_test.go create mode 100644 controllers/helmchart_semver.go create mode 100644 controllers/helmchart_semver_test.go create mode 100644 controllers/helmchart_version_source.go diff --git a/api/v1beta1/clustersummary_types.go b/api/v1beta1/clustersummary_types.go index a41f5b60..d318e6d2 100644 --- a/api/v1beta1/clustersummary_types.go +++ b/api/v1beta1/clustersummary_types.go @@ -129,6 +129,43 @@ type HelmChartSummary struct { // +optional // FailureMessage provides the specific error from the Helm engine for this release FailureMessage *string `json:"failureMessage,omitempty"` + + // ChartName, RepositoryName, RepoURL, and ChartVersion mirror the fully resolved + // (post-template) HelmChart entry that produced this release, captured at deploy time. + // Never set for Flux-source-backed charts (Flux owns version resolution there). + // +optional + ChartName string `json:"chartName,omitempty"` + + // +optional + RepositoryName string `json:"repositoryName,omitempty"` + + // +optional + RepoURL string `json:"repoURL,omitempty"` + + // +optional + ChartVersion string `json:"chartVersion,omitempty"` + + // CredentialsSecretRef is the resolved secret reference (if any) used to authenticate + // against RepoURL, captured at deploy time. Only the reference is stored, never secret + // contents. + // +optional + CredentialsSecretRef *corev1.SecretReference `json:"credentialsSecretRef,omitempty"` + + // LatestVersion is the highest version currently published upstream for this chart, if + // greater than ChartVersion. Populated by a periodic background check, independent of + // the reconcile loop. Detection only: Sveltos never mutates ChartVersion based on this. + // +optional + LatestVersion *string `json:"latestVersion,omitempty"` + + // LatestPatchVersion is the highest published version sharing ChartVersion's + // major.minor, if greater than ChartVersion. Distinguishes "a same-minor patch bump is + // available" from "a newer minor/major line exists" (LatestVersion). + // +optional + LatestPatchVersion *string `json:"latestPatchVersion,omitempty"` + + // LastCheckedTime is when LatestVersion/LatestPatchVersion were last evaluated. + // +optional + LastCheckedTime *metav1.Time `json:"lastCheckedTime,omitempty"` } // ClusterSummarySpec defines the desired state of ClusterSummary diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 7367cd4f..ae5338bc 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -21,11 +21,12 @@ limitations under the License. package v1beta1 import ( - apiv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" + + apiv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -867,6 +868,25 @@ func (in *HelmChartSummary) DeepCopyInto(out *HelmChartSummary) { *out = new(string) **out = **in } + if in.CredentialsSecretRef != nil { + in, out := &in.CredentialsSecretRef, &out.CredentialsSecretRef + *out = new(corev1.SecretReference) + **out = **in + } + if in.LatestVersion != nil { + in, out := &in.LatestVersion, &out.LatestVersion + *out = new(string) + **out = **in + } + if in.LatestPatchVersion != nil { + in, out := &in.LatestPatchVersion, &out.LatestPatchVersion + *out = new(string) + **out = **in + } + if in.LastCheckedTime != nil { + in, out := &in.LastCheckedTime, &out.LastCheckedTime + *out = (*in).DeepCopy() + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HelmChartSummary. diff --git a/cmd/main.go b/cmd/main.go index 91142bba..db9ecef7 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -75,33 +75,34 @@ import ( ) var ( - setupLog = ctrl.Log.WithName("setup") - diagnosticsAddress string - insecureDiagnostics bool - shardKey string - workers int - concurrentReconciles int - agentInMgmtCluster bool - reportMode controllers.ReportMode - tmpReportMode int - restConfigQPS float32 - restConfigBurst int - webhookPort int - syncPeriod time.Duration - conflictRetryTime time.Duration - healthErrorRetryTime time.Duration - version string - healthAddr string - profilerAddress string - driftDetectionConfigMap string - luaConfigMap string - capiOnboardAnnotation string - disableCaching bool - disableTelemetry bool - autoDeployDependencies bool - registry string - luaCallStackSize int - luaRegistrySize int + setupLog = ctrl.Log.WithName("setup") + diagnosticsAddress string + insecureDiagnostics bool + shardKey string + workers int + concurrentReconciles int + agentInMgmtCluster bool + reportMode controllers.ReportMode + tmpReportMode int + restConfigQPS float32 + restConfigBurst int + webhookPort int + syncPeriod time.Duration + conflictRetryTime time.Duration + healthErrorRetryTime time.Duration + version string + healthAddr string + profilerAddress string + driftDetectionConfigMap string + luaConfigMap string + capiOnboardAnnotation string + disableCaching bool + disableTelemetry bool + autoDeployDependencies bool + registry string + luaCallStackSize int + luaRegistrySize int + helmChartUpdateCheckInterval time.Duration ) const ( @@ -297,6 +298,10 @@ func initFlags(fs *pflag.FlagSet) { fmt.Sprintf("The minimum interval at which health check failures are retried. Default: %d seconds", defaultHealthErrorRetryTime)) + fs.DurationVar(&helmChartUpdateCheckInterval, "helm-chart-update-check-interval", time.Hour, + "Interval at which Sveltos checks whether newer versions of deployed Helm charts have been "+ + "published upstream (HTTP repositories and OCI registries). Set to 0 to disable.") + // AutoDeployDependencies enables automatic deployment of prerequisite profiles. // // Profile instances can specify dependencies on other profiles using the @@ -671,6 +676,15 @@ func startControllersAndWatchers(ctx context.Context, mgr manager.Manager) { setupLog.Error(err, "unable to create controller", "controller", "ClusterPromotion") os.Exit(1) } + + // Needs a fleet-wide view of every ClusterSummary to dedup chart keys correctly, so + // this only ever runs on the default (unsharded) deployment, same as the reconcilers + // started above. Running it per-shard would give no benefit (chart-key dedup is + // inherently fleet-wide) and would reintroduce cross-shard inconsistency. + if helmChartUpdateCheckInterval > 0 { + go controllers.RunHelmChartUpdateChecker(ctx, mgr.GetClient(), helmChartUpdateCheckInterval, + ctrl.Log.WithName("helm-chart-update-checker")) + } } clusterSummaryReconciler := getClusterSummaryReconciler(ctx, mgr) diff --git a/config/crd/bases/config.projectsveltos.io_clustersummaries.yaml b/config/crd/bases/config.projectsveltos.io_clustersummaries.yaml index eefd82a9..495c5e95 100644 --- a/config/crd/bases/config.projectsveltos.io_clustersummaries.yaml +++ b/config/crd/bases/config.projectsveltos.io_clustersummaries.yaml @@ -2262,15 +2262,56 @@ spec: directly managed by ClusterProfile. items: properties: + chartName: + description: |- + ChartName, RepositoryName, RepoURL, and ChartVersion mirror the fully resolved + (post-template) HelmChart entry that produced this release, captured at deploy time. + Never set for Flux-source-backed charts (Flux owns version resolution there). + type: string + chartVersion: + type: string conflictMessage: description: |- Status indicates whether ClusterSummary can manage the helm chart or there is a conflict type: string + credentialsSecretRef: + description: |- + CredentialsSecretRef is the resolved secret reference (if any) used to authenticate + against RepoURL, captured at deploy time. Only the reference is stored, never secret + contents. + properties: + name: + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which the + secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic failureMessage: description: FailureMessage provides the specific error from the Helm engine for this release type: string + lastCheckedTime: + description: LastCheckedTime is when LatestVersion/LatestPatchVersion + were last evaluated. + format: date-time + type: string + latestPatchVersion: + description: |- + LatestPatchVersion is the highest published version sharing ChartVersion's + major.minor, if greater than ChartVersion. Distinguishes "a same-minor patch bump is + available" from "a newer minor/major line exists" (LatestVersion). + type: string + latestVersion: + description: |- + LatestVersion is the highest version currently published upstream for this chart, if + greater than ChartVersion. Populated by a periodic background check, independent of + the reconcile loop. Detection only: Sveltos never mutates ChartVersion based on this. + type: string patchesHash: description: PatchesHash represents of a unique value for the patches section @@ -2285,6 +2326,10 @@ spec: be installed minLength: 1 type: string + repoURL: + type: string + repositoryName: + type: string status: description: |- Status indicates whether ClusterSummary can manage the helm diff --git a/controllers/clustersummary_predicates_test.go b/controllers/clustersummary_predicates_test.go index 3fbfae89..c4022d9e 100644 --- a/controllers/clustersummary_predicates_test.go +++ b/controllers/clustersummary_predicates_test.go @@ -31,6 +31,7 @@ import ( "k8s.io/klog/v2/textlogger" "sigs.k8s.io/controller-runtime/pkg/event" + configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" "github.com/projectsveltos/addon-controller/controllers" libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" ) @@ -338,3 +339,67 @@ var _ = Describe("ClusterProfile Predicates: FluxSourcePredicates", func() { Expect(result).To(BeFalse()) }) }) + +var _ = Describe("Clustersummary Predicates: ClusterSummaryPredicate", func() { + var logger logr.Logger + var clusterSummary *configv1beta1.ClusterSummary + + BeforeEach(func() { + logger = textlogger.NewLogger(textlogger.NewConfig()) + clusterSummary = &configv1beta1.ClusterSummary{ + ObjectMeta: metav1.ObjectMeta{ + Name: randomString(), + }, + Status: configv1beta1.ClusterSummaryStatus{ + HelmReleaseSummaries: []configv1beta1.HelmChartSummary{ + { + ReleaseName: randomString(), + ReleaseNamespace: randomString(), + Status: configv1beta1.HelmChartStatusManaging, + ChartName: randomString(), + RepositoryName: randomString(), + RepoURL: testRepoURLBitnami, + ChartVersion: testChartVersion100, + }, + }, + }, + } + }) + + It("Update returns false when only LatestVersion/LatestPatchVersion/LastCheckedTime change", func() { + clusterSummaryPredicate := controllers.ClusterSummaryPredicate{Logger: logger} + + oldClusterSummary := clusterSummary.DeepCopy() + + newLatest := "1.1.0" + lastChecked := metav1.Now() + clusterSummary.Status.HelmReleaseSummaries[0].LatestVersion = &newLatest + clusterSummary.Status.HelmReleaseSummaries[0].LastCheckedTime = &lastChecked + + e := event.UpdateEvent{ + ObjectNew: clusterSummary, + ObjectOld: oldClusterSummary, + } + + result := clusterSummaryPredicate.Update(e) + Expect(result).To(BeFalse()) + }) + + It("Update returns true when FeatureSummaries change", func() { + clusterSummaryPredicate := controllers.ClusterSummaryPredicate{Logger: logger} + + oldClusterSummary := clusterSummary.DeepCopy() + + clusterSummary.Status.FeatureSummaries = []configv1beta1.FeatureSummary{ + {FeatureID: libsveltosv1beta1.FeatureHelm, Status: libsveltosv1beta1.FeatureStatusProvisioned}, + } + + e := event.UpdateEvent{ + ObjectNew: clusterSummary, + ObjectOld: oldClusterSummary, + } + + result := clusterSummaryPredicate.Update(e) + Expect(result).To(BeTrue()) + }) +}) diff --git a/controllers/handlers_helm.go b/controllers/handlers_helm.go index 43206d12..8c2e4321 100644 --- a/controllers/handlers_helm.go +++ b/controllers/handlers_helm.go @@ -1325,7 +1325,7 @@ func deploySingleChart(ctx context.Context, c client.Client, dCtx *deploymentCon return nil, err } - valueHash, err := updateValueHashOnHelmChartSummary(ctx, instantiatedChart, dCtx, logger) + valueHash, err := updateValueHashOnHelmChartSummary(ctx, instantiatedChart, currentRelease, dCtx, logger) if err != nil { return nil, err } @@ -4656,7 +4656,7 @@ func getHelmChartValuesHash(ctx context.Context, c client.Client, instantiatedCh } func updateValueHashOnHelmChartSummary(ctx context.Context, requestedChart *configv1beta1.HelmChart, - dCtx *deploymentContext, logger logr.Logger) ([]byte, error) { + currentRelease *releaseInfo, dCtx *deploymentContext, logger logr.Logger) ([]byte, error) { c := getManagementClusterClient() @@ -4679,6 +4679,7 @@ func updateValueHashOnHelmChartSummary(ctx context.Context, requestedChart *conf rs.ReleaseNamespace == requestedChart.ReleaseNamespace { rs.ValuesHash = helmChartValuesHash + setResolvedHelmChartIdentity(ctx, c, dCtx.clusterSummary, rs, requestedChart, currentRelease, logger) } } @@ -4688,6 +4689,87 @@ func updateValueHashOnHelmChartSummary(ctx context.Context, requestedChart *conf return helmChartValuesHash, err } +// setResolvedHelmChartIdentity records, on rs, the fully resolved (post-template) chart +// identity and credentials used to produce this release, plus the actually deployed chart +// version. This lets a periodic, unrelated checker (see helmchart_outdated_check.go) compare +// against the upstream repository/registry without ever having to resolve Go templates +// itself. Never set for Flux-source-backed charts, since Sveltos ignores ChartVersion and +// leaves version resolution to Flux in that case. +func setResolvedHelmChartIdentity(ctx context.Context, c client.Client, clusterSummary *configv1beta1.ClusterSummary, + rs *configv1beta1.HelmChartSummary, requestedChart *configv1beta1.HelmChart, currentRelease *releaseInfo, + logger logr.Logger) { + + if isReferencingFluxSource(requestedChart) { + return + } + + previousChartName := rs.ChartName + previousChartVersion := rs.ChartVersion + + rs.ChartName = requestedChart.ChartName + rs.RepositoryName = requestedChart.RepositoryName + rs.RepoURL = requestedChart.RepositoryURL + + if currentRelease != nil { + rs.ChartVersion = currentRelease.ChartVersion + } + + if rs.ChartVersion != previousChartVersion { + clearStaleOutdatedVersionInfo(clusterSummary, rs, previousChartName, logger) + } + + rs.CredentialsSecretRef = nil + if requestedChart.RegistryCredentialsConfig != nil && requestedChart.RegistryCredentialsConfig.CredentialsSecretRef != nil { + // CredentialsSecretRef.Namespace may still be empty here (meaning "implicit cluster + // namespace", resolved on demand wherever this is used for a deploy, e.g. + // createRegistryClientOptions). Resolve it to a concrete namespace now, while + // clusterSummary is in scope, so a later, cluster-agnostic checker never needs to + // repeat this resolution. + secretRef := requestedChart.RegistryCredentialsConfig.CredentialsSecretRef + resolvedNamespace, err := libsveltostemplate.GetReferenceResourceNamespace(ctx, c, + clusterSummary.Spec.ClusterNamespace, clusterSummary.Spec.ClusterName, + secretRef.Namespace, clusterSummary.Spec.ClusterType) + if err != nil { + logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to resolve credentials secret namespace: %v", err)) + } else { + rs.CredentialsSecretRef = &corev1.SecretReference{ + Namespace: resolvedNamespace, + Name: secretRef.Name, + } + } + } +} + +// clearStaleOutdatedVersionInfo nils out rs's LatestVersion/LatestPatchVersion/LastCheckedTime +// and, if they were set, deletes the matching outdatedHelmChartGauge series. Called whenever a +// deploy changes ChartVersion: the periodic checker (helmchart_outdated_check.go) only +// re-evaluates a chart on its own interval, so without this a chart just upgraded to what was +// previously flagged as the latest version would keep reading as outdated - in both status and +// the metric - until the next pass. +func clearStaleOutdatedVersionInfo(clusterSummary *configv1beta1.ClusterSummary, rs *configv1beta1.HelmChartSummary, + previousChartName string, logger logr.Logger) { + + if rs.LatestVersion != nil || rs.LatestPatchVersion != nil { + profileOwnerRef, err := configv1beta1.GetProfileOwnerReference(clusterSummary) + if err != nil { + logger.V(logs.LogInfo).Info( + fmt.Sprintf("failed to get profile owner reference while clearing stale outdated-version info: %v", err)) + } else { + profileNamespace := "" + if profileOwnerRef.Kind == configv1beta1.ProfileKind { + profileNamespace = clusterSummary.Namespace + } + clearOutdatedHelmChart(profileOwnerRef.Kind, profileNamespace, profileOwnerRef.Name, + string(clusterSummary.Spec.ClusterType), clusterSummary.Spec.ClusterNamespace, clusterSummary.Spec.ClusterName, + previousChartName, rs.ReleaseNamespace, rs.ReleaseName) + } + } + + rs.LatestVersion = nil + rs.LatestPatchVersion = nil + rs.LastCheckedTime = nil +} + func getHelmChartPatchesHash(ctx context.Context, clusterSummary *configv1beta1.ClusterSummary, logger logr.Logger) ([]byte, error) { diff --git a/controllers/helmchart_outdated_check.go b/controllers/helmchart_outdated_check.go new file mode 100644 index 00000000..345b97ef --- /dev/null +++ b/controllers/helmchart_outdated_check.go @@ -0,0 +1,373 @@ +/* +Copyright 2025. projectsveltos.io. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + + configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" + logs "github.com/projectsveltos/libsveltos/lib/logsettings" +) + +const ( + // maxConcurrentHelmChartFetches bounds how many distinct chart repositories/registries are + // queried at once, so peak transient memory (index.yaml parsing) stays bounded to a handful + // of large indexes in flight rather than every distinct repo in the fleet at once. + maxConcurrentHelmChartFetches = 5 +) + +// helmChartRef records one HelmChartSummary entry, on one ClusterSummary, that references a +// given helmChartKey. Multiple ClusterSummary entries (across clusters and/or profiles) can +// point at the same key; they are fanned back out to after a single upstream fetch. Profile +// and cluster identity are captured here, at collection time, rather than re-derived later, +// since deriving profileKind/profileName requires the full ClusterSummary object (its +// OwnerReferences), not just its namespace/name. +type helmChartRef struct { + clusterSummaryNamespace string + clusterSummaryName string + releaseNamespace string + releaseName string + currentVersion string + credentialsSecretRef *corev1.SecretReference + + profileKind string + profileNamespace string + profileName string + clusterType string + clusterNamespace string + clusterName string +} + +// outdatedHelmChartMetricKey identifies one label combination of outdatedHelmChartGauge. +// Comparable (all string fields), so it can be used as a map key by +// reconcileOutdatedHelmChartMetric to track, across passes, which combinations are currently +// exposed as outdated. +type outdatedHelmChartMetricKey struct { + profileKind, profileNamespace, profileName string + clusterType, clusterNamespace, clusterName string + chartName, releaseNamespace, releaseName string +} + +func metricKeyForRef(ref *helmChartRef, chartName string) outdatedHelmChartMetricKey { + return outdatedHelmChartMetricKey{ + profileKind: ref.profileKind, profileNamespace: ref.profileNamespace, profileName: ref.profileName, + clusterType: ref.clusterType, clusterNamespace: ref.clusterNamespace, clusterName: ref.clusterName, + chartName: chartName, releaseNamespace: ref.releaseNamespace, releaseName: ref.releaseName, + } +} + +var ( + // previouslyOutdatedHelmCharts is the set of metric label combinations exposed as outdated by + // the most recently completed checkOutdatedHelmCharts pass (or carried forward from an even + // earlier pass, for a chart that could not be re-verified — see reconcileOutdatedHelmChartMetric). + // Only ever read/written from checkOutdatedHelmCharts, which RunHelmChartUpdateChecker drives + // sequentially from a single ticker loop — passes never overlap — so no lock is needed here. + previouslyOutdatedHelmCharts = map[outdatedHelmChartMetricKey]struct{}{} +) + +// outdatedHelmChartPassState accumulates, across the concurrent workers of one +// checkOutdatedHelmCharts pass, which metric label combinations were actually re-checked this +// pass and which of those came back outdated. Both are needed to reconcile the gauge correctly: +// a combination that was outdated last pass but wasn't re-checked this pass (its fetch failed) +// must be left alone, not cleared — clearing it would be reporting "confirmed fixed" when +// really nothing was learned. +type outdatedHelmChartPassState struct { + mu sync.Mutex + currentlyOutdated map[outdatedHelmChartMetricKey]struct{} + checkedThisPass map[outdatedHelmChartMetricKey]struct{} +} + +func newOutdatedHelmChartPassState() *outdatedHelmChartPassState { + return &outdatedHelmChartPassState{ + currentlyOutdated: map[outdatedHelmChartMetricKey]struct{}{}, + checkedThisPass: map[outdatedHelmChartMetricKey]struct{}{}, + } +} + +func (s *outdatedHelmChartPassState) markChecked(key *outdatedHelmChartMetricKey, outdated bool) { + s.mu.Lock() + defer s.mu.Unlock() + + s.checkedThisPass[*key] = struct{}{} + if outdated { + s.currentlyOutdated[*key] = struct{}{} + } +} + +// checkOutdatedHelmCharts runs one full pass: list every ClusterSummary, evaluate every +// HelmChartSummary entry it manages, and update both HelmReleaseSummaries[].LatestVersion/ +// LatestPatchVersion and the outdatedHelmChartGauge metric. Any single repository/registry +// fetch failure or timeout is logged and skipped; it never aborts the pass. +func checkOutdatedHelmCharts(ctx context.Context, c client.Client, logger logr.Logger) { + list := &configv1beta1.ClusterSummaryList{} + if err := c.List(ctx, list); err != nil { + logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to list ClusterSummaries, skipping this pass: %v", err)) + return + } + + refs := map[helmChartKey][]helmChartRef{} + for i := range list.Items { + collectHelmChartRefs(&list.Items[i], refs, logger) + } + + type work struct { + key helmChartKey + keyRefs []helmChartRef + } + workCh := make(chan work, len(refs)) + for key, keyRefs := range refs { + workCh <- work{key: key, keyRefs: keyRefs} + } + close(workCh) + + numWorkers := maxConcurrentHelmChartFetches + if len(refs) < numWorkers { + numWorkers = len(refs) + } + + passState := newOutdatedHelmChartPassState() + + var wg sync.WaitGroup + for range numWorkers { + wg.Add(1) + go func() { + defer wg.Done() + for w := range workCh { + processHelmChartKey(ctx, c, w.key, w.keyRefs, passState, logger) + } + }() + } + wg.Wait() + + reconcileOutdatedHelmChartMetric(passState) + trackHelmChartCheckCompleted(logger) +} + +// reconcileOutdatedHelmChartMetric updates previouslyOutdatedHelmCharts for the next pass and +// clears the gauge series for every combination confirmed, this pass, to no longer be outdated. +// Deliberately never calls outdatedHelmChartGauge.Reset(): doing so up front would make every +// still-outdated chart briefly vanish from /metrics while later workers in this same pass are +// still catching up to it, producing a false "not outdated" reading for the gap. Combinations +// found outdated this pass were already Set(1) by trackOutdatedHelmChart as they were found; +// this only needs to handle clearing the ones that are no longer outdated. +func reconcileOutdatedHelmChartMetric(passState *outdatedHelmChartPassState) { + next := make(map[outdatedHelmChartMetricKey]struct{}, len(passState.currentlyOutdated)) + for key := range passState.currentlyOutdated { + next[key] = struct{}{} + } + + for key := range previouslyOutdatedHelmCharts { + if _, stillOutdated := passState.currentlyOutdated[key]; stillOutdated { + continue // already carried forward above + } + if _, checked := passState.checkedThisPass[key]; !checked { + // Could not re-verify this pass (its chart key's fetch failed) -- leave state + // unchanged rather than guessing, and retry on the next pass. + next[key] = struct{}{} + continue + } + clearOutdatedHelmChart(key.profileKind, key.profileNamespace, key.profileName, + key.clusterType, key.clusterNamespace, key.clusterName, + key.chartName, key.releaseNamespace, key.releaseName) + } + + previouslyOutdatedHelmCharts = next +} + +// collectHelmChartRefs walks cs.Status.HelmReleaseSummaries and records a helmChartRef for +// every entry this ClusterSummary is the authoritative manager for (Status == Managing, so a +// non-owning ClusterSummary in a Conflict never double-reports the same release) and that has +// a resolved chart identity. Entries with no identity are either never successfully deployed +// yet, or Flux-source-backed (setResolvedHelmChartIdentity never populates either case) — both +// are silently skipped, not an error. +func collectHelmChartRefs(cs *configv1beta1.ClusterSummary, refs map[helmChartKey][]helmChartRef, logger logr.Logger) { + profileKind, profileNamespace, profileName := "", "", "" + if ref, err := configv1beta1.GetProfileOwnerReference(cs); err == nil { + profileKind = ref.Kind + profileName = ref.Name + if profileKind == configv1beta1.ProfileKind { + profileNamespace = cs.Namespace + } + } else { + logger.V(logs.LogVerbose).Info(fmt.Sprintf("failed to get profile owner reference for ClusterSummary %s/%s: %v", + cs.Namespace, cs.Name, err)) + } + + for i := range cs.Status.HelmReleaseSummaries { + rs := &cs.Status.HelmReleaseSummaries[i] + + if rs.Status != configv1beta1.HelmChartStatusManaging { + continue + } + if rs.ChartName == "" || rs.RepoURL == "" || rs.ChartVersion == "" { + continue + } + + key := helmChartKey{repositoryURL: rs.RepoURL, repositoryName: rs.RepositoryName, chartName: rs.ChartName} + refs[key] = append(refs[key], helmChartRef{ + clusterSummaryNamespace: cs.Namespace, + clusterSummaryName: cs.Name, + releaseNamespace: rs.ReleaseNamespace, + releaseName: rs.ReleaseName, + currentVersion: rs.ChartVersion, + credentialsSecretRef: rs.CredentialsSecretRef, + profileKind: profileKind, + profileNamespace: profileNamespace, + profileName: profileName, + clusterType: string(cs.Spec.ClusterType), + clusterNamespace: cs.Spec.ClusterNamespace, + clusterName: cs.Spec.ClusterName, + }) + } +} + +// processHelmChartKey fetches key's published versions once, then diffs and persists the +// result against every ClusterSummary referencing it. On a fetch failure, none of keyRefs are +// marked as checked in passState, so reconcileOutdatedHelmChartMetric leaves their previous +// metric state untouched rather than treating the failure as "confirmed no longer outdated". +func processHelmChartKey(ctx context.Context, c client.Client, key helmChartKey, keyRefs []helmChartRef, + passState *outdatedHelmChartPassState, logger logr.Logger) { + + // First-seen credentials win when multiple refs for the same key disagree (a documented + // limitation: this checker has no per-cluster template context to resolve which secret is + // "correct" when they genuinely differ). + var credentialsSecretRef *corev1.SecretReference + for i := range keyRefs { + if keyRefs[i].credentialsSecretRef != nil { + credentialsSecretRef = keyRefs[i].credentialsSecretRef + break + } + } + + versions, err := fetchAvailableVersions(ctx, c, key, credentialsSecretRef, logger) + if err != nil { + logger.V(logs.LogInfo).Info(fmt.Sprintf("skipping chart %q in %q: %v", key.chartName, key.repositoryURL, err)) + recordHelmChartCheckFailure() + return + } + + for i := range keyRefs { + applyDiffAndPersist(ctx, c, &keyRefs[i], key, versions, passState, logger) + } +} + +// applyDiffAndPersist diffs ref's currentVersion against versions and, if the result differs +// from what is already stored on the corresponding HelmChartSummary entry, patches it — this +// also clears a previously outdated entry once it is no longer outdated (e.g. the profile was +// updated to pin a newer version). On a diff error (unparseable currentVersion), ref is left out +// of passState entirely, same as a fetch failure: nothing was learned, so its previous metric +// state is left untouched rather than cleared. +func applyDiffAndPersist(ctx context.Context, c client.Client, ref *helmChartRef, key helmChartKey, versions []string, + passState *outdatedHelmChartPassState, logger logr.Logger) { + + latest, latestPatch, err := helmChartVersionDiff(ref.currentVersion, versions) + if err != nil { + logger.V(logs.LogDebug).Info(fmt.Sprintf("could not diff version for %q: %v", key.chartName, err)) + return + } + + metricKey := metricKeyForRef(ref, key.chartName) + + var latestStr, latestPatchStr *string + if latest != nil { + s := latest.String() + latestStr = &s + passState.markChecked(&metricKey, true) + trackOutdatedHelmChart(ref.profileKind, ref.profileNamespace, ref.profileName, + ref.clusterType, ref.clusterNamespace, ref.clusterName, + key.chartName, ref.releaseNamespace, ref.releaseName, logger) + } else { + passState.markChecked(&metricKey, false) + } + if latestPatch != nil { + s := latestPatch.String() + latestPatchStr = &s + } + + if err := patchHelmChartSummaryVersions(ctx, c, ref, latestStr, latestPatchStr, logger); err != nil { + logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to update HelmChartSummary for %s/%s: %v", + ref.releaseNamespace, ref.releaseName, err)) + } +} + +// patchHelmChartSummaryVersions updates the LatestVersion/LatestPatchVersion/LastCheckedTime +// fields on the named ClusterSummary's matching HelmChartSummary entry, mirroring the same +// Get -> mutate -> retry.RetryOnConflict(Status().Update()) pattern already used by +// updateValueHashOnHelmChartSummary. Always writes on a successful check, even when +// latest/latestPatch come back unchanged (in particular, unchanged-and-nil, the common case of +// a chart that is already up to date) -- LastCheckedTime advancing is the only signal that the +// checker is actually reaching this chart, as opposed to having stopped or never having checked +// it, so it can't be skipped just because the outdated-version verdict didn't change. +func patchHelmChartSummaryVersions(ctx context.Context, c client.Client, ref *helmChartRef, + latest, latestPatch *string, logger logr.Logger) error { + + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + cs := &configv1beta1.ClusterSummary{} + if err := c.Get(ctx, + types.NamespacedName{Namespace: ref.clusterSummaryNamespace, Name: ref.clusterSummaryName}, cs); err != nil { + return err + } + + for i := range cs.Status.HelmReleaseSummaries { + rs := &cs.Status.HelmReleaseSummaries[i] + if rs.ReleaseName != ref.releaseName || rs.ReleaseNamespace != ref.releaseNamespace { + continue + } + + rs.LatestVersion = latest + rs.LatestPatchVersion = latestPatch + now := metav1.Now() + rs.LastCheckedTime = &now + + return c.Status().Update(ctx, cs) + } + + logger.V(logs.LogDebug).Info(fmt.Sprintf("release %s/%s no longer found on ClusterSummary %s/%s", + ref.releaseNamespace, ref.releaseName, ref.clusterSummaryNamespace, ref.clusterSummaryName)) + return nil + }) +} + +// RunHelmChartUpdateChecker runs checkOutdatedHelmCharts once immediately, then on every tick +// of interval, until ctx is done. Only ever started for the default (unsharded) addon-controller +// deployment: see the shardKey == "" gate in cmd/main.go. +func RunHelmChartUpdateChecker(ctx context.Context, c client.Client, interval time.Duration, logger logr.Logger) { + checkOutdatedHelmCharts(ctx, c, logger) + + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + logger.Info("stopping helm chart update checker") + return + case <-ticker.C: + checkOutdatedHelmCharts(ctx, c, logger) + } + } +} diff --git a/controllers/helmchart_outdated_check_test.go b/controllers/helmchart_outdated_check_test.go new file mode 100644 index 00000000..dc3b423b --- /dev/null +++ b/controllers/helmchart_outdated_check_test.go @@ -0,0 +1,236 @@ +/* +Copyright 2025. projectsveltos.io. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package-internal (not controllers_test): collectHelmChartRefs is pure and dependency-free +// (no client, no envtest), and asserting on its output requires reading the unexported +// helmChartKey/helmChartRef fields it produces, so this is tested directly rather than through +// the Ginkgo/envtest suite used by the rest of this package's tests. +package controllers + +import ( + "context" + "testing" + + "github.com/go-logr/logr" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" +) + +const ( + testReleaseName = "r1" + testReleaseNamespace = "ns1" + testChartName = "nginx" + testChartVersion = "1.0.0" + testRepoURL = "https://example.com/charts" + testRepositoryName = "bitnami" + testBitnamiRepoURL = "https://charts.bitnami.com/bitnami" + testNamespace = "ns" + testClusterNamespace = "cluster-ns" + testClusterName = "cluster" + testClusterType = "Capi" + testProfileName = "profile" +) + +func TestCollectHelmChartRefs_SkipsConflictEntries(t *testing.T) { + cs := &configv1beta1.ClusterSummary{ + Status: configv1beta1.ClusterSummaryStatus{ + HelmReleaseSummaries: []configv1beta1.HelmChartSummary{ + { + ReleaseName: testReleaseName, + ReleaseNamespace: testReleaseNamespace, + Status: configv1beta1.HelmChartStatusConflict, + ChartName: testChartName, + RepoURL: testRepoURL, + ChartVersion: testChartVersion, + }, + }, + }, + } + + refs := map[helmChartKey][]helmChartRef{} + collectHelmChartRefs(cs, refs, logr.Discard()) + + if len(refs) != 0 { + t.Fatalf("expected no refs for a Conflict-status entry, got %d keys", len(refs)) + } +} + +func TestCollectHelmChartRefs_SkipsUnresolvedIdentity(t *testing.T) { + tests := []struct { + name string + rs configv1beta1.HelmChartSummary + }{ + { + name: "never deployed (no ChartName/RepoURL/ChartVersion)", + rs: configv1beta1.HelmChartSummary{ + ReleaseName: testReleaseName, ReleaseNamespace: testReleaseNamespace, Status: configv1beta1.HelmChartStatusManaging, + }, + }, + { + name: "flux-sourced (ChartName/RepoURL never populated)", + rs: configv1beta1.HelmChartSummary{ + ReleaseName: testReleaseName, ReleaseNamespace: testReleaseNamespace, Status: configv1beta1.HelmChartStatusManaging, + ChartVersion: testChartVersion, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cs := &configv1beta1.ClusterSummary{ + Status: configv1beta1.ClusterSummaryStatus{HelmReleaseSummaries: []configv1beta1.HelmChartSummary{tt.rs}}, + } + + refs := map[helmChartKey][]helmChartRef{} + collectHelmChartRefs(cs, refs, logr.Discard()) + + if len(refs) != 0 { + t.Fatalf("expected no refs, got %d keys", len(refs)) + } + }) + } +} + +func TestCollectHelmChartRefs_DedupsAcrossClusterSummaries(t *testing.T) { + makeClusterSummary := func(name string) *configv1beta1.ClusterSummary { + return &configv1beta1.ClusterSummary{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: testNamespace}, + Spec: configv1beta1.ClusterSummarySpec{ + ClusterNamespace: testClusterNamespace, ClusterName: name, ClusterType: testClusterType, + }, + Status: configv1beta1.ClusterSummaryStatus{ + HelmReleaseSummaries: []configv1beta1.HelmChartSummary{ + { + ReleaseName: testChartName, + ReleaseNamespace: "web", + Status: configv1beta1.HelmChartStatusManaging, + ChartName: testChartName, + RepositoryName: testRepositoryName, + RepoURL: testBitnamiRepoURL, + ChartVersion: testChartVersion, + }, + }, + }, + } + } + + refs := map[helmChartKey][]helmChartRef{} + collectHelmChartRefs(makeClusterSummary("cluster-a"), refs, logr.Discard()) + collectHelmChartRefs(makeClusterSummary("cluster-b"), refs, logr.Discard()) + + if len(refs) != 1 { + t.Fatalf("expected exactly one deduped chart key, got %d", len(refs)) + } + + for key, keyRefs := range refs { + if key.chartName != testChartName || key.repositoryName != testRepositoryName || key.repositoryURL != testBitnamiRepoURL { + t.Errorf("unexpected key: %+v", key) + } + if len(keyRefs) != 2 { + t.Fatalf("expected 2 refs for the deduped key, got %d", len(keyRefs)) + } + if keyRefs[0].clusterSummaryName == keyRefs[1].clusterSummaryName { + t.Errorf("expected refs from two distinct ClusterSummaries, got the same name twice: %q", + keyRefs[0].clusterSummaryName) + } + } +} + +func TestSetResolvedHelmChartIdentity_ClearsStaleOutdatedVersionInfoOnVersionChange(t *testing.T) { + newLatest := v1_1_0 + lastChecked := metav1.Now() + + clusterSummary := &configv1beta1.ClusterSummary{ + ObjectMeta: metav1.ObjectMeta{ + Name: controllerNameClusterSummary, Namespace: testNamespace, + OwnerReferences: []metav1.OwnerReference{ + {Kind: configv1beta1.ClusterProfileKind, APIVersion: configv1beta1.GroupVersion.String(), Name: testProfileName}, + }, + }, + Spec: configv1beta1.ClusterSummarySpec{ + ClusterNamespace: testClusterNamespace, ClusterName: testClusterName, ClusterType: testClusterType, + }, + } + + requestedChart := &configv1beta1.HelmChart{ + ChartName: testChartName, RepositoryName: testRepositoryName, RepositoryURL: testBitnamiRepoURL, + ReleaseName: testReleaseName, ReleaseNamespace: testReleaseNamespace, + } + + rs := &configv1beta1.HelmChartSummary{ + ReleaseName: testReleaseName, ReleaseNamespace: testReleaseNamespace, Status: configv1beta1.HelmChartStatusManaging, + ChartName: testChartName, RepositoryName: testRepositoryName, RepoURL: testBitnamiRepoURL, ChartVersion: testChartVersion, + LatestVersion: &newLatest, LastCheckedTime: &lastChecked, + } + + currentRelease := &releaseInfo{ChartVersion: newLatest} // upgrading to the version previously flagged as latest + + setResolvedHelmChartIdentity(context.TODO(), nil, clusterSummary, rs, requestedChart, currentRelease, logr.Discard()) + + if rs.ChartVersion != newLatest { + t.Errorf("expected ChartVersion to be updated to %q, got %q", newLatest, rs.ChartVersion) + } + if rs.LatestVersion != nil { + t.Errorf("expected LatestVersion to be cleared, got %v", *rs.LatestVersion) + } + if rs.LatestPatchVersion != nil { + t.Errorf("expected LatestPatchVersion to be cleared, got %v", *rs.LatestPatchVersion) + } + if rs.LastCheckedTime != nil { + t.Errorf("expected LastCheckedTime to be cleared, got %v", *rs.LastCheckedTime) + } +} + +func TestSetResolvedHelmChartIdentity_KeepsOutdatedVersionInfoWhenVersionUnchanged(t *testing.T) { + newLatest := v1_1_0 + lastChecked := metav1.Now() + + clusterSummary := &configv1beta1.ClusterSummary{ + ObjectMeta: metav1.ObjectMeta{ + Name: controllerNameClusterSummary, Namespace: testNamespace, + OwnerReferences: []metav1.OwnerReference{ + {Kind: configv1beta1.ClusterProfileKind, APIVersion: configv1beta1.GroupVersion.String(), Name: testProfileName}, + }, + }, + Spec: configv1beta1.ClusterSummarySpec{ + ClusterNamespace: testClusterNamespace, ClusterName: testClusterName, ClusterType: testClusterType, + }, + } + + requestedChart := &configv1beta1.HelmChart{ + ChartName: testChartName, RepositoryName: testRepositoryName, RepositoryURL: testBitnamiRepoURL, + ReleaseName: testReleaseName, ReleaseNamespace: testReleaseNamespace, + } + + rs := &configv1beta1.HelmChartSummary{ + ReleaseName: testReleaseName, ReleaseNamespace: testReleaseNamespace, Status: configv1beta1.HelmChartStatusManaging, + ChartName: testChartName, RepositoryName: testRepositoryName, RepoURL: testBitnamiRepoURL, ChartVersion: testChartVersion, + LatestVersion: &newLatest, LastCheckedTime: &lastChecked, + } + + // Same deployed version as before (e.g. a values-only reconcile) -- nothing to invalidate. + currentRelease := &releaseInfo{ChartVersion: testChartVersion} + + setResolvedHelmChartIdentity(context.TODO(), nil, clusterSummary, rs, requestedChart, currentRelease, logr.Discard()) + + if rs.LatestVersion == nil || *rs.LatestVersion != newLatest { + t.Errorf("expected LatestVersion to be left untouched at %q, got %v", newLatest, rs.LatestVersion) + } + if rs.LastCheckedTime == nil { + t.Errorf("expected LastCheckedTime to be left untouched, got nil") + } +} diff --git a/controllers/helmchart_semver.go b/controllers/helmchart_semver.go new file mode 100644 index 00000000..172f257e --- /dev/null +++ b/controllers/helmchart_semver.go @@ -0,0 +1,69 @@ +/* +Copyright 2025. projectsveltos.io. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "fmt" + + "github.com/Masterminds/semver/v3" +) + +// helmChartVersionDiff compares currentVersion against every version string in +// availableVersions and reports the highest version that is strictly newer, both overall and +// within currentVersion's own major.minor line. Unparseable entries in availableVersions are +// silently skipped (callers may log this themselves; kept out of this function so it stays a +// pure, easily table-tested comparison). Prerelease versions are only considered when +// currentVersion is itself a prerelease, so a stable pin is never flagged as outdated by a +// prerelease build. +// Returns (nil, nil, nil) when currentVersion is already the latest. +func helmChartVersionDiff(currentVersion string, availableVersions []string, +) (latestOverall, latestPatchSameMinor *semver.Version, err error) { + + current, err := semver.NewVersion(currentVersion) + if err != nil { + return nil, nil, fmt.Errorf("current version %q is not valid semver: %w", currentVersion, err) + } + + includePrerelease := current.Prerelease() != "" + + for i := range availableVersions { + candidate, parseErr := semver.NewVersion(availableVersions[i]) + if parseErr != nil { + continue + } + + if candidate.Prerelease() != "" && !includePrerelease { + continue + } + + if !candidate.GreaterThan(current) { + continue + } + + if latestOverall == nil || candidate.GreaterThan(latestOverall) { + latestOverall = candidate + } + + if candidate.Major() == current.Major() && candidate.Minor() == current.Minor() { + if latestPatchSameMinor == nil || candidate.GreaterThan(latestPatchSameMinor) { + latestPatchSameMinor = candidate + } + } + } + + return latestOverall, latestPatchSameMinor, nil +} diff --git a/controllers/helmchart_semver_test.go b/controllers/helmchart_semver_test.go new file mode 100644 index 00000000..1776695c --- /dev/null +++ b/controllers/helmchart_semver_test.go @@ -0,0 +1,127 @@ +/* +Copyright 2025. projectsveltos.io. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package-internal (not controllers_test): helmChartVersionDiff is a pure, dependency-free +// function, so it is table-tested directly with the standard library rather than through the +// Ginkgo/envtest suite used by the rest of this package's tests. +package controllers + +import "testing" + +const ( + v1_0_0 = "1.0.0" + v1_1_0 = "1.1.0" + v1_2_3 = "1.2.3" + v1_2_4 = "1.2.4" + v1_2_5 = "1.2.5" + v1_9_9 = "1.9.9" + v2_0_0 = "2.0.0" + notAVersion = "not-a-version" +) + +func TestHelmChartVersionDiff(t *testing.T) { + tests := []struct { + name string + current string + available []string + wantLatest string // "" means expect nil + wantLatestPatch string // "" means expect nil + wantErr bool + }{ + { + name: "newer minor and same-minor patch both found", + current: v1_2_3, + available: []string{v1_2_3, v1_2_4, "1.3.0", v2_0_0}, + wantLatest: v2_0_0, + wantLatestPatch: v1_2_4, + }, + { + name: "only a same-minor patch found", + current: v1_2_3, + available: []string{v1_2_3, v1_2_4, v1_2_5}, + wantLatest: v1_2_5, + wantLatestPatch: v1_2_5, + }, + { + name: "already the latest", + current: v2_0_0, + available: []string{v1_2_3, v1_9_9, v2_0_0}, + }, + { + name: "current not in available list but nothing newer", + current: v2_0_0, + available: []string{v1_9_9}, + }, + { + name: "unparseable current version", + current: notAVersion, + available: []string{v1_0_0}, + wantErr: true, + }, + { + name: "unparseable entries in available list are skipped", + current: v1_0_0, + available: []string{notAVersion, v1_1_0, "also-bad"}, + wantLatest: v1_1_0, + wantLatestPatch: "", + }, + { + name: "stable current excludes prereleases", + current: v1_0_0, + available: []string{"1.1.0-rc.1"}, + wantLatest: "", + }, + { + name: "prerelease current includes prereleases", + current: "1.0.0-rc.1", + available: []string{"1.0.0-rc.2", v1_0_0}, + wantLatest: v1_0_0, + wantLatestPatch: v1_0_0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + latest, latestPatch, err := helmChartVersionDiff(tt.current, tt.available) + + if tt.wantErr { + if err == nil { + t.Fatalf("expected an error, got none") + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if tt.wantLatest == "" { + if latest != nil { + t.Errorf("expected nil latest, got %v", latest) + } + } else if latest == nil || latest.String() != tt.wantLatest { + t.Errorf("expected latest %q, got %v", tt.wantLatest, latest) + } + + if tt.wantLatestPatch == "" { + if latestPatch != nil { + t.Errorf("expected nil latestPatch, got %v", latestPatch) + } + } else if latestPatch == nil || latestPatch.String() != tt.wantLatestPatch { + t.Errorf("expected latestPatch %q, got %v", tt.wantLatestPatch, latestPatch) + } + }) + } +} diff --git a/controllers/helmchart_version_source.go b/controllers/helmchart_version_source.go new file mode 100644 index 00000000..bac253c6 --- /dev/null +++ b/controllers/helmchart_version_source.go @@ -0,0 +1,225 @@ +/* +Copyright 2025. projectsveltos.io. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + "io" + "net/url" + "os" + "path" + "strings" + "time" + + "github.com/go-logr/logr" + "helm.sh/helm/v4/pkg/cli" + "helm.sh/helm/v4/pkg/getter" + "helm.sh/helm/v4/pkg/registry" + repo "helm.sh/helm/v4/pkg/repo/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + logs "github.com/projectsveltos/libsveltos/lib/logsettings" +) + +// helmChartKey identifies a distinct upstream chart to check, deduplicated across every +// HelmChartSummary entry, on every ClusterSummary, that references it. RepositoryURL, +// RepositoryName and ChartName here are always the fully resolved (post-template) values +// captured at deploy time by setResolvedHelmChartIdentity, never a template string. +type helmChartKey struct { + repositoryURL string + repositoryName string + chartName string +} + +const ( + // fetchAvailableVersionsTimeout bounds a single chart's fetch, so one slow or misbehaving + // repository/registry can never stall an entire checkOutdatedHelmCharts pass. + fetchAvailableVersionsTimeout = 30 * time.Second +) + +// resolveChartCredentials fetches secretRef (already resolved to a concrete namespace/name by +// setResolvedHelmChartIdentity at deploy time, so no cluster context is needed here) from the +// management cluster and extracts username/password for hostname. Returns empty strings, no +// error, when secretRef is nil (anonymous access). +func resolveChartCredentials(ctx context.Context, c client.Client, secretRef *corev1.SecretReference, + hostname string) (username, password string, err error) { + + if secretRef == nil { + return "", "", nil + } + + secret := &corev1.Secret{} + if getErr := c.Get(ctx, types.NamespacedName{Namespace: secretRef.Namespace, Name: secretRef.Name}, secret); getErr != nil { + return "", "", getErr + } + + return getUsernameAndPasswordFromSecret(hostname, secret) +} + +// fetchAvailableVersions returns every version string currently published for key, using the +// HTTP index.yaml path or the OCI registry Tags() path depending on key.repositoryURL. +// credentialsSecretRef, if non-nil, is used to authenticate the request; on any credential +// resolution failure this falls back to an anonymous request rather than failing outright, +// since that request may still succeed for a public chart. +func fetchAvailableVersions(ctx context.Context, c client.Client, key helmChartKey, + credentialsSecretRef *corev1.SecretReference, logger logr.Logger) ([]string, error) { + + fetchCtx, cancel := context.WithTimeout(ctx, fetchAvailableVersionsTimeout) + defer cancel() + + bareChartName := removeRepositoryNameFromChartName(key.chartName, key.repositoryName) + + parsedURL, err := url.Parse(key.repositoryURL) + if err != nil { + return nil, fmt.Errorf("failed to parse repository URL %q: %w", key.repositoryURL, err) + } + + username, password, credErr := resolveChartCredentials(fetchCtx, c, credentialsSecretRef, parsedURL.Host) + if credErr != nil { + logger.V(logs.LogDebug).Info(fmt.Sprintf("failed to resolve credentials for %s, trying anonymously: %v", + key.repositoryURL, credErr)) + } + + if registry.IsOCI(key.repositoryURL) { + return fetchOCIChartVersions(fetchCtx, key.repositoryURL, bareChartName, username, password) + } + + return fetchHTTPIndexChartVersions(fetchCtx, bareChartName, key.repositoryURL, username, password) +} + +// fetchHTTPIndexChartVersions always downloads a fresh index.yaml into a scratch directory and +// returns every published version string for chartName. Deliberately does not use +// repoAddOrUpdate/the shared repo storage cache used by the deploy path: that cache is +// download-once-per-process, which is correct for pinning a chart at deploy time but wrong for +// a periodic freshness check that must see newly published versions on every pass. +func fetchHTTPIndexChartVersions(ctx context.Context, chartName, repoURL, username, password string) ([]string, error) { + tmpDir, err := os.MkdirTemp("", "sveltos-helm-index-*") + if err != nil { + return nil, err + } + defer os.RemoveAll(tmpDir) // best-effort cleanup of a scratch temp dir + + settings := cli.New() + entry := &repo.Entry{Name: "chart", URL: repoURL, Username: username, Password: password} + chartRepo, err := repo.NewChartRepository(entry, getter.All(settings)) + if err != nil { + return nil, err + } + chartRepo.CachePath = tmpDir + + type downloadResult struct { + path string + err error + } + done := make(chan downloadResult, 1) + go func() { + p, dErr := chartRepo.DownloadIndexFile() + done <- downloadResult{p, dErr} + }() + + var indexPath string + select { + case r := <-done: + if r.err != nil { + return nil, r.err + } + indexPath = r.path + case <-ctx.Done(): + return nil, fmt.Errorf("timed out downloading index for repository %s", repoURL) + } + + idx, err := repo.LoadIndexFile(indexPath) + if err != nil { + return nil, err + } + + versions, ok := idx.Entries[chartName] + if !ok { + return nil, fmt.Errorf("chart %q not found in repository index %s", chartName, repoURL) + } + + out := make([]string, 0, len(versions)) + for i := range versions { + if versions[i] == nil || versions[i].Metadata == nil { + continue + } + out = append(out, versions[i].Version) + } + return out, nil +} + +// fetchOCIChartVersions lists every semver-parseable tag published for the chart identified by +// repoURL/repositoryName/bareChartName. Uses Helm's own registry.Client.Tags, which already +// filters to strict-semver tags and returns them sorted descending — no hand-rolled registry +// API calls needed. Plain-HTTP/insecure-TLS OCI registries are not supported by this check +// (a known, disclosed limitation): any failure here is treated as skippable by the caller. +func fetchOCIChartVersions(ctx context.Context, repoURL, bareChartName, username, password string, +) ([]string, error) { + + ref, err := buildOCIChartRef(repoURL, bareChartName) + if err != nil { + return nil, err + } + + opts := []registry.ClientOption{registry.ClientOptWriter(io.Discard)} + if username != "" && password != "" { + opts = append(opts, registry.ClientOptBasicAuth(username, password)) + } + + registryClient, err := registry.NewClient(opts...) + if err != nil { + return nil, err + } + + type tagsResult struct { + tags []string + err error + } + done := make(chan tagsResult, 1) + go func() { + t, tErr := registryClient.Tags(ref) + done <- tagsResult{t, tErr} + }() + + select { + case r := <-done: + return r.tags, r.err + case <-ctx.Done(): + return nil, fmt.Errorf("timed out listing tags for %s", ref) + } +} + +// buildOCIChartRef mirrors the OCI-ref construction in getHelmChartAndRepoName +// (handlers_helm.go), but is pure — no ClusterSummary/Flux resolution needed, since +// Flux-sourced charts never reach this point (setResolvedHelmChartIdentity never populates +// their identity fields). Returns a ref without the "oci://" prefix, as expected by +// registry.Client.Tags. +func buildOCIChartRef(repoURL, bareChartName string) (string, error) { + u, err := url.Parse(repoURL) + if err != nil { + return "", err + } + + // bareChartName is already stripped of any repository-name prefix by the caller + // (removeRepositoryNameFromChartName), so it can be joined onto the repo path directly. + u.Path = path.Join(u.Path, bareChartName) + + return strings.TrimPrefix(u.String(), "oci://"), nil +} diff --git a/controllers/metrics.go b/controllers/metrics.go index c298a6e4..98370833 100644 --- a/controllers/metrics.go +++ b/controllers/metrics.go @@ -44,6 +44,10 @@ const ( metricProfileNamespaceLabel = "profile_namespace" metricProfileNameLabel = "profile_name" + metricChartNameLabel = "chart_name" + metricReleaseNamespaceLabel = "release_namespace" + metricReleaseNameLabel = "release_name" + statusSuccess = "success" statusFailure = "failure" ) @@ -145,13 +149,57 @@ var ( []string{metricClusterTypeLabel, metricClusterNamespaceLabel, metricClusterNameLabel, metricFeatureLabel, metricProfileKindLabel, metricProfileNamespaceLabel, metricProfileNameLabel}, ) + + // outdatedHelmChartGauge is 1 when a ClusterSummary's managed HelmChart release is behind + // the latest version published upstream. Reset() once per checkOutdatedHelmCharts pass and + // re-Set(1) only for entries found outdated that pass, so stale series (charts that became + // up to date, or are no longer referenced) don't linger. Version strings are deliberately + // not labels here (high cardinality/churn) — they live only on HelmChartSummary. + outdatedHelmChartGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricNamespace, + Name: "outdated_helm_chart", + Help: "1 if a ClusterSummary's managed HelmChart release is behind the latest published upstream version", + }, + []string{metricProfileKindLabel, metricProfileNamespaceLabel, metricProfileNameLabel, + metricClusterTypeLabel, metricClusterNamespaceLabel, metricClusterNameLabel, + metricChartNameLabel, metricReleaseNamespaceLabel, metricReleaseNameLabel}, + ) + + // outdatedHelmChartCheckLastRunTimestampGauge records when checkOutdatedHelmCharts last + // completed a full pass, regardless of whether any chart was found outdated. Unlike + // outdatedHelmChartGauge (which describes chart state), this describes the checker's own + // health: if it stops advancing, the checker is stuck or has stopped running, which is + // indistinguishable from "everything is up to date" by looking at outdatedHelmChartGauge + // alone. + outdatedHelmChartCheckLastRunTimestampGauge = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: metricNamespace, + Name: "outdated_helm_chart_check_last_run_timestamp_seconds", + Help: "Unix timestamp of the last completed pass of the periodic outdated-Helm-chart checker", + }, + ) + + // outdatedHelmChartCheckFailuresCounter counts chart keys skipped in a + // checkOutdatedHelmCharts pass because their upstream repository/registry could not be + // queried (unreachable, auth failure, timeout, chart not found, etc). Not labeled by chart: + // this is meant as a coarse "is the checker healthy" signal, not a per-chart diagnostic — + // per-chart failures are logged instead. + outdatedHelmChartCheckFailuresCounter = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: metricNamespace, + Name: "outdated_helm_chart_check_failures_total", + Help: "Total number of chart keys skipped by the outdated-Helm-chart checker due to a fetch failure or timeout", + }, + ) ) //nolint:gochecknoinits // forced pattern, can't workaround func init() { // Register custom metrics with the global prometheus registry metrics.Registry.MustRegister(reconcileDurationHistogram, reconciliationCounter, reconcileOutcomeCounter, - driftCounter, matchingClustersGauge, reconcileConsecutiveFailuresGauge, reconcileLastSuccessTimestampGauge) + driftCounter, matchingClustersGauge, reconcileConsecutiveFailuresGauge, reconcileLastSuccessTimestampGauge, + outdatedHelmChartGauge, outdatedHelmChartCheckLastRunTimestampGauge, outdatedHelmChartCheckFailuresCounter) } func programDuration(elapsed time.Duration, clusterNamespace, clusterName, featureID string, @@ -281,6 +329,63 @@ func trackConsecutiveFailures(clusterNamespace, clusterName, featureID string, c clusterType, clusterNamespace, clusterName, featureID, consecutiveFailures)) } +// trackOutdatedHelmChart records that a ClusterSummary's managed HelmChart release currently +// has a newer version published upstream than its deployed ChartVersion. Only called when an +// outdated version was actually found. Pairs with clearOutdatedHelmChart, which +// checkOutdatedHelmCharts calls instead of resetting the whole gauge, so a chart's exposed +// state only ever changes when the real answer changes (see reconcileOutdatedHelmChartMetric). +func trackOutdatedHelmChart(profileKind, profileNamespace, profileName, clusterType, clusterNamespace, clusterName, + chartName, releaseNamespace, releaseName string, logger logr.Logger) { + + outdatedHelmChartGauge.With(prometheus.Labels{ + metricProfileKindLabel: profileKind, + metricProfileNamespaceLabel: profileNamespace, + metricProfileNameLabel: profileName, + metricClusterTypeLabel: clusterType, + metricClusterNamespaceLabel: clusterNamespace, + metricClusterNameLabel: clusterName, + metricChartNameLabel: chartName, + metricReleaseNamespaceLabel: releaseNamespace, + metricReleaseNameLabel: releaseName, + }).Set(1) + + logger.V(logs.LogVerbose).Info(fmt.Sprintf("tracking outdated helm chart %s (release %s/%s)", + chartName, releaseNamespace, releaseName)) +} + +// clearOutdatedHelmChart removes the gauge series for a chart confirmed, this pass, to no +// longer be outdated (or no longer referenced). Deletes the series rather than Set(0), matching +// outdatedHelmChartGauge's "absent means not outdated" convention. +func clearOutdatedHelmChart(profileKind, profileNamespace, profileName, clusterType, clusterNamespace, clusterName, + chartName, releaseNamespace, releaseName string) { + + outdatedHelmChartGauge.Delete(prometheus.Labels{ + metricProfileKindLabel: profileKind, + metricProfileNamespaceLabel: profileNamespace, + metricProfileNameLabel: profileName, + metricClusterTypeLabel: clusterType, + metricClusterNamespaceLabel: clusterNamespace, + metricClusterNameLabel: clusterName, + metricChartNameLabel: chartName, + metricReleaseNamespaceLabel: releaseNamespace, + metricReleaseNameLabel: releaseName, + }) +} + +// trackHelmChartCheckCompleted records that a checkOutdatedHelmCharts pass just finished, +// regardless of whether any individual chart fetch failed. Call once per pass. +func trackHelmChartCheckCompleted(logger logr.Logger) { + outdatedHelmChartCheckLastRunTimestampGauge.Set(float64(time.Now().Unix())) + + logger.V(logs.LogVerbose).Info("recorded completed outdated helm chart check pass") +} + +// recordHelmChartCheckFailure increments the coarse checker-health failure counter. Call once +// per chart key skipped due to a fetch failure or timeout, not once per affected ClusterSummary. +func recordHelmChartCheckFailure() { + outdatedHelmChartCheckFailuresCounter.Inc() +} + // trackLastSuccess records the timestamp of the most recent successful (Provisioned/Removed) terminal // outcome for a feature on a cluster. Only called from the success branches of updateFeatureStatus. func trackLastSuccess(clusterNamespace, clusterName, featureID string, clusterType libsveltosv1beta1.ClusterType, diff --git a/manifest/manifest.yaml b/manifest/manifest.yaml index 7e7d4eee..ebc8ac5c 100644 --- a/manifest/manifest.yaml +++ b/manifest/manifest.yaml @@ -8086,15 +8086,56 @@ spec: directly managed by ClusterProfile. items: properties: + chartName: + description: |- + ChartName, RepositoryName, RepoURL, and ChartVersion mirror the fully resolved + (post-template) HelmChart entry that produced this release, captured at deploy time. + Never set for Flux-source-backed charts (Flux owns version resolution there). + type: string + chartVersion: + type: string conflictMessage: description: |- Status indicates whether ClusterSummary can manage the helm chart or there is a conflict type: string + credentialsSecretRef: + description: |- + CredentialsSecretRef is the resolved secret reference (if any) used to authenticate + against RepoURL, captured at deploy time. Only the reference is stored, never secret + contents. + properties: + name: + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which the + secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic failureMessage: description: FailureMessage provides the specific error from the Helm engine for this release type: string + lastCheckedTime: + description: LastCheckedTime is when LatestVersion/LatestPatchVersion + were last evaluated. + format: date-time + type: string + latestPatchVersion: + description: |- + LatestPatchVersion is the highest published version sharing ChartVersion's + major.minor, if greater than ChartVersion. Distinguishes "a same-minor patch bump is + available" from "a newer minor/major line exists" (LatestVersion). + type: string + latestVersion: + description: |- + LatestVersion is the highest version currently published upstream for this chart, if + greater than ChartVersion. Populated by a periodic background check, independent of + the reconcile loop. Detection only: Sveltos never mutates ChartVersion based on this. + type: string patchesHash: description: PatchesHash represents of a unique value for the patches section @@ -8109,6 +8150,10 @@ spec: be installed minLength: 1 type: string + repoURL: + type: string + repositoryName: + type: string status: description: |- Status indicates whether ClusterSummary can manage the helm