diff --git a/api/v1beta1/clustersummary_types.go b/api/v1beta1/clustersummary_types.go index a41f5b60..d318e6d2 100644 --- a/api/v1beta1/clustersummary_types.go +++ b/api/v1beta1/clustersummary_types.go @@ -129,6 +129,43 @@ type HelmChartSummary struct { // +optional // FailureMessage provides the specific error from the Helm engine for this release FailureMessage *string `json:"failureMessage,omitempty"` + + // ChartName, RepositoryName, RepoURL, and ChartVersion mirror the fully resolved + // (post-template) HelmChart entry that produced this release, captured at deploy time. + // Never set for Flux-source-backed charts (Flux owns version resolution there). + // +optional + ChartName string `json:"chartName,omitempty"` + + // +optional + RepositoryName string `json:"repositoryName,omitempty"` + + // +optional + RepoURL string `json:"repoURL,omitempty"` + + // +optional + ChartVersion string `json:"chartVersion,omitempty"` + + // CredentialsSecretRef is the resolved secret reference (if any) used to authenticate + // against RepoURL, captured at deploy time. Only the reference is stored, never secret + // contents. + // +optional + CredentialsSecretRef *corev1.SecretReference `json:"credentialsSecretRef,omitempty"` + + // LatestVersion is the highest version currently published upstream for this chart, if + // greater than ChartVersion. Populated by a periodic background check, independent of + // the reconcile loop. Detection only: Sveltos never mutates ChartVersion based on this. + // +optional + LatestVersion *string `json:"latestVersion,omitempty"` + + // LatestPatchVersion is the highest published version sharing ChartVersion's + // major.minor, if greater than ChartVersion. Distinguishes "a same-minor patch bump is + // available" from "a newer minor/major line exists" (LatestVersion). + // +optional + LatestPatchVersion *string `json:"latestPatchVersion,omitempty"` + + // LastCheckedTime is when LatestVersion/LatestPatchVersion were last evaluated. + // +optional + LastCheckedTime *metav1.Time `json:"lastCheckedTime,omitempty"` } // ClusterSummarySpec defines the desired state of ClusterSummary diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 7367cd4f..ae5338bc 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -21,11 +21,12 @@ limitations under the License. package v1beta1 import ( - apiv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" + + apiv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -867,6 +868,25 @@ func (in *HelmChartSummary) DeepCopyInto(out *HelmChartSummary) { *out = new(string) **out = **in } + if in.CredentialsSecretRef != nil { + in, out := &in.CredentialsSecretRef, &out.CredentialsSecretRef + *out = new(corev1.SecretReference) + **out = **in + } + if in.LatestVersion != nil { + in, out := &in.LatestVersion, &out.LatestVersion + *out = new(string) + **out = **in + } + if in.LatestPatchVersion != nil { + in, out := &in.LatestPatchVersion, &out.LatestPatchVersion + *out = new(string) + **out = **in + } + if in.LastCheckedTime != nil { + in, out := &in.LastCheckedTime, &out.LastCheckedTime + *out = (*in).DeepCopy() + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HelmChartSummary. diff --git a/cmd/main.go b/cmd/main.go index 91142bba..db9ecef7 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -75,33 +75,34 @@ import ( ) var ( - setupLog = ctrl.Log.WithName("setup") - diagnosticsAddress string - insecureDiagnostics bool - shardKey string - workers int - concurrentReconciles int - agentInMgmtCluster bool - reportMode controllers.ReportMode - tmpReportMode int - restConfigQPS float32 - restConfigBurst int - webhookPort int - syncPeriod time.Duration - conflictRetryTime time.Duration - healthErrorRetryTime time.Duration - version string - healthAddr string - profilerAddress string - driftDetectionConfigMap string - luaConfigMap string - capiOnboardAnnotation string - disableCaching bool - disableTelemetry bool - autoDeployDependencies bool - registry string - luaCallStackSize int - luaRegistrySize int + setupLog = ctrl.Log.WithName("setup") + diagnosticsAddress string + insecureDiagnostics bool + shardKey string + workers int + concurrentReconciles int + agentInMgmtCluster bool + reportMode controllers.ReportMode + tmpReportMode int + restConfigQPS float32 + restConfigBurst int + webhookPort int + syncPeriod time.Duration + conflictRetryTime time.Duration + healthErrorRetryTime time.Duration + version string + healthAddr string + profilerAddress string + driftDetectionConfigMap string + luaConfigMap string + capiOnboardAnnotation string + disableCaching bool + disableTelemetry bool + autoDeployDependencies bool + registry string + luaCallStackSize int + luaRegistrySize int + helmChartUpdateCheckInterval time.Duration ) const ( @@ -297,6 +298,10 @@ func initFlags(fs *pflag.FlagSet) { fmt.Sprintf("The minimum interval at which health check failures are retried. Default: %d seconds", defaultHealthErrorRetryTime)) + fs.DurationVar(&helmChartUpdateCheckInterval, "helm-chart-update-check-interval", time.Hour, + "Interval at which Sveltos checks whether newer versions of deployed Helm charts have been "+ + "published upstream (HTTP repositories and OCI registries). Set to 0 to disable.") + // AutoDeployDependencies enables automatic deployment of prerequisite profiles. // // Profile instances can specify dependencies on other profiles using the @@ -671,6 +676,15 @@ func startControllersAndWatchers(ctx context.Context, mgr manager.Manager) { setupLog.Error(err, "unable to create controller", "controller", "ClusterPromotion") os.Exit(1) } + + // Needs a fleet-wide view of every ClusterSummary to dedup chart keys correctly, so + // this only ever runs on the default (unsharded) deployment, same as the reconcilers + // started above. Running it per-shard would give no benefit (chart-key dedup is + // inherently fleet-wide) and would reintroduce cross-shard inconsistency. + if helmChartUpdateCheckInterval > 0 { + go controllers.RunHelmChartUpdateChecker(ctx, mgr.GetClient(), helmChartUpdateCheckInterval, + ctrl.Log.WithName("helm-chart-update-checker")) + } } clusterSummaryReconciler := getClusterSummaryReconciler(ctx, mgr) diff --git a/config/crd/bases/config.projectsveltos.io_clustersummaries.yaml b/config/crd/bases/config.projectsveltos.io_clustersummaries.yaml index eefd82a9..495c5e95 100644 --- a/config/crd/bases/config.projectsveltos.io_clustersummaries.yaml +++ b/config/crd/bases/config.projectsveltos.io_clustersummaries.yaml @@ -2262,15 +2262,56 @@ spec: directly managed by ClusterProfile. items: properties: + chartName: + description: |- + ChartName, RepositoryName, RepoURL, and ChartVersion mirror the fully resolved + (post-template) HelmChart entry that produced this release, captured at deploy time. + Never set for Flux-source-backed charts (Flux owns version resolution there). + type: string + chartVersion: + type: string conflictMessage: description: |- Status indicates whether ClusterSummary can manage the helm chart or there is a conflict type: string + credentialsSecretRef: + description: |- + CredentialsSecretRef is the resolved secret reference (if any) used to authenticate + against RepoURL, captured at deploy time. Only the reference is stored, never secret + contents. + properties: + name: + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which the + secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic failureMessage: description: FailureMessage provides the specific error from the Helm engine for this release type: string + lastCheckedTime: + description: LastCheckedTime is when LatestVersion/LatestPatchVersion + were last evaluated. + format: date-time + type: string + latestPatchVersion: + description: |- + LatestPatchVersion is the highest published version sharing ChartVersion's + major.minor, if greater than ChartVersion. Distinguishes "a same-minor patch bump is + available" from "a newer minor/major line exists" (LatestVersion). + type: string + latestVersion: + description: |- + LatestVersion is the highest version currently published upstream for this chart, if + greater than ChartVersion. Populated by a periodic background check, independent of + the reconcile loop. Detection only: Sveltos never mutates ChartVersion based on this. + type: string patchesHash: description: PatchesHash represents of a unique value for the patches section @@ -2285,6 +2326,10 @@ spec: be installed minLength: 1 type: string + repoURL: + type: string + repositoryName: + type: string status: description: |- Status indicates whether ClusterSummary can manage the helm diff --git a/controllers/clustersummary_predicates_test.go b/controllers/clustersummary_predicates_test.go index 3fbfae89..c4022d9e 100644 --- a/controllers/clustersummary_predicates_test.go +++ b/controllers/clustersummary_predicates_test.go @@ -31,6 +31,7 @@ import ( "k8s.io/klog/v2/textlogger" "sigs.k8s.io/controller-runtime/pkg/event" + configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" "github.com/projectsveltos/addon-controller/controllers" libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" ) @@ -338,3 +339,67 @@ var _ = Describe("ClusterProfile Predicates: FluxSourcePredicates", func() { Expect(result).To(BeFalse()) }) }) + +var _ = Describe("Clustersummary Predicates: ClusterSummaryPredicate", func() { + var logger logr.Logger + var clusterSummary *configv1beta1.ClusterSummary + + BeforeEach(func() { + logger = textlogger.NewLogger(textlogger.NewConfig()) + clusterSummary = &configv1beta1.ClusterSummary{ + ObjectMeta: metav1.ObjectMeta{ + Name: randomString(), + }, + Status: configv1beta1.ClusterSummaryStatus{ + HelmReleaseSummaries: []configv1beta1.HelmChartSummary{ + { + ReleaseName: randomString(), + ReleaseNamespace: randomString(), + Status: configv1beta1.HelmChartStatusManaging, + ChartName: randomString(), + RepositoryName: randomString(), + RepoURL: testRepoURLBitnami, + ChartVersion: testChartVersion100, + }, + }, + }, + } + }) + + It("Update returns false when only LatestVersion/LatestPatchVersion/LastCheckedTime change", func() { + clusterSummaryPredicate := controllers.ClusterSummaryPredicate{Logger: logger} + + oldClusterSummary := clusterSummary.DeepCopy() + + newLatest := "1.1.0" + lastChecked := metav1.Now() + clusterSummary.Status.HelmReleaseSummaries[0].LatestVersion = &newLatest + clusterSummary.Status.HelmReleaseSummaries[0].LastCheckedTime = &lastChecked + + e := event.UpdateEvent{ + ObjectNew: clusterSummary, + ObjectOld: oldClusterSummary, + } + + result := clusterSummaryPredicate.Update(e) + Expect(result).To(BeFalse()) + }) + + It("Update returns true when FeatureSummaries change", func() { + clusterSummaryPredicate := controllers.ClusterSummaryPredicate{Logger: logger} + + oldClusterSummary := clusterSummary.DeepCopy() + + clusterSummary.Status.FeatureSummaries = []configv1beta1.FeatureSummary{ + {FeatureID: libsveltosv1beta1.FeatureHelm, Status: libsveltosv1beta1.FeatureStatusProvisioned}, + } + + e := event.UpdateEvent{ + ObjectNew: clusterSummary, + ObjectOld: oldClusterSummary, + } + + result := clusterSummaryPredicate.Update(e) + Expect(result).To(BeTrue()) + }) +}) diff --git a/controllers/handlers_helm.go b/controllers/handlers_helm.go index 43206d12..8c2e4321 100644 --- a/controllers/handlers_helm.go +++ b/controllers/handlers_helm.go @@ -1325,7 +1325,7 @@ func deploySingleChart(ctx context.Context, c client.Client, dCtx *deploymentCon return nil, err } - valueHash, err := updateValueHashOnHelmChartSummary(ctx, instantiatedChart, dCtx, logger) + valueHash, err := updateValueHashOnHelmChartSummary(ctx, instantiatedChart, currentRelease, dCtx, logger) if err != nil { return nil, err } @@ -4656,7 +4656,7 @@ func getHelmChartValuesHash(ctx context.Context, c client.Client, instantiatedCh } func updateValueHashOnHelmChartSummary(ctx context.Context, requestedChart *configv1beta1.HelmChart, - dCtx *deploymentContext, logger logr.Logger) ([]byte, error) { + currentRelease *releaseInfo, dCtx *deploymentContext, logger logr.Logger) ([]byte, error) { c := getManagementClusterClient() @@ -4679,6 +4679,7 @@ func updateValueHashOnHelmChartSummary(ctx context.Context, requestedChart *conf rs.ReleaseNamespace == requestedChart.ReleaseNamespace { rs.ValuesHash = helmChartValuesHash + setResolvedHelmChartIdentity(ctx, c, dCtx.clusterSummary, rs, requestedChart, currentRelease, logger) } } @@ -4688,6 +4689,87 @@ func updateValueHashOnHelmChartSummary(ctx context.Context, requestedChart *conf return helmChartValuesHash, err } +// setResolvedHelmChartIdentity records, on rs, the fully resolved (post-template) chart +// identity and credentials used to produce this release, plus the actually deployed chart +// version. This lets a periodic, unrelated checker (see helmchart_outdated_check.go) compare +// against the upstream repository/registry without ever having to resolve Go templates +// itself. Never set for Flux-source-backed charts, since Sveltos ignores ChartVersion and +// leaves version resolution to Flux in that case. +func setResolvedHelmChartIdentity(ctx context.Context, c client.Client, clusterSummary *configv1beta1.ClusterSummary, + rs *configv1beta1.HelmChartSummary, requestedChart *configv1beta1.HelmChart, currentRelease *releaseInfo, + logger logr.Logger) { + + if isReferencingFluxSource(requestedChart) { + return + } + + previousChartName := rs.ChartName + previousChartVersion := rs.ChartVersion + + rs.ChartName = requestedChart.ChartName + rs.RepositoryName = requestedChart.RepositoryName + rs.RepoURL = requestedChart.RepositoryURL + + if currentRelease != nil { + rs.ChartVersion = currentRelease.ChartVersion + } + + if rs.ChartVersion != previousChartVersion { + clearStaleOutdatedVersionInfo(clusterSummary, rs, previousChartName, logger) + } + + rs.CredentialsSecretRef = nil + if requestedChart.RegistryCredentialsConfig != nil && requestedChart.RegistryCredentialsConfig.CredentialsSecretRef != nil { + // CredentialsSecretRef.Namespace may still be empty here (meaning "implicit cluster + // namespace", resolved on demand wherever this is used for a deploy, e.g. + // createRegistryClientOptions). Resolve it to a concrete namespace now, while + // clusterSummary is in scope, so a later, cluster-agnostic checker never needs to + // repeat this resolution. + secretRef := requestedChart.RegistryCredentialsConfig.CredentialsSecretRef + resolvedNamespace, err := libsveltostemplate.GetReferenceResourceNamespace(ctx, c, + clusterSummary.Spec.ClusterNamespace, clusterSummary.Spec.ClusterName, + secretRef.Namespace, clusterSummary.Spec.ClusterType) + if err != nil { + logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to resolve credentials secret namespace: %v", err)) + } else { + rs.CredentialsSecretRef = &corev1.SecretReference{ + Namespace: resolvedNamespace, + Name: secretRef.Name, + } + } + } +} + +// clearStaleOutdatedVersionInfo nils out rs's LatestVersion/LatestPatchVersion/LastCheckedTime +// and, if they were set, deletes the matching outdatedHelmChartGauge series. Called whenever a +// deploy changes ChartVersion: the periodic checker (helmchart_outdated_check.go) only +// re-evaluates a chart on its own interval, so without this a chart just upgraded to what was +// previously flagged as the latest version would keep reading as outdated - in both status and +// the metric - until the next pass. +func clearStaleOutdatedVersionInfo(clusterSummary *configv1beta1.ClusterSummary, rs *configv1beta1.HelmChartSummary, + previousChartName string, logger logr.Logger) { + + if rs.LatestVersion != nil || rs.LatestPatchVersion != nil { + profileOwnerRef, err := configv1beta1.GetProfileOwnerReference(clusterSummary) + if err != nil { + logger.V(logs.LogInfo).Info( + fmt.Sprintf("failed to get profile owner reference while clearing stale outdated-version info: %v", err)) + } else { + profileNamespace := "" + if profileOwnerRef.Kind == configv1beta1.ProfileKind { + profileNamespace = clusterSummary.Namespace + } + clearOutdatedHelmChart(profileOwnerRef.Kind, profileNamespace, profileOwnerRef.Name, + string(clusterSummary.Spec.ClusterType), clusterSummary.Spec.ClusterNamespace, clusterSummary.Spec.ClusterName, + previousChartName, rs.ReleaseNamespace, rs.ReleaseName) + } + } + + rs.LatestVersion = nil + rs.LatestPatchVersion = nil + rs.LastCheckedTime = nil +} + func getHelmChartPatchesHash(ctx context.Context, clusterSummary *configv1beta1.ClusterSummary, logger logr.Logger) ([]byte, error) { diff --git a/controllers/helmchart_outdated_check.go b/controllers/helmchart_outdated_check.go new file mode 100644 index 00000000..345b97ef --- /dev/null +++ b/controllers/helmchart_outdated_check.go @@ -0,0 +1,373 @@ +/* +Copyright 2025. projectsveltos.io. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + + configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" + logs "github.com/projectsveltos/libsveltos/lib/logsettings" +) + +const ( + // maxConcurrentHelmChartFetches bounds how many distinct chart repositories/registries are + // queried at once, so peak transient memory (index.yaml parsing) stays bounded to a handful + // of large indexes in flight rather than every distinct repo in the fleet at once. + maxConcurrentHelmChartFetches = 5 +) + +// helmChartRef records one HelmChartSummary entry, on one ClusterSummary, that references a +// given helmChartKey. Multiple ClusterSummary entries (across clusters and/or profiles) can +// point at the same key; they are fanned back out to after a single upstream fetch. Profile +// and cluster identity are captured here, at collection time, rather than re-derived later, +// since deriving profileKind/profileName requires the full ClusterSummary object (its +// OwnerReferences), not just its namespace/name. +type helmChartRef struct { + clusterSummaryNamespace string + clusterSummaryName string + releaseNamespace string + releaseName string + currentVersion string + credentialsSecretRef *corev1.SecretReference + + profileKind string + profileNamespace string + profileName string + clusterType string + clusterNamespace string + clusterName string +} + +// outdatedHelmChartMetricKey identifies one label combination of outdatedHelmChartGauge. +// Comparable (all string fields), so it can be used as a map key by +// reconcileOutdatedHelmChartMetric to track, across passes, which combinations are currently +// exposed as outdated. +type outdatedHelmChartMetricKey struct { + profileKind, profileNamespace, profileName string + clusterType, clusterNamespace, clusterName string + chartName, releaseNamespace, releaseName string +} + +func metricKeyForRef(ref *helmChartRef, chartName string) outdatedHelmChartMetricKey { + return outdatedHelmChartMetricKey{ + profileKind: ref.profileKind, profileNamespace: ref.profileNamespace, profileName: ref.profileName, + clusterType: ref.clusterType, clusterNamespace: ref.clusterNamespace, clusterName: ref.clusterName, + chartName: chartName, releaseNamespace: ref.releaseNamespace, releaseName: ref.releaseName, + } +} + +var ( + // previouslyOutdatedHelmCharts is the set of metric label combinations exposed as outdated by + // the most recently completed checkOutdatedHelmCharts pass (or carried forward from an even + // earlier pass, for a chart that could not be re-verified — see reconcileOutdatedHelmChartMetric). + // Only ever read/written from checkOutdatedHelmCharts, which RunHelmChartUpdateChecker drives + // sequentially from a single ticker loop — passes never overlap — so no lock is needed here. + previouslyOutdatedHelmCharts = map[outdatedHelmChartMetricKey]struct{}{} +) + +// outdatedHelmChartPassState accumulates, across the concurrent workers of one +// checkOutdatedHelmCharts pass, which metric label combinations were actually re-checked this +// pass and which of those came back outdated. Both are needed to reconcile the gauge correctly: +// a combination that was outdated last pass but wasn't re-checked this pass (its fetch failed) +// must be left alone, not cleared — clearing it would be reporting "confirmed fixed" when +// really nothing was learned. +type outdatedHelmChartPassState struct { + mu sync.Mutex + currentlyOutdated map[outdatedHelmChartMetricKey]struct{} + checkedThisPass map[outdatedHelmChartMetricKey]struct{} +} + +func newOutdatedHelmChartPassState() *outdatedHelmChartPassState { + return &outdatedHelmChartPassState{ + currentlyOutdated: map[outdatedHelmChartMetricKey]struct{}{}, + checkedThisPass: map[outdatedHelmChartMetricKey]struct{}{}, + } +} + +func (s *outdatedHelmChartPassState) markChecked(key *outdatedHelmChartMetricKey, outdated bool) { + s.mu.Lock() + defer s.mu.Unlock() + + s.checkedThisPass[*key] = struct{}{} + if outdated { + s.currentlyOutdated[*key] = struct{}{} + } +} + +// checkOutdatedHelmCharts runs one full pass: list every ClusterSummary, evaluate every +// HelmChartSummary entry it manages, and update both HelmReleaseSummaries[].LatestVersion/ +// LatestPatchVersion and the outdatedHelmChartGauge metric. Any single repository/registry +// fetch failure or timeout is logged and skipped; it never aborts the pass. +func checkOutdatedHelmCharts(ctx context.Context, c client.Client, logger logr.Logger) { + list := &configv1beta1.ClusterSummaryList{} + if err := c.List(ctx, list); err != nil { + logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to list ClusterSummaries, skipping this pass: %v", err)) + return + } + + refs := map[helmChartKey][]helmChartRef{} + for i := range list.Items { + collectHelmChartRefs(&list.Items[i], refs, logger) + } + + type work struct { + key helmChartKey + keyRefs []helmChartRef + } + workCh := make(chan work, len(refs)) + for key, keyRefs := range refs { + workCh <- work{key: key, keyRefs: keyRefs} + } + close(workCh) + + numWorkers := maxConcurrentHelmChartFetches + if len(refs) < numWorkers { + numWorkers = len(refs) + } + + passState := newOutdatedHelmChartPassState() + + var wg sync.WaitGroup + for range numWorkers { + wg.Add(1) + go func() { + defer wg.Done() + for w := range workCh { + processHelmChartKey(ctx, c, w.key, w.keyRefs, passState, logger) + } + }() + } + wg.Wait() + + reconcileOutdatedHelmChartMetric(passState) + trackHelmChartCheckCompleted(logger) +} + +// reconcileOutdatedHelmChartMetric updates previouslyOutdatedHelmCharts for the next pass and +// clears the gauge series for every combination confirmed, this pass, to no longer be outdated. +// Deliberately never calls outdatedHelmChartGauge.Reset(): doing so up front would make every +// still-outdated chart briefly vanish from /metrics while later workers in this same pass are +// still catching up to it, producing a false "not outdated" reading for the gap. Combinations +// found outdated this pass were already Set(1) by trackOutdatedHelmChart as they were found; +// this only needs to handle clearing the ones that are no longer outdated. +func reconcileOutdatedHelmChartMetric(passState *outdatedHelmChartPassState) { + next := make(map[outdatedHelmChartMetricKey]struct{}, len(passState.currentlyOutdated)) + for key := range passState.currentlyOutdated { + next[key] = struct{}{} + } + + for key := range previouslyOutdatedHelmCharts { + if _, stillOutdated := passState.currentlyOutdated[key]; stillOutdated { + continue // already carried forward above + } + if _, checked := passState.checkedThisPass[key]; !checked { + // Could not re-verify this pass (its chart key's fetch failed) -- leave state + // unchanged rather than guessing, and retry on the next pass. + next[key] = struct{}{} + continue + } + clearOutdatedHelmChart(key.profileKind, key.profileNamespace, key.profileName, + key.clusterType, key.clusterNamespace, key.clusterName, + key.chartName, key.releaseNamespace, key.releaseName) + } + + previouslyOutdatedHelmCharts = next +} + +// collectHelmChartRefs walks cs.Status.HelmReleaseSummaries and records a helmChartRef for +// every entry this ClusterSummary is the authoritative manager for (Status == Managing, so a +// non-owning ClusterSummary in a Conflict never double-reports the same release) and that has +// a resolved chart identity. Entries with no identity are either never successfully deployed +// yet, or Flux-source-backed (setResolvedHelmChartIdentity never populates either case) — both +// are silently skipped, not an error. +func collectHelmChartRefs(cs *configv1beta1.ClusterSummary, refs map[helmChartKey][]helmChartRef, logger logr.Logger) { + profileKind, profileNamespace, profileName := "", "", "" + if ref, err := configv1beta1.GetProfileOwnerReference(cs); err == nil { + profileKind = ref.Kind + profileName = ref.Name + if profileKind == configv1beta1.ProfileKind { + profileNamespace = cs.Namespace + } + } else { + logger.V(logs.LogVerbose).Info(fmt.Sprintf("failed to get profile owner reference for ClusterSummary %s/%s: %v", + cs.Namespace, cs.Name, err)) + } + + for i := range cs.Status.HelmReleaseSummaries { + rs := &cs.Status.HelmReleaseSummaries[i] + + if rs.Status != configv1beta1.HelmChartStatusManaging { + continue + } + if rs.ChartName == "" || rs.RepoURL == "" || rs.ChartVersion == "" { + continue + } + + key := helmChartKey{repositoryURL: rs.RepoURL, repositoryName: rs.RepositoryName, chartName: rs.ChartName} + refs[key] = append(refs[key], helmChartRef{ + clusterSummaryNamespace: cs.Namespace, + clusterSummaryName: cs.Name, + releaseNamespace: rs.ReleaseNamespace, + releaseName: rs.ReleaseName, + currentVersion: rs.ChartVersion, + credentialsSecretRef: rs.CredentialsSecretRef, + profileKind: profileKind, + profileNamespace: profileNamespace, + profileName: profileName, + clusterType: string(cs.Spec.ClusterType), + clusterNamespace: cs.Spec.ClusterNamespace, + clusterName: cs.Spec.ClusterName, + }) + } +} + +// processHelmChartKey fetches key's published versions once, then diffs and persists the +// result against every ClusterSummary referencing it. On a fetch failure, none of keyRefs are +// marked as checked in passState, so reconcileOutdatedHelmChartMetric leaves their previous +// metric state untouched rather than treating the failure as "confirmed no longer outdated". +func processHelmChartKey(ctx context.Context, c client.Client, key helmChartKey, keyRefs []helmChartRef, + passState *outdatedHelmChartPassState, logger logr.Logger) { + + // First-seen credentials win when multiple refs for the same key disagree (a documented + // limitation: this checker has no per-cluster template context to resolve which secret is + // "correct" when they genuinely differ). + var credentialsSecretRef *corev1.SecretReference + for i := range keyRefs { + if keyRefs[i].credentialsSecretRef != nil { + credentialsSecretRef = keyRefs[i].credentialsSecretRef + break + } + } + + versions, err := fetchAvailableVersions(ctx, c, key, credentialsSecretRef, logger) + if err != nil { + logger.V(logs.LogInfo).Info(fmt.Sprintf("skipping chart %q in %q: %v", key.chartName, key.repositoryURL, err)) + recordHelmChartCheckFailure() + return + } + + for i := range keyRefs { + applyDiffAndPersist(ctx, c, &keyRefs[i], key, versions, passState, logger) + } +} + +// applyDiffAndPersist diffs ref's currentVersion against versions and, if the result differs +// from what is already stored on the corresponding HelmChartSummary entry, patches it — this +// also clears a previously outdated entry once it is no longer outdated (e.g. the profile was +// updated to pin a newer version). On a diff error (unparseable currentVersion), ref is left out +// of passState entirely, same as a fetch failure: nothing was learned, so its previous metric +// state is left untouched rather than cleared. +func applyDiffAndPersist(ctx context.Context, c client.Client, ref *helmChartRef, key helmChartKey, versions []string, + passState *outdatedHelmChartPassState, logger logr.Logger) { + + latest, latestPatch, err := helmChartVersionDiff(ref.currentVersion, versions) + if err != nil { + logger.V(logs.LogDebug).Info(fmt.Sprintf("could not diff version for %q: %v", key.chartName, err)) + return + } + + metricKey := metricKeyForRef(ref, key.chartName) + + var latestStr, latestPatchStr *string + if latest != nil { + s := latest.String() + latestStr = &s + passState.markChecked(&metricKey, true) + trackOutdatedHelmChart(ref.profileKind, ref.profileNamespace, ref.profileName, + ref.clusterType, ref.clusterNamespace, ref.clusterName, + key.chartName, ref.releaseNamespace, ref.releaseName, logger) + } else { + passState.markChecked(&metricKey, false) + } + if latestPatch != nil { + s := latestPatch.String() + latestPatchStr = &s + } + + if err := patchHelmChartSummaryVersions(ctx, c, ref, latestStr, latestPatchStr, logger); err != nil { + logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to update HelmChartSummary for %s/%s: %v", + ref.releaseNamespace, ref.releaseName, err)) + } +} + +// patchHelmChartSummaryVersions updates the LatestVersion/LatestPatchVersion/LastCheckedTime +// fields on the named ClusterSummary's matching HelmChartSummary entry, mirroring the same +// Get -> mutate -> retry.RetryOnConflict(Status().Update()) pattern already used by +// updateValueHashOnHelmChartSummary. Always writes on a successful check, even when +// latest/latestPatch come back unchanged (in particular, unchanged-and-nil, the common case of +// a chart that is already up to date) -- LastCheckedTime advancing is the only signal that the +// checker is actually reaching this chart, as opposed to having stopped or never having checked +// it, so it can't be skipped just because the outdated-version verdict didn't change. +func patchHelmChartSummaryVersions(ctx context.Context, c client.Client, ref *helmChartRef, + latest, latestPatch *string, logger logr.Logger) error { + + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + cs := &configv1beta1.ClusterSummary{} + if err := c.Get(ctx, + types.NamespacedName{Namespace: ref.clusterSummaryNamespace, Name: ref.clusterSummaryName}, cs); err != nil { + return err + } + + for i := range cs.Status.HelmReleaseSummaries { + rs := &cs.Status.HelmReleaseSummaries[i] + if rs.ReleaseName != ref.releaseName || rs.ReleaseNamespace != ref.releaseNamespace { + continue + } + + rs.LatestVersion = latest + rs.LatestPatchVersion = latestPatch + now := metav1.Now() + rs.LastCheckedTime = &now + + return c.Status().Update(ctx, cs) + } + + logger.V(logs.LogDebug).Info(fmt.Sprintf("release %s/%s no longer found on ClusterSummary %s/%s", + ref.releaseNamespace, ref.releaseName, ref.clusterSummaryNamespace, ref.clusterSummaryName)) + return nil + }) +} + +// RunHelmChartUpdateChecker runs checkOutdatedHelmCharts once immediately, then on every tick +// of interval, until ctx is done. Only ever started for the default (unsharded) addon-controller +// deployment: see the shardKey == "" gate in cmd/main.go. +func RunHelmChartUpdateChecker(ctx context.Context, c client.Client, interval time.Duration, logger logr.Logger) { + checkOutdatedHelmCharts(ctx, c, logger) + + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + logger.Info("stopping helm chart update checker") + return + case <-ticker.C: + checkOutdatedHelmCharts(ctx, c, logger) + } + } +} diff --git a/controllers/helmchart_outdated_check_test.go b/controllers/helmchart_outdated_check_test.go new file mode 100644 index 00000000..dc3b423b --- /dev/null +++ b/controllers/helmchart_outdated_check_test.go @@ -0,0 +1,236 @@ +/* +Copyright 2025. projectsveltos.io. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package-internal (not controllers_test): collectHelmChartRefs is pure and dependency-free +// (no client, no envtest), and asserting on its output requires reading the unexported +// helmChartKey/helmChartRef fields it produces, so this is tested directly rather than through +// the Ginkgo/envtest suite used by the rest of this package's tests. +package controllers + +import ( + "context" + "testing" + + "github.com/go-logr/logr" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" +) + +const ( + testReleaseName = "r1" + testReleaseNamespace = "ns1" + testChartName = "nginx" + testChartVersion = "1.0.0" + testRepoURL = "https://example.com/charts" + testRepositoryName = "bitnami" + testBitnamiRepoURL = "https://charts.bitnami.com/bitnami" + testNamespace = "ns" + testClusterNamespace = "cluster-ns" + testClusterName = "cluster" + testClusterType = "Capi" + testProfileName = "profile" +) + +func TestCollectHelmChartRefs_SkipsConflictEntries(t *testing.T) { + cs := &configv1beta1.ClusterSummary{ + Status: configv1beta1.ClusterSummaryStatus{ + HelmReleaseSummaries: []configv1beta1.HelmChartSummary{ + { + ReleaseName: testReleaseName, + ReleaseNamespace: testReleaseNamespace, + Status: configv1beta1.HelmChartStatusConflict, + ChartName: testChartName, + RepoURL: testRepoURL, + ChartVersion: testChartVersion, + }, + }, + }, + } + + refs := map[helmChartKey][]helmChartRef{} + collectHelmChartRefs(cs, refs, logr.Discard()) + + if len(refs) != 0 { + t.Fatalf("expected no refs for a Conflict-status entry, got %d keys", len(refs)) + } +} + +func TestCollectHelmChartRefs_SkipsUnresolvedIdentity(t *testing.T) { + tests := []struct { + name string + rs configv1beta1.HelmChartSummary + }{ + { + name: "never deployed (no ChartName/RepoURL/ChartVersion)", + rs: configv1beta1.HelmChartSummary{ + ReleaseName: testReleaseName, ReleaseNamespace: testReleaseNamespace, Status: configv1beta1.HelmChartStatusManaging, + }, + }, + { + name: "flux-sourced (ChartName/RepoURL never populated)", + rs: configv1beta1.HelmChartSummary{ + ReleaseName: testReleaseName, ReleaseNamespace: testReleaseNamespace, Status: configv1beta1.HelmChartStatusManaging, + ChartVersion: testChartVersion, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cs := &configv1beta1.ClusterSummary{ + Status: configv1beta1.ClusterSummaryStatus{HelmReleaseSummaries: []configv1beta1.HelmChartSummary{tt.rs}}, + } + + refs := map[helmChartKey][]helmChartRef{} + collectHelmChartRefs(cs, refs, logr.Discard()) + + if len(refs) != 0 { + t.Fatalf("expected no refs, got %d keys", len(refs)) + } + }) + } +} + +func TestCollectHelmChartRefs_DedupsAcrossClusterSummaries(t *testing.T) { + makeClusterSummary := func(name string) *configv1beta1.ClusterSummary { + return &configv1beta1.ClusterSummary{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: testNamespace}, + Spec: configv1beta1.ClusterSummarySpec{ + ClusterNamespace: testClusterNamespace, ClusterName: name, ClusterType: testClusterType, + }, + Status: configv1beta1.ClusterSummaryStatus{ + HelmReleaseSummaries: []configv1beta1.HelmChartSummary{ + { + ReleaseName: testChartName, + ReleaseNamespace: "web", + Status: configv1beta1.HelmChartStatusManaging, + ChartName: testChartName, + RepositoryName: testRepositoryName, + RepoURL: testBitnamiRepoURL, + ChartVersion: testChartVersion, + }, + }, + }, + } + } + + refs := map[helmChartKey][]helmChartRef{} + collectHelmChartRefs(makeClusterSummary("cluster-a"), refs, logr.Discard()) + collectHelmChartRefs(makeClusterSummary("cluster-b"), refs, logr.Discard()) + + if len(refs) != 1 { + t.Fatalf("expected exactly one deduped chart key, got %d", len(refs)) + } + + for key, keyRefs := range refs { + if key.chartName != testChartName || key.repositoryName != testRepositoryName || key.repositoryURL != testBitnamiRepoURL { + t.Errorf("unexpected key: %+v", key) + } + if len(keyRefs) != 2 { + t.Fatalf("expected 2 refs for the deduped key, got %d", len(keyRefs)) + } + if keyRefs[0].clusterSummaryName == keyRefs[1].clusterSummaryName { + t.Errorf("expected refs from two distinct ClusterSummaries, got the same name twice: %q", + keyRefs[0].clusterSummaryName) + } + } +} + +func TestSetResolvedHelmChartIdentity_ClearsStaleOutdatedVersionInfoOnVersionChange(t *testing.T) { + newLatest := v1_1_0 + lastChecked := metav1.Now() + + clusterSummary := &configv1beta1.ClusterSummary{ + ObjectMeta: metav1.ObjectMeta{ + Name: controllerNameClusterSummary, Namespace: testNamespace, + OwnerReferences: []metav1.OwnerReference{ + {Kind: configv1beta1.ClusterProfileKind, APIVersion: configv1beta1.GroupVersion.String(), Name: testProfileName}, + }, + }, + Spec: configv1beta1.ClusterSummarySpec{ + ClusterNamespace: testClusterNamespace, ClusterName: testClusterName, ClusterType: testClusterType, + }, + } + + requestedChart := &configv1beta1.HelmChart{ + ChartName: testChartName, RepositoryName: testRepositoryName, RepositoryURL: testBitnamiRepoURL, + ReleaseName: testReleaseName, ReleaseNamespace: testReleaseNamespace, + } + + rs := &configv1beta1.HelmChartSummary{ + ReleaseName: testReleaseName, ReleaseNamespace: testReleaseNamespace, Status: configv1beta1.HelmChartStatusManaging, + ChartName: testChartName, RepositoryName: testRepositoryName, RepoURL: testBitnamiRepoURL, ChartVersion: testChartVersion, + LatestVersion: &newLatest, LastCheckedTime: &lastChecked, + } + + currentRelease := &releaseInfo{ChartVersion: newLatest} // upgrading to the version previously flagged as latest + + setResolvedHelmChartIdentity(context.TODO(), nil, clusterSummary, rs, requestedChart, currentRelease, logr.Discard()) + + if rs.ChartVersion != newLatest { + t.Errorf("expected ChartVersion to be updated to %q, got %q", newLatest, rs.ChartVersion) + } + if rs.LatestVersion != nil { + t.Errorf("expected LatestVersion to be cleared, got %v", *rs.LatestVersion) + } + if rs.LatestPatchVersion != nil { + t.Errorf("expected LatestPatchVersion to be cleared, got %v", *rs.LatestPatchVersion) + } + if rs.LastCheckedTime != nil { + t.Errorf("expected LastCheckedTime to be cleared, got %v", *rs.LastCheckedTime) + } +} + +func TestSetResolvedHelmChartIdentity_KeepsOutdatedVersionInfoWhenVersionUnchanged(t *testing.T) { + newLatest := v1_1_0 + lastChecked := metav1.Now() + + clusterSummary := &configv1beta1.ClusterSummary{ + ObjectMeta: metav1.ObjectMeta{ + Name: controllerNameClusterSummary, Namespace: testNamespace, + OwnerReferences: []metav1.OwnerReference{ + {Kind: configv1beta1.ClusterProfileKind, APIVersion: configv1beta1.GroupVersion.String(), Name: testProfileName}, + }, + }, + Spec: configv1beta1.ClusterSummarySpec{ + ClusterNamespace: testClusterNamespace, ClusterName: testClusterName, ClusterType: testClusterType, + }, + } + + requestedChart := &configv1beta1.HelmChart{ + ChartName: testChartName, RepositoryName: testRepositoryName, RepositoryURL: testBitnamiRepoURL, + ReleaseName: testReleaseName, ReleaseNamespace: testReleaseNamespace, + } + + rs := &configv1beta1.HelmChartSummary{ + ReleaseName: testReleaseName, ReleaseNamespace: testReleaseNamespace, Status: configv1beta1.HelmChartStatusManaging, + ChartName: testChartName, RepositoryName: testRepositoryName, RepoURL: testBitnamiRepoURL, ChartVersion: testChartVersion, + LatestVersion: &newLatest, LastCheckedTime: &lastChecked, + } + + // Same deployed version as before (e.g. a values-only reconcile) -- nothing to invalidate. + currentRelease := &releaseInfo{ChartVersion: testChartVersion} + + setResolvedHelmChartIdentity(context.TODO(), nil, clusterSummary, rs, requestedChart, currentRelease, logr.Discard()) + + if rs.LatestVersion == nil || *rs.LatestVersion != newLatest { + t.Errorf("expected LatestVersion to be left untouched at %q, got %v", newLatest, rs.LatestVersion) + } + if rs.LastCheckedTime == nil { + t.Errorf("expected LastCheckedTime to be left untouched, got nil") + } +} diff --git a/controllers/helmchart_semver.go b/controllers/helmchart_semver.go new file mode 100644 index 00000000..172f257e --- /dev/null +++ b/controllers/helmchart_semver.go @@ -0,0 +1,69 @@ +/* +Copyright 2025. projectsveltos.io. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "fmt" + + "github.com/Masterminds/semver/v3" +) + +// helmChartVersionDiff compares currentVersion against every version string in +// availableVersions and reports the highest version that is strictly newer, both overall and +// within currentVersion's own major.minor line. Unparseable entries in availableVersions are +// silently skipped (callers may log this themselves; kept out of this function so it stays a +// pure, easily table-tested comparison). Prerelease versions are only considered when +// currentVersion is itself a prerelease, so a stable pin is never flagged as outdated by a +// prerelease build. +// Returns (nil, nil, nil) when currentVersion is already the latest. +func helmChartVersionDiff(currentVersion string, availableVersions []string, +) (latestOverall, latestPatchSameMinor *semver.Version, err error) { + + current, err := semver.NewVersion(currentVersion) + if err != nil { + return nil, nil, fmt.Errorf("current version %q is not valid semver: %w", currentVersion, err) + } + + includePrerelease := current.Prerelease() != "" + + for i := range availableVersions { + candidate, parseErr := semver.NewVersion(availableVersions[i]) + if parseErr != nil { + continue + } + + if candidate.Prerelease() != "" && !includePrerelease { + continue + } + + if !candidate.GreaterThan(current) { + continue + } + + if latestOverall == nil || candidate.GreaterThan(latestOverall) { + latestOverall = candidate + } + + if candidate.Major() == current.Major() && candidate.Minor() == current.Minor() { + if latestPatchSameMinor == nil || candidate.GreaterThan(latestPatchSameMinor) { + latestPatchSameMinor = candidate + } + } + } + + return latestOverall, latestPatchSameMinor, nil +} diff --git a/controllers/helmchart_semver_test.go b/controllers/helmchart_semver_test.go new file mode 100644 index 00000000..1776695c --- /dev/null +++ b/controllers/helmchart_semver_test.go @@ -0,0 +1,127 @@ +/* +Copyright 2025. projectsveltos.io. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package-internal (not controllers_test): helmChartVersionDiff is a pure, dependency-free +// function, so it is table-tested directly with the standard library rather than through the +// Ginkgo/envtest suite used by the rest of this package's tests. +package controllers + +import "testing" + +const ( + v1_0_0 = "1.0.0" + v1_1_0 = "1.1.0" + v1_2_3 = "1.2.3" + v1_2_4 = "1.2.4" + v1_2_5 = "1.2.5" + v1_9_9 = "1.9.9" + v2_0_0 = "2.0.0" + notAVersion = "not-a-version" +) + +func TestHelmChartVersionDiff(t *testing.T) { + tests := []struct { + name string + current string + available []string + wantLatest string // "" means expect nil + wantLatestPatch string // "" means expect nil + wantErr bool + }{ + { + name: "newer minor and same-minor patch both found", + current: v1_2_3, + available: []string{v1_2_3, v1_2_4, "1.3.0", v2_0_0}, + wantLatest: v2_0_0, + wantLatestPatch: v1_2_4, + }, + { + name: "only a same-minor patch found", + current: v1_2_3, + available: []string{v1_2_3, v1_2_4, v1_2_5}, + wantLatest: v1_2_5, + wantLatestPatch: v1_2_5, + }, + { + name: "already the latest", + current: v2_0_0, + available: []string{v1_2_3, v1_9_9, v2_0_0}, + }, + { + name: "current not in available list but nothing newer", + current: v2_0_0, + available: []string{v1_9_9}, + }, + { + name: "unparseable current version", + current: notAVersion, + available: []string{v1_0_0}, + wantErr: true, + }, + { + name: "unparseable entries in available list are skipped", + current: v1_0_0, + available: []string{notAVersion, v1_1_0, "also-bad"}, + wantLatest: v1_1_0, + wantLatestPatch: "", + }, + { + name: "stable current excludes prereleases", + current: v1_0_0, + available: []string{"1.1.0-rc.1"}, + wantLatest: "", + }, + { + name: "prerelease current includes prereleases", + current: "1.0.0-rc.1", + available: []string{"1.0.0-rc.2", v1_0_0}, + wantLatest: v1_0_0, + wantLatestPatch: v1_0_0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + latest, latestPatch, err := helmChartVersionDiff(tt.current, tt.available) + + if tt.wantErr { + if err == nil { + t.Fatalf("expected an error, got none") + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if tt.wantLatest == "" { + if latest != nil { + t.Errorf("expected nil latest, got %v", latest) + } + } else if latest == nil || latest.String() != tt.wantLatest { + t.Errorf("expected latest %q, got %v", tt.wantLatest, latest) + } + + if tt.wantLatestPatch == "" { + if latestPatch != nil { + t.Errorf("expected nil latestPatch, got %v", latestPatch) + } + } else if latestPatch == nil || latestPatch.String() != tt.wantLatestPatch { + t.Errorf("expected latestPatch %q, got %v", tt.wantLatestPatch, latestPatch) + } + }) + } +} diff --git a/controllers/helmchart_version_source.go b/controllers/helmchart_version_source.go new file mode 100644 index 00000000..bac253c6 --- /dev/null +++ b/controllers/helmchart_version_source.go @@ -0,0 +1,225 @@ +/* +Copyright 2025. projectsveltos.io. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + "io" + "net/url" + "os" + "path" + "strings" + "time" + + "github.com/go-logr/logr" + "helm.sh/helm/v4/pkg/cli" + "helm.sh/helm/v4/pkg/getter" + "helm.sh/helm/v4/pkg/registry" + repo "helm.sh/helm/v4/pkg/repo/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + logs "github.com/projectsveltos/libsveltos/lib/logsettings" +) + +// helmChartKey identifies a distinct upstream chart to check, deduplicated across every +// HelmChartSummary entry, on every ClusterSummary, that references it. RepositoryURL, +// RepositoryName and ChartName here are always the fully resolved (post-template) values +// captured at deploy time by setResolvedHelmChartIdentity, never a template string. +type helmChartKey struct { + repositoryURL string + repositoryName string + chartName string +} + +const ( + // fetchAvailableVersionsTimeout bounds a single chart's fetch, so one slow or misbehaving + // repository/registry can never stall an entire checkOutdatedHelmCharts pass. + fetchAvailableVersionsTimeout = 30 * time.Second +) + +// resolveChartCredentials fetches secretRef (already resolved to a concrete namespace/name by +// setResolvedHelmChartIdentity at deploy time, so no cluster context is needed here) from the +// management cluster and extracts username/password for hostname. Returns empty strings, no +// error, when secretRef is nil (anonymous access). +func resolveChartCredentials(ctx context.Context, c client.Client, secretRef *corev1.SecretReference, + hostname string) (username, password string, err error) { + + if secretRef == nil { + return "", "", nil + } + + secret := &corev1.Secret{} + if getErr := c.Get(ctx, types.NamespacedName{Namespace: secretRef.Namespace, Name: secretRef.Name}, secret); getErr != nil { + return "", "", getErr + } + + return getUsernameAndPasswordFromSecret(hostname, secret) +} + +// fetchAvailableVersions returns every version string currently published for key, using the +// HTTP index.yaml path or the OCI registry Tags() path depending on key.repositoryURL. +// credentialsSecretRef, if non-nil, is used to authenticate the request; on any credential +// resolution failure this falls back to an anonymous request rather than failing outright, +// since that request may still succeed for a public chart. +func fetchAvailableVersions(ctx context.Context, c client.Client, key helmChartKey, + credentialsSecretRef *corev1.SecretReference, logger logr.Logger) ([]string, error) { + + fetchCtx, cancel := context.WithTimeout(ctx, fetchAvailableVersionsTimeout) + defer cancel() + + bareChartName := removeRepositoryNameFromChartName(key.chartName, key.repositoryName) + + parsedURL, err := url.Parse(key.repositoryURL) + if err != nil { + return nil, fmt.Errorf("failed to parse repository URL %q: %w", key.repositoryURL, err) + } + + username, password, credErr := resolveChartCredentials(fetchCtx, c, credentialsSecretRef, parsedURL.Host) + if credErr != nil { + logger.V(logs.LogDebug).Info(fmt.Sprintf("failed to resolve credentials for %s, trying anonymously: %v", + key.repositoryURL, credErr)) + } + + if registry.IsOCI(key.repositoryURL) { + return fetchOCIChartVersions(fetchCtx, key.repositoryURL, bareChartName, username, password) + } + + return fetchHTTPIndexChartVersions(fetchCtx, bareChartName, key.repositoryURL, username, password) +} + +// fetchHTTPIndexChartVersions always downloads a fresh index.yaml into a scratch directory and +// returns every published version string for chartName. Deliberately does not use +// repoAddOrUpdate/the shared repo storage cache used by the deploy path: that cache is +// download-once-per-process, which is correct for pinning a chart at deploy time but wrong for +// a periodic freshness check that must see newly published versions on every pass. +func fetchHTTPIndexChartVersions(ctx context.Context, chartName, repoURL, username, password string) ([]string, error) { + tmpDir, err := os.MkdirTemp("", "sveltos-helm-index-*") + if err != nil { + return nil, err + } + defer os.RemoveAll(tmpDir) // best-effort cleanup of a scratch temp dir + + settings := cli.New() + entry := &repo.Entry{Name: "chart", URL: repoURL, Username: username, Password: password} + chartRepo, err := repo.NewChartRepository(entry, getter.All(settings)) + if err != nil { + return nil, err + } + chartRepo.CachePath = tmpDir + + type downloadResult struct { + path string + err error + } + done := make(chan downloadResult, 1) + go func() { + p, dErr := chartRepo.DownloadIndexFile() + done <- downloadResult{p, dErr} + }() + + var indexPath string + select { + case r := <-done: + if r.err != nil { + return nil, r.err + } + indexPath = r.path + case <-ctx.Done(): + return nil, fmt.Errorf("timed out downloading index for repository %s", repoURL) + } + + idx, err := repo.LoadIndexFile(indexPath) + if err != nil { + return nil, err + } + + versions, ok := idx.Entries[chartName] + if !ok { + return nil, fmt.Errorf("chart %q not found in repository index %s", chartName, repoURL) + } + + out := make([]string, 0, len(versions)) + for i := range versions { + if versions[i] == nil || versions[i].Metadata == nil { + continue + } + out = append(out, versions[i].Version) + } + return out, nil +} + +// fetchOCIChartVersions lists every semver-parseable tag published for the chart identified by +// repoURL/repositoryName/bareChartName. Uses Helm's own registry.Client.Tags, which already +// filters to strict-semver tags and returns them sorted descending — no hand-rolled registry +// API calls needed. Plain-HTTP/insecure-TLS OCI registries are not supported by this check +// (a known, disclosed limitation): any failure here is treated as skippable by the caller. +func fetchOCIChartVersions(ctx context.Context, repoURL, bareChartName, username, password string, +) ([]string, error) { + + ref, err := buildOCIChartRef(repoURL, bareChartName) + if err != nil { + return nil, err + } + + opts := []registry.ClientOption{registry.ClientOptWriter(io.Discard)} + if username != "" && password != "" { + opts = append(opts, registry.ClientOptBasicAuth(username, password)) + } + + registryClient, err := registry.NewClient(opts...) + if err != nil { + return nil, err + } + + type tagsResult struct { + tags []string + err error + } + done := make(chan tagsResult, 1) + go func() { + t, tErr := registryClient.Tags(ref) + done <- tagsResult{t, tErr} + }() + + select { + case r := <-done: + return r.tags, r.err + case <-ctx.Done(): + return nil, fmt.Errorf("timed out listing tags for %s", ref) + } +} + +// buildOCIChartRef mirrors the OCI-ref construction in getHelmChartAndRepoName +// (handlers_helm.go), but is pure — no ClusterSummary/Flux resolution needed, since +// Flux-sourced charts never reach this point (setResolvedHelmChartIdentity never populates +// their identity fields). Returns a ref without the "oci://" prefix, as expected by +// registry.Client.Tags. +func buildOCIChartRef(repoURL, bareChartName string) (string, error) { + u, err := url.Parse(repoURL) + if err != nil { + return "", err + } + + // bareChartName is already stripped of any repository-name prefix by the caller + // (removeRepositoryNameFromChartName), so it can be joined onto the repo path directly. + u.Path = path.Join(u.Path, bareChartName) + + return strings.TrimPrefix(u.String(), "oci://"), nil +} diff --git a/controllers/metrics.go b/controllers/metrics.go index c298a6e4..98370833 100644 --- a/controllers/metrics.go +++ b/controllers/metrics.go @@ -44,6 +44,10 @@ const ( metricProfileNamespaceLabel = "profile_namespace" metricProfileNameLabel = "profile_name" + metricChartNameLabel = "chart_name" + metricReleaseNamespaceLabel = "release_namespace" + metricReleaseNameLabel = "release_name" + statusSuccess = "success" statusFailure = "failure" ) @@ -145,13 +149,57 @@ var ( []string{metricClusterTypeLabel, metricClusterNamespaceLabel, metricClusterNameLabel, metricFeatureLabel, metricProfileKindLabel, metricProfileNamespaceLabel, metricProfileNameLabel}, ) + + // outdatedHelmChartGauge is 1 when a ClusterSummary's managed HelmChart release is behind + // the latest version published upstream. Reset() once per checkOutdatedHelmCharts pass and + // re-Set(1) only for entries found outdated that pass, so stale series (charts that became + // up to date, or are no longer referenced) don't linger. Version strings are deliberately + // not labels here (high cardinality/churn) — they live only on HelmChartSummary. + outdatedHelmChartGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricNamespace, + Name: "outdated_helm_chart", + Help: "1 if a ClusterSummary's managed HelmChart release is behind the latest published upstream version", + }, + []string{metricProfileKindLabel, metricProfileNamespaceLabel, metricProfileNameLabel, + metricClusterTypeLabel, metricClusterNamespaceLabel, metricClusterNameLabel, + metricChartNameLabel, metricReleaseNamespaceLabel, metricReleaseNameLabel}, + ) + + // outdatedHelmChartCheckLastRunTimestampGauge records when checkOutdatedHelmCharts last + // completed a full pass, regardless of whether any chart was found outdated. Unlike + // outdatedHelmChartGauge (which describes chart state), this describes the checker's own + // health: if it stops advancing, the checker is stuck or has stopped running, which is + // indistinguishable from "everything is up to date" by looking at outdatedHelmChartGauge + // alone. + outdatedHelmChartCheckLastRunTimestampGauge = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: metricNamespace, + Name: "outdated_helm_chart_check_last_run_timestamp_seconds", + Help: "Unix timestamp of the last completed pass of the periodic outdated-Helm-chart checker", + }, + ) + + // outdatedHelmChartCheckFailuresCounter counts chart keys skipped in a + // checkOutdatedHelmCharts pass because their upstream repository/registry could not be + // queried (unreachable, auth failure, timeout, chart not found, etc). Not labeled by chart: + // this is meant as a coarse "is the checker healthy" signal, not a per-chart diagnostic — + // per-chart failures are logged instead. + outdatedHelmChartCheckFailuresCounter = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: metricNamespace, + Name: "outdated_helm_chart_check_failures_total", + Help: "Total number of chart keys skipped by the outdated-Helm-chart checker due to a fetch failure or timeout", + }, + ) ) //nolint:gochecknoinits // forced pattern, can't workaround func init() { // Register custom metrics with the global prometheus registry metrics.Registry.MustRegister(reconcileDurationHistogram, reconciliationCounter, reconcileOutcomeCounter, - driftCounter, matchingClustersGauge, reconcileConsecutiveFailuresGauge, reconcileLastSuccessTimestampGauge) + driftCounter, matchingClustersGauge, reconcileConsecutiveFailuresGauge, reconcileLastSuccessTimestampGauge, + outdatedHelmChartGauge, outdatedHelmChartCheckLastRunTimestampGauge, outdatedHelmChartCheckFailuresCounter) } func programDuration(elapsed time.Duration, clusterNamespace, clusterName, featureID string, @@ -281,6 +329,63 @@ func trackConsecutiveFailures(clusterNamespace, clusterName, featureID string, c clusterType, clusterNamespace, clusterName, featureID, consecutiveFailures)) } +// trackOutdatedHelmChart records that a ClusterSummary's managed HelmChart release currently +// has a newer version published upstream than its deployed ChartVersion. Only called when an +// outdated version was actually found. Pairs with clearOutdatedHelmChart, which +// checkOutdatedHelmCharts calls instead of resetting the whole gauge, so a chart's exposed +// state only ever changes when the real answer changes (see reconcileOutdatedHelmChartMetric). +func trackOutdatedHelmChart(profileKind, profileNamespace, profileName, clusterType, clusterNamespace, clusterName, + chartName, releaseNamespace, releaseName string, logger logr.Logger) { + + outdatedHelmChartGauge.With(prometheus.Labels{ + metricProfileKindLabel: profileKind, + metricProfileNamespaceLabel: profileNamespace, + metricProfileNameLabel: profileName, + metricClusterTypeLabel: clusterType, + metricClusterNamespaceLabel: clusterNamespace, + metricClusterNameLabel: clusterName, + metricChartNameLabel: chartName, + metricReleaseNamespaceLabel: releaseNamespace, + metricReleaseNameLabel: releaseName, + }).Set(1) + + logger.V(logs.LogVerbose).Info(fmt.Sprintf("tracking outdated helm chart %s (release %s/%s)", + chartName, releaseNamespace, releaseName)) +} + +// clearOutdatedHelmChart removes the gauge series for a chart confirmed, this pass, to no +// longer be outdated (or no longer referenced). Deletes the series rather than Set(0), matching +// outdatedHelmChartGauge's "absent means not outdated" convention. +func clearOutdatedHelmChart(profileKind, profileNamespace, profileName, clusterType, clusterNamespace, clusterName, + chartName, releaseNamespace, releaseName string) { + + outdatedHelmChartGauge.Delete(prometheus.Labels{ + metricProfileKindLabel: profileKind, + metricProfileNamespaceLabel: profileNamespace, + metricProfileNameLabel: profileName, + metricClusterTypeLabel: clusterType, + metricClusterNamespaceLabel: clusterNamespace, + metricClusterNameLabel: clusterName, + metricChartNameLabel: chartName, + metricReleaseNamespaceLabel: releaseNamespace, + metricReleaseNameLabel: releaseName, + }) +} + +// trackHelmChartCheckCompleted records that a checkOutdatedHelmCharts pass just finished, +// regardless of whether any individual chart fetch failed. Call once per pass. +func trackHelmChartCheckCompleted(logger logr.Logger) { + outdatedHelmChartCheckLastRunTimestampGauge.Set(float64(time.Now().Unix())) + + logger.V(logs.LogVerbose).Info("recorded completed outdated helm chart check pass") +} + +// recordHelmChartCheckFailure increments the coarse checker-health failure counter. Call once +// per chart key skipped due to a fetch failure or timeout, not once per affected ClusterSummary. +func recordHelmChartCheckFailure() { + outdatedHelmChartCheckFailuresCounter.Inc() +} + // trackLastSuccess records the timestamp of the most recent successful (Provisioned/Removed) terminal // outcome for a feature on a cluster. Only called from the success branches of updateFeatureStatus. func trackLastSuccess(clusterNamespace, clusterName, featureID string, clusterType libsveltosv1beta1.ClusterType, diff --git a/manifest/manifest.yaml b/manifest/manifest.yaml index 7e7d4eee..ebc8ac5c 100644 --- a/manifest/manifest.yaml +++ b/manifest/manifest.yaml @@ -8086,15 +8086,56 @@ spec: directly managed by ClusterProfile. items: properties: + chartName: + description: |- + ChartName, RepositoryName, RepoURL, and ChartVersion mirror the fully resolved + (post-template) HelmChart entry that produced this release, captured at deploy time. + Never set for Flux-source-backed charts (Flux owns version resolution there). + type: string + chartVersion: + type: string conflictMessage: description: |- Status indicates whether ClusterSummary can manage the helm chart or there is a conflict type: string + credentialsSecretRef: + description: |- + CredentialsSecretRef is the resolved secret reference (if any) used to authenticate + against RepoURL, captured at deploy time. Only the reference is stored, never secret + contents. + properties: + name: + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which the + secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic failureMessage: description: FailureMessage provides the specific error from the Helm engine for this release type: string + lastCheckedTime: + description: LastCheckedTime is when LatestVersion/LatestPatchVersion + were last evaluated. + format: date-time + type: string + latestPatchVersion: + description: |- + LatestPatchVersion is the highest published version sharing ChartVersion's + major.minor, if greater than ChartVersion. Distinguishes "a same-minor patch bump is + available" from "a newer minor/major line exists" (LatestVersion). + type: string + latestVersion: + description: |- + LatestVersion is the highest version currently published upstream for this chart, if + greater than ChartVersion. Populated by a periodic background check, independent of + the reconcile loop. Detection only: Sveltos never mutates ChartVersion based on this. + type: string patchesHash: description: PatchesHash represents of a unique value for the patches section @@ -8109,6 +8150,10 @@ spec: be installed minLength: 1 type: string + repoURL: + type: string + repositoryName: + type: string status: description: |- Status indicates whether ClusterSummary can manage the helm