From 05b2244b36d1cc025c38f7c6ea13e4b133e79f49 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Mon, 5 Jan 2026 19:59:22 +0100 Subject: [PATCH] feat(operator): add secret replicator and reconciler improvements Add namespace-based secret replication with label selector approach. The implementation uses configurable secret name, namespace, and target namespace selector. Cache filtering optimizes memory usage. Co-Authored-By: Timofei Larkin Co-Authored-By: Claude Signed-off-by: Andrei Kvapil --- cmd/cozystack-operator/main.go | 42 +++++ .../cozyvaluesreplicator.go | 176 ++++++++++++++++++ internal/operator/package_reconciler.go | 1 + internal/operator/packagesource_reconciler.go | 23 +-- 4 files changed, 220 insertions(+), 22 deletions(-) create mode 100644 internal/cozyvaluesreplicator/cozyvaluesreplicator.go diff --git a/cmd/cozystack-operator/main.go b/cmd/cozystack-operator/main.go index d9c79057..13921f0c 100644 --- a/cmd/cozystack-operator/main.go +++ b/cmd/cozystack-operator/main.go @@ -32,12 +32,16 @@ import ( helmv2 "github.com/fluxcd/helm-controller/api/v2" sourcev1 "github.com/fluxcd/source-controller/api/v1" sourcewatcherv1beta1 "github.com/fluxcd/source-watcher/api/v2/v1beta1" + corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log" @@ -45,6 +49,7 @@ import ( metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" + "github.com/cozystack/cozystack/internal/cozyvaluesreplicator" "github.com/cozystack/cozystack/internal/fluxinstall" "github.com/cozystack/cozystack/internal/operator" // +kubebuilder:scaffold:imports @@ -73,6 +78,9 @@ func main() { var enableHTTP2 bool var installFlux bool var cozystackVersion string + var cozyValuesSecretName string + var cozyValuesSecretNamespace string + var cozyValuesNamespaceSelector string var platformSourceURL string var platformSourceName string var platformSourceRef string @@ -92,6 +100,9 @@ func main() { flag.StringVar(&platformSourceURL, "platform-source-url", "", "Platform source URL (oci:// or https://). If specified, generates OCIRepository or GitRepository resource.") flag.StringVar(&platformSourceName, "platform-source-name", "cozystack-packages", "Name for the generated platform source resource (default: cozystack-packages)") flag.StringVar(&platformSourceRef, "platform-source-ref", "", "Reference specification as key=value pairs (e.g., 'branch=main' or 'digest=sha256:...,tag=v1.0'). For OCI: digest, semver, semverFilter, tag. For Git: branch, tag, semver, name, commit.") + flag.StringVar(&cozyValuesSecretName, "cozy-values-secret-name", "cozystack-values", "The name of the secret containing cluster-wide configuration values.") + flag.StringVar(&cozyValuesSecretNamespace, "cozy-values-secret-namespace", "cozy-system", "The namespace of the secret containing cluster-wide configuration values.") + flag.StringVar(&cozyValuesNamespaceSelector, "cozy-values-namespace-selector", "cozystack.io/system=true", "The label selector for namespaces where the cluster-wide configuration values must be replicated.") opts := zap.Options{ Development: true, @@ -110,10 +121,29 @@ func main() { os.Exit(1) } + targetNSSelector, err := labels.Parse(cozyValuesNamespaceSelector) + if err != nil { + setupLog.Error(err, "could not parse namespace label selector") + os.Exit(1) + } + // Start the controller manager setupLog.Info("Starting controller manager") mgr, err := ctrl.NewManager(config, ctrl.Options{ Scheme: scheme, + Cache: cache.Options{ + ByObject: map[client.Object]cache.ByObject{ + // Cache only Secrets named (in any namespace) + &corev1.Secret{}: { + Field: fields.OneTermEqualSelector("metadata.name", cozyValuesSecretName), + }, + + // Cache only Namespaces that match a label selector + &corev1.Namespace{}: { + Label: targetNSSelector, + }, + }, + }, Metrics: metricsserver.Options{ BindAddress: metricsAddr, SecureServing: secureMetrics, @@ -187,6 +217,18 @@ func main() { os.Exit(1) } + // Setup CozyValuesReplicator reconciler + if err := (&cozyvaluesreplicator.SecretReplicatorReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + SourceNamespace: cozyValuesSecretNamespace, + SecretName: cozyValuesSecretName, + TargetNamespaceSelector: targetNSSelector, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "CozyValuesReplicator") + os.Exit(1) + } + // +kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { diff --git a/internal/cozyvaluesreplicator/cozyvaluesreplicator.go b/internal/cozyvaluesreplicator/cozyvaluesreplicator.go new file mode 100644 index 00000000..41b65347 --- /dev/null +++ b/internal/cozyvaluesreplicator/cozyvaluesreplicator.go @@ -0,0 +1,176 @@ +/* +Copyright 2025 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cozyvaluesreplicator + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +// SecretReplicatorReconciler replicates a source secret to namespaces matching a label selector. +type SecretReplicatorReconciler struct { + client.Client + Scheme *runtime.Scheme + + // Source of truth: + SourceNamespace string + SecretName string + + // Namespaces to replicate into: + // (e.g. labels.SelectorFromSet(labels.Set{"tenant":"true"}), or metav1.LabelSelectorAsSelector(...)) + TargetNamespaceSelector labels.Selector +} + +func (r *SecretReplicatorReconciler) SetupWithManager(mgr ctrl.Manager) error { + // 1) Primary watch for requirement (b): + // Reconcile any Secret named r.SecretName in any namespace (includes source too). + // This keeps Secrets in cache and causes "copy changed -> reconcile it" to happen. + secretNameOnly := predicate.NewPredicateFuncs(func(obj client.Object) bool { + return obj.GetName() == r.SecretName + }) + + // 2) Secondary watch for requirement (c): + // When the *source* Secret changes, fan-out reconcile requests to every matching namespace. + onlySourceSecret := predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { return isSourceSecret(e.Object, r) }, + UpdateFunc: func(e event.UpdateEvent) bool { return isSourceSecret(e.ObjectNew, r) }, + DeleteFunc: func(e event.DeleteEvent) bool { return isSourceSecret(e.Object, r) }, + GenericFunc: func(e event.GenericEvent) bool { + return isSourceSecret(e.Object, r) + }, + } + + // Fan-out mapper for source Secret events -> one request per matching target namespace. + fanOutOnSourceSecret := handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, _ client.Object) []reconcile.Request { + // List namespaces *from the cache* (because we also watch Namespaces below). + var nsList corev1.NamespaceList + if err := r.List(ctx, &nsList); err != nil { + // If list fails, best-effort: return nothing; reconcile will be retried by next event. + return nil + } + + reqs := make([]reconcile.Request, 0, len(nsList.Items)) + for i := range nsList.Items { + ns := &nsList.Items[i] + if ns.Name == r.SourceNamespace { + continue + } + if r.TargetNamespaceSelector != nil && !r.TargetNamespaceSelector.Matches(labels.Set(ns.Labels)) { + continue + } + reqs = append(reqs, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: ns.Name, + Name: r.SecretName, + }, + }) + } + return reqs + }) + + // 3) Namespace watch for requirement (a): + // When a namespace is created/updated to match selector, enqueue reconcile for the Secret copy in that namespace. + enqueueOnNamespaceMatch := handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []reconcile.Request { + ns, ok := obj.(*corev1.Namespace) + if !ok { + return nil + } + if ns.Name == r.SourceNamespace { + return nil + } + if r.TargetNamespaceSelector != nil && !r.TargetNamespaceSelector.Matches(labels.Set(ns.Labels)) { + return nil + } + return []reconcile.Request{{ + NamespacedName: types.NamespacedName{ + Namespace: ns.Name, + Name: r.SecretName, + }, + }} + }) + + // Only trigger from namespace events where the label match may be (or become) true. + // (You can keep this simple; it's fine if it fires on any update—your Reconcile should be idempotent.) + namespaceMayMatter := predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + ns, ok := e.Object.(*corev1.Namespace) + return ok && (r.TargetNamespaceSelector == nil || r.TargetNamespaceSelector.Matches(labels.Set(ns.Labels))) + }, + UpdateFunc: func(e event.UpdateEvent) bool { + oldNS, okOld := e.ObjectOld.(*corev1.Namespace) + newNS, okNew := e.ObjectNew.(*corev1.Namespace) + if !okOld || !okNew { + return false + } + // Fire if it matches now OR matched before (covers transitions both ways; reconcile can decide what to do). + oldMatch := r.TargetNamespaceSelector == nil || r.TargetNamespaceSelector.Matches(labels.Set(oldNS.Labels)) + newMatch := r.TargetNamespaceSelector == nil || r.TargetNamespaceSelector.Matches(labels.Set(newNS.Labels)) + return oldMatch || newMatch + }, + DeleteFunc: func(event.DeleteEvent) bool { return false }, // nothing to do on namespace delete + GenericFunc: func(event.GenericEvent) bool { return false }, + } + + return ctrl.NewControllerManagedBy(mgr). + // (b) Watch all Secrets with the chosen name; this also ensures Secret objects are cached. + For(&corev1.Secret{}, builder.WithPredicates(secretNameOnly)). + + // (c) Add a second watch on Secret, but only for the source secret, and fan-out to all namespaces. + Watches( + &corev1.Secret{}, + fanOutOnSourceSecret, + builder.WithPredicates(onlySourceSecret), + ). + + // (a) Watch Namespaces so they're cached and so "namespace appears / starts matching" enqueues reconcile. + Watches( + &corev1.Namespace{}, + enqueueOnNamespaceMatch, + builder.WithPredicates(namespaceMayMatter), + ). + Complete(r) +} + +func isSourceSecret(obj client.Object, r *SecretReplicatorReconciler) bool { + if obj == nil { + return false + } + return obj.GetNamespace() == r.SourceNamespace && obj.GetName() == r.SecretName +} + +func (r *SecretReplicatorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + if req.Name != r.SecretName || req.Namespace == r.SourceNamespace { + return ctrl.Result{}, nil + } + originalSecret := &corev1.Secret{} + r.Get(ctx, types.NamespacedName{Namespace: r.SourceNamespace, Name: r.SecretName}, originalSecret) + replicatedSecret := originalSecret.DeepCopy() + replicatedSecret.Namespace = req.Namespace + r.Update(ctx, replicatedSecret) + return ctrl.Result{}, nil +} diff --git a/internal/operator/package_reconciler.go b/internal/operator/package_reconciler.go index f46185fd..07237fe5 100644 --- a/internal/operator/package_reconciler.go +++ b/internal/operator/package_reconciler.go @@ -887,6 +887,7 @@ func (r *PackageReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). Named("cozystack-package"). For(&cozyv1alpha1.Package{}). + Owns(&helmv2.HelmRelease{}). Watches( &cozyv1alpha1.PackageSource{}, handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []reconcile.Request { diff --git a/internal/operator/packagesource_reconciler.go b/internal/operator/packagesource_reconciler.go index dfbc096d..e79370bd 100644 --- a/internal/operator/packagesource_reconciler.go +++ b/internal/operator/packagesource_reconciler.go @@ -31,9 +31,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" - "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/reconcile" ) // PackageSourceReconciler reconciles PackageSource resources @@ -409,26 +407,7 @@ func (r *PackageSourceReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). Named("cozystack-packagesource"). For(&cozyv1alpha1.PackageSource{}). - Watches( - &sourcewatcherv1beta1.ArtifactGenerator{}, - handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []reconcile.Request { - ag, ok := obj.(*sourcewatcherv1beta1.ArtifactGenerator) - if !ok { - return nil - } - // Find the PackageSource that owns this ArtifactGenerator by ownerReference - for _, ownerRef := range ag.OwnerReferences { - if ownerRef.Kind == "PackageSource" { - return []reconcile.Request{{ - NamespacedName: types.NamespacedName{ - Name: ownerRef.Name, - }, - }} - } - } - return nil - }), - ). + Owns(&sourcewatcherv1beta1.ArtifactGenerator{}). Complete(r) }