diff --git a/cmd/cozystack-operator/main.go b/cmd/cozystack-operator/main.go index e215b779..08f7a860 100644 --- a/cmd/cozystack-operator/main.go +++ b/cmd/cozystack-operator/main.go @@ -83,6 +83,11 @@ func main() { var disableTelemetry bool var telemetryEndpoint string var telemetryInterval string + var helmReleaseInterval string + var helmReleaseRetryInterval string + var helmReleaseInstallTimeout string + var helmReleaseUpgradeTimeout string + var helmReleaseMaxHistory int var cozyValuesSecretName string var cozyValuesSecretNamespace string var cozyValuesNamespaceSelector string @@ -107,6 +112,28 @@ func main() { "Endpoint for sending telemetry data") flag.StringVar(&telemetryInterval, "telemetry-interval", "15m", "Interval between telemetry data collection (e.g. 15m, 1h)") + flag.StringVar(&helmReleaseInterval, "helmrelease-interval", "5m", + "Reconcile interval applied to HelmReleases created by the Package reconciler. "+ + "Lower values speed up dependency-blocked retries (e.g. during E2E install) at the cost of "+ + "controller load. Production default 5m matches existing behaviour.") + flag.StringVar(&helmReleaseRetryInterval, "helmrelease-retry-interval", "30s", + "Retry interval applied to Install.Strategy and Upgrade.Strategy of HelmReleases created "+ + "by the Package reconciler. With Strategy.Name=RetryOnFailure, this controls how long the "+ + "controller waits between failed install/upgrade attempts. Decoupled from --helmrelease-interval "+ + "(which is the healthy reconcile cadence) so failures recover fast without polling healthy "+ + "releases at the same fast cadence.") + flag.StringVar(&helmReleaseInstallTimeout, "helmrelease-install-timeout", "10m", + "Timeout for the Helm install action of HelmReleases created by the Package reconciler "+ + "(Spec.Install.Timeout). Bounds how long an individual Kubernetes operation (Job/hook/wait) "+ + "may take during install.") + flag.StringVar(&helmReleaseUpgradeTimeout, "helmrelease-upgrade-timeout", "10m", + "Timeout for the Helm upgrade action of HelmReleases created by the Package reconciler "+ + "(Spec.Upgrade.Timeout). Bounds how long an individual Kubernetes operation (Job/hook/wait) "+ + "may take during upgrade.") + flag.IntVar(&helmReleaseMaxHistory, "helmrelease-max-history", 5, + "Number of release revisions Helm keeps for HelmReleases created by the Package reconciler "+ + "(Spec.MaxHistory). 0 means unlimited; 5 matches Helm's default. Lower values reduce "+ + "per-release Secret accumulation in clusters that bounce HRs frequently (e.g. E2E sandboxes).") flag.StringVar(&platformSourceURL, "platform-source-url", "", "Platform source URL (oci:// or https://). If specified, generates OCIRepository or GitRepository resource.") flag.StringVar(&platformSourceName, "platform-source-name", "cozystack-platform", "Name for the generated platform source resource and PackageSource") flag.StringVar(&platformSourceRef, "platform-source-ref", "", "Reference specification as key=value pairs (e.g., 'branch=main' or 'digest=sha256:...,tag=v1.0'). For OCI: digest, semver, semverFilter, tag. For Git: branch, tag, semver, name, commit.") @@ -122,6 +149,31 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + hrIntervalDuration, err := time.ParseDuration(helmReleaseInterval) + if err != nil { + setupLog.Error(err, "invalid --helmrelease-interval value", "value", helmReleaseInterval) + os.Exit(1) + } + hrRetryIntervalDuration, err := time.ParseDuration(helmReleaseRetryInterval) + if err != nil { + setupLog.Error(err, "invalid --helmrelease-retry-interval value", "value", helmReleaseRetryInterval) + os.Exit(1) + } + hrInstallTimeoutDuration, err := time.ParseDuration(helmReleaseInstallTimeout) + if err != nil { + setupLog.Error(err, "invalid --helmrelease-install-timeout value", "value", helmReleaseInstallTimeout) + os.Exit(1) + } + hrUpgradeTimeoutDuration, err := time.ParseDuration(helmReleaseUpgradeTimeout) + if err != nil { + setupLog.Error(err, "invalid --helmrelease-upgrade-timeout value", "value", helmReleaseUpgradeTimeout) + os.Exit(1) + } + if helmReleaseMaxHistory < 0 { + setupLog.Error(fmt.Errorf("--helmrelease-max-history must be >= 0"), "invalid value", "value", helmReleaseMaxHistory) + os.Exit(1) + } + config := ctrl.GetConfigOrDie() // Create a direct client (without cache) for pre-start operations @@ -258,8 +310,13 @@ func main() { // Setup Package reconciler if err := (&operator.PackageReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + HelmReleaseInterval: hrIntervalDuration, + HelmReleaseRetryInterval: hrRetryIntervalDuration, + HelmReleaseInstallTimeout: hrInstallTimeoutDuration, + HelmReleaseUpgradeTimeout: hrUpgradeTimeoutDuration, + HelmReleaseMaxHistory: helmReleaseMaxHistory, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Package") os.Exit(1) diff --git a/hack/e2e-install-cozystack.bats b/hack/e2e-install-cozystack.bats index cf66b73f..3a02fc8c 100644 --- a/hack/e2e-install-cozystack.bats +++ b/hack/e2e-install-cozystack.bats @@ -13,6 +13,7 @@ --install \ --namespace cozy-system \ --create-namespace \ + --set cozystackOperator.helmReleaseInterval=30s \ --wait \ --timeout 2m diff --git a/internal/operator/package_reconciler.go b/internal/operator/package_reconciler.go index 0e724e49..d13cce33 100644 --- a/internal/operator/package_reconciler.go +++ b/internal/operator/package_reconciler.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "strings" + "time" cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" helmv2 "github.com/fluxcd/helm-controller/api/v2" @@ -58,7 +59,12 @@ func parseCRDPolicy(install *cozyv1alpha1.ComponentInstall) helmv2.CRDsPolicy { // PackageReconciler reconciles Package resources type PackageReconciler struct { client.Client - Scheme *runtime.Scheme + Scheme *runtime.Scheme + HelmReleaseInterval time.Duration + HelmReleaseRetryInterval time.Duration + HelmReleaseInstallTimeout time.Duration + HelmReleaseUpgradeTimeout time.Duration + HelmReleaseMaxHistory int } // +kubebuilder:rbac:groups=cozystack.io,resources=packages,verbs=get;list;watch;create;update;patch;delete @@ -214,22 +220,30 @@ func (r *PackageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct Labels: labels, }, Spec: helmv2.HelmReleaseSpec{ - Interval: metav1.Duration{Duration: 5 * 60 * 1000000000}, // 5m + Interval: metav1.Duration{Duration: r.HelmReleaseInterval}, + MaxHistory: &r.HelmReleaseMaxHistory, ChartRef: &helmv2.CrossNamespaceSourceReference{ Kind: "ExternalArtifact", Name: artifactName, Namespace: "cozy-system", }, Install: &helmv2.Install{ - Timeout: &metav1.Duration{Duration: 10 * 60 * 1000000000}, // 10m - Remediation: &helmv2.InstallRemediation{ - Retries: -1, + Timeout: &metav1.Duration{Duration: r.HelmReleaseInstallTimeout}, + // Strategy=RetryOnFailure (with RetryInterval) replaces the previous + // Remediation{Retries:-1} setup. Functionally equivalent ("retry forever + // on failure"), but decouples retry timing from spec.Interval so failed + // installs recover at HelmReleaseRetryInterval (default 30s) without + // also polling healthy releases at the same fast cadence. + Strategy: &helmv2.InstallStrategy{ + Name: string(helmv2.ActionStrategyRetryOnFailure), + RetryInterval: &metav1.Duration{Duration: r.HelmReleaseRetryInterval}, }, }, Upgrade: &helmv2.Upgrade{ - Timeout: &metav1.Duration{Duration: 10 * 60 * 1000000000}, // 10m - Remediation: &helmv2.UpgradeRemediation{ - Retries: -1, + Timeout: &metav1.Duration{Duration: r.HelmReleaseUpgradeTimeout}, + Strategy: &helmv2.UpgradeStrategy{ + Name: string(helmv2.ActionStrategyRetryOnFailure), + RetryInterval: &metav1.Duration{Duration: r.HelmReleaseRetryInterval}, }, CRDs: parseCRDPolicy(component.Install), }, diff --git a/packages/core/installer/templates/cozystack-operator.yaml b/packages/core/installer/templates/cozystack-operator.yaml index aded0995..39d51210 100644 --- a/packages/core/installer/templates/cozystack-operator.yaml +++ b/packages/core/installer/templates/cozystack-operator.yaml @@ -68,6 +68,21 @@ spec: {{- if .Values.cozystackOperator.disableTelemetry }} - --disable-telemetry {{- end }} + {{- if .Values.cozystackOperator.helmReleaseInterval }} + - --helmrelease-interval={{ .Values.cozystackOperator.helmReleaseInterval }} + {{- end }} + {{- if .Values.cozystackOperator.helmReleaseRetryInterval }} + - --helmrelease-retry-interval={{ .Values.cozystackOperator.helmReleaseRetryInterval }} + {{- end }} + {{- if .Values.cozystackOperator.helmReleaseInstallTimeout }} + - --helmrelease-install-timeout={{ .Values.cozystackOperator.helmReleaseInstallTimeout }} + {{- end }} + {{- if .Values.cozystackOperator.helmReleaseUpgradeTimeout }} + - --helmrelease-upgrade-timeout={{ .Values.cozystackOperator.helmReleaseUpgradeTimeout }} + {{- end }} + {{- if .Values.cozystackOperator.helmReleaseMaxHistory }} + - --helmrelease-max-history={{ .Values.cozystackOperator.helmReleaseMaxHistory }} + {{- end }} - --platform-source-name=cozystack-platform - --platform-source-url={{ .Values.cozystackOperator.platformSourceUrl }} {{- if .Values.cozystackOperator.platformSourceRef }} diff --git a/packages/core/installer/values.yaml b/packages/core/installer/values.yaml index eef691ea..dcd0a913 100644 --- a/packages/core/installer/values.yaml +++ b/packages/core/installer/values.yaml @@ -4,6 +4,25 @@ cozystackOperator: image: ghcr.io/cozystack/cozystack/cozystack-operator:v1.3.0@sha256:62574f12486bb40c901cf5ed484cca264405ce5810196d86555cbb27cce1ba48 platformSourceUrl: 'oci://ghcr.io/cozystack/cozystack/cozystack-packages' platformSourceRef: 'digest=sha256:a0b9ef938446b3132d3d22ad2262beb1027c48c9037b6c2346fdc2f19acd3036' + # When non-empty, overrides the operator's --helmrelease-interval flag + # (operator default: 5m). E2E sets this to 30s; production should leave empty. + helmReleaseInterval: "" + # When non-empty, overrides the operator's --helmrelease-retry-interval flag + # (operator default: 30s). Controls how fast failed install/upgrade attempts + # are retried. Decoupled from helmReleaseInterval (healthy reconcile cadence). + helmReleaseRetryInterval: "" + # When non-empty, overrides the operator's --helmrelease-install-timeout flag + # (operator default: 10m). Bounds individual k8s operations (job/hook/wait) + # during Helm install. Charts with long-running install hooks may need longer. + helmReleaseInstallTimeout: "" + # When non-empty, overrides the operator's --helmrelease-upgrade-timeout flag + # (operator default: 10m). Same semantics as helmReleaseInstallTimeout, for + # the Helm upgrade action. + helmReleaseUpgradeTimeout: "" + # When non-empty, overrides the operator's --helmrelease-max-history flag + # (operator default: 5). Number of release revisions Helm keeps per HR. + # Use 0 for unlimited; lower values reduce Secret accumulation in test clusters. + helmReleaseMaxHistory: "" # Generic variant configuration (only used when cozystackOperator.variant=generic) cozystack: # Kubernetes API server host (IP only, no protocol/port)