fix(harbor): merge BucketInfo at values root, drop targetPath

The previous fix (a21c18f9) sourced BucketInfo via Flux valuesFrom with
`targetPath: bucket.bucketInfo`. Flux runs values with `targetPath`
through Helm's `strvals.ParseInto`, which splits the value on commas as
list separators. The COSI BucketInfo JSON is comma-rich, so values
resolution bailed:

  could not resolve Secret chart values reference
  'tenant-X/harbor-X-registry-bucket' with key 'BucketInfo':
  key "\"spec\":{\"bucketName\":\"bucket-1785...\""
  has no value (cannot end with ,)

Drop `targetPath`. With only `valuesKey: BucketInfo`, helm-controller
unmarshals the value as YAML and merges at the chart's values root, so
JSON commas stay nested instead of being split. The system chart's
bucket-secret.yaml now reads `.Values.spec.bucketName` /
`.Values.spec.secretS3.{accessKeyID,accessSecretKey,endpoint}`,
guarded by nested `with` blocks so the registry-s3 Secret renders only
when secretS3 has been populated.

Gating semantics from the previous fix are preserved: with the default
`optional: false`, helm-controller still refuses to compose values
until the COSI BucketAccess controller writes `BucketInfo` into the
Secret, and the value's content still drives the HR config-digest.

Verified via `helm template` against the system chart with three
scenarios: missing `spec` and `spec` without `secretS3` render
nothing; full BucketInfo renders all five S3 env keys correctly.

Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
Assisted-By: Claude <noreply@anthropic.com>

.github/workflows/pull-requests.yaml

@@ -227,27 +227,18 @@ jobs:
           until make SANDBOX_NAME=$SANDBOX_NAME prepare-env; do
             attempt=$((attempt + 1))
             if [ $attempt -ge 3 ]; then
-              echo "❌ Attempt $attempt failed, exiting..."
+              echo "Attempt $attempt failed, exiting..."
               exit 1
             fi
-            echo "❌ Attempt $attempt failed, retrying..."
+            echo "Attempt $attempt failed, retrying..."
           done
-          echo "✅ The task completed successfully after $attempt attempts"
+          echo "Prepare environment completed after $attempt attempts"
 
       # ▸ Install Cozystack
       - name: Install Cozystack into sandbox
         run: |
           cd /tmp/$SANDBOX_NAME
-          attempt=0
-          until make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME install-cozystack; do
-            attempt=$((attempt + 1))
-            if [ $attempt -ge 3 ]; then
-              echo "❌ Attempt $attempt failed, exiting..."
-              exit 1
-            fi
-            echo "❌ Attempt $attempt failed, retrying..."
-          done
-          echo "✅ The task completed successfully after $attempt attempts"
+          make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME install-cozystack
 
       - name: Run OpenAPI tests
         run: |
@@ -262,23 +253,18 @@ jobs:
           failed_tests=""
           for app in $(ls hack/e2e-apps/*.bats | xargs -n1 basename | cut -d. -f1); do
             echo "::group::Testing $app"
-            attempt=0
-            success=false
-            until [ $attempt -ge 3 ]; do
-              if make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-apps-$app; then
-                success=true
-                break
-              fi
-              attempt=$((attempt + 1))
-              echo "❌ Attempt $attempt failed, retrying..."
-            done
-            if [ "$success" = true ]; then
+            if make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-apps-$app; then
+              echo "::endgroup::"
               echo "✅ Test $app completed successfully"
             else
-              echo "❌ Test $app failed after $attempt attempts"
+              echo "::endgroup::"
+              echo "❌ Test $app failed (no retry — see diagnostics below)"
               failed_tests="$failed_tests $app"
+              echo "::group::Diagnostics for $app"
+              kubectl get hr -A -o wide 2>&1 | tail -50 || true
+              kubectl get events -A --sort-by=.lastTimestamp 2>&1 | tail -30 || true
+              echo "::endgroup::"
             fi
-            echo "::endgroup::"
           done
           if [ -n "$failed_tests" ]; then
             echo "❌ Failed tests:$failed_tests"

cmd/cozystack-operator/main.go

@@ -83,6 +83,11 @@ func main() {
 	var disableTelemetry bool
 	var telemetryEndpoint string
 	var telemetryInterval string
+	var helmReleaseInterval string
+	var helmReleaseRetryInterval string
+	var helmReleaseInstallTimeout string
+	var helmReleaseUpgradeTimeout string
+	var helmReleaseMaxHistory int
 	var cozyValuesSecretName string
 	var cozyValuesSecretNamespace string
 	var cozyValuesNamespaceSelector string
@@ -107,6 +112,28 @@ func main() {
 		"Endpoint for sending telemetry data")
 	flag.StringVar(&telemetryInterval, "telemetry-interval", "15m",
 		"Interval between telemetry data collection (e.g. 15m, 1h)")
+	flag.StringVar(&helmReleaseInterval, "helmrelease-interval", "5m",
+		"Reconcile interval applied to HelmReleases created by the Package reconciler. "+
+			"Lower values speed up dependency-blocked retries (e.g. during E2E install) at the cost of "+
+			"controller load. Production default 5m matches existing behaviour.")
+	flag.StringVar(&helmReleaseRetryInterval, "helmrelease-retry-interval", "30s",
+		"Retry interval applied to Install.Strategy and Upgrade.Strategy of HelmReleases created "+
+			"by the Package reconciler. With Strategy.Name=RetryOnFailure, this controls how long the "+
+			"controller waits between failed install/upgrade attempts. Decoupled from --helmrelease-interval "+
+			"(which is the healthy reconcile cadence) so failures recover fast without polling healthy "+
+			"releases at the same fast cadence.")
+	flag.StringVar(&helmReleaseInstallTimeout, "helmrelease-install-timeout", "10m",
+		"Timeout for the Helm install action of HelmReleases created by the Package reconciler "+
+			"(Spec.Install.Timeout). Bounds how long an individual Kubernetes operation (Job/hook/wait) "+
+			"may take during install.")
+	flag.StringVar(&helmReleaseUpgradeTimeout, "helmrelease-upgrade-timeout", "10m",
+		"Timeout for the Helm upgrade action of HelmReleases created by the Package reconciler "+
+			"(Spec.Upgrade.Timeout). Bounds how long an individual Kubernetes operation (Job/hook/wait) "+
+			"may take during upgrade.")
+	flag.IntVar(&helmReleaseMaxHistory, "helmrelease-max-history", 5,
+		"Number of release revisions Helm keeps for HelmReleases created by the Package reconciler "+
+			"(Spec.MaxHistory). 0 means unlimited; 5 matches Helm's default. Lower values reduce "+
+			"per-release Secret accumulation in clusters that bounce HRs frequently (e.g. E2E sandboxes).")
 	flag.StringVar(&platformSourceURL, "platform-source-url", "", "Platform source URL (oci:// or https://). If specified, generates OCIRepository or GitRepository resource.")
 	flag.StringVar(&platformSourceName, "platform-source-name", "cozystack-platform", "Name for the generated platform source resource and PackageSource")
 	flag.StringVar(&platformSourceRef, "platform-source-ref", "", "Reference specification as key=value pairs (e.g., 'branch=main' or 'digest=sha256:...,tag=v1.0'). For OCI: digest, semver, semverFilter, tag. For Git: branch, tag, semver, name, commit.")
@@ -122,6 +149,31 @@ func main() {
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
+	hrIntervalDuration, err := time.ParseDuration(helmReleaseInterval)
+	if err != nil {
+		setupLog.Error(err, "invalid --helmrelease-interval value", "value", helmReleaseInterval)
+		os.Exit(1)
+	}
+	hrRetryIntervalDuration, err := time.ParseDuration(helmReleaseRetryInterval)
+	if err != nil {
+		setupLog.Error(err, "invalid --helmrelease-retry-interval value", "value", helmReleaseRetryInterval)
+		os.Exit(1)
+	}
+	hrInstallTimeoutDuration, err := time.ParseDuration(helmReleaseInstallTimeout)
+	if err != nil {
+		setupLog.Error(err, "invalid --helmrelease-install-timeout value", "value", helmReleaseInstallTimeout)
+		os.Exit(1)
+	}
+	hrUpgradeTimeoutDuration, err := time.ParseDuration(helmReleaseUpgradeTimeout)
+	if err != nil {
+		setupLog.Error(err, "invalid --helmrelease-upgrade-timeout value", "value", helmReleaseUpgradeTimeout)
+		os.Exit(1)
+	}
+	if helmReleaseMaxHistory < 0 {
+		setupLog.Error(fmt.Errorf("--helmrelease-max-history must be >= 0"), "invalid value", "value", helmReleaseMaxHistory)
+		os.Exit(1)
+	}
+
 	config := ctrl.GetConfigOrDie()
 
 	// Create a direct client (without cache) for pre-start operations
@@ -258,8 +310,13 @@ func main() {
 
 	// Setup Package reconciler
 	if err := (&operator.PackageReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
+		Client:                    mgr.GetClient(),
+		Scheme:                    mgr.GetScheme(),
+		HelmReleaseInterval:       hrIntervalDuration,
+		HelmReleaseRetryInterval:  hrRetryIntervalDuration,
+		HelmReleaseInstallTimeout: hrInstallTimeoutDuration,
+		HelmReleaseUpgradeTimeout: hrUpgradeTimeoutDuration,
+		HelmReleaseMaxHistory:     helmReleaseMaxHistory,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Package")
 		os.Exit(1)

hack/cozyreport-summary.sh

@@ -0,0 +1,66 @@
+#!/bin/sh
+# Emit a human-readable summary of "what is broken" to a single file.
+# Reads the live cluster (not the report dir) so it can use kubectl JSONPath.
+# Usage: cozyreport-summary.sh > summary.txt
+set -eu
+
+echo "# Cozystack E2E Diagnostic Summary"
+echo "Generated: $(date -Iseconds)"
+echo
+
+echo "## HelmReleases not Ready"
+echo
+kubectl get hr -A --no-headers 2>/dev/null \
+  | awk '$4 != "True" {printf "  %s/%s — %s\n", $1, $2, $5}' \
+  | head -40
+echo
+
+echo "## Pods not Running/Succeeded"
+echo
+kubectl get pod -A --no-headers 2>/dev/null \
+  | awk '$4 !~ /Running|Succeeded|Completed/ {printf "  %s/%s — %s (restarts=%s, age=%s)\n", $1, $2, $4, $5, $6}' \
+  | head -40
+echo
+
+echo "## ImagePullBackOff / ErrImagePull"
+echo
+kubectl get pod -A --no-headers 2>/dev/null \
+  | awk '$4 ~ /ImagePullBackOff|ErrImagePull/ {printf "  %s/%s — %s\n", $1, $2, $4}'
+echo
+
+echo "## OOMKilled in last 30 min"
+echo
+kubectl get events -A --field-selector reason=OOMKilling --sort-by=.lastTimestamp 2>/dev/null \
+  | tail -20
+echo
+
+echo "## Recent Warning events (top 30)"
+echo
+kubectl get events -A --field-selector type=Warning --sort-by=.lastTimestamp 2>/dev/null \
+  | tail -30
+echo
+
+echo "## cert-manager: Certificates not Ready"
+echo
+if kubectl get crd certificates.cert-manager.io >/dev/null 2>&1; then
+  kubectl get certificates.cert-manager.io -A --no-headers 2>/dev/null \
+    | awk '$3 != "True" {printf "  %s/%s — Ready=%s\n", $1, $2, $3}'
+fi
+echo
+
+echo "## Flux Sources not Ready"
+echo
+for kind in helmrepositories.source.toolkit.fluxcd.io ocirepositories.source.toolkit.fluxcd.io gitrepositories.source.toolkit.fluxcd.io; do
+  kubectl get $kind -A --no-headers 2>/dev/null \
+    | awk -v k="${kind%%.*}" '$4 != "True" {printf "  %s %s/%s — Ready=%s\n", k, $1, $2, $4}'
+done
+echo
+
+echo "## Storage: PVCs not Bound, PVs not Bound"
+echo
+kubectl get pvc -A --no-headers 2>/dev/null | awk '$3 != "Bound" {printf "  PVC %s/%s — %s\n", $1, $2, $3}'
+kubectl get pv --no-headers 2>/dev/null    | awk '$5 != "Bound" {printf "  PV %s — %s\n", $1, $5}'
+echo
+
+echo "## Node Conditions"
+kubectl get nodes -o custom-columns=NAME:.metadata.name,READY:.status.conditions[?\(@.type==\"Ready\"\)].status,DISK:.status.conditions[?\(@.type==\"DiskPressure\"\)].status,MEM:.status.conditions[?\(@.type==\"MemoryPressure\"\)].status 2>/dev/null

hack/cozyreport.sh

@@ -9,10 +9,13 @@ command -V kubectl >/dev/null || exit $?
 command -V tar >/dev/null || exit $?
 
 # -- cozystack module
-
 echo "Collecting Cozystack information..."
 mkdir -p $REPORT_DIR/cozystack
 kubectl get deploy -n cozy-system cozystack -o jsonpath='{.spec.template.spec.containers[0].image}' > $REPORT_DIR/cozystack/image.txt 2>&1
+if kubectl get deploy -n cozy-system cozystack-operator >/dev/null 2>&1; then
+  kubectl logs -n cozy-system deploy/cozystack-operator --tail=2000 > $REPORT_DIR/cozystack/operator.log 2>&1
+  kubectl logs -n cozy-system deploy/cozystack-operator --tail=2000 --previous > $REPORT_DIR/cozystack/operator-previous.log 2>&1 || true
+fi
 kubectl get cm -n cozy-system --no-headers | awk '$1 ~ /^cozystack/' |
   while read NAME _; do
     DIR=$REPORT_DIR/cozystack/configs
@@ -20,6 +23,46 @@ kubectl get cm -n cozy-system --no-headers | awk '$1 ~ /^cozystack/' |
     kubectl get cm -n cozy-system $NAME -o yaml > $DIR/$NAME.yaml 2>&1
   done
 
+# -- flux module
+echo "Collecting Flux controller state..."
+mkdir -p $REPORT_DIR/flux
+for ctrl in helm-controller source-controller notification-controller kustomize-controller; do
+  if kubectl get deploy -n cozy-fluxcd $ctrl >/dev/null 2>&1; then
+    kubectl logs -n cozy-fluxcd deploy/$ctrl --tail=2000 > $REPORT_DIR/flux/$ctrl.log 2>&1
+    kubectl logs -n cozy-fluxcd deploy/$ctrl --tail=2000 --previous > $REPORT_DIR/flux/$ctrl-previous.log 2>&1 || true
+  fi
+done
+
+echo "Collecting Flux sources..."
+for kind in helmrepositories.source.toolkit.fluxcd.io ocirepositories.source.toolkit.fluxcd.io gitrepositories.source.toolkit.fluxcd.io externalartifacts.source.toolkit.fluxcd.io; do
+  short=${kind%%.*}
+  kubectl get $kind -A > $REPORT_DIR/flux/$short.txt 2>&1
+  kubectl get $kind -A -o yaml > $REPORT_DIR/flux/$short.yaml 2>&1
+done
+
+# -- cert-manager module
+if kubectl get crd certificates.cert-manager.io >/dev/null 2>&1; then
+  echo "Collecting cert-manager state..."
+  DIR=$REPORT_DIR/cert-manager
+  mkdir -p $DIR
+  kubectl get certificates.cert-manager.io -A > $DIR/certificates.txt 2>&1
+  kubectl get certificaterequests.cert-manager.io -A > $DIR/certificaterequests.txt 2>&1
+  kubectl get orders.acme.cert-manager.io -A > $DIR/orders.txt 2>&1
+  kubectl get challenges.acme.cert-manager.io -A > $DIR/challenges.txt 2>&1
+  # Per non-Ready cert: full yaml + describe
+  kubectl get certificates.cert-manager.io -A --no-headers 2>/dev/null | awk '$3 != "True"' | \
+    while read NAMESPACE NAME _; do
+      cdir=$DIR/certificates/$NAMESPACE/$NAME
+      mkdir -p $cdir
+      kubectl get certificates.cert-manager.io -n $NAMESPACE $NAME -o yaml > $cdir/cert.yaml 2>&1
+      kubectl describe certificates.cert-manager.io -n $NAMESPACE $NAME > $cdir/describe.txt 2>&1
+    done
+  if kubectl get deploy -n cozy-cert-manager cert-manager >/dev/null 2>&1; then
+    kubectl logs -n cozy-cert-manager deploy/cert-manager --tail=2000 > $DIR/cert-manager.log 2>&1
+    kubectl logs -n cozy-cert-manager deploy/cert-manager-webhook --tail=2000 > $DIR/cert-manager-webhook.log 2>&1
+  fi
+fi
+
 # -- kubernetes module
 
 echo "Collecting Kubernetes information..."
@@ -46,6 +89,13 @@ kubectl get ns --no-headers | awk '$2 != "Active"' |
     kubectl describe ns $NAME > $DIR/describe.txt 2>&1
   done
 
+echo "Collecting events..."
+kubectl get events -A --sort-by=.lastTimestamp > $REPORT_DIR/kubernetes/events.txt 2>&1
+# Filter to warning-class and recent for quick triage
+kubectl get events -A --sort-by=.lastTimestamp \
+  -o jsonpath='{range .items[?(@.type!="Normal")]}{.lastTimestamp}{"\t"}{.involvedObject.namespace}/{.involvedObject.kind}/{.involvedObject.name}{"\t"}{.reason}{"\t"}{.message}{"\n"}{end}' \
+  > $REPORT_DIR/kubernetes/events-warnings.txt 2>&1
+
 echo "Collecting helmreleases..."
 kubectl get hr -A > $REPORT_DIR/kubernetes/helmreleases.txt 2>&1
 kubectl get hr -A --no-headers | awk '$4 != "True"' | \
@@ -54,6 +104,13 @@ kubectl get hr -A --no-headers | awk '$4 != "True"' | \
     mkdir -p $DIR
     kubectl get hr -n $NAMESPACE $NAME -o yaml > $DIR/hr.yaml 2>&1
     kubectl describe hr -n $NAMESPACE $NAME > $DIR/describe.txt 2>&1
+    # Helm storage secrets: latest revision per release.
+    kubectl get secret -n $NAMESPACE -l owner=helm,name=$NAME --sort-by='.metadata.creationTimestamp' --no-headers 2>/dev/null | \
+      tail -1 | awk '{print $1}' | while read SECRET; do
+        [ -z "$SECRET" ] && continue
+        kubectl get secret -n $NAMESPACE $SECRET -o jsonpath='{.data.release}' 2>/dev/null \
+          | base64 -d | base64 -d | gzip -d > $DIR/helm-release.json 2>&1 || true
+      done
   done
 
 echo "Collecting packages..."
@@ -76,6 +133,23 @@ kubectl get packagesources --no-headers | awk '$3 != "True"' | \
     kubectl describe packagesource $NAME > $DIR/describe.txt 2>&1
   done
 
+echo "Collecting cozystack apps..."
+DIR=$REPORT_DIR/cozystack-apps
+mkdir -p $DIR
+for kind in applications.apps.cozystack.io applicationdefinitions.apps.cozystack.io tenants.apps.cozystack.io; do
+  short=${kind%%.*}
+  if kubectl get crd $kind >/dev/null 2>&1; then
+    kubectl get $kind -A > $DIR/$short.txt 2>&1
+    kubectl get $kind -A --no-headers 2>/dev/null | awk 'NF >= 3 && $NF != "True" && $NF != "Ready"' | \
+      while read NAMESPACE NAME _; do
+        d=$DIR/$short/$NAMESPACE/$NAME
+        mkdir -p $d
+        kubectl get $kind -n $NAMESPACE $NAME -o yaml > $d/$short.yaml 2>&1
+        kubectl describe $kind -n $NAMESPACE $NAME > $d/describe.txt 2>&1
+      done
+  fi
+done
+
 echo "Collecting pods..."
 kubectl get pod -A -o wide > $REPORT_DIR/kubernetes/pods.txt 2>&1
 kubectl get pod -A --no-headers | awk '$4 !~ /Running|Succeeded|Completed/' |
@@ -157,8 +231,28 @@ if kubectl get deploy -n cozy-linstor linstor-controller >/dev/null 2>&1; then
   kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor --no-color r l > $DIR/resources.txt 2>&1
 fi
 
+# -- sandbox-host module
+
+echo "Collecting sandbox host context..."
+DIR=$REPORT_DIR/sandbox-host
+mkdir -p $DIR
+df -h > $DIR/df.txt 2>&1
+free -m > $DIR/free.txt 2>&1
+ps auxww > $DIR/ps.txt 2>&1
+dmesg | tail -200 > $DIR/dmesg.txt 2>&1 || true
+if [ -f /workspace/talosconfig ]; then
+  for node in 192.168.123.11 192.168.123.12 192.168.123.13; do
+    talosctl --talosconfig /workspace/talosconfig -n $node dmesg --tail=200 > $DIR/talos-$node-dmesg.txt 2>&1 || true
+    talosctl --talosconfig /workspace/talosconfig -n $node logs kubelet --tail=500 > $DIR/talos-$node-kubelet.log 2>&1 || true
+    talosctl --talosconfig /workspace/talosconfig -n $node logs containerd --tail=500 > $DIR/talos-$node-containerd.log 2>&1 || true
+  done
+fi
+
 # -- finalization
 
+echo "Generating summary..."
+hack/cozyreport-summary.sh > $REPORT_DIR/summary.txt 2>&1 || true
+
 echo "Creating archive..."
 tar -czf $REPORT_NAME.tgz -C $REPORT_PDIR .
 echo "Report created: $REPORT_NAME.tgz"

hack/e2e-apps/bucket.bats

@@ -20,8 +20,11 @@ EOF
 
   # Wait for the bucket to be ready
   kubectl -n tenant-test wait hr bucket-${name} --timeout=100s --for=condition=ready
+  timeout 60 sh -ec "until kubectl -n tenant-test get bucketclaims.objectstorage.k8s.io bucket-${name} >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait bucketclaims.objectstorage.k8s.io bucket-${name} --timeout=300s --for=jsonpath='{.status.bucketReady}'
+  timeout 60 sh -ec "until kubectl -n tenant-test get bucketaccesses.objectstorage.k8s.io bucket-${name}-admin >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait bucketaccesses.objectstorage.k8s.io bucket-${name}-admin --timeout=300s --for=jsonpath='{.status.accessGranted}'
+  timeout 60 sh -ec "until kubectl -n tenant-test get bucketaccesses.objectstorage.k8s.io bucket-${name}-viewer >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait bucketaccesses.objectstorage.k8s.io bucket-${name}-viewer --timeout=300s --for=jsonpath='{.status.accessGranted}'
 
   # Get admin (readwrite) credentials

hack/e2e-apps/clickhouse.bats

@@ -35,9 +35,12 @@ spec:
   resources: {}
   resourcesPreset: "nano"
 EOF
-  sleep 5
+  # Wait for the operator to materialise the HelmRelease before kubectl wait
+  # kicks in (kubectl wait errors immediately if the object does not exist yet).
+  timeout 60 sh -ec "until kubectl -n tenant-test get hr clickhouse-$name >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait hr clickhouse-$name --timeout=20s --for=condition=ready
   timeout 180 sh -ec "until kubectl -n tenant-test get svc chendpoint-clickhouse-$name -o jsonpath='{.spec.ports[*].port}' | grep -q '8123 9000'; do sleep 10; done"
+  timeout 60 sh -ec "until kubectl -n tenant-test get statefulset.apps/chi-clickhouse-$name-clickhouse-0-0 >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait statefulset.apps/chi-clickhouse-$name-clickhouse-0-0 --timeout=120s --for=jsonpath='{.status.replicas}'=1
   timeout 80 sh -ec "until kubectl -n tenant-test get endpoints chi-clickhouse-$name-clickhouse-0-0 -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done"
   timeout 100 sh -ec "until kubectl -n tenant-test get svc chi-clickhouse-$name-clickhouse-0-0 -o jsonpath='{.spec.ports[*].port}' | grep -q '9000 8123 9009'; do sleep 10; done"

hack/e2e-apps/external-dns.bats

@@ -18,7 +18,9 @@ spec:
     - example.com
 EOF
 
-  sleep 5
+  # Wait for the operator to materialise the HelmRelease before kubectl wait
+  # kicks in (kubectl wait errors immediately if the object does not exist yet).
+  timeout 60 sh -ec "until kubectl -n tenant-test get hr ${name} >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait hr ${name} --timeout=120s --for=condition=ready
   kubectl -n tenant-test wait hr ${name}-system --timeout=120s --for=condition=ready
   timeout 60 sh -ec "until kubectl -n tenant-test get deploy -l app.kubernetes.io/instance=${name}-system -o jsonpath='{.items[0].status.readyReplicas}' | grep -q '1'; do sleep 5; done"
@@ -41,7 +43,9 @@ spec:
     - example.org
 EOF
 
-  sleep 5
+  # Wait for the operator to materialise the HelmRelease before kubectl wait
+  # kicks in (kubectl wait errors immediately if the object does not exist yet).
+  timeout 60 sh -ec "until kubectl -n tenant-test get hr ${name} >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait hr ${name} --timeout=120s --for=condition=ready
   kubectl -n tenant-test wait hr ${name}-system --timeout=120s --for=condition=ready
   timeout 60 sh -ec "until kubectl -n tenant-test get deploy -l app.kubernetes.io/instance=${name}-system -o jsonpath='{.items[0].status.readyReplicas}' | grep -q '1'; do sleep 5; done"

hack/e2e-apps/foundationdb.bats

@@ -42,7 +42,9 @@ spec:
   imageType: "unified"
   automaticReplacements: true
 EOF
-  sleep 15
+  # Wait for the operator to materialise the HelmRelease before kubectl wait
+  # kicks in (kubectl wait errors immediately if the object does not exist yet).
+  timeout 60 sh -ec "until kubectl -n tenant-test get hr foundationdb-$name >/dev/null 2>&1; do sleep 2; done"
 
   # Wait for HelmRelease to be ready
   kubectl -n tenant-test wait hr foundationdb-$name --timeout=300s --for=condition=ready

hack/e2e-apps/harbor.bats

@@ -38,12 +38,16 @@ spec:
     size: 1Gi
     replicas: 1
 EOF
-  sleep 5
+  # Wait for the operator to materialise the HelmRelease before kubectl wait
+  # kicks in (kubectl wait errors immediately if the object does not exist yet).
+  timeout 60 sh -ec "until kubectl -n tenant-test get hr $release >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait hr $release --timeout=60s --for=condition=ready
 
   # Wait for COSI to provision bucket
+  timeout 60 sh -ec "until kubectl -n tenant-test get bucketclaims.objectstorage.k8s.io $release-registry >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait bucketclaims.objectstorage.k8s.io $release-registry \
     --timeout=300s --for=jsonpath='{.status.bucketReady}'=true
+  timeout 60 sh -ec "until kubectl -n tenant-test get bucketaccesses.objectstorage.k8s.io $release-registry >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait bucketaccesses.objectstorage.k8s.io $release-registry \
     --timeout=60s --for=jsonpath='{.status.accessGranted}'=true
 
@@ -64,8 +68,11 @@ EOF
     kubectl -n tenant-test get secret $release-registry-bucket -o jsonpath='{.data.BucketInfo}' 2>&1 | base64 -d 2>&1 || true
     false
   }
+  timeout 60 sh -ec "until kubectl -n tenant-test get deploy $release-core >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait deploy $release-core --timeout=120s --for=condition=available
+  timeout 60 sh -ec "until kubectl -n tenant-test get deploy $release-registry >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait deploy $release-registry --timeout=120s --for=condition=available
+  timeout 60 sh -ec "until kubectl -n tenant-test get deploy $release-portal >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait deploy $release-portal --timeout=120s --for=condition=available
   kubectl -n tenant-test get secret $release-credentials -o jsonpath='{.data.admin-password}' | base64 --decode | grep -q '.'
   kubectl -n tenant-test get secret $release-credentials -o jsonpath='{.data.url}' | base64 --decode | grep -q 'https://'

hack/e2e-apps/kafka.bats

@@ -39,8 +39,11 @@ spec:
       partitions: 1
       replicas: 2
 EOF
-  sleep 5
+  # Wait for the operator to materialise the HelmRelease before kubectl wait
+  # kicks in (kubectl wait errors immediately if the object does not exist yet).
+  timeout 60 sh -ec "until kubectl -n tenant-test get hr kafka-$name >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait hr kafka-$name --timeout=30s --for=condition=ready
+  timeout 60 sh -ec "until kubectl -n tenant-test get kafkas test >/dev/null 2>&1; do sleep 2; done"
   kubectl wait kafkas -n tenant-test test --timeout=300s --for=condition=ready
   timeout 60 sh -ec "until kubectl -n tenant-test get pvc data-kafka-$name-zookeeper-0; do sleep 10; done"
   kubectl -n tenant-test wait pvc data-kafka-$name-zookeeper-0 --timeout=50s --for=jsonpath='{.status.phase}'=Bound

hack/e2e-apps/mariadb.bats

@@ -35,13 +35,17 @@ spec:
   resources: {}
   resourcesPreset: "nano"
 EOF
-  sleep 5
+  # Wait for the operator to materialise the HelmRelease before kubectl wait
+  # kicks in (kubectl wait errors immediately if the object does not exist yet).
+  timeout 60 sh -ec "until kubectl -n tenant-test get hr mariadb-$name >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait hr mariadb-$name --timeout=30s --for=condition=ready
   timeout 80 sh -ec "until kubectl -n tenant-test get svc mariadb-$name -o jsonpath='{.spec.ports[0].port}' | grep -q '3306'; do sleep 10; done"
   timeout 80 sh -ec "until kubectl -n tenant-test get endpoints mariadb-$name -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done"
+  timeout 60 sh -ec "until kubectl -n tenant-test get statefulset.apps/mariadb-$name >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait statefulset.apps/mariadb-$name --timeout=110s --for=jsonpath='{.status.replicas}'=2
   timeout 80 sh -ec "until kubectl -n tenant-test get svc mariadb-$name-metrics -o jsonpath='{.spec.ports[0].port}' | grep -q '9104'; do sleep 10; done"
   timeout 40 sh -ec "until kubectl -n tenant-test get endpoints mariadb-$name-metrics -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done"
+  timeout 60 sh -ec "until kubectl -n tenant-test get deployment.apps/mariadb-$name-metrics >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait deployment.apps/mariadb-$name-metrics --timeout=90s --for=jsonpath='{.status.replicas}'=1
   kubectl -n tenant-test delete mariadbs.apps.cozystack.io $name
 }

hack/e2e-apps/mongodb.bats

@@ -26,7 +26,9 @@ spec:
   backup:
     enabled: false
 EOF
-  sleep 5
+  # Wait for the operator to materialise the HelmRelease before kubectl wait
+  # kicks in (kubectl wait errors immediately if the object does not exist yet).
+  timeout 60 sh -ec "until kubectl -n tenant-test get hr mongodb-$name >/dev/null 2>&1; do sleep 2; done"
   # Wait for HelmRelease
   kubectl -n tenant-test wait hr mongodb-$name --timeout=60s --for=condition=ready
   # Wait for MongoDB service (port 27017)
@@ -34,6 +36,7 @@ EOF
   # Wait for endpoints
   timeout 180 sh -ec "until kubectl -n tenant-test get endpoints mongodb-$name-rs0 -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done"
   # Wait for StatefulSet replicas
+  timeout 60 sh -ec "until kubectl -n tenant-test get statefulset.apps/mongodb-$name-rs0 >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait statefulset.apps/mongodb-$name-rs0 --timeout=300s --for=jsonpath='{.status.replicas}'=1
   # Cleanup
   kubectl -n tenant-test delete mongodbs.apps.cozystack.io $name

hack/e2e-apps/openbao.bats

@@ -18,7 +18,9 @@ spec:
   external: false
   ui: true
 EOF
-  sleep 5
+  # Wait for the operator to materialise the HelmRelease before kubectl wait
+  # kicks in (kubectl wait errors immediately if the object does not exist yet).
+  timeout 60 sh -ec "until kubectl -n tenant-test get hr openbao-$name >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait hr openbao-$name --timeout=60s --for=condition=ready
   kubectl -n tenant-test wait hr openbao-$name-system --timeout=120s --for=condition=ready
 
@@ -53,7 +55,9 @@ EOF
   kubectl -n tenant-test exec openbao-$name-0 -- bao operator unseal "$unseal_key"
 
   # Now wait for pod to become ready (readiness probe checks seal status)
+  timeout 60 sh -ec "until kubectl -n tenant-test get sts openbao-$name >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait sts openbao-$name --timeout=90s --for=jsonpath='{.status.readyReplicas}'=1
+  timeout 60 sh -ec "until kubectl -n tenant-test get pvc data-openbao-$name-0 >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait pvc data-openbao-$name-0 --timeout=50s --for=jsonpath='{.status.phase}'=Bound
   kubectl -n tenant-test delete openbao.apps.cozystack.io $name
   kubectl -n tenant-test delete pvc data-openbao-$name-0 --ignore-not-found

hack/e2e-apps/postgres.bats

@@ -40,8 +40,11 @@ spec:
   resources: {}
   resourcesPreset: "nano"
 EOF
-  sleep 5
+  # Wait for the operator to materialise the HelmRelease before kubectl wait
+  # kicks in (kubectl wait errors immediately if the object does not exist yet).
+  timeout 60 sh -ec "until kubectl -n tenant-test get hr postgres-$name >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait hr postgres-$name --timeout=100s --for=condition=ready
+  timeout 60 sh -ec "until kubectl -n tenant-test get job.batch postgres-$name-init-job >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait job.batch postgres-$name-init-job --timeout=50s --for=condition=Complete
   timeout 40 sh -ec "until kubectl -n tenant-test get svc postgres-$name-r -o jsonpath='{.spec.ports[0].port}' | grep -q '5432'; do sleep 10; done"
   timeout 40 sh -ec "until kubectl -n tenant-test get svc postgres-$name-ro -o jsonpath='{.spec.ports[0].port}' | grep -q '5432'; do sleep 10; done"

hack/e2e-apps/qdrant.bats

@@ -17,10 +17,14 @@ spec:
   resources: {}
   external: false
 EOF
-  sleep 5
+  # Wait for the operator to materialise the HelmRelease before kubectl wait
+  # kicks in (kubectl wait errors immediately if the object does not exist yet).
+  timeout 60 sh -ec "until kubectl -n tenant-test get hr qdrant-$name >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait hr qdrant-$name --timeout=60s --for=condition=ready
   kubectl -n tenant-test wait hr qdrant-$name-system --timeout=120s --for=condition=ready
+  timeout 60 sh -ec "until kubectl -n tenant-test get sts qdrant-$name >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait sts qdrant-$name --timeout=90s --for=jsonpath='{.status.readyReplicas}'=1
+  timeout 60 sh -ec "until kubectl -n tenant-test get pvc qdrant-storage-qdrant-$name-0 >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait pvc qdrant-storage-qdrant-$name-0 --timeout=50s --for=jsonpath='{.status.phase}'=Bound
   kubectl -n tenant-test delete qdrant.apps.cozystack.io $name
 }

hack/e2e-apps/redis.bats

@@ -18,10 +18,15 @@ spec:
   resources: {}
   resourcesPreset: "nano"
 EOF
-  sleep 5
+  # Wait for the operator to materialise the HelmRelease before kubectl wait
+  # kicks in (kubectl wait errors immediately if the object does not exist yet).
+  timeout 60 sh -ec "until kubectl -n tenant-test get hr redis-$name >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait hr redis-$name --timeout=20s --for=condition=ready
+  timeout 60 sh -ec "until kubectl -n tenant-test get pvc redisfailover-persistent-data-rfr-redis-$name-0 >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait pvc redisfailover-persistent-data-rfr-redis-$name-0 --timeout=50s --for=jsonpath='{.status.phase}'=Bound
+  timeout 60 sh -ec "until kubectl -n tenant-test get deploy rfs-redis-$name >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait deploy rfs-redis-$name --timeout=90s --for=condition=available
+  timeout 60 sh -ec "until kubectl -n tenant-test get sts rfr-redis-$name >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait sts rfr-redis-$name --timeout=90s --for=jsonpath='{.status.replicas}'=2
   kubectl -n tenant-test delete redis.apps.cozystack.io $name
 }

hack/e2e-apps/run-kubernetes.sh

@@ -227,6 +227,11 @@ EOF
     exit 1
   fi
 
+  # TODO(e2e-replace-fixed-timeouts): genuine retry loop. This validates an
+  # external HTTP path (MetalLB-advertised LB IP -> in-tenant ingress ->
+  # backend pod) which is not visible to the Kubernetes API as a single
+  # condition, so kubectl wait cannot replace it. The 20x3s = 60s budget is
+  # capped with `lb_ok=false` then asserted below.
   lb_ok=false
   for i in $(seq 1 20); do
     echo "Attempt $i"

hack/e2e-apps/vminstance.bats

@@ -2,8 +2,15 @@
 
 @test "Create a VM Disk" {
   name='test'
-  kubectl -n tenant-test delete vminstances.apps.cozystack.io $name --ignore-not-found --timeout=2m || true
-  kubectl -n tenant-test delete vmdisks.apps.cozystack.io $name --ignore-not-found --timeout=2m || true
+  # Delete any leftover from a previous run and BLOCK until removal completes.
+  # `kubectl apply` on a resource still in finalizer-drain silently no-ops
+  # ("Detected changes to resource ... which is currently being deleted"),
+  # which then races a NotFound on the downstream HR wait. The previous
+  # `|| true` swallowed timeout errors here and let the test continue with
+  # the old VMDisk still draining. Bumped timeout to 3m and removed `|| true`
+  # so a true delete failure surfaces immediately.
+  kubectl -n tenant-test delete vminstances.apps.cozystack.io $name --ignore-not-found --timeout=3m
+  kubectl -n tenant-test delete vmdisks.apps.cozystack.io $name --ignore-not-found --timeout=3m
   kubectl apply -f - <<EOF
 apiVersion: apps.cozystack.io/v1alpha1
 kind: VMDisk
@@ -18,16 +25,22 @@ spec:
   storage: 5Gi
   storageClass: replicated
 EOF
-  sleep 5
+  # Wait for the operator to materialise the HelmRelease before kubectl wait
+  # kicks in (kubectl wait errors immediately if the object does not exist yet).
+  timeout 60 sh -ec "until kubectl -n tenant-test get hr vm-disk-$name >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait hr vm-disk-$name --timeout=5s --for=condition=ready
+  timeout 120 sh -ec "until kubectl -n tenant-test get dv vm-disk-$name >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait dv vm-disk-$name --timeout=250s --for=condition=ready
+  timeout 120 sh -ec "until kubectl -n tenant-test get pvc vm-disk-$name >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait pvc vm-disk-$name --timeout=200s --for=jsonpath='{.status.phase}'=Bound
 }
 
 @test "Create a VM Instance" {
   diskName='test'
   name='test'
-  kubectl -n tenant-test delete vminstances.apps.cozystack.io $name --ignore-not-found --timeout=2m || true
+  # Same delete-finalizer-drain race as in "Create a VM Disk" above —
+  # block until removal completes, surface timeouts loudly.
+  kubectl -n tenant-test delete vminstances.apps.cozystack.io $name --ignore-not-found --timeout=3m
   kubectl apply -f - <<EOF
 apiVersion: apps.cozystack.io/v1alpha1
 kind: VMInstance
@@ -59,10 +72,17 @@ spec:
           - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPht0dPk5qQ+54g1hSX7A6AUxXJW5T6n/3d7Ga2F8gTF test@test
   cloudInitSeed: ""
 EOF
-  sleep 5
-  timeout 20 sh -ec "until kubectl -n tenant-test get vmi vm-instance-$name -o jsonpath='{.status.interfaces[0].ipAddress}' | grep -q '[0-9]'; do sleep 5; done"
+  # Wait for the operator to materialise the HelmRelease before downstream
+  # waits proceed (kubectl wait errors immediately if the HR does not exist).
+  timeout 60 sh -ec "until kubectl -n tenant-test get hr vm-instance-$name >/dev/null 2>&1; do sleep 2; done"
+  # Nested KubeVirt VM startup (virt-launcher + libvirt + cloud-init DHCP)
+  # routinely takes 30-60s under runner load; the previous 20s was unrealistic
+  # and produced flakes. 120s is a comfortable upper bound for nested virt.
+  timeout 120 sh -ec "until kubectl -n tenant-test get vmi vm-instance-$name -o jsonpath='{.status.interfaces[0].ipAddress}' | grep -q '[0-9]'; do sleep 2; done"
   kubectl -n tenant-test wait hr vm-instance-$name --timeout=5s --for=condition=ready
-  kubectl -n tenant-test wait vm vm-instance-$name --timeout=20s --for=condition=ready
-  kubectl -n tenant-test delete vminstances.apps.cozystack.io $name 
-  kubectl -n tenant-test delete vmdisks.apps.cozystack.io $diskName 
+  # VM ready follows IP assignment closely; 60s gives buffer for the qemu-guest-agent.
+  timeout 120 sh -ec "until kubectl -n tenant-test get vm vm-instance-$name >/dev/null 2>&1; do sleep 2; done"
+  kubectl -n tenant-test wait vm vm-instance-$name --timeout=60s --for=condition=ready
+  kubectl -n tenant-test delete vminstances.apps.cozystack.io $name --timeout=3m
+  kubectl -n tenant-test delete vmdisks.apps.cozystack.io $diskName --timeout=3m
 }

hack/e2e-install-cozystack.bats

@@ -7,12 +7,52 @@
   fi
 }
 
+@test "Pre-pull platform images" {
+  # Cluster-member workloads (OVN raft, LINSTOR) fail if replicas start at
+  # different times due to image-pull stagger across nodes. Pre-pull these
+  # images to every node so all replicas start with images already cached.
+  #
+  # Source images directly from the rendered charts so version bumps stay in
+  # sync automatically. yq walks every PodSpec-shaped object and emits the
+  # images of each container — this scopes the result to images the kubelet
+  # actually pulls (skips configmap fields and CRD examples that happen to
+  # contain an `image:` key). Add a chart here when a new peer-sensitive
+  # workload is found.
+  {
+    helm template packages/system/kubeovn
+    helm template packages/system/linstor
+  } | yq -N '
+      (..|select(has("containers"))|.containers[]|.image),
+      (..|select(has("initContainers"))|.initContainers[]|.image)
+    ' | hack/e2e-prepull-images.sh
+}
+
 @test "Install Cozystack" {
-  # Install cozy-installer chart (operator installs CRDs on startup via --install-crds)
+  # Pre-create the cozy-system namespace with the labels the operator pod needs.
+  # The chart no longer ships a Namespace resource (helm v3's --create-namespace
+  # collides with chart-defined Namespace). On Talos clusters that enforce PSA
+  # baseline-by-default, the cozystack-operator pod (hostNetwork=true) is
+  # rejected unless the namespace is labelled enforce=privileged BEFORE the
+  # pod is scheduled. A pre-install hook can't bootstrap that label because
+  # the hook itself runs in the same namespace and faces the same PSA gate.
+  # So apply the namespace from the caller (this bats / production install docs)
+  # — same pattern as kube-prometheus-stack, argo-cd, cert-manager.
+  kubectl apply -f - <<'EOF'
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: cozy-system
+  labels:
+    cozystack.io/system: "true"
+    cozystack.io/deletion-protected: "true"
+    pod-security.kubernetes.io/enforce: privileged
+EOF
+
+  # Install cozy-installer chart (operator installs CRDs on startup via --install-crds).
   helm upgrade installer packages/core/installer \
     --install \
     --namespace cozy-system \
-    --create-namespace \
+    --set cozystackOperator.helmReleaseInterval=30s \
     --wait \
     --timeout 2m
 
@@ -51,11 +91,31 @@ spec:
             - cozystack.external-dns-application
 EOF
 
+  # Launch storage + LB configuration in the background. It waits for its
+  # own prerequisites (linstor-controller deploy, MetalLB CRDs) and finishes
+  # while the parallel HR wait below is still running, so the cost overlaps
+  # with the platform reconcile instead of compounding it.
+  hack/e2e-post-install-prep.sh > /tmp/post-install-prep.log 2>&1 &
+  POST_PREP_PID=$!
+
   # Wait until HelmReleases appear & reconcile them
   timeout 180 sh -ec 'until [ $(kubectl get hr -A --no-headers 2>/dev/null | wc -l) -gt 10 ]; do sleep 1; done'
+  # TODO(e2e-replace-fixed-timeouts): genuine sleep. The threshold of 10 is a
+  # heuristic for "enough HRs visible to start waiting"; the snapshot below
+  # uses whatever HRs have appeared by then. There is no objective k8s API
+  # signal for "all platform HRs have been emitted" without hard-coding the
+  # expected list, so the 5s pad lets a few late-arrivals join the snapshot.
   sleep 5
   kubectl get hr -A | awk 'NR>1 {print "kubectl wait --timeout=15m --for=condition=ready -n "$1" hr/"$2" &"} END {print "wait"}' | sh -ex
 
+  echo "Waiting for post-install-prep to complete"
+  if ! wait $POST_PREP_PID; then
+    cat /tmp/post-install-prep.log >&2
+    echo "post-install-prep failed" >&2
+    exit 1
+  fi
+  cat /tmp/post-install-prep.log
+
   # Fail the test if any HelmRelease is not Ready
   if kubectl get hr -A | grep -v " True " | grep -v NAME; then
     kubectl get hr -A
@@ -69,80 +129,8 @@ EOF
   kubectl wait deployment/capi-controller-manager deployment/capi-kamaji-controller-manager deployment/capi-kubeadm-bootstrap-controller-manager deployment/capi-operator-cluster-api-operator deployment/capk-controller-manager -n cozy-cluster-api --timeout=2m --for=condition=available
 }
 
-@test "Wait for LINSTOR and configure storage" {
-  # Linstor controller and nodes
-  kubectl wait deployment/linstor-controller -n cozy-linstor --timeout=5m --for=condition=available
-  timeout 60 sh -ec 'until [ $(kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor node list | grep -c Online) -eq 3 ]; do sleep 1; done'
-
-  created_pools=$(kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor sp l -s data --pastable | awk '$2 == "data" {printf " " $4} END{printf " "}')
-  for node in srv1 srv2 srv3; do
-    case $created_pools in
-      *" $node "*) echo "Storage pool 'data' already exists on node $node"; continue;;
-    esac
-    kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor ps cdp zfs ${node} /dev/vdc --pool-name data --storage-pool data
-  done
-
-  # Storage classes
-  kubectl apply -f - <<'EOF'
----
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  name: local
-  annotations:
-    storageclass.kubernetes.io/is-default-class: "true"
-provisioner: linstor.csi.linbit.com
-parameters:
-  linstor.csi.linbit.com/storagePool: "data"
-  linstor.csi.linbit.com/layerList: "storage"
-  linstor.csi.linbit.com/allowRemoteVolumeAccess: "false"
-volumeBindingMode: WaitForFirstConsumer
-allowVolumeExpansion: true
----
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  name: replicated
-provisioner: linstor.csi.linbit.com
-parameters:
-  linstor.csi.linbit.com/storagePool: "data"
-  linstor.csi.linbit.com/autoPlace: "3"
-  linstor.csi.linbit.com/layerList: "drbd storage"
-  linstor.csi.linbit.com/allowRemoteVolumeAccess: "true"
-  property.linstor.csi.linbit.com/DrbdOptions/auto-quorum: suspend-io
-  property.linstor.csi.linbit.com/DrbdOptions/Resource/on-no-data-accessible: suspend-io
-  property.linstor.csi.linbit.com/DrbdOptions/Resource/on-suspended-primary-outdated: force-secondary
-  property.linstor.csi.linbit.com/DrbdOptions/Net/rr-conflict: retry-connect
-volumeBindingMode: Immediate
-allowVolumeExpansion: true
-EOF
-}
-
-@test "Wait for MetalLB and configure address pool" {
-  # MetalLB address pool
-  kubectl apply -f - <<'EOF'
----
-apiVersion: metallb.io/v1beta1
-kind: L2Advertisement
-metadata:
-  name: cozystack
-  namespace: cozy-metallb
-spec:
-  ipAddressPools: [cozystack]
----
-apiVersion: metallb.io/v1beta1
-kind: IPAddressPool
-metadata:
-  name: cozystack
-  namespace: cozy-metallb
-spec:
-  addresses: [192.168.123.200-192.168.123.250]
-  autoAssign: true
-  avoidBuggyIPs: false
-EOF
-}
-
 @test "Check Cozystack API service" {
+  timeout 60 sh -ec 'until kubectl get apiservices/v1alpha1.apps.cozystack.io apiservices/v1alpha1.core.cozystack.io >/dev/null 2>&1; do sleep 2; done'
   kubectl wait --for=condition=Available apiservices/v1alpha1.apps.cozystack.io apiservices/v1alpha1.core.cozystack.io --timeout=2m
 }
 
@@ -174,15 +162,22 @@ EOF
   kubectl wait deploy/root-ingress-controller -n tenant-root --timeout=5m --for=condition=available
 
   # etcd statefulset
+  timeout 60 sh -ec 'until kubectl get sts/etcd -n tenant-root >/dev/null 2>&1; do sleep 2; done'
   kubectl wait sts/etcd -n tenant-root --for=jsonpath='{.status.readyReplicas}'=3 --timeout=5m
 
   # VictoriaMetrics components
+  timeout 60 sh -ec 'until kubectl get vmalert/vmalert-shortterm -n tenant-root >/dev/null 2>&1; do sleep 2; done'
+  timeout 60 sh -ec 'until kubectl get vmalertmanager/alertmanager -n tenant-root >/dev/null 2>&1; do sleep 2; done'
   kubectl wait vmalert/vmalert-shortterm vmalertmanager/alertmanager -n tenant-root --for=jsonpath='{.status.updateStatus}'=operational --timeout=15m
+  timeout 60 sh -ec 'until kubectl get vlclusters/generic -n tenant-root >/dev/null 2>&1; do sleep 2; done'
   kubectl wait vlclusters/generic -n tenant-root --for=jsonpath='{.status.updateStatus}'=operational --timeout=5m
+  timeout 60 sh -ec 'until kubectl get vmcluster/shortterm vmcluster/longterm -n tenant-root >/dev/null 2>&1; do sleep 2; done'
   kubectl wait vmcluster/shortterm vmcluster/longterm -n tenant-root --for=jsonpath='{.status.updateStatus}'=operational --timeout=5m
 
   # Grafana
+  timeout 60 sh -ec 'until kubectl get clusters.postgresql.cnpg.io/grafana-db -n tenant-root >/dev/null 2>&1; do sleep 2; done'
   kubectl wait clusters.postgresql.cnpg.io/grafana-db -n tenant-root --for=condition=ready --timeout=5m
+  timeout 60 sh -ec 'until kubectl get deploy/grafana-deployment -n tenant-root >/dev/null 2>&1; do sleep 2; done'
   kubectl wait deploy/grafana-deployment -n tenant-root --for=condition=available --timeout=5m
 
   # Verify Grafana via ingress
@@ -273,6 +268,7 @@ spec:
   seaweedfs: false
 EOF
   kubectl wait hr/tenant-test -n tenant-root --timeout=1m --for=condition=ready
+  timeout 60 sh -ec 'until kubectl get namespace tenant-test >/dev/null 2>&1; do sleep 2; done'
   kubectl wait namespace tenant-test --timeout=20s --for=jsonpath='{.status.phase}'=Active
   # Wait for ResourceQuota to appear and assert values
   timeout 60 sh -ec 'until [ "$(kubectl get quota -n tenant-test --no-headers 2>/dev/null | wc -l)" -ge 1 ]; do sleep 1; done'

hack/e2e-post-install-prep.sh

@@ -0,0 +1,96 @@
+#!/bin/sh
+# Runs LINSTOR pool + StorageClass + MetalLB pool configuration as soon as
+# their respective prerequisites are reachable. Designed to run in the
+# background during the platform HR reconcile wait, so its wall-clock cost
+# overlaps with the wait instead of compounding it.
+set -eu
+
+echo "[post-install-prep] waiting for linstor HelmRelease object to exist"
+timeout 60 sh -ec 'until kubectl get hr/linstor -n cozy-linstor >/dev/null 2>&1; do sleep 2; done'
+
+echo "[post-install-prep] waiting for linstor HelmRelease to be Ready"
+kubectl wait helmrelease/linstor -n cozy-linstor --for=condition=Ready --timeout=15m
+
+echo "[post-install-prep] waiting for linstor-controller deployment to exist"
+timeout 300 sh -ec 'until kubectl get deploy/linstor-controller -n cozy-linstor >/dev/null 2>&1; do sleep 2; done'
+
+echo "[post-install-prep] waiting for linstor-controller to be Available"
+kubectl wait deployment/linstor-controller -n cozy-linstor --timeout=5m --for=condition=available
+
+echo "[post-install-prep] waiting for 3 LINSTOR nodes Online"
+# TODO(e2e-replace-fixed-timeouts): genuine poll. LINSTOR node membership is
+# reported by the linstor binary inside the controller pod, not via a
+# Kubernetes API condition, so kubectl wait cannot subscribe to it.
+timeout 60 sh -ec 'until [ $(kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor node list | grep -c Online) -eq 3 ]; do sleep 1; done'
+
+echo "[post-install-prep] creating LINSTOR storage pools (parallel across nodes)"
+created_pools=$(kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor sp l -s data --pastable | awk '$2 == "data" {printf " " $4} END{printf " "}')
+for node in srv1 srv2 srv3; do
+  case $created_pools in
+    *" $node "*) echo "  pool 'data' already exists on $node"; continue;;
+  esac
+  kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor ps cdp zfs ${node} /dev/vdc --pool-name data --storage-pool data &
+done
+wait
+
+echo "[post-install-prep] applying StorageClasses"
+kubectl apply -f - <<'EOF'
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: local
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "true"
+provisioner: linstor.csi.linbit.com
+parameters:
+  linstor.csi.linbit.com/storagePool: "data"
+  linstor.csi.linbit.com/layerList: "storage"
+  linstor.csi.linbit.com/allowRemoteVolumeAccess: "false"
+volumeBindingMode: WaitForFirstConsumer
+allowVolumeExpansion: true
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: replicated
+provisioner: linstor.csi.linbit.com
+parameters:
+  linstor.csi.linbit.com/storagePool: "data"
+  linstor.csi.linbit.com/autoPlace: "3"
+  linstor.csi.linbit.com/layerList: "drbd storage"
+  linstor.csi.linbit.com/allowRemoteVolumeAccess: "true"
+  property.linstor.csi.linbit.com/DrbdOptions/auto-quorum: suspend-io
+  property.linstor.csi.linbit.com/DrbdOptions/Resource/on-no-data-accessible: suspend-io
+  property.linstor.csi.linbit.com/DrbdOptions/Resource/on-suspended-primary-outdated: force-secondary
+  property.linstor.csi.linbit.com/DrbdOptions/Net/rr-conflict: retry-connect
+volumeBindingMode: Immediate
+allowVolumeExpansion: true
+EOF
+
+echo "[post-install-prep] waiting for MetalLB CRDs"
+timeout 300 sh -ec 'until kubectl get crd ipaddresspools.metallb.io l2advertisements.metallb.io >/dev/null 2>&1; do sleep 2; done'
+
+echo "[post-install-prep] applying MetalLB IPAddressPool"
+kubectl apply -f - <<'EOF'
+---
+apiVersion: metallb.io/v1beta1
+kind: L2Advertisement
+metadata:
+  name: cozystack
+  namespace: cozy-metallb
+spec:
+  ipAddressPools: [cozystack]
+---
+apiVersion: metallb.io/v1beta1
+kind: IPAddressPool
+metadata:
+  name: cozystack
+  namespace: cozy-metallb
+spec:
+  addresses: [192.168.123.200-192.168.123.250]
+  autoAssign: true
+  avoidBuggyIPs: false
+EOF
+
+echo "[post-install-prep] done"

hack/e2e-prepull-images.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+# Pre-pull images to all cluster nodes.
+#
+# Reads image references from stdin, one per line. Empty lines and lines
+# starting with '#' are ignored.
+#
+# Some workloads (OVN raft, LINSTOR) are sensitive to peer-readiness at
+# startup: if nodes pull the image at different speeds, one replica boots
+# before its peers are reachable, exhausts its raft connection retries, and
+# the HelmRelease install times out. Pre-pulling eliminates the pull stagger
+# so all replicas start within milliseconds of each other.
+#
+# Implementation: a DaemonSet with one regular container per image (parallel
+# pulls — total time = max of any one image rather than sum). kubectl rollout
+# status blocks until all pods are Ready (= all containers running = all
+# images cached on every node), then we delete the DaemonSet.
+
+set -euo pipefail
+
+mapfile -t images < <(grep -Ev '^[[:space:]]*(#|$)' | sort -u)
+
+if [[ ${#images[@]} -eq 0 ]]; then
+  echo "e2e-prepull-images: no images on stdin, nothing to do" >&2
+  exit 0
+fi
+
+echo "Pre-pulling ${#images[@]} image(s):"
+printf '  %s\n' "${images[@]}"
+
+containers=""
+i=0
+for image in "${images[@]}"; do
+  containers+="
+      - name: pull-${i}
+        image: ${image}
+        command: [\"sleep\", \"infinity\"]
+        imagePullPolicy: IfNotPresent
+        resources:
+          requests:
+            cpu: 1m
+            memory: 1Mi"
+  i=$((i + 1))
+done
+
+kubectl apply -f - <<EOF
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: e2e-image-prepuller
+  namespace: kube-system
+  labels:
+    app: e2e-image-prepuller
+spec:
+  selector:
+    matchLabels:
+      app: e2e-image-prepuller
+  template:
+    metadata:
+      labels:
+        app: e2e-image-prepuller
+    spec:
+      # hostNetwork bypasses the CNI: this script runs BEFORE Cozystack
+      # installs kube-ovn, so the cluster has no working CNI yet and a normal
+      # pod would stay ContainerCreating with NetworkPluginNotReady, never
+      # reaching image-pull. With hostNetwork the pod sandbox is created on
+      # the host's namespace and the kubelet pulls images right away.
+      # Same pattern the cozystack-operator pod uses for the same reason.
+      hostNetwork: true
+      # No need for an SA token — this pod just sleeps while images pull.
+      automountServiceAccountToken: false
+      tolerations:
+      # Reach every node including control-plane and nodes still coming up.
+      - operator: Exists
+      containers:${containers}
+EOF
+
+echo "Waiting for e2e-image-prepuller DaemonSet to become ready..."
+kubectl rollout status daemonset/e2e-image-prepuller -n kube-system --timeout=10m
+
+kubectl delete daemonset e2e-image-prepuller -n kube-system --ignore-not-found
+echo "Image pre-pull complete."

hack/e2e-test-openapi.bats

@@ -14,8 +14,13 @@
 
 @test "Test OpenAPI v2 endpoint (protobuf)" {
   (
-    kubectl proxy --port=21234 & sleep 0.5
-    trap "kill $!" EXIT
+    kubectl proxy --port=21234 &
+    proxy_pid=$!
+    trap "kill $proxy_pid" EXIT
+    # Wait for the proxy to actually be listening rather than guessing with
+    # a fixed sleep. nc -z is non-destructive and exits 0 the moment the
+    # listener accepts a connection.
+    timeout 10 sh -ec 'until nc -z localhost 21234; do sleep 0.1; done'
     curl -sS --fail 'http://localhost:21234/openapi/v2?timeout=32s' -H 'Accept: application/com.github.proto-openapi.spec.v2@v1.0+protobuf' > /dev/null
   )
 }

internal/operator/package_reconciler.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"strings"
+	"time"
 
 	cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
 	helmv2 "github.com/fluxcd/helm-controller/api/v2"
@@ -58,7 +59,12 @@ func parseCRDPolicy(install *cozyv1alpha1.ComponentInstall) helmv2.CRDsPolicy {
 // PackageReconciler reconciles Package resources
 type PackageReconciler struct {
 	client.Client
-	Scheme *runtime.Scheme
+	Scheme                    *runtime.Scheme
+	HelmReleaseInterval       time.Duration
+	HelmReleaseRetryInterval  time.Duration
+	HelmReleaseInstallTimeout time.Duration
+	HelmReleaseUpgradeTimeout time.Duration
+	HelmReleaseMaxHistory     int
 }
 
 // +kubebuilder:rbac:groups=cozystack.io,resources=packages,verbs=get;list;watch;create;update;patch;delete
@@ -214,22 +220,30 @@ func (r *PackageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 				Labels:    labels,
 			},
 			Spec: helmv2.HelmReleaseSpec{
-				Interval: metav1.Duration{Duration: 5 * 60 * 1000000000}, // 5m
+				Interval:   metav1.Duration{Duration: r.HelmReleaseInterval},
+				MaxHistory: &r.HelmReleaseMaxHistory,
 				ChartRef: &helmv2.CrossNamespaceSourceReference{
 					Kind:      "ExternalArtifact",
 					Name:      artifactName,
 					Namespace: "cozy-system",
 				},
 				Install: &helmv2.Install{
-					Timeout: &metav1.Duration{Duration: 10 * 60 * 1000000000}, // 10m
-					Remediation: &helmv2.InstallRemediation{
-						Retries: -1,
+					Timeout: &metav1.Duration{Duration: r.HelmReleaseInstallTimeout},
+					// Strategy=RetryOnFailure (with RetryInterval) replaces the previous
+					// Remediation{Retries:-1} setup. Functionally equivalent ("retry forever
+					// on failure"), but decouples retry timing from spec.Interval so failed
+					// installs recover at HelmReleaseRetryInterval (default 30s) without
+					// also polling healthy releases at the same fast cadence.
+					Strategy: &helmv2.InstallStrategy{
+						Name:          string(helmv2.ActionStrategyRetryOnFailure),
+						RetryInterval: &metav1.Duration{Duration: r.HelmReleaseRetryInterval},
 					},
 				},
 				Upgrade: &helmv2.Upgrade{
-					Timeout: &metav1.Duration{Duration: 10 * 60 * 1000000000}, // 10m
-					Remediation: &helmv2.UpgradeRemediation{
-						Retries: -1,
+					Timeout: &metav1.Duration{Duration: r.HelmReleaseUpgradeTimeout},
+					Strategy: &helmv2.UpgradeStrategy{
+						Name:          string(helmv2.ActionStrategyRetryOnFailure),
+						RetryInterval: &metav1.Duration{Duration: r.HelmReleaseRetryInterval},
 					},
 					CRDs: parseCRDPolicy(component.Install),
 				},

packages/apps/harbor/templates/harbor.yaml

@@ -65,9 +65,15 @@ spec:
     name: {{ .Release.Name }}-credentials
     valuesKey: redis-password
     targetPath: harbor.redis.external.password
+  # BucketInfo is the JSON blob populated by the COSI BucketAccess controller.
+  # Merge it at the values root (no targetPath) so Flux unmarshals it as YAML
+  # instead of running it through Helm's strvals parser, which would split the
+  # JSON on commas. This also gates reconciliation on the Secret being ready and
+  # forces a config-digest change when its contents change.
+  - kind: Secret
+    name: {{ .Release.Name }}-registry-bucket
+    valuesKey: BucketInfo
   values:
-    bucket:
-      secretName: {{ .Release.Name }}-registry-bucket
     db:
       replicas: {{ .Values.database.replicas }}
       size: {{ .Values.database.size }}

packages/core/installer/templates/cozystack-operator.yaml

@@ -4,16 +4,6 @@
 {{- end -}}
 ---
 apiVersion: v1
-kind: Namespace
-metadata:
-  name: cozy-system
-  labels:
-    cozystack.io/system: "true"
-    pod-security.kubernetes.io/enforce: privileged
-  annotations:
-    helm.sh/resource-policy: keep
----
-apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: cozystack
@@ -68,6 +58,21 @@ spec:
         {{- if .Values.cozystackOperator.disableTelemetry }}
         - --disable-telemetry
         {{- end }}
+        {{- if .Values.cozystackOperator.helmReleaseInterval }}
+        - --helmrelease-interval={{ .Values.cozystackOperator.helmReleaseInterval }}
+        {{- end }}
+        {{- if .Values.cozystackOperator.helmReleaseRetryInterval }}
+        - --helmrelease-retry-interval={{ .Values.cozystackOperator.helmReleaseRetryInterval }}
+        {{- end }}
+        {{- if .Values.cozystackOperator.helmReleaseInstallTimeout }}
+        - --helmrelease-install-timeout={{ .Values.cozystackOperator.helmReleaseInstallTimeout }}
+        {{- end }}
+        {{- if .Values.cozystackOperator.helmReleaseUpgradeTimeout }}
+        - --helmrelease-upgrade-timeout={{ .Values.cozystackOperator.helmReleaseUpgradeTimeout }}
+        {{- end }}
+        {{- if .Values.cozystackOperator.helmReleaseMaxHistory }}
+        - --helmrelease-max-history={{ .Values.cozystackOperator.helmReleaseMaxHistory }}
+        {{- end }}
         - --platform-source-name=cozystack-platform
         - --platform-source-url={{ .Values.cozystackOperator.platformSourceUrl }}
         {{- if .Values.cozystackOperator.platformSourceRef }}

packages/core/installer/values.yaml

@@ -4,6 +4,25 @@ cozystackOperator:
   image: ghcr.io/cozystack/cozystack/cozystack-operator:v1.3.0@sha256:62574f12486bb40c901cf5ed484cca264405ce5810196d86555cbb27cce1ba48
   platformSourceUrl: 'oci://ghcr.io/cozystack/cozystack/cozystack-packages'
   platformSourceRef: 'digest=sha256:a0b9ef938446b3132d3d22ad2262beb1027c48c9037b6c2346fdc2f19acd3036'
+  # When non-empty, overrides the operator's --helmrelease-interval flag
+  # (operator default: 5m). E2E sets this to 30s; production should leave empty.
+  helmReleaseInterval: ""
+  # When non-empty, overrides the operator's --helmrelease-retry-interval flag
+  # (operator default: 30s). Controls how fast failed install/upgrade attempts
+  # are retried. Decoupled from helmReleaseInterval (healthy reconcile cadence).
+  helmReleaseRetryInterval: ""
+  # When non-empty, overrides the operator's --helmrelease-install-timeout flag
+  # (operator default: 10m). Bounds individual k8s operations (job/hook/wait)
+  # during Helm install. Charts with long-running install hooks may need longer.
+  helmReleaseInstallTimeout: ""
+  # When non-empty, overrides the operator's --helmrelease-upgrade-timeout flag
+  # (operator default: 10m). Same semantics as helmReleaseInstallTimeout, for
+  # the Helm upgrade action.
+  helmReleaseUpgradeTimeout: ""
+  # When non-empty, overrides the operator's --helmrelease-max-history flag
+  # (operator default: 5). Number of release revisions Helm keeps per HR.
+  # Use 0 for unlimited; lower values reduce Secret accumulation in test clusters.
+  helmReleaseMaxHistory: ""
 # Generic variant configuration (only used when cozystackOperator.variant=generic)
 cozystack:
   # Kubernetes API server host (IP only, no protocol/port)

packages/system/harbor/templates/bucket-secret.yaml

@@ -1,20 +1,17 @@
-{{- if .Values.bucket.secretName }}
-{{- $existingSecret := lookup "v1" "Secret" .Release.Namespace .Values.bucket.secretName }}
-{{- $bucketInfo := fromJson (b64dec (index $existingSecret.data "BucketInfo")) }}
-{{- $accessKeyID := index $bucketInfo.spec.secretS3 "accessKeyID" }}
-{{- $accessSecretKey := index $bucketInfo.spec.secretS3 "accessSecretKey" }}
-{{- $endpoint := index $bucketInfo.spec.secretS3 "endpoint" }}
-{{- $bucketName := $bucketInfo.spec.bucketName }}
+{{- /* COSI BucketInfo is merged at the values root via Flux valuesFrom (no targetPath). */ -}}
+{{- with .Values.spec }}
+{{- with .secretS3 }}
 ---
 apiVersion: v1
 kind: Secret
 metadata:
-  name: {{ .Values.harbor.fullnameOverride }}-registry-s3
+  name: {{ $.Values.harbor.fullnameOverride }}-registry-s3
 type: Opaque
 stringData:
-  REGISTRY_STORAGE_S3_ACCESSKEY: {{ $accessKeyID | quote }}
-  REGISTRY_STORAGE_S3_SECRETKEY: {{ $accessSecretKey | quote }}
-  REGISTRY_STORAGE_S3_REGIONENDPOINT: {{ $endpoint | quote }}
-  REGISTRY_STORAGE_S3_BUCKET: {{ $bucketName | quote }}
+  REGISTRY_STORAGE_S3_ACCESSKEY: {{ index . "accessKeyID" | quote }}
+  REGISTRY_STORAGE_S3_SECRETKEY: {{ index . "accessSecretKey" | quote }}
+  REGISTRY_STORAGE_S3_REGIONENDPOINT: {{ index . "endpoint" | quote }}
+  REGISTRY_STORAGE_S3_BUCKET: {{ $.Values.spec.bucketName | quote }}
   REGISTRY_STORAGE_S3_REGION: "us-east-1"
 {{- end }}
+{{- end }}

packages/system/harbor/values.yaml

@@ -1,6 +1,4 @@
 harbor: {}
-bucket:
-  secretName: ""
 db:
   replicas: 2
   size: 5Gi

packages/system/objectstorage-controller/images/objectstorage/Dockerfile

@@ -2,12 +2,15 @@
 
 FROM alpine AS source
 ARG COMMIT_REF=v0.2.2
-RUN apk add --no-cache curl tar
+RUN apk add --no-cache curl tar git
 WORKDIR /src
 
 RUN curl -sSL https://github.com/kubernetes-sigs/container-object-storage-interface/archive/${COMMIT_REF}.tar.gz \
         | tar -xz --strip-components=1
 
+COPY patches /patches
+RUN git apply /patches/*.diff
+
 FROM --platform=$BUILDPLATFORM docker.io/golang:1.24 AS builder
 ARG TARGETOS
 ARG TARGETARCH

packages/system/objectstorage-controller/images/objectstorage/patches/91-bucketaccess-conflict-retry.diff

@@ -0,0 +1,39 @@
+diff --git a/sidecar/pkg/bucketaccess/bucketaccess_controller.go b/sidecar/pkg/bucketaccess/bucketaccess_controller.go
+index a4db5c4..b351f9e 100644
+--- a/sidecar/pkg/bucketaccess/bucketaccess_controller.go
++++ b/sidecar/pkg/bucketaccess/bucketaccess_controller.go
+@@ -31,6 +31,7 @@ import (
+ 	kube "k8s.io/client-go/kubernetes"
+ 	kubecorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
+ 	"k8s.io/client-go/tools/record"
++	"k8s.io/client-go/util/retry"
+ 	"k8s.io/klog/v2"
+ 	cosiapi "sigs.k8s.io/container-object-storage-interface/client/apis"
+ 	"sigs.k8s.io/container-object-storage-interface/client/apis/objectstorage/consts"
+@@ -273,11 +274,22 @@ func (bal *BucketAccessListener) Add(ctx context.Context, inputBucketAccess *v1a
+ 		}
+ 	}
+ 
+-	if controllerutil.AddFinalizer(bucket, consts.BABucketFinalizer) {
+-		_, err = bal.buckets().Update(ctx, bucket, metav1.UpdateOptions{})
+-		if err != nil {
+-			return bal.recordError(inputBucketAccess, v1.EventTypeWarning, v1alpha1.FailedGrantAccess, err)
++	// Re-fetch and update inside RetryOnConflict to handle the case where the Bucket
++	// reconciler in the same controller process mutates the Bucket between our Get
++	// and Update, which surfaces as "the object has been modified" on the parent
++	// Bucket and emits FailedGrantAccess on the BucketAccess.
++	if err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
++		latest, getErr := bal.buckets().Get(ctx, bucketClaim.Status.BucketName, metav1.GetOptions{})
++		if getErr != nil {
++			return getErr
++		}
++		if !controllerutil.AddFinalizer(latest, consts.BABucketFinalizer) {
++			return nil
+ 		}
++		_, updateErr := bal.buckets().Update(ctx, latest, metav1.UpdateOptions{})
++		return updateErr
++	}); err != nil {
++		return bal.recordError(inputBucketAccess, v1.EventTypeWarning, v1alpha1.FailedGrantAccess, err)
+ 	}
+ 
+ 	if controllerutil.AddFinalizer(bucketAccess, consts.BAFinalizer) {