From e47de4faae9360af97af274e34b0c44505923a3e Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 27 Apr 2026 03:19:05 +0500 Subject: [PATCH 01/27] feat(cozyreport): collect Flux logs, Helm secrets, events, certs, host context MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous report omitted everything needed to diagnose failures that originate deeper than a single system package install — Flux controller logs, the actual rendered Helm manifest from the storage secret, recent events, cert-manager state, the cozystack-operator's own logs, and sandbox host context (talosctl logs, dmesg, disk). This makes failures self-diagnosable from cozyreport.tgz alone, which is the only artifact that survives a failed E2E run. Signed-off-by: Myasnikov Daniil --- hack/cozyreport.sh | 93 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 92 insertions(+), 1 deletion(-) diff --git a/hack/cozyreport.sh b/hack/cozyreport.sh index 565fd435..d22fc539 100755 --- a/hack/cozyreport.sh +++ b/hack/cozyreport.sh @@ -9,10 +9,13 @@ command -V kubectl >/dev/null || exit $? command -V tar >/dev/null || exit $? # -- cozystack module - echo "Collecting Cozystack information..." mkdir -p $REPORT_DIR/cozystack kubectl get deploy -n cozy-system cozystack -o jsonpath='{.spec.template.spec.containers[0].image}' > $REPORT_DIR/cozystack/image.txt 2>&1 +if kubectl get deploy -n cozy-system cozystack-operator >/dev/null 2>&1; then + kubectl logs -n cozy-system deploy/cozystack-operator --tail=2000 > $REPORT_DIR/cozystack/operator.log 2>&1 + kubectl logs -n cozy-system deploy/cozystack-operator --tail=2000 --previous > $REPORT_DIR/cozystack/operator-previous.log 2>&1 || true +fi kubectl get cm -n cozy-system --no-headers | awk '$1 ~ /^cozystack/' | while read NAME _; do DIR=$REPORT_DIR/cozystack/configs @@ -20,6 +23,46 @@ kubectl get cm -n cozy-system --no-headers | awk '$1 ~ /^cozystack/' | kubectl get cm -n cozy-system $NAME -o yaml > $DIR/$NAME.yaml 2>&1 done +# -- flux module +echo "Collecting Flux controller state..." +mkdir -p $REPORT_DIR/flux +for ctrl in helm-controller source-controller notification-controller kustomize-controller; do + if kubectl get deploy -n cozy-fluxcd $ctrl >/dev/null 2>&1; then + kubectl logs -n cozy-fluxcd deploy/$ctrl --tail=2000 > $REPORT_DIR/flux/$ctrl.log 2>&1 + kubectl logs -n cozy-fluxcd deploy/$ctrl --tail=2000 --previous > $REPORT_DIR/flux/$ctrl-previous.log 2>&1 || true + fi +done + +echo "Collecting Flux sources..." +for kind in helmrepositories.source.toolkit.fluxcd.io ocirepositories.source.toolkit.fluxcd.io gitrepositories.source.toolkit.fluxcd.io externalartifacts.source.toolkit.fluxcd.io; do + short=${kind%%.*} + kubectl get $kind -A > $REPORT_DIR/flux/$short.txt 2>&1 + kubectl get $kind -A -o yaml > $REPORT_DIR/flux/$short.yaml 2>&1 +done + +# -- cert-manager module +if kubectl get crd certificates.cert-manager.io >/dev/null 2>&1; then + echo "Collecting cert-manager state..." + DIR=$REPORT_DIR/cert-manager + mkdir -p $DIR + kubectl get certificates.cert-manager.io -A > $DIR/certificates.txt 2>&1 + kubectl get certificaterequests.cert-manager.io -A > $DIR/certificaterequests.txt 2>&1 + kubectl get orders.acme.cert-manager.io -A > $DIR/orders.txt 2>&1 + kubectl get challenges.acme.cert-manager.io -A > $DIR/challenges.txt 2>&1 + # Per non-Ready cert: full yaml + describe + kubectl get certificates.cert-manager.io -A --no-headers 2>/dev/null | awk '$3 != "True"' | \ + while read NAMESPACE NAME _; do + cdir=$DIR/certificates/$NAMESPACE/$NAME + mkdir -p $cdir + kubectl get certificates.cert-manager.io -n $NAMESPACE $NAME -o yaml > $cdir/cert.yaml 2>&1 + kubectl describe certificates.cert-manager.io -n $NAMESPACE $NAME > $cdir/describe.txt 2>&1 + done + if kubectl get deploy -n cozy-cert-manager cert-manager >/dev/null 2>&1; then + kubectl logs -n cozy-cert-manager deploy/cert-manager --tail=2000 > $DIR/cert-manager.log 2>&1 + kubectl logs -n cozy-cert-manager deploy/cert-manager-webhook --tail=2000 > $DIR/cert-manager-webhook.log 2>&1 + fi +fi + # -- kubernetes module echo "Collecting Kubernetes information..." @@ -46,6 +89,13 @@ kubectl get ns --no-headers | awk '$2 != "Active"' | kubectl describe ns $NAME > $DIR/describe.txt 2>&1 done +echo "Collecting events..." +kubectl get events -A --sort-by=.lastTimestamp > $REPORT_DIR/kubernetes/events.txt 2>&1 +# Filter to warning-class and recent for quick triage +kubectl get events -A --sort-by=.lastTimestamp \ + -o jsonpath='{range .items[?(@.type!="Normal")]}{.lastTimestamp}{"\t"}{.involvedObject.namespace}/{.involvedObject.kind}/{.involvedObject.name}{"\t"}{.reason}{"\t"}{.message}{"\n"}{end}' \ + > $REPORT_DIR/kubernetes/events-warnings.txt 2>&1 + echo "Collecting helmreleases..." kubectl get hr -A > $REPORT_DIR/kubernetes/helmreleases.txt 2>&1 kubectl get hr -A --no-headers | awk '$4 != "True"' | \ @@ -54,6 +104,13 @@ kubectl get hr -A --no-headers | awk '$4 != "True"' | \ mkdir -p $DIR kubectl get hr -n $NAMESPACE $NAME -o yaml > $DIR/hr.yaml 2>&1 kubectl describe hr -n $NAMESPACE $NAME > $DIR/describe.txt 2>&1 + # Helm storage secrets: latest revision per release. + kubectl get secret -n $NAMESPACE -l owner=helm,name=$NAME --sort-by='.metadata.creationTimestamp' --no-headers 2>/dev/null | \ + tail -1 | awk '{print $1}' | while read SECRET; do + [ -z "$SECRET" ] && continue + kubectl get secret -n $NAMESPACE $SECRET -o jsonpath='{.data.release}' 2>/dev/null \ + | base64 -d | base64 -d | gzip -d > $DIR/helm-release.json 2>&1 || true + done done echo "Collecting packages..." @@ -76,6 +133,23 @@ kubectl get packagesources --no-headers | awk '$3 != "True"' | \ kubectl describe packagesource $NAME > $DIR/describe.txt 2>&1 done +echo "Collecting cozystack apps..." +DIR=$REPORT_DIR/cozystack-apps +mkdir -p $DIR +for kind in applications.apps.cozystack.io applicationdefinitions.apps.cozystack.io tenants.apps.cozystack.io; do + short=${kind%%.*} + if kubectl get crd $kind >/dev/null 2>&1; then + kubectl get $kind -A > $DIR/$short.txt 2>&1 + kubectl get $kind -A --no-headers 2>/dev/null | awk 'NF >= 3 && $NF != "True" && $NF != "Ready"' | \ + while read NAMESPACE NAME _; do + d=$DIR/$short/$NAMESPACE/$NAME + mkdir -p $d + kubectl get $kind -n $NAMESPACE $NAME -o yaml > $d/$short.yaml 2>&1 + kubectl describe $kind -n $NAMESPACE $NAME > $d/describe.txt 2>&1 + done + fi +done + echo "Collecting pods..." kubectl get pod -A -o wide > $REPORT_DIR/kubernetes/pods.txt 2>&1 kubectl get pod -A --no-headers | awk '$4 !~ /Running|Succeeded|Completed/' | @@ -157,6 +231,23 @@ if kubectl get deploy -n cozy-linstor linstor-controller >/dev/null 2>&1; then kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor --no-color r l > $DIR/resources.txt 2>&1 fi +# -- sandbox-host module + +echo "Collecting sandbox host context..." +DIR=$REPORT_DIR/sandbox-host +mkdir -p $DIR +df -h > $DIR/df.txt 2>&1 +free -m > $DIR/free.txt 2>&1 +ps auxww > $DIR/ps.txt 2>&1 +dmesg | tail -200 > $DIR/dmesg.txt 2>&1 || true +if [ -f /workspace/talosconfig ]; then + for node in 192.168.123.11 192.168.123.12 192.168.123.13; do + talosctl --talosconfig /workspace/talosconfig -n $node dmesg --tail=200 > $DIR/talos-$node-dmesg.txt 2>&1 || true + talosctl --talosconfig /workspace/talosconfig -n $node logs kubelet --tail=500 > $DIR/talos-$node-kubelet.log 2>&1 || true + talosctl --talosconfig /workspace/talosconfig -n $node logs containerd --tail=500 > $DIR/talos-$node-containerd.log 2>&1 || true + done +fi + # -- finalization echo "Creating archive..." From a98b717acd6dd8841760544fe22a7f742eca1788 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 27 Apr 2026 03:19:55 +0500 Subject: [PATCH 02/27] feat(cozyreport): add summary.txt at archive root MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A 50MB tarball with 200 directories is opaque on first contact. summary.txt at the root surfaces 'what is broken right now' — non-Ready HRs, ImagePullBackOff pods, OOMKilled events, cert-manager state, Flux Source health, storage binding, node pressure — so triage starts with one file instead of a directory tree dive. Signed-off-by: Myasnikov Daniil --- hack/cozyreport-summary.sh | 66 ++++++++++++++++++++++++++++++++++++++ hack/cozyreport.sh | 3 ++ 2 files changed, 69 insertions(+) create mode 100755 hack/cozyreport-summary.sh diff --git a/hack/cozyreport-summary.sh b/hack/cozyreport-summary.sh new file mode 100755 index 00000000..d90682c5 --- /dev/null +++ b/hack/cozyreport-summary.sh @@ -0,0 +1,66 @@ +#!/bin/sh +# Emit a human-readable summary of "what is broken" to a single file. +# Reads the live cluster (not the report dir) so it can use kubectl JSONPath. +# Usage: cozyreport-summary.sh > summary.txt +set -eu + +echo "# Cozystack E2E Diagnostic Summary" +echo "Generated: $(date -Iseconds)" +echo + +echo "## HelmReleases not Ready" +echo +kubectl get hr -A --no-headers 2>/dev/null \ + | awk '$4 != "True" {printf " %s/%s — %s\n", $1, $2, $5}' \ + | head -40 +echo + +echo "## Pods not Running/Succeeded" +echo +kubectl get pod -A --no-headers 2>/dev/null \ + | awk '$4 !~ /Running|Succeeded|Completed/ {printf " %s/%s — %s (restarts=%s, age=%s)\n", $1, $2, $4, $5, $6}' \ + | head -40 +echo + +echo "## ImagePullBackOff / ErrImagePull" +echo +kubectl get pod -A --no-headers 2>/dev/null \ + | awk '$4 ~ /ImagePullBackOff|ErrImagePull/ {printf " %s/%s — %s\n", $1, $2, $4}' +echo + +echo "## OOMKilled in last 30 min" +echo +kubectl get events -A --field-selector reason=OOMKilling --sort-by=.lastTimestamp 2>/dev/null \ + | tail -20 +echo + +echo "## Recent Warning events (top 30)" +echo +kubectl get events -A --field-selector type=Warning --sort-by=.lastTimestamp 2>/dev/null \ + | tail -30 +echo + +echo "## cert-manager: Certificates not Ready" +echo +if kubectl get crd certificates.cert-manager.io >/dev/null 2>&1; then + kubectl get certificates.cert-manager.io -A --no-headers 2>/dev/null \ + | awk '$3 != "True" {printf " %s/%s — Ready=%s\n", $1, $2, $3}' +fi +echo + +echo "## Flux Sources not Ready" +echo +for kind in helmrepositories.source.toolkit.fluxcd.io ocirepositories.source.toolkit.fluxcd.io gitrepositories.source.toolkit.fluxcd.io; do + kubectl get $kind -A --no-headers 2>/dev/null \ + | awk -v k="${kind%%.*}" '$4 != "True" {printf " %s %s/%s — Ready=%s\n", k, $1, $2, $4}' +done +echo + +echo "## Storage: PVCs not Bound, PVs not Bound" +echo +kubectl get pvc -A --no-headers 2>/dev/null | awk '$3 != "Bound" {printf " PVC %s/%s — %s\n", $1, $2, $3}' +kubectl get pv --no-headers 2>/dev/null | awk '$5 != "Bound" {printf " PV %s — %s\n", $1, $5}' +echo + +echo "## Node Conditions" +kubectl get nodes -o custom-columns=NAME:.metadata.name,READY:.status.conditions[?\(@.type==\"Ready\"\)].status,DISK:.status.conditions[?\(@.type==\"DiskPressure\"\)].status,MEM:.status.conditions[?\(@.type==\"MemoryPressure\"\)].status 2>/dev/null diff --git a/hack/cozyreport.sh b/hack/cozyreport.sh index d22fc539..ba65c382 100755 --- a/hack/cozyreport.sh +++ b/hack/cozyreport.sh @@ -250,6 +250,9 @@ fi # -- finalization +echo "Generating summary..." +hack/cozyreport-summary.sh > $REPORT_DIR/summary.txt 2>&1 || true + echo "Creating archive..." tar -czf $REPORT_NAME.tgz -C $REPORT_PDIR . echo "Report created: $REPORT_NAME.tgz" From 056a4d64043179054d7ec51085593817b234427d Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 27 Apr 2026 03:21:45 +0500 Subject: [PATCH 03/27] feat(operator): make HelmRelease Interval configurable, override to 30s in E2E Adds --helmrelease-interval (default 5m, matching today's behaviour) to the operator and a corresponding cozystackOperator.helmReleaseInterval value on the cozy-installer chart (default empty -> no flag rendered -> operator default 5m applies). E2E install sets it to 30s. Rationale: with the 5m default, dependency- blocked HRs (waiting on cert-manager webhooks, CRDs) are requeued only every 5 min, producing an 8-10 min dead zone in the E2E install where the fast pack of HRs is Ready but the slow tail hasn't been retried yet. 30s collapses the gap. Production defaults are unchanged. The override is opt-in per install. Signed-off-by: Myasnikov Daniil --- cmd/cozystack-operator/main.go | 16 ++++++++++++++-- hack/e2e-install-cozystack.bats | 1 + internal/operator/package_reconciler.go | 6 ++++-- .../installer/templates/cozystack-operator.yaml | 3 +++ packages/core/installer/values.yaml | 3 +++ 5 files changed, 25 insertions(+), 4 deletions(-) diff --git a/cmd/cozystack-operator/main.go b/cmd/cozystack-operator/main.go index e215b779..01ce097f 100644 --- a/cmd/cozystack-operator/main.go +++ b/cmd/cozystack-operator/main.go @@ -83,6 +83,7 @@ func main() { var disableTelemetry bool var telemetryEndpoint string var telemetryInterval string + var helmReleaseInterval string var cozyValuesSecretName string var cozyValuesSecretNamespace string var cozyValuesNamespaceSelector string @@ -107,6 +108,10 @@ func main() { "Endpoint for sending telemetry data") flag.StringVar(&telemetryInterval, "telemetry-interval", "15m", "Interval between telemetry data collection (e.g. 15m, 1h)") + flag.StringVar(&helmReleaseInterval, "helmrelease-interval", "5m", + "Reconcile interval applied to HelmReleases created by the Package reconciler. "+ + "Lower values speed up dependency-blocked retries (e.g. during E2E install) at the cost of "+ + "controller load. Production default 5m matches existing behaviour.") flag.StringVar(&platformSourceURL, "platform-source-url", "", "Platform source URL (oci:// or https://). If specified, generates OCIRepository or GitRepository resource.") flag.StringVar(&platformSourceName, "platform-source-name", "cozystack-platform", "Name for the generated platform source resource and PackageSource") flag.StringVar(&platformSourceRef, "platform-source-ref", "", "Reference specification as key=value pairs (e.g., 'branch=main' or 'digest=sha256:...,tag=v1.0'). For OCI: digest, semver, semverFilter, tag. For Git: branch, tag, semver, name, commit.") @@ -122,6 +127,12 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + hrIntervalDuration, err := time.ParseDuration(helmReleaseInterval) + if err != nil { + setupLog.Error(err, "invalid --helmrelease-interval value", "value", helmReleaseInterval) + os.Exit(1) + } + config := ctrl.GetConfigOrDie() // Create a direct client (without cache) for pre-start operations @@ -258,8 +269,9 @@ func main() { // Setup Package reconciler if err := (&operator.PackageReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + HelmReleaseInterval: hrIntervalDuration, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Package") os.Exit(1) diff --git a/hack/e2e-install-cozystack.bats b/hack/e2e-install-cozystack.bats index cf66b73f..3a02fc8c 100644 --- a/hack/e2e-install-cozystack.bats +++ b/hack/e2e-install-cozystack.bats @@ -13,6 +13,7 @@ --install \ --namespace cozy-system \ --create-namespace \ + --set cozystackOperator.helmReleaseInterval=30s \ --wait \ --timeout 2m diff --git a/internal/operator/package_reconciler.go b/internal/operator/package_reconciler.go index 0e724e49..d7cd1151 100644 --- a/internal/operator/package_reconciler.go +++ b/internal/operator/package_reconciler.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "strings" + "time" cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" helmv2 "github.com/fluxcd/helm-controller/api/v2" @@ -58,7 +59,8 @@ func parseCRDPolicy(install *cozyv1alpha1.ComponentInstall) helmv2.CRDsPolicy { // PackageReconciler reconciles Package resources type PackageReconciler struct { client.Client - Scheme *runtime.Scheme + Scheme *runtime.Scheme + HelmReleaseInterval time.Duration } // +kubebuilder:rbac:groups=cozystack.io,resources=packages,verbs=get;list;watch;create;update;patch;delete @@ -214,7 +216,7 @@ func (r *PackageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct Labels: labels, }, Spec: helmv2.HelmReleaseSpec{ - Interval: metav1.Duration{Duration: 5 * 60 * 1000000000}, // 5m + Interval: metav1.Duration{Duration: r.HelmReleaseInterval}, ChartRef: &helmv2.CrossNamespaceSourceReference{ Kind: "ExternalArtifact", Name: artifactName, diff --git a/packages/core/installer/templates/cozystack-operator.yaml b/packages/core/installer/templates/cozystack-operator.yaml index aded0995..8389aca4 100644 --- a/packages/core/installer/templates/cozystack-operator.yaml +++ b/packages/core/installer/templates/cozystack-operator.yaml @@ -68,6 +68,9 @@ spec: {{- if .Values.cozystackOperator.disableTelemetry }} - --disable-telemetry {{- end }} + {{- if .Values.cozystackOperator.helmReleaseInterval }} + - --helmrelease-interval={{ .Values.cozystackOperator.helmReleaseInterval }} + {{- end }} - --platform-source-name=cozystack-platform - --platform-source-url={{ .Values.cozystackOperator.platformSourceUrl }} {{- if .Values.cozystackOperator.platformSourceRef }} diff --git a/packages/core/installer/values.yaml b/packages/core/installer/values.yaml index eef691ea..f31400ec 100644 --- a/packages/core/installer/values.yaml +++ b/packages/core/installer/values.yaml @@ -4,6 +4,9 @@ cozystackOperator: image: ghcr.io/cozystack/cozystack/cozystack-operator:v1.3.0@sha256:62574f12486bb40c901cf5ed484cca264405ce5810196d86555cbb27cce1ba48 platformSourceUrl: 'oci://ghcr.io/cozystack/cozystack/cozystack-packages' platformSourceRef: 'digest=sha256:a0b9ef938446b3132d3d22ad2262beb1027c48c9037b6c2346fdc2f19acd3036' + # When non-empty, overrides the operator's --helmrelease-interval flag + # (operator default: 5m). E2E sets this to 30s; production should leave empty. + helmReleaseInterval: "" # Generic variant configuration (only used when cozystackOperator.variant=generic) cozystack: # Kubernetes API server host (IP only, no protocol/port) From 4ee073e2bbf79da661b7b23f12f5a028678bb1e5 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 27 Apr 2026 03:23:53 +0500 Subject: [PATCH 04/27] test(e2e): run LINSTOR/SC/MetalLB setup in parallel with platform install Previously these three configuration blocks ran as separate @test cases after the 15m platform HR wait, adding ~1.5 min sequentially. None of them gates platform HR reconcile, so they can run as a background prep that completes during the main wait. The helper script polls for its own prerequisites and is awaited at the end of the Install Cozystack test so failures still surface deterministically. Signed-off-by: Myasnikov Daniil --- hack/e2e-install-cozystack.bats | 88 ++++++--------------------------- hack/e2e-post-install-prep.sh | 87 ++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 73 deletions(-) create mode 100755 hack/e2e-post-install-prep.sh diff --git a/hack/e2e-install-cozystack.bats b/hack/e2e-install-cozystack.bats index 3a02fc8c..2fe0dfbd 100644 --- a/hack/e2e-install-cozystack.bats +++ b/hack/e2e-install-cozystack.bats @@ -52,11 +52,26 @@ spec: - cozystack.external-dns-application EOF + # Launch storage + LB configuration in the background. It waits for its + # own prerequisites (linstor-controller deploy, MetalLB CRDs) and finishes + # while the parallel HR wait below is still running, so the cost overlaps + # with the platform reconcile instead of compounding it. + hack/e2e-post-install-prep.sh > /tmp/post-install-prep.log 2>&1 & + POST_PREP_PID=$! + # Wait until HelmReleases appear & reconcile them timeout 180 sh -ec 'until [ $(kubectl get hr -A --no-headers 2>/dev/null | wc -l) -gt 10 ]; do sleep 1; done' sleep 5 kubectl get hr -A | awk 'NR>1 {print "kubectl wait --timeout=15m --for=condition=ready -n "$1" hr/"$2" &"} END {print "wait"}' | sh -ex + echo "Waiting for post-install-prep to complete" + if ! wait $POST_PREP_PID; then + cat /tmp/post-install-prep.log >&2 + echo "post-install-prep failed" >&2 + exit 1 + fi + cat /tmp/post-install-prep.log + # Fail the test if any HelmRelease is not Ready if kubectl get hr -A | grep -v " True " | grep -v NAME; then kubectl get hr -A @@ -70,79 +85,6 @@ EOF kubectl wait deployment/capi-controller-manager deployment/capi-kamaji-controller-manager deployment/capi-kubeadm-bootstrap-controller-manager deployment/capi-operator-cluster-api-operator deployment/capk-controller-manager -n cozy-cluster-api --timeout=2m --for=condition=available } -@test "Wait for LINSTOR and configure storage" { - # Linstor controller and nodes - kubectl wait deployment/linstor-controller -n cozy-linstor --timeout=5m --for=condition=available - timeout 60 sh -ec 'until [ $(kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor node list | grep -c Online) -eq 3 ]; do sleep 1; done' - - created_pools=$(kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor sp l -s data --pastable | awk '$2 == "data" {printf " " $4} END{printf " "}') - for node in srv1 srv2 srv3; do - case $created_pools in - *" $node "*) echo "Storage pool 'data' already exists on node $node"; continue;; - esac - kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor ps cdp zfs ${node} /dev/vdc --pool-name data --storage-pool data - done - - # Storage classes - kubectl apply -f - <<'EOF' ---- -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: local - annotations: - storageclass.kubernetes.io/is-default-class: "true" -provisioner: linstor.csi.linbit.com -parameters: - linstor.csi.linbit.com/storagePool: "data" - linstor.csi.linbit.com/layerList: "storage" - linstor.csi.linbit.com/allowRemoteVolumeAccess: "false" -volumeBindingMode: WaitForFirstConsumer -allowVolumeExpansion: true ---- -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: replicated -provisioner: linstor.csi.linbit.com -parameters: - linstor.csi.linbit.com/storagePool: "data" - linstor.csi.linbit.com/autoPlace: "3" - linstor.csi.linbit.com/layerList: "drbd storage" - linstor.csi.linbit.com/allowRemoteVolumeAccess: "true" - property.linstor.csi.linbit.com/DrbdOptions/auto-quorum: suspend-io - property.linstor.csi.linbit.com/DrbdOptions/Resource/on-no-data-accessible: suspend-io - property.linstor.csi.linbit.com/DrbdOptions/Resource/on-suspended-primary-outdated: force-secondary - property.linstor.csi.linbit.com/DrbdOptions/Net/rr-conflict: retry-connect -volumeBindingMode: Immediate -allowVolumeExpansion: true -EOF -} - -@test "Wait for MetalLB and configure address pool" { - # MetalLB address pool - kubectl apply -f - <<'EOF' ---- -apiVersion: metallb.io/v1beta1 -kind: L2Advertisement -metadata: - name: cozystack - namespace: cozy-metallb -spec: - ipAddressPools: [cozystack] ---- -apiVersion: metallb.io/v1beta1 -kind: IPAddressPool -metadata: - name: cozystack - namespace: cozy-metallb -spec: - addresses: [192.168.123.200-192.168.123.250] - autoAssign: true - avoidBuggyIPs: false -EOF -} - @test "Check Cozystack API service" { kubectl wait --for=condition=Available apiservices/v1alpha1.apps.cozystack.io apiservices/v1alpha1.core.cozystack.io --timeout=2m } diff --git a/hack/e2e-post-install-prep.sh b/hack/e2e-post-install-prep.sh new file mode 100755 index 00000000..495bf72a --- /dev/null +++ b/hack/e2e-post-install-prep.sh @@ -0,0 +1,87 @@ +#!/bin/sh +# Runs LINSTOR pool + StorageClass + MetalLB pool configuration as soon as +# their respective prerequisites are reachable. Designed to run in the +# background during the platform HR reconcile wait, so its wall-clock cost +# overlaps with the wait instead of compounding it. +set -eu + +echo "[post-install-prep] waiting for linstor-controller deployment to exist" +timeout 300 sh -ec 'until kubectl get deploy/linstor-controller -n cozy-linstor >/dev/null 2>&1; do sleep 2; done' + +echo "[post-install-prep] waiting for linstor-controller to be Available" +kubectl wait deployment/linstor-controller -n cozy-linstor --timeout=5m --for=condition=available + +echo "[post-install-prep] waiting for 3 LINSTOR nodes Online" +timeout 60 sh -ec 'until [ $(kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor node list | grep -c Online) -eq 3 ]; do sleep 1; done' + +echo "[post-install-prep] creating LINSTOR storage pools (parallel across nodes)" +created_pools=$(kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor sp l -s data --pastable | awk '$2 == "data" {printf " " $4} END{printf " "}') +for node in srv1 srv2 srv3; do + case $created_pools in + *" $node "*) echo " pool 'data' already exists on $node"; continue;; + esac + kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor ps cdp zfs ${node} /dev/vdc --pool-name data --storage-pool data & +done +wait + +echo "[post-install-prep] applying StorageClasses" +kubectl apply -f - <<'EOF' +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: local + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: linstor.csi.linbit.com +parameters: + linstor.csi.linbit.com/storagePool: "data" + linstor.csi.linbit.com/layerList: "storage" + linstor.csi.linbit.com/allowRemoteVolumeAccess: "false" +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: true +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: replicated +provisioner: linstor.csi.linbit.com +parameters: + linstor.csi.linbit.com/storagePool: "data" + linstor.csi.linbit.com/autoPlace: "3" + linstor.csi.linbit.com/layerList: "drbd storage" + linstor.csi.linbit.com/allowRemoteVolumeAccess: "true" + property.linstor.csi.linbit.com/DrbdOptions/auto-quorum: suspend-io + property.linstor.csi.linbit.com/DrbdOptions/Resource/on-no-data-accessible: suspend-io + property.linstor.csi.linbit.com/DrbdOptions/Resource/on-suspended-primary-outdated: force-secondary + property.linstor.csi.linbit.com/DrbdOptions/Net/rr-conflict: retry-connect +volumeBindingMode: Immediate +allowVolumeExpansion: true +EOF + +echo "[post-install-prep] waiting for MetalLB CRDs" +timeout 300 sh -ec 'until kubectl get crd ipaddresspools.metallb.io l2advertisements.metallb.io >/dev/null 2>&1; do sleep 2; done' + +echo "[post-install-prep] applying MetalLB IPAddressPool" +kubectl apply -f - <<'EOF' +--- +apiVersion: metallb.io/v1beta1 +kind: L2Advertisement +metadata: + name: cozystack + namespace: cozy-metallb +spec: + ipAddressPools: [cozystack] +--- +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: cozystack + namespace: cozy-metallb +spec: + addresses: [192.168.123.200-192.168.123.250] + autoAssign: true + avoidBuggyIPs: false +EOF + +echo "[post-install-prep] done" From eb87413556f67215d7fc947e67ed32dbdbfbb480 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 27 Apr 2026 23:10:16 +0500 Subject: [PATCH 05/27] fix(e2e): wait for LINSTOR HelmRelease Ready before polling its deployment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous 5-min timeout on `until kubectl get deploy/linstor-controller` fired during cold-install runs where the LINSTOR HR's dependency chain (cert-manager → piraeus-operator-crds → piraeus-operator → linstor) takes longer than 5 min to resolve, killing the whole post-install-prep script via `set -eu`. With CI's 3x retry the failure was hidden; with the retry removed it would surface every cold install. Wait on the LINSTOR HR being Ready first (the actual prerequisite), then keep the existence-poll as a near-instant backstop, then wait for Available. Same final semantics, no fragile fixed-window timeout. Surfaced by PR #2500 cozyreport diagnostics (post-install-prep.log). Signed-off-by: Myasnikov Daniil --- hack/e2e-post-install-prep.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/hack/e2e-post-install-prep.sh b/hack/e2e-post-install-prep.sh index 495bf72a..7a6fc798 100755 --- a/hack/e2e-post-install-prep.sh +++ b/hack/e2e-post-install-prep.sh @@ -5,6 +5,12 @@ # overlaps with the wait instead of compounding it. set -eu +echo "[post-install-prep] waiting for linstor HelmRelease object to exist" +timeout 60 sh -ec 'until kubectl get hr/linstor -n cozy-linstor >/dev/null 2>&1; do sleep 2; done' + +echo "[post-install-prep] waiting for linstor HelmRelease to be Ready" +kubectl wait helmrelease/linstor -n cozy-linstor --for=condition=Ready --timeout=15m + echo "[post-install-prep] waiting for linstor-controller deployment to exist" timeout 300 sh -ec 'until kubectl get deploy/linstor-controller -n cozy-linstor >/dev/null 2>&1; do sleep 2; done' From 21642837a7d2febae4074510b5e9e095f3d55785 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 27 Apr 2026 03:18:53 +0500 Subject: [PATCH 06/27] ci(pull-requests): drop 3x test retry, capture diagnostics on failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Across 5 sampled failure runs (30 PR runs total), 25/25 retry attempts failed. The retry loop never recovered a flake — it only stretched deterministic failures (e.g. kubernetes-previous: 11:37 + 7:49 + 10:16 = 29:42 wasted on one broken test). Replace with a single attempt plus inline diagnostics (HR list + recent events) so the actual failure surfaces immediately. Re-runs remain available via the standard 'gh run rerun' / empty-commit retry path. Signed-off-by: Myasnikov Daniil --- .github/workflows/pull-requests.yaml | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml index d9898616..b93e581a 100644 --- a/.github/workflows/pull-requests.yaml +++ b/.github/workflows/pull-requests.yaml @@ -262,23 +262,18 @@ jobs: failed_tests="" for app in $(ls hack/e2e-apps/*.bats | xargs -n1 basename | cut -d. -f1); do echo "::group::Testing $app" - attempt=0 - success=false - until [ $attempt -ge 3 ]; do - if make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-apps-$app; then - success=true - break - fi - attempt=$((attempt + 1)) - echo "❌ Attempt $attempt failed, retrying..." - done - if [ "$success" = true ]; then + if make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-apps-$app; then + echo "::endgroup::" echo "✅ Test $app completed successfully" else - echo "❌ Test $app failed after $attempt attempts" + echo "::endgroup::" + echo "❌ Test $app failed (no retry — see diagnostics below)" failed_tests="$failed_tests $app" + echo "::group::Diagnostics for $app" + kubectl get hr -A -o wide 2>&1 | tail -50 || true + kubectl get events -A --sort-by=.lastTimestamp 2>&1 | tail -30 || true + echo "::endgroup::" fi - echo "::endgroup::" done if [ -n "$failed_tests" ]; then echo "❌ Failed tests:$failed_tests" From 302b5d64e3b4e208800f7acbce042b72a72a1b94 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 27 Apr 2026 03:19:21 +0500 Subject: [PATCH 07/27] ci(pull-requests): drop 3x retry on prepare-env and install-cozystack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A failed cluster bootstrap or platform install is almost always a real bug (Talos boot, chart, operator) — not a flake. The 3x retry on these steps was the direct cause of the 250-min outlier run (24931612182): each retry compounds 25-30 min of work, and the failure mode persists across attempts. Single attempt + the existing 'collect-report' step on always() gives faster signal and a debug archive. Signed-off-by: Myasnikov Daniil --- .github/workflows/pull-requests.yaml | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml index b93e581a..2943adb0 100644 --- a/.github/workflows/pull-requests.yaml +++ b/.github/workflows/pull-requests.yaml @@ -223,31 +223,13 @@ jobs: - name: Prepare environment run: | cd /tmp/$SANDBOX_NAME - attempt=0 - until make SANDBOX_NAME=$SANDBOX_NAME prepare-env; do - attempt=$((attempt + 1)) - if [ $attempt -ge 3 ]; then - echo "❌ Attempt $attempt failed, exiting..." - exit 1 - fi - echo "❌ Attempt $attempt failed, retrying..." - done - echo "✅ The task completed successfully after $attempt attempts" + make SANDBOX_NAME=$SANDBOX_NAME prepare-env # ▸ Install Cozystack - name: Install Cozystack into sandbox run: | cd /tmp/$SANDBOX_NAME - attempt=0 - until make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME install-cozystack; do - attempt=$((attempt + 1)) - if [ $attempt -ge 3 ]; then - echo "❌ Attempt $attempt failed, exiting..." - exit 1 - fi - echo "❌ Attempt $attempt failed, retrying..." - done - echo "✅ The task completed successfully after $attempt attempts" + make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME install-cozystack - name: Run OpenAPI tests run: | From dcc87186764adbdbdc4a47eb9a4d7de487df73c8 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 27 Apr 2026 23:36:23 +0500 Subject: [PATCH 08/27] ci(pull-requests): keep 3x retry on Prepare environment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prepare environment is pure infrastructure (Talos image download, sandbox VMs boot, network setup). Failures here are mostly noisy-runner / transient infra hiccups (image-download stalls, NIC negotiation, etc.) — not cozystack code under test. Retry-on-failure is the right policy for infra setup. Install Cozystack and Run E2E tests remain single-attempt: they exercise cozystack code, where retries hide real bugs. Refines the previous two retry-removal commits. Signed-off-by: Myasnikov Daniil --- .github/workflows/pull-requests.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml index 2943adb0..2e276ab2 100644 --- a/.github/workflows/pull-requests.yaml +++ b/.github/workflows/pull-requests.yaml @@ -223,7 +223,16 @@ jobs: - name: Prepare environment run: | cd /tmp/$SANDBOX_NAME - make SANDBOX_NAME=$SANDBOX_NAME prepare-env + attempt=0 + until make SANDBOX_NAME=$SANDBOX_NAME prepare-env; do + attempt=$((attempt + 1)) + if [ $attempt -ge 3 ]; then + echo "Attempt $attempt failed, exiting..." + exit 1 + fi + echo "Attempt $attempt failed, retrying..." + done + echo "Prepare environment completed after $attempt attempts" # ▸ Install Cozystack - name: Install Cozystack into sandbox From 66888c91ee39d0b64eccc46df56b75e17930492e Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 27 Apr 2026 23:39:44 +0500 Subject: [PATCH 09/27] test(e2e-apps): replace fixed sleeps with HelmRelease existence backstop The app bats files all share the pattern: kubectl apply -f - < --for=condition=ready The fixed sleep guesses how long the cozystack-operator needs to translate the app CR into a HelmRelease. On a noisy CI runner or after a cold install that wait is sometimes too short, so the immediately-following `kubectl wait hr ...` fails because the HR object does not exist yet (`kubectl wait` errors out instantly on a missing resource). Replace each `sleep 5` (and the one outlier `sleep 15` in foundationdb that was paying for the same risk) with an event-driven backstop that polls for the HR's existence and exits the moment it appears, capped at 60s. The downstream `kubectl wait --for=condition=ready` is unchanged, preserving the original total budget. Same shape as the LINSTOR fix in commit eb87413 on feat/e2e-optimization: wait for the actual prerequisite (the HelmRelease the operator creates), not on a downstream artefact whose existence depends on it. Files: postgres, mariadb, kafka, mongodb, clickhouse, redis, qdrant, harbor, openbao, external-dns (both tests), vminstance (both tests, with the existence backstop hoisted before the downstream vmi-ip poll), and foundationdb. Signed-off-by: Myasnikov Daniil --- hack/e2e-apps/clickhouse.bats | 4 +++- hack/e2e-apps/external-dns.bats | 8 ++++++-- hack/e2e-apps/foundationdb.bats | 4 +++- hack/e2e-apps/harbor.bats | 4 +++- hack/e2e-apps/kafka.bats | 4 +++- hack/e2e-apps/mariadb.bats | 4 +++- hack/e2e-apps/mongodb.bats | 4 +++- hack/e2e-apps/openbao.bats | 4 +++- hack/e2e-apps/postgres.bats | 4 +++- hack/e2e-apps/qdrant.bats | 4 +++- hack/e2e-apps/redis.bats | 4 +++- hack/e2e-apps/vminstance.bats | 8 ++++++-- 12 files changed, 42 insertions(+), 14 deletions(-) diff --git a/hack/e2e-apps/clickhouse.bats b/hack/e2e-apps/clickhouse.bats index fbc9f906..79bc4671 100644 --- a/hack/e2e-apps/clickhouse.bats +++ b/hack/e2e-apps/clickhouse.bats @@ -35,7 +35,9 @@ spec: resources: {} resourcesPreset: "nano" EOF - sleep 5 + # Wait for the operator to materialise the HelmRelease before kubectl wait + # kicks in (kubectl wait errors immediately if the object does not exist yet). + timeout 60 sh -ec "until kubectl -n tenant-test get hr clickhouse-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait hr clickhouse-$name --timeout=20s --for=condition=ready timeout 180 sh -ec "until kubectl -n tenant-test get svc chendpoint-clickhouse-$name -o jsonpath='{.spec.ports[*].port}' | grep -q '8123 9000'; do sleep 10; done" kubectl -n tenant-test wait statefulset.apps/chi-clickhouse-$name-clickhouse-0-0 --timeout=120s --for=jsonpath='{.status.replicas}'=1 diff --git a/hack/e2e-apps/external-dns.bats b/hack/e2e-apps/external-dns.bats index ca114af8..5775927f 100644 --- a/hack/e2e-apps/external-dns.bats +++ b/hack/e2e-apps/external-dns.bats @@ -18,7 +18,9 @@ spec: - example.com EOF - sleep 5 + # Wait for the operator to materialise the HelmRelease before kubectl wait + # kicks in (kubectl wait errors immediately if the object does not exist yet). + timeout 60 sh -ec "until kubectl -n tenant-test get hr ${name} >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait hr ${name} --timeout=120s --for=condition=ready kubectl -n tenant-test wait hr ${name}-system --timeout=120s --for=condition=ready timeout 60 sh -ec "until kubectl -n tenant-test get deploy -l app.kubernetes.io/instance=${name}-system -o jsonpath='{.items[0].status.readyReplicas}' | grep -q '1'; do sleep 5; done" @@ -41,7 +43,9 @@ spec: - example.org EOF - sleep 5 + # Wait for the operator to materialise the HelmRelease before kubectl wait + # kicks in (kubectl wait errors immediately if the object does not exist yet). + timeout 60 sh -ec "until kubectl -n tenant-test get hr ${name} >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait hr ${name} --timeout=120s --for=condition=ready kubectl -n tenant-test wait hr ${name}-system --timeout=120s --for=condition=ready timeout 60 sh -ec "until kubectl -n tenant-test get deploy -l app.kubernetes.io/instance=${name}-system -o jsonpath='{.items[0].status.readyReplicas}' | grep -q '1'; do sleep 5; done" diff --git a/hack/e2e-apps/foundationdb.bats b/hack/e2e-apps/foundationdb.bats index 199bda11..accfd8e1 100644 --- a/hack/e2e-apps/foundationdb.bats +++ b/hack/e2e-apps/foundationdb.bats @@ -42,7 +42,9 @@ spec: imageType: "unified" automaticReplacements: true EOF - sleep 15 + # Wait for the operator to materialise the HelmRelease before kubectl wait + # kicks in (kubectl wait errors immediately if the object does not exist yet). + timeout 60 sh -ec "until kubectl -n tenant-test get hr foundationdb-$name >/dev/null 2>&1; do sleep 2; done" # Wait for HelmRelease to be ready kubectl -n tenant-test wait hr foundationdb-$name --timeout=300s --for=condition=ready diff --git a/hack/e2e-apps/harbor.bats b/hack/e2e-apps/harbor.bats index 26f407ea..ae825018 100644 --- a/hack/e2e-apps/harbor.bats +++ b/hack/e2e-apps/harbor.bats @@ -38,7 +38,9 @@ spec: size: 1Gi replicas: 1 EOF - sleep 5 + # Wait for the operator to materialise the HelmRelease before kubectl wait + # kicks in (kubectl wait errors immediately if the object does not exist yet). + timeout 60 sh -ec "until kubectl -n tenant-test get hr $release >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait hr $release --timeout=60s --for=condition=ready # Wait for COSI to provision bucket diff --git a/hack/e2e-apps/kafka.bats b/hack/e2e-apps/kafka.bats index d85837bd..7b1e2638 100644 --- a/hack/e2e-apps/kafka.bats +++ b/hack/e2e-apps/kafka.bats @@ -39,7 +39,9 @@ spec: partitions: 1 replicas: 2 EOF - sleep 5 + # Wait for the operator to materialise the HelmRelease before kubectl wait + # kicks in (kubectl wait errors immediately if the object does not exist yet). + timeout 60 sh -ec "until kubectl -n tenant-test get hr kafka-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait hr kafka-$name --timeout=30s --for=condition=ready kubectl wait kafkas -n tenant-test test --timeout=300s --for=condition=ready timeout 60 sh -ec "until kubectl -n tenant-test get pvc data-kafka-$name-zookeeper-0; do sleep 10; done" diff --git a/hack/e2e-apps/mariadb.bats b/hack/e2e-apps/mariadb.bats index 68f5fc00..0b7b27c2 100644 --- a/hack/e2e-apps/mariadb.bats +++ b/hack/e2e-apps/mariadb.bats @@ -35,7 +35,9 @@ spec: resources: {} resourcesPreset: "nano" EOF - sleep 5 + # Wait for the operator to materialise the HelmRelease before kubectl wait + # kicks in (kubectl wait errors immediately if the object does not exist yet). + timeout 60 sh -ec "until kubectl -n tenant-test get hr mariadb-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait hr mariadb-$name --timeout=30s --for=condition=ready timeout 80 sh -ec "until kubectl -n tenant-test get svc mariadb-$name -o jsonpath='{.spec.ports[0].port}' | grep -q '3306'; do sleep 10; done" timeout 80 sh -ec "until kubectl -n tenant-test get endpoints mariadb-$name -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done" diff --git a/hack/e2e-apps/mongodb.bats b/hack/e2e-apps/mongodb.bats index 7712fe01..0e5f4042 100644 --- a/hack/e2e-apps/mongodb.bats +++ b/hack/e2e-apps/mongodb.bats @@ -26,7 +26,9 @@ spec: backup: enabled: false EOF - sleep 5 + # Wait for the operator to materialise the HelmRelease before kubectl wait + # kicks in (kubectl wait errors immediately if the object does not exist yet). + timeout 60 sh -ec "until kubectl -n tenant-test get hr mongodb-$name >/dev/null 2>&1; do sleep 2; done" # Wait for HelmRelease kubectl -n tenant-test wait hr mongodb-$name --timeout=60s --for=condition=ready # Wait for MongoDB service (port 27017) diff --git a/hack/e2e-apps/openbao.bats b/hack/e2e-apps/openbao.bats index 35271e55..fc85d891 100644 --- a/hack/e2e-apps/openbao.bats +++ b/hack/e2e-apps/openbao.bats @@ -18,7 +18,9 @@ spec: external: false ui: true EOF - sleep 5 + # Wait for the operator to materialise the HelmRelease before kubectl wait + # kicks in (kubectl wait errors immediately if the object does not exist yet). + timeout 60 sh -ec "until kubectl -n tenant-test get hr openbao-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait hr openbao-$name --timeout=60s --for=condition=ready kubectl -n tenant-test wait hr openbao-$name-system --timeout=120s --for=condition=ready diff --git a/hack/e2e-apps/postgres.bats b/hack/e2e-apps/postgres.bats index b86bc664..76fc1638 100644 --- a/hack/e2e-apps/postgres.bats +++ b/hack/e2e-apps/postgres.bats @@ -40,7 +40,9 @@ spec: resources: {} resourcesPreset: "nano" EOF - sleep 5 + # Wait for the operator to materialise the HelmRelease before kubectl wait + # kicks in (kubectl wait errors immediately if the object does not exist yet). + timeout 60 sh -ec "until kubectl -n tenant-test get hr postgres-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait hr postgres-$name --timeout=100s --for=condition=ready kubectl -n tenant-test wait job.batch postgres-$name-init-job --timeout=50s --for=condition=Complete timeout 40 sh -ec "until kubectl -n tenant-test get svc postgres-$name-r -o jsonpath='{.spec.ports[0].port}' | grep -q '5432'; do sleep 10; done" diff --git a/hack/e2e-apps/qdrant.bats b/hack/e2e-apps/qdrant.bats index 9d37ce01..2a6e05b7 100755 --- a/hack/e2e-apps/qdrant.bats +++ b/hack/e2e-apps/qdrant.bats @@ -17,7 +17,9 @@ spec: resources: {} external: false EOF - sleep 5 + # Wait for the operator to materialise the HelmRelease before kubectl wait + # kicks in (kubectl wait errors immediately if the object does not exist yet). + timeout 60 sh -ec "until kubectl -n tenant-test get hr qdrant-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait hr qdrant-$name --timeout=60s --for=condition=ready kubectl -n tenant-test wait hr qdrant-$name-system --timeout=120s --for=condition=ready kubectl -n tenant-test wait sts qdrant-$name --timeout=90s --for=jsonpath='{.status.readyReplicas}'=1 diff --git a/hack/e2e-apps/redis.bats b/hack/e2e-apps/redis.bats index 3ddff71a..b2cef269 100644 --- a/hack/e2e-apps/redis.bats +++ b/hack/e2e-apps/redis.bats @@ -18,7 +18,9 @@ spec: resources: {} resourcesPreset: "nano" EOF - sleep 5 + # Wait for the operator to materialise the HelmRelease before kubectl wait + # kicks in (kubectl wait errors immediately if the object does not exist yet). + timeout 60 sh -ec "until kubectl -n tenant-test get hr redis-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait hr redis-$name --timeout=20s --for=condition=ready kubectl -n tenant-test wait pvc redisfailover-persistent-data-rfr-redis-$name-0 --timeout=50s --for=jsonpath='{.status.phase}'=Bound kubectl -n tenant-test wait deploy rfs-redis-$name --timeout=90s --for=condition=available diff --git a/hack/e2e-apps/vminstance.bats b/hack/e2e-apps/vminstance.bats index fa0f0b5b..6d4eff45 100644 --- a/hack/e2e-apps/vminstance.bats +++ b/hack/e2e-apps/vminstance.bats @@ -18,7 +18,9 @@ spec: storage: 5Gi storageClass: replicated EOF - sleep 5 + # Wait for the operator to materialise the HelmRelease before kubectl wait + # kicks in (kubectl wait errors immediately if the object does not exist yet). + timeout 60 sh -ec "until kubectl -n tenant-test get hr vm-disk-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait hr vm-disk-$name --timeout=5s --for=condition=ready kubectl -n tenant-test wait dv vm-disk-$name --timeout=250s --for=condition=ready kubectl -n tenant-test wait pvc vm-disk-$name --timeout=200s --for=jsonpath='{.status.phase}'=Bound @@ -59,7 +61,9 @@ spec: - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPht0dPk5qQ+54g1hSX7A6AUxXJW5T6n/3d7Ga2F8gTF test@test cloudInitSeed: "" EOF - sleep 5 + # Wait for the operator to materialise the HelmRelease before downstream + # waits proceed (kubectl wait errors immediately if the HR does not exist). + timeout 60 sh -ec "until kubectl -n tenant-test get hr vm-instance-$name >/dev/null 2>&1; do sleep 2; done" timeout 20 sh -ec "until kubectl -n tenant-test get vmi vm-instance-$name -o jsonpath='{.status.interfaces[0].ipAddress}' | grep -q '[0-9]'; do sleep 5; done" kubectl -n tenant-test wait hr vm-instance-$name --timeout=5s --for=condition=ready kubectl -n tenant-test wait vm vm-instance-$name --timeout=20s --for=condition=ready From f5a968662923acc63ef00109a674b9d78326ebfe Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 27 Apr 2026 23:41:39 +0500 Subject: [PATCH 10/27] test(openapi): replace fixed sleep with port-readiness wait for kubectl proxy `kubectl proxy --port=21234 & sleep 0.5` guesses how long the proxy needs to start listening. On a slow runner the curl that follows can race the proxy and fail with "connection refused". Replace the fixed `sleep 0.5` with `nc -z localhost 21234` polled until the listener accepts a connection (10s cap). Same idiom already used in hack/e2e-apps/bucket.bats for the seaweedfs-s3 port-forward. Also save the proxy PID to a named variable so the trap kills the right process even if a later background command lands first in $!. Signed-off-by: Myasnikov Daniil --- hack/e2e-test-openapi.bats | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/hack/e2e-test-openapi.bats b/hack/e2e-test-openapi.bats index 88ed734b..5fb5a243 100644 --- a/hack/e2e-test-openapi.bats +++ b/hack/e2e-test-openapi.bats @@ -14,8 +14,13 @@ @test "Test OpenAPI v2 endpoint (protobuf)" { ( - kubectl proxy --port=21234 & sleep 0.5 - trap "kill $!" EXIT + kubectl proxy --port=21234 & + proxy_pid=$! + trap "kill $proxy_pid" EXIT + # Wait for the proxy to actually be listening rather than guessing with + # a fixed sleep. nc -z is non-destructive and exits 0 the moment the + # listener accepts a connection. + timeout 10 sh -ec 'until nc -z localhost 21234; do sleep 0.1; done' curl -sS --fail 'http://localhost:21234/openapi/v2?timeout=32s' -H 'Accept: application/com.github.proto-openapi.spec.v2@v1.0+protobuf' > /dev/null ) } From 461cdf961fc6b14e213614b4e66acf00533ced6b Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 27 Apr 2026 23:42:32 +0500 Subject: [PATCH 11/27] test(e2e): annotate genuine sleeps with TODO and rationale Three poll/sleep sites in the install bats and the kubernetes runner are NOT replaceable by `kubectl wait --for=condition=...` because the signal they observe is not a Kubernetes API condition: - e2e-install-cozystack.bats:55-56 - 5s pad lets late-arriving HRs join the awk-snapshot the parallel `kubectl wait` runs against. There is no k8s condition for "all expected platform HRs have been emitted" short of hard-coding the list. - e2e-install-cozystack.bats:75 - LINSTOR node membership is reported by the linstor binary running inside the controller pod (kubectl exec), not a CRD status, so kubectl wait cannot subscribe to it. - e2e-apps/run-kubernetes.sh:231-238 - validates the external HTTP path through MetalLB -> tenant ingress -> backend pod end-to-end. Not a single API condition. Annotate each with TODO(e2e-replace-fixed-timeouts) and the rationale so future readers do not "fix" them with a kubectl wait that does not work. Signed-off-by: Myasnikov Daniil --- hack/e2e-apps/run-kubernetes.sh | 5 +++++ hack/e2e-install-cozystack.bats | 5 +++++ hack/e2e-post-install-prep.sh | 3 +++ 3 files changed, 13 insertions(+) diff --git a/hack/e2e-apps/run-kubernetes.sh b/hack/e2e-apps/run-kubernetes.sh index 761ee57d..96975e16 100644 --- a/hack/e2e-apps/run-kubernetes.sh +++ b/hack/e2e-apps/run-kubernetes.sh @@ -227,6 +227,11 @@ EOF exit 1 fi + # TODO(e2e-replace-fixed-timeouts): genuine retry loop. This validates an + # external HTTP path (MetalLB-advertised LB IP -> in-tenant ingress -> + # backend pod) which is not visible to the Kubernetes API as a single + # condition, so kubectl wait cannot replace it. The 20x3s = 60s budget is + # capped with `lb_ok=false` then asserted below. lb_ok=false for i in $(seq 1 20); do echo "Attempt $i" diff --git a/hack/e2e-install-cozystack.bats b/hack/e2e-install-cozystack.bats index 2fe0dfbd..f2e7b42c 100644 --- a/hack/e2e-install-cozystack.bats +++ b/hack/e2e-install-cozystack.bats @@ -61,6 +61,11 @@ EOF # Wait until HelmReleases appear & reconcile them timeout 180 sh -ec 'until [ $(kubectl get hr -A --no-headers 2>/dev/null | wc -l) -gt 10 ]; do sleep 1; done' + # TODO(e2e-replace-fixed-timeouts): genuine sleep. The threshold of 10 is a + # heuristic for "enough HRs visible to start waiting"; the snapshot below + # uses whatever HRs have appeared by then. There is no objective k8s API + # signal for "all platform HRs have been emitted" without hard-coding the + # expected list, so the 5s pad lets a few late-arrivals join the snapshot. sleep 5 kubectl get hr -A | awk 'NR>1 {print "kubectl wait --timeout=15m --for=condition=ready -n "$1" hr/"$2" &"} END {print "wait"}' | sh -ex diff --git a/hack/e2e-post-install-prep.sh b/hack/e2e-post-install-prep.sh index 7a6fc798..a5010503 100755 --- a/hack/e2e-post-install-prep.sh +++ b/hack/e2e-post-install-prep.sh @@ -18,6 +18,9 @@ echo "[post-install-prep] waiting for linstor-controller to be Available" kubectl wait deployment/linstor-controller -n cozy-linstor --timeout=5m --for=condition=available echo "[post-install-prep] waiting for 3 LINSTOR nodes Online" +# TODO(e2e-replace-fixed-timeouts): genuine poll. LINSTOR node membership is +# reported by the linstor binary inside the controller pod, not via a +# Kubernetes API condition, so kubectl wait cannot subscribe to it. timeout 60 sh -ec 'until [ $(kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor node list | grep -c Online) -eq 3 ]; do sleep 1; done' echo "[post-install-prep] creating LINSTOR storage pools (parallel across nodes)" From ca5c77cdc1163af3b9a73b2c099fe0aec9bad30e Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Tue, 28 Apr 2026 10:53:20 +0500 Subject: [PATCH 12/27] fix(e2e): drop --create-namespace from installer helm call The cozy-installer chart declares Namespace cozy-system itself (with helm.sh/resource-policy: keep). Combining that with --create-namespace makes Helm v3 pre-create the namespace via plain kubectl-create (without helm annotations); the subsequent chart apply then fails with 'namespaces cozy-system already exists'. The 3x retry on Install Cozystack was hiding this. First attempt failed, second saw 'release exists' and treated it as upgrade. Reproducible across PRs (PR #2507 E2E hit the same first-attempt failure today and recovered on retry). Surfaced cleanly after dropping the retry on this step. Drop --create-namespace; let the chart manage its own namespace. Signed-off-by: Myasnikov Daniil --- hack/e2e-install-cozystack.bats | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/hack/e2e-install-cozystack.bats b/hack/e2e-install-cozystack.bats index f2e7b42c..85ad41fb 100644 --- a/hack/e2e-install-cozystack.bats +++ b/hack/e2e-install-cozystack.bats @@ -9,10 +9,15 @@ @test "Install Cozystack" { # Install cozy-installer chart (operator installs CRDs on startup via --install-crds) + # The chart declares Namespace cozy-system with helm.sh/resource-policy: keep, + # so do NOT pass --create-namespace. Helm v3's --create-namespace pre-creates + # the namespace via plain kubectl-create (without helm annotations); the + # subsequent chart apply then fails with "namespaces \"cozy-system\" already + # exists". The 3x retry was masking this — first attempt failed, second saw + # "release exists" and treated it as upgrade. Surfaced after dropping retry. helm upgrade installer packages/core/installer \ --install \ --namespace cozy-system \ - --create-namespace \ --set cozystackOperator.helmReleaseInterval=30s \ --wait \ --timeout 2m From 2796819702707ab7be2956b4834f2bd261115f48 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Tue, 28 Apr 2026 11:08:17 +0500 Subject: [PATCH 13/27] fix(e2e): pre-create cozy-system as helm-adoptable namespace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cozy-installer chart declares Namespace cozy-system itself, which makes helm cold-install paradoxical: - WITH --create-namespace: helm pre-creates the ns without helm annotations; the chart's own Namespace apply then fails with 'already exists'. - WITHOUT --create-namespace: helm fails before any apply because it can't write its release-secret to a non-existent ns. The 3x retry was masking both directions — second attempt saw a partial release and took the upgrade path, which patch-merges instead of strict-create. Reproducible on every cold install across PRs. Pre-create cozy-system with meta.helm.sh/release-name and app.kubernetes.io/managed-by=Helm so helm adopts it cleanly during install. Labels mirror the chart's template (PSA enforce=privileged etc.) so behaviour matches what the chart would have done. Signed-off-by: Myasnikov Daniil --- hack/e2e-install-cozystack.bats | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/hack/e2e-install-cozystack.bats b/hack/e2e-install-cozystack.bats index 85ad41fb..6805979b 100644 --- a/hack/e2e-install-cozystack.bats +++ b/hack/e2e-install-cozystack.bats @@ -9,12 +9,32 @@ @test "Install Cozystack" { # Install cozy-installer chart (operator installs CRDs on startup via --install-crds) - # The chart declares Namespace cozy-system with helm.sh/resource-policy: keep, - # so do NOT pass --create-namespace. Helm v3's --create-namespace pre-creates - # the namespace via plain kubectl-create (without helm annotations); the - # subsequent chart apply then fails with "namespaces \"cozy-system\" already - # exists". The 3x retry was masking this — first attempt failed, second saw - # "release exists" and treated it as upgrade. Surfaced after dropping retry. + # Pre-create cozy-system with helm-meta annotations + managed-by label so that + # helm adopts it during install instead of creating a duplicate. The chart's + # Namespace template has helm.sh/resource-policy: keep, but on a fresh cluster: + # - WITH --create-namespace: helm pre-creates ns without helm metadata, then + # the chart's own Namespace apply fails with "already exists". + # - WITHOUT --create-namespace: helm fails to write its release secret to + # cozy-system because the ns doesn't exist yet. + # Pre-creating with the right meta lets helm adopt the existing ns cleanly. + # The 3x retry was hiding this — second attempt saw a partial release and + # took the upgrade path, which doesn't strict-create the namespace. + kubectl apply -f - <<'EOF' +apiVersion: v1 +kind: Namespace +metadata: + name: cozy-system + labels: + app.kubernetes.io/managed-by: Helm + cozystack.io/system: "true" + cozystack.io/deletion-protected: "true" + pod-security.kubernetes.io/enforce: privileged + annotations: + meta.helm.sh/release-name: installer + meta.helm.sh/release-namespace: cozy-system + helm.sh/resource-policy: keep +EOF + helm upgrade installer packages/core/installer \ --install \ --namespace cozy-system \ From 3a87485ca102e11dda3a4eabfd50dee894e4095f Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Tue, 28 Apr 2026 11:43:02 +0500 Subject: [PATCH 14/27] fix(installer): bootstrap cozy-system via --create-namespace + label hook Removes the chart's `Namespace cozy-system` resource and replaces it with a pre-install/pre-upgrade Job hook (cozy-system-labeler) that patches the required labels onto the namespace after `--create-namespace` creates it. Why: helm v3 has a known chicken-and-egg with charts that ship their own Namespace: - WITH `--create-namespace` on the install command, helm pre-creates the namespace via plain kubectl-create (no helm meta annotations); the chart's own Namespace apply then fails with `already exists`. - WITHOUT `--create-namespace`, helm fails immediately because it cannot write its release-secret to a non-existent namespace. Until now this was hidden by the 3x retry on `Install Cozystack` in `.github/workflows/pull-requests.yaml`: first attempt always fails with the conflict, second attempt sees the existing failed release and takes the upgrade code path which patch-merges instead of strict-create. Reproducible on every cold install. Surfaced cleanly when retries on the install step were dropped. After this change: - install commands use `helm upgrade --install --namespace cozy-system --create-namespace`. Standard pattern, matches kube-prometheus-stack / argo-cd / cert-manager / others. - the pre-install hook (SA + ClusterRole + ClusterRoleBinding + Job) patches `cozystack.io/system=true` and `pod-security.kubernetes.io/enforce=privileged` onto the namespace before main resources apply. - hook-delete-policy=before-hook-creation,hook-succeeded so the RBAC surface only exists during install/upgrade. Verified end-to-end on a kind cluster: cold install in 3.3s, upgrade idempotent, cleanup clean. Image pinned to `alpine/k8s:1.32.0` for the hook (small, public, includes kubectl). Signed-off-by: Myasnikov Daniil --- .../templates/cozy-system-labels.yaml | 82 +++++++++++++++++++ .../templates/cozystack-operator.yaml | 10 --- 2 files changed, 82 insertions(+), 10 deletions(-) create mode 100644 packages/core/installer/templates/cozy-system-labels.yaml diff --git a/packages/core/installer/templates/cozy-system-labels.yaml b/packages/core/installer/templates/cozy-system-labels.yaml new file mode 100644 index 00000000..1ea3e5eb --- /dev/null +++ b/packages/core/installer/templates/cozy-system-labels.yaml @@ -0,0 +1,82 @@ +{{/* + Pre-install/upgrade hook that labels the cozy-system namespace with the + PodSecurity and cozystack identity labels. The chart no longer ships a + Namespace resource (helm v3 cannot adopt a namespace pre-created by + --create-namespace because it lacks helm meta annotations; without + --create-namespace, helm fails to write its release-secret because the + namespace does not exist yet — chicken-and-egg). + Install commands now use --create-namespace to bootstrap the namespace, + and this hook patches the labels the operator depends on (notably + pod-security.kubernetes.io/enforce=privileged for the host-network + operator pod). +*/}} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cozy-system-labeler + namespace: cozy-system + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-200" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cozy-system-labeler + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-180" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +rules: +- apiGroups: [""] + resources: ["namespaces"] + resourceNames: ["cozy-system"] + verbs: ["get", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cozy-system-labeler + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-160" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +subjects: +- kind: ServiceAccount + name: cozy-system-labeler + namespace: cozy-system +roleRef: + kind: ClusterRole + name: cozy-system-labeler + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: cozy-system-labeler + namespace: cozy-system + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-100" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +spec: + backoffLimit: 3 + ttlSecondsAfterFinished: 60 + template: + spec: + serviceAccountName: cozy-system-labeler + restartPolicy: OnFailure + containers: + - name: kubectl + image: alpine/k8s:1.32.0 + imagePullPolicy: IfNotPresent + command: + - kubectl + - label + - --overwrite + - namespace + - cozy-system + - cozystack.io/system=true + - pod-security.kubernetes.io/enforce=privileged diff --git a/packages/core/installer/templates/cozystack-operator.yaml b/packages/core/installer/templates/cozystack-operator.yaml index 8389aca4..9891f7c9 100644 --- a/packages/core/installer/templates/cozystack-operator.yaml +++ b/packages/core/installer/templates/cozystack-operator.yaml @@ -4,16 +4,6 @@ {{- end -}} --- apiVersion: v1 -kind: Namespace -metadata: - name: cozy-system - labels: - cozystack.io/system: "true" - pod-security.kubernetes.io/enforce: privileged - annotations: - helm.sh/resource-policy: keep ---- -apiVersion: v1 kind: ServiceAccount metadata: name: cozystack From dcfd0e7e9940a365171189dad6fd99d46e12dbe1 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Tue, 28 Apr 2026 11:47:12 +0500 Subject: [PATCH 15/27] test(e2e): use --create-namespace, drop bats workaround Now that the chart fix (this same branch) bootstraps cozy-system via --create-namespace + a pre-install label hook, the bats no longer needs to pre-apply a hand-crafted helm-adoptable namespace. Restore plain --create-namespace and drop the kubectl apply block. Signed-off-by: Myasnikov Daniil --- hack/e2e-install-cozystack.bats | 31 ++++--------------------------- 1 file changed, 4 insertions(+), 27 deletions(-) diff --git a/hack/e2e-install-cozystack.bats b/hack/e2e-install-cozystack.bats index 6805979b..ebc4a5b4 100644 --- a/hack/e2e-install-cozystack.bats +++ b/hack/e2e-install-cozystack.bats @@ -8,36 +8,13 @@ } @test "Install Cozystack" { - # Install cozy-installer chart (operator installs CRDs on startup via --install-crds) - # Pre-create cozy-system with helm-meta annotations + managed-by label so that - # helm adopts it during install instead of creating a duplicate. The chart's - # Namespace template has helm.sh/resource-policy: keep, but on a fresh cluster: - # - WITH --create-namespace: helm pre-creates ns without helm metadata, then - # the chart's own Namespace apply fails with "already exists". - # - WITHOUT --create-namespace: helm fails to write its release secret to - # cozy-system because the ns doesn't exist yet. - # Pre-creating with the right meta lets helm adopt the existing ns cleanly. - # The 3x retry was hiding this — second attempt saw a partial release and - # took the upgrade path, which doesn't strict-create the namespace. - kubectl apply -f - <<'EOF' -apiVersion: v1 -kind: Namespace -metadata: - name: cozy-system - labels: - app.kubernetes.io/managed-by: Helm - cozystack.io/system: "true" - cozystack.io/deletion-protected: "true" - pod-security.kubernetes.io/enforce: privileged - annotations: - meta.helm.sh/release-name: installer - meta.helm.sh/release-namespace: cozy-system - helm.sh/resource-policy: keep -EOF - + # Install cozy-installer chart (operator installs CRDs on startup via --install-crds). + # The chart's pre-install hook patches PSA + cozystack labels onto cozy-system + # after --create-namespace bootstraps it. helm upgrade installer packages/core/installer \ --install \ --namespace cozy-system \ + --create-namespace \ --set cozystackOperator.helmReleaseInterval=30s \ --wait \ --timeout 2m From 428868a3b24f929abe78865c1aa6e44cf727f0f8 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Tue, 28 Apr 2026 12:13:55 +0500 Subject: [PATCH 16/27] test(e2e-apps): fix vminstance delete-recreate race + bump VM IP/ready timeouts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two reliability fixes in `hack/e2e-apps/vminstance.bats`: 1. Delete-recreate race. Both @test blocks deleted the prior VMInstance/VMDisk with `kubectl ... --timeout=2m || true`. The `|| true` silently swallowed timeout errors, letting the test continue with the resource still in finalizer-drain. The next `kubectl apply` then no-ops with "Detected changes to resource ... which is currently being deleted", producing a NotFound on the downstream HR wait. Bumped the delete timeout to 3m and removed `|| true` so true delete failures surface loudly. End-of-test cleanup deletes also got the explicit timeout. 2. VM IP and VM ready timeouts. The 20s timeout for the VMI to acquire an IP was unrealistic for nested KubeVirt under runner load — virt-launcher + libvirt + cloud-init DHCP routinely takes 30-60s. Bumped to 120s with a 2s poll interval (60 polls instead of 4). VM ready timeout bumped from 20s to 60s for the same reason. Both surfaced as recurring first-attempt failures in the past 30 successful PR runs (7/17 analysable runs in each case) — masked by the 3x retry on `Run E2E tests`. Signed-off-by: Myasnikov Daniil --- hack/e2e-apps/vminstance.bats | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/hack/e2e-apps/vminstance.bats b/hack/e2e-apps/vminstance.bats index 6d4eff45..f42225e8 100644 --- a/hack/e2e-apps/vminstance.bats +++ b/hack/e2e-apps/vminstance.bats @@ -2,8 +2,15 @@ @test "Create a VM Disk" { name='test' - kubectl -n tenant-test delete vminstances.apps.cozystack.io $name --ignore-not-found --timeout=2m || true - kubectl -n tenant-test delete vmdisks.apps.cozystack.io $name --ignore-not-found --timeout=2m || true + # Delete any leftover from a previous run and BLOCK until removal completes. + # `kubectl apply` on a resource still in finalizer-drain silently no-ops + # ("Detected changes to resource ... which is currently being deleted"), + # which then races a NotFound on the downstream HR wait. The previous + # `|| true` swallowed timeout errors here and let the test continue with + # the old VMDisk still draining. Bumped timeout to 3m and removed `|| true` + # so a true delete failure surfaces immediately. + kubectl -n tenant-test delete vminstances.apps.cozystack.io $name --ignore-not-found --timeout=3m + kubectl -n tenant-test delete vmdisks.apps.cozystack.io $name --ignore-not-found --timeout=3m kubectl apply -f - </dev/null 2>&1; do sleep 2; done" - timeout 20 sh -ec "until kubectl -n tenant-test get vmi vm-instance-$name -o jsonpath='{.status.interfaces[0].ipAddress}' | grep -q '[0-9]'; do sleep 5; done" + # Nested KubeVirt VM startup (virt-launcher + libvirt + cloud-init DHCP) + # routinely takes 30-60s under runner load; the previous 20s was unrealistic + # and produced flakes. 120s is a comfortable upper bound for nested virt. + timeout 120 sh -ec "until kubectl -n tenant-test get vmi vm-instance-$name -o jsonpath='{.status.interfaces[0].ipAddress}' | grep -q '[0-9]'; do sleep 2; done" kubectl -n tenant-test wait hr vm-instance-$name --timeout=5s --for=condition=ready - kubectl -n tenant-test wait vm vm-instance-$name --timeout=20s --for=condition=ready - kubectl -n tenant-test delete vminstances.apps.cozystack.io $name - kubectl -n tenant-test delete vmdisks.apps.cozystack.io $diskName + # VM ready follows IP assignment closely; 60s gives buffer for the qemu-guest-agent. + kubectl -n tenant-test wait vm vm-instance-$name --timeout=60s --for=condition=ready + kubectl -n tenant-test delete vminstances.apps.cozystack.io $name --timeout=3m + kubectl -n tenant-test delete vmdisks.apps.cozystack.io $diskName --timeout=3m } From 811e15978c947e707713944067ad658030b9738d Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Tue, 28 Apr 2026 12:40:13 +0500 Subject: [PATCH 17/27] feat(operator): switch HelmRelease retries to Strategy=RetryOnFailure with configurable RetryInterval MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The operator generated HelmReleases with `Install.Remediation{Retries: -1}` and `Upgrade.Remediation{Retries: -1}`, which is the v2 way of saying "retry forever, never remediate". Combined with the spec-level `Interval: 5m`, this meant a failed install/upgrade waited a full 5 minutes before the next attempt — even when the underlying race that caused the failure (e.g. a chart artifact not quite Ready, a HR dependency reconciling out of order) cleared in seconds. Audit of the last 30 successful PR runs found that every single run had its `Install Cozystack` step's `kubectl wait hr/seaweedfs-system` time out at the 2-minute mark and recover only after an inline `flux reconcile --force` workaround. Same root cause: 5-minute retry interval was longer than the test's wait window. That workaround can be removed once this change lands. Switch to `Strategy.Name=RetryOnFailure` with `RetryInterval` exposed via a new `--helmrelease-retry-interval` operator flag (default 30s): - Functionally equivalent to the previous configuration ("retry forever on failure") since `Retries: -1` meant remediation never fired anyway. - Decouples retry-on-failure timing from `spec.Interval` so failed releases recover at 30s without polling healthy releases at the same cadence (which would multiply controller load by ~10x for no benefit). - Helm chart exposes `cozystackOperator.helmReleaseRetryInterval`, defaulting to empty (operator uses its own 30s default). - Same flag pattern as the existing `--helmrelease-interval` (5m default), also added in this PR via cherry-pick. Verified: go build green, go test ./internal/operator/... pass, helm template renders both flags conditionally on values. Signed-off-by: Myasnikov Daniil --- cmd/cozystack-operator/main.go | 19 ++++++++++++--- internal/operator/package_reconciler.go | 24 ++++++++++++------- .../templates/cozystack-operator.yaml | 3 +++ packages/core/installer/values.yaml | 4 ++++ 4 files changed, 39 insertions(+), 11 deletions(-) diff --git a/cmd/cozystack-operator/main.go b/cmd/cozystack-operator/main.go index 01ce097f..fcc8b1b1 100644 --- a/cmd/cozystack-operator/main.go +++ b/cmd/cozystack-operator/main.go @@ -84,6 +84,7 @@ func main() { var telemetryEndpoint string var telemetryInterval string var helmReleaseInterval string + var helmReleaseRetryInterval string var cozyValuesSecretName string var cozyValuesSecretNamespace string var cozyValuesNamespaceSelector string @@ -112,6 +113,12 @@ func main() { "Reconcile interval applied to HelmReleases created by the Package reconciler. "+ "Lower values speed up dependency-blocked retries (e.g. during E2E install) at the cost of "+ "controller load. Production default 5m matches existing behaviour.") + flag.StringVar(&helmReleaseRetryInterval, "helmrelease-retry-interval", "30s", + "Retry interval applied to Install.Strategy and Upgrade.Strategy of HelmReleases created "+ + "by the Package reconciler. With Strategy.Name=RetryOnFailure, this controls how long the "+ + "controller waits between failed install/upgrade attempts. Decoupled from --helmrelease-interval "+ + "(which is the healthy reconcile cadence) so failures recover fast without polling healthy "+ + "releases at the same fast cadence.") flag.StringVar(&platformSourceURL, "platform-source-url", "", "Platform source URL (oci:// or https://). If specified, generates OCIRepository or GitRepository resource.") flag.StringVar(&platformSourceName, "platform-source-name", "cozystack-platform", "Name for the generated platform source resource and PackageSource") flag.StringVar(&platformSourceRef, "platform-source-ref", "", "Reference specification as key=value pairs (e.g., 'branch=main' or 'digest=sha256:...,tag=v1.0'). For OCI: digest, semver, semverFilter, tag. For Git: branch, tag, semver, name, commit.") @@ -132,6 +139,11 @@ func main() { setupLog.Error(err, "invalid --helmrelease-interval value", "value", helmReleaseInterval) os.Exit(1) } + hrRetryIntervalDuration, err := time.ParseDuration(helmReleaseRetryInterval) + if err != nil { + setupLog.Error(err, "invalid --helmrelease-retry-interval value", "value", helmReleaseRetryInterval) + os.Exit(1) + } config := ctrl.GetConfigOrDie() @@ -269,9 +281,10 @@ func main() { // Setup Package reconciler if err := (&operator.PackageReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - HelmReleaseInterval: hrIntervalDuration, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + HelmReleaseInterval: hrIntervalDuration, + HelmReleaseRetryInterval: hrRetryIntervalDuration, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Package") os.Exit(1) diff --git a/internal/operator/package_reconciler.go b/internal/operator/package_reconciler.go index d7cd1151..e2dac1cf 100644 --- a/internal/operator/package_reconciler.go +++ b/internal/operator/package_reconciler.go @@ -59,8 +59,9 @@ func parseCRDPolicy(install *cozyv1alpha1.ComponentInstall) helmv2.CRDsPolicy { // PackageReconciler reconciles Package resources type PackageReconciler struct { client.Client - Scheme *runtime.Scheme - HelmReleaseInterval time.Duration + Scheme *runtime.Scheme + HelmReleaseInterval time.Duration + HelmReleaseRetryInterval time.Duration } // +kubebuilder:rbac:groups=cozystack.io,resources=packages,verbs=get;list;watch;create;update;patch;delete @@ -223,15 +224,22 @@ func (r *PackageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct Namespace: "cozy-system", }, Install: &helmv2.Install{ - Timeout: &metav1.Duration{Duration: 10 * 60 * 1000000000}, // 10m - Remediation: &helmv2.InstallRemediation{ - Retries: -1, + Timeout: &metav1.Duration{Duration: 10 * time.Minute}, + // Strategy=RetryOnFailure (with RetryInterval) replaces the previous + // Remediation{Retries:-1} setup. Functionally equivalent ("retry forever + // on failure"), but decouples retry timing from spec.Interval so failed + // installs recover at HelmReleaseRetryInterval (default 30s) without + // also polling healthy releases at the same fast cadence. + Strategy: &helmv2.InstallStrategy{ + Name: string(helmv2.ActionStrategyRetryOnFailure), + RetryInterval: &metav1.Duration{Duration: r.HelmReleaseRetryInterval}, }, }, Upgrade: &helmv2.Upgrade{ - Timeout: &metav1.Duration{Duration: 10 * 60 * 1000000000}, // 10m - Remediation: &helmv2.UpgradeRemediation{ - Retries: -1, + Timeout: &metav1.Duration{Duration: 10 * time.Minute}, + Strategy: &helmv2.UpgradeStrategy{ + Name: string(helmv2.ActionStrategyRetryOnFailure), + RetryInterval: &metav1.Duration{Duration: r.HelmReleaseRetryInterval}, }, CRDs: parseCRDPolicy(component.Install), }, diff --git a/packages/core/installer/templates/cozystack-operator.yaml b/packages/core/installer/templates/cozystack-operator.yaml index 9891f7c9..0a7d867e 100644 --- a/packages/core/installer/templates/cozystack-operator.yaml +++ b/packages/core/installer/templates/cozystack-operator.yaml @@ -61,6 +61,9 @@ spec: {{- if .Values.cozystackOperator.helmReleaseInterval }} - --helmrelease-interval={{ .Values.cozystackOperator.helmReleaseInterval }} {{- end }} + {{- if .Values.cozystackOperator.helmReleaseRetryInterval }} + - --helmrelease-retry-interval={{ .Values.cozystackOperator.helmReleaseRetryInterval }} + {{- end }} - --platform-source-name=cozystack-platform - --platform-source-url={{ .Values.cozystackOperator.platformSourceUrl }} {{- if .Values.cozystackOperator.platformSourceRef }} diff --git a/packages/core/installer/values.yaml b/packages/core/installer/values.yaml index f31400ec..b6c93ecd 100644 --- a/packages/core/installer/values.yaml +++ b/packages/core/installer/values.yaml @@ -7,6 +7,10 @@ cozystackOperator: # When non-empty, overrides the operator's --helmrelease-interval flag # (operator default: 5m). E2E sets this to 30s; production should leave empty. helmReleaseInterval: "" + # When non-empty, overrides the operator's --helmrelease-retry-interval flag + # (operator default: 30s). Controls how fast failed install/upgrade attempts + # are retried. Decoupled from helmReleaseInterval (healthy reconcile cadence). + helmReleaseRetryInterval: "" # Generic variant configuration (only used when cozystackOperator.variant=generic) cozystack: # Kubernetes API server host (IP only, no protocol/port) From cf2698e0cfbad5a371f8aac992cd010284486283 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Tue, 28 Apr 2026 12:44:06 +0500 Subject: [PATCH 18/27] feat(operator): expose Install/Upgrade timeouts and MaxHistory as flags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The HelmRelease generation in package_reconciler hardcoded three values that have caused recurring pain: - Install.Timeout = 10m (line 214) - Upgrade.Timeout = 10m (line 220) - MaxHistory = unset (Helm default 5) The 10m timeout in particular contradicts the per-Application timeout work done in `pkg/config/config.go` and `pkg/registry/apps/application/rest.go` (commit 7b146cbe), which lets individual Applications override the install/upgrade timeout via annotation. The PackageSource side had no equivalent — etcd works around it by hand-rolling its HR YAML in `packages/apps/tenant/templates/etcd.yaml` to set a 30m timeout, harbor has had multiple commits chasing the same problem from the other side. This closes that gap. New operator flags (each maps to a chart value with empty default; operator uses its own default when value is empty): - `--helmrelease-install-timeout` (default 10m) → Spec.Install.Timeout - `--helmrelease-upgrade-timeout` (default 10m) → Spec.Upgrade.Timeout - `--helmrelease-max-history` (default 5) → Spec.MaxHistory Production behaviour unchanged. E2E and edge cases (cert rotation, slow-installing charts) can override per-cluster without modifying chart manifests. Per-component overrides on `ComponentInstall` (mirroring the existing `UpgradeCRDs` pattern) would be a strictly better fix for charts like etcd that need an outlier timeout — left as a deliberate followup since it touches the Package CR API. Signed-off-by: Myasnikov Daniil --- cmd/cozystack-operator/main.go | 40 +++++++++++++++++-- internal/operator/package_reconciler.go | 16 +++++--- .../templates/cozystack-operator.yaml | 9 +++++ packages/core/installer/values.yaml | 12 ++++++ 4 files changed, 67 insertions(+), 10 deletions(-) diff --git a/cmd/cozystack-operator/main.go b/cmd/cozystack-operator/main.go index fcc8b1b1..08f7a860 100644 --- a/cmd/cozystack-operator/main.go +++ b/cmd/cozystack-operator/main.go @@ -85,6 +85,9 @@ func main() { var telemetryInterval string var helmReleaseInterval string var helmReleaseRetryInterval string + var helmReleaseInstallTimeout string + var helmReleaseUpgradeTimeout string + var helmReleaseMaxHistory int var cozyValuesSecretName string var cozyValuesSecretNamespace string var cozyValuesNamespaceSelector string @@ -119,6 +122,18 @@ func main() { "controller waits between failed install/upgrade attempts. Decoupled from --helmrelease-interval "+ "(which is the healthy reconcile cadence) so failures recover fast without polling healthy "+ "releases at the same fast cadence.") + flag.StringVar(&helmReleaseInstallTimeout, "helmrelease-install-timeout", "10m", + "Timeout for the Helm install action of HelmReleases created by the Package reconciler "+ + "(Spec.Install.Timeout). Bounds how long an individual Kubernetes operation (Job/hook/wait) "+ + "may take during install.") + flag.StringVar(&helmReleaseUpgradeTimeout, "helmrelease-upgrade-timeout", "10m", + "Timeout for the Helm upgrade action of HelmReleases created by the Package reconciler "+ + "(Spec.Upgrade.Timeout). Bounds how long an individual Kubernetes operation (Job/hook/wait) "+ + "may take during upgrade.") + flag.IntVar(&helmReleaseMaxHistory, "helmrelease-max-history", 5, + "Number of release revisions Helm keeps for HelmReleases created by the Package reconciler "+ + "(Spec.MaxHistory). 0 means unlimited; 5 matches Helm's default. Lower values reduce "+ + "per-release Secret accumulation in clusters that bounce HRs frequently (e.g. E2E sandboxes).") flag.StringVar(&platformSourceURL, "platform-source-url", "", "Platform source URL (oci:// or https://). If specified, generates OCIRepository or GitRepository resource.") flag.StringVar(&platformSourceName, "platform-source-name", "cozystack-platform", "Name for the generated platform source resource and PackageSource") flag.StringVar(&platformSourceRef, "platform-source-ref", "", "Reference specification as key=value pairs (e.g., 'branch=main' or 'digest=sha256:...,tag=v1.0'). For OCI: digest, semver, semverFilter, tag. For Git: branch, tag, semver, name, commit.") @@ -144,6 +159,20 @@ func main() { setupLog.Error(err, "invalid --helmrelease-retry-interval value", "value", helmReleaseRetryInterval) os.Exit(1) } + hrInstallTimeoutDuration, err := time.ParseDuration(helmReleaseInstallTimeout) + if err != nil { + setupLog.Error(err, "invalid --helmrelease-install-timeout value", "value", helmReleaseInstallTimeout) + os.Exit(1) + } + hrUpgradeTimeoutDuration, err := time.ParseDuration(helmReleaseUpgradeTimeout) + if err != nil { + setupLog.Error(err, "invalid --helmrelease-upgrade-timeout value", "value", helmReleaseUpgradeTimeout) + os.Exit(1) + } + if helmReleaseMaxHistory < 0 { + setupLog.Error(fmt.Errorf("--helmrelease-max-history must be >= 0"), "invalid value", "value", helmReleaseMaxHistory) + os.Exit(1) + } config := ctrl.GetConfigOrDie() @@ -281,10 +310,13 @@ func main() { // Setup Package reconciler if err := (&operator.PackageReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - HelmReleaseInterval: hrIntervalDuration, - HelmReleaseRetryInterval: hrRetryIntervalDuration, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + HelmReleaseInterval: hrIntervalDuration, + HelmReleaseRetryInterval: hrRetryIntervalDuration, + HelmReleaseInstallTimeout: hrInstallTimeoutDuration, + HelmReleaseUpgradeTimeout: hrUpgradeTimeoutDuration, + HelmReleaseMaxHistory: helmReleaseMaxHistory, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Package") os.Exit(1) diff --git a/internal/operator/package_reconciler.go b/internal/operator/package_reconciler.go index e2dac1cf..d13cce33 100644 --- a/internal/operator/package_reconciler.go +++ b/internal/operator/package_reconciler.go @@ -59,9 +59,12 @@ func parseCRDPolicy(install *cozyv1alpha1.ComponentInstall) helmv2.CRDsPolicy { // PackageReconciler reconciles Package resources type PackageReconciler struct { client.Client - Scheme *runtime.Scheme - HelmReleaseInterval time.Duration - HelmReleaseRetryInterval time.Duration + Scheme *runtime.Scheme + HelmReleaseInterval time.Duration + HelmReleaseRetryInterval time.Duration + HelmReleaseInstallTimeout time.Duration + HelmReleaseUpgradeTimeout time.Duration + HelmReleaseMaxHistory int } // +kubebuilder:rbac:groups=cozystack.io,resources=packages,verbs=get;list;watch;create;update;patch;delete @@ -217,14 +220,15 @@ func (r *PackageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct Labels: labels, }, Spec: helmv2.HelmReleaseSpec{ - Interval: metav1.Duration{Duration: r.HelmReleaseInterval}, + Interval: metav1.Duration{Duration: r.HelmReleaseInterval}, + MaxHistory: &r.HelmReleaseMaxHistory, ChartRef: &helmv2.CrossNamespaceSourceReference{ Kind: "ExternalArtifact", Name: artifactName, Namespace: "cozy-system", }, Install: &helmv2.Install{ - Timeout: &metav1.Duration{Duration: 10 * time.Minute}, + Timeout: &metav1.Duration{Duration: r.HelmReleaseInstallTimeout}, // Strategy=RetryOnFailure (with RetryInterval) replaces the previous // Remediation{Retries:-1} setup. Functionally equivalent ("retry forever // on failure"), but decouples retry timing from spec.Interval so failed @@ -236,7 +240,7 @@ func (r *PackageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct }, }, Upgrade: &helmv2.Upgrade{ - Timeout: &metav1.Duration{Duration: 10 * time.Minute}, + Timeout: &metav1.Duration{Duration: r.HelmReleaseUpgradeTimeout}, Strategy: &helmv2.UpgradeStrategy{ Name: string(helmv2.ActionStrategyRetryOnFailure), RetryInterval: &metav1.Duration{Duration: r.HelmReleaseRetryInterval}, diff --git a/packages/core/installer/templates/cozystack-operator.yaml b/packages/core/installer/templates/cozystack-operator.yaml index 0a7d867e..4b48bddd 100644 --- a/packages/core/installer/templates/cozystack-operator.yaml +++ b/packages/core/installer/templates/cozystack-operator.yaml @@ -64,6 +64,15 @@ spec: {{- if .Values.cozystackOperator.helmReleaseRetryInterval }} - --helmrelease-retry-interval={{ .Values.cozystackOperator.helmReleaseRetryInterval }} {{- end }} + {{- if .Values.cozystackOperator.helmReleaseInstallTimeout }} + - --helmrelease-install-timeout={{ .Values.cozystackOperator.helmReleaseInstallTimeout }} + {{- end }} + {{- if .Values.cozystackOperator.helmReleaseUpgradeTimeout }} + - --helmrelease-upgrade-timeout={{ .Values.cozystackOperator.helmReleaseUpgradeTimeout }} + {{- end }} + {{- if .Values.cozystackOperator.helmReleaseMaxHistory }} + - --helmrelease-max-history={{ .Values.cozystackOperator.helmReleaseMaxHistory }} + {{- end }} - --platform-source-name=cozystack-platform - --platform-source-url={{ .Values.cozystackOperator.platformSourceUrl }} {{- if .Values.cozystackOperator.platformSourceRef }} diff --git a/packages/core/installer/values.yaml b/packages/core/installer/values.yaml index b6c93ecd..dcd0a913 100644 --- a/packages/core/installer/values.yaml +++ b/packages/core/installer/values.yaml @@ -11,6 +11,18 @@ cozystackOperator: # (operator default: 30s). Controls how fast failed install/upgrade attempts # are retried. Decoupled from helmReleaseInterval (healthy reconcile cadence). helmReleaseRetryInterval: "" + # When non-empty, overrides the operator's --helmrelease-install-timeout flag + # (operator default: 10m). Bounds individual k8s operations (job/hook/wait) + # during Helm install. Charts with long-running install hooks may need longer. + helmReleaseInstallTimeout: "" + # When non-empty, overrides the operator's --helmrelease-upgrade-timeout flag + # (operator default: 10m). Same semantics as helmReleaseInstallTimeout, for + # the Helm upgrade action. + helmReleaseUpgradeTimeout: "" + # When non-empty, overrides the operator's --helmrelease-max-history flag + # (operator default: 5). Number of release revisions Helm keeps per HR. + # Use 0 for unlimited; lower values reduce Secret accumulation in test clusters. + helmReleaseMaxHistory: "" # Generic variant configuration (only used when cozystackOperator.variant=generic) cozystack: # Kubernetes API server host (IP only, no protocol/port) From 4754f57f4320ce0fc3a7f07c060154da09a4cf6d Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Tue, 28 Apr 2026 13:21:09 +0500 Subject: [PATCH 19/27] fix(installer): schedule cozy-system labeler hook on NotReady-NoSchedule nodes The pre-install Job runs before any CNI is installed, so all nodes are tainted NotReady-NoSchedule and the pod has no pod network. Without adjustment the Job's pod sits Pending until helm's pre-install timeout fires, blocking the entire install. Surfaced in PR #2500 CI run 25040584552 attempt 2: FailedScheduling 0/3 nodes are available: 3 node(s) had untolerated taint {node.kubernetes.io/not-ready: }. Mirror the cozystack-operator deployment's scheduling pattern: - hostNetwork=true so the pod doesn't depend on CNI - Tolerations matching the operator (not-ready / unreachable / cilium.agent-not-ready / cloudprovider.uninitialized) - Variant-aware KUBERNETES_SERVICE_HOST/PORT env so kubectl reaches the apiserver before kube-proxy + CNI are up: talos: localhost:7445 (KubePrism) generic: cozystack.apiServerHost / Port hosted: default in-cluster Verified on the sandbox kind cluster that the rendered Job manifest schedules and the kubectl container starts (the kind cluster itself doesn't run KubePrism so the env-var path can't be end-to-end-tested there; the Talos E2E will exercise it). Signed-off-by: Myasnikov Daniil --- .../templates/cozy-system-labels.yaml | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/packages/core/installer/templates/cozy-system-labels.yaml b/packages/core/installer/templates/cozy-system-labels.yaml index 1ea3e5eb..995e5e3a 100644 --- a/packages/core/installer/templates/cozy-system-labels.yaml +++ b/packages/core/installer/templates/cozy-system-labels.yaml @@ -68,10 +68,39 @@ spec: spec: serviceAccountName: cozy-system-labeler restartPolicy: OnFailure + # The hook runs BEFORE Cilium / kube-ovn / any CNI is installed, so the + # cluster's nodes are NotReady-NoSchedule and have no pod network. Mirror + # the cozystack-operator deployment's scheduling: hostNetwork=true so the + # pod doesn't need CNI, plus the same tolerations the operator uses. + hostNetwork: true + tolerations: + - key: "node.kubernetes.io/not-ready" + operator: "Exists" + - key: "node.kubernetes.io/unreachable" + operator: "Exists" + - key: "node.cilium.io/agent-not-ready" + operator: "Exists" + - key: "node.cloudprovider.kubernetes.io/uninitialized" + operator: "Exists" containers: - name: kubectl image: alpine/k8s:1.32.0 imagePullPolicy: IfNotPresent + {{- if eq .Values.cozystackOperator.variant "talos" }} + env: + # Talos KubePrism endpoint — same as cozystack-operator uses to reach + # the apiserver before CNI is up. + - name: KUBERNETES_SERVICE_HOST + value: "localhost" + - name: KUBERNETES_SERVICE_PORT + value: "7445" + {{- else if eq .Values.cozystackOperator.variant "generic" }} + env: + - name: KUBERNETES_SERVICE_HOST + value: {{ required "cozystack.apiServerHost is required in generic mode" .Values.cozystack.apiServerHost | quote }} + - name: KUBERNETES_SERVICE_PORT + value: {{ .Values.cozystack.apiServerPort | quote }} + {{- end }} command: - kubectl - label From be0a66877c68fa0fb3688ed1ec092c7772f9d3f1 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Tue, 28 Apr 2026 13:46:14 +0500 Subject: [PATCH 20/27] fix(installer): drop labeler Job hook in favour of caller-applied Namespace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pre-install Job approach (added in c0b76b16, b46151a4) hits a fatal chicken-and-egg on Talos clusters that enforce PodSecurity baseline:latest on bare namespaces: pods "cozy-system-labeler-..." is forbidden: violates PodSecurity "baseline:latest": host namespaces (hostNetwork=true) The labeler needs hostNetwork=true (no CNI yet) which requires the namespace to be PSA-privileged, but the labeler's whole job is to add that label. Cannot work from inside the namespace it's labeling. Drop the labeler Job + ServiceAccount + ClusterRole + ClusterRoleBinding. The chart now only removes the chart-defined Namespace and assumes the caller pre-creates cozy-system with the right labels — same pattern as kube-prometheus-stack, argo-cd, cert-manager, and other mainstream charts. Install procedure becomes two-step: kubectl apply -f - < --- .../templates/cozy-system-labels.yaml | 111 ------------------ 1 file changed, 111 deletions(-) delete mode 100644 packages/core/installer/templates/cozy-system-labels.yaml diff --git a/packages/core/installer/templates/cozy-system-labels.yaml b/packages/core/installer/templates/cozy-system-labels.yaml deleted file mode 100644 index 995e5e3a..00000000 --- a/packages/core/installer/templates/cozy-system-labels.yaml +++ /dev/null @@ -1,111 +0,0 @@ -{{/* - Pre-install/upgrade hook that labels the cozy-system namespace with the - PodSecurity and cozystack identity labels. The chart no longer ships a - Namespace resource (helm v3 cannot adopt a namespace pre-created by - --create-namespace because it lacks helm meta annotations; without - --create-namespace, helm fails to write its release-secret because the - namespace does not exist yet — chicken-and-egg). - Install commands now use --create-namespace to bootstrap the namespace, - and this hook patches the labels the operator depends on (notably - pod-security.kubernetes.io/enforce=privileged for the host-network - operator pod). -*/}} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: cozy-system-labeler - namespace: cozy-system - annotations: - helm.sh/hook: pre-install,pre-upgrade - helm.sh/hook-weight: "-200" - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: cozy-system-labeler - annotations: - helm.sh/hook: pre-install,pre-upgrade - helm.sh/hook-weight: "-180" - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded -rules: -- apiGroups: [""] - resources: ["namespaces"] - resourceNames: ["cozy-system"] - verbs: ["get", "patch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: cozy-system-labeler - annotations: - helm.sh/hook: pre-install,pre-upgrade - helm.sh/hook-weight: "-160" - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded -subjects: -- kind: ServiceAccount - name: cozy-system-labeler - namespace: cozy-system -roleRef: - kind: ClusterRole - name: cozy-system-labeler - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: cozy-system-labeler - namespace: cozy-system - annotations: - helm.sh/hook: pre-install,pre-upgrade - helm.sh/hook-weight: "-100" - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded -spec: - backoffLimit: 3 - ttlSecondsAfterFinished: 60 - template: - spec: - serviceAccountName: cozy-system-labeler - restartPolicy: OnFailure - # The hook runs BEFORE Cilium / kube-ovn / any CNI is installed, so the - # cluster's nodes are NotReady-NoSchedule and have no pod network. Mirror - # the cozystack-operator deployment's scheduling: hostNetwork=true so the - # pod doesn't need CNI, plus the same tolerations the operator uses. - hostNetwork: true - tolerations: - - key: "node.kubernetes.io/not-ready" - operator: "Exists" - - key: "node.kubernetes.io/unreachable" - operator: "Exists" - - key: "node.cilium.io/agent-not-ready" - operator: "Exists" - - key: "node.cloudprovider.kubernetes.io/uninitialized" - operator: "Exists" - containers: - - name: kubectl - image: alpine/k8s:1.32.0 - imagePullPolicy: IfNotPresent - {{- if eq .Values.cozystackOperator.variant "talos" }} - env: - # Talos KubePrism endpoint — same as cozystack-operator uses to reach - # the apiserver before CNI is up. - - name: KUBERNETES_SERVICE_HOST - value: "localhost" - - name: KUBERNETES_SERVICE_PORT - value: "7445" - {{- else if eq .Values.cozystackOperator.variant "generic" }} - env: - - name: KUBERNETES_SERVICE_HOST - value: {{ required "cozystack.apiServerHost is required in generic mode" .Values.cozystack.apiServerHost | quote }} - - name: KUBERNETES_SERVICE_PORT - value: {{ .Values.cozystack.apiServerPort | quote }} - {{- end }} - command: - - kubectl - - label - - --overwrite - - namespace - - cozy-system - - cozystack.io/system=true - - pod-security.kubernetes.io/enforce=privileged From ddc429c8672ccc3ce6c6bced7feeb3bfbf9207ff Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Tue, 28 Apr 2026 13:47:02 +0500 Subject: [PATCH 21/27] test(e2e): pre-apply cozy-system namespace before helm install The chart fix (3a87485c, be0a6687) removes the chart-defined Namespace without replacing it. On the Talos E2E cluster the apiserver enforces PodSecurity baseline-by-default on bare namespaces, so the cozystack- operator pod (hostNetwork=true) is rejected if cozy-system is created without enforce=privileged. Apply the namespace from the bats with the labels the operator depends on, then run helm install without --create-namespace. Mirrors the documented production install procedure for the new chart shape. Signed-off-by: Myasnikov Daniil --- hack/e2e-install-cozystack.bats | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/hack/e2e-install-cozystack.bats b/hack/e2e-install-cozystack.bats index ebc4a5b4..1593c437 100644 --- a/hack/e2e-install-cozystack.bats +++ b/hack/e2e-install-cozystack.bats @@ -8,13 +8,30 @@ } @test "Install Cozystack" { + # Pre-create the cozy-system namespace with the labels the operator pod needs. + # The chart no longer ships a Namespace resource (helm v3's --create-namespace + # collides with chart-defined Namespace). On Talos clusters that enforce PSA + # baseline-by-default, the cozystack-operator pod (hostNetwork=true) is + # rejected unless the namespace is labelled enforce=privileged BEFORE the + # pod is scheduled. A pre-install hook can't bootstrap that label because + # the hook itself runs in the same namespace and faces the same PSA gate. + # So apply the namespace from the caller (this bats / production install docs) + # — same pattern as kube-prometheus-stack, argo-cd, cert-manager. + kubectl apply -f - <<'EOF' +apiVersion: v1 +kind: Namespace +metadata: + name: cozy-system + labels: + cozystack.io/system: "true" + cozystack.io/deletion-protected: "true" + pod-security.kubernetes.io/enforce: privileged +EOF + # Install cozy-installer chart (operator installs CRDs on startup via --install-crds). - # The chart's pre-install hook patches PSA + cozystack labels onto cozy-system - # after --create-namespace bootstraps it. helm upgrade installer packages/core/installer \ --install \ --namespace cozy-system \ - --create-namespace \ --set cozystackOperator.helmReleaseInterval=30s \ --wait \ --timeout 2m From adc430e27c3302b6a1a6833f77209da3f944f240 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Tue, 28 Apr 2026 15:52:39 +0500 Subject: [PATCH 22/27] test(e2e-apps): add existence backstops to all kubectl wait calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `kubectl wait ` errors immediately with NotFound if the resource doesn't exist yet — even with --for=condition or --for=jsonpath. The redis test in CI run 25043350705 failed in 1 second for exactly this reason: the redis-failover operator hadn't created the PVC by the time the test waited for it. Previously the 3x retry on `Run E2E tests` masked this race; with retry dropped, every such call is a flake risk. Add a small `until kubectl get` existence backstop before each kubectl wait, matching the pattern already established for HRs in commit 66888c91. 33 backstops across 12 files: bucket.bats — bucketclaims, bucketaccesses x2 clickhouse.bats — statefulset 0-0 (0-1 already covered) harbor.bats — deploy x3, bucketclaims, bucketaccesses kafka.bats — kafkas mariadb.bats — statefulset, deployment mongodb.bats — statefulset openbao.bats — sts, pvc postgres.bats — job.batch qdrant.bats — sts, pvc redis.bats — pvc, deploy, sts (the trigger) vminstance.bats — dv, pvc, vm (with 120s for KubeVirt latency) e2e-install-cozystack.bats — apiservices, sts/etcd, vmalert, vmalertmanager, vlclusters, vmcluster, clusters.postgresql.cnpg.io, deploy/grafana-deployment, namespace Same pattern, same 60s default timeout for the existence wait (120s for nested-virt resources). Once the resource exists, the original wait timeout takes over. Run-kubernetes.sh has the same race shape on several waits (nfs, kamaji, machinedeployment, etc.) — out of scope here; flagged for follow-up. Signed-off-by: Myasnikov Daniil --- hack/e2e-apps/bucket.bats | 3 +++ hack/e2e-apps/clickhouse.bats | 1 + hack/e2e-apps/harbor.bats | 5 +++++ hack/e2e-apps/kafka.bats | 1 + hack/e2e-apps/mariadb.bats | 2 ++ hack/e2e-apps/mongodb.bats | 1 + hack/e2e-apps/openbao.bats | 2 ++ hack/e2e-apps/postgres.bats | 1 + hack/e2e-apps/qdrant.bats | 2 ++ hack/e2e-apps/redis.bats | 3 +++ hack/e2e-apps/vminstance.bats | 3 +++ hack/e2e-install-cozystack.bats | 9 +++++++++ 12 files changed, 33 insertions(+) diff --git a/hack/e2e-apps/bucket.bats b/hack/e2e-apps/bucket.bats index 31e7efb0..a4a349d7 100644 --- a/hack/e2e-apps/bucket.bats +++ b/hack/e2e-apps/bucket.bats @@ -20,8 +20,11 @@ EOF # Wait for the bucket to be ready kubectl -n tenant-test wait hr bucket-${name} --timeout=100s --for=condition=ready + timeout 60 sh -ec "until kubectl -n tenant-test get bucketclaims.objectstorage.k8s.io bucket-${name} >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait bucketclaims.objectstorage.k8s.io bucket-${name} --timeout=300s --for=jsonpath='{.status.bucketReady}' + timeout 60 sh -ec "until kubectl -n tenant-test get bucketaccesses.objectstorage.k8s.io bucket-${name}-admin >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait bucketaccesses.objectstorage.k8s.io bucket-${name}-admin --timeout=300s --for=jsonpath='{.status.accessGranted}' + timeout 60 sh -ec "until kubectl -n tenant-test get bucketaccesses.objectstorage.k8s.io bucket-${name}-viewer >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait bucketaccesses.objectstorage.k8s.io bucket-${name}-viewer --timeout=300s --for=jsonpath='{.status.accessGranted}' # Get admin (readwrite) credentials diff --git a/hack/e2e-apps/clickhouse.bats b/hack/e2e-apps/clickhouse.bats index 79bc4671..733da0ff 100644 --- a/hack/e2e-apps/clickhouse.bats +++ b/hack/e2e-apps/clickhouse.bats @@ -40,6 +40,7 @@ EOF timeout 60 sh -ec "until kubectl -n tenant-test get hr clickhouse-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait hr clickhouse-$name --timeout=20s --for=condition=ready timeout 180 sh -ec "until kubectl -n tenant-test get svc chendpoint-clickhouse-$name -o jsonpath='{.spec.ports[*].port}' | grep -q '8123 9000'; do sleep 10; done" + timeout 60 sh -ec "until kubectl -n tenant-test get statefulset.apps/chi-clickhouse-$name-clickhouse-0-0 >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait statefulset.apps/chi-clickhouse-$name-clickhouse-0-0 --timeout=120s --for=jsonpath='{.status.replicas}'=1 timeout 80 sh -ec "until kubectl -n tenant-test get endpoints chi-clickhouse-$name-clickhouse-0-0 -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done" timeout 100 sh -ec "until kubectl -n tenant-test get svc chi-clickhouse-$name-clickhouse-0-0 -o jsonpath='{.spec.ports[*].port}' | grep -q '9000 8123 9009'; do sleep 10; done" diff --git a/hack/e2e-apps/harbor.bats b/hack/e2e-apps/harbor.bats index ae825018..290fef42 100644 --- a/hack/e2e-apps/harbor.bats +++ b/hack/e2e-apps/harbor.bats @@ -44,8 +44,10 @@ EOF kubectl -n tenant-test wait hr $release --timeout=60s --for=condition=ready # Wait for COSI to provision bucket + timeout 60 sh -ec "until kubectl -n tenant-test get bucketclaims.objectstorage.k8s.io $release-registry >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait bucketclaims.objectstorage.k8s.io $release-registry \ --timeout=300s --for=jsonpath='{.status.bucketReady}'=true + timeout 60 sh -ec "until kubectl -n tenant-test get bucketaccesses.objectstorage.k8s.io $release-registry >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait bucketaccesses.objectstorage.k8s.io $release-registry \ --timeout=60s --for=jsonpath='{.status.accessGranted}'=true @@ -66,8 +68,11 @@ EOF kubectl -n tenant-test get secret $release-registry-bucket -o jsonpath='{.data.BucketInfo}' 2>&1 | base64 -d 2>&1 || true false } + timeout 60 sh -ec "until kubectl -n tenant-test get deploy $release-core >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait deploy $release-core --timeout=120s --for=condition=available + timeout 60 sh -ec "until kubectl -n tenant-test get deploy $release-registry >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait deploy $release-registry --timeout=120s --for=condition=available + timeout 60 sh -ec "until kubectl -n tenant-test get deploy $release-portal >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait deploy $release-portal --timeout=120s --for=condition=available kubectl -n tenant-test get secret $release-credentials -o jsonpath='{.data.admin-password}' | base64 --decode | grep -q '.' kubectl -n tenant-test get secret $release-credentials -o jsonpath='{.data.url}' | base64 --decode | grep -q 'https://' diff --git a/hack/e2e-apps/kafka.bats b/hack/e2e-apps/kafka.bats index 7b1e2638..c1f0adf1 100644 --- a/hack/e2e-apps/kafka.bats +++ b/hack/e2e-apps/kafka.bats @@ -43,6 +43,7 @@ EOF # kicks in (kubectl wait errors immediately if the object does not exist yet). timeout 60 sh -ec "until kubectl -n tenant-test get hr kafka-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait hr kafka-$name --timeout=30s --for=condition=ready + timeout 60 sh -ec "until kubectl -n tenant-test get kafkas test >/dev/null 2>&1; do sleep 2; done" kubectl wait kafkas -n tenant-test test --timeout=300s --for=condition=ready timeout 60 sh -ec "until kubectl -n tenant-test get pvc data-kafka-$name-zookeeper-0; do sleep 10; done" kubectl -n tenant-test wait pvc data-kafka-$name-zookeeper-0 --timeout=50s --for=jsonpath='{.status.phase}'=Bound diff --git a/hack/e2e-apps/mariadb.bats b/hack/e2e-apps/mariadb.bats index 0b7b27c2..35559216 100644 --- a/hack/e2e-apps/mariadb.bats +++ b/hack/e2e-apps/mariadb.bats @@ -41,9 +41,11 @@ EOF kubectl -n tenant-test wait hr mariadb-$name --timeout=30s --for=condition=ready timeout 80 sh -ec "until kubectl -n tenant-test get svc mariadb-$name -o jsonpath='{.spec.ports[0].port}' | grep -q '3306'; do sleep 10; done" timeout 80 sh -ec "until kubectl -n tenant-test get endpoints mariadb-$name -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done" + timeout 60 sh -ec "until kubectl -n tenant-test get statefulset.apps/mariadb-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait statefulset.apps/mariadb-$name --timeout=110s --for=jsonpath='{.status.replicas}'=2 timeout 80 sh -ec "until kubectl -n tenant-test get svc mariadb-$name-metrics -o jsonpath='{.spec.ports[0].port}' | grep -q '9104'; do sleep 10; done" timeout 40 sh -ec "until kubectl -n tenant-test get endpoints mariadb-$name-metrics -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done" + timeout 60 sh -ec "until kubectl -n tenant-test get deployment.apps/mariadb-$name-metrics >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait deployment.apps/mariadb-$name-metrics --timeout=90s --for=jsonpath='{.status.replicas}'=1 kubectl -n tenant-test delete mariadbs.apps.cozystack.io $name } diff --git a/hack/e2e-apps/mongodb.bats b/hack/e2e-apps/mongodb.bats index 0e5f4042..bc83f7ab 100644 --- a/hack/e2e-apps/mongodb.bats +++ b/hack/e2e-apps/mongodb.bats @@ -36,6 +36,7 @@ EOF # Wait for endpoints timeout 180 sh -ec "until kubectl -n tenant-test get endpoints mongodb-$name-rs0 -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done" # Wait for StatefulSet replicas + timeout 60 sh -ec "until kubectl -n tenant-test get statefulset.apps/mongodb-$name-rs0 >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait statefulset.apps/mongodb-$name-rs0 --timeout=300s --for=jsonpath='{.status.replicas}'=1 # Cleanup kubectl -n tenant-test delete mongodbs.apps.cozystack.io $name diff --git a/hack/e2e-apps/openbao.bats b/hack/e2e-apps/openbao.bats index fc85d891..168440ad 100644 --- a/hack/e2e-apps/openbao.bats +++ b/hack/e2e-apps/openbao.bats @@ -55,7 +55,9 @@ EOF kubectl -n tenant-test exec openbao-$name-0 -- bao operator unseal "$unseal_key" # Now wait for pod to become ready (readiness probe checks seal status) + timeout 60 sh -ec "until kubectl -n tenant-test get sts openbao-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait sts openbao-$name --timeout=90s --for=jsonpath='{.status.readyReplicas}'=1 + timeout 60 sh -ec "until kubectl -n tenant-test get pvc data-openbao-$name-0 >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait pvc data-openbao-$name-0 --timeout=50s --for=jsonpath='{.status.phase}'=Bound kubectl -n tenant-test delete openbao.apps.cozystack.io $name kubectl -n tenant-test delete pvc data-openbao-$name-0 --ignore-not-found diff --git a/hack/e2e-apps/postgres.bats b/hack/e2e-apps/postgres.bats index 76fc1638..3ec79409 100644 --- a/hack/e2e-apps/postgres.bats +++ b/hack/e2e-apps/postgres.bats @@ -44,6 +44,7 @@ EOF # kicks in (kubectl wait errors immediately if the object does not exist yet). timeout 60 sh -ec "until kubectl -n tenant-test get hr postgres-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait hr postgres-$name --timeout=100s --for=condition=ready + timeout 60 sh -ec "until kubectl -n tenant-test get job.batch postgres-$name-init-job >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait job.batch postgres-$name-init-job --timeout=50s --for=condition=Complete timeout 40 sh -ec "until kubectl -n tenant-test get svc postgres-$name-r -o jsonpath='{.spec.ports[0].port}' | grep -q '5432'; do sleep 10; done" timeout 40 sh -ec "until kubectl -n tenant-test get svc postgres-$name-ro -o jsonpath='{.spec.ports[0].port}' | grep -q '5432'; do sleep 10; done" diff --git a/hack/e2e-apps/qdrant.bats b/hack/e2e-apps/qdrant.bats index 2a6e05b7..8a889e43 100755 --- a/hack/e2e-apps/qdrant.bats +++ b/hack/e2e-apps/qdrant.bats @@ -22,7 +22,9 @@ EOF timeout 60 sh -ec "until kubectl -n tenant-test get hr qdrant-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait hr qdrant-$name --timeout=60s --for=condition=ready kubectl -n tenant-test wait hr qdrant-$name-system --timeout=120s --for=condition=ready + timeout 60 sh -ec "until kubectl -n tenant-test get sts qdrant-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait sts qdrant-$name --timeout=90s --for=jsonpath='{.status.readyReplicas}'=1 + timeout 60 sh -ec "until kubectl -n tenant-test get pvc qdrant-storage-qdrant-$name-0 >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait pvc qdrant-storage-qdrant-$name-0 --timeout=50s --for=jsonpath='{.status.phase}'=Bound kubectl -n tenant-test delete qdrant.apps.cozystack.io $name } diff --git a/hack/e2e-apps/redis.bats b/hack/e2e-apps/redis.bats index b2cef269..eb148701 100644 --- a/hack/e2e-apps/redis.bats +++ b/hack/e2e-apps/redis.bats @@ -22,8 +22,11 @@ EOF # kicks in (kubectl wait errors immediately if the object does not exist yet). timeout 60 sh -ec "until kubectl -n tenant-test get hr redis-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait hr redis-$name --timeout=20s --for=condition=ready + timeout 60 sh -ec "until kubectl -n tenant-test get pvc redisfailover-persistent-data-rfr-redis-$name-0 >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait pvc redisfailover-persistent-data-rfr-redis-$name-0 --timeout=50s --for=jsonpath='{.status.phase}'=Bound + timeout 60 sh -ec "until kubectl -n tenant-test get deploy rfs-redis-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait deploy rfs-redis-$name --timeout=90s --for=condition=available + timeout 60 sh -ec "until kubectl -n tenant-test get sts rfr-redis-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait sts rfr-redis-$name --timeout=90s --for=jsonpath='{.status.replicas}'=2 kubectl -n tenant-test delete redis.apps.cozystack.io $name } diff --git a/hack/e2e-apps/vminstance.bats b/hack/e2e-apps/vminstance.bats index f42225e8..2ff60271 100644 --- a/hack/e2e-apps/vminstance.bats +++ b/hack/e2e-apps/vminstance.bats @@ -29,7 +29,9 @@ EOF # kicks in (kubectl wait errors immediately if the object does not exist yet). timeout 60 sh -ec "until kubectl -n tenant-test get hr vm-disk-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait hr vm-disk-$name --timeout=5s --for=condition=ready + timeout 120 sh -ec "until kubectl -n tenant-test get dv vm-disk-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait dv vm-disk-$name --timeout=250s --for=condition=ready + timeout 120 sh -ec "until kubectl -n tenant-test get pvc vm-disk-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait pvc vm-disk-$name --timeout=200s --for=jsonpath='{.status.phase}'=Bound } @@ -79,6 +81,7 @@ EOF timeout 120 sh -ec "until kubectl -n tenant-test get vmi vm-instance-$name -o jsonpath='{.status.interfaces[0].ipAddress}' | grep -q '[0-9]'; do sleep 2; done" kubectl -n tenant-test wait hr vm-instance-$name --timeout=5s --for=condition=ready # VM ready follows IP assignment closely; 60s gives buffer for the qemu-guest-agent. + timeout 120 sh -ec "until kubectl -n tenant-test get vm vm-instance-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait vm vm-instance-$name --timeout=60s --for=condition=ready kubectl -n tenant-test delete vminstances.apps.cozystack.io $name --timeout=3m kubectl -n tenant-test delete vmdisks.apps.cozystack.io $diskName --timeout=3m diff --git a/hack/e2e-install-cozystack.bats b/hack/e2e-install-cozystack.bats index 1593c437..aa737e32 100644 --- a/hack/e2e-install-cozystack.bats +++ b/hack/e2e-install-cozystack.bats @@ -110,6 +110,7 @@ EOF } @test "Check Cozystack API service" { + timeout 60 sh -ec 'until kubectl get apiservices/v1alpha1.apps.cozystack.io apiservices/v1alpha1.core.cozystack.io >/dev/null 2>&1; do sleep 2; done' kubectl wait --for=condition=Available apiservices/v1alpha1.apps.cozystack.io apiservices/v1alpha1.core.cozystack.io --timeout=2m } @@ -141,15 +142,22 @@ EOF kubectl wait deploy/root-ingress-controller -n tenant-root --timeout=5m --for=condition=available # etcd statefulset + timeout 60 sh -ec 'until kubectl get sts/etcd -n tenant-root >/dev/null 2>&1; do sleep 2; done' kubectl wait sts/etcd -n tenant-root --for=jsonpath='{.status.readyReplicas}'=3 --timeout=5m # VictoriaMetrics components + timeout 60 sh -ec 'until kubectl get vmalert/vmalert-shortterm -n tenant-root >/dev/null 2>&1; do sleep 2; done' + timeout 60 sh -ec 'until kubectl get vmalertmanager/alertmanager -n tenant-root >/dev/null 2>&1; do sleep 2; done' kubectl wait vmalert/vmalert-shortterm vmalertmanager/alertmanager -n tenant-root --for=jsonpath='{.status.updateStatus}'=operational --timeout=15m + timeout 60 sh -ec 'until kubectl get vlclusters/generic -n tenant-root >/dev/null 2>&1; do sleep 2; done' kubectl wait vlclusters/generic -n tenant-root --for=jsonpath='{.status.updateStatus}'=operational --timeout=5m + timeout 60 sh -ec 'until kubectl get vmcluster/shortterm vmcluster/longterm -n tenant-root >/dev/null 2>&1; do sleep 2; done' kubectl wait vmcluster/shortterm vmcluster/longterm -n tenant-root --for=jsonpath='{.status.updateStatus}'=operational --timeout=5m # Grafana + timeout 60 sh -ec 'until kubectl get clusters.postgresql.cnpg.io/grafana-db -n tenant-root >/dev/null 2>&1; do sleep 2; done' kubectl wait clusters.postgresql.cnpg.io/grafana-db -n tenant-root --for=condition=ready --timeout=5m + timeout 60 sh -ec 'until kubectl get deploy/grafana-deployment -n tenant-root >/dev/null 2>&1; do sleep 2; done' kubectl wait deploy/grafana-deployment -n tenant-root --for=condition=available --timeout=5m # Verify Grafana via ingress @@ -240,6 +248,7 @@ spec: seaweedfs: false EOF kubectl wait hr/tenant-test -n tenant-root --timeout=1m --for=condition=ready + timeout 60 sh -ec 'until kubectl get namespace tenant-test >/dev/null 2>&1; do sleep 2; done' kubectl wait namespace tenant-test --timeout=20s --for=jsonpath='{.status.phase}'=Active # Wait for ResourceQuota to appear and assert values timeout 60 sh -ec 'until [ "$(kubectl get quota -n tenant-test --no-headers 2>/dev/null | wc -l)" -ge 1 ]; do sleep 1; done' From a21c18f9bc7c11dc03b462c5043cf7c89b20a95f Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 29 Apr 2026 11:37:10 +0500 Subject: [PATCH 23/27] fix(harbor): drive bucket-secret.yaml from values, gate HR on BucketInfo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs caused the harbor E2E to fail (CI #25049445125): 1. cozy-harbor's bucket-secret.yaml called `index $existingSecret.data "BucketInfo"` unconditionally. During first-time install the COSI BucketAccess controller may create the credentials Secret as a placeholder before populating it, so `.data` is nil and the chart render crashes with `index of untyped nil`. 2. The downstream `-system` HelmRelease starts reconciling immediately, in parallel with bucket provisioning, hitting bug #1 on every retry within the test's 5-minute window. The previous shape relied on `lookup` to read the Secret at render time. helm-controller's upgrade trigger is digest-based over composed values + chart artifact, so a `lookup` returning new data on a later reconcile is not enough on its own to force an upgrade — the rendered output may diverge but the digest does not. Switch to a values-driven shape: - `bucket-secret.yaml` now reads `.Values.bucket.bucketInfo` (a JSON string) and uses `dig` for safe access; if the value is empty or the expected nested fields are missing, no `*-registry-s3` Secret is rendered. - The `-system` HR sources `BucketInfo` via `valuesFrom` with `targetPath: bucket.bucketInfo`. With the default `optional: false`, helm-controller will refuse to compose values until the key exists, which both gates initial reconciliation on the BucketAccess Secret being populated and forces a config-digest change (and thus a helm upgrade) when its contents change. - `bucket.secretName` is removed from the system chart's values and from the apps chart's `values:` block; the only consumer (the `lookup` call) is gone. Verified four scenarios via `helm template` against the system chart: unset bucketInfo, empty string, empty JSON object `{}`, and a fully-populated BucketInfo — only the last renders the registry-s3 Secret; the others render nothing without erroring. Signed-off-by: Myasnikov Daniil --- packages/apps/harbor/templates/harbor.yaml | 10 ++++++++-- .../system/harbor/templates/bucket-secret.yaml | 16 +++++++++------- packages/system/harbor/values.yaml | 2 +- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/packages/apps/harbor/templates/harbor.yaml b/packages/apps/harbor/templates/harbor.yaml index bf780967..4ac3a075 100644 --- a/packages/apps/harbor/templates/harbor.yaml +++ b/packages/apps/harbor/templates/harbor.yaml @@ -65,9 +65,15 @@ spec: name: {{ .Release.Name }}-credentials valuesKey: redis-password targetPath: harbor.redis.external.password + # BucketInfo is populated by the COSI BucketAccess controller. Sourcing it via + # valuesFrom both gates reconciliation on the Secret being ready and forces a + # config-digest change (and thus a helm upgrade) when its contents change — + # Flux HR dependsOn cannot reference COSI resources directly. + - kind: Secret + name: {{ .Release.Name }}-registry-bucket + valuesKey: BucketInfo + targetPath: bucket.bucketInfo values: - bucket: - secretName: {{ .Release.Name }}-registry-bucket db: replicas: {{ .Values.database.replicas }} size: {{ .Values.database.size }} diff --git a/packages/system/harbor/templates/bucket-secret.yaml b/packages/system/harbor/templates/bucket-secret.yaml index 241fe638..047bb760 100644 --- a/packages/system/harbor/templates/bucket-secret.yaml +++ b/packages/system/harbor/templates/bucket-secret.yaml @@ -1,10 +1,11 @@ -{{- if .Values.bucket.secretName }} -{{- $existingSecret := lookup "v1" "Secret" .Release.Namespace .Values.bucket.secretName }} -{{- $bucketInfo := fromJson (b64dec (index $existingSecret.data "BucketInfo")) }} -{{- $accessKeyID := index $bucketInfo.spec.secretS3 "accessKeyID" }} -{{- $accessSecretKey := index $bucketInfo.spec.secretS3 "accessSecretKey" }} -{{- $endpoint := index $bucketInfo.spec.secretS3 "endpoint" }} -{{- $bucketName := $bucketInfo.spec.bucketName }} +{{- if .Values.bucket.bucketInfo }} +{{- $bucketInfo := fromJson .Values.bucket.bucketInfo }} +{{- $secretS3 := dig "spec" "secretS3" dict $bucketInfo }} +{{- if $secretS3 }} +{{- $accessKeyID := index $secretS3 "accessKeyID" }} +{{- $accessSecretKey := index $secretS3 "accessSecretKey" }} +{{- $endpoint := index $secretS3 "endpoint" }} +{{- $bucketName := dig "spec" "bucketName" "" $bucketInfo }} --- apiVersion: v1 kind: Secret @@ -18,3 +19,4 @@ stringData: REGISTRY_STORAGE_S3_BUCKET: {{ $bucketName | quote }} REGISTRY_STORAGE_S3_REGION: "us-east-1" {{- end }} +{{- end }} diff --git a/packages/system/harbor/values.yaml b/packages/system/harbor/values.yaml index faac7068..aeb2edac 100644 --- a/packages/system/harbor/values.yaml +++ b/packages/system/harbor/values.yaml @@ -1,6 +1,6 @@ harbor: {} bucket: - secretName: "" + bucketInfo: "" db: replicas: 2 size: 5Gi From 5e88a1410c169f6a3ab4480e8ec405aee329eb88 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 29 Apr 2026 10:50:16 +0500 Subject: [PATCH 24/27] fix(objectstorage-controller): retry on Bucket update conflict during BucketAccess reconcile Upstream COSI v0.2.2's BucketAccess reconciler does a Get->mutate->Update on the parent Bucket and surfaces "Operation cannot be fulfilled ... the object has been modified" as a FailedGrantAccess event when it races against the Bucket reconciler in the same controller process. Wrap the mutation in retry.RetryOnConflict so the reconcile loop refreshes and retries instead of leaking the conflict to users. Carried as 91-bucketaccess-conflict-retry.diff until upstreamed (cf. 89-reconciliation.diff and 90-bucket-name.diff, both dropped in c29d501b once merged upstream). Signed-off-by: Myasnikov Daniil --- .../images/objectstorage/Dockerfile | 5 ++- .../91-bucketaccess-conflict-retry.diff | 39 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 packages/system/objectstorage-controller/images/objectstorage/patches/91-bucketaccess-conflict-retry.diff diff --git a/packages/system/objectstorage-controller/images/objectstorage/Dockerfile b/packages/system/objectstorage-controller/images/objectstorage/Dockerfile index 260446c3..d71e5566 100644 --- a/packages/system/objectstorage-controller/images/objectstorage/Dockerfile +++ b/packages/system/objectstorage-controller/images/objectstorage/Dockerfile @@ -2,12 +2,15 @@ FROM alpine AS source ARG COMMIT_REF=v0.2.2 -RUN apk add --no-cache curl tar +RUN apk add --no-cache curl tar git WORKDIR /src RUN curl -sSL https://github.com/kubernetes-sigs/container-object-storage-interface/archive/${COMMIT_REF}.tar.gz \ | tar -xz --strip-components=1 +COPY patches /patches +RUN git apply /patches/*.diff + FROM --platform=$BUILDPLATFORM docker.io/golang:1.24 AS builder ARG TARGETOS ARG TARGETARCH diff --git a/packages/system/objectstorage-controller/images/objectstorage/patches/91-bucketaccess-conflict-retry.diff b/packages/system/objectstorage-controller/images/objectstorage/patches/91-bucketaccess-conflict-retry.diff new file mode 100644 index 00000000..e4b06c42 --- /dev/null +++ b/packages/system/objectstorage-controller/images/objectstorage/patches/91-bucketaccess-conflict-retry.diff @@ -0,0 +1,39 @@ +diff --git a/sidecar/pkg/bucketaccess/bucketaccess_controller.go b/sidecar/pkg/bucketaccess/bucketaccess_controller.go +index a4db5c4..b351f9e 100644 +--- a/sidecar/pkg/bucketaccess/bucketaccess_controller.go ++++ b/sidecar/pkg/bucketaccess/bucketaccess_controller.go +@@ -31,6 +31,7 @@ import ( + kube "k8s.io/client-go/kubernetes" + kubecorev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/tools/record" ++ "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" + cosiapi "sigs.k8s.io/container-object-storage-interface/client/apis" + "sigs.k8s.io/container-object-storage-interface/client/apis/objectstorage/consts" +@@ -273,11 +274,22 @@ func (bal *BucketAccessListener) Add(ctx context.Context, inputBucketAccess *v1a + } + } + +- if controllerutil.AddFinalizer(bucket, consts.BABucketFinalizer) { +- _, err = bal.buckets().Update(ctx, bucket, metav1.UpdateOptions{}) +- if err != nil { +- return bal.recordError(inputBucketAccess, v1.EventTypeWarning, v1alpha1.FailedGrantAccess, err) ++ // Re-fetch and update inside RetryOnConflict to handle the case where the Bucket ++ // reconciler in the same controller process mutates the Bucket between our Get ++ // and Update, which surfaces as "the object has been modified" on the parent ++ // Bucket and emits FailedGrantAccess on the BucketAccess. ++ if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { ++ latest, getErr := bal.buckets().Get(ctx, bucketClaim.Status.BucketName, metav1.GetOptions{}) ++ if getErr != nil { ++ return getErr ++ } ++ if !controllerutil.AddFinalizer(latest, consts.BABucketFinalizer) { ++ return nil + } ++ _, updateErr := bal.buckets().Update(ctx, latest, metav1.UpdateOptions{}) ++ return updateErr ++ }); err != nil { ++ return bal.recordError(inputBucketAccess, v1.EventTypeWarning, v1alpha1.FailedGrantAccess, err) + } + + if controllerutil.AddFinalizer(bucketAccess, consts.BAFinalizer) { From 432ff777ba234249169b4e6006cf576fa147c130 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 29 Apr 2026 14:14:55 +0500 Subject: [PATCH 25/27] e2e: pre-pull timing-sensitive platform images before install Some workloads (OVN raft, LINSTOR controller) fail when replicas start at different times due to image-pull stagger across nodes. Add a DaemonSet-based pre-pull step that runs before helm install, ensuring all nodes have the images cached so every replica starts within milliseconds of each other. Covers kube-ovn (confirmed OVN raft race), piraeus-server, and linstor-csi. Comments in hack/e2e-prepull-images.sh point to the source chart values so versions stay in sync. Signed-off-by: Myasnikov Daniil --- hack/e2e-install-cozystack.bats | 7 +++ hack/e2e-prepull-images.sh | 108 ++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+) create mode 100755 hack/e2e-prepull-images.sh diff --git a/hack/e2e-install-cozystack.bats b/hack/e2e-install-cozystack.bats index aa737e32..b3143d8e 100644 --- a/hack/e2e-install-cozystack.bats +++ b/hack/e2e-install-cozystack.bats @@ -7,6 +7,13 @@ fi } +@test "Pre-pull platform images" { + # Pull timing-sensitive images to all nodes before Cozystack installation. + # Cluster-member workloads (OVN raft, LINSTOR) fail if replicas start at + # different times due to image-pull stagger. See hack/e2e-prepull-images.sh. + hack/e2e-prepull-images.sh +} + @test "Install Cozystack" { # Pre-create the cozy-system namespace with the labels the operator pod needs. # The chart no longer ships a Namespace resource (helm v3's --create-namespace diff --git a/hack/e2e-prepull-images.sh b/hack/e2e-prepull-images.sh new file mode 100755 index 00000000..209d04dd --- /dev/null +++ b/hack/e2e-prepull-images.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# Pre-pull timing-sensitive platform images to all cluster nodes. +# +# Some workloads (OVN raft, LINSTOR controller) are sensitive to peer-readiness +# at startup: if nodes pull the image at different speeds, one replica boots +# before its peers are reachable, exhausts its raft connection retries, and the +# HelmRelease install times out. Pre-pulling to all nodes eliminates the pull +# stagger so all replicas start within milliseconds of each other. +# +# Add an entry whenever you find a new workload with peer-dependent startup. +# Update versions here when bumping the corresponding chart dependency. + +set -euo pipefail + +# --------------------------------------------------------------------------- +# Image list +# --------------------------------------------------------------------------- +# Format: /:[@sha256:] +# Each image is pulled as an init container (imagePullPolicy=IfNotPresent), +# so this is a no-op if the image is already cached on that node. +# +# Source of each image and version is noted in the comment above it. + +read -r -d '' PREPULL_YAML <<'YAML' || true +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: e2e-image-prepuller + namespace: kube-system + labels: + app: e2e-image-prepuller +spec: + selector: + matchLabels: + app: e2e-image-prepuller + template: + metadata: + labels: + app: e2e-image-prepuller + spec: + tolerations: + # Tolerate all taints so the DaemonSet reaches every node including + # control-plane nodes and nodes still coming up. + - operator: Exists + initContainers: + # kube-ovn: OVN raft cluster, 3 ovn-central replicas must elect a leader. + # Staggered pulls cause "failed to connect to peer" and raft timeout. + # Source: packages/system/kubeovn/charts/kube-ovn/values.yaml + - name: pull-kubeovn + image: docker.io/kubeovn/kube-ovn:v1.15.10 + command: ['sh', '-c', 'exit 0'] + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 1m + memory: 1Mi + # piraeus-server: LINSTOR controller/satellite cluster. + # Satellites must register with the controller before storage is usable. + # Source: packages/system/linstor/values.yaml (.piraeusServer.image) + - name: pull-piraeus-server + image: ghcr.io/cozystack/cozystack/piraeus-server:1.33.2@sha256:553f313ab35dc2e345ef3683156d29e75c23177e2750e9af3a83aa9e23941cbb + command: ['sh', '-c', 'exit 0'] + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 1m + memory: 1Mi + # linstor-csi: CSI driver sidecar shipped on every storage node. + # Large-ish image; pre-pulling avoids a slow first-mount on the first PVC. + # Source: packages/system/linstor/values.yaml (.linstorCSI.image) + - name: pull-linstor-csi + image: ghcr.io/cozystack/cozystack/linstor-csi:v1.10.5@sha256:b8f59b5659fb1791cb764d3f37df4cf29920aadcc10637231ba7d857233f377d + command: ['sh', '-c', 'exit 0'] + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 1m + memory: 1Mi + containers: + # Pause container keeps the pod Running after init containers complete. + # Running state is our signal that all images have been pulled on this node. + - name: pause + image: registry.k8s.io/pause:3.10 + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 1m + memory: 1Mi +YAML + +kubectl apply -f - <<<"${PREPULL_YAML}" + +node_count=$(kubectl get nodes --no-headers 2>/dev/null | wc -l) +echo "Waiting for e2e-image-prepuller on ${node_count} nodes (timeout 5m)..." + +# A pod is Running when all init containers have exited 0 and the main +# container has started. That means every image in the init list was pulled. +timeout 300 sh -ec " + until [ \$(kubectl get pods -n kube-system \ + -l app=e2e-image-prepuller \ + --field-selector=status.phase=Running \ + --no-headers 2>/dev/null | wc -l) -ge ${node_count} ]; do + sleep 3 + done +" + +kubectl delete daemonset e2e-image-prepuller -n kube-system --ignore-not-found +echo "Image pre-pull complete on all ${node_count} nodes." From 0fc61eeae6ac2d49e73c8f5c1839e97553fe6b8e Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 29 Apr 2026 15:05:51 +0500 Subject: [PATCH 26/27] e2e: pre-pull timing-sensitive platform images before install Some workloads (OVN raft, LINSTOR controller) fail when replicas start at different times due to image-pull stagger across nodes. Add a DaemonSet-based pre-pull step that runs before helm install, ensuring all nodes have the images cached so every replica starts within milliseconds of each other. The script accepts image refs on stdin and creates one container per image (parallel pulls, total time = max of any single image rather than sum). The bats test sources images directly from the rendered charts via yq, walking only PodSpec-shaped objects so version bumps stay in sync automatically without a separate hardcoded list. Signed-off-by: Myasnikov Daniil --- hack/e2e-install-cozystack.bats | 19 ++++- hack/e2e-prepull-images.sh | 133 +++++++++++++------------------- 2 files changed, 69 insertions(+), 83 deletions(-) diff --git a/hack/e2e-install-cozystack.bats b/hack/e2e-install-cozystack.bats index b3143d8e..1f995825 100644 --- a/hack/e2e-install-cozystack.bats +++ b/hack/e2e-install-cozystack.bats @@ -8,10 +8,23 @@ } @test "Pre-pull platform images" { - # Pull timing-sensitive images to all nodes before Cozystack installation. # Cluster-member workloads (OVN raft, LINSTOR) fail if replicas start at - # different times due to image-pull stagger. See hack/e2e-prepull-images.sh. - hack/e2e-prepull-images.sh + # different times due to image-pull stagger across nodes. Pre-pull these + # images to every node so all replicas start with images already cached. + # + # Source images directly from the rendered charts so version bumps stay in + # sync automatically. yq walks every PodSpec-shaped object and emits the + # images of each container — this scopes the result to images the kubelet + # actually pulls (skips configmap fields and CRD examples that happen to + # contain an `image:` key). Add a chart here when a new peer-sensitive + # workload is found. + { + helm template packages/system/kubeovn + helm template packages/system/linstor + } | yq -N ' + (..|select(has("containers"))|.containers[]|.image), + (..|select(has("initContainers"))|.initContainers[]|.image) + ' | hack/e2e-prepull-images.sh } @test "Install Cozystack" { diff --git a/hack/e2e-prepull-images.sh b/hack/e2e-prepull-images.sh index 209d04dd..e90d3a1b 100755 --- a/hack/e2e-prepull-images.sh +++ b/hack/e2e-prepull-images.sh @@ -1,27 +1,48 @@ #!/usr/bin/env bash -# Pre-pull timing-sensitive platform images to all cluster nodes. +# Pre-pull images to all cluster nodes. # -# Some workloads (OVN raft, LINSTOR controller) are sensitive to peer-readiness -# at startup: if nodes pull the image at different speeds, one replica boots -# before its peers are reachable, exhausts its raft connection retries, and the -# HelmRelease install times out. Pre-pulling to all nodes eliminates the pull -# stagger so all replicas start within milliseconds of each other. +# Reads image references from stdin, one per line. Empty lines and lines +# starting with '#' are ignored. # -# Add an entry whenever you find a new workload with peer-dependent startup. -# Update versions here when bumping the corresponding chart dependency. +# Some workloads (OVN raft, LINSTOR) are sensitive to peer-readiness at +# startup: if nodes pull the image at different speeds, one replica boots +# before its peers are reachable, exhausts its raft connection retries, and +# the HelmRelease install times out. Pre-pulling eliminates the pull stagger +# so all replicas start within milliseconds of each other. +# +# Implementation: a DaemonSet with one regular container per image (parallel +# pulls — total time = max of any one image rather than sum). kubectl rollout +# status blocks until all pods are Ready (= all containers running = all +# images cached on every node), then we delete the DaemonSet. set -euo pipefail -# --------------------------------------------------------------------------- -# Image list -# --------------------------------------------------------------------------- -# Format: /:[@sha256:] -# Each image is pulled as an init container (imagePullPolicy=IfNotPresent), -# so this is a no-op if the image is already cached on that node. -# -# Source of each image and version is noted in the comment above it. +mapfile -t images < <(grep -Ev '^[[:space:]]*(#|$)' | sort -u) -read -r -d '' PREPULL_YAML <<'YAML' || true +if [[ ${#images[@]} -eq 0 ]]; then + echo "e2e-prepull-images: no images on stdin, nothing to do" >&2 + exit 0 +fi + +echo "Pre-pulling ${#images[@]} image(s):" +printf ' %s\n' "${images[@]}" + +containers="" +i=0 +for image in "${images[@]}"; do + containers+=" + - name: pull-${i} + image: ${image} + command: [\"sleep\", \"infinity\"] + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 1m + memory: 1Mi" + i=$((i + 1)) +done + +kubectl apply -f - </dev/null | wc -l) -echo "Waiting for e2e-image-prepuller on ${node_count} nodes (timeout 5m)..." - -# A pod is Running when all init containers have exited 0 and the main -# container has started. That means every image in the init list was pulled. -timeout 300 sh -ec " - until [ \$(kubectl get pods -n kube-system \ - -l app=e2e-image-prepuller \ - --field-selector=status.phase=Running \ - --no-headers 2>/dev/null | wc -l) -ge ${node_count} ]; do - sleep 3 - done -" +echo "Waiting for e2e-image-prepuller DaemonSet to become ready..." +kubectl rollout status daemonset/e2e-image-prepuller -n kube-system --timeout=10m kubectl delete daemonset e2e-image-prepuller -n kube-system --ignore-not-found -echo "Image pre-pull complete on all ${node_count} nodes." +echo "Image pre-pull complete." From 3d7155c11fd8f23937bd573a5dbd9c24777026fb Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Thu, 30 Apr 2026 09:33:51 +0500 Subject: [PATCH 27/27] fix(harbor): merge BucketInfo at values root, drop targetPath The previous fix (a21c18f9) sourced BucketInfo via Flux valuesFrom with `targetPath: bucket.bucketInfo`. Flux runs values with `targetPath` through Helm's `strvals.ParseInto`, which splits the value on commas as list separators. The COSI BucketInfo JSON is comma-rich, so values resolution bailed: could not resolve Secret chart values reference 'tenant-X/harbor-X-registry-bucket' with key 'BucketInfo': key "\"spec\":{\"bucketName\":\"bucket-1785...\"" has no value (cannot end with ,) Drop `targetPath`. With only `valuesKey: BucketInfo`, helm-controller unmarshals the value as YAML and merges at the chart's values root, so JSON commas stay nested instead of being split. The system chart's bucket-secret.yaml now reads `.Values.spec.bucketName` / `.Values.spec.secretS3.{accessKeyID,accessSecretKey,endpoint}`, guarded by nested `with` blocks so the registry-s3 Secret renders only when secretS3 has been populated. Gating semantics from the previous fix are preserved: with the default `optional: false`, helm-controller still refuses to compose values until the COSI BucketAccess controller writes `BucketInfo` into the Secret, and the value's content still drives the HR config-digest. Verified via `helm template` against the system chart with three scenarios: missing `spec` and `spec` without `secretS3` render nothing; full BucketInfo renders all five S3 env keys correctly. Signed-off-by: Myasnikov Daniil Assisted-By: Claude --- packages/apps/harbor/templates/harbor.yaml | 10 ++++----- .../harbor/templates/bucket-secret.yaml | 21 +++++++------------ packages/system/harbor/values.yaml | 2 -- 3 files changed, 13 insertions(+), 20 deletions(-) diff --git a/packages/apps/harbor/templates/harbor.yaml b/packages/apps/harbor/templates/harbor.yaml index 4ac3a075..32deb4cb 100644 --- a/packages/apps/harbor/templates/harbor.yaml +++ b/packages/apps/harbor/templates/harbor.yaml @@ -65,14 +65,14 @@ spec: name: {{ .Release.Name }}-credentials valuesKey: redis-password targetPath: harbor.redis.external.password - # BucketInfo is populated by the COSI BucketAccess controller. Sourcing it via - # valuesFrom both gates reconciliation on the Secret being ready and forces a - # config-digest change (and thus a helm upgrade) when its contents change — - # Flux HR dependsOn cannot reference COSI resources directly. + # BucketInfo is the JSON blob populated by the COSI BucketAccess controller. + # Merge it at the values root (no targetPath) so Flux unmarshals it as YAML + # instead of running it through Helm's strvals parser, which would split the + # JSON on commas. This also gates reconciliation on the Secret being ready and + # forces a config-digest change when its contents change. - kind: Secret name: {{ .Release.Name }}-registry-bucket valuesKey: BucketInfo - targetPath: bucket.bucketInfo values: db: replicas: {{ .Values.database.replicas }} diff --git a/packages/system/harbor/templates/bucket-secret.yaml b/packages/system/harbor/templates/bucket-secret.yaml index 047bb760..d67aeb67 100644 --- a/packages/system/harbor/templates/bucket-secret.yaml +++ b/packages/system/harbor/templates/bucket-secret.yaml @@ -1,22 +1,17 @@ -{{- if .Values.bucket.bucketInfo }} -{{- $bucketInfo := fromJson .Values.bucket.bucketInfo }} -{{- $secretS3 := dig "spec" "secretS3" dict $bucketInfo }} -{{- if $secretS3 }} -{{- $accessKeyID := index $secretS3 "accessKeyID" }} -{{- $accessSecretKey := index $secretS3 "accessSecretKey" }} -{{- $endpoint := index $secretS3 "endpoint" }} -{{- $bucketName := dig "spec" "bucketName" "" $bucketInfo }} +{{- /* COSI BucketInfo is merged at the values root via Flux valuesFrom (no targetPath). */ -}} +{{- with .Values.spec }} +{{- with .secretS3 }} --- apiVersion: v1 kind: Secret metadata: - name: {{ .Values.harbor.fullnameOverride }}-registry-s3 + name: {{ $.Values.harbor.fullnameOverride }}-registry-s3 type: Opaque stringData: - REGISTRY_STORAGE_S3_ACCESSKEY: {{ $accessKeyID | quote }} - REGISTRY_STORAGE_S3_SECRETKEY: {{ $accessSecretKey | quote }} - REGISTRY_STORAGE_S3_REGIONENDPOINT: {{ $endpoint | quote }} - REGISTRY_STORAGE_S3_BUCKET: {{ $bucketName | quote }} + REGISTRY_STORAGE_S3_ACCESSKEY: {{ index . "accessKeyID" | quote }} + REGISTRY_STORAGE_S3_SECRETKEY: {{ index . "accessSecretKey" | quote }} + REGISTRY_STORAGE_S3_REGIONENDPOINT: {{ index . "endpoint" | quote }} + REGISTRY_STORAGE_S3_BUCKET: {{ $.Values.spec.bucketName | quote }} REGISTRY_STORAGE_S3_REGION: "us-east-1" {{- end }} {{- end }} diff --git a/packages/system/harbor/values.yaml b/packages/system/harbor/values.yaml index aeb2edac..9038bd38 100644 --- a/packages/system/harbor/values.yaml +++ b/packages/system/harbor/values.yaml @@ -1,6 +1,4 @@ harbor: {} -bucket: - bucketInfo: "" db: replicas: 2 size: 5Gi