From 8ff2ba3337db7cf682a82a5de9285305b818c33b Mon Sep 17 00:00:00 2001
From: Andrei Kvapil <kvapss@gmail.com>
Date: Fri, 20 Mar 2026 16:22:41 +0100
Subject: [PATCH] [tests] Fix flaky E2E timeouts, add diagnostics and retry
 cleanup

Increase HelmRelease readiness timeouts that were too short for a
freshly-started QEMU/Talos sandbox:
- qdrant: 60s -> 180s
- redis: 20s -> 120s
- vminstance vm-disk: 5s -> 120s, vm-instance: 5s -> 180s
- mariadb: 30s -> 120s
- clickhouse: 20s -> 120s
- kafka: 30s -> 120s
- mongodb: 60s -> 120s

Add diagnostic output (HelmRelease yaml, pods, events) on failure in
all affected bats files, following the pattern already used in
harbor.bats.

Add cleanup before retry in the E2E workflow: introduce
hack/e2e-cleanup-app.sh and a cleanup-test-app-% Makefile target that
removes leftover CRs and HelmReleases from tenant-test namespace before
each subsequent attempt, so retries do not run against a dirty cluster.

Co-Authored-By: Claude <noreply@anthropic.com>
Signed-off-by: Andrei Kvapil <kvapss@gmail.com>
---
 .github/workflows/pull-requests.yaml |  3 ++-
 hack/e2e-apps/clickhouse.bats        | 10 +++++++-
 hack/e2e-apps/kafka.bats             | 10 +++++++-
 hack/e2e-apps/mariadb.bats           | 10 +++++++-
 hack/e2e-apps/mongodb.bats           | 10 +++++++-
 hack/e2e-apps/qdrant.bats            | 10 +++++++-
 hack/e2e-apps/redis.bats             | 10 +++++++-
 hack/e2e-apps/vminstance.bats        | 22 +++++++++++++++--
 hack/e2e-cleanup-app.sh              | 37 ++++++++++++++++++++++++++++
 packages/core/testing/Makefile       |  3 +++
 10 files changed, 116 insertions(+), 9 deletions(-)
 create mode 100755 hack/e2e-cleanup-app.sh

diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml
index 1f0f733a..e4cd358a 100644
--- a/.github/workflows/pull-requests.yaml
+++ b/.github/workflows/pull-requests.yaml
@@ -251,7 +251,8 @@ jobs:
                 break
               fi
               attempt=$((attempt + 1))
-              echo "❌ Attempt $attempt failed, retrying..."
+              echo "❌ Attempt $attempt failed, cleaning up before retry..."
+              make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME cleanup-test-app-$app || true
             done
             if [ "$success" = true ]; then
               echo "✅ Test $app completed successfully"
diff --git a/hack/e2e-apps/clickhouse.bats b/hack/e2e-apps/clickhouse.bats
index 1684c0c4..56d7b129 100644
--- a/hack/e2e-apps/clickhouse.bats
+++ b/hack/e2e-apps/clickhouse.bats
@@ -35,7 +35,15 @@ spec:
   resourcesPreset: "nano"
 EOF
   sleep 5
-  kubectl -n tenant-test wait hr clickhouse-$name --timeout=20s --for=condition=ready
+  kubectl -n tenant-test wait hr clickhouse-$name --timeout=120s --for=condition=ready || {
+    echo "=== HelmRelease status ===" >&2
+    kubectl -n tenant-test get hr clickhouse-$name -o yaml 2>&1 || true
+    echo "=== Pods ===" >&2
+    kubectl -n tenant-test get pods 2>&1 || true
+    echo "=== Events ===" >&2
+    kubectl -n tenant-test get events --sort-by='.lastTimestamp' 2>&1 | tail -30 || true
+    false
+  }
   timeout 180 sh -ec "until kubectl -n tenant-test get svc chendpoint-clickhouse-$name -o jsonpath='{.spec.ports[*].port}' | grep -q '8123 9000'; do sleep 10; done"
   kubectl -n tenant-test wait statefulset.apps/chi-clickhouse-$name-clickhouse-0-0 --timeout=120s --for=jsonpath='{.status.replicas}'=1
   timeout 80 sh -ec "until kubectl -n tenant-test get endpoints chi-clickhouse-$name-clickhouse-0-0 -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done"
diff --git a/hack/e2e-apps/kafka.bats b/hack/e2e-apps/kafka.bats
index ceaf2cde..fd7d0a4a 100644
--- a/hack/e2e-apps/kafka.bats
+++ b/hack/e2e-apps/kafka.bats
@@ -38,7 +38,15 @@ spec:
       replicas: 2
 EOF
   sleep 5
-  kubectl -n tenant-test wait hr kafka-$name --timeout=30s --for=condition=ready
+  kubectl -n tenant-test wait hr kafka-$name --timeout=120s --for=condition=ready || {
+    echo "=== HelmRelease status ===" >&2
+    kubectl -n tenant-test get hr kafka-$name -o yaml 2>&1 || true
+    echo "=== Pods ===" >&2
+    kubectl -n tenant-test get pods 2>&1 || true
+    echo "=== Events ===" >&2
+    kubectl -n tenant-test get events --sort-by='.lastTimestamp' 2>&1 | tail -30 || true
+    false
+  }
   kubectl wait kafkas -n tenant-test test --timeout=60s --for=condition=ready
   timeout 60 sh -ec "until kubectl -n tenant-test get pvc data-kafka-$name-zookeeper-0; do sleep 10; done"
   kubectl -n tenant-test wait pvc data-kafka-$name-zookeeper-0 --timeout=50s --for=jsonpath='{.status.phase}'=Bound
diff --git a/hack/e2e-apps/mariadb.bats b/hack/e2e-apps/mariadb.bats
index 5b11c4c0..1b6f620b 100644
--- a/hack/e2e-apps/mariadb.bats
+++ b/hack/e2e-apps/mariadb.bats
@@ -35,7 +35,15 @@ spec:
   resourcesPreset: "nano"
 EOF
   sleep 5
-  kubectl -n tenant-test wait hr mariadb-$name --timeout=30s --for=condition=ready
+  kubectl -n tenant-test wait hr mariadb-$name --timeout=120s --for=condition=ready || {
+    echo "=== HelmRelease status ===" >&2
+    kubectl -n tenant-test get hr mariadb-$name -o yaml 2>&1 || true
+    echo "=== Pods ===" >&2
+    kubectl -n tenant-test get pods 2>&1 || true
+    echo "=== Events ===" >&2
+    kubectl -n tenant-test get events --sort-by='.lastTimestamp' 2>&1 | tail -30 || true
+    false
+  }
   timeout 80 sh -ec "until kubectl -n tenant-test get svc mariadb-$name -o jsonpath='{.spec.ports[0].port}' | grep -q '3306'; do sleep 10; done"
   timeout 80 sh -ec "until kubectl -n tenant-test get endpoints mariadb-$name -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done"
   kubectl -n tenant-test wait statefulset.apps/mariadb-$name --timeout=110s --for=jsonpath='{.status.replicas}'=2
diff --git a/hack/e2e-apps/mongodb.bats b/hack/e2e-apps/mongodb.bats
index db58b2cf..744e477d 100644
--- a/hack/e2e-apps/mongodb.bats
+++ b/hack/e2e-apps/mongodb.bats
@@ -27,7 +27,15 @@ spec:
 EOF
   sleep 5
   # Wait for HelmRelease
-  kubectl -n tenant-test wait hr mongodb-$name --timeout=60s --for=condition=ready
+  kubectl -n tenant-test wait hr mongodb-$name --timeout=120s --for=condition=ready || {
+    echo "=== HelmRelease status ===" >&2
+    kubectl -n tenant-test get hr mongodb-$name -o yaml 2>&1 || true
+    echo "=== Pods ===" >&2
+    kubectl -n tenant-test get pods 2>&1 || true
+    echo "=== Events ===" >&2
+    kubectl -n tenant-test get events --sort-by='.lastTimestamp' 2>&1 | tail -30 || true
+    false
+  }
   # Wait for MongoDB service (port 27017)
   timeout 120 sh -ec "until kubectl -n tenant-test get svc mongodb-$name-rs0 -o jsonpath='{.spec.ports[0].port}' | grep -q '27017'; do sleep 10; done"
   # Wait for endpoints
diff --git a/hack/e2e-apps/qdrant.bats b/hack/e2e-apps/qdrant.bats
index 63b82a3b..8880edfd 100755
--- a/hack/e2e-apps/qdrant.bats
+++ b/hack/e2e-apps/qdrant.bats
@@ -17,7 +17,15 @@ spec:
   external: false
 EOF
   sleep 5
-  kubectl -n tenant-test wait hr qdrant-$name --timeout=60s --for=condition=ready
+  kubectl -n tenant-test wait hr qdrant-$name --timeout=180s --for=condition=ready || {
+    echo "=== HelmRelease status ===" >&2
+    kubectl -n tenant-test get hr qdrant-$name -o yaml 2>&1 || true
+    echo "=== Pods ===" >&2
+    kubectl -n tenant-test get pods 2>&1 || true
+    echo "=== Events ===" >&2
+    kubectl -n tenant-test get events --sort-by='.lastTimestamp' 2>&1 | tail -30 || true
+    false
+  }
   kubectl -n tenant-test wait hr qdrant-$name-system --timeout=120s --for=condition=ready
   kubectl -n tenant-test wait sts qdrant-$name --timeout=90s --for=jsonpath='{.status.readyReplicas}'=1
   kubectl -n tenant-test wait pvc qdrant-storage-qdrant-$name-0 --timeout=50s --for=jsonpath='{.status.phase}'=Bound
diff --git a/hack/e2e-apps/redis.bats b/hack/e2e-apps/redis.bats
index 050f407b..463f8d52 100644
--- a/hack/e2e-apps/redis.bats
+++ b/hack/e2e-apps/redis.bats
@@ -18,7 +18,15 @@ spec:
   resourcesPreset: "nano"
 EOF
   sleep 5
-  kubectl -n tenant-test wait hr redis-$name --timeout=20s --for=condition=ready
+  kubectl -n tenant-test wait hr redis-$name --timeout=120s --for=condition=ready || {
+    echo "=== HelmRelease status ===" >&2
+    kubectl -n tenant-test get hr redis-$name -o yaml 2>&1 || true
+    echo "=== Pods ===" >&2
+    kubectl -n tenant-test get pods 2>&1 || true
+    echo "=== Events ===" >&2
+    kubectl -n tenant-test get events --sort-by='.lastTimestamp' 2>&1 | tail -30 || true
+    false
+  }
   kubectl -n tenant-test wait pvc redisfailover-persistent-data-rfr-redis-$name-0 --timeout=50s --for=jsonpath='{.status.phase}'=Bound
   kubectl -n tenant-test wait deploy rfs-redis-$name --timeout=90s --for=condition=available
   kubectl -n tenant-test wait sts rfr-redis-$name --timeout=90s --for=jsonpath='{.status.replicas}'=2
diff --git a/hack/e2e-apps/vminstance.bats b/hack/e2e-apps/vminstance.bats
index 60513e8e..2463a65d 100644
--- a/hack/e2e-apps/vminstance.bats
+++ b/hack/e2e-apps/vminstance.bats
@@ -17,7 +17,15 @@ spec:
   storageClass: replicated
 EOF
   sleep 5
-  kubectl -n tenant-test wait hr vm-disk-$name --timeout=5s --for=condition=ready
+  kubectl -n tenant-test wait hr vm-disk-$name --timeout=120s --for=condition=ready || {
+    echo "=== HelmRelease status ===" >&2
+    kubectl -n tenant-test get hr vm-disk-$name -o yaml 2>&1 || true
+    echo "=== Pods ===" >&2
+    kubectl -n tenant-test get pods 2>&1 || true
+    echo "=== Events ===" >&2
+    kubectl -n tenant-test get events --sort-by='.lastTimestamp' 2>&1 | tail -30 || true
+    false
+  }
   kubectl -n tenant-test wait dv vm-disk-$name --timeout=250s --for=condition=ready
   kubectl -n tenant-test wait pvc vm-disk-$name --timeout=200s --for=jsonpath='{.status.phase}'=Bound
 }
@@ -58,7 +66,17 @@ spec:
 EOF
   sleep 5
   timeout 20 sh -ec "until kubectl -n tenant-test get vmi vm-instance-$name -o jsonpath='{.status.interfaces[0].ipAddress}' | grep -q '[0-9]'; do sleep 5; done"
-  kubectl -n tenant-test wait hr vm-instance-$name --timeout=5s --for=condition=ready
+  kubectl -n tenant-test wait hr vm-instance-$name --timeout=180s --for=condition=ready || {
+    echo "=== HelmRelease status ===" >&2
+    kubectl -n tenant-test get hr vm-instance-$name -o yaml 2>&1 || true
+    echo "=== VM/VMI status ===" >&2
+    kubectl -n tenant-test get vmdisk,vm,virtualmachine,virtualmachineinstance 2>&1 || true
+    echo "=== Pods ===" >&2
+    kubectl -n tenant-test get pods 2>&1 || true
+    echo "=== Events ===" >&2
+    kubectl -n tenant-test get events --sort-by='.lastTimestamp' 2>&1 | tail -30 || true
+    false
+  }
   kubectl -n tenant-test wait vm vm-instance-$name --timeout=20s --for=condition=ready
   kubectl -n tenant-test delete vminstances.apps.cozystack.io $name 
   kubectl -n tenant-test delete vmdisks.apps.cozystack.io $diskName 
diff --git a/hack/e2e-cleanup-app.sh b/hack/e2e-cleanup-app.sh
new file mode 100755
index 00000000..544b7885
--- /dev/null
+++ b/hack/e2e-cleanup-app.sh
@@ -0,0 +1,37 @@
+#!/bin/sh
+# Cleanup leftover app resources in tenant-test namespace before retrying an E2E test.
+# Usage: e2e-cleanup-app.sh <app-name>
+# Example: e2e-cleanup-app.sh qdrant
+
+set -e
+
+APP="$1"
+if [ -z "$APP" ]; then
+	echo "Usage: $0 <app-name>" >&2
+	exit 1
+fi
+
+NS="tenant-test"
+
+echo "=== Cleaning up leftover resources for app '$APP' in namespace '$NS' ==="
+
+# Delete the custom resource(s) by guessing the Kind from app name
+# Each bats test creates a resource matching the app name; delete all CR types
+kubectl api-resources --verbs=list --namespaced -o name 2>/dev/null |
+	grep '\.apps\.cozystack\.io' |
+	while read -r resource; do
+		kubectl -n "$NS" delete "$resource" --all --ignore-not-found --wait=false 2>/dev/null || true
+	done
+
+# Remove all HelmReleases matching the app prefix
+kubectl -n "$NS" get hr --no-headers -o custom-columns=':metadata.name' 2>/dev/null |
+	grep "^${APP}-" |
+	while read -r hr; do
+		echo "Deleting HelmRelease $hr"
+		kubectl -n "$NS" delete hr "$hr" --ignore-not-found --wait=false 2>/dev/null || true
+	done
+
+# Give controllers a moment to process deletions
+sleep 10
+
+echo "=== Cleanup done ==="
diff --git a/packages/core/testing/Makefile b/packages/core/testing/Makefile
index ce21a8e1..f8002834 100644
--- a/packages/core/testing/Makefile
+++ b/packages/core/testing/Makefile
@@ -42,6 +42,9 @@ test-openapi:
 test-apps-%:
 	docker exec "${SANDBOX_NAME}" sh -c 'cd /workspace && hack/cozytest.sh hack/e2e-apps/$*.bats'
 
+cleanup-test-app-%:
+	docker exec "${SANDBOX_NAME}" hack/e2e-cleanup-app.sh "$*"
+
 collect-report: ## Collect the test report from the sandbox.
 	docker exec "${SANDBOX_NAME}" sh -c 'cd /workspace && hack/cozyreport.sh cozyreport'
 	mkdir -p ../../../_out