[tests] Fix flaky E2E timeouts, add diagnostics and retry cleanup

Increase HelmRelease readiness timeouts that were too short for a
freshly-started QEMU/Talos sandbox:
- qdrant: 60s -> 180s
- redis: 20s -> 120s
- vminstance vm-disk: 5s -> 120s, vm-instance: 5s -> 180s
- mariadb: 30s -> 120s
- clickhouse: 20s -> 120s
- kafka: 30s -> 120s
- mongodb: 60s -> 120s

Add diagnostic output (HelmRelease yaml, pods, events) on failure in
all affected bats files, following the pattern already used in
harbor.bats.

Add cleanup before retry in the E2E workflow: introduce
hack/e2e-cleanup-app.sh and a cleanup-test-app-% Makefile target that
removes leftover CRs and HelmReleases from tenant-test namespace before
each subsequent attempt, so retries do not run against a dirty cluster.

Co-Authored-By: Claude <noreply@anthropic.com>
Signed-off-by: Andrei Kvapil <kvapss@gmail.com>

.github/workflows/pull-requests.yaml

@@ -251,7 +251,8 @@ jobs:
                 break
               fi
               attempt=$((attempt + 1))
-              echo "❌ Attempt $attempt failed, retrying..."
+              echo "❌ Attempt $attempt failed, cleaning up before retry..."
+              make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME cleanup-test-app-$app || true
             done
             if [ "$success" = true ]; then
               echo "✅ Test $app completed successfully"

hack/e2e-apps/clickhouse.bats

@@ -35,7 +35,15 @@ spec:
   resourcesPreset: "nano"
 EOF
   sleep 5
-  kubectl -n tenant-test wait hr clickhouse-$name --timeout=20s --for=condition=ready
+  kubectl -n tenant-test wait hr clickhouse-$name --timeout=120s --for=condition=ready || {
+    echo "=== HelmRelease status ===" >&2
+    kubectl -n tenant-test get hr clickhouse-$name -o yaml 2>&1 || true
+    echo "=== Pods ===" >&2
+    kubectl -n tenant-test get pods 2>&1 || true
+    echo "=== Events ===" >&2
+    kubectl -n tenant-test get events --sort-by='.lastTimestamp' 2>&1 | tail -30 || true
+    false
+  }
   timeout 180 sh -ec "until kubectl -n tenant-test get svc chendpoint-clickhouse-$name -o jsonpath='{.spec.ports[*].port}' | grep -q '8123 9000'; do sleep 10; done"
   kubectl -n tenant-test wait statefulset.apps/chi-clickhouse-$name-clickhouse-0-0 --timeout=120s --for=jsonpath='{.status.replicas}'=1
   timeout 80 sh -ec "until kubectl -n tenant-test get endpoints chi-clickhouse-$name-clickhouse-0-0 -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done"

hack/e2e-apps/kafka.bats

@@ -38,7 +38,15 @@ spec:
       replicas: 2
 EOF
   sleep 5
-  kubectl -n tenant-test wait hr kafka-$name --timeout=30s --for=condition=ready
+  kubectl -n tenant-test wait hr kafka-$name --timeout=120s --for=condition=ready || {
+    echo "=== HelmRelease status ===" >&2
+    kubectl -n tenant-test get hr kafka-$name -o yaml 2>&1 || true
+    echo "=== Pods ===" >&2
+    kubectl -n tenant-test get pods 2>&1 || true
+    echo "=== Events ===" >&2
+    kubectl -n tenant-test get events --sort-by='.lastTimestamp' 2>&1 | tail -30 || true
+    false
+  }
   kubectl wait kafkas -n tenant-test test --timeout=60s --for=condition=ready
   timeout 60 sh -ec "until kubectl -n tenant-test get pvc data-kafka-$name-zookeeper-0; do sleep 10; done"
   kubectl -n tenant-test wait pvc data-kafka-$name-zookeeper-0 --timeout=50s --for=jsonpath='{.status.phase}'=Bound

hack/e2e-apps/mariadb.bats

@@ -35,7 +35,15 @@ spec:
   resourcesPreset: "nano"
 EOF
   sleep 5
-  kubectl -n tenant-test wait hr mariadb-$name --timeout=30s --for=condition=ready
+  kubectl -n tenant-test wait hr mariadb-$name --timeout=120s --for=condition=ready || {
+    echo "=== HelmRelease status ===" >&2
+    kubectl -n tenant-test get hr mariadb-$name -o yaml 2>&1 || true
+    echo "=== Pods ===" >&2
+    kubectl -n tenant-test get pods 2>&1 || true
+    echo "=== Events ===" >&2
+    kubectl -n tenant-test get events --sort-by='.lastTimestamp' 2>&1 | tail -30 || true
+    false
+  }
   timeout 80 sh -ec "until kubectl -n tenant-test get svc mariadb-$name -o jsonpath='{.spec.ports[0].port}' | grep -q '3306'; do sleep 10; done"
   timeout 80 sh -ec "until kubectl -n tenant-test get endpoints mariadb-$name -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done"
   kubectl -n tenant-test wait statefulset.apps/mariadb-$name --timeout=110s --for=jsonpath='{.status.replicas}'=2

hack/e2e-apps/mongodb.bats

@@ -27,7 +27,15 @@ spec:
 EOF
   sleep 5
   # Wait for HelmRelease
-  kubectl -n tenant-test wait hr mongodb-$name --timeout=60s --for=condition=ready
+  kubectl -n tenant-test wait hr mongodb-$name --timeout=120s --for=condition=ready || {
+    echo "=== HelmRelease status ===" >&2
+    kubectl -n tenant-test get hr mongodb-$name -o yaml 2>&1 || true
+    echo "=== Pods ===" >&2
+    kubectl -n tenant-test get pods 2>&1 || true
+    echo "=== Events ===" >&2
+    kubectl -n tenant-test get events --sort-by='.lastTimestamp' 2>&1 | tail -30 || true
+    false
+  }
   # Wait for MongoDB service (port 27017)
   timeout 120 sh -ec "until kubectl -n tenant-test get svc mongodb-$name-rs0 -o jsonpath='{.spec.ports[0].port}' | grep -q '27017'; do sleep 10; done"
   # Wait for endpoints

hack/e2e-apps/qdrant.bats

@@ -17,7 +17,15 @@ spec:
   external: false
 EOF
   sleep 5
-  kubectl -n tenant-test wait hr qdrant-$name --timeout=60s --for=condition=ready
+  kubectl -n tenant-test wait hr qdrant-$name --timeout=180s --for=condition=ready || {
+    echo "=== HelmRelease status ===" >&2
+    kubectl -n tenant-test get hr qdrant-$name -o yaml 2>&1 || true
+    echo "=== Pods ===" >&2
+    kubectl -n tenant-test get pods 2>&1 || true
+    echo "=== Events ===" >&2
+    kubectl -n tenant-test get events --sort-by='.lastTimestamp' 2>&1 | tail -30 || true
+    false
+  }
   kubectl -n tenant-test wait hr qdrant-$name-system --timeout=120s --for=condition=ready
   kubectl -n tenant-test wait sts qdrant-$name --timeout=90s --for=jsonpath='{.status.readyReplicas}'=1
   kubectl -n tenant-test wait pvc qdrant-storage-qdrant-$name-0 --timeout=50s --for=jsonpath='{.status.phase}'=Bound

hack/e2e-apps/redis.bats

@@ -18,7 +18,15 @@ spec:
   resourcesPreset: "nano"
 EOF
   sleep 5
-  kubectl -n tenant-test wait hr redis-$name --timeout=20s --for=condition=ready
+  kubectl -n tenant-test wait hr redis-$name --timeout=120s --for=condition=ready || {
+    echo "=== HelmRelease status ===" >&2
+    kubectl -n tenant-test get hr redis-$name -o yaml 2>&1 || true
+    echo "=== Pods ===" >&2
+    kubectl -n tenant-test get pods 2>&1 || true
+    echo "=== Events ===" >&2
+    kubectl -n tenant-test get events --sort-by='.lastTimestamp' 2>&1 | tail -30 || true
+    false
+  }
   kubectl -n tenant-test wait pvc redisfailover-persistent-data-rfr-redis-$name-0 --timeout=50s --for=jsonpath='{.status.phase}'=Bound
   kubectl -n tenant-test wait deploy rfs-redis-$name --timeout=90s --for=condition=available
   kubectl -n tenant-test wait sts rfr-redis-$name --timeout=90s --for=jsonpath='{.status.replicas}'=2

hack/e2e-apps/vminstance.bats

@@ -17,7 +17,15 @@ spec:
   storageClass: replicated
 EOF
   sleep 5
-  kubectl -n tenant-test wait hr vm-disk-$name --timeout=5s --for=condition=ready
+  kubectl -n tenant-test wait hr vm-disk-$name --timeout=120s --for=condition=ready || {
+    echo "=== HelmRelease status ===" >&2
+    kubectl -n tenant-test get hr vm-disk-$name -o yaml 2>&1 || true
+    echo "=== Pods ===" >&2
+    kubectl -n tenant-test get pods 2>&1 || true
+    echo "=== Events ===" >&2
+    kubectl -n tenant-test get events --sort-by='.lastTimestamp' 2>&1 | tail -30 || true
+    false
+  }
   kubectl -n tenant-test wait dv vm-disk-$name --timeout=250s --for=condition=ready
   kubectl -n tenant-test wait pvc vm-disk-$name --timeout=200s --for=jsonpath='{.status.phase}'=Bound
 }
@@ -58,7 +66,17 @@ spec:
 EOF
   sleep 5
   timeout 20 sh -ec "until kubectl -n tenant-test get vmi vm-instance-$name -o jsonpath='{.status.interfaces[0].ipAddress}' | grep -q '[0-9]'; do sleep 5; done"
-  kubectl -n tenant-test wait hr vm-instance-$name --timeout=5s --for=condition=ready
+  kubectl -n tenant-test wait hr vm-instance-$name --timeout=180s --for=condition=ready || {
+    echo "=== HelmRelease status ===" >&2
+    kubectl -n tenant-test get hr vm-instance-$name -o yaml 2>&1 || true
+    echo "=== VM/VMI status ===" >&2
+    kubectl -n tenant-test get vmdisk,vm,virtualmachine,virtualmachineinstance 2>&1 || true
+    echo "=== Pods ===" >&2
+    kubectl -n tenant-test get pods 2>&1 || true
+    echo "=== Events ===" >&2
+    kubectl -n tenant-test get events --sort-by='.lastTimestamp' 2>&1 | tail -30 || true
+    false
+  }
   kubectl -n tenant-test wait vm vm-instance-$name --timeout=20s --for=condition=ready
   kubectl -n tenant-test delete vminstances.apps.cozystack.io $name 
   kubectl -n tenant-test delete vmdisks.apps.cozystack.io $diskName 

hack/e2e-cleanup-app.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+# Cleanup leftover app resources in tenant-test namespace before retrying an E2E test.
+# Usage: e2e-cleanup-app.sh <app-name>
+# Example: e2e-cleanup-app.sh qdrant
+
+set -e
+
+APP="$1"
+if [ -z "$APP" ]; then
+	echo "Usage: $0 <app-name>" >&2
+	exit 1
+fi
+
+NS="tenant-test"
+
+echo "=== Cleaning up leftover resources for app '$APP' in namespace '$NS' ==="
+
+# Delete the custom resource(s) by guessing the Kind from app name
+# Each bats test creates a resource matching the app name; delete all CR types
+kubectl api-resources --verbs=list --namespaced -o name 2>/dev/null |
+	grep '\.apps\.cozystack\.io' |
+	while read -r resource; do
+		kubectl -n "$NS" delete "$resource" --all --ignore-not-found --wait=false 2>/dev/null || true
+	done
+
+# Remove all HelmReleases matching the app prefix
+kubectl -n "$NS" get hr --no-headers -o custom-columns=':metadata.name' 2>/dev/null |
+	grep "^${APP}-" |
+	while read -r hr; do
+		echo "Deleting HelmRelease $hr"
+		kubectl -n "$NS" delete hr "$hr" --ignore-not-found --wait=false 2>/dev/null || true
+	done
+
+# Give controllers a moment to process deletions
+sleep 10
+
+echo "=== Cleanup done ==="

packages/core/testing/Makefile

@@ -42,6 +42,9 @@ test-openapi:
 test-apps-%:
 	docker exec "${SANDBOX_NAME}" sh -c 'cd /workspace && hack/cozytest.sh hack/e2e-apps/$*.bats'
 
+cleanup-test-app-%:
+	docker exec "${SANDBOX_NAME}" hack/e2e-cleanup-app.sh "$*"
+
 collect-report: ## Collect the test report from the sandbox.
 	docker exec "${SANDBOX_NAME}" sh -c 'cd /workspace && hack/cozyreport.sh cozyreport'
 	mkdir -p ../../../_out