From 64b370bfefc5a70ab38eebf1a7153741dac1223d Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Mon, 23 Mar 2026 16:54:09 +0300 Subject: [PATCH 01/20] [tests] Stabilize E2E kubernetes tests Three changes to reduce flakiness in kubernetes E2E tests: - Increase node Ready timeout from 2m to 5m to accommodate resource-constrained CI runners - Fail fast when nodes are not Ready instead of continuing to LB/NFS tests that will also fail, saving ~7 minutes per attempt - Delete stale Kubernetes resources at test start to ensure retries provision from scratch instead of patching stuck state Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- hack/e2e-apps/run-kubernetes.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/hack/e2e-apps/run-kubernetes.sh b/hack/e2e-apps/run-kubernetes.sh index dbdebd5e..739e637a 100644 --- a/hack/e2e-apps/run-kubernetes.sh +++ b/hack/e2e-apps/run-kubernetes.sh @@ -4,6 +4,10 @@ run_kubernetes_test() { local port="$3" local k8s_version=$(yq "$version_expr" packages/apps/kubernetes/files/versions.yaml) + # Clean up stale resources from a previous failed retry to ensure fresh provisioning + kubectl delete kuberneteses.apps.cozystack.io "${test_name}" \ + -n tenant-test --ignore-not-found --timeout=2m || true + kubectl apply -f - < Date: Mon, 23 Mar 2026 17:25:01 +0300 Subject: [PATCH 02/20] [tests] Add pre-cleanup, fix port-forward race, fix temp leak - Add pre-cleanup of stale resources to all app E2E tests so retries start fresh instead of patching stuck state - Wait for port-forward to be ready before using it in kubernetes tests (fixes race condition) - Reduce version check sleep from 5s to 1s for faster retries - Clean up temp directory on test failure in cozytest.sh Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- hack/cozytest.sh | 1 + hack/e2e-apps/bucket.bats | 1 + hack/e2e-apps/clickhouse.bats | 1 + hack/e2e-apps/foundationdb.bats | 1 + hack/e2e-apps/kafka.bats | 1 + hack/e2e-apps/mariadb.bats | 1 + hack/e2e-apps/mongodb.bats | 1 + hack/e2e-apps/openbao.bats | 1 + hack/e2e-apps/postgres.bats | 1 + hack/e2e-apps/qdrant.bats | 1 + hack/e2e-apps/redis.bats | 1 + hack/e2e-apps/run-kubernetes.sh | 4 +++- hack/e2e-apps/vminstance.bats | 2 ++ 13 files changed, 16 insertions(+), 1 deletion(-) diff --git a/hack/cozytest.sh b/hack/cozytest.sh index 363a4d9f..a3955fd1 100755 --- a/hack/cozytest.sh +++ b/hack/cozytest.sh @@ -65,6 +65,7 @@ run_one() { echo "----- captured output -----------------------------------------" grep -v '^__RC__' "$log" echo "$LINE" + rm -rf "$tmp" exit "$rc" fi diff --git a/hack/e2e-apps/bucket.bats b/hack/e2e-apps/bucket.bats index 9239d780..a606bf68 100644 --- a/hack/e2e-apps/bucket.bats +++ b/hack/e2e-apps/bucket.bats @@ -3,6 +3,7 @@ @test "Create and Verify Seeweedfs Bucket" { # Create the bucket resource with readwrite and readonly users name='test' + kubectl -n tenant-test delete bucket.apps.cozystack.io $name --ignore-not-found --timeout=2m || true kubectl apply -f - < /dev/null 2>&1 &' + # Wait for port-forward to be ready before using it + timeout 15 sh -ec 'until curl -sk https://localhost:'"${port}"' >/dev/null 2>&1; do sleep 1; done' # Verify the Kubernetes version matches what we expect (retry for up to 20 seconds) - timeout 20 sh -ec 'until kubectl --kubeconfig tenantkubeconfig-'"${test_name}"' version 2>/dev/null | grep -Fq "Server Version: ${k8s_version}"; do sleep 5; done' + timeout 20 sh -ec 'until kubectl --kubeconfig tenantkubeconfig-'"${test_name}"' version 2>/dev/null | grep -Fq "Server Version: ${k8s_version}"; do sleep 1; done' # Wait for at least 2 nodes to join (timeout after 8 minutes) timeout 8m bash -c ' diff --git a/hack/e2e-apps/vminstance.bats b/hack/e2e-apps/vminstance.bats index 60513e8e..9f32fcc2 100644 --- a/hack/e2e-apps/vminstance.bats +++ b/hack/e2e-apps/vminstance.bats @@ -2,6 +2,8 @@ @test "Create a VM Disk" { name='test' + kubectl -n tenant-test delete vminstances.apps.cozystack.io $name --ignore-not-found --timeout=2m || true + kubectl -n tenant-test delete vmdisks.apps.cozystack.io $name --ignore-not-found --timeout=2m || true kubectl apply -f - < Date: Mon, 23 Mar 2026 17:33:27 +0300 Subject: [PATCH 03/20] [tests] Fix port-forward leak and LB retry off-by-one - Add EXIT trap in run-kubernetes.sh to clean up port-forward process and tenantkubeconfig file on any exit path - Fix LoadBalancer retry loop: success on attempt 20 was falsely reported as failure due to $i -eq 20 check after break - Kill stale port-forward in bucket.bats before retry Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- hack/e2e-apps/bucket.bats | 1 + hack/e2e-apps/run-kubernetes.sh | 11 +++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/hack/e2e-apps/bucket.bats b/hack/e2e-apps/bucket.bats index a606bf68..31e7efb0 100644 --- a/hack/e2e-apps/bucket.bats +++ b/hack/e2e-apps/bucket.bats @@ -4,6 +4,7 @@ # Create the bucket resource with readwrite and readonly users name='test' kubectl -n tenant-test delete bucket.apps.cozystack.io $name --ignore-not-found --timeout=2m || true + pkill -f "port-forward.*8333:" 2>/dev/null || true kubectl apply -f - </dev/null || true; rm -f "tenantkubeconfig-${test_name}"' EXIT + # Clean up stale resources from a previous failed retry to ensure fresh provisioning kubectl delete kuberneteses.apps.cozystack.io "${test_name}" \ -n tenant-test --ignore-not-found --timeout=2m || true @@ -224,13 +227,17 @@ if [ -z "$LB_ADDR" ]; then exit 1 fi + lb_ok=false for i in $(seq 1 20); do echo "Attempt $i" - curl --silent --fail "http://${LB_ADDR}" && break + if curl --silent --fail "http://${LB_ADDR}"; then + lb_ok=true + break + fi sleep 3 done - if [ "$i" -eq 20 ]; then + if [ "$lb_ok" != true ]; then echo "LoadBalancer not reachable" >&2 exit 1 fi From c9b04c4ba33d17a578d1cab5bd19900ece3d38eb Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Mon, 23 Mar 2026 17:42:23 +0300 Subject: [PATCH 04/20] [tests] Fix review findings: vminstance pre-cleanup, indentation, --ignore-not-found - Add pre-cleanup for VMInstance in second vminstance.bats test - Fix indentation of LB_ADDR block in run-kubernetes.sh - Add --ignore-not-found to final Kubernetes resource cleanup Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- hack/e2e-apps/run-kubernetes.sh | 20 ++++++++++---------- hack/e2e-apps/vminstance.bats | 1 + 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/hack/e2e-apps/run-kubernetes.sh b/hack/e2e-apps/run-kubernetes.sh index 15e52ac5..aef63f30 100644 --- a/hack/e2e-apps/run-kubernetes.sh +++ b/hack/e2e-apps/run-kubernetes.sh @@ -216,16 +216,16 @@ EOF done " -LB_ADDR=$( - kubectl get svc --kubeconfig "tenantkubeconfig-${test_name}" "${test_name}-backend" \ - -n tenant-test \ - -o jsonpath='{.status.loadBalancer.ingress[0].ip}{.status.loadBalancer.ingress[0].hostname}' -) + LB_ADDR=$( + kubectl get svc --kubeconfig "tenantkubeconfig-${test_name}" "${test_name}-backend" \ + -n tenant-test \ + -o jsonpath='{.status.loadBalancer.ingress[0].ip}{.status.loadBalancer.ingress[0].hostname}' + ) -if [ -z "$LB_ADDR" ]; then - echo "LoadBalancer address is empty" >&2 - exit 1 -fi + if [ -z "$LB_ADDR" ]; then + echo "LoadBalancer address is empty" >&2 + exit 1 + fi lb_ok=false for i in $(seq 1 20); do @@ -318,6 +318,6 @@ EOF kubectl wait hr kubernetes-${test_name}-ingress-nginx -n tenant-test --timeout=5m --for=condition=ready # Clean up by deleting the Kubernetes resource - kubectl -n tenant-test delete kuberneteses.apps.cozystack.io $test_name + kubectl -n tenant-test delete kuberneteses.apps.cozystack.io $test_name --ignore-not-found } diff --git a/hack/e2e-apps/vminstance.bats b/hack/e2e-apps/vminstance.bats index 9f32fcc2..fa0f0b5b 100644 --- a/hack/e2e-apps/vminstance.bats +++ b/hack/e2e-apps/vminstance.bats @@ -27,6 +27,7 @@ EOF @test "Create a VM Instance" { diskName='test' name='test' + kubectl -n tenant-test delete vminstances.apps.cozystack.io $name --ignore-not-found --timeout=2m || true kubectl apply -f - < Date: Mon, 23 Mar 2026 17:47:48 +0300 Subject: [PATCH 05/20] [tests] Include Kubernetes resource cleanup in EXIT trap Expand the EXIT trap in run_kubernetes_test to also delete the Kubernetes tenant resource. Previously, early exit paths (exit 1 on node readiness failure, LB failure, etc.) would skip the final kubectl delete, leaving the resource behind. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- hack/e2e-apps/run-kubernetes.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/e2e-apps/run-kubernetes.sh b/hack/e2e-apps/run-kubernetes.sh index aef63f30..1c90f9fc 100644 --- a/hack/e2e-apps/run-kubernetes.sh +++ b/hack/e2e-apps/run-kubernetes.sh @@ -4,8 +4,8 @@ run_kubernetes_test() { local port="$3" local k8s_version=$(yq "$version_expr" packages/apps/kubernetes/files/versions.yaml) - # Ensure port-forward and temp files are cleaned up on any exit - trap 'pkill -f "port-forward.*${port}:" 2>/dev/null || true; rm -f "tenantkubeconfig-${test_name}"' EXIT + # Ensure all resources are cleaned up on any exit (port-forward, kubeconfig, k8s resource) + trap 'pkill -f "port-forward.*${port}:" 2>/dev/null || true; rm -f "tenantkubeconfig-${test_name}"; kubectl -n tenant-test delete kuberneteses.apps.cozystack.io "${test_name}" --ignore-not-found --timeout=2m 2>/dev/null || true' EXIT # Clean up stale resources from a previous failed retry to ensure fresh provisioning kubectl delete kuberneteses.apps.cozystack.io "${test_name}" \ From 8f2c452ff3a5f8d8bca17542f8d0f627d21aece2 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Mon, 23 Mar 2026 17:51:57 +0300 Subject: [PATCH 06/20] [tests] Replace EXIT trap with explicit cleanup function Use a _k8s_test_cleanup helper called at every exit point instead of a process-scoped EXIT trap. This avoids any potential trap scope issues and makes cleanup behavior explicit and visible at each error path. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- hack/e2e-apps/run-kubernetes.sh | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/hack/e2e-apps/run-kubernetes.sh b/hack/e2e-apps/run-kubernetes.sh index 1c90f9fc..7aa84cef 100644 --- a/hack/e2e-apps/run-kubernetes.sh +++ b/hack/e2e-apps/run-kubernetes.sh @@ -4,8 +4,12 @@ run_kubernetes_test() { local port="$3" local k8s_version=$(yq "$version_expr" packages/apps/kubernetes/files/versions.yaml) - # Ensure all resources are cleaned up on any exit (port-forward, kubeconfig, k8s resource) - trap 'pkill -f "port-forward.*${port}:" 2>/dev/null || true; rm -f "tenantkubeconfig-${test_name}"; kubectl -n tenant-test delete kuberneteses.apps.cozystack.io "${test_name}" --ignore-not-found --timeout=2m 2>/dev/null || true' EXIT + # Cleanup helper — called at every exit point and at the end of the test + _k8s_test_cleanup() { + pkill -f "port-forward.*${port}:" 2>/dev/null || true + rm -f "tenantkubeconfig-${test_name}" + kubectl -n tenant-test delete kuberneteses.apps.cozystack.io "${test_name}" --ignore-not-found --timeout=2m 2>/dev/null || true + } # Clean up stale resources from a previous failed retry to ensure fresh provisioning kubectl delete kuberneteses.apps.cozystack.io "${test_name}" \ @@ -115,7 +119,7 @@ EOF # Dump debug info and fail fast — no point running LB/NFS tests without Ready nodes kubectl --kubeconfig "tenantkubeconfig-${test_name}" describe nodes kubectl -n tenant-test get hr - exit 1 + _k8s_test_cleanup; exit 1 fi kubectl --kubeconfig "tenantkubeconfig-${test_name}" get nodes -o wide @@ -139,7 +143,7 @@ EOF if [ "$node_ok" != true ]; then echo "Kubelet versions did not match expected ${k8s_version}" >&2 - exit 1 + _k8s_test_cleanup; exit 1 fi @@ -224,7 +228,7 @@ EOF if [ -z "$LB_ADDR" ]; then echo "LoadBalancer address is empty" >&2 - exit 1 + _k8s_test_cleanup; exit 1 fi lb_ok=false @@ -239,7 +243,7 @@ EOF if [ "$lb_ok" != true ]; then echo "LoadBalancer not reachable" >&2 - exit 1 + _k8s_test_cleanup; exit 1 fi # Cleanup @@ -302,7 +306,7 @@ EOF echo "NFS mount test failed: expected 'nfs-mount-ok', got '$nfs_result'" >&2 kubectl --kubeconfig "tenantkubeconfig-${test_name}" delete pod nfs-test-pod -n tenant-test --wait=false 2>/dev/null || true kubectl --kubeconfig "tenantkubeconfig-${test_name}" delete pvc nfs-test-pvc -n tenant-test --wait=false 2>/dev/null || true - exit 1 + _k8s_test_cleanup; exit 1 fi # Cleanup NFS test resources in tenant cluster @@ -317,7 +321,7 @@ EOF done kubectl wait hr kubernetes-${test_name}-ingress-nginx -n tenant-test --timeout=5m --for=condition=ready - # Clean up by deleting the Kubernetes resource - kubectl -n tenant-test delete kuberneteses.apps.cozystack.io $test_name --ignore-not-found + # Clean up all test resources (port-forward, kubeconfig, k8s resource) + _k8s_test_cleanup } From a610d1331dc4d92fe6a0d7ea44c53f74d79f5d73 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Mon, 23 Mar 2026 17:55:30 +0300 Subject: [PATCH 07/20] [tests] Add EXIT trap as safety net for unexpected set -e exits The explicit _k8s_test_cleanup calls cover known error paths, but set -e can terminate the script at any kubectl/wait command. Add an EXIT trap to catch these unexpected exits. The function runs in a cozytest.sh subshell, so the trap is scoped and does not affect parent traps. Removed 2>/dev/null from kubectl delete in cleanup to preserve useful error output. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- hack/e2e-apps/run-kubernetes.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hack/e2e-apps/run-kubernetes.sh b/hack/e2e-apps/run-kubernetes.sh index 7aa84cef..52ae6e4c 100644 --- a/hack/e2e-apps/run-kubernetes.sh +++ b/hack/e2e-apps/run-kubernetes.sh @@ -4,12 +4,14 @@ run_kubernetes_test() { local port="$3" local k8s_version=$(yq "$version_expr" packages/apps/kubernetes/files/versions.yaml) - # Cleanup helper — called at every exit point and at the end of the test + # Cleanup helper — idempotent, safe to call multiple times _k8s_test_cleanup() { pkill -f "port-forward.*${port}:" 2>/dev/null || true rm -f "tenantkubeconfig-${test_name}" - kubectl -n tenant-test delete kuberneteses.apps.cozystack.io "${test_name}" --ignore-not-found --timeout=2m 2>/dev/null || true + kubectl -n tenant-test delete kuberneteses.apps.cozystack.io "${test_name}" --ignore-not-found --timeout=2m || true } + # Catch unexpected set -e exits (runs in subshell via cozytest.sh, does not affect parent traps) + trap '_k8s_test_cleanup' EXIT # Clean up stale resources from a previous failed retry to ensure fresh provisioning kubectl delete kuberneteses.apps.cozystack.io "${test_name}" \ From 262db2f176909d96994f982382a9cec021888fd4 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Mon, 23 Mar 2026 18:45:50 +0300 Subject: [PATCH 08/20] [tests] Increase CAPI deployment timeouts in install-cozystack On resource-constrained CI runners, HelmReleases may take close to 15 minutes to reconcile, leaving very little time for CAPI deployments to start and become available. Increase timeouts: - CAPI deployment discovery: 60s -> 180s - CAPI deployment availability: 1m -> 5m Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- hack/e2e-install-cozystack.bats | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/e2e-install-cozystack.bats b/hack/e2e-install-cozystack.bats index 7656b898..42198ae0 100644 --- a/hack/e2e-install-cozystack.bats +++ b/hack/e2e-install-cozystack.bats @@ -65,8 +65,8 @@ EOF @test "Wait for Cluster‑API provider deployments" { # Wait for Cluster‑API provider deployments - timeout 60 sh -ec 'until kubectl get deploy -n cozy-cluster-api capi-controller-manager capi-kamaji-controller-manager capi-kubeadm-bootstrap-controller-manager capi-operator-cluster-api-operator capk-controller-manager >/dev/null 2>&1; do sleep 1; done' - kubectl wait deployment/capi-controller-manager deployment/capi-kamaji-controller-manager deployment/capi-kubeadm-bootstrap-controller-manager deployment/capi-operator-cluster-api-operator deployment/capk-controller-manager -n cozy-cluster-api --timeout=1m --for=condition=available + timeout 180 sh -ec 'until kubectl get deploy -n cozy-cluster-api capi-controller-manager capi-kamaji-controller-manager capi-kubeadm-bootstrap-controller-manager capi-operator-cluster-api-operator capk-controller-manager >/dev/null 2>&1; do sleep 1; done' + kubectl wait deployment/capi-controller-manager deployment/capi-kamaji-controller-manager deployment/capi-kubeadm-bootstrap-controller-manager deployment/capi-operator-cluster-api-operator deployment/capk-controller-manager -n cozy-cluster-api --timeout=5m --for=condition=available } @test "Wait for LINSTOR and configure storage" { From 8477e72ad2e996b05e0910ff3073ffa12c1aac11 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Mon, 23 Mar 2026 20:40:45 +0300 Subject: [PATCH 10/20] [ci] Add timeout-minutes to Build and E2E jobs Without explicit timeouts, stuck jobs use the GitHub Actions default of 360 minutes (6 hours). This blocks the concurrency group and prevents new runs from starting. Set reasonable limits: - Build: 30 minutes (normally ~5 min) - E2E: 120 minutes (normally ~60 min) Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- .github/workflows/pull-requests.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml index 1f0f733a..76dbfc17 100644 --- a/.github/workflows/pull-requests.yaml +++ b/.github/workflows/pull-requests.yaml @@ -29,6 +29,7 @@ jobs: build: name: Build runs-on: [self-hosted] + timeout-minutes: 30 permissions: contents: read packages: write @@ -150,6 +151,7 @@ jobs: name: "E2E Tests" runs-on: ${{ contains(github.event.pull_request.labels.*.name, 'debug') && 'self-hosted' || 'oracle-vm-24cpu-96gb-x86-64' }} #runs-on: [oracle-vm-32cpu-128gb-x86-64] + timeout-minutes: 120 permissions: contents: read packages: read From 8c11ece4ce782879a0fb09ce512f459f6526a3a4 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Tue, 24 Mar 2026 03:23:16 +0300 Subject: [PATCH 13/20] [tests] Improve NFS test: increase PVC timeout, add debug on failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Increase NFS PVC Bound timeout from 2m to 5m — RWX via kubevirt CSI provisions an NFS server pod which takes time - Add pod describe and events dump when NFS test pod fails to complete, making the root cause visible in CI logs Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- hack/e2e-apps/run-kubernetes.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/hack/e2e-apps/run-kubernetes.sh b/hack/e2e-apps/run-kubernetes.sh index 52ae6e4c..44765f8f 100644 --- a/hack/e2e-apps/run-kubernetes.sh +++ b/hack/e2e-apps/run-kubernetes.sh @@ -274,8 +274,8 @@ spec: storage: 1Gi EOF - # Wait for PVC to be bound - kubectl --kubeconfig "tenantkubeconfig-${test_name}" wait pvc nfs-test-pvc -n tenant-test --timeout=2m --for=jsonpath='{.status.phase}'=Bound + # Wait for PVC to be bound (RWX via kubevirt CSI provisions an NFS server pod, needs time) + kubectl --kubeconfig "tenantkubeconfig-${test_name}" wait pvc nfs-test-pvc -n tenant-test --timeout=5m --for=jsonpath='{.status.phase}'=Bound # Create Pod that writes and reads data from NFS volume kubectl --kubeconfig "tenantkubeconfig-${test_name}" apply -f - <&2 + kubectl --kubeconfig "tenantkubeconfig-${test_name}" describe pod nfs-test-pod -n tenant-test >&2 || true + kubectl --kubeconfig "tenantkubeconfig-${test_name}" get events -n tenant-test --sort-by='.lastTimestamp' >&2 || true + _k8s_test_cleanup; exit 1 + fi # Verify NFS data integrity nfs_result=$(kubectl --kubeconfig "tenantkubeconfig-${test_name}" logs nfs-test-pod -n tenant-test) From 30e36a22253e4fa3294be6441690cb497576f998 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Tue, 24 Mar 2026 13:44:38 +0300 Subject: [PATCH 14/20] [tests] Remove port-forward timeout that kills NFS tests Root cause of NFS test failures: port-forward had `timeout 500s` (8.3 minutes), but tests after it take up to 26 minutes (node join 8m + node Ready 5m + LB test 2.5m + NFS 10m). Port-forward was guaranteed to die mid-test. Remove the timeout entirely. Cleanup is already handled by: - EXIT trap (_k8s_test_cleanup does pkill port-forward) - Explicit cleanup at every exit point - Job-level timeout-minutes: 120 as backstop Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- hack/e2e-apps/run-kubernetes.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hack/e2e-apps/run-kubernetes.sh b/hack/e2e-apps/run-kubernetes.sh index 44765f8f..e2cefc33 100644 --- a/hack/e2e-apps/run-kubernetes.sh +++ b/hack/e2e-apps/run-kubernetes.sh @@ -104,7 +104,8 @@ EOF sleep 1 # Set up port forwarding to the Kubernetes API server - bash -c 'timeout 500s kubectl port-forward service/kubernetes-'"${test_name}"' -n tenant-test '"${port}"':6443 > /dev/null 2>&1 &' + # No timeout — cleanup is handled by EXIT trap (_k8s_test_cleanup) and job-level timeout-minutes + kubectl port-forward service/kubernetes-"${test_name}" -n tenant-test "${port}":6443 > /dev/null 2>&1 & # Wait for port-forward to be ready before using it timeout 15 sh -ec 'until curl -sk https://localhost:'"${port}"' >/dev/null 2>&1; do sleep 1; done' # Verify the Kubernetes version matches what we expect (retry for up to 20 seconds) From 72cad3b7d6a763db492dd22d20bb7ca1465bc374 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Tue, 24 Mar 2026 18:25:50 +0300 Subject: [PATCH 17/20] [tests] Optimize cleanup and timeouts to reduce E2E duration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous changes increased worst-case E2E time by ~60 minutes due to blocking cleanup (--timeout=2m × multiple calls) and generous timeouts on retry. Changes: - Use --wait=false in _k8s_test_cleanup (non-blocking fire-and-forget) - Pre-cleanup: delete --wait=false + wait --for=delete (only blocks once) - Node Ready timeout: 5m -> 3m (was 2m originally) - NFS PVC timeout: 5m -> 3m (was 2m originally) - CAPI discovery: 180s -> 120s, availability: 5m -> 2m (was 1m originally) Estimated savings: ~40 minutes on worst-case retry path. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- hack/e2e-apps/run-kubernetes.sh | 15 +++++++-------- hack/e2e-install-cozystack.bats | 4 ++-- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/hack/e2e-apps/run-kubernetes.sh b/hack/e2e-apps/run-kubernetes.sh index e2cefc33..5bad4bb8 100644 --- a/hack/e2e-apps/run-kubernetes.sh +++ b/hack/e2e-apps/run-kubernetes.sh @@ -4,18 +4,17 @@ run_kubernetes_test() { local port="$3" local k8s_version=$(yq "$version_expr" packages/apps/kubernetes/files/versions.yaml) - # Cleanup helper — idempotent, safe to call multiple times + # Cleanup helper — idempotent, non-blocking _k8s_test_cleanup() { pkill -f "port-forward.*${port}:" 2>/dev/null || true rm -f "tenantkubeconfig-${test_name}" - kubectl -n tenant-test delete kuberneteses.apps.cozystack.io "${test_name}" --ignore-not-found --timeout=2m || true + kubectl -n tenant-test delete kuberneteses.apps.cozystack.io "${test_name}" --ignore-not-found --wait=false 2>/dev/null || true } - # Catch unexpected set -e exits (runs in subshell via cozytest.sh, does not affect parent traps) trap '_k8s_test_cleanup' EXIT - # Clean up stale resources from a previous failed retry to ensure fresh provisioning - kubectl delete kuberneteses.apps.cozystack.io "${test_name}" \ - -n tenant-test --ignore-not-found --timeout=2m || true + # Clean up stale resources from a previous failed retry and wait for deletion + kubectl -n tenant-test delete kuberneteses.apps.cozystack.io "${test_name}" --ignore-not-found --wait=false 2>/dev/null || true + kubectl -n tenant-test wait kuberneteses.apps.cozystack.io "${test_name}" --for=delete --timeout=2m 2>/dev/null || true kubectl apply -f - </dev/null 2>&1; do sleep 1; done' - kubectl wait deployment/capi-controller-manager deployment/capi-kamaji-controller-manager deployment/capi-kubeadm-bootstrap-controller-manager deployment/capi-operator-cluster-api-operator deployment/capk-controller-manager -n cozy-cluster-api --timeout=5m --for=condition=available + timeout 120 sh -ec 'until kubectl get deploy -n cozy-cluster-api capi-controller-manager capi-kamaji-controller-manager capi-kubeadm-bootstrap-controller-manager capi-operator-cluster-api-operator capk-controller-manager >/dev/null 2>&1; do sleep 1; done' + kubectl wait deployment/capi-controller-manager deployment/capi-kamaji-controller-manager deployment/capi-kubeadm-bootstrap-controller-manager deployment/capi-operator-cluster-api-operator deployment/capk-controller-manager -n cozy-cluster-api --timeout=2m --for=condition=available } @test "Wait for LINSTOR and configure storage" { From 67b41801529af546a7b121757ac336ba171af7f5 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Tue, 24 Mar 2026 21:30:06 +0300 Subject: [PATCH 20/20] [tests] Fix: remove EXIT trap that broke successful kubernetes tests Root cause: EXIT trap called _k8s_test_cleanup after subshell exit, but variables ${port} and ${test_name} were not available in that context. set -u caught the unset variable and exited with error, marking a successful test as failed. This caused every kubernetes test to "fail" on attempt 1, then retry 2 more times with real failures (NFS PVC timeout because the cluster was being deleted). Fix: remove trap and cleanup function entirely. Pre-cleanup at test start handles stale resources. Inline cleanup at test end handles normal path. On failure, resources are cleaned up by pre-cleanup on the next retry. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- hack/e2e-apps/run-kubernetes.sh | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/hack/e2e-apps/run-kubernetes.sh b/hack/e2e-apps/run-kubernetes.sh index 5bad4bb8..be6dcd6f 100644 --- a/hack/e2e-apps/run-kubernetes.sh +++ b/hack/e2e-apps/run-kubernetes.sh @@ -4,15 +4,7 @@ run_kubernetes_test() { local port="$3" local k8s_version=$(yq "$version_expr" packages/apps/kubernetes/files/versions.yaml) - # Cleanup helper — idempotent, non-blocking - _k8s_test_cleanup() { - pkill -f "port-forward.*${port}:" 2>/dev/null || true - rm -f "tenantkubeconfig-${test_name}" - kubectl -n tenant-test delete kuberneteses.apps.cozystack.io "${test_name}" --ignore-not-found --wait=false 2>/dev/null || true - } - trap '_k8s_test_cleanup' EXIT - - # Clean up stale resources from a previous failed retry and wait for deletion + # Clean up stale resources from a previous failed retry kubectl -n tenant-test delete kuberneteses.apps.cozystack.io "${test_name}" --ignore-not-found --wait=false 2>/dev/null || true kubectl -n tenant-test wait kuberneteses.apps.cozystack.io "${test_name}" --for=delete --timeout=2m 2>/dev/null || true @@ -103,7 +95,7 @@ EOF sleep 1 # Set up port forwarding to the Kubernetes API server - # No timeout — cleanup is handled by EXIT trap (_k8s_test_cleanup) and job-level timeout-minutes + # No timeout — process is killed at end of test or by job-level timeout-minutes kubectl port-forward service/kubernetes-"${test_name}" -n tenant-test "${port}":6443 > /dev/null 2>&1 & # Wait for port-forward to be ready before using it timeout 15 sh -ec 'until curl -sk https://localhost:'"${port}"' >/dev/null 2>&1; do sleep 1; done' @@ -121,7 +113,7 @@ EOF # Dump debug info and fail fast — no point running LB/NFS tests without Ready nodes kubectl --kubeconfig "tenantkubeconfig-${test_name}" describe nodes kubectl -n tenant-test get hr - _k8s_test_cleanup; exit 1 + exit 1 fi kubectl --kubeconfig "tenantkubeconfig-${test_name}" get nodes -o wide @@ -145,7 +137,7 @@ EOF if [ "$node_ok" != true ]; then echo "Kubelet versions did not match expected ${k8s_version}" >&2 - _k8s_test_cleanup; exit 1 + exit 1 fi @@ -230,7 +222,7 @@ EOF if [ -z "$LB_ADDR" ]; then echo "LoadBalancer address is empty" >&2 - _k8s_test_cleanup; exit 1 + exit 1 fi lb_ok=false @@ -245,7 +237,7 @@ EOF if [ "$lb_ok" != true ]; then echo "LoadBalancer not reachable" >&2 - _k8s_test_cleanup; exit 1 + exit 1 fi # Cleanup @@ -304,7 +296,7 @@ EOF echo "=== NFS test pod did not complete ===" >&2 kubectl --kubeconfig "tenantkubeconfig-${test_name}" describe pod nfs-test-pod -n tenant-test >&2 || true kubectl --kubeconfig "tenantkubeconfig-${test_name}" get events -n tenant-test --sort-by='.lastTimestamp' >&2 || true - _k8s_test_cleanup; exit 1 + exit 1 fi # Verify NFS data integrity @@ -313,7 +305,7 @@ EOF echo "NFS mount test failed: expected 'nfs-mount-ok', got '$nfs_result'" >&2 kubectl --kubeconfig "tenantkubeconfig-${test_name}" delete pod nfs-test-pod -n tenant-test --wait=false 2>/dev/null || true kubectl --kubeconfig "tenantkubeconfig-${test_name}" delete pvc nfs-test-pvc -n tenant-test --wait=false 2>/dev/null || true - _k8s_test_cleanup; exit 1 + exit 1 fi # Cleanup NFS test resources in tenant cluster @@ -328,7 +320,9 @@ EOF done kubectl wait hr kubernetes-${test_name}-ingress-nginx -n tenant-test --timeout=5m --for=condition=ready - # Clean up all test resources (port-forward, kubeconfig, k8s resource) - _k8s_test_cleanup + # Clean up + pkill -f "port-forward.*${port}:" 2>/dev/null || true + rm -f "tenantkubeconfig-${test_name}" + kubectl -n tenant-test delete kuberneteses.apps.cozystack.io "${test_name}" --ignore-not-found --wait=false 2>/dev/null || true }