vrr/cozystack
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[tests] Stabilize E2E kubernetes tests (#2262)

Browse source 
## What this PR does

Reduces flakiness in E2E tests by addressing the most common failure
modes.

### Kubernetes tenant tests (`run-kubernetes.sh`)

- Increase node Ready timeout from 2m to 5m — CI runners are shared and
resource-constrained
- Fail fast when nodes are not Ready — saves ~7 minutes per failed
attempt by not running LB/NFS tests that will also fail
- Delete stale Kubernetes resources at test start — retries provision
from scratch instead of patching stuck state
- Wait for port-forward to be ready before using it (fixes race
condition)
- Reduce version check polling interval from 5s to 1s

### All app tests (postgres, redis, kafka, clickhouse, mariadb, mongodb,
qdrant, foundationdb, openbao, vminstance, bucket)

- Add pre-cleanup of stale resources from previous failed retries so
each attempt starts clean

### Test runner (`cozytest.sh`)

- Clean up temp directory on test failure (was only cleaned on success,
leaking `/tmp` dirs)

### Release note

```release-note
NONE
```

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **Tests**
* Improved e2e robustness by deleting stale resources before creation
(ignore-if-missing, bounded time) and killing leftover port-forwards.
* Added an exit cleanup to consistently remove temporary artifacts and
teardown port-forwards on any exit.
* Added readiness polling for forwarded endpoints (curl with timeout);
made API/version retries more responsive.
* Extended node readiness wait (2m → 5m), dump debug info and fail fast
if unready.
  * Made load‑balancer reachability checking explicit and more reliable.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
Kirill Ilin 2026-03-26 18:57:14 +05:00 • committed by GitHub
commit dfe16ed9b4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 58 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/pull-requests.yaml vendored
View file

@ -29,6 +29,7 @@ jobs:
build:
name: Build
runs-on: [self-hosted]
timeout-minutes: 30
permissions:
contents: read
packages: write
@ -150,6 +151,7 @@ jobs:
name: "E2E Tests"
runs-on: ${{ contains(github.event.pull_request.labels.*.name, 'debug') && 'self-hosted' || 'oracle-vm-24cpu-96gb-x86-64' }}
#runs-on: [oracle-vm-32cpu-128gb-x86-64]
timeout-minutes: 120
permissions:
contents: read
packages: read

1
hack/cozytest.sh
View file

@ -65,6 +65,7 @@ run_one() {
echo "----- captured output -----------------------------------------"
grep -v '^__RC__' "$log"
echo "$LINE"
rm -rf "$tmp"
exit "$rc"
fi

2
hack/e2e-apps/bucket.bats
View file

@ -3,6 +3,8 @@
@test "Create and Verify Seeweedfs Bucket" {
# Create the bucket resource with readwrite and readonly users
name='test'
kubectl -n tenant-test delete bucket.apps.cozystack.io $name --ignore-not-found --timeout=2m || true
pkill -f "port-forward.*8333:" 2>/dev/null || true
kubectl apply -f - <<EOF
apiVersion: apps.cozystack.io/v1alpha1
kind: Bucket

1
hack/e2e-apps/clickhouse.bats
View file

@ -2,6 +2,7 @@
@test "Create DB ClickHouse" {
name='test'
kubectl -n tenant-test delete clickhouse.apps.cozystack.io $name --ignore-not-found --timeout=2m || true
kubectl apply -f- <<EOF
apiVersion: apps.cozystack.io/v1alpha1
kind: ClickHouse

1
hack/e2e-apps/foundationdb.bats
View file

@ -2,6 +2,7 @@
@test "Create DB FoundationDB" {
name='test'
kubectl -n tenant-test delete foundationdb.apps.cozystack.io $name --ignore-not-found --timeout=2m || true
kubectl apply -f - <<EOF
apiVersion: apps.cozystack.io/v1alpha1
kind: FoundationDB

1
hack/e2e-apps/kafka.bats
View file

@ -2,6 +2,7 @@
@test "Create Kafka" {
name='test'
kubectl -n tenant-test delete kafka.apps.cozystack.io $name --ignore-not-found --timeout=2m || true
kubectl apply -f- <<EOF
apiVersion: apps.cozystack.io/v1alpha1
kind: Kafka

1
hack/e2e-apps/mariadb.bats
View file

@ -2,6 +2,7 @@
@test "Create DB MariaDB" {
name='test'
kubectl -n tenant-test delete mariadbs.apps.cozystack.io $name --ignore-not-found --timeout=2m || true
kubectl apply -f- <<EOF
apiVersion: apps.cozystack.io/v1alpha1
kind: MariaDB

1
hack/e2e-apps/mongodb.bats
View file

@ -2,6 +2,7 @@
@test "Create DB MongoDB" {
name='test'
kubectl -n tenant-test delete mongodbs.apps.cozystack.io $name --ignore-not-found --timeout=2m || true
kubectl apply -f - <<EOF
apiVersion: apps.cozystack.io/v1alpha1
kind: MongoDB

1
hack/e2e-apps/openbao.bats
View file

@ -2,6 +2,7 @@
@test "Create OpenBAO (standalone)" {
name='test'
kubectl -n tenant-test delete openbao.apps.cozystack.io $name --ignore-not-found --timeout=2m || true
kubectl apply -f- <<EOF
apiVersion: apps.cozystack.io/v1alpha1
kind: OpenBAO

1
hack/e2e-apps/postgres.bats
View file

@ -2,6 +2,7 @@
@test "Create DB PostgreSQL" {
name='test'
kubectl -n tenant-test delete postgreses.apps.cozystack.io $name --ignore-not-found --timeout=2m || true
kubectl apply -f - <<EOF
apiVersion: apps.cozystack.io/v1alpha1
kind: Postgres

1
hack/e2e-apps/qdrant.bats
View file

@ -2,6 +2,7 @@
@test "Create Qdrant" {
name='test'
kubectl -n tenant-test delete qdrant.apps.cozystack.io $name --ignore-not-found --timeout=2m || true
kubectl apply -f- <<EOF
apiVersion: apps.cozystack.io/v1alpha1
kind: Qdrant

1
hack/e2e-apps/redis.bats
View file

@ -2,6 +2,7 @@
@test "Create Redis" {
name='test'
kubectl -n tenant-test delete redis.apps.cozystack.io $name --ignore-not-found --timeout=2m || true
kubectl apply -f- <<EOF
apiVersion: apps.cozystack.io/v1alpha1
kind: Redis

59
hack/e2e-apps/run-kubernetes.sh
View file

@ -4,6 +4,10 @@ run_kubernetes_test() {
local port="$3"
local k8s_version=$(yq "$version_expr" packages/apps/kubernetes/files/versions.yaml)
# Clean up stale resources from a previous failed retry
kubectl -n tenant-test delete kuberneteses.apps.cozystack.io "${test_name}" --ignore-not-found --wait=false 2>/dev/null || true
kubectl -n tenant-test wait kuberneteses.apps.cozystack.io "${test_name}" --for=delete --timeout=2m 2>/dev/null || true
kubectl apply -f - <<EOF
apiVersion: apps.cozystack.io/v1alpha1
kind: Kubernetes
@ -91,9 +95,12 @@ EOF
sleep 1
# Set up port forwarding to the Kubernetes API server
bash -c 'timeout 500s kubectl port-forward service/kubernetes-'"${test_name}"' -n tenant-test '"${port}"':6443 > /dev/null 2>&1 &'
# No timeout — process is killed at end of test or by job-level timeout-minutes
kubectl port-forward service/kubernetes-"${test_name}" -n tenant-test "${port}":6443 > /dev/null 2>&1 &
# Wait for port-forward to be ready before using it
timeout 15 sh -ec 'until curl -sk https://localhost:'"${port}"' >/dev/null 2>&1; do sleep 1; done'
# Verify the Kubernetes version matches what we expect (retry for up to 20 seconds)
timeout 20 sh -ec 'until kubectl --kubeconfig tenantkubeconfig-'"${test_name}"' version 2>/dev/null | grep -Fq "Server Version: ${k8s_version}"; do sleep 5; done'
timeout 20 sh -ec 'until kubectl --kubeconfig tenantkubeconfig-'"${test_name}"' version 2>/dev/null | grep -Fq "Server Version: ${k8s_version}"; do sleep 1; done'
# Wait for at least 2 nodes to join (timeout after 8 minutes)
timeout 8m bash -c '
@ -102,10 +109,11 @@ EOF
done
'
# Verify the nodes are ready
if ! kubectl --kubeconfig "tenantkubeconfig-${test_name}" wait node --all --timeout=2m --for=condition=Ready; then
# Additional debug messages
if ! kubectl --kubeconfig "tenantkubeconfig-${test_name}" wait node --all --timeout=3m --for=condition=Ready; then
# Dump debug info and fail fast — no point running LB/NFS tests without Ready nodes
kubectl --kubeconfig "tenantkubeconfig-${test_name}" describe nodes
kubectl -n tenant-test get hr
exit 1
fi
kubectl --kubeconfig "tenantkubeconfig-${test_name}" get nodes -o wide
@ -206,24 +214,28 @@ EOF
done
"
LB_ADDR=$(
kubectl get svc --kubeconfig "tenantkubeconfig-${test_name}" "${test_name}-backend" \
-n tenant-test \
-o jsonpath='{.status.loadBalancer.ingress[0].ip}{.status.loadBalancer.ingress[0].hostname}'
)
LB_ADDR=$(
kubectl get svc --kubeconfig "tenantkubeconfig-${test_name}" "${test_name}-backend" \
-n tenant-test \
-o jsonpath='{.status.loadBalancer.ingress[0].ip}{.status.loadBalancer.ingress[0].hostname}'
)
if [ -z "$LB_ADDR" ]; then
echo "LoadBalancer address is empty" >&2
exit 1
fi
if [ -z "$LB_ADDR" ]; then
echo "LoadBalancer address is empty" >&2
exit 1
fi
lb_ok=false
for i in $(seq 1 20); do
echo "Attempt $i"
curl --silent --fail "http://${LB_ADDR}" && break
if curl --silent --fail "http://${LB_ADDR}"; then
lb_ok=true
break
fi
sleep 3
done
if [ "$i" -eq 20 ]; then
if [ "$lb_ok" != true ]; then
echo "LoadBalancer not reachable" >&2
exit 1
fi
@ -254,8 +266,8 @@ spec:
storage: 1Gi
EOF
# Wait for PVC to be bound
kubectl --kubeconfig "tenantkubeconfig-${test_name}" wait pvc nfs-test-pvc -n tenant-test --timeout=2m --for=jsonpath='{.status.phase}'=Bound
# Wait for PVC to be bound (RWX via kubevirt CSI provisions an NFS server pod, needs time)
kubectl --kubeconfig "tenantkubeconfig-${test_name}" wait pvc nfs-test-pvc -n tenant-test --timeout=3m --for=jsonpath='{.status.phase}'=Bound
# Create Pod that writes and reads data from NFS volume
kubectl --kubeconfig "tenantkubeconfig-${test_name}" apply -f - <<EOF
@ -280,7 +292,12 @@ spec:
EOF
# Wait for Pod to complete successfully
kubectl --kubeconfig "tenantkubeconfig-${test_name}" wait pod nfs-test-pod -n tenant-test --timeout=5m --for=jsonpath='{.status.phase}'=Succeeded
if ! kubectl --kubeconfig "tenantkubeconfig-${test_name}" wait pod nfs-test-pod -n tenant-test --timeout=5m --for=jsonpath='{.status.phase}'=Succeeded; then
echo "=== NFS test pod did not complete ===" >&2
kubectl --kubeconfig "tenantkubeconfig-${test_name}" describe pod nfs-test-pod -n tenant-test >&2 || true
kubectl --kubeconfig "tenantkubeconfig-${test_name}" get events -n tenant-test --sort-by='.lastTimestamp' >&2 || true
exit 1
fi
# Verify NFS data integrity
nfs_result=$(kubectl --kubeconfig "tenantkubeconfig-${test_name}" logs nfs-test-pod -n tenant-test)
@ -303,7 +320,9 @@ EOF
done
kubectl wait hr kubernetes-${test_name}-ingress-nginx -n tenant-test --timeout=5m --for=condition=ready
# Clean up by deleting the Kubernetes resource
kubectl -n tenant-test delete kuberneteses.apps.cozystack.io $test_name
# Clean up
pkill -f "port-forward.*${port}:" 2>/dev/null || true
rm -f "tenantkubeconfig-${test_name}"
kubectl -n tenant-test delete kuberneteses.apps.cozystack.io "${test_name}" --ignore-not-found --wait=false 2>/dev/null || true
}

3
hack/e2e-apps/vminstance.bats
View file

@ -2,6 +2,8 @@
@test "Create a VM Disk" {
name='test'
kubectl -n tenant-test delete vminstances.apps.cozystack.io $name --ignore-not-found --timeout=2m || true
kubectl -n tenant-test delete vmdisks.apps.cozystack.io $name --ignore-not-found --timeout=2m || true
kubectl apply -f - <<EOF
apiVersion: apps.cozystack.io/v1alpha1
kind: VMDisk
@ -25,6 +27,7 @@ EOF
@test "Create a VM Instance" {
diskName='test'
name='test'
kubectl -n tenant-test delete vminstances.apps.cozystack.io $name --ignore-not-found --timeout=2m || true
kubectl apply -f - <<EOF
apiVersion: apps.cozystack.io/v1alpha1
kind: VMInstance

4
hack/e2e-install-cozystack.bats
View file

@ -65,8 +65,8 @@ EOF
@test "Wait for Cluster‑API provider deployments" {
# Wait for Cluster‑API provider deployments
timeout 60 sh -ec 'until kubectl get deploy -n cozy-cluster-api capi-controller-manager capi-kamaji-controller-manager capi-kubeadm-bootstrap-controller-manager capi-operator-cluster-api-operator capk-controller-manager >/dev/null 2>&1; do sleep 1; done'
kubectl wait deployment/capi-controller-manager deployment/capi-kamaji-controller-manager deployment/capi-kubeadm-bootstrap-controller-manager deployment/capi-operator-cluster-api-operator deployment/capk-controller-manager -n cozy-cluster-api --timeout=1m --for=condition=available
timeout 120 sh -ec 'until kubectl get deploy -n cozy-cluster-api capi-controller-manager capi-kamaji-controller-manager capi-kubeadm-bootstrap-controller-manager capi-operator-cluster-api-operator capk-controller-manager >/dev/null 2>&1; do sleep 1; done'
kubectl wait deployment/capi-controller-manager deployment/capi-kamaji-controller-manager deployment/capi-kubeadm-bootstrap-controller-manager deployment/capi-operator-cluster-api-operator deployment/capk-controller-manager -n cozy-cluster-api --timeout=2m --for=condition=available
}
@test "Wait for LINSTOR and configure storage" {