diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml index d9898616..2e276ab2 100644 --- a/.github/workflows/pull-requests.yaml +++ b/.github/workflows/pull-requests.yaml @@ -227,27 +227,18 @@ jobs: until make SANDBOX_NAME=$SANDBOX_NAME prepare-env; do attempt=$((attempt + 1)) if [ $attempt -ge 3 ]; then - echo "❌ Attempt $attempt failed, exiting..." + echo "Attempt $attempt failed, exiting..." exit 1 fi - echo "❌ Attempt $attempt failed, retrying..." + echo "Attempt $attempt failed, retrying..." done - echo "✅ The task completed successfully after $attempt attempts" + echo "Prepare environment completed after $attempt attempts" # ▸ Install Cozystack - name: Install Cozystack into sandbox run: | cd /tmp/$SANDBOX_NAME - attempt=0 - until make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME install-cozystack; do - attempt=$((attempt + 1)) - if [ $attempt -ge 3 ]; then - echo "❌ Attempt $attempt failed, exiting..." - exit 1 - fi - echo "❌ Attempt $attempt failed, retrying..." - done - echo "✅ The task completed successfully after $attempt attempts" + make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME install-cozystack - name: Run OpenAPI tests run: | @@ -262,23 +253,18 @@ jobs: failed_tests="" for app in $(ls hack/e2e-apps/*.bats | xargs -n1 basename | cut -d. -f1); do echo "::group::Testing $app" - attempt=0 - success=false - until [ $attempt -ge 3 ]; do - if make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-apps-$app; then - success=true - break - fi - attempt=$((attempt + 1)) - echo "❌ Attempt $attempt failed, retrying..." - done - if [ "$success" = true ]; then + if make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-apps-$app; then + echo "::endgroup::" echo "✅ Test $app completed successfully" else - echo "❌ Test $app failed after $attempt attempts" + echo "::endgroup::" + echo "❌ Test $app failed (no retry — see diagnostics below)" failed_tests="$failed_tests $app" + echo "::group::Diagnostics for $app" + kubectl get hr -A -o wide 2>&1 | tail -50 || true + kubectl get events -A --sort-by=.lastTimestamp 2>&1 | tail -30 || true + echo "::endgroup::" fi - echo "::endgroup::" done if [ -n "$failed_tests" ]; then echo "❌ Failed tests:$failed_tests" diff --git a/cmd/cozystack-operator/main.go b/cmd/cozystack-operator/main.go index e215b779..08f7a860 100644 --- a/cmd/cozystack-operator/main.go +++ b/cmd/cozystack-operator/main.go @@ -83,6 +83,11 @@ func main() { var disableTelemetry bool var telemetryEndpoint string var telemetryInterval string + var helmReleaseInterval string + var helmReleaseRetryInterval string + var helmReleaseInstallTimeout string + var helmReleaseUpgradeTimeout string + var helmReleaseMaxHistory int var cozyValuesSecretName string var cozyValuesSecretNamespace string var cozyValuesNamespaceSelector string @@ -107,6 +112,28 @@ func main() { "Endpoint for sending telemetry data") flag.StringVar(&telemetryInterval, "telemetry-interval", "15m", "Interval between telemetry data collection (e.g. 15m, 1h)") + flag.StringVar(&helmReleaseInterval, "helmrelease-interval", "5m", + "Reconcile interval applied to HelmReleases created by the Package reconciler. "+ + "Lower values speed up dependency-blocked retries (e.g. during E2E install) at the cost of "+ + "controller load. Production default 5m matches existing behaviour.") + flag.StringVar(&helmReleaseRetryInterval, "helmrelease-retry-interval", "30s", + "Retry interval applied to Install.Strategy and Upgrade.Strategy of HelmReleases created "+ + "by the Package reconciler. With Strategy.Name=RetryOnFailure, this controls how long the "+ + "controller waits between failed install/upgrade attempts. Decoupled from --helmrelease-interval "+ + "(which is the healthy reconcile cadence) so failures recover fast without polling healthy "+ + "releases at the same fast cadence.") + flag.StringVar(&helmReleaseInstallTimeout, "helmrelease-install-timeout", "10m", + "Timeout for the Helm install action of HelmReleases created by the Package reconciler "+ + "(Spec.Install.Timeout). Bounds how long an individual Kubernetes operation (Job/hook/wait) "+ + "may take during install.") + flag.StringVar(&helmReleaseUpgradeTimeout, "helmrelease-upgrade-timeout", "10m", + "Timeout for the Helm upgrade action of HelmReleases created by the Package reconciler "+ + "(Spec.Upgrade.Timeout). Bounds how long an individual Kubernetes operation (Job/hook/wait) "+ + "may take during upgrade.") + flag.IntVar(&helmReleaseMaxHistory, "helmrelease-max-history", 5, + "Number of release revisions Helm keeps for HelmReleases created by the Package reconciler "+ + "(Spec.MaxHistory). 0 means unlimited; 5 matches Helm's default. Lower values reduce "+ + "per-release Secret accumulation in clusters that bounce HRs frequently (e.g. E2E sandboxes).") flag.StringVar(&platformSourceURL, "platform-source-url", "", "Platform source URL (oci:// or https://). If specified, generates OCIRepository or GitRepository resource.") flag.StringVar(&platformSourceName, "platform-source-name", "cozystack-platform", "Name for the generated platform source resource and PackageSource") flag.StringVar(&platformSourceRef, "platform-source-ref", "", "Reference specification as key=value pairs (e.g., 'branch=main' or 'digest=sha256:...,tag=v1.0'). For OCI: digest, semver, semverFilter, tag. For Git: branch, tag, semver, name, commit.") @@ -122,6 +149,31 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + hrIntervalDuration, err := time.ParseDuration(helmReleaseInterval) + if err != nil { + setupLog.Error(err, "invalid --helmrelease-interval value", "value", helmReleaseInterval) + os.Exit(1) + } + hrRetryIntervalDuration, err := time.ParseDuration(helmReleaseRetryInterval) + if err != nil { + setupLog.Error(err, "invalid --helmrelease-retry-interval value", "value", helmReleaseRetryInterval) + os.Exit(1) + } + hrInstallTimeoutDuration, err := time.ParseDuration(helmReleaseInstallTimeout) + if err != nil { + setupLog.Error(err, "invalid --helmrelease-install-timeout value", "value", helmReleaseInstallTimeout) + os.Exit(1) + } + hrUpgradeTimeoutDuration, err := time.ParseDuration(helmReleaseUpgradeTimeout) + if err != nil { + setupLog.Error(err, "invalid --helmrelease-upgrade-timeout value", "value", helmReleaseUpgradeTimeout) + os.Exit(1) + } + if helmReleaseMaxHistory < 0 { + setupLog.Error(fmt.Errorf("--helmrelease-max-history must be >= 0"), "invalid value", "value", helmReleaseMaxHistory) + os.Exit(1) + } + config := ctrl.GetConfigOrDie() // Create a direct client (without cache) for pre-start operations @@ -258,8 +310,13 @@ func main() { // Setup Package reconciler if err := (&operator.PackageReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + HelmReleaseInterval: hrIntervalDuration, + HelmReleaseRetryInterval: hrRetryIntervalDuration, + HelmReleaseInstallTimeout: hrInstallTimeoutDuration, + HelmReleaseUpgradeTimeout: hrUpgradeTimeoutDuration, + HelmReleaseMaxHistory: helmReleaseMaxHistory, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Package") os.Exit(1) diff --git a/hack/cozyreport-summary.sh b/hack/cozyreport-summary.sh new file mode 100755 index 00000000..d90682c5 --- /dev/null +++ b/hack/cozyreport-summary.sh @@ -0,0 +1,66 @@ +#!/bin/sh +# Emit a human-readable summary of "what is broken" to a single file. +# Reads the live cluster (not the report dir) so it can use kubectl JSONPath. +# Usage: cozyreport-summary.sh > summary.txt +set -eu + +echo "# Cozystack E2E Diagnostic Summary" +echo "Generated: $(date -Iseconds)" +echo + +echo "## HelmReleases not Ready" +echo +kubectl get hr -A --no-headers 2>/dev/null \ + | awk '$4 != "True" {printf " %s/%s — %s\n", $1, $2, $5}' \ + | head -40 +echo + +echo "## Pods not Running/Succeeded" +echo +kubectl get pod -A --no-headers 2>/dev/null \ + | awk '$4 !~ /Running|Succeeded|Completed/ {printf " %s/%s — %s (restarts=%s, age=%s)\n", $1, $2, $4, $5, $6}' \ + | head -40 +echo + +echo "## ImagePullBackOff / ErrImagePull" +echo +kubectl get pod -A --no-headers 2>/dev/null \ + | awk '$4 ~ /ImagePullBackOff|ErrImagePull/ {printf " %s/%s — %s\n", $1, $2, $4}' +echo + +echo "## OOMKilled in last 30 min" +echo +kubectl get events -A --field-selector reason=OOMKilling --sort-by=.lastTimestamp 2>/dev/null \ + | tail -20 +echo + +echo "## Recent Warning events (top 30)" +echo +kubectl get events -A --field-selector type=Warning --sort-by=.lastTimestamp 2>/dev/null \ + | tail -30 +echo + +echo "## cert-manager: Certificates not Ready" +echo +if kubectl get crd certificates.cert-manager.io >/dev/null 2>&1; then + kubectl get certificates.cert-manager.io -A --no-headers 2>/dev/null \ + | awk '$3 != "True" {printf " %s/%s — Ready=%s\n", $1, $2, $3}' +fi +echo + +echo "## Flux Sources not Ready" +echo +for kind in helmrepositories.source.toolkit.fluxcd.io ocirepositories.source.toolkit.fluxcd.io gitrepositories.source.toolkit.fluxcd.io; do + kubectl get $kind -A --no-headers 2>/dev/null \ + | awk -v k="${kind%%.*}" '$4 != "True" {printf " %s %s/%s — Ready=%s\n", k, $1, $2, $4}' +done +echo + +echo "## Storage: PVCs not Bound, PVs not Bound" +echo +kubectl get pvc -A --no-headers 2>/dev/null | awk '$3 != "Bound" {printf " PVC %s/%s — %s\n", $1, $2, $3}' +kubectl get pv --no-headers 2>/dev/null | awk '$5 != "Bound" {printf " PV %s — %s\n", $1, $5}' +echo + +echo "## Node Conditions" +kubectl get nodes -o custom-columns=NAME:.metadata.name,READY:.status.conditions[?\(@.type==\"Ready\"\)].status,DISK:.status.conditions[?\(@.type==\"DiskPressure\"\)].status,MEM:.status.conditions[?\(@.type==\"MemoryPressure\"\)].status 2>/dev/null diff --git a/hack/cozyreport.sh b/hack/cozyreport.sh index 565fd435..ba65c382 100755 --- a/hack/cozyreport.sh +++ b/hack/cozyreport.sh @@ -9,10 +9,13 @@ command -V kubectl >/dev/null || exit $? command -V tar >/dev/null || exit $? # -- cozystack module - echo "Collecting Cozystack information..." mkdir -p $REPORT_DIR/cozystack kubectl get deploy -n cozy-system cozystack -o jsonpath='{.spec.template.spec.containers[0].image}' > $REPORT_DIR/cozystack/image.txt 2>&1 +if kubectl get deploy -n cozy-system cozystack-operator >/dev/null 2>&1; then + kubectl logs -n cozy-system deploy/cozystack-operator --tail=2000 > $REPORT_DIR/cozystack/operator.log 2>&1 + kubectl logs -n cozy-system deploy/cozystack-operator --tail=2000 --previous > $REPORT_DIR/cozystack/operator-previous.log 2>&1 || true +fi kubectl get cm -n cozy-system --no-headers | awk '$1 ~ /^cozystack/' | while read NAME _; do DIR=$REPORT_DIR/cozystack/configs @@ -20,6 +23,46 @@ kubectl get cm -n cozy-system --no-headers | awk '$1 ~ /^cozystack/' | kubectl get cm -n cozy-system $NAME -o yaml > $DIR/$NAME.yaml 2>&1 done +# -- flux module +echo "Collecting Flux controller state..." +mkdir -p $REPORT_DIR/flux +for ctrl in helm-controller source-controller notification-controller kustomize-controller; do + if kubectl get deploy -n cozy-fluxcd $ctrl >/dev/null 2>&1; then + kubectl logs -n cozy-fluxcd deploy/$ctrl --tail=2000 > $REPORT_DIR/flux/$ctrl.log 2>&1 + kubectl logs -n cozy-fluxcd deploy/$ctrl --tail=2000 --previous > $REPORT_DIR/flux/$ctrl-previous.log 2>&1 || true + fi +done + +echo "Collecting Flux sources..." +for kind in helmrepositories.source.toolkit.fluxcd.io ocirepositories.source.toolkit.fluxcd.io gitrepositories.source.toolkit.fluxcd.io externalartifacts.source.toolkit.fluxcd.io; do + short=${kind%%.*} + kubectl get $kind -A > $REPORT_DIR/flux/$short.txt 2>&1 + kubectl get $kind -A -o yaml > $REPORT_DIR/flux/$short.yaml 2>&1 +done + +# -- cert-manager module +if kubectl get crd certificates.cert-manager.io >/dev/null 2>&1; then + echo "Collecting cert-manager state..." + DIR=$REPORT_DIR/cert-manager + mkdir -p $DIR + kubectl get certificates.cert-manager.io -A > $DIR/certificates.txt 2>&1 + kubectl get certificaterequests.cert-manager.io -A > $DIR/certificaterequests.txt 2>&1 + kubectl get orders.acme.cert-manager.io -A > $DIR/orders.txt 2>&1 + kubectl get challenges.acme.cert-manager.io -A > $DIR/challenges.txt 2>&1 + # Per non-Ready cert: full yaml + describe + kubectl get certificates.cert-manager.io -A --no-headers 2>/dev/null | awk '$3 != "True"' | \ + while read NAMESPACE NAME _; do + cdir=$DIR/certificates/$NAMESPACE/$NAME + mkdir -p $cdir + kubectl get certificates.cert-manager.io -n $NAMESPACE $NAME -o yaml > $cdir/cert.yaml 2>&1 + kubectl describe certificates.cert-manager.io -n $NAMESPACE $NAME > $cdir/describe.txt 2>&1 + done + if kubectl get deploy -n cozy-cert-manager cert-manager >/dev/null 2>&1; then + kubectl logs -n cozy-cert-manager deploy/cert-manager --tail=2000 > $DIR/cert-manager.log 2>&1 + kubectl logs -n cozy-cert-manager deploy/cert-manager-webhook --tail=2000 > $DIR/cert-manager-webhook.log 2>&1 + fi +fi + # -- kubernetes module echo "Collecting Kubernetes information..." @@ -46,6 +89,13 @@ kubectl get ns --no-headers | awk '$2 != "Active"' | kubectl describe ns $NAME > $DIR/describe.txt 2>&1 done +echo "Collecting events..." +kubectl get events -A --sort-by=.lastTimestamp > $REPORT_DIR/kubernetes/events.txt 2>&1 +# Filter to warning-class and recent for quick triage +kubectl get events -A --sort-by=.lastTimestamp \ + -o jsonpath='{range .items[?(@.type!="Normal")]}{.lastTimestamp}{"\t"}{.involvedObject.namespace}/{.involvedObject.kind}/{.involvedObject.name}{"\t"}{.reason}{"\t"}{.message}{"\n"}{end}' \ + > $REPORT_DIR/kubernetes/events-warnings.txt 2>&1 + echo "Collecting helmreleases..." kubectl get hr -A > $REPORT_DIR/kubernetes/helmreleases.txt 2>&1 kubectl get hr -A --no-headers | awk '$4 != "True"' | \ @@ -54,6 +104,13 @@ kubectl get hr -A --no-headers | awk '$4 != "True"' | \ mkdir -p $DIR kubectl get hr -n $NAMESPACE $NAME -o yaml > $DIR/hr.yaml 2>&1 kubectl describe hr -n $NAMESPACE $NAME > $DIR/describe.txt 2>&1 + # Helm storage secrets: latest revision per release. + kubectl get secret -n $NAMESPACE -l owner=helm,name=$NAME --sort-by='.metadata.creationTimestamp' --no-headers 2>/dev/null | \ + tail -1 | awk '{print $1}' | while read SECRET; do + [ -z "$SECRET" ] && continue + kubectl get secret -n $NAMESPACE $SECRET -o jsonpath='{.data.release}' 2>/dev/null \ + | base64 -d | base64 -d | gzip -d > $DIR/helm-release.json 2>&1 || true + done done echo "Collecting packages..." @@ -76,6 +133,23 @@ kubectl get packagesources --no-headers | awk '$3 != "True"' | \ kubectl describe packagesource $NAME > $DIR/describe.txt 2>&1 done +echo "Collecting cozystack apps..." +DIR=$REPORT_DIR/cozystack-apps +mkdir -p $DIR +for kind in applications.apps.cozystack.io applicationdefinitions.apps.cozystack.io tenants.apps.cozystack.io; do + short=${kind%%.*} + if kubectl get crd $kind >/dev/null 2>&1; then + kubectl get $kind -A > $DIR/$short.txt 2>&1 + kubectl get $kind -A --no-headers 2>/dev/null | awk 'NF >= 3 && $NF != "True" && $NF != "Ready"' | \ + while read NAMESPACE NAME _; do + d=$DIR/$short/$NAMESPACE/$NAME + mkdir -p $d + kubectl get $kind -n $NAMESPACE $NAME -o yaml > $d/$short.yaml 2>&1 + kubectl describe $kind -n $NAMESPACE $NAME > $d/describe.txt 2>&1 + done + fi +done + echo "Collecting pods..." kubectl get pod -A -o wide > $REPORT_DIR/kubernetes/pods.txt 2>&1 kubectl get pod -A --no-headers | awk '$4 !~ /Running|Succeeded|Completed/' | @@ -157,8 +231,28 @@ if kubectl get deploy -n cozy-linstor linstor-controller >/dev/null 2>&1; then kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor --no-color r l > $DIR/resources.txt 2>&1 fi +# -- sandbox-host module + +echo "Collecting sandbox host context..." +DIR=$REPORT_DIR/sandbox-host +mkdir -p $DIR +df -h > $DIR/df.txt 2>&1 +free -m > $DIR/free.txt 2>&1 +ps auxww > $DIR/ps.txt 2>&1 +dmesg | tail -200 > $DIR/dmesg.txt 2>&1 || true +if [ -f /workspace/talosconfig ]; then + for node in 192.168.123.11 192.168.123.12 192.168.123.13; do + talosctl --talosconfig /workspace/talosconfig -n $node dmesg --tail=200 > $DIR/talos-$node-dmesg.txt 2>&1 || true + talosctl --talosconfig /workspace/talosconfig -n $node logs kubelet --tail=500 > $DIR/talos-$node-kubelet.log 2>&1 || true + talosctl --talosconfig /workspace/talosconfig -n $node logs containerd --tail=500 > $DIR/talos-$node-containerd.log 2>&1 || true + done +fi + # -- finalization +echo "Generating summary..." +hack/cozyreport-summary.sh > $REPORT_DIR/summary.txt 2>&1 || true + echo "Creating archive..." tar -czf $REPORT_NAME.tgz -C $REPORT_PDIR . echo "Report created: $REPORT_NAME.tgz" diff --git a/hack/e2e-apps/bucket.bats b/hack/e2e-apps/bucket.bats index 31e7efb0..a4a349d7 100644 --- a/hack/e2e-apps/bucket.bats +++ b/hack/e2e-apps/bucket.bats @@ -20,8 +20,11 @@ EOF # Wait for the bucket to be ready kubectl -n tenant-test wait hr bucket-${name} --timeout=100s --for=condition=ready + timeout 60 sh -ec "until kubectl -n tenant-test get bucketclaims.objectstorage.k8s.io bucket-${name} >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait bucketclaims.objectstorage.k8s.io bucket-${name} --timeout=300s --for=jsonpath='{.status.bucketReady}' + timeout 60 sh -ec "until kubectl -n tenant-test get bucketaccesses.objectstorage.k8s.io bucket-${name}-admin >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait bucketaccesses.objectstorage.k8s.io bucket-${name}-admin --timeout=300s --for=jsonpath='{.status.accessGranted}' + timeout 60 sh -ec "until kubectl -n tenant-test get bucketaccesses.objectstorage.k8s.io bucket-${name}-viewer >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait bucketaccesses.objectstorage.k8s.io bucket-${name}-viewer --timeout=300s --for=jsonpath='{.status.accessGranted}' # Get admin (readwrite) credentials diff --git a/hack/e2e-apps/clickhouse.bats b/hack/e2e-apps/clickhouse.bats index fbc9f906..733da0ff 100644 --- a/hack/e2e-apps/clickhouse.bats +++ b/hack/e2e-apps/clickhouse.bats @@ -35,9 +35,12 @@ spec: resources: {} resourcesPreset: "nano" EOF - sleep 5 + # Wait for the operator to materialise the HelmRelease before kubectl wait + # kicks in (kubectl wait errors immediately if the object does not exist yet). + timeout 60 sh -ec "until kubectl -n tenant-test get hr clickhouse-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait hr clickhouse-$name --timeout=20s --for=condition=ready timeout 180 sh -ec "until kubectl -n tenant-test get svc chendpoint-clickhouse-$name -o jsonpath='{.spec.ports[*].port}' | grep -q '8123 9000'; do sleep 10; done" + timeout 60 sh -ec "until kubectl -n tenant-test get statefulset.apps/chi-clickhouse-$name-clickhouse-0-0 >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait statefulset.apps/chi-clickhouse-$name-clickhouse-0-0 --timeout=120s --for=jsonpath='{.status.replicas}'=1 timeout 80 sh -ec "until kubectl -n tenant-test get endpoints chi-clickhouse-$name-clickhouse-0-0 -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done" timeout 100 sh -ec "until kubectl -n tenant-test get svc chi-clickhouse-$name-clickhouse-0-0 -o jsonpath='{.spec.ports[*].port}' | grep -q '9000 8123 9009'; do sleep 10; done" diff --git a/hack/e2e-apps/external-dns.bats b/hack/e2e-apps/external-dns.bats index ca114af8..5775927f 100644 --- a/hack/e2e-apps/external-dns.bats +++ b/hack/e2e-apps/external-dns.bats @@ -18,7 +18,9 @@ spec: - example.com EOF - sleep 5 + # Wait for the operator to materialise the HelmRelease before kubectl wait + # kicks in (kubectl wait errors immediately if the object does not exist yet). + timeout 60 sh -ec "until kubectl -n tenant-test get hr ${name} >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait hr ${name} --timeout=120s --for=condition=ready kubectl -n tenant-test wait hr ${name}-system --timeout=120s --for=condition=ready timeout 60 sh -ec "until kubectl -n tenant-test get deploy -l app.kubernetes.io/instance=${name}-system -o jsonpath='{.items[0].status.readyReplicas}' | grep -q '1'; do sleep 5; done" @@ -41,7 +43,9 @@ spec: - example.org EOF - sleep 5 + # Wait for the operator to materialise the HelmRelease before kubectl wait + # kicks in (kubectl wait errors immediately if the object does not exist yet). + timeout 60 sh -ec "until kubectl -n tenant-test get hr ${name} >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait hr ${name} --timeout=120s --for=condition=ready kubectl -n tenant-test wait hr ${name}-system --timeout=120s --for=condition=ready timeout 60 sh -ec "until kubectl -n tenant-test get deploy -l app.kubernetes.io/instance=${name}-system -o jsonpath='{.items[0].status.readyReplicas}' | grep -q '1'; do sleep 5; done" diff --git a/hack/e2e-apps/foundationdb.bats b/hack/e2e-apps/foundationdb.bats index 199bda11..accfd8e1 100644 --- a/hack/e2e-apps/foundationdb.bats +++ b/hack/e2e-apps/foundationdb.bats @@ -42,7 +42,9 @@ spec: imageType: "unified" automaticReplacements: true EOF - sleep 15 + # Wait for the operator to materialise the HelmRelease before kubectl wait + # kicks in (kubectl wait errors immediately if the object does not exist yet). + timeout 60 sh -ec "until kubectl -n tenant-test get hr foundationdb-$name >/dev/null 2>&1; do sleep 2; done" # Wait for HelmRelease to be ready kubectl -n tenant-test wait hr foundationdb-$name --timeout=300s --for=condition=ready diff --git a/hack/e2e-apps/harbor.bats b/hack/e2e-apps/harbor.bats index 26f407ea..290fef42 100644 --- a/hack/e2e-apps/harbor.bats +++ b/hack/e2e-apps/harbor.bats @@ -38,12 +38,16 @@ spec: size: 1Gi replicas: 1 EOF - sleep 5 + # Wait for the operator to materialise the HelmRelease before kubectl wait + # kicks in (kubectl wait errors immediately if the object does not exist yet). + timeout 60 sh -ec "until kubectl -n tenant-test get hr $release >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait hr $release --timeout=60s --for=condition=ready # Wait for COSI to provision bucket + timeout 60 sh -ec "until kubectl -n tenant-test get bucketclaims.objectstorage.k8s.io $release-registry >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait bucketclaims.objectstorage.k8s.io $release-registry \ --timeout=300s --for=jsonpath='{.status.bucketReady}'=true + timeout 60 sh -ec "until kubectl -n tenant-test get bucketaccesses.objectstorage.k8s.io $release-registry >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait bucketaccesses.objectstorage.k8s.io $release-registry \ --timeout=60s --for=jsonpath='{.status.accessGranted}'=true @@ -64,8 +68,11 @@ EOF kubectl -n tenant-test get secret $release-registry-bucket -o jsonpath='{.data.BucketInfo}' 2>&1 | base64 -d 2>&1 || true false } + timeout 60 sh -ec "until kubectl -n tenant-test get deploy $release-core >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait deploy $release-core --timeout=120s --for=condition=available + timeout 60 sh -ec "until kubectl -n tenant-test get deploy $release-registry >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait deploy $release-registry --timeout=120s --for=condition=available + timeout 60 sh -ec "until kubectl -n tenant-test get deploy $release-portal >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait deploy $release-portal --timeout=120s --for=condition=available kubectl -n tenant-test get secret $release-credentials -o jsonpath='{.data.admin-password}' | base64 --decode | grep -q '.' kubectl -n tenant-test get secret $release-credentials -o jsonpath='{.data.url}' | base64 --decode | grep -q 'https://' diff --git a/hack/e2e-apps/kafka.bats b/hack/e2e-apps/kafka.bats index d85837bd..c1f0adf1 100644 --- a/hack/e2e-apps/kafka.bats +++ b/hack/e2e-apps/kafka.bats @@ -39,8 +39,11 @@ spec: partitions: 1 replicas: 2 EOF - sleep 5 + # Wait for the operator to materialise the HelmRelease before kubectl wait + # kicks in (kubectl wait errors immediately if the object does not exist yet). + timeout 60 sh -ec "until kubectl -n tenant-test get hr kafka-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait hr kafka-$name --timeout=30s --for=condition=ready + timeout 60 sh -ec "until kubectl -n tenant-test get kafkas test >/dev/null 2>&1; do sleep 2; done" kubectl wait kafkas -n tenant-test test --timeout=300s --for=condition=ready timeout 60 sh -ec "until kubectl -n tenant-test get pvc data-kafka-$name-zookeeper-0; do sleep 10; done" kubectl -n tenant-test wait pvc data-kafka-$name-zookeeper-0 --timeout=50s --for=jsonpath='{.status.phase}'=Bound diff --git a/hack/e2e-apps/mariadb.bats b/hack/e2e-apps/mariadb.bats index 68f5fc00..35559216 100644 --- a/hack/e2e-apps/mariadb.bats +++ b/hack/e2e-apps/mariadb.bats @@ -35,13 +35,17 @@ spec: resources: {} resourcesPreset: "nano" EOF - sleep 5 + # Wait for the operator to materialise the HelmRelease before kubectl wait + # kicks in (kubectl wait errors immediately if the object does not exist yet). + timeout 60 sh -ec "until kubectl -n tenant-test get hr mariadb-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait hr mariadb-$name --timeout=30s --for=condition=ready timeout 80 sh -ec "until kubectl -n tenant-test get svc mariadb-$name -o jsonpath='{.spec.ports[0].port}' | grep -q '3306'; do sleep 10; done" timeout 80 sh -ec "until kubectl -n tenant-test get endpoints mariadb-$name -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done" + timeout 60 sh -ec "until kubectl -n tenant-test get statefulset.apps/mariadb-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait statefulset.apps/mariadb-$name --timeout=110s --for=jsonpath='{.status.replicas}'=2 timeout 80 sh -ec "until kubectl -n tenant-test get svc mariadb-$name-metrics -o jsonpath='{.spec.ports[0].port}' | grep -q '9104'; do sleep 10; done" timeout 40 sh -ec "until kubectl -n tenant-test get endpoints mariadb-$name-metrics -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done" + timeout 60 sh -ec "until kubectl -n tenant-test get deployment.apps/mariadb-$name-metrics >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait deployment.apps/mariadb-$name-metrics --timeout=90s --for=jsonpath='{.status.replicas}'=1 kubectl -n tenant-test delete mariadbs.apps.cozystack.io $name } diff --git a/hack/e2e-apps/mongodb.bats b/hack/e2e-apps/mongodb.bats index 7712fe01..bc83f7ab 100644 --- a/hack/e2e-apps/mongodb.bats +++ b/hack/e2e-apps/mongodb.bats @@ -26,7 +26,9 @@ spec: backup: enabled: false EOF - sleep 5 + # Wait for the operator to materialise the HelmRelease before kubectl wait + # kicks in (kubectl wait errors immediately if the object does not exist yet). + timeout 60 sh -ec "until kubectl -n tenant-test get hr mongodb-$name >/dev/null 2>&1; do sleep 2; done" # Wait for HelmRelease kubectl -n tenant-test wait hr mongodb-$name --timeout=60s --for=condition=ready # Wait for MongoDB service (port 27017) @@ -34,6 +36,7 @@ EOF # Wait for endpoints timeout 180 sh -ec "until kubectl -n tenant-test get endpoints mongodb-$name-rs0 -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done" # Wait for StatefulSet replicas + timeout 60 sh -ec "until kubectl -n tenant-test get statefulset.apps/mongodb-$name-rs0 >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait statefulset.apps/mongodb-$name-rs0 --timeout=300s --for=jsonpath='{.status.replicas}'=1 # Cleanup kubectl -n tenant-test delete mongodbs.apps.cozystack.io $name diff --git a/hack/e2e-apps/openbao.bats b/hack/e2e-apps/openbao.bats index 35271e55..168440ad 100644 --- a/hack/e2e-apps/openbao.bats +++ b/hack/e2e-apps/openbao.bats @@ -18,7 +18,9 @@ spec: external: false ui: true EOF - sleep 5 + # Wait for the operator to materialise the HelmRelease before kubectl wait + # kicks in (kubectl wait errors immediately if the object does not exist yet). + timeout 60 sh -ec "until kubectl -n tenant-test get hr openbao-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait hr openbao-$name --timeout=60s --for=condition=ready kubectl -n tenant-test wait hr openbao-$name-system --timeout=120s --for=condition=ready @@ -53,7 +55,9 @@ EOF kubectl -n tenant-test exec openbao-$name-0 -- bao operator unseal "$unseal_key" # Now wait for pod to become ready (readiness probe checks seal status) + timeout 60 sh -ec "until kubectl -n tenant-test get sts openbao-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait sts openbao-$name --timeout=90s --for=jsonpath='{.status.readyReplicas}'=1 + timeout 60 sh -ec "until kubectl -n tenant-test get pvc data-openbao-$name-0 >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait pvc data-openbao-$name-0 --timeout=50s --for=jsonpath='{.status.phase}'=Bound kubectl -n tenant-test delete openbao.apps.cozystack.io $name kubectl -n tenant-test delete pvc data-openbao-$name-0 --ignore-not-found diff --git a/hack/e2e-apps/postgres.bats b/hack/e2e-apps/postgres.bats index b86bc664..3ec79409 100644 --- a/hack/e2e-apps/postgres.bats +++ b/hack/e2e-apps/postgres.bats @@ -40,8 +40,11 @@ spec: resources: {} resourcesPreset: "nano" EOF - sleep 5 + # Wait for the operator to materialise the HelmRelease before kubectl wait + # kicks in (kubectl wait errors immediately if the object does not exist yet). + timeout 60 sh -ec "until kubectl -n tenant-test get hr postgres-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait hr postgres-$name --timeout=100s --for=condition=ready + timeout 60 sh -ec "until kubectl -n tenant-test get job.batch postgres-$name-init-job >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait job.batch postgres-$name-init-job --timeout=50s --for=condition=Complete timeout 40 sh -ec "until kubectl -n tenant-test get svc postgres-$name-r -o jsonpath='{.spec.ports[0].port}' | grep -q '5432'; do sleep 10; done" timeout 40 sh -ec "until kubectl -n tenant-test get svc postgres-$name-ro -o jsonpath='{.spec.ports[0].port}' | grep -q '5432'; do sleep 10; done" diff --git a/hack/e2e-apps/qdrant.bats b/hack/e2e-apps/qdrant.bats index 9d37ce01..8a889e43 100755 --- a/hack/e2e-apps/qdrant.bats +++ b/hack/e2e-apps/qdrant.bats @@ -17,10 +17,14 @@ spec: resources: {} external: false EOF - sleep 5 + # Wait for the operator to materialise the HelmRelease before kubectl wait + # kicks in (kubectl wait errors immediately if the object does not exist yet). + timeout 60 sh -ec "until kubectl -n tenant-test get hr qdrant-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait hr qdrant-$name --timeout=60s --for=condition=ready kubectl -n tenant-test wait hr qdrant-$name-system --timeout=120s --for=condition=ready + timeout 60 sh -ec "until kubectl -n tenant-test get sts qdrant-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait sts qdrant-$name --timeout=90s --for=jsonpath='{.status.readyReplicas}'=1 + timeout 60 sh -ec "until kubectl -n tenant-test get pvc qdrant-storage-qdrant-$name-0 >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait pvc qdrant-storage-qdrant-$name-0 --timeout=50s --for=jsonpath='{.status.phase}'=Bound kubectl -n tenant-test delete qdrant.apps.cozystack.io $name } diff --git a/hack/e2e-apps/redis.bats b/hack/e2e-apps/redis.bats index 3ddff71a..eb148701 100644 --- a/hack/e2e-apps/redis.bats +++ b/hack/e2e-apps/redis.bats @@ -18,10 +18,15 @@ spec: resources: {} resourcesPreset: "nano" EOF - sleep 5 + # Wait for the operator to materialise the HelmRelease before kubectl wait + # kicks in (kubectl wait errors immediately if the object does not exist yet). + timeout 60 sh -ec "until kubectl -n tenant-test get hr redis-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait hr redis-$name --timeout=20s --for=condition=ready + timeout 60 sh -ec "until kubectl -n tenant-test get pvc redisfailover-persistent-data-rfr-redis-$name-0 >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait pvc redisfailover-persistent-data-rfr-redis-$name-0 --timeout=50s --for=jsonpath='{.status.phase}'=Bound + timeout 60 sh -ec "until kubectl -n tenant-test get deploy rfs-redis-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait deploy rfs-redis-$name --timeout=90s --for=condition=available + timeout 60 sh -ec "until kubectl -n tenant-test get sts rfr-redis-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait sts rfr-redis-$name --timeout=90s --for=jsonpath='{.status.replicas}'=2 kubectl -n tenant-test delete redis.apps.cozystack.io $name } diff --git a/hack/e2e-apps/run-kubernetes.sh b/hack/e2e-apps/run-kubernetes.sh index 761ee57d..96975e16 100644 --- a/hack/e2e-apps/run-kubernetes.sh +++ b/hack/e2e-apps/run-kubernetes.sh @@ -227,6 +227,11 @@ EOF exit 1 fi + # TODO(e2e-replace-fixed-timeouts): genuine retry loop. This validates an + # external HTTP path (MetalLB-advertised LB IP -> in-tenant ingress -> + # backend pod) which is not visible to the Kubernetes API as a single + # condition, so kubectl wait cannot replace it. The 20x3s = 60s budget is + # capped with `lb_ok=false` then asserted below. lb_ok=false for i in $(seq 1 20); do echo "Attempt $i" diff --git a/hack/e2e-apps/vminstance.bats b/hack/e2e-apps/vminstance.bats index fa0f0b5b..2ff60271 100644 --- a/hack/e2e-apps/vminstance.bats +++ b/hack/e2e-apps/vminstance.bats @@ -2,8 +2,15 @@ @test "Create a VM Disk" { name='test' - kubectl -n tenant-test delete vminstances.apps.cozystack.io $name --ignore-not-found --timeout=2m || true - kubectl -n tenant-test delete vmdisks.apps.cozystack.io $name --ignore-not-found --timeout=2m || true + # Delete any leftover from a previous run and BLOCK until removal completes. + # `kubectl apply` on a resource still in finalizer-drain silently no-ops + # ("Detected changes to resource ... which is currently being deleted"), + # which then races a NotFound on the downstream HR wait. The previous + # `|| true` swallowed timeout errors here and let the test continue with + # the old VMDisk still draining. Bumped timeout to 3m and removed `|| true` + # so a true delete failure surfaces immediately. + kubectl -n tenant-test delete vminstances.apps.cozystack.io $name --ignore-not-found --timeout=3m + kubectl -n tenant-test delete vmdisks.apps.cozystack.io $name --ignore-not-found --timeout=3m kubectl apply -f - </dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait hr vm-disk-$name --timeout=5s --for=condition=ready + timeout 120 sh -ec "until kubectl -n tenant-test get dv vm-disk-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait dv vm-disk-$name --timeout=250s --for=condition=ready + timeout 120 sh -ec "until kubectl -n tenant-test get pvc vm-disk-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait pvc vm-disk-$name --timeout=200s --for=jsonpath='{.status.phase}'=Bound } @test "Create a VM Instance" { diskName='test' name='test' - kubectl -n tenant-test delete vminstances.apps.cozystack.io $name --ignore-not-found --timeout=2m || true + # Same delete-finalizer-drain race as in "Create a VM Disk" above — + # block until removal completes, surface timeouts loudly. + kubectl -n tenant-test delete vminstances.apps.cozystack.io $name --ignore-not-found --timeout=3m kubectl apply -f - </dev/null 2>&1; do sleep 2; done" + # Nested KubeVirt VM startup (virt-launcher + libvirt + cloud-init DHCP) + # routinely takes 30-60s under runner load; the previous 20s was unrealistic + # and produced flakes. 120s is a comfortable upper bound for nested virt. + timeout 120 sh -ec "until kubectl -n tenant-test get vmi vm-instance-$name -o jsonpath='{.status.interfaces[0].ipAddress}' | grep -q '[0-9]'; do sleep 2; done" kubectl -n tenant-test wait hr vm-instance-$name --timeout=5s --for=condition=ready - kubectl -n tenant-test wait vm vm-instance-$name --timeout=20s --for=condition=ready - kubectl -n tenant-test delete vminstances.apps.cozystack.io $name - kubectl -n tenant-test delete vmdisks.apps.cozystack.io $diskName + # VM ready follows IP assignment closely; 60s gives buffer for the qemu-guest-agent. + timeout 120 sh -ec "until kubectl -n tenant-test get vm vm-instance-$name >/dev/null 2>&1; do sleep 2; done" + kubectl -n tenant-test wait vm vm-instance-$name --timeout=60s --for=condition=ready + kubectl -n tenant-test delete vminstances.apps.cozystack.io $name --timeout=3m + kubectl -n tenant-test delete vmdisks.apps.cozystack.io $diskName --timeout=3m } diff --git a/hack/e2e-install-cozystack.bats b/hack/e2e-install-cozystack.bats index cf66b73f..1f995825 100644 --- a/hack/e2e-install-cozystack.bats +++ b/hack/e2e-install-cozystack.bats @@ -7,12 +7,52 @@ fi } +@test "Pre-pull platform images" { + # Cluster-member workloads (OVN raft, LINSTOR) fail if replicas start at + # different times due to image-pull stagger across nodes. Pre-pull these + # images to every node so all replicas start with images already cached. + # + # Source images directly from the rendered charts so version bumps stay in + # sync automatically. yq walks every PodSpec-shaped object and emits the + # images of each container — this scopes the result to images the kubelet + # actually pulls (skips configmap fields and CRD examples that happen to + # contain an `image:` key). Add a chart here when a new peer-sensitive + # workload is found. + { + helm template packages/system/kubeovn + helm template packages/system/linstor + } | yq -N ' + (..|select(has("containers"))|.containers[]|.image), + (..|select(has("initContainers"))|.initContainers[]|.image) + ' | hack/e2e-prepull-images.sh +} + @test "Install Cozystack" { - # Install cozy-installer chart (operator installs CRDs on startup via --install-crds) + # Pre-create the cozy-system namespace with the labels the operator pod needs. + # The chart no longer ships a Namespace resource (helm v3's --create-namespace + # collides with chart-defined Namespace). On Talos clusters that enforce PSA + # baseline-by-default, the cozystack-operator pod (hostNetwork=true) is + # rejected unless the namespace is labelled enforce=privileged BEFORE the + # pod is scheduled. A pre-install hook can't bootstrap that label because + # the hook itself runs in the same namespace and faces the same PSA gate. + # So apply the namespace from the caller (this bats / production install docs) + # — same pattern as kube-prometheus-stack, argo-cd, cert-manager. + kubectl apply -f - <<'EOF' +apiVersion: v1 +kind: Namespace +metadata: + name: cozy-system + labels: + cozystack.io/system: "true" + cozystack.io/deletion-protected: "true" + pod-security.kubernetes.io/enforce: privileged +EOF + + # Install cozy-installer chart (operator installs CRDs on startup via --install-crds). helm upgrade installer packages/core/installer \ --install \ --namespace cozy-system \ - --create-namespace \ + --set cozystackOperator.helmReleaseInterval=30s \ --wait \ --timeout 2m @@ -51,11 +91,31 @@ spec: - cozystack.external-dns-application EOF + # Launch storage + LB configuration in the background. It waits for its + # own prerequisites (linstor-controller deploy, MetalLB CRDs) and finishes + # while the parallel HR wait below is still running, so the cost overlaps + # with the platform reconcile instead of compounding it. + hack/e2e-post-install-prep.sh > /tmp/post-install-prep.log 2>&1 & + POST_PREP_PID=$! + # Wait until HelmReleases appear & reconcile them timeout 180 sh -ec 'until [ $(kubectl get hr -A --no-headers 2>/dev/null | wc -l) -gt 10 ]; do sleep 1; done' + # TODO(e2e-replace-fixed-timeouts): genuine sleep. The threshold of 10 is a + # heuristic for "enough HRs visible to start waiting"; the snapshot below + # uses whatever HRs have appeared by then. There is no objective k8s API + # signal for "all platform HRs have been emitted" without hard-coding the + # expected list, so the 5s pad lets a few late-arrivals join the snapshot. sleep 5 kubectl get hr -A | awk 'NR>1 {print "kubectl wait --timeout=15m --for=condition=ready -n "$1" hr/"$2" &"} END {print "wait"}' | sh -ex + echo "Waiting for post-install-prep to complete" + if ! wait $POST_PREP_PID; then + cat /tmp/post-install-prep.log >&2 + echo "post-install-prep failed" >&2 + exit 1 + fi + cat /tmp/post-install-prep.log + # Fail the test if any HelmRelease is not Ready if kubectl get hr -A | grep -v " True " | grep -v NAME; then kubectl get hr -A @@ -69,80 +129,8 @@ EOF kubectl wait deployment/capi-controller-manager deployment/capi-kamaji-controller-manager deployment/capi-kubeadm-bootstrap-controller-manager deployment/capi-operator-cluster-api-operator deployment/capk-controller-manager -n cozy-cluster-api --timeout=2m --for=condition=available } -@test "Wait for LINSTOR and configure storage" { - # Linstor controller and nodes - kubectl wait deployment/linstor-controller -n cozy-linstor --timeout=5m --for=condition=available - timeout 60 sh -ec 'until [ $(kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor node list | grep -c Online) -eq 3 ]; do sleep 1; done' - - created_pools=$(kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor sp l -s data --pastable | awk '$2 == "data" {printf " " $4} END{printf " "}') - for node in srv1 srv2 srv3; do - case $created_pools in - *" $node "*) echo "Storage pool 'data' already exists on node $node"; continue;; - esac - kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor ps cdp zfs ${node} /dev/vdc --pool-name data --storage-pool data - done - - # Storage classes - kubectl apply -f - <<'EOF' ---- -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: local - annotations: - storageclass.kubernetes.io/is-default-class: "true" -provisioner: linstor.csi.linbit.com -parameters: - linstor.csi.linbit.com/storagePool: "data" - linstor.csi.linbit.com/layerList: "storage" - linstor.csi.linbit.com/allowRemoteVolumeAccess: "false" -volumeBindingMode: WaitForFirstConsumer -allowVolumeExpansion: true ---- -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: replicated -provisioner: linstor.csi.linbit.com -parameters: - linstor.csi.linbit.com/storagePool: "data" - linstor.csi.linbit.com/autoPlace: "3" - linstor.csi.linbit.com/layerList: "drbd storage" - linstor.csi.linbit.com/allowRemoteVolumeAccess: "true" - property.linstor.csi.linbit.com/DrbdOptions/auto-quorum: suspend-io - property.linstor.csi.linbit.com/DrbdOptions/Resource/on-no-data-accessible: suspend-io - property.linstor.csi.linbit.com/DrbdOptions/Resource/on-suspended-primary-outdated: force-secondary - property.linstor.csi.linbit.com/DrbdOptions/Net/rr-conflict: retry-connect -volumeBindingMode: Immediate -allowVolumeExpansion: true -EOF -} - -@test "Wait for MetalLB and configure address pool" { - # MetalLB address pool - kubectl apply -f - <<'EOF' ---- -apiVersion: metallb.io/v1beta1 -kind: L2Advertisement -metadata: - name: cozystack - namespace: cozy-metallb -spec: - ipAddressPools: [cozystack] ---- -apiVersion: metallb.io/v1beta1 -kind: IPAddressPool -metadata: - name: cozystack - namespace: cozy-metallb -spec: - addresses: [192.168.123.200-192.168.123.250] - autoAssign: true - avoidBuggyIPs: false -EOF -} - @test "Check Cozystack API service" { + timeout 60 sh -ec 'until kubectl get apiservices/v1alpha1.apps.cozystack.io apiservices/v1alpha1.core.cozystack.io >/dev/null 2>&1; do sleep 2; done' kubectl wait --for=condition=Available apiservices/v1alpha1.apps.cozystack.io apiservices/v1alpha1.core.cozystack.io --timeout=2m } @@ -174,15 +162,22 @@ EOF kubectl wait deploy/root-ingress-controller -n tenant-root --timeout=5m --for=condition=available # etcd statefulset + timeout 60 sh -ec 'until kubectl get sts/etcd -n tenant-root >/dev/null 2>&1; do sleep 2; done' kubectl wait sts/etcd -n tenant-root --for=jsonpath='{.status.readyReplicas}'=3 --timeout=5m # VictoriaMetrics components + timeout 60 sh -ec 'until kubectl get vmalert/vmalert-shortterm -n tenant-root >/dev/null 2>&1; do sleep 2; done' + timeout 60 sh -ec 'until kubectl get vmalertmanager/alertmanager -n tenant-root >/dev/null 2>&1; do sleep 2; done' kubectl wait vmalert/vmalert-shortterm vmalertmanager/alertmanager -n tenant-root --for=jsonpath='{.status.updateStatus}'=operational --timeout=15m + timeout 60 sh -ec 'until kubectl get vlclusters/generic -n tenant-root >/dev/null 2>&1; do sleep 2; done' kubectl wait vlclusters/generic -n tenant-root --for=jsonpath='{.status.updateStatus}'=operational --timeout=5m + timeout 60 sh -ec 'until kubectl get vmcluster/shortterm vmcluster/longterm -n tenant-root >/dev/null 2>&1; do sleep 2; done' kubectl wait vmcluster/shortterm vmcluster/longterm -n tenant-root --for=jsonpath='{.status.updateStatus}'=operational --timeout=5m # Grafana + timeout 60 sh -ec 'until kubectl get clusters.postgresql.cnpg.io/grafana-db -n tenant-root >/dev/null 2>&1; do sleep 2; done' kubectl wait clusters.postgresql.cnpg.io/grafana-db -n tenant-root --for=condition=ready --timeout=5m + timeout 60 sh -ec 'until kubectl get deploy/grafana-deployment -n tenant-root >/dev/null 2>&1; do sleep 2; done' kubectl wait deploy/grafana-deployment -n tenant-root --for=condition=available --timeout=5m # Verify Grafana via ingress @@ -273,6 +268,7 @@ spec: seaweedfs: false EOF kubectl wait hr/tenant-test -n tenant-root --timeout=1m --for=condition=ready + timeout 60 sh -ec 'until kubectl get namespace tenant-test >/dev/null 2>&1; do sleep 2; done' kubectl wait namespace tenant-test --timeout=20s --for=jsonpath='{.status.phase}'=Active # Wait for ResourceQuota to appear and assert values timeout 60 sh -ec 'until [ "$(kubectl get quota -n tenant-test --no-headers 2>/dev/null | wc -l)" -ge 1 ]; do sleep 1; done' diff --git a/hack/e2e-post-install-prep.sh b/hack/e2e-post-install-prep.sh new file mode 100755 index 00000000..a5010503 --- /dev/null +++ b/hack/e2e-post-install-prep.sh @@ -0,0 +1,96 @@ +#!/bin/sh +# Runs LINSTOR pool + StorageClass + MetalLB pool configuration as soon as +# their respective prerequisites are reachable. Designed to run in the +# background during the platform HR reconcile wait, so its wall-clock cost +# overlaps with the wait instead of compounding it. +set -eu + +echo "[post-install-prep] waiting for linstor HelmRelease object to exist" +timeout 60 sh -ec 'until kubectl get hr/linstor -n cozy-linstor >/dev/null 2>&1; do sleep 2; done' + +echo "[post-install-prep] waiting for linstor HelmRelease to be Ready" +kubectl wait helmrelease/linstor -n cozy-linstor --for=condition=Ready --timeout=15m + +echo "[post-install-prep] waiting for linstor-controller deployment to exist" +timeout 300 sh -ec 'until kubectl get deploy/linstor-controller -n cozy-linstor >/dev/null 2>&1; do sleep 2; done' + +echo "[post-install-prep] waiting for linstor-controller to be Available" +kubectl wait deployment/linstor-controller -n cozy-linstor --timeout=5m --for=condition=available + +echo "[post-install-prep] waiting for 3 LINSTOR nodes Online" +# TODO(e2e-replace-fixed-timeouts): genuine poll. LINSTOR node membership is +# reported by the linstor binary inside the controller pod, not via a +# Kubernetes API condition, so kubectl wait cannot subscribe to it. +timeout 60 sh -ec 'until [ $(kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor node list | grep -c Online) -eq 3 ]; do sleep 1; done' + +echo "[post-install-prep] creating LINSTOR storage pools (parallel across nodes)" +created_pools=$(kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor sp l -s data --pastable | awk '$2 == "data" {printf " " $4} END{printf " "}') +for node in srv1 srv2 srv3; do + case $created_pools in + *" $node "*) echo " pool 'data' already exists on $node"; continue;; + esac + kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor ps cdp zfs ${node} /dev/vdc --pool-name data --storage-pool data & +done +wait + +echo "[post-install-prep] applying StorageClasses" +kubectl apply -f - <<'EOF' +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: local + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: linstor.csi.linbit.com +parameters: + linstor.csi.linbit.com/storagePool: "data" + linstor.csi.linbit.com/layerList: "storage" + linstor.csi.linbit.com/allowRemoteVolumeAccess: "false" +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: true +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: replicated +provisioner: linstor.csi.linbit.com +parameters: + linstor.csi.linbit.com/storagePool: "data" + linstor.csi.linbit.com/autoPlace: "3" + linstor.csi.linbit.com/layerList: "drbd storage" + linstor.csi.linbit.com/allowRemoteVolumeAccess: "true" + property.linstor.csi.linbit.com/DrbdOptions/auto-quorum: suspend-io + property.linstor.csi.linbit.com/DrbdOptions/Resource/on-no-data-accessible: suspend-io + property.linstor.csi.linbit.com/DrbdOptions/Resource/on-suspended-primary-outdated: force-secondary + property.linstor.csi.linbit.com/DrbdOptions/Net/rr-conflict: retry-connect +volumeBindingMode: Immediate +allowVolumeExpansion: true +EOF + +echo "[post-install-prep] waiting for MetalLB CRDs" +timeout 300 sh -ec 'until kubectl get crd ipaddresspools.metallb.io l2advertisements.metallb.io >/dev/null 2>&1; do sleep 2; done' + +echo "[post-install-prep] applying MetalLB IPAddressPool" +kubectl apply -f - <<'EOF' +--- +apiVersion: metallb.io/v1beta1 +kind: L2Advertisement +metadata: + name: cozystack + namespace: cozy-metallb +spec: + ipAddressPools: [cozystack] +--- +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: cozystack + namespace: cozy-metallb +spec: + addresses: [192.168.123.200-192.168.123.250] + autoAssign: true + avoidBuggyIPs: false +EOF + +echo "[post-install-prep] done" diff --git a/hack/e2e-prepull-images.sh b/hack/e2e-prepull-images.sh new file mode 100755 index 00000000..e90d3a1b --- /dev/null +++ b/hack/e2e-prepull-images.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Pre-pull images to all cluster nodes. +# +# Reads image references from stdin, one per line. Empty lines and lines +# starting with '#' are ignored. +# +# Some workloads (OVN raft, LINSTOR) are sensitive to peer-readiness at +# startup: if nodes pull the image at different speeds, one replica boots +# before its peers are reachable, exhausts its raft connection retries, and +# the HelmRelease install times out. Pre-pulling eliminates the pull stagger +# so all replicas start within milliseconds of each other. +# +# Implementation: a DaemonSet with one regular container per image (parallel +# pulls — total time = max of any one image rather than sum). kubectl rollout +# status blocks until all pods are Ready (= all containers running = all +# images cached on every node), then we delete the DaemonSet. + +set -euo pipefail + +mapfile -t images < <(grep -Ev '^[[:space:]]*(#|$)' | sort -u) + +if [[ ${#images[@]} -eq 0 ]]; then + echo "e2e-prepull-images: no images on stdin, nothing to do" >&2 + exit 0 +fi + +echo "Pre-pulling ${#images[@]} image(s):" +printf ' %s\n' "${images[@]}" + +containers="" +i=0 +for image in "${images[@]}"; do + containers+=" + - name: pull-${i} + image: ${image} + command: [\"sleep\", \"infinity\"] + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 1m + memory: 1Mi" + i=$((i + 1)) +done + +kubectl apply -f - < /dev/null ) } diff --git a/internal/operator/package_reconciler.go b/internal/operator/package_reconciler.go index 0e724e49..d13cce33 100644 --- a/internal/operator/package_reconciler.go +++ b/internal/operator/package_reconciler.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "strings" + "time" cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" helmv2 "github.com/fluxcd/helm-controller/api/v2" @@ -58,7 +59,12 @@ func parseCRDPolicy(install *cozyv1alpha1.ComponentInstall) helmv2.CRDsPolicy { // PackageReconciler reconciles Package resources type PackageReconciler struct { client.Client - Scheme *runtime.Scheme + Scheme *runtime.Scheme + HelmReleaseInterval time.Duration + HelmReleaseRetryInterval time.Duration + HelmReleaseInstallTimeout time.Duration + HelmReleaseUpgradeTimeout time.Duration + HelmReleaseMaxHistory int } // +kubebuilder:rbac:groups=cozystack.io,resources=packages,verbs=get;list;watch;create;update;patch;delete @@ -214,22 +220,30 @@ func (r *PackageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct Labels: labels, }, Spec: helmv2.HelmReleaseSpec{ - Interval: metav1.Duration{Duration: 5 * 60 * 1000000000}, // 5m + Interval: metav1.Duration{Duration: r.HelmReleaseInterval}, + MaxHistory: &r.HelmReleaseMaxHistory, ChartRef: &helmv2.CrossNamespaceSourceReference{ Kind: "ExternalArtifact", Name: artifactName, Namespace: "cozy-system", }, Install: &helmv2.Install{ - Timeout: &metav1.Duration{Duration: 10 * 60 * 1000000000}, // 10m - Remediation: &helmv2.InstallRemediation{ - Retries: -1, + Timeout: &metav1.Duration{Duration: r.HelmReleaseInstallTimeout}, + // Strategy=RetryOnFailure (with RetryInterval) replaces the previous + // Remediation{Retries:-1} setup. Functionally equivalent ("retry forever + // on failure"), but decouples retry timing from spec.Interval so failed + // installs recover at HelmReleaseRetryInterval (default 30s) without + // also polling healthy releases at the same fast cadence. + Strategy: &helmv2.InstallStrategy{ + Name: string(helmv2.ActionStrategyRetryOnFailure), + RetryInterval: &metav1.Duration{Duration: r.HelmReleaseRetryInterval}, }, }, Upgrade: &helmv2.Upgrade{ - Timeout: &metav1.Duration{Duration: 10 * 60 * 1000000000}, // 10m - Remediation: &helmv2.UpgradeRemediation{ - Retries: -1, + Timeout: &metav1.Duration{Duration: r.HelmReleaseUpgradeTimeout}, + Strategy: &helmv2.UpgradeStrategy{ + Name: string(helmv2.ActionStrategyRetryOnFailure), + RetryInterval: &metav1.Duration{Duration: r.HelmReleaseRetryInterval}, }, CRDs: parseCRDPolicy(component.Install), }, diff --git a/packages/apps/harbor/templates/harbor.yaml b/packages/apps/harbor/templates/harbor.yaml index bf780967..32deb4cb 100644 --- a/packages/apps/harbor/templates/harbor.yaml +++ b/packages/apps/harbor/templates/harbor.yaml @@ -65,9 +65,15 @@ spec: name: {{ .Release.Name }}-credentials valuesKey: redis-password targetPath: harbor.redis.external.password + # BucketInfo is the JSON blob populated by the COSI BucketAccess controller. + # Merge it at the values root (no targetPath) so Flux unmarshals it as YAML + # instead of running it through Helm's strvals parser, which would split the + # JSON on commas. This also gates reconciliation on the Secret being ready and + # forces a config-digest change when its contents change. + - kind: Secret + name: {{ .Release.Name }}-registry-bucket + valuesKey: BucketInfo values: - bucket: - secretName: {{ .Release.Name }}-registry-bucket db: replicas: {{ .Values.database.replicas }} size: {{ .Values.database.size }} diff --git a/packages/core/installer/templates/cozystack-operator.yaml b/packages/core/installer/templates/cozystack-operator.yaml index aded0995..4b48bddd 100644 --- a/packages/core/installer/templates/cozystack-operator.yaml +++ b/packages/core/installer/templates/cozystack-operator.yaml @@ -4,16 +4,6 @@ {{- end -}} --- apiVersion: v1 -kind: Namespace -metadata: - name: cozy-system - labels: - cozystack.io/system: "true" - pod-security.kubernetes.io/enforce: privileged - annotations: - helm.sh/resource-policy: keep ---- -apiVersion: v1 kind: ServiceAccount metadata: name: cozystack @@ -68,6 +58,21 @@ spec: {{- if .Values.cozystackOperator.disableTelemetry }} - --disable-telemetry {{- end }} + {{- if .Values.cozystackOperator.helmReleaseInterval }} + - --helmrelease-interval={{ .Values.cozystackOperator.helmReleaseInterval }} + {{- end }} + {{- if .Values.cozystackOperator.helmReleaseRetryInterval }} + - --helmrelease-retry-interval={{ .Values.cozystackOperator.helmReleaseRetryInterval }} + {{- end }} + {{- if .Values.cozystackOperator.helmReleaseInstallTimeout }} + - --helmrelease-install-timeout={{ .Values.cozystackOperator.helmReleaseInstallTimeout }} + {{- end }} + {{- if .Values.cozystackOperator.helmReleaseUpgradeTimeout }} + - --helmrelease-upgrade-timeout={{ .Values.cozystackOperator.helmReleaseUpgradeTimeout }} + {{- end }} + {{- if .Values.cozystackOperator.helmReleaseMaxHistory }} + - --helmrelease-max-history={{ .Values.cozystackOperator.helmReleaseMaxHistory }} + {{- end }} - --platform-source-name=cozystack-platform - --platform-source-url={{ .Values.cozystackOperator.platformSourceUrl }} {{- if .Values.cozystackOperator.platformSourceRef }} diff --git a/packages/core/installer/values.yaml b/packages/core/installer/values.yaml index eef691ea..dcd0a913 100644 --- a/packages/core/installer/values.yaml +++ b/packages/core/installer/values.yaml @@ -4,6 +4,25 @@ cozystackOperator: image: ghcr.io/cozystack/cozystack/cozystack-operator:v1.3.0@sha256:62574f12486bb40c901cf5ed484cca264405ce5810196d86555cbb27cce1ba48 platformSourceUrl: 'oci://ghcr.io/cozystack/cozystack/cozystack-packages' platformSourceRef: 'digest=sha256:a0b9ef938446b3132d3d22ad2262beb1027c48c9037b6c2346fdc2f19acd3036' + # When non-empty, overrides the operator's --helmrelease-interval flag + # (operator default: 5m). E2E sets this to 30s; production should leave empty. + helmReleaseInterval: "" + # When non-empty, overrides the operator's --helmrelease-retry-interval flag + # (operator default: 30s). Controls how fast failed install/upgrade attempts + # are retried. Decoupled from helmReleaseInterval (healthy reconcile cadence). + helmReleaseRetryInterval: "" + # When non-empty, overrides the operator's --helmrelease-install-timeout flag + # (operator default: 10m). Bounds individual k8s operations (job/hook/wait) + # during Helm install. Charts with long-running install hooks may need longer. + helmReleaseInstallTimeout: "" + # When non-empty, overrides the operator's --helmrelease-upgrade-timeout flag + # (operator default: 10m). Same semantics as helmReleaseInstallTimeout, for + # the Helm upgrade action. + helmReleaseUpgradeTimeout: "" + # When non-empty, overrides the operator's --helmrelease-max-history flag + # (operator default: 5). Number of release revisions Helm keeps per HR. + # Use 0 for unlimited; lower values reduce Secret accumulation in test clusters. + helmReleaseMaxHistory: "" # Generic variant configuration (only used when cozystackOperator.variant=generic) cozystack: # Kubernetes API server host (IP only, no protocol/port) diff --git a/packages/system/harbor/templates/bucket-secret.yaml b/packages/system/harbor/templates/bucket-secret.yaml index 241fe638..d67aeb67 100644 --- a/packages/system/harbor/templates/bucket-secret.yaml +++ b/packages/system/harbor/templates/bucket-secret.yaml @@ -1,20 +1,17 @@ -{{- if .Values.bucket.secretName }} -{{- $existingSecret := lookup "v1" "Secret" .Release.Namespace .Values.bucket.secretName }} -{{- $bucketInfo := fromJson (b64dec (index $existingSecret.data "BucketInfo")) }} -{{- $accessKeyID := index $bucketInfo.spec.secretS3 "accessKeyID" }} -{{- $accessSecretKey := index $bucketInfo.spec.secretS3 "accessSecretKey" }} -{{- $endpoint := index $bucketInfo.spec.secretS3 "endpoint" }} -{{- $bucketName := $bucketInfo.spec.bucketName }} +{{- /* COSI BucketInfo is merged at the values root via Flux valuesFrom (no targetPath). */ -}} +{{- with .Values.spec }} +{{- with .secretS3 }} --- apiVersion: v1 kind: Secret metadata: - name: {{ .Values.harbor.fullnameOverride }}-registry-s3 + name: {{ $.Values.harbor.fullnameOverride }}-registry-s3 type: Opaque stringData: - REGISTRY_STORAGE_S3_ACCESSKEY: {{ $accessKeyID | quote }} - REGISTRY_STORAGE_S3_SECRETKEY: {{ $accessSecretKey | quote }} - REGISTRY_STORAGE_S3_REGIONENDPOINT: {{ $endpoint | quote }} - REGISTRY_STORAGE_S3_BUCKET: {{ $bucketName | quote }} + REGISTRY_STORAGE_S3_ACCESSKEY: {{ index . "accessKeyID" | quote }} + REGISTRY_STORAGE_S3_SECRETKEY: {{ index . "accessSecretKey" | quote }} + REGISTRY_STORAGE_S3_REGIONENDPOINT: {{ index . "endpoint" | quote }} + REGISTRY_STORAGE_S3_BUCKET: {{ $.Values.spec.bucketName | quote }} REGISTRY_STORAGE_S3_REGION: "us-east-1" {{- end }} +{{- end }} diff --git a/packages/system/harbor/values.yaml b/packages/system/harbor/values.yaml index faac7068..9038bd38 100644 --- a/packages/system/harbor/values.yaml +++ b/packages/system/harbor/values.yaml @@ -1,6 +1,4 @@ harbor: {} -bucket: - secretName: "" db: replicas: 2 size: 5Gi diff --git a/packages/system/objectstorage-controller/images/objectstorage/Dockerfile b/packages/system/objectstorage-controller/images/objectstorage/Dockerfile index 260446c3..d71e5566 100644 --- a/packages/system/objectstorage-controller/images/objectstorage/Dockerfile +++ b/packages/system/objectstorage-controller/images/objectstorage/Dockerfile @@ -2,12 +2,15 @@ FROM alpine AS source ARG COMMIT_REF=v0.2.2 -RUN apk add --no-cache curl tar +RUN apk add --no-cache curl tar git WORKDIR /src RUN curl -sSL https://github.com/kubernetes-sigs/container-object-storage-interface/archive/${COMMIT_REF}.tar.gz \ | tar -xz --strip-components=1 +COPY patches /patches +RUN git apply /patches/*.diff + FROM --platform=$BUILDPLATFORM docker.io/golang:1.24 AS builder ARG TARGETOS ARG TARGETARCH diff --git a/packages/system/objectstorage-controller/images/objectstorage/patches/91-bucketaccess-conflict-retry.diff b/packages/system/objectstorage-controller/images/objectstorage/patches/91-bucketaccess-conflict-retry.diff new file mode 100644 index 00000000..e4b06c42 --- /dev/null +++ b/packages/system/objectstorage-controller/images/objectstorage/patches/91-bucketaccess-conflict-retry.diff @@ -0,0 +1,39 @@ +diff --git a/sidecar/pkg/bucketaccess/bucketaccess_controller.go b/sidecar/pkg/bucketaccess/bucketaccess_controller.go +index a4db5c4..b351f9e 100644 +--- a/sidecar/pkg/bucketaccess/bucketaccess_controller.go ++++ b/sidecar/pkg/bucketaccess/bucketaccess_controller.go +@@ -31,6 +31,7 @@ import ( + kube "k8s.io/client-go/kubernetes" + kubecorev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/tools/record" ++ "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" + cosiapi "sigs.k8s.io/container-object-storage-interface/client/apis" + "sigs.k8s.io/container-object-storage-interface/client/apis/objectstorage/consts" +@@ -273,11 +274,22 @@ func (bal *BucketAccessListener) Add(ctx context.Context, inputBucketAccess *v1a + } + } + +- if controllerutil.AddFinalizer(bucket, consts.BABucketFinalizer) { +- _, err = bal.buckets().Update(ctx, bucket, metav1.UpdateOptions{}) +- if err != nil { +- return bal.recordError(inputBucketAccess, v1.EventTypeWarning, v1alpha1.FailedGrantAccess, err) ++ // Re-fetch and update inside RetryOnConflict to handle the case where the Bucket ++ // reconciler in the same controller process mutates the Bucket between our Get ++ // and Update, which surfaces as "the object has been modified" on the parent ++ // Bucket and emits FailedGrantAccess on the BucketAccess. ++ if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { ++ latest, getErr := bal.buckets().Get(ctx, bucketClaim.Status.BucketName, metav1.GetOptions{}) ++ if getErr != nil { ++ return getErr ++ } ++ if !controllerutil.AddFinalizer(latest, consts.BABucketFinalizer) { ++ return nil + } ++ _, updateErr := bal.buckets().Update(ctx, latest, metav1.UpdateOptions{}) ++ return updateErr ++ }); err != nil { ++ return bal.recordError(inputBucketAccess, v1.EventTypeWarning, v1alpha1.FailedGrantAccess, err) + } + + if controllerutil.AddFinalizer(bucketAccess, consts.BAFinalizer) {