e2e: pre-pull timing-sensitive platform images before install

Some workloads (OVN raft, LINSTOR controller) fail when replicas start
at different times due to image-pull stagger across nodes. Add a
DaemonSet-based pre-pull step that runs before helm install, ensuring
all nodes have the images cached so every replica starts within
milliseconds of each other.

Covers kube-ovn (confirmed OVN raft race), piraeus-server, and
linstor-csi. Comments in hack/e2e-prepull-images.sh point to the
source chart values so versions stay in sync.

Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
This commit is contained in:
Myasnikov Daniil 2026-04-29 14:14:55 +05:00
parent 5e88a1410c
commit 432ff777ba
No known key found for this signature in database
GPG key ID: FA953A439C637F04
2 changed files with 115 additions and 0 deletions

View file

@ -7,6 +7,13 @@
fi
}
@test "Pre-pull platform images" {
# Pull timing-sensitive images to all nodes before Cozystack installation.
# Cluster-member workloads (OVN raft, LINSTOR) fail if replicas start at
# different times due to image-pull stagger. See hack/e2e-prepull-images.sh.
hack/e2e-prepull-images.sh
}
@test "Install Cozystack" {
# Pre-create the cozy-system namespace with the labels the operator pod needs.
# The chart no longer ships a Namespace resource (helm v3's --create-namespace

108
hack/e2e-prepull-images.sh Executable file
View file

@ -0,0 +1,108 @@
#!/usr/bin/env bash
# Pre-pull timing-sensitive platform images to all cluster nodes.
#
# Some workloads (OVN raft, LINSTOR controller) are sensitive to peer-readiness
# at startup: if nodes pull the image at different speeds, one replica boots
# before its peers are reachable, exhausts its raft connection retries, and the
# HelmRelease install times out. Pre-pulling to all nodes eliminates the pull
# stagger so all replicas start within milliseconds of each other.
#
# Add an entry whenever you find a new workload with peer-dependent startup.
# Update versions here when bumping the corresponding chart dependency.
set -euo pipefail
# ---------------------------------------------------------------------------
# Image list
# ---------------------------------------------------------------------------
# Format: <registry>/<repo>:<tag>[@sha256:<digest>]
# Each image is pulled as an init container (imagePullPolicy=IfNotPresent),
# so this is a no-op if the image is already cached on that node.
#
# Source of each image and version is noted in the comment above it.
read -r -d '' PREPULL_YAML <<'YAML' || true
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: e2e-image-prepuller
namespace: kube-system
labels:
app: e2e-image-prepuller
spec:
selector:
matchLabels:
app: e2e-image-prepuller
template:
metadata:
labels:
app: e2e-image-prepuller
spec:
tolerations:
# Tolerate all taints so the DaemonSet reaches every node including
# control-plane nodes and nodes still coming up.
- operator: Exists
initContainers:
# kube-ovn: OVN raft cluster, 3 ovn-central replicas must elect a leader.
# Staggered pulls cause "failed to connect to peer" and raft timeout.
# Source: packages/system/kubeovn/charts/kube-ovn/values.yaml
- name: pull-kubeovn
image: docker.io/kubeovn/kube-ovn:v1.15.10
command: ['sh', '-c', 'exit 0']
imagePullPolicy: IfNotPresent
resources:
requests:
cpu: 1m
memory: 1Mi
# piraeus-server: LINSTOR controller/satellite cluster.
# Satellites must register with the controller before storage is usable.
# Source: packages/system/linstor/values.yaml (.piraeusServer.image)
- name: pull-piraeus-server
image: ghcr.io/cozystack/cozystack/piraeus-server:1.33.2@sha256:553f313ab35dc2e345ef3683156d29e75c23177e2750e9af3a83aa9e23941cbb
command: ['sh', '-c', 'exit 0']
imagePullPolicy: IfNotPresent
resources:
requests:
cpu: 1m
memory: 1Mi
# linstor-csi: CSI driver sidecar shipped on every storage node.
# Large-ish image; pre-pulling avoids a slow first-mount on the first PVC.
# Source: packages/system/linstor/values.yaml (.linstorCSI.image)
- name: pull-linstor-csi
image: ghcr.io/cozystack/cozystack/linstor-csi:v1.10.5@sha256:b8f59b5659fb1791cb764d3f37df4cf29920aadcc10637231ba7d857233f377d
command: ['sh', '-c', 'exit 0']
imagePullPolicy: IfNotPresent
resources:
requests:
cpu: 1m
memory: 1Mi
containers:
# Pause container keeps the pod Running after init containers complete.
# Running state is our signal that all images have been pulled on this node.
- name: pause
image: registry.k8s.io/pause:3.10
imagePullPolicy: IfNotPresent
resources:
requests:
cpu: 1m
memory: 1Mi
YAML
kubectl apply -f - <<<"${PREPULL_YAML}"
node_count=$(kubectl get nodes --no-headers 2>/dev/null | wc -l)
echo "Waiting for e2e-image-prepuller on ${node_count} nodes (timeout 5m)..."
# A pod is Running when all init containers have exited 0 and the main
# container has started. That means every image in the init list was pulled.
timeout 300 sh -ec "
until [ \$(kubectl get pods -n kube-system \
-l app=e2e-image-prepuller \
--field-selector=status.phase=Running \
--no-headers 2>/dev/null | wc -l) -ge ${node_count} ]; do
sleep 3
done
"
kubectl delete daemonset e2e-image-prepuller -n kube-system --ignore-not-found
echo "Image pre-pull complete on all ${node_count} nodes."