0fc61eeae6
Some workloads (OVN raft, LINSTOR controller) fail when replicas start at different times due to image-pull stagger across nodes. Add a DaemonSet-based pre-pull step that runs before helm install, ensuring all nodes have the images cached so every replica starts within milliseconds of each other. The script accepts image refs on stdin and creates one container per image (parallel pulls, total time = max of any single image rather than sum). The bats test sources images directly from the rendered charts via yq, walking only PodSpec-shaped objects so version bumps stay in sync automatically without a separate hardcoded list. Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
81 lines
2.6 KiB
Bash
Executable file
81 lines
2.6 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Pre-pull images to all cluster nodes.
|
|
#
|
|
# Reads image references from stdin, one per line. Empty lines and lines
|
|
# starting with '#' are ignored.
|
|
#
|
|
# Some workloads (OVN raft, LINSTOR) are sensitive to peer-readiness at
|
|
# startup: if nodes pull the image at different speeds, one replica boots
|
|
# before its peers are reachable, exhausts its raft connection retries, and
|
|
# the HelmRelease install times out. Pre-pulling eliminates the pull stagger
|
|
# so all replicas start within milliseconds of each other.
|
|
#
|
|
# Implementation: a DaemonSet with one regular container per image (parallel
|
|
# pulls — total time = max of any one image rather than sum). kubectl rollout
|
|
# status blocks until all pods are Ready (= all containers running = all
|
|
# images cached on every node), then we delete the DaemonSet.
|
|
|
|
set -euo pipefail
|
|
|
|
mapfile -t images < <(grep -Ev '^[[:space:]]*(#|$)' | sort -u)
|
|
|
|
if [[ ${#images[@]} -eq 0 ]]; then
|
|
echo "e2e-prepull-images: no images on stdin, nothing to do" >&2
|
|
exit 0
|
|
fi
|
|
|
|
echo "Pre-pulling ${#images[@]} image(s):"
|
|
printf ' %s\n' "${images[@]}"
|
|
|
|
containers=""
|
|
i=0
|
|
for image in "${images[@]}"; do
|
|
containers+="
|
|
- name: pull-${i}
|
|
image: ${image}
|
|
command: [\"sleep\", \"infinity\"]
|
|
imagePullPolicy: IfNotPresent
|
|
resources:
|
|
requests:
|
|
cpu: 1m
|
|
memory: 1Mi"
|
|
i=$((i + 1))
|
|
done
|
|
|
|
kubectl apply -f - <<EOF
|
|
apiVersion: apps/v1
|
|
kind: DaemonSet
|
|
metadata:
|
|
name: e2e-image-prepuller
|
|
namespace: kube-system
|
|
labels:
|
|
app: e2e-image-prepuller
|
|
spec:
|
|
selector:
|
|
matchLabels:
|
|
app: e2e-image-prepuller
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: e2e-image-prepuller
|
|
spec:
|
|
# hostNetwork bypasses the CNI: this script runs BEFORE Cozystack
|
|
# installs kube-ovn, so the cluster has no working CNI yet and a normal
|
|
# pod would stay ContainerCreating with NetworkPluginNotReady, never
|
|
# reaching image-pull. With hostNetwork the pod sandbox is created on
|
|
# the host's namespace and the kubelet pulls images right away.
|
|
# Same pattern the cozystack-operator pod uses for the same reason.
|
|
hostNetwork: true
|
|
# No need for an SA token — this pod just sleeps while images pull.
|
|
automountServiceAccountToken: false
|
|
tolerations:
|
|
# Reach every node including control-plane and nodes still coming up.
|
|
- operator: Exists
|
|
containers:${containers}
|
|
EOF
|
|
|
|
echo "Waiting for e2e-image-prepuller DaemonSet to become ready..."
|
|
kubectl rollout status daemonset/e2e-image-prepuller -n kube-system --timeout=10m
|
|
|
|
kubectl delete daemonset e2e-image-prepuller -n kube-system --ignore-not-found
|
|
echo "Image pre-pull complete."
|