From 56f0484eb0783ba0b5451c1b8ea45b95e3e63bb3 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Tue, 31 Mar 2026 11:15:40 +0200 Subject: [PATCH 1/2] fix(multus): build custom image with DEL cache fix When CNI ADD never completes, multus DEL fails because delegate plugins require runtime state that only exists after a successful ADD. This creates a deadlock where containerd cannot release sandbox name reservations, permanently blocking pod creation on the affected node. Build a custom multus-cni image with a patch that returns success on DEL when no cache file exists (ADD was never completed). Upstream PR: https://github.com/k8snetworkplumbingwg/multus-cni/pull/1498 Tracking issue: https://github.com/cozystack/cozystack/issues/2310 Co-Authored-By: Claude Signed-off-by: Andrei Kvapil (cherry picked from commit 094b80ae16aef967b8f57167ee4f06c89b4a7d78) --- Makefile | 1 + packages/system/multus/Makefile | 12 +++++ packages/system/multus/images/multus-cni.json | 54 +++++++++++++++++++ .../multus/images/multus-cni/Dockerfile | 25 +++++++++ .../patches/fix-del-without-cache.diff | 49 +++++++++++++++++ .../templates/multus-daemonset-thick.yml | 4 +- 6 files changed, 143 insertions(+), 2 deletions(-) create mode 100644 packages/system/multus/images/multus-cni.json create mode 100644 packages/system/multus/images/multus-cni/Dockerfile create mode 100644 packages/system/multus/images/multus-cni/patches/fix-del-without-cache.diff diff --git a/Makefile b/Makefile index ce6eb7b3..59a55bfb 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,7 @@ build: build-deps make -C packages/system/dashboard image make -C packages/system/metallb image make -C packages/system/kamaji image + make -C packages/system/multus image make -C packages/system/bucket image make -C packages/system/objectstorage-controller image make -C packages/system/grafana-operator image diff --git a/packages/system/multus/Makefile b/packages/system/multus/Makefile index 9b34606d..399af728 100644 --- a/packages/system/multus/Makefile +++ b/packages/system/multus/Makefile @@ -11,3 +11,15 @@ update: wget -q https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/refs/tags/$(RELEASE)/deployments/multus-daemonset-thick.yml -O templates/multus-daemonset-thick.yml sed -i '/multus-cni/s/snapshot-thick/$(RELEASE)-thick/' templates/multus-daemonset-thick.yml && \ patch --no-backup-if-mismatch -p4 < patches/customize-deployment.patch + $(SED_INPLACE) "/ARG VERSION/ s|=.*|=$(RELEASE)|g" images/multus-cni/Dockerfile + +image: + docker buildx build images/multus-cni \ + --tag $(REGISTRY)/multus-cni:$(call settag,$(TAG)) \ + --cache-from type=registry,ref=$(REGISTRY)/multus-cni:latest \ + --cache-to type=inline \ + --metadata-file images/multus-cni.json \ + $(BUILDX_ARGS) + DIGEST=$$(yq e '."containerimage.digest"' images/multus-cni.json -o json -r) && \ + sed -i "s|image: .*multus-cni.*|image: $(REGISTRY)/multus-cni:$(TAG)@$${DIGEST}|g" templates/multus-daemonset-thick.yml + rm -f images/multus-cni.json diff --git a/packages/system/multus/images/multus-cni.json b/packages/system/multus/images/multus-cni.json new file mode 100644 index 00000000..9848b688 --- /dev/null +++ b/packages/system/multus/images/multus-cni.json @@ -0,0 +1,54 @@ +{ + "buildx.build.provenance": { + "buildType": "https://mobyproject.org/buildkit@v1", + "materials": [ + { + "uri": "pkg:docker/debian@stable-slim?platform=linux%2Famd64", + "digest": { + "sha256": "99fc6d2a0882fcbcdc452948d2d54eab91faafc7db037df82425edcdcf950e1f" + } + }, + { + "uri": "pkg:docker/golang@1.24?platform=linux%2Famd64", + "digest": { + "sha256": "d2d2bc1c84f7e60d7d2438a3836ae7d0c847f4888464e7ec9ba3a1339a1ee804" + } + } + ], + "invocation": { + "configSource": { + "entryPoint": "Dockerfile" + }, + "parameters": { + "frontend": "dockerfile.v0", + "args": { + "label:org.opencontainers.image.source": "https://github.com/cozystack/cozystack" + }, + "locals": [ + { + "name": "context" + }, + { + "name": "dockerfile" + } + ] + }, + "environment": { + "platform": "linux/amd64" + } + } + }, + "buildx.build.ref": "buildkit-builder/buildkit-builder0/84w636wch59pt7uaw1mvwfg2w", + "containerimage.config.digest": "sha256:407e75f5f1122abc86ec12ea1d70b233941f1c431ee03cf9b7cae55d890b9c98", + "containerimage.descriptor": { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:4b7d9081bdb9d6608ff9b941364fb3c71bf56a352ca3da1bbe14e6b64e6aa459", + "size": 1098, + "platform": { + "architecture": "amd64", + "os": "linux" + } + }, + "containerimage.digest": "sha256:4b7d9081bdb9d6608ff9b941364fb3c71bf56a352ca3da1bbe14e6b64e6aa459", + "image.name": "ghcr.io/cozystack/cozystack/multus-cni:latest" +} \ No newline at end of file diff --git a/packages/system/multus/images/multus-cni/Dockerfile b/packages/system/multus/images/multus-cni/Dockerfile new file mode 100644 index 00000000..feecbc6c --- /dev/null +++ b/packages/system/multus/images/multus-cni/Dockerfile @@ -0,0 +1,25 @@ +FROM --platform=$BUILDPLATFORM golang:1.24 AS build + +ARG VERSION=v4.2.3 +ARG TARGETPLATFORM + +WORKDIR /usr/src/multus-cni + +RUN curl -sSL https://github.com/k8snetworkplumbingwg/multus-cni/archive/refs/tags/${VERSION}.tar.gz | \ + tar -xzf - --strip=1 + +COPY patches /patches +RUN git init && git config user.email "build@cozystack" && git config user.name "build" && \ + git add -A && git commit -m "init" && git tag ${VERSION} && \ + git apply /patches/*.diff + +RUN ./hack/build-go.sh + +FROM debian:stable-slim +LABEL org.opencontainers.image.source=https://github.com/cozystack/cozystack +COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin +COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE +COPY --from=build /usr/src/multus-cni/bin/cert-approver / +WORKDIR / + +ENTRYPOINT ["/usr/src/multus-cni/bin/multus-daemon"] diff --git a/packages/system/multus/images/multus-cni/patches/fix-del-without-cache.diff b/packages/system/multus/images/multus-cni/patches/fix-del-without-cache.diff new file mode 100644 index 00000000..7595ddb2 --- /dev/null +++ b/packages/system/multus/images/multus-cni/patches/fix-del-without-cache.diff @@ -0,0 +1,49 @@ +diff --git a/pkg/multus/multus.go b/pkg/multus/multus.go +index a7941fe..eff0406 100644 +--- a/pkg/multus/multus.go ++++ b/pkg/multus/multus.go +@@ -954,33 +954,19 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er + } + + if !useCacheConf { +- // Fetch delegates again if cache is not exist and pod info can be read +- if os.IsNotExist(err) && pod != nil { +- if in.ClusterNetwork != "" { +- _, err = k8s.GetDefaultNetworks(pod, in, kubeClient, nil) +- if err != nil { +- return cmdErr(k8sArgs, "failed to get clusterNetwork/defaultNetworks: %v", err) +- } +- // First delegate is always the master plugin +- in.Delegates[0].MasterPlugin = true +- } +- +- // Get pod annotation and so on +- _, _, err := k8s.TryLoadPodDelegates(pod, in, kubeClient, nil) +- if err != nil { +- if len(in.Delegates) == 0 { +- // No delegate available so send error +- return cmdErr(k8sArgs, "failed to get delegates: %v", err) +- } +- // Get clusterNetwork before, so continue to delete +- logging.Errorf("Multus: failed to get delegates: %v, but continue to delete clusterNetwork", err) +- } +- } else { +- // The options to continue with a delete have been exhausted (cachefile + API query didn't work) +- // We cannot exit with an error as this may cause a sandbox to never get deleted. +- logging.Errorf("Multus: failed to get the cached delegates file: %v, cannot properly delete", err) ++ // If cache does not exist, ADD was never completed for this sandbox. ++ // Attempting DEL with a config reconstructed from the API will fail because ++ // delegate plugins (e.g. kube-ovn) require runtime state that only exists ++ // after a successful ADD (socket paths, cached results, etc.). ++ // Return success to allow containerd to release the sandbox name reservation. ++ if os.IsNotExist(err) { ++ logging.Verbosef("Multus: no cache found for container %s (ADD was never completed), skip DEL", args.ContainerID) + return nil + } ++ // The options to continue with a delete have been exhausted ++ // We cannot exit with an error as this may cause a sandbox to never get deleted. ++ logging.Errorf("Multus: failed to get the cached delegates file: %v, cannot properly delete", err) ++ return nil + } + + // set CNIVersion in delegate CNI config if there is no CNIVersion and multus conf have CNIVersion. diff --git a/packages/system/multus/templates/multus-daemonset-thick.yml b/packages/system/multus/templates/multus-daemonset-thick.yml index 2f00de85..97144d67 100644 --- a/packages/system/multus/templates/multus-daemonset-thick.yml +++ b/packages/system/multus/templates/multus-daemonset-thick.yml @@ -154,7 +154,7 @@ spec: serviceAccountName: multus containers: - name: kube-multus - image: ghcr.io/k8snetworkplumbingwg/multus-cni:v4.2.3-thick + image: ghcr.io/cozystack/cozystack/multus-cni:latest@sha256:4b7d9081bdb9d6608ff9b941364fb3c71bf56a352ca3da1bbe14e6b64e6aa459 command: [ "/usr/src/multus-cni/bin/multus-daemon" ] resources: requests: @@ -200,7 +200,7 @@ spec: fieldPath: spec.nodeName initContainers: - name: install-multus-binary - image: ghcr.io/k8snetworkplumbingwg/multus-cni:v4.2.3-thick + image: ghcr.io/cozystack/cozystack/multus-cni:latest@sha256:4b7d9081bdb9d6608ff9b941364fb3c71bf56a352ca3da1bbe14e6b64e6aa459 command: - "/usr/src/multus-cni/bin/install_multus" - "-d" From 3ab03d3bce4c4a8f76ed64793589f38760b393b8 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Tue, 31 Mar 2026 11:15:45 +0200 Subject: [PATCH 2/2] chore: remove build artifact Co-Authored-By: Claude Signed-off-by: Andrei Kvapil (cherry picked from commit 1cd564330cb7137c2994afa1b5ca78a3b062dc08) --- packages/system/multus/images/multus-cni.json | 54 ------------------- 1 file changed, 54 deletions(-) delete mode 100644 packages/system/multus/images/multus-cni.json diff --git a/packages/system/multus/images/multus-cni.json b/packages/system/multus/images/multus-cni.json deleted file mode 100644 index 9848b688..00000000 --- a/packages/system/multus/images/multus-cni.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "buildx.build.provenance": { - "buildType": "https://mobyproject.org/buildkit@v1", - "materials": [ - { - "uri": "pkg:docker/debian@stable-slim?platform=linux%2Famd64", - "digest": { - "sha256": "99fc6d2a0882fcbcdc452948d2d54eab91faafc7db037df82425edcdcf950e1f" - } - }, - { - "uri": "pkg:docker/golang@1.24?platform=linux%2Famd64", - "digest": { - "sha256": "d2d2bc1c84f7e60d7d2438a3836ae7d0c847f4888464e7ec9ba3a1339a1ee804" - } - } - ], - "invocation": { - "configSource": { - "entryPoint": "Dockerfile" - }, - "parameters": { - "frontend": "dockerfile.v0", - "args": { - "label:org.opencontainers.image.source": "https://github.com/cozystack/cozystack" - }, - "locals": [ - { - "name": "context" - }, - { - "name": "dockerfile" - } - ] - }, - "environment": { - "platform": "linux/amd64" - } - } - }, - "buildx.build.ref": "buildkit-builder/buildkit-builder0/84w636wch59pt7uaw1mvwfg2w", - "containerimage.config.digest": "sha256:407e75f5f1122abc86ec12ea1d70b233941f1c431ee03cf9b7cae55d890b9c98", - "containerimage.descriptor": { - "mediaType": "application/vnd.docker.distribution.manifest.v2+json", - "digest": "sha256:4b7d9081bdb9d6608ff9b941364fb3c71bf56a352ca3da1bbe14e6b64e6aa459", - "size": 1098, - "platform": { - "architecture": "amd64", - "os": "linux" - } - }, - "containerimage.digest": "sha256:4b7d9081bdb9d6608ff9b941364fb3c71bf56a352ca3da1bbe14e6b64e6aa459", - "image.name": "ghcr.io/cozystack/cozystack/multus-cni:latest" -} \ No newline at end of file