diff --git a/Makefile b/Makefile index 9aff5ac6..d0f5dd69 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,7 @@ build: build-deps make -C packages/system/backup-controller image make -C packages/system/lineage-controller-webhook image make -C packages/system/cilium image + make -C packages/system/linstor image make -C packages/system/kubeovn-webhook image make -C packages/system/kubeovn-plunger image make -C packages/system/dashboard image diff --git a/packages/system/linstor/Makefile b/packages/system/linstor/Makefile index 5de22194..da895085 100644 --- a/packages/system/linstor/Makefile +++ b/packages/system/linstor/Makefile @@ -1,4 +1,23 @@ export NAME=linstor export NAMESPACE=cozy-$(NAME) +include ../../../scripts/common-envs.mk include ../../../scripts/package.mk + +LINSTOR_VERSION ?= 1.32.3 + +image: + docker buildx build images/piraeus-server \ + --build-arg LINSTOR_VERSION=$(LINSTOR_VERSION) \ + --build-arg K8S_AWAIT_ELECTION_VERSION=v0.4.2 \ + --tag $(REGISTRY)/piraeus-server:$(call settag,$(LINSTOR_VERSION)) \ + --tag $(REGISTRY)/piraeus-server:$(call settag,$(LINSTOR_VERSION)-$(TAG)) \ + --cache-from type=registry,ref=$(REGISTRY)/piraeus-server:latest \ + --cache-to type=inline \ + --metadata-file images/piraeus-server.json \ + $(BUILDX_ARGS) + REPOSITORY="$(REGISTRY)/piraeus-server" \ + yq -i '.piraeusServer.image.repository = strenv(REPOSITORY)' values.yaml + TAG="$(call settag,$(LINSTOR_VERSION))@$$(yq e '."containerimage.digest"' images/piraeus-server.json -o json -r)" \ + yq -i '.piraeusServer.image.tag = strenv(TAG)' values.yaml + rm -f images/piraeus-server.json diff --git a/packages/system/linstor/hack/plunger/plunger-satellite.sh b/packages/system/linstor/hack/plunger/plunger-satellite.sh index 688cfd65..4a488a51 100755 --- a/packages/system/linstor/hack/plunger/plunger-satellite.sh +++ b/packages/system/linstor/hack/plunger/plunger-satellite.sh @@ -10,10 +10,125 @@ trap terminate SIGINT SIGQUIT SIGTERM echo "Starting Linstor per-satellite plunger" +INTERVAL_SEC="${INTERVAL_SEC:-30}" +STALL_ITERS="${STALL_ITERS:-4}" +STATE_FILE="${STATE_FILE:-/run/drbd-sync-watch.state}" + +log() { printf '%s %s\n' "$(date -Is)" "$*" >&2; } + +drbd_status_json() { + drbdsetup status --json 2>/dev/null || true +} + +# Detect DRBD resources where resync is stuck: +# - at least one local device is Inconsistent +# - there is an active SyncTarget peer +# - there are other peers suspended with resync-suspended:dependency +# Output format: " " +drbd_stall_candidates() { + jq -r ' + .[]? + | . as $r + | select(any($r.devices[]?; ."disk-state" == "Inconsistent")) + | ( + [ $r.connections[]? + | . as $c + | $c.peer_devices[]? + | select(."replication-state" == "SyncTarget") + | { peer: $c.name, pct: (."percent-in-sync" // empty) } + ] | .[0]? + ) as $sync + | select($sync != null and ($sync.pct|tostring) != "") + | select(any($r.connections[]?.peer_devices[]?; ."resync-suspended" == "dependency")) + | "\($r.name) \($sync.peer) \($sync.pct)" + ' +} + +drbd_stall_load_state() { + [ -f "$STATE_FILE" ] && cat "$STATE_FILE" || true +} + +drbd_stall_save_state() { + local tmp="${STATE_FILE}.tmp" + cat >"$tmp" + mv "$tmp" "$STATE_FILE" +} + +# Break stalled resync by disconnecting the current SyncTarget peer. +# After reconnect, DRBD will typically pick another eligible peer and continue syncing. +drbd_stall_act() { + local res="$1" + local peer="$2" + local pct="$3" + log "STALL detected: res=$res sync_peer=$peer percent_in_sync=$pct -> disconnect/connect" + drbdadm disconnect "${res}:${peer}" && drbdadm connect "$res" || log "WARN: action failed for ${res}:${peer}" +} + +# Track percent-in-sync progress across iterations. +# If progress does not change for STALL_ITERS loops, trigger reconnect. +drbd_fix_stalled_sync() { + local now prev json out + now="$(date +%s)" + prev="$(drbd_stall_load_state)" + + json="$(drbd_status_json)" + [ -n "$json" ] || return 0 + + out="$(printf '%s' "$json" | drbd_stall_candidates)" + + local new_state="" + local acts="" + + while IFS= read -r line; do + [ -n "$line" ] || continue + set -- $line + local res="$1" peer="$2" pct="$3" + local key="${res} ${peer}" + + local prev_line + prev_line="$(printf '%s\n' "$prev" | awk -v k="$key" '$1" "$2==k {print; exit}')" + + local cnt last_act prev_pct prev_cnt prev_act + if [ -n "$prev_line" ]; then + set -- $prev_line + prev_pct="$3" + prev_cnt="$4" + prev_act="$5" + if [ "$pct" = "$prev_pct" ]; then + cnt=$((prev_cnt + 1)) + else + cnt=1 + fi + last_act="$prev_act" + else + cnt=1 + last_act=0 + fi + + if [ "$cnt" -ge "$STALL_ITERS" ]; then + acts="${acts}${res} ${peer} ${pct}"$'\n' + cnt=0 + last_act="$now" + fi + + new_state="${new_state}${res} ${peer} ${pct} ${cnt} ${last_act}"$'\n' + done <<< "$out" + + if [ -n "$acts" ]; then + while IFS= read -r a; do + [ -n "$a" ] || continue + set -- $a + drbd_stall_act "$1" "$2" "$3" + done <<< "$acts" + fi + + printf '%s' "$new_state" | drbd_stall_save_state +} + while true; do # timeout at the start of the loop to give a chance for the fresh linstor-satellite instance to cleanup itself - sleep 30 & + sleep "$INTERVAL_SEC" & pid=$! wait $pid @@ -21,7 +136,7 @@ while true; do # the `/` path could not be a backing file for a loop device, so it's a good indicator of a stuck loop device # TODO describe the issue in more detail # Using the direct /usr/sbin/losetup as the linstor-satellite image has own wrapper in /usr/local - stale_loopbacks=$(/usr/sbin/losetup --json | jq -r '.[][] | select(."back-file" == "/" or ."back-file" == "/ (deleted)").name' ) + stale_loopbacks=$(/usr/sbin/losetup --json | jq -r '.[][] | select(."back-file" == "/" or ."back-file" == "/ (deleted)").name') for stale_device in $stale_loopbacks; do ( echo "Detaching stuck loop device ${stale_device}" set -x @@ -39,4 +154,7 @@ while true; do drbdadm up "${secondary}" || echo "Command failed" ); done + # Detect and fix stalled DRBD resync by switching SyncTarget peer + drbd_fix_stalled_sync || true + done diff --git a/packages/system/linstor/images/piraeus-server/Dockerfile b/packages/system/linstor/images/piraeus-server/Dockerfile new file mode 100644 index 00000000..049a3f47 --- /dev/null +++ b/packages/system/linstor/images/piraeus-server/Dockerfile @@ -0,0 +1,172 @@ +ARG DISTRO=bookworm +ARG LINSTOR_VERSION + +# ------------------------------------------------------------------------------ +# Build linstor-server from source +FROM debian:bookworm AS builder + +ARG LINSTOR_VERSION +ARG VERSION=${LINSTOR_VERSION} +ARG DISTRO + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update \ + && apt-get -y upgrade \ + && apt-get -y install build-essential git default-jdk-headless python3-all debhelper wget unzip && \ + wget https://services.gradle.org/distributions/gradle-8.9-bin.zip -O /tmp/gradle.zip && \ + unzip -d /opt /tmp/gradle.zip && \ + rm /tmp/gradle.zip && \ + ln -s /opt/gradle-8.9/bin/gradle /usr/local/bin/gradle + +RUN git clone https://github.com/LINBIT/linstor-server.git /linstor-server +WORKDIR /linstor-server +RUN git checkout v${VERSION} + +# Apply patches +COPY patches /patches +RUN git apply /patches/*.diff && \ + git config user.email "build@cozystack.io" && \ + git config user.name "Cozystack Builder" && \ + git add -A && \ + git commit -m "Apply patches" + +# Initialize git submodules +RUN git submodule update --init --recursive || make check-submods + +# Pre-download ALL dependencies before make tarball +# This ensures all transitive dependencies are cached, including optional ones like AWS SDK +RUN ./gradlew getProtoc +RUN ./gradlew generateJava +RUN ./gradlew --no-daemon --gradle-user-home .gradlehome downloadDependencies + +# Manually create tarball without removing caches +# make tarball removes .gradlehome/caches/[0-9]* which deletes dependencies +# So we'll do the steps manually but keep the caches +RUN make check-submods versioninfo gen-java FORCE=1 VERSION=${VERSION} +RUN make server/jar.deps controller/jar.deps satellite/jar.deps jclcrypto/jar.deps FORCE=1 VERSION=${VERSION} +RUN make .filelist FORCE=1 VERSION=${VERSION} PRESERVE_DEBIAN=1 +# Don't remove caches - we need them for offline build +RUN rm -Rf .gradlehome/wrapper .gradlehome/native .gradlehome/.tmp || true +RUN mkdir -p ./libs +RUN make tgz VERSION=${VERSION} + +# Extract tarball and build DEB packages from it +RUN mv linstor-server-${VERSION}.tar.gz /linstor-server_${VERSION}.orig.tar.gz \ + && tar -C / -xvf /linstor-server_${VERSION}.orig.tar.gz + +WORKDIR /linstor-server-${VERSION} +# Verify .gradlehome is present in extracted tarball +RUN test -d .gradlehome && echo ".gradlehome found in tarball" || (echo ".gradlehome not found in tarball!" && exit 1) + +# Build DEB packages from tarball +# Override GRADLE_FLAGS to remove --offline flag, allowing Gradle to download missing dependencies +RUN sed -i 's/GRADLE_FLAGS = --offline/GRADLE_FLAGS =/' debian/rules || true +RUN LD_LIBRARY_PATH='' dpkg-buildpackage -rfakeroot -b -uc + +# Copy built .deb packages to a location accessible from final image +# dpkg-buildpackage creates packages in parent directory +RUN mkdir -p /packages-output && \ + find .. -maxdepth 1 -name "linstor-*.deb" -exec cp {} /packages-output/ \; && \ + test -n "$(ls -A /packages-output)" || (echo "ERROR: No linstor .deb packages found after build." && exit 1) + +# ------------------------------------------------------------------------------ +# Final image +FROM debian:${DISTRO} + +LABEL maintainer="Roland Kammerer " + +ARG LINSTOR_VERSION +ARG DISTRO + +# Copy built .deb packages from builder stage +# dpkg-buildpackage creates packages in parent directory, we copied them to /packages-output +COPY --from=builder /packages-output/ /packages/ + +RUN { echo 'APT::Install-Recommends "false";' ; echo 'APT::Install-Suggests "false";' ; } > /etc/apt/apt.conf.d/99_piraeus + +RUN --mount=type=cache,target=/var/cache,sharing=private \ + --mount=type=cache,target=/var/lib/apt/lists,sharing=private \ + --mount=type=tmpfs,target=/var/log \ + # Install wget first for downloading keyring + apt-get update && apt-get install -y wget ca-certificates && \ + # Enable contrib repos for zfsutils \ + . /etc/os-release && \ + sed -i -r 's/^Components: (.*)$/Components: \1 contrib/' /etc/apt/sources.list.d/debian.sources && \ + echo "deb http://deb.debian.org/debian $VERSION_CODENAME-backports contrib" > /etc/apt/sources.list.d/backports.list && \ + wget https://packages.linbit.com/public/linbit-keyring.deb -O /var/cache/linbit-keyring.deb && \ + dpkg -i /var/cache/linbit-keyring.deb && \ + echo "deb http://packages.linbit.com/public $VERSION_CODENAME misc" > /etc/apt/sources.list.d/linbit.list && \ + apt-get update && \ + # Install useful utilities and general dependencies + apt-get install -y udev drbd-utils jq net-tools iputils-ping iproute2 dnsutils netcat-traditional sysstat curl util-linux && \ + # Install dependencies for optional features \ + apt-get install -y \ + # cryptsetup: luks layer + cryptsetup \ + # e2fsprogs: LINSTOR can create file systems \ + e2fsprogs \ + # lsscsi: exos layer \ + lsscsi \ + # lvm2: manage lvm storage pools \ + lvm2 \ + # multipath-tools: exos layer \ + multipath-tools \ + # nvme-cli: nvme layer + nvme-cli \ + # procps: used by LINSTOR to find orphaned send/receive processes \ + procps \ + # socat: used with thin-send-recv to send snapshots to another LINSTOR cluster + socat \ + # thin-send-recv: used to send/receive snapshots of LVM thin volumes \ + thin-send-recv \ + # xfsprogs: LINSTOR can create file systems; xfs deps \ + xfsprogs \ + # zstd: used with thin-send-recv to send snapshots to another LINSTOR cluster \ + zstd \ + # zfsutils-linux: for zfs storage pools \ + zfsutils-linux/$VERSION_CODENAME-backports \ + && \ + # remove udev, no need for it in the container \ + apt-get remove -y udev && \ + # Install linstor packages from built .deb files and linstor-client from repository + apt-get install -y default-jre-headless python3-all python3-natsort linstor-client \ + && ls packages/*.deb >/dev/null && (dpkg -i packages/*.deb || apt-get install -f -y) \ + && rm -rf /packages \ + && sed -i 's/"-Djdk.tls.acknowledgeCloseNotify=true"//g' /usr/share/linstor-server/bin/Controller \ + && apt-get clean + +# Log directory need to be group writable. OpenShift assigns random UID and GID, without extra RBAC changes we can only influence the GID. +RUN mkdir /var/log/linstor-controller && \ + chown 0:1000 /var/log/linstor-controller && \ + chmod -R 0775 /var/log/linstor-controller && \ + # Ensure we log to files in containers, otherwise SOS reports won't show any logs at all + sed -i 's###' /usr/share/linstor-server/lib/conf/logback.xml + + +RUN lvmconfig --type current --mergedconfig --config 'activation { udev_sync = 0 udev_rules = 0 monitoring = 0 } devices { global_filter = [ "r|^/dev/drbd|" ] obtain_device_list_from_udev = 0}' > /etc/lvm/lvm.conf.new && mv /etc/lvm/lvm.conf.new /etc/lvm/lvm.conf +RUN echo 'global { usage-count no; }' > /etc/drbd.d/global_common.conf + +# controller +EXPOSE 3376/tcp 3377/tcp 3370/tcp 3371/tcp + +# satellite +EXPOSE 3366/tcp 3367/tcp + +RUN wget https://raw.githubusercontent.com/piraeusdatastore/piraeus/refs/heads/master/dockerfiles/piraeus-server/entry.sh -O /usr/bin/piraeus-entry.sh \ + && chmod +x /usr/bin/piraeus-entry.sh + +ARG K8S_AWAIT_ELECTION_VERSION=v0.4.2 +# TARGETARCH is a docker special variable: https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope +ARG TARGETARCH + +RUN wget https://github.com/LINBIT/k8s-await-election/releases/download/${K8S_AWAIT_ELECTION_VERSION}/k8s-await-election-${K8S_AWAIT_ELECTION_VERSION}-linux-${TARGETARCH}.tar.gz -O - | tar -xvz -C /usr/bin/ + +ARG LOSETUP_CONTAINER_VERSION=v1.0.1 +RUN wget "https://github.com/LINBIT/losetup-container/releases/download/${LOSETUP_CONTAINER_VERSION}/losetup-container-$(uname -m)-unknown-linux-gnu.tar.gz" -O - | tar -xvz -C /usr/local/sbin && \ + printf '#!/bin/sh\nLOSETUP_CONTAINER_ORIGINAL_LOSETUP=%s exec /usr/local/sbin/losetup-container "$@"\n' $(command -v losetup) > /usr/local/sbin/losetup && \ + chmod +x /usr/local/sbin/losetup + +RUN wget "https://dl.k8s.io/$(wget -O - https://dl.k8s.io/release/stable.txt)/bin/linux/${TARGETARCH}/kubectl" -O /usr/local/bin/kubectl && chmod +x /usr/local/bin/kubectl + +CMD ["startSatellite"] +ENTRYPOINT ["/usr/bin/k8s-await-election", "/usr/bin/piraeus-entry.sh"] diff --git a/packages/system/linstor/images/piraeus-server/patches/README.md b/packages/system/linstor/images/piraeus-server/patches/README.md new file mode 100644 index 00000000..c219eaf3 --- /dev/null +++ b/packages/system/linstor/images/piraeus-server/patches/README.md @@ -0,0 +1,12 @@ +# LINSTOR Server Patches + +Custom patches for piraeus-server (linstor-server) v1.32.3. + +- **adjust-on-resfile-change.diff** — Use actual device path in res file during toggle-disk; fix LUKS data offset + - Upstream: [#473](https://github.com/LINBIT/linstor-server/pull/473), [#472](https://github.com/LINBIT/linstor-server/pull/472) +- **allow-toggle-disk-retry.diff** — Allow retry and cancellation of failed toggle-disk operations + - Upstream: [#475](https://github.com/LINBIT/linstor-server/pull/475) +- **force-metadata-check-on-disk-add.diff** — Create metadata during toggle-disk from diskless to diskful + - Upstream: [#474](https://github.com/LINBIT/linstor-server/pull/474) +- **skip-adjust-when-device-inaccessible.diff** — Skip DRBD adjust/res file regeneration when child layer device is inaccessible + - Upstream: [#471](https://github.com/LINBIT/linstor-server/pull/471) diff --git a/packages/system/linstor/images/piraeus-server/patches/adjust-on-resfile-change.diff b/packages/system/linstor/images/piraeus-server/patches/adjust-on-resfile-change.diff new file mode 100644 index 00000000..6788efaf --- /dev/null +++ b/packages/system/linstor/images/piraeus-server/patches/adjust-on-resfile-change.diff @@ -0,0 +1,48 @@ +diff --git a/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/ConfFileBuilder.java b/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/ConfFileBuilder.java +index 36c52ccf8..c0bb7b967 100644 +--- a/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/ConfFileBuilder.java ++++ b/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/ConfFileBuilder.java +@@ -894,12 +894,16 @@ public class ConfFileBuilder + if (((Volume) vlmData.getVolume()).getFlags().isUnset(localAccCtx, Volume.Flags.DELETE)) + { + final String disk; ++ // Check if we're in toggle-disk operation (adding disk to diskless resource) ++ boolean isDiskAdding = vlmData.getVolume().getAbsResource().getStateFlags().isSomeSet( ++ localAccCtx, Resource.Flags.DISK_ADD_REQUESTED, Resource.Flags.DISK_ADDING); + if ((!isPeerRsc && vlmData.getDataDevice() == null) || + (isPeerRsc && + // FIXME: vlmData.getRscLayerObject().getFlags should be used here + vlmData.getVolume().getAbsResource().disklessForDrbdPeers(accCtx) + ) || +- (!isPeerRsc && ++ // For toggle-disk: if dataDevice is set and we're adding disk, use the actual device ++ (!isPeerRsc && !isDiskAdding && + // FIXME: vlmData.getRscLayerObject().getFlags should be used here + vlmData.getVolume().getAbsResource().isDrbdDiskless(accCtx) + ) +diff --git a/satellite/src/main/java/com/linbit/linstor/layer/luks/CryptSetupCommands.java b/satellite/src/main/java/com/linbit/linstor/layer/luks/CryptSetupCommands.java +index 54dd5c19f..018de58cf 100644 +--- a/satellite/src/main/java/com/linbit/linstor/layer/luks/CryptSetupCommands.java ++++ b/satellite/src/main/java/com/linbit/linstor/layer/luks/CryptSetupCommands.java +@@ -34,6 +34,9 @@ public class CryptSetupCommands implements Luks + private static final Version V2_1_0 = new Version(2, 1, 0); + private static final Version V2_0_0 = new Version(2, 0, 0); + private static final String PBDKF_MAX_MEMORY_KIB = "262144"; // 256 MiB ++ // Fixed LUKS2 data offset in 512-byte sectors (16 MiB = 32768 sectors) ++ // This ensures consistent LUKS header size across all nodes regardless of system defaults ++ private static final String LUKS2_DATA_OFFSET_SECTORS = "32768"; + + @SuppressWarnings("unused") + private final ErrorReporter errorReporter; +@@ -78,6 +81,11 @@ public class CryptSetupCommands implements Luks + command.add(CRYPTSETUP); + command.add("-q"); + command.add("luksFormat"); ++ // Always specify explicit offset to ensure consistent LUKS header size across all nodes ++ // Without this, different systems may create LUKS with different header sizes (16MiB vs 32MiB) ++ // which causes "Low.dev. smaller than requested DRBD-dev. size" errors during toggle-disk ++ command.add("--offset"); ++ command.add(LUKS2_DATA_OFFSET_SECTORS); + if (version.greaterOrEqual(V2_0_0)) + { + command.add("--pbkdf-memory"); diff --git a/packages/system/linstor/images/piraeus-server/patches/allow-toggle-disk-retry.diff b/packages/system/linstor/images/piraeus-server/patches/allow-toggle-disk-retry.diff new file mode 100644 index 00000000..4f4f2fc6 --- /dev/null +++ b/packages/system/linstor/images/piraeus-server/patches/allow-toggle-disk-retry.diff @@ -0,0 +1,235 @@ +diff --git a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java +index 1a6f7b7f0..bd447e049 100644 +--- a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java ++++ b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java +@@ -58,7 +58,9 @@ import com.linbit.linstor.stateflags.StateFlags; + import com.linbit.linstor.storage.StorageException; + import com.linbit.linstor.storage.data.adapter.drbd.DrbdRscData; + import com.linbit.linstor.storage.interfaces.categories.resource.AbsRscLayerObject; ++import com.linbit.linstor.storage.interfaces.categories.resource.VlmProviderObject; + import com.linbit.linstor.storage.kinds.DeviceLayerKind; ++import com.linbit.linstor.storage.kinds.DeviceProviderKind; + import com.linbit.linstor.storage.utils.LayerUtils; + import com.linbit.linstor.tasks.AutoDiskfulTask; + import com.linbit.linstor.utils.layer.LayerRscUtils; +@@ -317,21 +319,90 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL + + Resource rsc = ctrlApiDataLoader.loadRsc(nodeName, rscName, true); + ++ // Allow retry of the same operation if the previous attempt failed ++ // (the requested flag remains set for retry on reconnection, but we should also allow manual retry) ++ // Also allow cancellation of a failed operation by requesting the opposite operation + if (hasDiskAddRequested(rsc)) + { +- throw new ApiRcException(ApiCallRcImpl.simpleEntry( +- ApiConsts.FAIL_RSC_BUSY, +- "Addition of disk to resource already requested", +- true +- )); ++ if (removeDisk) ++ { ++ // User wants to cancel the failed add-disk operation and go back to diskless ++ // Use the existing disk removal flow to properly cleanup storage on satellite ++ errorReporter.logInfo( ++ "Toggle Disk cancel on %s/%s - cancelling failed DISK_ADD_REQUESTED, reverting to diskless", ++ nodeNameStr, rscNameStr); ++ unmarkDiskAddRequested(rsc); ++ // Also clear DISK_ADDING if it was set ++ unmarkDiskAdding(rsc); ++ ++ // Set storage pool to diskless pool (overwrite the diskful pool that was set) ++ Props rscProps = ctrlPropsHelper.getProps(rsc); ++ rscProps.map().put(ApiConsts.KEY_STOR_POOL_NAME, LinStor.DISKLESS_STOR_POOL_NAME); ++ ++ // Set DISK_REMOVE_REQUESTED to use the existing disk removal flow ++ // This will: ++ // 1. updateAndAdjustDisk sets DISK_REMOVING flag ++ // 2. Satellite sees DISK_REMOVING and deletes LUKS/storage devices ++ // 3. finishOperation rebuilds layer stack as diskless ++ // We keep the existing layer data so satellite can properly cleanup ++ markDiskRemoveRequested(rsc); ++ ++ ctrlTransactionHelper.commit(); ++ ++ // Use existing disk removal flow - this will properly cleanup storage on satellite ++ return Flux ++ .just(ApiCallRcImpl.singleApiCallRc( ++ ApiConsts.MODIFIED, ++ "Cancelling disk addition, reverting to diskless" ++ )) ++ .concatWith(updateAndAdjustDisk(nodeName, rscName, true, toggleIntoTiebreakerRef, context)) ++ .concatWith(ctrlRscDfnApiCallHandler.get().updateProps(rsc.getResourceDefinition())); ++ } ++ // If adding disk and DISK_ADD_REQUESTED is already set, treat as retry ++ // First clean up partially created storage by removing and recreating layer data ++ errorReporter.logInfo( ++ "Toggle Disk retry on %s/%s - DISK_ADD_REQUESTED already set, cleaning up and retrying", ++ nodeNameStr, rscNameStr); ++ ++ // Remove old layer data and recreate to ensure clean state ++ // This forces satellite to delete any partially created storage and start fresh ++ LayerPayload payload = new LayerPayload(); ++ copyDrbdNodeIdIfExists(rsc, payload); ++ List layerList = removeLayerData(rsc); ++ ctrlLayerStackHelper.ensureStackDataExists(rsc, layerList, payload); ++ ++ ctrlTransactionHelper.commit(); ++ return Flux ++ .just(new ApiCallRcImpl()) ++ .concatWith(updateAndAdjustDisk(nodeName, rscName, false, toggleIntoTiebreakerRef, context)) ++ .concatWith(ctrlRscDfnApiCallHandler.get().updateProps(rsc.getResourceDefinition())); + } + if (hasDiskRemoveRequested(rsc)) + { +- throw new ApiRcException(ApiCallRcImpl.simpleEntry( +- ApiConsts.FAIL_RSC_BUSY, +- "Removal of disk from resource already requested", +- true +- )); ++ if (!removeDisk) ++ { ++ // User wants to cancel the failed remove-disk operation ++ errorReporter.logInfo( ++ "Toggle Disk cancel on %s/%s - cancelling failed DISK_REMOVE_REQUESTED", ++ nodeNameStr, rscNameStr); ++ unmarkDiskRemoveRequested(rsc); ++ ctrlTransactionHelper.commit(); ++ return Flux.just( ++ ApiCallRcImpl.singleApiCallRc( ++ ApiConsts.MODIFIED, ++ "Cancelled disk removal request" ++ ) ++ ); ++ } ++ // If removing disk and DISK_REMOVE_REQUESTED is already set, treat as retry ++ errorReporter.logInfo( ++ "Toggle Disk retry on %s/%s - DISK_REMOVE_REQUESTED already set, continuing operation", ++ nodeNameStr, rscNameStr); ++ ctrlTransactionHelper.commit(); ++ return Flux ++ .just(new ApiCallRcImpl()) ++ .concatWith(updateAndAdjustDisk(nodeName, rscName, true, toggleIntoTiebreakerRef, context)) ++ .concatWith(ctrlRscDfnApiCallHandler.get().updateProps(rsc.getResourceDefinition())); + } + + if (!removeDisk && !ctrlVlmCrtApiHelper.isDiskless(rsc)) +@@ -342,17 +413,43 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL + true + )); + } ++ ResourceDefinition rscDfn = rsc.getResourceDefinition(); ++ AccessContext peerCtx = peerAccCtx.get(); ++ + if (removeDisk && ctrlVlmCrtApiHelper.isDiskless(rsc)) + { ++ // Resource is marked as diskless - check if it has orphaned storage layers that need cleanup ++ AbsRscLayerObject layerData = getLayerData(peerCtx, rsc); ++ if (layerData != null && (LayerUtils.hasLayer(layerData, DeviceLayerKind.LUKS) || ++ hasNonDisklessStorageLayer(layerData))) ++ { ++ // Resource is marked as diskless but has orphaned storage layers - need cleanup ++ // Use the existing disk removal flow to properly cleanup storage on satellite ++ errorReporter.logInfo( ++ "Toggle Disk cleanup on %s/%s - resource is diskless but has orphaned storage layers, cleaning up", ++ nodeNameStr, rscNameStr); ++ ++ // Set DISK_REMOVE_REQUESTED to use the existing disk removal flow ++ // This will trigger proper satellite cleanup via DISK_REMOVING flag ++ markDiskRemoveRequested(rsc); ++ ++ ctrlTransactionHelper.commit(); ++ ++ // Use existing disk removal flow - this will properly cleanup storage on satellite ++ return Flux ++ .just(ApiCallRcImpl.singleApiCallRc( ++ ApiConsts.MODIFIED, ++ "Cleaning up orphaned storage layers" ++ )) ++ .concatWith(updateAndAdjustDisk(nodeName, rscName, true, toggleIntoTiebreakerRef, context)) ++ .concatWith(ctrlRscDfnApiCallHandler.get().updateProps(rsc.getResourceDefinition())); ++ } + throw new ApiRcException(ApiCallRcImpl.simpleEntry( + ApiConsts.WARN_RSC_ALREADY_DISKLESS, + "Resource already diskless", + true + )); + } +- +- ResourceDefinition rscDfn = rsc.getResourceDefinition(); +- AccessContext peerCtx = peerAccCtx.get(); + if (removeDisk) + { + // Prevent removal of the last disk +@@ -1324,6 +1421,30 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL + } + } + ++ private void unmarkDiskAddRequested(Resource rsc) ++ { ++ try ++ { ++ rsc.getStateFlags().disableFlags(apiCtx, Resource.Flags.DISK_ADD_REQUESTED); ++ } ++ catch (AccessDeniedException | DatabaseException exc) ++ { ++ throw new ImplementationError(exc); ++ } ++ } ++ ++ private void unmarkDiskRemoveRequested(Resource rsc) ++ { ++ try ++ { ++ rsc.getStateFlags().disableFlags(apiCtx, Resource.Flags.DISK_REMOVE_REQUESTED); ++ } ++ catch (AccessDeniedException | DatabaseException exc) ++ { ++ throw new ImplementationError(exc); ++ } ++ } ++ + private void markDiskAdded(Resource rscData) + { + try +@@ -1389,6 +1510,41 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL + return layerData; + } + ++ /** ++ * Check if the layer stack has a non-diskless STORAGE layer. ++ * This is used to detect orphaned storage layers that need cleanup. ++ */ ++ private boolean hasNonDisklessStorageLayer(AbsRscLayerObject layerDataRef) ++ { ++ boolean hasNonDiskless = false; ++ if (layerDataRef != null) ++ { ++ if (layerDataRef.getLayerKind() == DeviceLayerKind.STORAGE) ++ { ++ for (VlmProviderObject vlmData : layerDataRef.getVlmLayerObjects().values()) ++ { ++ if (vlmData.getProviderKind() != DeviceProviderKind.DISKLESS) ++ { ++ hasNonDiskless = true; ++ break; ++ } ++ } ++ } ++ if (!hasNonDiskless) ++ { ++ for (AbsRscLayerObject child : layerDataRef.getChildren()) ++ { ++ if (hasNonDisklessStorageLayer(child)) ++ { ++ hasNonDiskless = true; ++ break; ++ } ++ } ++ } ++ } ++ return hasNonDiskless; ++ } ++ + private LockGuard createLockGuard() + { + return lockGuardFactory.buildDeferred(LockType.WRITE, LockObj.NODES_MAP, LockObj.RSC_DFN_MAP); diff --git a/packages/system/linstor/images/piraeus-server/patches/force-metadata-check-on-disk-add.diff b/packages/system/linstor/images/piraeus-server/patches/force-metadata-check-on-disk-add.diff new file mode 100644 index 00000000..0c413005 --- /dev/null +++ b/packages/system/linstor/images/piraeus-server/patches/force-metadata-check-on-disk-add.diff @@ -0,0 +1,63 @@ +diff --git a/satellite/src/main/java/com/linbit/linstor/layer/drbd/DrbdLayer.java b/satellite/src/main/java/com/linbit/linstor/layer/drbd/DrbdLayer.java +index a302ee835..01967a31f 100644 +--- a/satellite/src/main/java/com/linbit/linstor/layer/drbd/DrbdLayer.java ++++ b/satellite/src/main/java/com/linbit/linstor/layer/drbd/DrbdLayer.java +@@ -371,10 +371,13 @@ public class DrbdLayer implements DeviceLayer + boolean isDiskless = drbdRscData.getAbsResource().isDrbdDiskless(workerCtx); + StateFlags rscFlags = drbdRscData.getAbsResource().getStateFlags(); + boolean isDiskRemoving = rscFlags.isSet(workerCtx, Resource.Flags.DISK_REMOVING); ++ // Check if we're in toggle-disk operation (adding disk to diskless resource) ++ boolean isDiskAdding = rscFlags.isSomeSet(workerCtx, Resource.Flags.DISK_ADD_REQUESTED, Resource.Flags.DISK_ADDING); + + boolean contProcess = isDiskless; + +- boolean processChildren = !isDiskless || isDiskRemoving; ++ // Process children when: has disk, removing disk, OR adding disk (toggle-disk) ++ boolean processChildren = !isDiskless || isDiskRemoving || isDiskAdding; + // do not process children when ONLY DRBD_DELETE flag is set (DELETE flag is still unset) + processChildren &= (!rscFlags.isSet(workerCtx, Resource.Flags.DRBD_DELETE) || + rscFlags.isSet(workerCtx, Resource.Flags.DELETE)); +@@ -570,7 +573,11 @@ public class DrbdLayer implements DeviceLayer + { + // hasMetaData needs to be run after child-resource processed + List> createMetaData = new ArrayList<>(); +- if (!drbdRscData.getAbsResource().isDrbdDiskless(workerCtx) && !skipDisk) ++ // Check if we're in toggle-disk operation (adding disk to diskless resource) ++ boolean isDiskAddingForMd = drbdRscData.getAbsResource().getStateFlags() ++ .isSomeSet(workerCtx, Resource.Flags.DISK_ADD_REQUESTED, Resource.Flags.DISK_ADDING); ++ // Create metadata when: has disk OR adding disk (toggle-disk), and skipDisk is disabled ++ if ((!drbdRscData.getAbsResource().isDrbdDiskless(workerCtx) || isDiskAddingForMd) && !skipDisk) + { + // do not try to create meta data while the resource is diskless or skipDisk is enabled + for (DrbdVlmData drbdVlmData : checkMetaData) +@@ -988,8 +995,10 @@ public class DrbdLayer implements DeviceLayer + { + List> checkMetaData = new ArrayList<>(); + Resource rsc = drbdRscData.getAbsResource(); ++ // Include DISK_ADD_REQUESTED/DISK_ADDING for toggle-disk scenario where we need to check/create metadata + if (!rsc.isDrbdDiskless(workerCtx) || +- rsc.getStateFlags().isSet(workerCtx, Resource.Flags.DISK_REMOVING) ++ rsc.getStateFlags().isSet(workerCtx, Resource.Flags.DISK_REMOVING) || ++ rsc.getStateFlags().isSomeSet(workerCtx, Resource.Flags.DISK_ADD_REQUESTED, Resource.Flags.DISK_ADDING) + ) + { + // using a dedicated list to prevent concurrentModificationException +@@ -1177,9 +1186,16 @@ public class DrbdLayer implements DeviceLayer + + boolean hasMetaData; + ++ // Check if we need to verify/create metadata ++ // Force metadata check when: ++ // 1. checkMetaData is enabled ++ // 2. volume doesn't have disk yet (diskless -> diskful transition) ++ // 3. DISK_ADD_REQUESTED/DISK_ADDING flag is set (retry scenario where storage exists but no metadata) ++ boolean isDiskAddingState = drbdVlmData.getRscLayerObject().getAbsResource().getStateFlags() ++ .isSomeSet(workerCtx, Resource.Flags.DISK_ADD_REQUESTED, Resource.Flags.DISK_ADDING); + if (drbdVlmData.checkMetaData() || +- // when adding a disk, DRBD believes that it is diskless but we still need to create metadata +- !drbdVlmData.hasDisk()) ++ !drbdVlmData.hasDisk() || ++ isDiskAddingState) + { + if (mdUtils.hasMetaData()) + { diff --git a/packages/system/linstor/images/piraeus-server/patches/skip-adjust-when-device-inaccessible.diff b/packages/system/linstor/images/piraeus-server/patches/skip-adjust-when-device-inaccessible.diff new file mode 100644 index 00000000..09e7ccf9 --- /dev/null +++ b/packages/system/linstor/images/piraeus-server/patches/skip-adjust-when-device-inaccessible.diff @@ -0,0 +1,93 @@ +diff --git a/satellite/src/main/java/com/linbit/linstor/layer/drbd/DrbdLayer.java b/satellite/src/main/java/com/linbit/linstor/layer/drbd/DrbdLayer.java +index 01967a3..871d830 100644 +--- a/satellite/src/main/java/com/linbit/linstor/layer/drbd/DrbdLayer.java ++++ b/satellite/src/main/java/com/linbit/linstor/layer/drbd/DrbdLayer.java +@@ -592,7 +592,29 @@ public class DrbdLayer implements DeviceLayer + // The .res file might not have been generated in the prepare method since it was + // missing information from the child-layers. Now that we have processed them, we + // need to make sure the .res file exists in all circumstances. +- regenerateResFile(drbdRscData); ++ // However, if the underlying devices are not accessible (e.g., LUKS device is closed ++ // during resource deletion), we skip regenerating the res file to avoid errors ++ boolean canRegenerateResFile = true; ++ if (!skipDisk && !drbdRscData.getAbsResource().isDrbdDiskless(workerCtx)) ++ { ++ AbsRscLayerObject dataChild = drbdRscData.getChildBySuffix(RscLayerSuffixes.SUFFIX_DATA); ++ if (dataChild != null) ++ { ++ for (DrbdVlmData drbdVlmData : drbdRscData.getVlmLayerObjects().values()) ++ { ++ VlmProviderObject childVlm = dataChild.getVlmProviderObject(drbdVlmData.getVlmNr()); ++ if (childVlm == null || !childVlm.exists() || childVlm.getDevicePath() == null) ++ { ++ canRegenerateResFile = false; ++ break; ++ } ++ } ++ } ++ } ++ if (canRegenerateResFile) ++ { ++ regenerateResFile(drbdRscData); ++ } + + // createMetaData needs rendered resFile + for (DrbdVlmData drbdVlmData : createMetaData) +@@ -766,19 +788,47 @@ public class DrbdLayer implements DeviceLayer + + if (drbdRscData.isAdjustRequired()) + { +- try ++ // Check if underlying devices are accessible before adjusting ++ // This is important for encrypted resources (LUKS) where the device ++ // might be closed during deletion ++ boolean canAdjust = true; ++ if (!skipDisk && !drbdRscData.getAbsResource().isDrbdDiskless(workerCtx)) + { +- drbdUtils.adjust( +- drbdRscData, +- false, +- skipDisk, +- false +- ); ++ AbsRscLayerObject dataChild = drbdRscData.getChildBySuffix(RscLayerSuffixes.SUFFIX_DATA); ++ if (dataChild != null) ++ { ++ for (DrbdVlmData drbdVlmData : drbdRscData.getVlmLayerObjects().values()) ++ { ++ VlmProviderObject childVlm = dataChild.getVlmProviderObject(drbdVlmData.getVlmNr()); ++ if (childVlm == null || !childVlm.exists() || childVlm.getDevicePath() == null) ++ { ++ canAdjust = false; ++ break; ++ } ++ } ++ } + } +- catch (ExtCmdFailedException extCmdExc) ++ ++ if (canAdjust) ++ { ++ try ++ { ++ drbdUtils.adjust( ++ drbdRscData, ++ false, ++ skipDisk, ++ false ++ ); ++ } ++ catch (ExtCmdFailedException extCmdExc) ++ { ++ restoreBackupResFile(drbdRscData); ++ throw extCmdExc; ++ } ++ } ++ else + { +- restoreBackupResFile(drbdRscData); +- throw extCmdExc; ++ drbdRscData.setAdjustRequired(false); + } + } + diff --git a/packages/system/linstor/templates/_helpers.tpl b/packages/system/linstor/templates/_helpers.tpl deleted file mode 100644 index 20d43863..00000000 --- a/packages/system/linstor/templates/_helpers.tpl +++ /dev/null @@ -1,24 +0,0 @@ -{{- define "cozy.linstor.version" -}} -{{- $piraeusConfigMap := lookup "v1" "ConfigMap" "cozy-linstor" "piraeus-operator-image-config"}} -{{- if not $piraeusConfigMap }} - {{- fail "Piraeus controller is not yet installed, ConfigMap cozy-linstor/piraeus-operator-image-config is missing" }} -{{- end }} -{{- $piraeusImagesConfig := $piraeusConfigMap | dig "data" "0_piraeus_datastore_images.yaml" nil | required "No image config" | fromYaml }} -base: {{ $piraeusImagesConfig.base | required "No image base in piraeus config" }} -controller: - image: {{ $piraeusImagesConfig | dig "components" "linstor-controller" "image" nil | required "No controller image" }} - tag: {{ $piraeusImagesConfig | dig "components" "linstor-controller" "tag" nil | required "No controller tag" }} -satellite: - image: {{ $piraeusImagesConfig | dig "components" "linstor-satellite" "image" nil | required "No satellite image" }} - tag: {{ $piraeusImagesConfig | dig "components" "linstor-satellite" "tag" nil | required "No satellite tag" }} -{{- end -}} - -{{- define "cozy.linstor.version.controller" -}} -{{- $version := (include "cozy.linstor.version" .) | fromYaml }} -{{- printf "%s/%s:%s" $version.base $version.controller.image $version.controller.tag }} -{{- end -}} - -{{- define "cozy.linstor.version.satellite" -}} -{{- $version := (include "cozy.linstor.version" .) | fromYaml }} -{{- printf "%s/%s:%s" $version.base $version.satellite.image $version.satellite.tag }} -{{- end -}} diff --git a/packages/system/linstor/templates/cluster.yaml b/packages/system/linstor/templates/cluster.yaml index 68af90fd..584fd850 100644 --- a/packages/system/linstor/templates/cluster.yaml +++ b/packages/system/linstor/templates/cluster.yaml @@ -27,8 +27,10 @@ spec: podTemplate: spec: containers: + - name: linstor-controller + image: {{ .Values.piraeusServer.image.repository }}:{{ .Values.piraeusServer.image.tag }} - name: plunger - image: {{ include "cozy.linstor.version.controller" . }} + image: {{ .Values.piraeusServer.image.repository }}:{{ .Values.piraeusServer.image.tag }} command: - "/scripts/plunger-controller.sh" securityContext: diff --git a/packages/system/linstor/templates/satellites-cozy.yaml b/packages/system/linstor/templates/satellites-cozy.yaml index a4c1baa7..c621126d 100644 --- a/packages/system/linstor/templates/satellites-cozy.yaml +++ b/packages/system/linstor/templates/satellites-cozy.yaml @@ -13,6 +13,7 @@ spec: hostNetwork: true containers: - name: linstor-satellite + image: {{ .Values.piraeusServer.image.repository }}:{{ .Values.piraeusServer.image.tag }} securityContext: # real-world installations need some debugging from time to time readOnlyRootFilesystem: false diff --git a/packages/system/linstor/templates/satellites-plunger.yaml b/packages/system/linstor/templates/satellites-plunger.yaml index e3cfa3b1..ab71298b 100644 --- a/packages/system/linstor/templates/satellites-plunger.yaml +++ b/packages/system/linstor/templates/satellites-plunger.yaml @@ -11,7 +11,7 @@ spec: spec: containers: - name: plunger - image: {{ include "cozy.linstor.version.satellite" . }} + image: {{ .Values.piraeusServer.image.repository }}:{{ .Values.piraeusServer.image.tag }} command: - "/scripts/plunger-satellite.sh" securityContext: @@ -48,7 +48,7 @@ spec: name: script-volume readOnly: true - name: drbd-logger - image: {{ include "cozy.linstor.version.satellite" . }} + image: {{ .Values.piraeusServer.image.repository }}:{{ .Values.piraeusServer.image.tag }} command: - "/scripts/plunger-drbd-logger.sh" securityContext: diff --git a/packages/system/linstor/values.yaml b/packages/system/linstor/values.yaml index 8b137891..23179bce 100644 --- a/packages/system/linstor/values.yaml +++ b/packages/system/linstor/values.yaml @@ -1 +1,4 @@ - +piraeusServer: + image: + repository: ghcr.io/cozystack/cozystack/piraeus-server + tag: latest@sha256:417532baa2801288147cd9ac9ae260751c1a7754f0b829725d09b72a770c111a