From a6ac0485a8ec31b441be02a20c44230f8fc225aa Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Wed, 29 Apr 2026 17:30:08 +0200 Subject: [PATCH] fix(linstor): plunger demotes idle-Primary DRBD resources DRBD auto-promotes a resource to Primary during the provisioning path (e.g. mkfs.ext4 against a freshly created replica), but auto-demote sometimes fails to fire on close. The resource then stays Primary on a node that has no consumer, and CSI on the legitimate target node fails NodePublishVolume with "failed to set source device readwrite". The only recovery is an explicit drbdadm secondary on the stuck node. Add a per-satellite plunger handler that detects local Primaries where every device has open == false and demotes them. Both the detection and the action are local to the satellite, so no peer coordination is needed. open:no makes drbdadm secondary safe; if a real consumer is about to arrive, auto-promote will redo it. Upstream tracker: piraeusdatastore/piraeus-operator#942 Co-Authored-By: Claude Signed-off-by: Andrei Kvapil --- .../linstor/hack/plunger/plunger-satellite.sh | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/packages/system/linstor/hack/plunger/plunger-satellite.sh b/packages/system/linstor/hack/plunger/plunger-satellite.sh index 4a488a51..da4da04d 100755 --- a/packages/system/linstor/hack/plunger/plunger-satellite.sh +++ b/packages/system/linstor/hack/plunger/plunger-satellite.sh @@ -20,6 +20,34 @@ drbd_status_json() { drbdsetup status --json 2>/dev/null || true } +# Detect DRBD resources stuck in Primary on this node with no consumer. +# Auto-promote can promote a resource to Primary (e.g. during mkfs.ext4 +# at provisioning time), but auto-demote sometimes fails to fire on +# close. The resource then stays Primary on a node that has no +# consumer and blocks NodePublishVolume on the actual target node with +# "failed to set source device readwrite". Upstream tracker: +# https://github.com/piraeusdatastore/piraeus-operator/issues/942 +# +# `open == false` on every device of the resource guarantees nobody is +# using it, so `drbdadm secondary` is safe — at worst it's a no-op +# that auto-promote will redo when a real consumer arrives. +drbd_fix_idle_primary() { + local json + json="$(drbd_status_json)" + [ -n "$json" ] || return 0 + + printf '%s' "$json" | jq -r ' + .[]? + | select(.role == "Primary") + | select(all(.devices[]?; .open == false)) + | .name + ' | while IFS= read -r res; do + [ -n "$res" ] || continue + log "Idle Primary detected: res=$res open:no -> drbdadm secondary" + drbdadm secondary "$res" || log "WARN: secondary failed for $res" + done +} + # Detect DRBD resources where resync is stuck: # - at least one local device is Inconsistent # - there is an active SyncTarget peer @@ -154,6 +182,9 @@ while true; do drbdadm up "${secondary}" || echo "Command failed" ); done + # Detect and demote idle Primary resources stuck after auto-promote + drbd_fix_idle_primary || true + # Detect and fix stalled DRBD resync by switching SyncTarget peer drbd_fix_stalled_sync || true