fix(monitoring): mark PVE nodes offline when the whole instance is unreachable

pollPVEInstance returned early on a GetNodes error, so a host that
stopped answering entirely (shutdown, network loss) kept its last
successful snapshot in state forever: node row frozen at online with
stale uptime. The existing 60s offline grace policy only ran for
cluster-reported peer-down and empty-poll cycles, never for a whole
instance going dark.

Route the poll error path through the same grace policy: nodes of an
unreachable instance stay online (connection degraded) within the
grace window and flip to offline with cleared uptime and CPU once it
lapses, matching the cluster-reported node-down behavior.

Verified live: registered a synthetic PVE instance, killed it, and the
node flips to offline one poll cycle after the grace period.

Fixes #1441
This commit is contained in:
rcourtman 2026-06-10 16:08:12 +01:00
parent c511c935b3
commit 02955619f5
2 changed files with 153 additions and 0 deletions

View file

@ -613,6 +613,25 @@ func (m *Monitor) preserveNodesWhenEmpty(instanceName string, modelNodes []model
// Mark connection health as degraded to reflect polling failure
m.setProviderConnectionHealth(InstanceTypePVE, instanceName, false)
return m.preserveOrExpireNodes(prevInstanceNodes)
}
// markPVEInstanceNodesUnreachable applies the node offline grace policy to an
// instance whose poll failed outright (connection refused, timeout, all
// cluster endpoints down). Without this, the early return on a poll error
// leaves the last successful snapshot in state forever, so a shut-down host
// keeps showing its final online status (#1441).
func (m *Monitor) markPVEInstanceNodesUnreachable(instanceName string) {
_, prevInstanceNodes := m.snapshotPrevNodes(instanceName)
if len(prevInstanceNodes) == 0 {
return
}
m.state.UpdateNodesForInstance(instanceName, m.preserveOrExpireNodes(prevInstanceNodes))
}
// preserveOrExpireNodes keeps recently seen nodes online through transient
// gaps and marks nodes offline once the grace period has lapsed.
func (m *Monitor) preserveOrExpireNodes(prevInstanceNodes []models.Node) []models.Node {
preserved := make([]models.Node, 0, len(prevInstanceNodes))
now := time.Now()
for _, prevNode := range prevInstanceNodes {
@ -1044,6 +1063,7 @@ func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, clie
nodes, updatedClient, err := m.fetchPVENodes(ctx, instanceName, instanceCfg, client)
if err != nil {
pollErr = err
m.markPVEInstanceNodesUnreachable(instanceName)
return
}
client = updatedClient

View file

@ -0,0 +1,133 @@
package monitoring
import (
"context"
"fmt"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)
func TestPreserveOrExpireNodes(t *testing.T) {
m := &Monitor{
state: models.NewState(),
nodeLastOnline: make(map[string]time.Time),
}
t.Run("within grace keeps node online with degraded health", func(t *testing.T) {
preserved := m.preserveOrExpireNodes([]models.Node{{
ID: "inst-node1",
Name: "node1",
Status: "online",
ConnectionHealth: "error",
Uptime: 1234,
LastSeen: time.Now().Add(-10 * time.Second),
}})
if len(preserved) != 1 {
t.Fatalf("preserved = %d nodes, want 1", len(preserved))
}
if preserved[0].Status != "online" {
t.Fatalf("Status = %q, want online within grace", preserved[0].Status)
}
if preserved[0].ConnectionHealth != "degraded" {
t.Fatalf("ConnectionHealth = %q, want degraded within grace", preserved[0].ConnectionHealth)
}
})
t.Run("past grace marks node offline", func(t *testing.T) {
preserved := m.preserveOrExpireNodes([]models.Node{{
ID: "inst-node1",
Name: "node1",
Status: "online",
ConnectionHealth: "healthy",
Uptime: 1234,
LastSeen: time.Now().Add(-2 * nodeOfflineGracePeriod),
}})
if len(preserved) != 1 {
t.Fatalf("preserved = %d nodes, want 1", len(preserved))
}
if preserved[0].Status != "offline" {
t.Fatalf("Status = %q, want offline past grace", preserved[0].Status)
}
if preserved[0].ConnectionHealth != "error" {
t.Fatalf("ConnectionHealth = %q, want error past grace", preserved[0].ConnectionHealth)
}
if preserved[0].Uptime != 0 {
t.Fatalf("Uptime = %d, want 0 past grace", preserved[0].Uptime)
}
})
}
type unreachablePVEClient struct {
mockPVEClientExtended
}
func (c *unreachablePVEClient) GetNodes(ctx context.Context) ([]proxmox.Node, error) {
return nil, fmt.Errorf("connection refused")
}
// Regression test for #1441: when the whole instance stops answering (host
// shut down), the poll error path must still run the offline grace policy
// instead of freezing the last online snapshot in state forever.
func TestPollPVEInstanceMarksNodesOfflineWhenUnreachable(t *testing.T) {
m := &Monitor{
config: &config.Config{
PVEInstances: []config.PVEInstance{
// Non-default port so the portless fallback path stays out of the way.
{Name: "pve-test", Host: "https://localhost:9999"},
},
},
state: models.NewState(),
pveClients: make(map[string]PVEClientInterface),
nodeLastOnline: make(map[string]time.Time),
nodeSnapshots: make(map[string]NodeMemorySnapshot),
guestSnapshots: make(map[string]GuestMemorySnapshot),
metricsHistory: NewMetricsHistory(32, time.Hour),
lastClusterCheck: make(map[string]time.Time),
lastPhysicalDiskPoll: make(map[string]time.Time),
lastPVEBackupPoll: make(map[string]time.Time),
authFailures: make(map[string]int),
lastAuthAttempt: make(map[string]time.Time),
pollStatusMap: make(map[string]*pollStatus),
nodePendingUpdatesCache: make(map[string]pendingUpdatesCache),
instanceInfoCache: make(map[string]*instanceInfo),
lastOutcome: make(map[string]taskOutcome),
failureCounts: make(map[string]int),
alertManager: alerts.NewManager(),
notificationMgr: notifications.NewNotificationManager(""),
}
defer m.alertManager.Stop()
defer m.notificationMgr.Stop()
// Seed state as if a previous poll saw the node online, longer ago than
// the offline grace period.
m.state.UpdateNodesForInstance("pve-test", []models.Node{{
ID: "pve-test-node1",
Name: "node1",
Instance: "pve-test",
Status: "online",
Type: "node",
Uptime: 4242,
LastSeen: time.Now().Add(-2 * nodeOfflineGracePeriod),
}})
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
m.pollPVEInstance(ctx, "pve-test", &unreachablePVEClient{})
nodes := m.state.GetSnapshot().Nodes
if len(nodes) != 1 {
t.Fatalf("state nodes = %d, want 1", len(nodes))
}
if nodes[0].Status != "offline" {
t.Fatalf("node Status = %q after unreachable poll past grace, want offline", nodes[0].Status)
}
if nodes[0].ConnectionHealth != "error" {
t.Fatalf("node ConnectionHealth = %q, want error", nodes[0].ConnectionHealth)
}
}