mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-07-10 00:14:38 +00:00
fix(monitoring): mark PVE nodes offline when the whole instance is unreachable
pollPVEInstance returned early on a GetNodes error, so a host that stopped answering entirely (shutdown, network loss) kept its last successful snapshot in state forever: node row frozen at online with stale uptime. The existing 60s offline grace policy only ran for cluster-reported peer-down and empty-poll cycles, never for a whole instance going dark. Route the poll error path through the same grace policy: nodes of an unreachable instance stay online (connection degraded) within the grace window and flip to offline with cleared uptime and CPU once it lapses, matching the cluster-reported node-down behavior. Verified live: registered a synthetic PVE instance, killed it, and the node flips to offline one poll cycle after the grace period. Fixes #1441
This commit is contained in:
parent
c511c935b3
commit
02955619f5
2 changed files with 153 additions and 0 deletions
|
|
@ -613,6 +613,25 @@ func (m *Monitor) preserveNodesWhenEmpty(instanceName string, modelNodes []model
|
|||
// Mark connection health as degraded to reflect polling failure
|
||||
m.setProviderConnectionHealth(InstanceTypePVE, instanceName, false)
|
||||
|
||||
return m.preserveOrExpireNodes(prevInstanceNodes)
|
||||
}
|
||||
|
||||
// markPVEInstanceNodesUnreachable applies the node offline grace policy to an
|
||||
// instance whose poll failed outright (connection refused, timeout, all
|
||||
// cluster endpoints down). Without this, the early return on a poll error
|
||||
// leaves the last successful snapshot in state forever, so a shut-down host
|
||||
// keeps showing its final online status (#1441).
|
||||
func (m *Monitor) markPVEInstanceNodesUnreachable(instanceName string) {
|
||||
_, prevInstanceNodes := m.snapshotPrevNodes(instanceName)
|
||||
if len(prevInstanceNodes) == 0 {
|
||||
return
|
||||
}
|
||||
m.state.UpdateNodesForInstance(instanceName, m.preserveOrExpireNodes(prevInstanceNodes))
|
||||
}
|
||||
|
||||
// preserveOrExpireNodes keeps recently seen nodes online through transient
|
||||
// gaps and marks nodes offline once the grace period has lapsed.
|
||||
func (m *Monitor) preserveOrExpireNodes(prevInstanceNodes []models.Node) []models.Node {
|
||||
preserved := make([]models.Node, 0, len(prevInstanceNodes))
|
||||
now := time.Now()
|
||||
for _, prevNode := range prevInstanceNodes {
|
||||
|
|
@ -1044,6 +1063,7 @@ func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, clie
|
|||
nodes, updatedClient, err := m.fetchPVENodes(ctx, instanceName, instanceCfg, client)
|
||||
if err != nil {
|
||||
pollErr = err
|
||||
m.markPVEInstanceNodesUnreachable(instanceName)
|
||||
return
|
||||
}
|
||||
client = updatedClient
|
||||
|
|
|
|||
133
internal/monitoring/monitor_pve_unreachable_test.go
Normal file
133
internal/monitoring/monitor_pve_unreachable_test.go
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
package monitoring
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
|
||||
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
|
||||
)
|
||||
|
||||
func TestPreserveOrExpireNodes(t *testing.T) {
|
||||
m := &Monitor{
|
||||
state: models.NewState(),
|
||||
nodeLastOnline: make(map[string]time.Time),
|
||||
}
|
||||
|
||||
t.Run("within grace keeps node online with degraded health", func(t *testing.T) {
|
||||
preserved := m.preserveOrExpireNodes([]models.Node{{
|
||||
ID: "inst-node1",
|
||||
Name: "node1",
|
||||
Status: "online",
|
||||
ConnectionHealth: "error",
|
||||
Uptime: 1234,
|
||||
LastSeen: time.Now().Add(-10 * time.Second),
|
||||
}})
|
||||
if len(preserved) != 1 {
|
||||
t.Fatalf("preserved = %d nodes, want 1", len(preserved))
|
||||
}
|
||||
if preserved[0].Status != "online" {
|
||||
t.Fatalf("Status = %q, want online within grace", preserved[0].Status)
|
||||
}
|
||||
if preserved[0].ConnectionHealth != "degraded" {
|
||||
t.Fatalf("ConnectionHealth = %q, want degraded within grace", preserved[0].ConnectionHealth)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("past grace marks node offline", func(t *testing.T) {
|
||||
preserved := m.preserveOrExpireNodes([]models.Node{{
|
||||
ID: "inst-node1",
|
||||
Name: "node1",
|
||||
Status: "online",
|
||||
ConnectionHealth: "healthy",
|
||||
Uptime: 1234,
|
||||
LastSeen: time.Now().Add(-2 * nodeOfflineGracePeriod),
|
||||
}})
|
||||
if len(preserved) != 1 {
|
||||
t.Fatalf("preserved = %d nodes, want 1", len(preserved))
|
||||
}
|
||||
if preserved[0].Status != "offline" {
|
||||
t.Fatalf("Status = %q, want offline past grace", preserved[0].Status)
|
||||
}
|
||||
if preserved[0].ConnectionHealth != "error" {
|
||||
t.Fatalf("ConnectionHealth = %q, want error past grace", preserved[0].ConnectionHealth)
|
||||
}
|
||||
if preserved[0].Uptime != 0 {
|
||||
t.Fatalf("Uptime = %d, want 0 past grace", preserved[0].Uptime)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
type unreachablePVEClient struct {
|
||||
mockPVEClientExtended
|
||||
}
|
||||
|
||||
func (c *unreachablePVEClient) GetNodes(ctx context.Context) ([]proxmox.Node, error) {
|
||||
return nil, fmt.Errorf("connection refused")
|
||||
}
|
||||
|
||||
// Regression test for #1441: when the whole instance stops answering (host
|
||||
// shut down), the poll error path must still run the offline grace policy
|
||||
// instead of freezing the last online snapshot in state forever.
|
||||
func TestPollPVEInstanceMarksNodesOfflineWhenUnreachable(t *testing.T) {
|
||||
m := &Monitor{
|
||||
config: &config.Config{
|
||||
PVEInstances: []config.PVEInstance{
|
||||
// Non-default port so the portless fallback path stays out of the way.
|
||||
{Name: "pve-test", Host: "https://localhost:9999"},
|
||||
},
|
||||
},
|
||||
state: models.NewState(),
|
||||
pveClients: make(map[string]PVEClientInterface),
|
||||
nodeLastOnline: make(map[string]time.Time),
|
||||
nodeSnapshots: make(map[string]NodeMemorySnapshot),
|
||||
guestSnapshots: make(map[string]GuestMemorySnapshot),
|
||||
metricsHistory: NewMetricsHistory(32, time.Hour),
|
||||
lastClusterCheck: make(map[string]time.Time),
|
||||
lastPhysicalDiskPoll: make(map[string]time.Time),
|
||||
lastPVEBackupPoll: make(map[string]time.Time),
|
||||
authFailures: make(map[string]int),
|
||||
lastAuthAttempt: make(map[string]time.Time),
|
||||
pollStatusMap: make(map[string]*pollStatus),
|
||||
nodePendingUpdatesCache: make(map[string]pendingUpdatesCache),
|
||||
instanceInfoCache: make(map[string]*instanceInfo),
|
||||
lastOutcome: make(map[string]taskOutcome),
|
||||
failureCounts: make(map[string]int),
|
||||
alertManager: alerts.NewManager(),
|
||||
notificationMgr: notifications.NewNotificationManager(""),
|
||||
}
|
||||
defer m.alertManager.Stop()
|
||||
defer m.notificationMgr.Stop()
|
||||
|
||||
// Seed state as if a previous poll saw the node online, longer ago than
|
||||
// the offline grace period.
|
||||
m.state.UpdateNodesForInstance("pve-test", []models.Node{{
|
||||
ID: "pve-test-node1",
|
||||
Name: "node1",
|
||||
Instance: "pve-test",
|
||||
Status: "online",
|
||||
Type: "node",
|
||||
Uptime: 4242,
|
||||
LastSeen: time.Now().Add(-2 * nodeOfflineGracePeriod),
|
||||
}})
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
m.pollPVEInstance(ctx, "pve-test", &unreachablePVEClient{})
|
||||
|
||||
nodes := m.state.GetSnapshot().Nodes
|
||||
if len(nodes) != 1 {
|
||||
t.Fatalf("state nodes = %d, want 1", len(nodes))
|
||||
}
|
||||
if nodes[0].Status != "offline" {
|
||||
t.Fatalf("node Status = %q after unreachable poll past grace, want offline", nodes[0].Status)
|
||||
}
|
||||
if nodes[0].ConnectionHealth != "error" {
|
||||
t.Fatalf("node ConnectionHealth = %q, want error", nodes[0].ConnectionHealth)
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue