diff --git a/internal/monitoring/monitor_pve.go b/internal/monitoring/monitor_pve.go index 032ce12118..24f447e1e3 100644 --- a/internal/monitoring/monitor_pve.go +++ b/internal/monitoring/monitor_pve.go @@ -837,6 +837,12 @@ func (m *Monitor) pollGuestsWithFallback( return nil } +// hostAgentLinkSettleWindow is how long after monitor start disk alert +// evaluation waits for nodes that have no linked host agent yet. Agent +// linkage only exists once agents re-report after a reload, and evaluating +// before then treats missing --disk-exclude patterns as no exclusions. +const hostAgentLinkSettleWindow = 2 * time.Minute + func (m *Monitor) maybePollPhysicalDisksAsync( ctx context.Context, instanceName string, @@ -939,6 +945,13 @@ func (m *Monitor) maybePollPhysicalDisksAsync( for _, h := range hosts { hostByID[h.ID] = h } + // Host-agent linkage is rebuilt from live agent reports, and a full + // monitor reload (every settings save) starts from empty host state. + // A disk poll that runs before a node's agent has re-reported sees no + // exclusions and raises alerts that the next cycle immediately + // resolves (#1674). Track which nodes are linked so alert evaluation + // can wait out that window for the rest. + linkedNodes := make(map[string]bool) // Also map node name → linked host agent SMART inventory so a node whose // Proxmox disk query fails can still populate its physical disks from // the agent's smartctl view (#1516). @@ -947,6 +960,7 @@ func (m *Monitor) maybePollPhysicalDisksAsync( if n.LinkedAgentID == "" || n.Instance != inst { continue } + linkedNodes[n.Name] = true if linkedHost, ok := hostByID[n.LinkedAgentID]; ok { if len(linkedHost.DiskExclude) > 0 { diskExcludeByNode[n.Name] = linkedHost.DiskExclude @@ -1110,10 +1124,18 @@ func (m *Monitor) maybePollPhysicalDisksAsync( allDisks[index] = preserveUnavailablePhysicalDiskEvidence(allDisks[index], previous) } } + awaitingAgentLinks := time.Since(m.startTime) < hostAgentLinkSettleWindow for _, disk := range allDisks { if !polledNodes[disk.Node] { continue } + if awaitingAgentLinks && !linkedNodes[disk.Node] { + log.Debug(). + Str("node", disk.Node). + Str("disk", disk.DevPath). + Msg("Deferring disk alert evaluation until host-agent linkage settles") + continue + } log.Debug(). Str("node", disk.Node). diff --git a/internal/monitoring/monitor_pve_disk_alert_settle_test.go b/internal/monitoring/monitor_pve_disk_alert_settle_test.go new file mode 100644 index 0000000000..a92c01c796 --- /dev/null +++ b/internal/monitoring/monitor_pve_disk_alert_settle_test.go @@ -0,0 +1,162 @@ +package monitoring + +import ( + "context" + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/alerts" + "github.com/rcourtman/pulse-go-rewrite/internal/config" + "github.com/rcourtman/pulse-go-rewrite/internal/models" + "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox" +) + +type wornDiskPVEClient struct { + fakeStorageClient +} + +func (*wornDiskPVEClient) GetDisks(ctx context.Context, node string) ([]proxmox.Disk, error) { + return []proxmox.Disk{{ + DevPath: "/dev/sdb", + Model: "WDC WDS500G1R0A", + Serial: "1234", + Type: "sata", + Health: "PASSED", + Wearout: 1, + Size: 500107862016, + }}, nil +} + +// node1DiskAlerts filters to alerts raised for the test disk so persisted +// active alerts loaded from a previous test's manager cannot leak in. +func node1DiskAlerts(m *Monitor) []alerts.Alert { + var matched []alerts.Alert + for _, alert := range m.alertManager.GetActiveAlerts() { + if (alert.Type == "disk-wearout" || alert.Type == "disk-health") && alert.Node == "node1" { + matched = append(matched, alert) + } + } + return matched +} + +func waitForPhysicalDisks(t *testing.T, m *Monitor) { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for { + if len(m.state.GetSnapshot().PhysicalDisks) > 0 { + return + } + if time.Now().After(deadline) { + t.Fatal("expected physical disks to land in state") + } + time.Sleep(10 * time.Millisecond) + } +} + +// A monitor reload rebuilds host state from live agent reports, so a disk +// poll that runs before a node's agent has re-reported must not evaluate +// disk alerts for that node: it would treat the agent's --disk-exclude +// patterns as absent and fire alerts the next cycle resolves (#1674). +func TestMaybePollPhysicalDisksAsync_DefersAlertsUntilAgentLinkSettles(t *testing.T) { + m := &Monitor{ + state: models.NewState(), + lastPhysicalDiskPoll: make(map[string]time.Time), + alertManager: alerts.NewManager(), + startTime: time.Now(), + } + + m.state.UpdateNodesForInstance("pve1", []models.Node{{ + ID: "pve1-node1", + Name: "node1", + Instance: "pve1", + }}) + + m.maybePollPhysicalDisksAsync( + context.Background(), + "pve1", + &config.PVEInstance{}, + &wornDiskPVEClient{}, + []proxmox.Node{{Node: "node1", Status: "online"}}, + map[string]string{"node1": "online"}, + []models.Node{{Name: "node1"}}, + ) + + waitForPhysicalDisks(t, m) + if alertsNow := node1DiskAlerts(m); len(alertsNow) != 0 { + t.Fatalf("expected no disk alerts during the agent-link settle window, got %+v", alertsNow) + } +} + +func TestMaybePollPhysicalDisksAsync_EvaluatesAlertsOnceSettled(t *testing.T) { + m := &Monitor{ + state: models.NewState(), + lastPhysicalDiskPoll: make(map[string]time.Time), + alertManager: alerts.NewManager(), + startTime: time.Now().Add(-2 * hostAgentLinkSettleWindow), + } + + m.state.UpdateNodesForInstance("pve1", []models.Node{{ + ID: "pve1-node1", + Name: "node1", + Instance: "pve1", + }}) + + m.maybePollPhysicalDisksAsync( + context.Background(), + "pve1", + &config.PVEInstance{}, + &wornDiskPVEClient{}, + []proxmox.Node{{Node: "node1", Status: "online"}}, + map[string]string{"node1": "online"}, + []models.Node{{Name: "node1"}}, + ) + + waitForPhysicalDisks(t, m) + found := false + for _, alert := range node1DiskAlerts(m) { + if alert.Type == "disk-wearout" { + found = true + } + } + if !found { + t.Fatalf("expected a disk-wearout alert once the settle window has passed, got %+v", m.alertManager.GetActiveAlerts()) + } +} + +// A node whose agent is already linked evaluates immediately even inside the +// settle window, and the agent's --disk-exclude patterns suppress the alert. +func TestMaybePollPhysicalDisksAsync_LinkedNodeSuppressesExcludedDisk(t *testing.T) { + m := &Monitor{ + state: models.NewState(), + lastPhysicalDiskPoll: make(map[string]time.Time), + alertManager: alerts.NewManager(), + startTime: time.Now(), + } + + m.state.UpsertHost(models.Host{ + ID: "host-1", + Hostname: "node1", + DiskExclude: []string{"sdb"}, + }) + m.state.UpdateNodesForInstance("pve1", []models.Node{{ + ID: "pve1-node1", + Name: "node1", + Instance: "pve1", + LinkedAgentID: "host-1", + }}) + + m.maybePollPhysicalDisksAsync( + context.Background(), + "pve1", + &config.PVEInstance{}, + &wornDiskPVEClient{}, + []proxmox.Node{{Node: "node1", Status: "online"}}, + map[string]string{"node1": "online"}, + []models.Node{{Name: "node1"}}, + ) + + waitForPhysicalDisks(t, m) + if alertsNow := node1DiskAlerts(m); len(alertsNow) != 0 { + t.Fatalf("expected agent disk exclusions to suppress the wearout alert, got %+v", alertsNow) + } +}