Resolve guest custom-rule alert paths

Fix snapshot, backup, powered-off, and config-reevaluation guest threshold resolution by routing them through shared guest alert context instead of override-only fallback paths.\n\nFixes #1418
This commit is contained in:
rcourtman 2026-04-15 17:25:07 +01:00
parent f72ff5df5c
commit 183fd41dd0
4 changed files with 585 additions and 17 deletions

View file

@ -82,6 +82,14 @@ runtime must migrate the active alert, history entry, acknowledgment record,
suppression/rate-limit/flapping tracking, and guest per-disk metric identity
to the current canonical state instead of reopening a duplicate alert or
resolving only the stale node-scoped identity.
That same guest-threshold owner also governs guest-derived lifecycle and
posture alerts. Snapshot age, backup age, powered-off state, and
configuration-change reevaluation must all construct a canonical lightweight
guest snapshot and route threshold resolution through the shared
guest-defaults → filter-driven custom rules → guest-override chain.
Passing `nil` guest context or resolving only overrides/defaults is forbidden
because it silently bypasses custom guest rules and makes guest lifecycle
alerting diverge from running-guest metric truth.
That same guest-alert owner also has to retire per-disk guest alerts when the
guest stops, disk alerting is disabled, or the reported disk set changes.
Canonical guest disk identity is only valid while the guest still exposes that

View file

@ -2162,22 +2162,34 @@ func (m *Manager) reevaluateActiveAlertsLocked() {
alertsToResolve = append(alertsToResolve, alertID)
continue
}
// We need to evaluate custom rules, but we don't have the guest object here.
// For now, we'll mark these alerts for re-evaluation by the monitor.
// The next poll cycle will properly evaluate them with custom rules.
thresholds := m.resolveGuestThresholdOverride(cloneThresholdConfig(m.config.GuestDefaults), nil, resourceID)
if thresholds.Disabled {
guestThresholds := m.getGuestThresholds(guestSnapshotFromAlert(alert, resourceID), resourceID)
if guestThresholds.Disabled {
alertsToResolve = append(alertsToResolve, alertID)
continue
}
// If no custom rule context is available, reevaluation still uses the
// shared default+override resolution path and waits for the next poll
// to apply filter-driven guest rules.
// Note: This doesn't consider custom rules - those will be evaluated
// on the next poll cycle when we have the full guest object
threshold = getThresholdForMetric(thresholds, metricType)
switch alert.Type {
case "snapshot-age":
if !snapshotAlertStillTriggered(alert, m.resolvedSnapshotAlertConfigNoLock(guestThresholds)) {
alertsToResolve = append(alertsToResolve, alertID)
}
continue
case "backup-age":
if !backupAlertStillTriggered(alert, m.resolvedBackupAlertConfigNoLock(guestThresholds)) {
alertsToResolve = append(alertsToResolve, alertID)
}
continue
case "powered-off":
if guestThresholds.DisableConnectivity {
alertsToResolve = append(alertsToResolve, alertID)
continue
}
alert.Level = normalizePoweredOffSeverity(guestThresholds.PoweredOffSeverity)
continue
}
threshold = getThresholdForMetric(guestThresholds, metricType)
}
// If no threshold found or threshold is disabled (trigger <= 0), resolve the alert
@ -2260,6 +2272,83 @@ func (m *Manager) reevaluateActiveAlertsLocked() {
}
}
func (m *Manager) resolvedSnapshotAlertConfigNoLock(thresholds ThresholdConfig) SnapshotAlertConfig {
cfg := m.config.SnapshotDefaults
if thresholds.Snapshot != nil {
cfg = *thresholds.Snapshot
}
return cfg
}
func (m *Manager) resolvedBackupAlertConfigNoLock(thresholds ThresholdConfig) BackupAlertConfig {
cfg := m.config.BackupDefaults
if thresholds.Backup != nil {
cfg = *thresholds.Backup
}
if cfg.AlertOrphaned == nil {
alertOrphaned := true
cfg.AlertOrphaned = &alertOrphaned
}
return cfg
}
func snapshotAlertStillTriggered(alert *Alert, cfg SnapshotAlertConfig) bool {
if alert == nil || !cfg.Enabled {
return false
}
ageValue, _ := metadataFloatValue(alert.Metadata, "snapshotAgeDays")
sizeValue, _ := metadataFloatValue(alert.Metadata, "snapshotSizeGiB")
if cfg.CriticalDays > 0 && ageValue >= float64(cfg.CriticalDays) {
return true
}
if cfg.WarningDays > 0 && ageValue >= float64(cfg.WarningDays) {
return true
}
if cfg.CriticalSizeGiB > 0 && sizeValue >= cfg.CriticalSizeGiB {
return true
}
if cfg.WarningSizeGiB > 0 && sizeValue >= cfg.WarningSizeGiB {
return true
}
return false
}
func backupAlertStillTriggered(alert *Alert, cfg BackupAlertConfig) bool {
if alert == nil || !cfg.Enabled {
return false
}
vmid := metadataStringValue(alert.Metadata, "guestVmid")
if vmid == "" {
if parsed := metadataIntValue(alert.Metadata["guestVmid"]); parsed > 0 {
vmid = strconv.Itoa(parsed)
}
}
if backupIgnoreVMID(vmid, cfg.IgnoreVMIDs) {
return false
}
if metadataBoolValue(alert.Metadata, "orphaned") && cfg.AlertOrphaned != nil && !*cfg.AlertOrphaned {
return false
}
ageValue, ok := metadataFloatValue(alert.Metadata, "ageDays")
if !ok {
ageValue = alert.Value
}
if cfg.CriticalDays > 0 && ageValue >= float64(cfg.CriticalDays) {
return true
}
if cfg.WarningDays > 0 && ageValue >= float64(cfg.WarningDays) {
return true
}
return false
}
// ReevaluateGuestAlert reevaluates a specific guest's alerts with full threshold resolution including custom rules
// This should be called by the monitor with the current guest state
func (m *Manager) ReevaluateGuestAlert(guest any, guestID string) {
@ -6094,8 +6183,10 @@ func (m *Manager) CheckSnapshotsForInstance(instanceName string, snapshots []mod
// Determine thresholds for this snapshot
resourceID := fmt.Sprintf("%s:%s:%d", snapshot.Instance, snapshot.Node, snapshot.VMID)
guestName := strings.TrimSpace(guestNames[BuildGuestKey(snapshot.Instance, snapshot.Node, snapshot.VMID)])
guestContext := guestSnapshotFromIdentity(resourceID, guestName, snapshot.Node, snapshot.Instance, snapshot.Type, "")
m.mu.RLock()
gh := m.getGuestThresholds(nil, resourceID)
gh := m.getGuestThresholds(guestContext, resourceID)
m.mu.RUnlock()
if gh.Disabled {
@ -6153,7 +6244,6 @@ func (m *Manager) CheckSnapshotsForInstance(instanceName string, snapshots []mod
alertID := fmt.Sprintf("snapshot-age-%s", snapshot.ID)
guestKey := BuildGuestKey(snapshot.Instance, snapshot.Node, snapshot.VMID)
guestName := strings.TrimSpace(guestNames[guestKey])
guestType := "VM"
if strings.EqualFold(snapshot.Type, "lxc") {
@ -6484,9 +6574,14 @@ func (m *Manager) CheckBackups(
// Determine thresholds for this backup
currentBackupCfg := backupCfg
if record.lookup.ResourceID != "" {
guestContext := guestSnapshotFromLookup(record.lookup, record.fallbackName)
guestResourceID := strings.TrimSpace(record.lookup.ResourceID)
if guestResourceID == "" {
guestResourceID = guestContext.ID
}
if guestResourceID != "" {
m.mu.RLock()
gh := m.getGuestThresholds(nil, record.lookup.ResourceID)
gh := m.getGuestThresholds(guestContext, guestResourceID)
m.mu.RUnlock()
if gh.Disabled {
continue
@ -6576,6 +6671,12 @@ func (m *Manager) CheckBackups(
"lastBackupTime": record.lastTime,
"ageDays": ageDays,
"thresholdDays": threshold,
"guestName": displayName,
"guestType": record.lookup.Type,
"guestInstance": instance,
"guestNode": node,
"guestVmid": metadataIntValue(record.vmID),
"orphaned": record.vmID != "" && guestResourceID == "",
}
specResourceID := canonicalBackupSubjectResourceID(alertKey, *record)
specResourceType := canonicalBackupSubjectResourceType(*record)
@ -7865,6 +7966,17 @@ func metadataIntValue(value interface{}) int {
return 0
}
func metadataFloatValue(metadata map[string]interface{}, key string) (float64, bool) {
if metadata == nil {
return 0, false
}
value, ok := metadata[key]
if !ok {
return 0, false
}
return numericConditionValue(value)
}
func metadataStringValue(metadata map[string]interface{}, key string) string {
if metadata == nil {
return ""
@ -9223,7 +9335,7 @@ func (m *Manager) clearStorageOfflineAlert(storage models.Storage) {
// checkGuestPoweredOff creates an alert for powered-off guests
func (m *Manager) checkGuestPoweredOff(guestID, name, node, instanceName, guestType string, monitorOnly bool) {
m.mu.RLock()
thresholds := m.resolveGuestThresholdOverride(cloneThresholdConfig(m.config.GuestDefaults), nil, guestID)
thresholds := m.getGuestThresholds(guestSnapshotFromIdentity(guestID, name, node, instanceName, guestType, "stopped"), guestID)
m.mu.RUnlock()
m.checkGuestPoweredOffWithThresholds(guestID, name, node, instanceName, guestType, thresholds, monitorOnly)
}

View file

@ -1,6 +1,10 @@
package alerts
import "github.com/rcourtman/pulse-go-rewrite/internal/models"
import (
"strings"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
)
type guestKind uint8
@ -126,12 +130,124 @@ func guestSnapshotFromContainer(container models.Container) guestSnapshot {
}.normalizeCollections()
}
func guestKindFromType(guestType string) guestKind {
switch strings.ToLower(strings.TrimSpace(guestType)) {
case "qemu", "vm":
return guestKindVM
case "lxc", "container", "system-container":
return guestKindContainer
default:
return guestKindUnknown
}
}
func guestSnapshotFromIdentity(resourceID, name, node, instance, guestType, status string) guestSnapshot {
snapshot := guestSnapshot{
Kind: guestKindFromType(guestType),
ID: strings.TrimSpace(resourceID),
Name: strings.TrimSpace(name),
Node: strings.TrimSpace(node),
Instance: strings.TrimSpace(instance),
Status: strings.TrimSpace(status),
}
if ident, ok := guestOverrideIdentityFromGuestOrID(nil, resourceID); ok {
if snapshot.VMID <= 0 {
snapshot.VMID = ident.vmid
}
if snapshot.Instance == "" {
snapshot.Instance = ident.instance
}
if snapshot.Node == "" {
snapshot.Node = ident.node
}
}
if snapshot.Instance == "" {
snapshot.Instance = snapshot.Node
}
return snapshot.normalizeCollections()
}
func guestSnapshotFromLookup(lookup GuestLookup, fallbackName string) guestSnapshot {
resourceID := strings.TrimSpace(lookup.ResourceID)
if resourceID == "" && lookup.Instance != "" && lookup.Node != "" && lookup.VMID > 0 {
resourceID = BuildGuestKey(lookup.Instance, lookup.Node, lookup.VMID)
}
name := strings.TrimSpace(lookup.Name)
if name == "" {
name = strings.TrimSpace(fallbackName)
}
snapshot := guestSnapshotFromIdentity(resourceID, name, lookup.Node, lookup.Instance, lookup.Type, "")
if snapshot.VMID <= 0 && lookup.VMID > 0 {
snapshot.VMID = lookup.VMID
}
return snapshot.normalizeCollections()
}
func guestSnapshotFromAlert(alert *Alert, resourceID string) guestSnapshot {
if alert == nil {
return guestSnapshotFromIdentity(resourceID, "", "", "", "", "")
}
name := metadataStringValue(alert.Metadata, "guestName")
if name == "" {
name = alert.ResourceName
}
node := metadataStringValue(alert.Metadata, "guestNode")
if node == "" {
node = alert.Node
}
instance := metadataStringValue(alert.Metadata, "guestInstance")
if instance == "" {
instance = alert.Instance
}
guestType := metadataStringValue(alert.Metadata, "guestType")
if guestType == "" {
guestType = metadataStringValue(alert.Metadata, "resourceType")
}
status := metadataStringValue(alert.Metadata, "guestStatus")
if status == "" {
status = metadataStringValue(alert.Metadata, "status")
}
snapshot := guestSnapshotFromIdentity(resourceID, name, node, instance, guestType, status)
if snapshot.VMID <= 0 {
snapshot.VMID = metadataIntValue(alert.Metadata["guestVmid"])
}
return snapshot.normalizeCollections()
}
func extractGuestSnapshot(guest any) (guestSnapshot, bool) {
switch g := guest.(type) {
case models.VM:
return guestSnapshotFromVM(g), true
case *models.VM:
if g == nil {
return emptyGuestSnapshot(), false
}
return guestSnapshotFromVM(*g), true
case models.Container:
return guestSnapshotFromContainer(g), true
case *models.Container:
if g == nil {
return emptyGuestSnapshot(), false
}
return guestSnapshotFromContainer(*g), true
case guestSnapshot:
return g.normalizeCollections(), true
case *guestSnapshot:
if g == nil {
return emptyGuestSnapshot(), false
}
return g.normalizeCollections(), true
default:
return emptyGuestSnapshot(), false
}

View file

@ -2,9 +2,11 @@ package alerts
import (
"testing"
"time"
alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/recovery"
)
func boolPtr(v bool) *bool {
@ -318,3 +320,333 @@ func TestCheckStorageOfflineUsesSharedThresholdResolution(t *testing.T) {
t.Fatalf("expected storage offline confirmations to clear when shared thresholds disable connectivity")
}
}
func TestCheckSnapshotsUsesGuestContextForCustomRules(t *testing.T) {
m := newTestManager(t)
m.ClearActiveAlerts()
cfg := AlertConfig{
Enabled: true,
SnapshotDefaults: SnapshotAlertConfig{
Enabled: true,
WarningDays: 7,
CriticalDays: 14,
},
CustomRules: []CustomAlertRule{
{
Name: "db-snapshots",
Enabled: true,
Priority: 10,
FilterConditions: FilterStack{
LogicalOperator: "AND",
Filters: []FilterCondition{
{Type: "text", Field: "name", Value: "db"},
},
},
Thresholds: ThresholdConfig{
Snapshot: &SnapshotAlertConfig{
Enabled: true,
WarningDays: 15,
CriticalDays: 20,
},
},
},
},
}
m.UpdateConfig(cfg)
m.mu.Lock()
m.config.TimeThresholds = map[string]int{}
m.mu.Unlock()
now := time.Now()
snapshots := []models.GuestSnapshot{
{
ID: "inst-node-100-weekly",
Name: "weekly",
Node: "node",
Instance: "inst",
Type: "qemu",
VMID: 100,
Time: now.Add(-10 * 24 * time.Hour),
},
{
ID: "inst-node-101-weekly",
Name: "weekly",
Node: "node",
Instance: "inst",
Type: "qemu",
VMID: 101,
Time: now.Add(-10 * 24 * time.Hour),
},
}
guestNames := map[string]string{
BuildGuestKey("inst", "node", 100): "db-server",
BuildGuestKey("inst", "node", 101): "web-server",
}
m.CheckSnapshotsForInstance("inst", snapshots, guestNames)
m.mu.RLock()
_, dbExists := testLookupActiveAlert(t, m, "snapshot-age-inst-node-100-weekly")
_, webExists := testLookupActiveAlert(t, m, "snapshot-age-inst-node-101-weekly")
m.mu.RUnlock()
if dbExists {
t.Fatalf("expected db snapshot alert to be suppressed by custom rule thresholds")
}
if !webExists {
t.Fatalf("expected non-matching snapshot alert to use default thresholds")
}
}
func TestCheckBackupsUsesGuestContextForCustomRules(t *testing.T) {
m := newTestManager(t)
m.ClearActiveAlerts()
cfg := AlertConfig{
Enabled: true,
BackupDefaults: BackupAlertConfig{
Enabled: true,
WarningDays: 7,
CriticalDays: 14,
},
CustomRules: []CustomAlertRule{
{
Name: "db-backups",
Enabled: true,
Priority: 10,
FilterConditions: FilterStack{
LogicalOperator: "AND",
Filters: []FilterCondition{
{Type: "text", Field: "name", Value: "db"},
},
},
Thresholds: ThresholdConfig{
Backup: &BackupAlertConfig{
Enabled: true,
WarningDays: 15,
CriticalDays: 20,
},
},
},
},
}
m.UpdateConfig(cfg)
now := time.Now()
rollups := []recovery.ProtectionRollup{
{
RollupID: "db-rollup",
SubjectRef: &recovery.ExternalRef{
Type: "proxmox-vm",
Namespace: "inst",
Name: "db-server",
ID: BuildGuestKey("inst", "node", 100),
Class: "node",
},
LastSuccessAt: ptrTime(now.Add(-10 * 24 * time.Hour)),
LastOutcome: recovery.OutcomeSuccess,
Providers: []recovery.Provider{recovery.ProviderProxmoxPVE},
},
{
RollupID: "web-rollup",
SubjectRef: &recovery.ExternalRef{
Type: "proxmox-vm",
Namespace: "inst",
Name: "web-server",
ID: BuildGuestKey("inst", "node", 101),
Class: "node",
},
LastSuccessAt: ptrTime(now.Add(-10 * 24 * time.Hour)),
LastOutcome: recovery.OutcomeSuccess,
Providers: []recovery.Provider{recovery.ProviderProxmoxPVE},
},
}
guest100 := GuestLookup{
ResourceID: BuildGuestKey("inst", "node", 100),
Name: "db-server",
Instance: "inst",
Node: "node",
Type: "qemu",
VMID: 100,
}
guest101 := GuestLookup{
ResourceID: BuildGuestKey("inst", "node", 101),
Name: "web-server",
Instance: "inst",
Node: "node",
Type: "qemu",
VMID: 101,
}
guestsByKey := map[string]GuestLookup{
guest100.ResourceID: guest100,
guest101.ResourceID: guest101,
}
guestsByVMID := map[string][]GuestLookup{
"100": {guest100},
"101": {guest101},
}
m.CheckBackups(rollups, guestsByKey, guestsByVMID)
m.mu.RLock()
_, dbExists := testLookupActiveAlert(t, m, "backup-age-"+sanitizeAlertKey(guest100.ResourceID))
_, webExists := testLookupActiveAlert(t, m, "backup-age-"+sanitizeAlertKey(guest101.ResourceID))
m.mu.RUnlock()
if dbExists {
t.Fatalf("expected db backup alert to be suppressed by custom rule thresholds")
}
if !webExists {
t.Fatalf("expected non-matching backup alert to use default thresholds")
}
}
func TestReevaluateActiveAlertsUsesGuestContextForMetricCustomRules(t *testing.T) {
m := newTestManager(t)
resourceID := BuildGuestKey("pve1", "node1", 100)
m.mu.Lock()
m.config.Enabled = true
m.config.GuestDefaults = ThresholdConfig{
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
}
m.config.CustomRules = []CustomAlertRule{
{
Name: "db-metrics",
Enabled: true,
Priority: 10,
FilterConditions: FilterStack{
LogicalOperator: "AND",
Filters: []FilterCondition{
{Type: "text", Field: "name", Value: "db"},
},
},
Thresholds: ThresholdConfig{
CPU: &HysteresisThreshold{Trigger: 95, Clear: 90},
},
},
}
state, alert := testNewCanonicalAlert(resourceID, canonicalMetricSpecID(resourceID, "cpu"), string(alertspecs.AlertSpecKindMetricThreshold), "cpu")
alert.Value = 90
alert.Threshold = 80
alert.ResourceName = "db-server"
alert.Node = "node1"
alert.Instance = "pve1"
alert.Metadata = map[string]interface{}{
"resourceType": "vm",
}
m.setActiveAlertNoLock(state, alert)
m.reevaluateActiveAlertsLocked()
m.mu.Unlock()
m.mu.RLock()
_, exists := m.activeAlerts[state]
m.mu.RUnlock()
if exists {
t.Fatalf("expected guest metric alert to resolve when custom rule raises the trigger")
}
}
func TestReevaluateActiveAlertsUsesGuestContextForBackupCustomRules(t *testing.T) {
m := newTestManager(t)
resourceID := BuildGuestKey("pve1", "node1", 100)
m.mu.Lock()
m.config.Enabled = true
m.config.BackupDefaults = BackupAlertConfig{
Enabled: true,
WarningDays: 7,
CriticalDays: 14,
}
m.config.CustomRules = []CustomAlertRule{
{
Name: "db-backup-reeval",
Enabled: true,
Priority: 10,
FilterConditions: FilterStack{
LogicalOperator: "AND",
Filters: []FilterCondition{
{Type: "text", Field: "name", Value: "db"},
},
},
Thresholds: ThresholdConfig{
Backup: &BackupAlertConfig{
Enabled: true,
WarningDays: 15,
CriticalDays: 20,
},
},
},
}
state, alert := testNewCanonicalAlert(resourceID, resourceID+"-backup-age", string(alertspecs.AlertSpecKindPostureThreshold), "backup-age")
alert.Value = 10
alert.Threshold = 7
alert.ResourceName = "db-server backup"
alert.Node = "node1"
alert.Instance = "pve1"
alert.Metadata = map[string]interface{}{
"ageDays": 10.0,
"guestName": "db-server",
"guestType": "qemu",
"guestInstance": "pve1",
"guestNode": "node1",
"guestVmid": 100,
"orphaned": false,
}
m.setActiveAlertNoLock(state, alert)
m.reevaluateActiveAlertsLocked()
m.mu.Unlock()
m.mu.RLock()
_, exists := m.activeAlerts[state]
m.mu.RUnlock()
if exists {
t.Fatalf("expected guest backup alert to resolve when custom rule raises backup thresholds")
}
}
func TestReevaluateActiveAlertsUsesGuestContextForPoweredOffCustomRules(t *testing.T) {
m := newTestManager(t)
resourceID := BuildGuestKey("pve1", "node1", 100)
m.mu.Lock()
m.config.Enabled = true
m.config.CustomRules = []CustomAlertRule{
{
Name: "db-no-powered-off",
Enabled: true,
Priority: 10,
FilterConditions: FilterStack{
LogicalOperator: "AND",
Filters: []FilterCondition{
{Type: "text", Field: "name", Value: "db"},
},
},
Thresholds: ThresholdConfig{
DisableConnectivity: true,
},
},
}
state, alert := testNewCanonicalAlert(resourceID, canonicalPoweredStateSpecID(resourceID), string(alertspecs.AlertSpecKindPoweredState), "powered-off")
alert.ResourceName = "db-server"
alert.Node = "node1"
alert.Instance = "pve1"
alert.Metadata = map[string]interface{}{
"resourceType": "vm",
}
m.setActiveAlertNoLock(state, alert)
m.reevaluateActiveAlertsLocked()
m.mu.Unlock()
m.mu.RLock()
_, exists := m.activeAlerts[state]
m.mu.RUnlock()
if exists {
t.Fatalf("expected guest powered-off alert to resolve when custom rule disables connectivity")
}
}