From 7ce2cfaa66225d79a61e55fd967e26585a0d160f Mon Sep 17 00:00:00 2001 From: rcourtman Date: Wed, 6 May 2026 14:17:14 +0100 Subject: [PATCH] refactor: split Proxmox guest alerts Move guest metric projection, per-disk guest metric evaluation, powered-off lifecycle alerts, Pulse tag controls, relaxed thresholds, and guest suppression cleanup into internal/alerts/guest.go. Record guest.go as the Proxmox guest alert owner in the alerts subsystem contract and add a Pulse-tag characterization test. Proof: go test ./internal/alerts/... --- .../v6/internal/subsystems/alerts.md | 6 + internal/alerts/alerts.go | 594 ----------------- internal/alerts/guest.go | 604 ++++++++++++++++++ internal/alerts/guest_snapshot_test.go | 13 + 4 files changed, 623 insertions(+), 594 deletions(-) create mode 100644 internal/alerts/guest.go diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index daab68306..7861da3ef 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -66,6 +66,7 @@ operator-facing alert routing behavior for live runtime alerts. 44. `internal/alerts/disk_health.go` 45. `internal/alerts/metric_runtime.go` 46. `internal/alerts/health_assessment.go` +47. `internal/alerts/guest.go` ## Shared Boundaries @@ -288,6 +289,11 @@ alert synchronization, and ZFS device assessment construction for host and storage checkers; future shared health-assessment behavior should extend that owner rather than reappearing inside resource-specific evaluators or the central Manager file. +Proxmox guest alert evaluation now lives in `internal/alerts/guest.go`. That +file owns guest metric projection, per-disk guest metric alerts, guest +powered-off lifecycle alerts, Pulse tag controls, relaxed guest thresholds, and +guest suppression cleanup; future guest-specific alert behavior should extend +that checker owner rather than expanding the central Manager file. Commercial alert handoffs now follow the same shared navigation boundary. `frontend-modern/src/components/Alerts/InvestigateAlertButton.tsx` may resolve the canonical `ai_alerts` destination from the shared license/commercial diff --git a/internal/alerts/alerts.go b/internal/alerts/alerts.go index 071eb6fc7..cd93a7dea 100644 --- a/internal/alerts/alerts.go +++ b/internal/alerts/alerts.go @@ -13,7 +13,6 @@ import ( alertconfig "github.com/rcourtman/pulse-go-rewrite/internal/alerts/config" alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs" - "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" "github.com/rcourtman/pulse-go-rewrite/internal/utils" "github.com/rs/zerolog/log" ) @@ -1044,87 +1043,6 @@ func (m *Manager) reevaluateActiveAlertsLocked() { } } -// ReevaluateGuestAlert reevaluates a specific guest's alerts with full threshold resolution including custom rules -// This should be called by the monitor with the current guest state -func (m *Manager) ReevaluateGuestAlert(guest any, guestID string) { - m.mu.Lock() - defer m.mu.Unlock() - - // Get the correct thresholds for this guest (includes custom rules evaluation) - thresholds := m.getGuestThresholds(guest, guestID) - - // Check all metric types for this guest - metricTypes := []string{"cpu", "memory", "disk", "diskRead", "diskWrite", "networkIn", "networkOut"} - - for _, metricType := range metricTypes { - alertID := canonicalMetricStateID(guestID, metricType) - alert, exists := m.getActiveAlertNoLock(alertID) - if !exists { - alert, exists = m.getActiveAlertNoLock(fmt.Sprintf("%s-%s", guestID, metricType)) - } - if !exists { - continue - } - trackingKey := canonicalTrackingKeyForAlert(alert) - if trackingKey == "" { - trackingKey = alertID - } - - // Get the threshold for this metric - var threshold *HysteresisThreshold - switch metricType { - case "cpu": - threshold = thresholds.CPU - case "memory": - threshold = thresholds.Memory - case "disk": - threshold = thresholds.Disk - case "diskRead": - threshold = thresholds.DiskRead - case "diskWrite": - threshold = thresholds.DiskWrite - case "networkIn": - threshold = thresholds.NetworkIn - case "networkOut": - threshold = thresholds.NetworkOut - } - - // If threshold is disabled or doesn't exist, clear the alert - if threshold == nil || threshold.Trigger <= 0 { - m.clearAlertNoLock(trackingKey) - // Also clear any pending alert for this metric - if _, isPending := m.pendingAlerts[trackingKey]; isPending { - delete(m.pendingAlerts, trackingKey) - log.Debug(). - Str("alertID", alertID). - Msg("Cleared pending alert - threshold disabled") - } - log.Info(). - Str("alertID", alertID). - Str("metric", metricType). - Msg("Cleared alert - threshold disabled") - continue - } - - // Check if alert should be cleared based on new threshold - clearThreshold := threshold.Clear - if clearThreshold <= 0 { - clearThreshold = threshold.Trigger - } - - if alert.Value <= clearThreshold || alert.Value < threshold.Trigger { - m.clearAlertNoLock(trackingKey) - log.Info(). - Str("alertID", alertID). - Str("metric", metricType). - Float64("value", alert.Value). - Float64("trigger", threshold.Trigger). - Float64("clear", clearThreshold). - Msg("Cleared alert - value now below threshold after config change") - } - } -} - // GetConfig returns the current alert configuration func (m *Manager) GetConfig() AlertConfig { m.mu.RLock() @@ -1132,306 +1050,6 @@ func (m *Manager) GetConfig() AlertConfig { return m.config } -// CheckGuest checks a guest (VM or container) against thresholds -func (m *Manager) CheckGuest(guest any, instanceName string) { - m.mu.RLock() - enabled := m.config.Enabled - disableAllGuests := m.config.DisableAllGuests - disableAllGuestsOffline := m.config.DisableAllGuestsOffline - ignoredGuestPrefixes := m.config.IgnoredGuestPrefixes - guestTagWhitelist := m.config.GuestTagWhitelist - guestTagBlacklist := m.config.GuestTagBlacklist - m.mu.RUnlock() - - if !enabled { - log.Debug().Msg("checkGuest: alerts disabled globally") - return - } - if disableAllGuests { - log.Debug().Msg("checkGuest: all guest alerts disabled") - return - } - - snapshot, ok := extractGuestSnapshot(guest) - if !ok { - log.Debug(). - Str("type", fmt.Sprintf("%T", guest)). - Msg("CheckGuest: unsupported guest type") - return - } - - guestID := snapshot.ID - name := snapshot.Name - node := snapshot.Node - guestType := snapshot.displayType() - status := snapshot.Status - cpu := snapshot.CPUPercent - memUsage := snapshot.MemUsage - diskUsage := snapshot.DiskUsage - diskRead := snapshot.DiskRead - diskWrite := snapshot.DiskWrite - netIn := snapshot.NetworkIn - netOut := snapshot.NetworkOut - disks := snapshot.Disks - tags := snapshot.Tags - - // Debug logging for high memory VMs - if snapshot.Kind == guestKindVM && memUsage > 85 { - log.Debug(). - Str("vm", name). - Float64("memUsage", memUsage). - Str("status", status). - Msg("VM with high memory detected in CheckGuest") - } - - // Check ignored prefixes - for _, prefix := range ignoredGuestPrefixes { - if prefix != "" && strings.HasPrefix(name, prefix) { - if cleared := m.suppressGuestAlerts(guestID); cleared { - m.saveActiveAlertsAsync("ignored-prefix") - } - return - } - } - - settings := parsePulseTags(tags) - if settings.Suppress { - if cleared := m.suppressGuestAlerts(guestID); cleared { - m.saveActiveAlertsAsync("pulse-no-alerts") - } - log.Debug(). - Str("guestID", guestID). - Msg("Pulse no-alerts tag active; suppressing guest alerts") - return - } - - // Custom Tag Filtering - if len(guestTagBlacklist) > 0 || len(guestTagWhitelist) > 0 { - // Normalize tags once for checking - normalizedTags := make(map[string]bool) - for _, tag := range tags { - normalizedTags[strings.ToLower(strings.TrimSpace(tag))] = true - } - - // Check Blacklist - for _, block := range guestTagBlacklist { - if normalizedTags[strings.ToLower(strings.TrimSpace(block))] { - if cleared := m.suppressGuestAlerts(guestID); cleared { - m.saveActiveAlertsAsync("tag-blacklist") - } - log.Debug().Str("guestID", guestID).Msg("guest suppressed by tag blacklist") - return - } - } - - // Check Whitelist - if len(guestTagWhitelist) > 0 { - found := false - for _, allow := range guestTagWhitelist { - if normalizedTags[strings.ToLower(strings.TrimSpace(allow))] { - found = true - break - } - } - if !found { - if cleared := m.suppressGuestAlerts(guestID); cleared { - m.saveActiveAlertsAsync("tag-whitelist") - } - log.Debug().Str("guestID", guestID).Msg("guest suppressed by tag whitelist (required tag not found)") - return - } - } - } - - monitorOnly := settings.MonitorOnly - if monitorOnly || m.guestHasMonitorOnlyAlerts(guestID) { - log.Debug(). - Str("guest", name). - Bool("monitorOnly", monitorOnly). - Msg("Pulse monitor-only status applied") - } - - // Handle non-running guests - // Proxmox VM states: running, stopped, paused, suspended - if status != "running" { - // Check for powered-off state and generate alert if configured - if status == "stopped" { - if disableAllGuestsOffline { - // Clear any pending powered-off tracking and alerts when globally disabled - m.mu.Lock() - delete(m.offlineConfirmations, guestID) - m.mu.Unlock() - m.clearAlert(canonicalPoweredStateStateID(guestID)) - } else { - m.mu.RLock() - thresholds := m.getGuestThresholds(guest, guestID) - m.mu.RUnlock() - m.checkGuestPoweredOffWithThresholds(guestID, name, node, instanceName, guestType, thresholds, monitorOnly) - } - } else { - // For paused/suspended, clear powered-off alert - m.clearGuestPoweredOffAlert(guestID, name) - } - - alertsCleared := m.clearGuestMetricAlerts(guestID) - - if alertsCleared > 0 { - log.Debug(). - Str("guest", name). - Str("status", status). - Int("alertsCleared", alertsCleared). - Msg("Cleared metric alerts for non-running guest") - m.saveActiveAlertsAsync("guest-not-running") - } - return - } - - // If guest is running, clear any powered-off alert - m.clearGuestPoweredOffAlert(guestID, name) - - // Get thresholds (check custom rules, then overrides, then defaults) - m.mu.RLock() - thresholds := m.getGuestThresholds(guest, guestID) - m.mu.RUnlock() - - if settings.Relaxed { - thresholds = applyRelaxedGuestThresholds(thresholds) - log.Info(). - Str("guest", name). - Float64("trigger", thresholds.CPU.Trigger). - Msg("Applied relaxed thresholds for pulse-relaxed tag") - } - - // If alerts are disabled for this guest, clear any existing alerts and return - if thresholds.Disabled { - if alertsCleared := m.clearGuestMetricAlerts(guestID); alertsCleared > 0 { - log.Info(). - Str("guest", name). - Int("alertsCleared", alertsCleared). - Msg("Cleared guest metric alerts because alerts are disabled") - m.saveActiveAlertsAsync("guest-alerts-disabled") - } - return - } - - // Check each metric - log.Debug(). - Str("guest", name). - Float64("cpu", cpu). - Float64("memory", memUsage). - Float64("disk", diskUsage). - Interface("thresholds", thresholds). - Msg("Checking guest thresholds") - - // Evaluate standard metrics through unified path - var evalOpts *metricOptions - if monitorOnly { - evalOpts = &metricOptions{MonitorOnly: true} - } - m.evaluateUnifiedMetrics(&UnifiedResourceInput{ - ID: guestID, - Type: snapshot.resourceType(), - Name: name, - Node: node, - Instance: instanceName, - CPU: &UnifiedResourceMetric{Percent: cpu}, - Memory: &UnifiedResourceMetric{Percent: memUsage}, - Disk: &UnifiedResourceMetric{Percent: diskUsage}, - DiskRead: &UnifiedResourceMetric{Value: float64(diskRead) / 1024 / 1024}, - DiskWrite: &UnifiedResourceMetric{Value: float64(diskWrite) / 1024 / 1024}, - NetworkIn: &UnifiedResourceMetric{Value: float64(netIn) / 1024 / 1024}, - NetworkOut: &UnifiedResourceMetric{Value: float64(netOut) / 1024 / 1024}, - }, thresholds, evalOpts) - - if thresholds.Disk != nil && thresholds.Disk.Trigger > 0 && len(disks) > 0 { - seenDiskKeys := make(map[string]struct{}) - seenDiskResources := make(map[string]struct{}) - for idx, disk := range disks { - if disk.Total <= 0 { - continue - } - if disk.Usage < 0 { - continue - } - - label := strings.TrimSpace(disk.Mountpoint) - if label == "" { - label = strings.TrimSpace(disk.Device) - } - if label == "" { - label = fmt.Sprintf("Disk %d", idx+1) - } - - keySource := label - if disk.Device != "" && !strings.EqualFold(disk.Device, label) { - keySource = fmt.Sprintf("%s-%s", label, disk.Device) - } - sanitizedKey := sanitizeAlertKey(keySource) - if sanitizedKey == "" { - sanitizedKey = fmt.Sprintf("disk-%d", idx+1) - } - - // Avoid duplicate checks if two disks resolve to the same key - if _, exists := seenDiskKeys[sanitizedKey]; exists { - continue - } - seenDiskKeys[sanitizedKey] = struct{}{} - - perDiskResourceID := fmt.Sprintf("%s-disk-%s", guestID, sanitizedKey) - seenDiskResources[perDiskResourceID] = struct{}{} - message := fmt.Sprintf("%s disk (%s) at %.1f%%", guestType, label, disk.Usage) - - log.Debug(). - Str("guest", name). - Str("node", node). - Str("instance", instanceName). - Str("diskLabel", label). - Float64("usage", disk.Usage). - Msg("Evaluating individual disk for alert thresholds") - - metadata := map[string]interface{}{ - "mountpoint": disk.Mountpoint, - "device": disk.Device, - "diskType": disk.Type, - "totalBytes": disk.Total, - "usedBytes": disk.Used, - "freeBytes": disk.Free, - "diskIndex": idx, - "label": label, - } - resourceType, ok := unifiedMetricResourceType(snapshot.resourceType()) - if !ok { - m.checkMetric(perDiskResourceID, name, node, instanceName, snapshot.resourceType(), "disk", disk.Usage, thresholds.Disk, &metricOptions{ - Metadata: metadata, - Message: message, - MonitorOnly: monitorOnly, - }) - continue - } - spec, err := buildCanonicalMetricSpec(perDiskResourceID, name, resourceType, "disk", thresholds.Disk) - if err != nil { - log.Warn(). - Err(err). - Str("resourceID", perDiskResourceID). - Str("guest", name). - Msg("Skipping invalid canonical guest disk metric spec") - continue - } - - m.checkMetricWithCanonicalSpec(spec, name, node, instanceName, snapshot.resourceType(), disk.Usage, thresholds.Disk, &metricOptions{ - Metadata: metadata, - Message: message, - MonitorOnly: monitorOnly, - }) - } - if cleared := m.cleanupGuestDiskAlerts(guestID, seenDiskResources); cleared > 0 { - m.saveActiveAlertsAsync("guest-disk-set-changed") - } - } else if cleared := m.cleanupGuestDiskAlerts(guestID, nil); cleared > 0 { - m.saveActiveAlertsAsync("guest-disk-alerts-cleared") - } -} - func asyncSaveActiveAlerts(reason string, save func() error) { go func() { defer func() { @@ -1707,102 +1325,6 @@ func (m *Manager) clearResourceOfflineAlert(resourceID, resourceName, host, reso Msg(resourceKind + " instance is back online") } -// checkGuestPoweredOff creates an alert for powered-off guests -func (m *Manager) checkGuestPoweredOff(guestID, name, node, instanceName, guestType string, monitorOnly bool) { - m.mu.RLock() - thresholds := m.getGuestThresholds(guestSnapshotFromIdentity(guestID, name, node, instanceName, guestType, "stopped"), guestID) - m.mu.RUnlock() - m.checkGuestPoweredOffWithThresholds(guestID, name, node, instanceName, guestType, thresholds, monitorOnly) -} - -func (m *Manager) checkGuestPoweredOffWithThresholds(guestID, name, node, instanceName, guestType string, thresholds ThresholdConfig, monitorOnly bool) { - alertID := fmt.Sprintf("guest-powered-off-%s", guestID) - severity := NormalizePoweredOffSeverity(thresholds.PoweredOffSeverity) - resourceType := unifiedresources.ResourceTypeVM - if strings.EqualFold(guestType, "container") { - resourceType = unifiedresources.ResourceTypeSystemContainer - } - spec, err := buildCanonicalPoweredStateSpec(guestID, name, resourceType, severity, 2, thresholds.Disabled || thresholds.DisableConnectivity) - if err != nil { - log.Warn(). - Err(err). - Str("guest", name). - Str("guestID", guestID). - Msg("Skipping invalid canonical guest powered-state spec") - return - } - - m.evaluateCanonicalLifecycleAlert(canonicalLifecycleAlertParams{ - Spec: spec, - Evidence: alertspecs.AlertEvidence{ - ObservedAt: time.Now(), - PoweredState: &alertspecs.PoweredStateEvidence{ - Expected: alertspecs.PowerStateOn, - Observed: alertspecs.PowerStateOff, - }, - }, - Tracking: m.offlineConfirmations, - TrackingKey: guestID, - AlertID: alertID, - AlertType: "powered-off", - ResourceID: guestID, - ResourceName: name, - Node: node, - Instance: instanceName, - Message: fmt.Sprintf("%s '%s' is powered off", guestType, name), - Metadata: map[string]interface{}{ - "monitorOnly": monitorOnly, - "resourceType": strings.ToLower(guestType), - }, - AddToRecent: true, - AddToHistory: true, - DispatchAsync: false, - }) -} - -// clearGuestPoweredOffAlert removes powered-off alert when guest starts running -func (m *Manager) clearGuestPoweredOffAlert(guestID, name string) { - alertID := canonicalPoweredStateStateID(guestID) - - m.mu.Lock() - defer m.mu.Unlock() - - // Reset confirmation count when guest comes back online - if count, exists := m.offlineConfirmations[guestID]; exists && count > 0 { - log.Debug(). - Str("guest", name). - Int("previousCount", count). - Msg("Guest is running, resetting powered-off confirmation count") - delete(m.offlineConfirmations, guestID) - } - - // Check if powered-off alert exists - alert, exists := m.getActiveAlertNoLock(alertID) - if !exists { - return - } - - // Remove from active alerts - m.removeActiveAlertNoLock(alertID) - - downtime := time.Since(alert.StartTime) - resolvedAlert := &ResolvedAlert{ - Alert: alert, - ResolvedTime: time.Now(), - } - m.addRecentlyResolvedWithPrimaryLock(resolvedAlert) - - // Send recovery notification (async to avoid deadlock — callback acquires m.mu.RLock - // via ShouldSuppressResolvedNotification, and we currently hold m.mu.Lock) - m.safeCallResolvedAlertCallback(alert, effectiveAlertID(alert, alertID), true) - - // Log recovery - log.Info(). - Str("guest", name). - Dur("downtime", downtime). - Msg("Guest is now running") -} - // ClearAlert removes an alert from active alerts (but keeps in history) func (m *Manager) ClearAlert(alertID string) bool { m.mu.Lock() @@ -2209,122 +1731,6 @@ func ensureHysteresisThreshold(threshold *HysteresisThreshold) *HysteresisThresh return threshold } -type pulseTagSettings struct { - Suppress bool - MonitorOnly bool - Relaxed bool -} - -func parsePulseTags(tags []string) pulseTagSettings { - settings := pulseTagSettings{} - for _, raw := range tags { - tag := strings.TrimSpace(strings.ToLower(raw)) - switch tag { - case "pulse-no-alerts": - settings.Suppress = true - case "pulse-monitor-only": - settings.MonitorOnly = true - case "pulse-relaxed": - settings.Relaxed = true - } - } - return settings -} - -func applyRelaxedGuestThresholds(cfg ThresholdConfig) ThresholdConfig { - relaxed := cloneThresholdConfig(cfg) - - adjust := func(th **HysteresisThreshold, minTrigger float64) { - if *th == nil { - *th = &HysteresisThreshold{Trigger: minTrigger, Clear: minTrigger - 5} - return - } - ensureHysteresisThreshold(*th) - if (*th).Trigger < minTrigger { - (*th).Trigger = minTrigger - } - if (*th).Clear >= (*th).Trigger { - (*th).Clear = (*th).Trigger - 5 - } - if (*th).Clear < 0 { - (*th).Clear = 0 - } - } - - adjust(&relaxed.CPU, 95) - adjust(&relaxed.Memory, 92) - adjust(&relaxed.Disk, 95) - - return relaxed -} - -func (m *Manager) suppressGuestAlerts(guestID string) bool { - m.mu.Lock() - defer m.mu.Unlock() - - cleared := false - - for storageKey, alert := range m.activeAlerts { - alertID := effectiveAlertID(alert, storageKey) - trackingKey := canonicalTrackingKeyForAlert(alert) - if alert == nil { - continue - } - if alert.ResourceID == guestID || strings.HasPrefix(alert.ResourceID, guestID+"/") || strings.HasPrefix(alertID, guestID) { - m.clearAlertNoLock(alertID) - delete(m.recentAlerts, trackingKey) - delete(m.pendingAlerts, trackingKey) - delete(m.suppressedUntil, trackingKey) - delete(m.alertRateLimit, trackingKey) - cleared = true - } - } - - for key := range m.pendingAlerts { - if strings.HasPrefix(key, guestID) { - delete(m.pendingAlerts, key) - } - } - for key := range m.recentAlerts { - if strings.HasPrefix(key, guestID) { - delete(m.recentAlerts, key) - } - } - for key := range m.suppressedUntil { - if strings.HasPrefix(key, guestID) { - delete(m.suppressedUntil, key) - } - } - for key := range m.alertRateLimit { - if strings.HasPrefix(key, guestID) { - delete(m.alertRateLimit, key) - } - } - - delete(m.offlineConfirmations, guestID) - - return cleared -} - -func (m *Manager) guestHasMonitorOnlyAlerts(guestID string) bool { - m.mu.RLock() - defer m.mu.RUnlock() - - for _, alert := range m.activeAlerts { - if alert == nil { - continue - } - if alert.ResourceID != guestID { - continue - } - if isMonitorOnlyAlert(alert) { - return true - } - } - - return false -} - // escalationChecker runs periodically to check for alerts that need escalation and cleanup func (m *Manager) escalationChecker() { ticker := time.NewTicker(1 * time.Minute) diff --git a/internal/alerts/guest.go b/internal/alerts/guest.go new file mode 100644 index 000000000..64764f7be --- /dev/null +++ b/internal/alerts/guest.go @@ -0,0 +1,604 @@ +package alerts + +import ( + "fmt" + "strings" + "time" + + alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs" + "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" + "github.com/rs/zerolog/log" +) + +// ReevaluateGuestAlert reevaluates a specific guest's alerts with full threshold resolution including custom rules. +// This should be called by the monitor with the current guest state. +func (m *Manager) ReevaluateGuestAlert(guest any, guestID string) { + m.mu.Lock() + defer m.mu.Unlock() + + // Get the correct thresholds for this guest (includes custom rules evaluation) + thresholds := m.getGuestThresholds(guest, guestID) + + // Check all metric types for this guest + metricTypes := []string{"cpu", "memory", "disk", "diskRead", "diskWrite", "networkIn", "networkOut"} + + for _, metricType := range metricTypes { + alertID := canonicalMetricStateID(guestID, metricType) + alert, exists := m.getActiveAlertNoLock(alertID) + if !exists { + alert, exists = m.getActiveAlertNoLock(fmt.Sprintf("%s-%s", guestID, metricType)) + } + if !exists { + continue + } + trackingKey := canonicalTrackingKeyForAlert(alert) + if trackingKey == "" { + trackingKey = alertID + } + + // Get the threshold for this metric + var threshold *HysteresisThreshold + switch metricType { + case "cpu": + threshold = thresholds.CPU + case "memory": + threshold = thresholds.Memory + case "disk": + threshold = thresholds.Disk + case "diskRead": + threshold = thresholds.DiskRead + case "diskWrite": + threshold = thresholds.DiskWrite + case "networkIn": + threshold = thresholds.NetworkIn + case "networkOut": + threshold = thresholds.NetworkOut + } + + // If threshold is disabled or doesn't exist, clear the alert + if threshold == nil || threshold.Trigger <= 0 { + m.clearAlertNoLock(trackingKey) + // Also clear any pending alert for this metric + if _, isPending := m.pendingAlerts[trackingKey]; isPending { + delete(m.pendingAlerts, trackingKey) + log.Debug(). + Str("alertID", alertID). + Msg("Cleared pending alert - threshold disabled") + } + log.Info(). + Str("alertID", alertID). + Str("metric", metricType). + Msg("Cleared alert - threshold disabled") + continue + } + + // Check if alert should be cleared based on new threshold + clearThreshold := threshold.Clear + if clearThreshold <= 0 { + clearThreshold = threshold.Trigger + } + + if alert.Value <= clearThreshold || alert.Value < threshold.Trigger { + m.clearAlertNoLock(trackingKey) + log.Info(). + Str("alertID", alertID). + Str("metric", metricType). + Float64("value", alert.Value). + Float64("trigger", threshold.Trigger). + Float64("clear", clearThreshold). + Msg("Cleared alert - value now below threshold after config change") + } + } +} + +// CheckGuest checks a guest (VM or container) against thresholds. +func (m *Manager) CheckGuest(guest any, instanceName string) { + m.mu.RLock() + enabled := m.config.Enabled + disableAllGuests := m.config.DisableAllGuests + disableAllGuestsOffline := m.config.DisableAllGuestsOffline + ignoredGuestPrefixes := m.config.IgnoredGuestPrefixes + guestTagWhitelist := m.config.GuestTagWhitelist + guestTagBlacklist := m.config.GuestTagBlacklist + m.mu.RUnlock() + + if !enabled { + log.Debug().Msg("checkGuest: alerts disabled globally") + return + } + if disableAllGuests { + log.Debug().Msg("checkGuest: all guest alerts disabled") + return + } + + snapshot, ok := extractGuestSnapshot(guest) + if !ok { + log.Debug(). + Str("type", fmt.Sprintf("%T", guest)). + Msg("CheckGuest: unsupported guest type") + return + } + + guestID := snapshot.ID + name := snapshot.Name + node := snapshot.Node + guestType := snapshot.displayType() + status := snapshot.Status + cpu := snapshot.CPUPercent + memUsage := snapshot.MemUsage + diskUsage := snapshot.DiskUsage + diskRead := snapshot.DiskRead + diskWrite := snapshot.DiskWrite + netIn := snapshot.NetworkIn + netOut := snapshot.NetworkOut + disks := snapshot.Disks + tags := snapshot.Tags + + // Debug logging for high memory VMs + if snapshot.Kind == guestKindVM && memUsage > 85 { + log.Debug(). + Str("vm", name). + Float64("memUsage", memUsage). + Str("status", status). + Msg("VM with high memory detected in CheckGuest") + } + + // Check ignored prefixes + for _, prefix := range ignoredGuestPrefixes { + if prefix != "" && strings.HasPrefix(name, prefix) { + if cleared := m.suppressGuestAlerts(guestID); cleared { + m.saveActiveAlertsAsync("ignored-prefix") + } + return + } + } + + settings := parsePulseTags(tags) + if settings.Suppress { + if cleared := m.suppressGuestAlerts(guestID); cleared { + m.saveActiveAlertsAsync("pulse-no-alerts") + } + log.Debug(). + Str("guestID", guestID). + Msg("Pulse no-alerts tag active; suppressing guest alerts") + return + } + + // Custom Tag Filtering + if len(guestTagBlacklist) > 0 || len(guestTagWhitelist) > 0 { + // Normalize tags once for checking + normalizedTags := make(map[string]bool) + for _, tag := range tags { + normalizedTags[strings.ToLower(strings.TrimSpace(tag))] = true + } + + // Check Blacklist + for _, block := range guestTagBlacklist { + if normalizedTags[strings.ToLower(strings.TrimSpace(block))] { + if cleared := m.suppressGuestAlerts(guestID); cleared { + m.saveActiveAlertsAsync("tag-blacklist") + } + log.Debug().Str("guestID", guestID).Msg("guest suppressed by tag blacklist") + return + } + } + + // Check Whitelist + if len(guestTagWhitelist) > 0 { + found := false + for _, allow := range guestTagWhitelist { + if normalizedTags[strings.ToLower(strings.TrimSpace(allow))] { + found = true + break + } + } + if !found { + if cleared := m.suppressGuestAlerts(guestID); cleared { + m.saveActiveAlertsAsync("tag-whitelist") + } + log.Debug().Str("guestID", guestID).Msg("guest suppressed by tag whitelist (required tag not found)") + return + } + } + } + + monitorOnly := settings.MonitorOnly + if monitorOnly || m.guestHasMonitorOnlyAlerts(guestID) { + log.Debug(). + Str("guest", name). + Bool("monitorOnly", monitorOnly). + Msg("Pulse monitor-only status applied") + } + + // Handle non-running guests + // Proxmox VM states: running, stopped, paused, suspended + if status != "running" { + // Check for powered-off state and generate alert if configured + if status == "stopped" { + if disableAllGuestsOffline { + // Clear any pending powered-off tracking and alerts when globally disabled + m.mu.Lock() + delete(m.offlineConfirmations, guestID) + m.mu.Unlock() + m.clearAlert(canonicalPoweredStateStateID(guestID)) + } else { + m.mu.RLock() + thresholds := m.getGuestThresholds(guest, guestID) + m.mu.RUnlock() + m.checkGuestPoweredOffWithThresholds(guestID, name, node, instanceName, guestType, thresholds, monitorOnly) + } + } else { + // For paused/suspended, clear powered-off alert + m.clearGuestPoweredOffAlert(guestID, name) + } + + alertsCleared := m.clearGuestMetricAlerts(guestID) + + if alertsCleared > 0 { + log.Debug(). + Str("guest", name). + Str("status", status). + Int("alertsCleared", alertsCleared). + Msg("Cleared metric alerts for non-running guest") + m.saveActiveAlertsAsync("guest-not-running") + } + return + } + + // If guest is running, clear any powered-off alert + m.clearGuestPoweredOffAlert(guestID, name) + + // Get thresholds (check custom rules, then overrides, then defaults) + m.mu.RLock() + thresholds := m.getGuestThresholds(guest, guestID) + m.mu.RUnlock() + + if settings.Relaxed { + thresholds = applyRelaxedGuestThresholds(thresholds) + log.Info(). + Str("guest", name). + Float64("trigger", thresholds.CPU.Trigger). + Msg("Applied relaxed thresholds for pulse-relaxed tag") + } + + // If alerts are disabled for this guest, clear any existing alerts and return + if thresholds.Disabled { + if alertsCleared := m.clearGuestMetricAlerts(guestID); alertsCleared > 0 { + log.Info(). + Str("guest", name). + Int("alertsCleared", alertsCleared). + Msg("Cleared guest metric alerts because alerts are disabled") + m.saveActiveAlertsAsync("guest-alerts-disabled") + } + return + } + + // Check each metric + log.Debug(). + Str("guest", name). + Float64("cpu", cpu). + Float64("memory", memUsage). + Float64("disk", diskUsage). + Interface("thresholds", thresholds). + Msg("Checking guest thresholds") + + // Evaluate standard metrics through unified path + var evalOpts *metricOptions + if monitorOnly { + evalOpts = &metricOptions{MonitorOnly: true} + } + m.evaluateUnifiedMetrics(&UnifiedResourceInput{ + ID: guestID, + Type: snapshot.resourceType(), + Name: name, + Node: node, + Instance: instanceName, + CPU: &UnifiedResourceMetric{Percent: cpu}, + Memory: &UnifiedResourceMetric{Percent: memUsage}, + Disk: &UnifiedResourceMetric{Percent: diskUsage}, + DiskRead: &UnifiedResourceMetric{Value: float64(diskRead) / 1024 / 1024}, + DiskWrite: &UnifiedResourceMetric{Value: float64(diskWrite) / 1024 / 1024}, + NetworkIn: &UnifiedResourceMetric{Value: float64(netIn) / 1024 / 1024}, + NetworkOut: &UnifiedResourceMetric{Value: float64(netOut) / 1024 / 1024}, + }, thresholds, evalOpts) + + if thresholds.Disk != nil && thresholds.Disk.Trigger > 0 && len(disks) > 0 { + seenDiskKeys := make(map[string]struct{}) + seenDiskResources := make(map[string]struct{}) + for idx, disk := range disks { + if disk.Total <= 0 { + continue + } + if disk.Usage < 0 { + continue + } + + label := strings.TrimSpace(disk.Mountpoint) + if label == "" { + label = strings.TrimSpace(disk.Device) + } + if label == "" { + label = fmt.Sprintf("Disk %d", idx+1) + } + + keySource := label + if disk.Device != "" && !strings.EqualFold(disk.Device, label) { + keySource = fmt.Sprintf("%s-%s", label, disk.Device) + } + sanitizedKey := sanitizeAlertKey(keySource) + if sanitizedKey == "" { + sanitizedKey = fmt.Sprintf("disk-%d", idx+1) + } + + // Avoid duplicate checks if two disks resolve to the same key + if _, exists := seenDiskKeys[sanitizedKey]; exists { + continue + } + seenDiskKeys[sanitizedKey] = struct{}{} + + perDiskResourceID := fmt.Sprintf("%s-disk-%s", guestID, sanitizedKey) + seenDiskResources[perDiskResourceID] = struct{}{} + message := fmt.Sprintf("%s disk (%s) at %.1f%%", guestType, label, disk.Usage) + + log.Debug(). + Str("guest", name). + Str("node", node). + Str("instance", instanceName). + Str("diskLabel", label). + Float64("usage", disk.Usage). + Msg("Evaluating individual disk for alert thresholds") + + metadata := map[string]interface{}{ + "mountpoint": disk.Mountpoint, + "device": disk.Device, + "diskType": disk.Type, + "totalBytes": disk.Total, + "usedBytes": disk.Used, + "freeBytes": disk.Free, + "diskIndex": idx, + "label": label, + } + resourceType, ok := unifiedMetricResourceType(snapshot.resourceType()) + if !ok { + m.checkMetric(perDiskResourceID, name, node, instanceName, snapshot.resourceType(), "disk", disk.Usage, thresholds.Disk, &metricOptions{ + Metadata: metadata, + Message: message, + MonitorOnly: monitorOnly, + }) + continue + } + spec, err := buildCanonicalMetricSpec(perDiskResourceID, name, resourceType, "disk", thresholds.Disk) + if err != nil { + log.Warn(). + Err(err). + Str("resourceID", perDiskResourceID). + Str("guest", name). + Msg("Skipping invalid canonical guest disk metric spec") + continue + } + + m.checkMetricWithCanonicalSpec(spec, name, node, instanceName, snapshot.resourceType(), disk.Usage, thresholds.Disk, &metricOptions{ + Metadata: metadata, + Message: message, + MonitorOnly: monitorOnly, + }) + } + if cleared := m.cleanupGuestDiskAlerts(guestID, seenDiskResources); cleared > 0 { + m.saveActiveAlertsAsync("guest-disk-set-changed") + } + } else if cleared := m.cleanupGuestDiskAlerts(guestID, nil); cleared > 0 { + m.saveActiveAlertsAsync("guest-disk-alerts-cleared") + } +} + +// checkGuestPoweredOff creates an alert for powered-off guests. +func (m *Manager) checkGuestPoweredOff(guestID, name, node, instanceName, guestType string, monitorOnly bool) { + m.mu.RLock() + thresholds := m.getGuestThresholds(guestSnapshotFromIdentity(guestID, name, node, instanceName, guestType, "stopped"), guestID) + m.mu.RUnlock() + m.checkGuestPoweredOffWithThresholds(guestID, name, node, instanceName, guestType, thresholds, monitorOnly) +} + +func (m *Manager) checkGuestPoweredOffWithThresholds(guestID, name, node, instanceName, guestType string, thresholds ThresholdConfig, monitorOnly bool) { + alertID := fmt.Sprintf("guest-powered-off-%s", guestID) + severity := NormalizePoweredOffSeverity(thresholds.PoweredOffSeverity) + resourceType := unifiedresources.ResourceTypeVM + if strings.EqualFold(guestType, "container") { + resourceType = unifiedresources.ResourceTypeSystemContainer + } + spec, err := buildCanonicalPoweredStateSpec(guestID, name, resourceType, severity, 2, thresholds.Disabled || thresholds.DisableConnectivity) + if err != nil { + log.Warn(). + Err(err). + Str("guest", name). + Str("guestID", guestID). + Msg("Skipping invalid canonical guest powered-state spec") + return + } + + m.evaluateCanonicalLifecycleAlert(canonicalLifecycleAlertParams{ + Spec: spec, + Evidence: alertspecs.AlertEvidence{ + ObservedAt: time.Now(), + PoweredState: &alertspecs.PoweredStateEvidence{ + Expected: alertspecs.PowerStateOn, + Observed: alertspecs.PowerStateOff, + }, + }, + Tracking: m.offlineConfirmations, + TrackingKey: guestID, + AlertID: alertID, + AlertType: "powered-off", + ResourceID: guestID, + ResourceName: name, + Node: node, + Instance: instanceName, + Message: fmt.Sprintf("%s '%s' is powered off", guestType, name), + Metadata: map[string]interface{}{ + "monitorOnly": monitorOnly, + "resourceType": strings.ToLower(guestType), + }, + AddToRecent: true, + AddToHistory: true, + DispatchAsync: false, + }) +} + +// clearGuestPoweredOffAlert removes powered-off alert when guest starts running. +func (m *Manager) clearGuestPoweredOffAlert(guestID, name string) { + alertID := canonicalPoweredStateStateID(guestID) + + m.mu.Lock() + defer m.mu.Unlock() + + // Reset confirmation count when guest comes back online + if count, exists := m.offlineConfirmations[guestID]; exists && count > 0 { + log.Debug(). + Str("guest", name). + Int("previousCount", count). + Msg("Guest is running, resetting powered-off confirmation count") + delete(m.offlineConfirmations, guestID) + } + + // Check if powered-off alert exists + alert, exists := m.getActiveAlertNoLock(alertID) + if !exists { + return + } + + // Remove from active alerts + m.removeActiveAlertNoLock(alertID) + + downtime := time.Since(alert.StartTime) + resolvedAlert := &ResolvedAlert{ + Alert: alert, + ResolvedTime: time.Now(), + } + m.addRecentlyResolvedWithPrimaryLock(resolvedAlert) + + // Send recovery notification (async to avoid deadlock because callback acquires m.mu.RLock + // via ShouldSuppressResolvedNotification, and we currently hold m.mu.Lock) + m.safeCallResolvedAlertCallback(alert, effectiveAlertID(alert, alertID), true) + + // Log recovery + log.Info(). + Str("guest", name). + Dur("downtime", downtime). + Msg("Guest is now running") +} + +type pulseTagSettings struct { + Suppress bool + MonitorOnly bool + Relaxed bool +} + +func parsePulseTags(tags []string) pulseTagSettings { + settings := pulseTagSettings{} + for _, raw := range tags { + tag := strings.TrimSpace(strings.ToLower(raw)) + switch tag { + case "pulse-no-alerts": + settings.Suppress = true + case "pulse-monitor-only": + settings.MonitorOnly = true + case "pulse-relaxed": + settings.Relaxed = true + } + } + return settings +} + +func applyRelaxedGuestThresholds(cfg ThresholdConfig) ThresholdConfig { + relaxed := cloneThresholdConfig(cfg) + + adjust := func(th **HysteresisThreshold, minTrigger float64) { + if *th == nil { + *th = &HysteresisThreshold{Trigger: minTrigger, Clear: minTrigger - 5} + return + } + ensureHysteresisThreshold(*th) + if (*th).Trigger < minTrigger { + (*th).Trigger = minTrigger + } + if (*th).Clear >= (*th).Trigger { + (*th).Clear = (*th).Trigger - 5 + } + if (*th).Clear < 0 { + (*th).Clear = 0 + } + } + + adjust(&relaxed.CPU, 95) + adjust(&relaxed.Memory, 92) + adjust(&relaxed.Disk, 95) + + return relaxed +} + +func (m *Manager) suppressGuestAlerts(guestID string) bool { + m.mu.Lock() + defer m.mu.Unlock() + + cleared := false + + for storageKey, alert := range m.activeAlerts { + alertID := effectiveAlertID(alert, storageKey) + trackingKey := canonicalTrackingKeyForAlert(alert) + if alert == nil { + continue + } + if alert.ResourceID == guestID || strings.HasPrefix(alert.ResourceID, guestID+"/") || strings.HasPrefix(alertID, guestID) { + m.clearAlertNoLock(alertID) + delete(m.recentAlerts, trackingKey) + delete(m.pendingAlerts, trackingKey) + delete(m.suppressedUntil, trackingKey) + delete(m.alertRateLimit, trackingKey) + cleared = true + } + } + + for key := range m.pendingAlerts { + if strings.HasPrefix(key, guestID) { + delete(m.pendingAlerts, key) + } + } + for key := range m.recentAlerts { + if strings.HasPrefix(key, guestID) { + delete(m.recentAlerts, key) + } + } + for key := range m.suppressedUntil { + if strings.HasPrefix(key, guestID) { + delete(m.suppressedUntil, key) + } + } + for key := range m.alertRateLimit { + if strings.HasPrefix(key, guestID) { + delete(m.alertRateLimit, key) + } + } + + delete(m.offlineConfirmations, guestID) + + return cleared +} + +func (m *Manager) guestHasMonitorOnlyAlerts(guestID string) bool { + m.mu.RLock() + defer m.mu.RUnlock() + + for _, alert := range m.activeAlerts { + if alert == nil { + continue + } + if alert.ResourceID != guestID { + continue + } + if isMonitorOnlyAlert(alert) { + return true + } + } + + return false +} diff --git a/internal/alerts/guest_snapshot_test.go b/internal/alerts/guest_snapshot_test.go index f50456864..9ca7cb9a7 100644 --- a/internal/alerts/guest_snapshot_test.go +++ b/internal/alerts/guest_snapshot_test.go @@ -19,6 +19,19 @@ func TestGuestSnapshotResourceTypeUsesCanonicalSystemContainer(t *testing.T) { } } +func TestParsePulseTagsRecognizesGuestRuntimeControls(t *testing.T) { + settings := parsePulseTags([]string{ + " PULSE-NO-ALERTS ", + "pulse-monitor-only", + "pulse-relaxed", + "unrelated", + }) + + if !settings.Suppress || !settings.MonitorOnly || !settings.Relaxed { + t.Fatalf("parsePulseTags() = %+v, want all guest runtime controls enabled", settings) + } +} + func TestEmptyGuestSnapshot_NormalizesCollections(t *testing.T) { snapshot := emptyGuestSnapshot() if snapshot.Disks == nil || snapshot.Tags == nil {