diff --git a/internal/alerts/alerts.go b/internal/alerts/alerts.go index 6d892a2b0..bc930a0d3 100644 --- a/internal/alerts/alerts.go +++ b/internal/alerts/alerts.go @@ -1116,6 +1116,7 @@ func (m *Manager) UpdateConfig(config AlertConfig) { normalizeStorageDefaults(&config) normalizeDockerDefaults(&config) normalizePMGDefaults(&config) + normalizePBSDefaults(&config) normalizeSnapshotDefaults(&config) normalizeBackupDefaults(&config) normalizeNodeDefaults(&config) @@ -1308,6 +1309,32 @@ func normalizePMGDefaults(config *AlertConfig) { } } +// normalizePBSDefaults ensures PBS server threshold defaults exist. +// Trigger=0 is allowed and means "disable alerting for this metric". +func normalizePBSDefaults(config *AlertConfig) { + if config.PBSDefaults.CPU == nil || config.PBSDefaults.CPU.Trigger < 0 { + config.PBSDefaults.CPU = &HysteresisThreshold{Trigger: 80, Clear: 75} + } else if config.PBSDefaults.CPU.Trigger == 0 { + config.PBSDefaults.CPU.Clear = 0 + } else if config.PBSDefaults.CPU.Clear <= 0 { + config.PBSDefaults.CPU.Clear = config.PBSDefaults.CPU.Trigger - 5 + if config.PBSDefaults.CPU.Clear <= 0 { + config.PBSDefaults.CPU.Clear = 75 + } + } + + if config.PBSDefaults.Memory == nil || config.PBSDefaults.Memory.Trigger < 0 { + config.PBSDefaults.Memory = &HysteresisThreshold{Trigger: 85, Clear: 80} + } else if config.PBSDefaults.Memory.Trigger == 0 { + config.PBSDefaults.Memory.Clear = 0 + } else if config.PBSDefaults.Memory.Clear <= 0 { + config.PBSDefaults.Memory.Clear = config.PBSDefaults.Memory.Trigger - 5 + if config.PBSDefaults.Memory.Clear <= 0 { + config.PBSDefaults.Memory.Clear = 80 + } + } +} + // normalizeSnapshotDefaults ensures snapshot alert thresholds are valid func normalizeSnapshotDefaults(config *AlertConfig) { if config.SnapshotDefaults.WarningDays < 0 { @@ -1530,6 +1557,8 @@ func validateHysteresisThresholds(config *AlertConfig) { ensureValidHysteresis(config.NodeDefaults.CPU, "node.cpu") ensureValidHysteresis(config.NodeDefaults.Memory, "node.memory") ensureValidHysteresis(config.NodeDefaults.Temperature, "node.temperature") + ensureValidHysteresis(config.PBSDefaults.CPU, "pbs.cpu") + ensureValidHysteresis(config.PBSDefaults.Memory, "pbs.memory") ensureValidHysteresis(&config.StorageDefault, "storage") } diff --git a/internal/alerts/alerts_test.go b/internal/alerts/alerts_test.go index db893e4b2..58f50d8e6 100644 --- a/internal/alerts/alerts_test.go +++ b/internal/alerts/alerts_test.go @@ -3049,6 +3049,65 @@ func TestNormalizeStorageDefaultsPreservesZeroTrigger(t *testing.T) { }) } +func TestNormalizePBSDefaultsPreservesZeroTrigger(t *testing.T) { + t.Run("nil PBSDefaults get factory defaults", func(t *testing.T) { + m := newTestManager(t) + + cfg := AlertConfig{ + Enabled: true, + PBSDefaults: ThresholdConfig{}, + } + + m.UpdateConfig(cfg) + + m.mu.RLock() + defer m.mu.RUnlock() + + if m.config.PBSDefaults.CPU == nil { + t.Fatal("CPU defaults should be set") + } + if m.config.PBSDefaults.CPU.Trigger != 80 { + t.Errorf("CPU trigger = %v, want 80", m.config.PBSDefaults.CPU.Trigger) + } + if m.config.PBSDefaults.Memory == nil { + t.Fatal("Memory defaults should be set") + } + if m.config.PBSDefaults.Memory.Trigger != 85 { + t.Errorf("Memory trigger = %v, want 85", m.config.PBSDefaults.Memory.Trigger) + } + }) + + t.Run("Trigger=0 preserved to disable alerting", func(t *testing.T) { + m := newTestManager(t) + + cfg := AlertConfig{ + Enabled: true, + PBSDefaults: ThresholdConfig{ + CPU: &HysteresisThreshold{Trigger: 99, Clear: 94}, + Memory: &HysteresisThreshold{Trigger: 0, Clear: 50}, + }, + } + + m.UpdateConfig(cfg) + + m.mu.RLock() + defer m.mu.RUnlock() + + if m.config.PBSDefaults.CPU.Trigger != 99 { + t.Errorf("CPU trigger = %v, want 99", m.config.PBSDefaults.CPU.Trigger) + } + if m.config.PBSDefaults.Memory == nil { + t.Fatal("Memory threshold should be preserved") + } + if m.config.PBSDefaults.Memory.Trigger != 0 { + t.Errorf("Memory trigger = %v, want 0", m.config.PBSDefaults.Memory.Trigger) + } + if m.config.PBSDefaults.Memory.Clear != 0 { + t.Errorf("Memory clear = %v, want 0", m.config.PBSDefaults.Memory.Clear) + } + }) +} + // TestNormalizeNodeDefaultsTemperaturePreservesZeroTrigger verifies that setting // NodeDefaults.Temperature threshold to 0 is preserved to disable temperature alerting. func TestNormalizeNodeDefaultsTemperaturePreservesZeroTrigger(t *testing.T) { @@ -15739,6 +15798,38 @@ func TestCheckPBSComprehensive(t *testing.T) { } }) + t.Run("disabled PBS memory threshold clears existing memory alert", func(t *testing.T) { + m := newTestManager(t) + + m.mu.Lock() + m.config.TimeThreshold = 0 + m.config.TimeThresholds = map[string]int{} + m.config.PBSDefaults = ThresholdConfig{ + CPU: &HysteresisThreshold{Trigger: 99.0, Clear: 94.0}, + Memory: &HysteresisThreshold{Trigger: 0, Clear: 0}, + } + m.activeAlerts["pbs1-memory"] = &Alert{ID: "pbs1-memory", Type: "memory"} + m.mu.Unlock() + + pbs := models.PBSInstance{ + ID: "pbs1", + Name: "pbs", + Host: "pbs", + Status: "online", + Memory: 90.8, + } + + m.CheckPBS(pbs) + + m.mu.RLock() + _, exists := m.activeAlerts["pbs1-memory"] + m.mu.RUnlock() + + if exists { + t.Fatal("expected PBS memory alert to be cleared when memory threshold is disabled") + } + }) + t.Run("skips metrics when PBS is offline", func(t *testing.T) { // t.Parallel() m := newTestManager(t) diff --git a/internal/config/persistence.go b/internal/config/persistence.go index 31e4195a1..21b559944 100644 --- a/internal/config/persistence.go +++ b/internal/config/persistence.go @@ -536,6 +536,10 @@ func (c *ConfigPersistence) LoadAlertConfig() (*alerts.AlertConfig, error) { Memory: &alerts.HysteresisThreshold{Trigger: 85, Clear: 80}, Disk: &alerts.HysteresisThreshold{Trigger: 90, Clear: 85}, }, + PBSDefaults: alerts.ThresholdConfig{ + CPU: &alerts.HysteresisThreshold{Trigger: 80, Clear: 75}, + Memory: &alerts.HysteresisThreshold{Trigger: 85, Clear: 80}, + }, StorageDefault: alerts.HysteresisThreshold{Trigger: 85, Clear: 80}, TimeThreshold: 5, TimeThresholds: map[string]int{ @@ -709,6 +713,27 @@ func (c *ConfigPersistence) LoadAlertConfig() (*alerts.AlertConfig, error) { config.HostDefaults.Disk.Clear = 85 } } + // PBS defaults: Allow Trigger=0 to disable specific alerts + if config.PBSDefaults.CPU == nil || config.PBSDefaults.CPU.Trigger < 0 { + config.PBSDefaults.CPU = &alerts.HysteresisThreshold{Trigger: 80, Clear: 75} + } else if config.PBSDefaults.CPU.Trigger == 0 { + config.PBSDefaults.CPU.Clear = 0 + } else if config.PBSDefaults.CPU.Clear <= 0 { + config.PBSDefaults.CPU.Clear = config.PBSDefaults.CPU.Trigger - 5 + if config.PBSDefaults.CPU.Clear <= 0 { + config.PBSDefaults.CPU.Clear = 75 + } + } + if config.PBSDefaults.Memory == nil || config.PBSDefaults.Memory.Trigger < 0 { + config.PBSDefaults.Memory = &alerts.HysteresisThreshold{Trigger: 85, Clear: 80} + } else if config.PBSDefaults.Memory.Trigger == 0 { + config.PBSDefaults.Memory.Clear = 0 + } else if config.PBSDefaults.Memory.Clear <= 0 { + config.PBSDefaults.Memory.Clear = config.PBSDefaults.Memory.Trigger - 5 + if config.PBSDefaults.Memory.Clear <= 0 { + config.PBSDefaults.Memory.Clear = 80 + } + } if config.TimeThreshold <= 0 { config.TimeThreshold = 5 } diff --git a/internal/config/persistence_alerts_normalization_test.go b/internal/config/persistence_alerts_normalization_test.go index 47ed8af8f..80b28469b 100644 --- a/internal/config/persistence_alerts_normalization_test.go +++ b/internal/config/persistence_alerts_normalization_test.go @@ -162,6 +162,35 @@ func TestLoadAlertConfig_Normalization(t *testing.T) { assert.Equal(t, 0.0, cfg.HostDefaults.CPU.Clear) }, }, + { + name: "PBSDefaults missing", + input: map[string]interface{}{ + "pbsDefaults": map[string]interface{}{}, + }, + verify: func(t *testing.T, cfg *alerts.AlertConfig) { + require.NotNil(t, cfg.PBSDefaults.CPU) + require.NotNil(t, cfg.PBSDefaults.Memory) + assert.Equal(t, 80.0, cfg.PBSDefaults.CPU.Trigger) + assert.Equal(t, 85.0, cfg.PBSDefaults.Memory.Trigger) + }, + }, + { + name: "PBSDefaults memory zero", + input: map[string]interface{}{ + "pbsDefaults": map[string]interface{}{ + "cpu": map[string]interface{}{"trigger": 99}, + "memory": map[string]interface{}{"trigger": 0, "clear": 50}, + }, + }, + verify: func(t *testing.T, cfg *alerts.AlertConfig) { + require.NotNil(t, cfg.PBSDefaults.CPU) + require.NotNil(t, cfg.PBSDefaults.Memory) + assert.Equal(t, 99.0, cfg.PBSDefaults.CPU.Trigger) + assert.Equal(t, 94.0, cfg.PBSDefaults.CPU.Clear) + assert.Equal(t, 0.0, cfg.PBSDefaults.Memory.Trigger) + assert.Equal(t, 0.0, cfg.PBSDefaults.Memory.Clear) + }, + }, { name: "TimeThreshold and TimeThresholds", input: map[string]interface{}{ diff --git a/internal/notifications/email_template.go b/internal/notifications/email_template.go index adcf21c30..41576455c 100644 --- a/internal/notifications/email_template.go +++ b/internal/notifications/email_template.go @@ -36,6 +36,25 @@ func alertNodeDisplay(alert *alerts.Alert) string { return alert.Node } +func alertResourceTypeDisplay(alert *alerts.Alert) string { + if alert == nil || alert.Metadata == nil { + return "" + } + resourceType, ok := alert.Metadata["resourceType"].(string) + if !ok { + return "" + } + return strings.TrimSpace(resourceType) +} + +func alertMetricContext(alert *alerts.Alert) string { + context := fmt.Sprintf("%s on %s", alert.Type, alertNodeDisplay(alert)) + if resourceType := alertResourceTypeDisplay(alert); resourceType != "" { + return fmt.Sprintf("%s %s", resourceType, context) + } + return context +} + // EmailTemplate generates a professional HTML email template for alerts func EmailTemplate(alertList []*alerts.Alert, isSingle bool) (subject, htmlBody, textBody string) { if isSingle && len(alertList) == 1 { @@ -67,6 +86,18 @@ func singleAlertTemplate(alert *alerts.Alert) (subject, htmlBody, textBody strin alertType = titleCase(alertType) } + resourceType := alertResourceTypeDisplay(alert) + resourceTypeHTML := "" + resourceTypeText := "" + if resourceType != "" { + resourceTypeHTML = fmt.Sprintf(` +
+ Resource Type + %s +
`, resourceType) + resourceTypeText = fmt.Sprintf("- Resource Type: %s\n", resourceType) + } + subject = fmt.Sprintf("[Pulse Alert] %s: %s on %s", titleCase(string(alert.Level)), alertType, alert.ResourceName) @@ -138,6 +169,7 @@ func singleAlertTemplate(alert *alerts.Alert) (subject, htmlBody, textBody strin Alert Type %s + %s
Node %s @@ -171,6 +203,7 @@ func singleAlertTemplate(alert *alerts.Alert) (subject, htmlBody, textBody strin formatMetricThreshold(alert.Type, alert.Threshold), alert.ResourceID, alertType, + resourceTypeHTML, alertNodeDisplay(alert), alert.Instance, alert.StartTime.Format("Jan 2, 2006 at 3:04 PM"), @@ -188,6 +221,8 @@ Current Value: %s (Threshold: %s) Message: %s Details: +- Resource ID: %s +%s- Alert Metric: %s - Node: %s - Instance: %s - Started: %s @@ -203,6 +238,9 @@ View alerts and configure settings in your Pulse dashboard.`, formatMetricValue(alert.Type, alert.Value), formatMetricThreshold(alert.Type, alert.Threshold), alert.Message, + alert.ResourceID, + resourceTypeText, + alert.Type, alertNodeDisplay(alert), alert.Instance, alert.StartTime.Format("Jan 2, 2006 at 3:04 PM"), @@ -247,7 +285,7 @@ func groupedAlertTemplate(alertList []*alerts.Alert) (subject, htmlBody, textBod
%s
-
%s on %s
+
%s
@@ -264,7 +302,7 @@ func groupedAlertTemplate(alertList []*alerts.Alert) (subject, htmlBody, textBod `, levelColor, alert.ResourceName, - alert.Type, alertNodeDisplay(alert), + alertMetricContext(alert), levelColor, alert.Level, formatMetricValue(alert.Type, alert.Value), formatMetricThreshold(alert.Type, alert.Threshold), formatDuration(time.Since(alert.StartTime)), @@ -370,7 +408,11 @@ func groupedAlertTemplate(alertList []*alerts.Alert) (subject, htmlBody, textBod textBuilder.WriteString("─────────────────────────────────────────────────────────────\n") for i, alert := range alertList { + resourceType := alertResourceTypeDisplay(alert) textBuilder.WriteString(fmt.Sprintf("\n%d. %s (%s)\n", i+1, alert.ResourceName, alert.ResourceID)) + if resourceType != "" { + textBuilder.WriteString(fmt.Sprintf(" Resource Type: %s\n", resourceType)) + } textBuilder.WriteString(fmt.Sprintf(" Level: %s | Type: %s\n", strings.ToUpper(string(alert.Level)), alert.Type)) textBuilder.WriteString(fmt.Sprintf(" Value: %s (Threshold: %s)\n", formatMetricValue(alert.Type, alert.Value), formatMetricThreshold(alert.Type, alert.Threshold))) textBuilder.WriteString(fmt.Sprintf(" Node: %s | Started: %s ago\n", alertNodeDisplay(alert), formatDuration(time.Since(alert.StartTime)))) diff --git a/internal/notifications/email_template_test.go b/internal/notifications/email_template_test.go index 5311477f9..c9e9553c4 100644 --- a/internal/notifications/email_template_test.go +++ b/internal/notifications/email_template_test.go @@ -704,3 +704,39 @@ func TestEmailTemplate(t *testing.T) { }) } } + +func TestEmailTemplateIncludesResourceType(t *testing.T) { + t.Parallel() + + alert := &alerts.Alert{ + ID: "host:pbs-memory", + Level: "warning", + Type: "memory", + ResourceID: "host:pbs", + ResourceName: "pbs", + Node: "pbs", + Value: 90.8, + Threshold: 85.0, + StartTime: time.Now(), + Message: "Host memory at 90.8%", + Metadata: map[string]interface{}{ + "resourceType": "Host", + }, + } + + _, singleHTML, singleText := EmailTemplate([]*alerts.Alert{alert}, true) + if !strings.Contains(singleHTML, "Resource Type") || !strings.Contains(singleHTML, "Host") { + t.Fatalf("single alert HTML should include resource type, got %s", singleHTML) + } + if !strings.Contains(singleText, "Resource Type: Host") { + t.Fatalf("single alert text should include resource type, got %s", singleText) + } + + _, groupedHTML, groupedText := EmailTemplate([]*alerts.Alert{alert}, false) + if !strings.Contains(groupedHTML, "Host memory on pbs") { + t.Fatalf("grouped alert HTML should include typed metric context, got %s", groupedHTML) + } + if !strings.Contains(groupedText, "Resource Type: Host") { + t.Fatalf("grouped alert text should include resource type, got %s", groupedText) + } +}