vrr/Pulse
2
0
Fork 0
mirror of https://github.com/rcourtman/Pulse.git synced 2026-07-10 00:14:38 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

Clarify PBS alert threshold identity

Browse source 
Refs #1448
This commit is contained in:
rcourtman 2026-05-24 22:13:58 +01:00
parent a8f6dbd8b4
commit 117ec901da
6 changed files with 254 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

29
internal/alerts/alerts.go
View file

@ -1116,6 +1116,7 @@ func (m *Manager) UpdateConfig(config AlertConfig) {
normalizeStorageDefaults(&config)
normalizeDockerDefaults(&config)
normalizePMGDefaults(&config)
normalizePBSDefaults(&config)
normalizeSnapshotDefaults(&config)
normalizeBackupDefaults(&config)
normalizeNodeDefaults(&config)
@ -1308,6 +1309,32 @@ func normalizePMGDefaults(config *AlertConfig) {
}
}
// normalizePBSDefaults ensures PBS server threshold defaults exist.
// Trigger=0 is allowed and means "disable alerting for this metric".
func normalizePBSDefaults(config *AlertConfig) {
if config.PBSDefaults.CPU == nil || config.PBSDefaults.CPU.Trigger < 0 {
config.PBSDefaults.CPU = &HysteresisThreshold{Trigger: 80, Clear: 75}
} else if config.PBSDefaults.CPU.Trigger == 0 {
config.PBSDefaults.CPU.Clear = 0
} else if config.PBSDefaults.CPU.Clear <= 0 {
config.PBSDefaults.CPU.Clear = config.PBSDefaults.CPU.Trigger - 5
if config.PBSDefaults.CPU.Clear <= 0 {
config.PBSDefaults.CPU.Clear = 75
}
}
if config.PBSDefaults.Memory == nil || config.PBSDefaults.Memory.Trigger < 0 {
config.PBSDefaults.Memory = &HysteresisThreshold{Trigger: 85, Clear: 80}
} else if config.PBSDefaults.Memory.Trigger == 0 {
config.PBSDefaults.Memory.Clear = 0
} else if config.PBSDefaults.Memory.Clear <= 0 {
config.PBSDefaults.Memory.Clear = config.PBSDefaults.Memory.Trigger - 5
if config.PBSDefaults.Memory.Clear <= 0 {
config.PBSDefaults.Memory.Clear = 80
}
}
}
// normalizeSnapshotDefaults ensures snapshot alert thresholds are valid
func normalizeSnapshotDefaults(config *AlertConfig) {
if config.SnapshotDefaults.WarningDays < 0 {
@ -1530,6 +1557,8 @@ func validateHysteresisThresholds(config *AlertConfig) {
ensureValidHysteresis(config.NodeDefaults.CPU, "node.cpu")
ensureValidHysteresis(config.NodeDefaults.Memory, "node.memory")
ensureValidHysteresis(config.NodeDefaults.Temperature, "node.temperature")
ensureValidHysteresis(config.PBSDefaults.CPU, "pbs.cpu")
ensureValidHysteresis(config.PBSDefaults.Memory, "pbs.memory")
ensureValidHysteresis(&config.StorageDefault, "storage")
}

91
internal/alerts/alerts_test.go
View file

@ -3049,6 +3049,65 @@ func TestNormalizeStorageDefaultsPreservesZeroTrigger(t *testing.T) {
})
}
func TestNormalizePBSDefaultsPreservesZeroTrigger(t *testing.T) {
t.Run("nil PBSDefaults get factory defaults", func(t *testing.T) {
m := newTestManager(t)
cfg := AlertConfig{
Enabled: true,
PBSDefaults: ThresholdConfig{},
}
m.UpdateConfig(cfg)
m.mu.RLock()
defer m.mu.RUnlock()
if m.config.PBSDefaults.CPU == nil {
t.Fatal("CPU defaults should be set")
}
if m.config.PBSDefaults.CPU.Trigger != 80 {
t.Errorf("CPU trigger = %v, want 80", m.config.PBSDefaults.CPU.Trigger)
}
if m.config.PBSDefaults.Memory == nil {
t.Fatal("Memory defaults should be set")
}
if m.config.PBSDefaults.Memory.Trigger != 85 {
t.Errorf("Memory trigger = %v, want 85", m.config.PBSDefaults.Memory.Trigger)
}
})
t.Run("Trigger=0 preserved to disable alerting", func(t *testing.T) {
m := newTestManager(t)
cfg := AlertConfig{
Enabled: true,
PBSDefaults: ThresholdConfig{
CPU: &HysteresisThreshold{Trigger: 99, Clear: 94},
Memory: &HysteresisThreshold{Trigger: 0, Clear: 50},
},
}
m.UpdateConfig(cfg)
m.mu.RLock()
defer m.mu.RUnlock()
if m.config.PBSDefaults.CPU.Trigger != 99 {
t.Errorf("CPU trigger = %v, want 99", m.config.PBSDefaults.CPU.Trigger)
}
if m.config.PBSDefaults.Memory == nil {
t.Fatal("Memory threshold should be preserved")
}
if m.config.PBSDefaults.Memory.Trigger != 0 {
t.Errorf("Memory trigger = %v, want 0", m.config.PBSDefaults.Memory.Trigger)
}
if m.config.PBSDefaults.Memory.Clear != 0 {
t.Errorf("Memory clear = %v, want 0", m.config.PBSDefaults.Memory.Clear)
}
})
}
// TestNormalizeNodeDefaultsTemperaturePreservesZeroTrigger verifies that setting
// NodeDefaults.Temperature threshold to 0 is preserved to disable temperature alerting.
func TestNormalizeNodeDefaultsTemperaturePreservesZeroTrigger(t *testing.T) {
@ -15739,6 +15798,38 @@ func TestCheckPBSComprehensive(t *testing.T) {
}
})
t.Run("disabled PBS memory threshold clears existing memory alert", func(t *testing.T) {
m := newTestManager(t)
m.mu.Lock()
m.config.TimeThreshold = 0
m.config.TimeThresholds = map[string]int{}
m.config.PBSDefaults = ThresholdConfig{
CPU: &HysteresisThreshold{Trigger: 99.0, Clear: 94.0},
Memory: &HysteresisThreshold{Trigger: 0, Clear: 0},
}
m.activeAlerts["pbs1-memory"] = &Alert{ID: "pbs1-memory", Type: "memory"}
m.mu.Unlock()
pbs := models.PBSInstance{
ID: "pbs1",
Name: "pbs",
Host: "pbs",
Status: "online",
Memory: 90.8,
}
m.CheckPBS(pbs)
m.mu.RLock()
_, exists := m.activeAlerts["pbs1-memory"]
m.mu.RUnlock()
if exists {
t.Fatal("expected PBS memory alert to be cleared when memory threshold is disabled")
}
})
t.Run("skips metrics when PBS is offline", func(t *testing.T) {
// t.Parallel()
m := newTestManager(t)

25
internal/config/persistence.go
View file

@ -536,6 +536,10 @@ func (c *ConfigPersistence) LoadAlertConfig() (*alerts.AlertConfig, error) {
Memory: &alerts.HysteresisThreshold{Trigger: 85, Clear: 80},
Disk: &alerts.HysteresisThreshold{Trigger: 90, Clear: 85},
},
PBSDefaults: alerts.ThresholdConfig{
CPU: &alerts.HysteresisThreshold{Trigger: 80, Clear: 75},
Memory: &alerts.HysteresisThreshold{Trigger: 85, Clear: 80},
},
StorageDefault: alerts.HysteresisThreshold{Trigger: 85, Clear: 80},
TimeThreshold: 5,
TimeThresholds: map[string]int{
@ -709,6 +713,27 @@ func (c *ConfigPersistence) LoadAlertConfig() (*alerts.AlertConfig, error) {
config.HostDefaults.Disk.Clear = 85
}
}
// PBS defaults: Allow Trigger=0 to disable specific alerts
if config.PBSDefaults.CPU == nil || config.PBSDefaults.CPU.Trigger < 0 {
config.PBSDefaults.CPU = &alerts.HysteresisThreshold{Trigger: 80, Clear: 75}
} else if config.PBSDefaults.CPU.Trigger == 0 {
config.PBSDefaults.CPU.Clear = 0
} else if config.PBSDefaults.CPU.Clear <= 0 {
config.PBSDefaults.CPU.Clear = config.PBSDefaults.CPU.Trigger - 5
if config.PBSDefaults.CPU.Clear <= 0 {
config.PBSDefaults.CPU.Clear = 75
}
}
if config.PBSDefaults.Memory == nil || config.PBSDefaults.Memory.Trigger < 0 {
config.PBSDefaults.Memory = &alerts.HysteresisThreshold{Trigger: 85, Clear: 80}
} else if config.PBSDefaults.Memory.Trigger == 0 {
config.PBSDefaults.Memory.Clear = 0
} else if config.PBSDefaults.Memory.Clear <= 0 {
config.PBSDefaults.Memory.Clear = config.PBSDefaults.Memory.Trigger - 5
if config.PBSDefaults.Memory.Clear <= 0 {
config.PBSDefaults.Memory.Clear = 80
}
}
if config.TimeThreshold <= 0 {
config.TimeThreshold = 5
}

29
internal/config/persistence_alerts_normalization_test.go
View file

@ -162,6 +162,35 @@ func TestLoadAlertConfig_Normalization(t *testing.T) {
assert.Equal(t, 0.0, cfg.HostDefaults.CPU.Clear)
},
},
{
name: "PBSDefaults missing",
input: map[string]interface{}{
"pbsDefaults": map[string]interface{}{},
},
verify: func(t *testing.T, cfg *alerts.AlertConfig) {
require.NotNil(t, cfg.PBSDefaults.CPU)
require.NotNil(t, cfg.PBSDefaults.Memory)
assert.Equal(t, 80.0, cfg.PBSDefaults.CPU.Trigger)
assert.Equal(t, 85.0, cfg.PBSDefaults.Memory.Trigger)
},
},
{
name: "PBSDefaults memory zero",
input: map[string]interface{}{
"pbsDefaults": map[string]interface{}{
"cpu": map[string]interface{}{"trigger": 99},
"memory": map[string]interface{}{"trigger": 0, "clear": 50},
},
},
verify: func(t *testing.T, cfg *alerts.AlertConfig) {
require.NotNil(t, cfg.PBSDefaults.CPU)
require.NotNil(t, cfg.PBSDefaults.Memory)
assert.Equal(t, 99.0, cfg.PBSDefaults.CPU.Trigger)
assert.Equal(t, 94.0, cfg.PBSDefaults.CPU.Clear)
assert.Equal(t, 0.0, cfg.PBSDefaults.Memory.Trigger)
assert.Equal(t, 0.0, cfg.PBSDefaults.Memory.Clear)
},
},
{
name: "TimeThreshold and TimeThresholds",
input: map[string]interface{}{

46
internal/notifications/email_template.go
View file

@ -36,6 +36,25 @@ func alertNodeDisplay(alert *alerts.Alert) string {
return alert.Node
}
func alertResourceTypeDisplay(alert *alerts.Alert) string {
if alert == nil || alert.Metadata == nil {
return ""
}
resourceType, ok := alert.Metadata["resourceType"].(string)
if !ok {
return ""
}
return strings.TrimSpace(resourceType)
}
func alertMetricContext(alert *alerts.Alert) string {
context := fmt.Sprintf("%s on %s", alert.Type, alertNodeDisplay(alert))
if resourceType := alertResourceTypeDisplay(alert); resourceType != "" {
return fmt.Sprintf("%s %s", resourceType, context)
}
return context
}
// EmailTemplate generates a professional HTML email template for alerts
func EmailTemplate(alertList []*alerts.Alert, isSingle bool) (subject, htmlBody, textBody string) {
if isSingle && len(alertList) == 1 {
@ -67,6 +86,18 @@ func singleAlertTemplate(alert *alerts.Alert) (subject, htmlBody, textBody strin
alertType = titleCase(alertType)
}
resourceType := alertResourceTypeDisplay(alert)
resourceTypeHTML := ""
resourceTypeText := ""
if resourceType != "" {
resourceTypeHTML = fmt.Sprintf(`
<div class="detail-row">
<span class="detail-label">Resource Type</span>
<span class="detail-value">%s</span>
</div>`, resourceType)
resourceTypeText = fmt.Sprintf("- Resource Type: %s\n", resourceType)
}
subject = fmt.Sprintf("[Pulse Alert] %s: %s on %s",
titleCase(string(alert.Level)), alertType, alert.ResourceName)
@ -138,6 +169,7 @@ func singleAlertTemplate(alert *alerts.Alert) (subject, htmlBody, textBody strin
<span class="detail-label">Alert Type</span>
<span class="detail-value">%s</span>
</div>
%s
<div class="detail-row">
<span class="detail-label">Node</span>
<span class="detail-value">%s</span>
@ -171,6 +203,7 @@ func singleAlertTemplate(alert *alerts.Alert) (subject, htmlBody, textBody strin
formatMetricThreshold(alert.Type, alert.Threshold),
alert.ResourceID,
alertType,
resourceTypeHTML,
alertNodeDisplay(alert),
alert.Instance,
alert.StartTime.Format("Jan 2, 2006 at 3:04 PM"),
@ -188,6 +221,8 @@ Current Value: %s (Threshold: %s)
Message: %s
Details:
- Resource ID: %s
%s- Alert Metric: %s
- Node: %s
- Instance: %s
- Started: %s
@ -203,6 +238,9 @@ View alerts and configure settings in your Pulse dashboard.`,
formatMetricValue(alert.Type, alert.Value),
formatMetricThreshold(alert.Type, alert.Threshold),
alert.Message,
alert.ResourceID,
resourceTypeText,
alert.Type,
alertNodeDisplay(alert),
alert.Instance,
alert.StartTime.Format("Jan 2, 2006 at 3:04 PM"),
@ -247,7 +285,7 @@ func groupedAlertTemplate(alertList []*alerts.Alert) (subject, htmlBody, textBod
<span style="display: inline-block; width: 8px; height: 8px; background: %s; border-radius: 50%%; margin-right: 10px;"></span>
<div>
<div style="font-weight: 500; color: #1a1a1a;">%s</div>
<div style="font-size: 12px; color: #666; margin-top: 2px;">%s on %s</div>
<div style="font-size: 12px; color: #666; margin-top: 2px;">%s</div>
</div>
</div>
</td>
@ -264,7 +302,7 @@ func groupedAlertTemplate(alertList []*alerts.Alert) (subject, htmlBody, textBod
</tr>`,
levelColor,
alert.ResourceName,
alert.Type, alertNodeDisplay(alert),
alertMetricContext(alert),
levelColor, alert.Level,
formatMetricValue(alert.Type, alert.Value), formatMetricThreshold(alert.Type, alert.Threshold),
formatDuration(time.Since(alert.StartTime)),
@ -370,7 +408,11 @@ func groupedAlertTemplate(alertList []*alerts.Alert) (subject, htmlBody, textBod
textBuilder.WriteString("─────────────────────────────────────────────────────────────\n")
for i, alert := range alertList {
resourceType := alertResourceTypeDisplay(alert)
textBuilder.WriteString(fmt.Sprintf("\n%d. %s (%s)\n", i+1, alert.ResourceName, alert.ResourceID))
if resourceType != "" {
textBuilder.WriteString(fmt.Sprintf(" Resource Type: %s\n", resourceType))
}
textBuilder.WriteString(fmt.Sprintf(" Level: %s | Type: %s\n", strings.ToUpper(string(alert.Level)), alert.Type))
textBuilder.WriteString(fmt.Sprintf(" Value: %s (Threshold: %s)\n", formatMetricValue(alert.Type, alert.Value), formatMetricThreshold(alert.Type, alert.Threshold)))
textBuilder.WriteString(fmt.Sprintf(" Node: %s | Started: %s ago\n", alertNodeDisplay(alert), formatDuration(time.Since(alert.StartTime))))

36
internal/notifications/email_template_test.go
View file

@ -704,3 +704,39 @@ func TestEmailTemplate(t *testing.T) {
})
}
}
func TestEmailTemplateIncludesResourceType(t *testing.T) {
t.Parallel()
alert := &alerts.Alert{
ID: "host:pbs-memory",
Level: "warning",
Type: "memory",
ResourceID: "host:pbs",
ResourceName: "pbs",
Node: "pbs",
Value: 90.8,
Threshold: 85.0,
StartTime: time.Now(),
Message: "Host memory at 90.8%",
Metadata: map[string]interface{}{
"resourceType": "Host",
},
}
_, singleHTML, singleText := EmailTemplate([]*alerts.Alert{alert}, true)
if !strings.Contains(singleHTML, "Resource Type") || !strings.Contains(singleHTML, "Host") {
t.Fatalf("single alert HTML should include resource type, got %s", singleHTML)
}
if !strings.Contains(singleText, "Resource Type: Host") {
t.Fatalf("single alert text should include resource type, got %s", singleText)
}
_, groupedHTML, groupedText := EmailTemplate([]*alerts.Alert{alert}, false)
if !strings.Contains(groupedHTML, "Host memory on pbs") {
t.Fatalf("grouped alert HTML should include typed metric context, got %s", groupedHTML)
}
if !strings.Contains(groupedText, "Resource Type: Host") {
t.Fatalf("grouped alert text should include resource type, got %s", groupedText)
}
}