mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-07-10 00:14:38 +00:00
Fix unreachable critical escalation for high percentage thresholds
The critical (severity-escalation) threshold was hardcoded as Trigger + 10 for all metric types. For percentage metrics (cpu, memory, disk, usage) with high triggers, this made critical escalation unreachable: a CPU trigger of 95% produced a critical threshold of 105%, which is impossible for a 0-100% metric. Fix: add computeCriticalThreshold() helper that caps the critical threshold at 99 for percentage metrics. Non-percentage metrics (temperature, diskRead, diskWrite, networkIn, networkOut) keep the Trigger + 10 offset unchanged. Applied to all three code paths that derive critical from trigger: - buildCanonicalMetricSpec (canonical evaluation) - checkMetric new alert creation (legacy path) - checkMetric existing alert update (legacy path)
This commit is contained in:
parent
06ea25e5ce
commit
e09fe06201
3 changed files with 81 additions and 3 deletions
55
internal/alerts/critical_threshold_test.go
Normal file
55
internal/alerts/critical_threshold_test.go
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
package alerts
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/alerts/config"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
|
||||
)
|
||||
|
||||
func TestComputeCriticalThreshold_PercentageMetric(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
trigger float64
|
||||
metricType string
|
||||
want float64
|
||||
}{
|
||||
{name: "cpu low trigger", trigger: 80, metricType: "cpu", want: 90},
|
||||
{name: "cpu mid trigger", trigger: 85, metricType: "cpu", want: 95},
|
||||
{name: "cpu high trigger capped", trigger: 95, metricType: "cpu", want: 99},
|
||||
{name: "cpu 90 capped at boundary", trigger: 90, metricType: "cpu", want: 99},
|
||||
{name: "memory high trigger capped", trigger: 93, metricType: "memory", want: 99},
|
||||
{name: "disk high trigger capped", trigger: 91, metricType: "disk", want: 99},
|
||||
{name: "usage high trigger capped", trigger: 97, metricType: "usage", want: 99},
|
||||
{name: "temperature not capped", trigger: 95, metricType: "temperature", want: 105},
|
||||
{name: "diskRead not capped", trigger: 100, metricType: "diskRead", want: 110},
|
||||
{name: "networkIn not capped", trigger: 50, metricType: "networkIn", want: 60},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := computeCriticalThreshold(tt.trigger, tt.metricType)
|
||||
if got != tt.want {
|
||||
t.Errorf("computeCriticalThreshold(%.1f, %q) = %.1f, want %.1f", tt.trigger, tt.metricType, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildCanonicalMetricSpec_CriticalCappedForHighTrigger(t *testing.T) {
|
||||
threshold := &config.HysteresisThreshold{Trigger: 95, Clear: 90}
|
||||
|
||||
spec, err := buildCanonicalMetricSpec("res-1", "CPU", unifiedresources.ResourceTypeAgent, "cpu", threshold)
|
||||
if err != nil {
|
||||
t.Fatalf("buildCanonicalMetricSpec: %v", err)
|
||||
}
|
||||
if spec.MetricThreshold.Critical == nil {
|
||||
t.Fatal("expected critical threshold to be set")
|
||||
}
|
||||
if *spec.MetricThreshold.Critical > 99 {
|
||||
t.Errorf("critical threshold = %.1f, should be capped at 99 for percentage metric", *spec.MetricThreshold.Critical)
|
||||
}
|
||||
if *spec.MetricThreshold.Critical != 99 {
|
||||
t.Errorf("critical threshold = %.1f, want 99", *spec.MetricThreshold.Critical)
|
||||
}
|
||||
}
|
||||
|
|
@ -325,7 +325,7 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
|
|||
applyCanonicalIdentity(alert, canonicalSpecID, string(alertspecs.AlertSpecKindMetricThreshold))
|
||||
|
||||
// Set level based on how much over threshold
|
||||
if value >= threshold.Trigger+10 {
|
||||
if value >= computeCriticalThreshold(threshold.Trigger, metricType) {
|
||||
alert.Level = AlertLevelCritical
|
||||
}
|
||||
|
||||
|
|
@ -431,7 +431,7 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
|
|||
|
||||
// Update level if needed
|
||||
oldLevel := existingAlert.Level
|
||||
if value >= threshold.Trigger+10 {
|
||||
if value >= computeCriticalThreshold(threshold.Trigger, metricType) {
|
||||
existingAlert.Level = AlertLevelCritical
|
||||
} else {
|
||||
existingAlert.Level = AlertLevelWarning
|
||||
|
|
|
|||
|
|
@ -301,6 +301,29 @@ func unifiedStorageUsageResourceType(typeKey string) bool {
|
|||
}
|
||||
}
|
||||
|
||||
// isPercentageMetric returns true for metrics whose values are bounded 0–100.
|
||||
// The critical threshold for these must stay below 100 or escalation will never fire.
|
||||
func isPercentageMetric(metricType string) bool {
|
||||
switch metricType {
|
||||
case "cpu", "memory", "disk", "usage":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// computeCriticalThreshold derives the critical (severity-escalation) threshold
|
||||
// from the trigger value. For percentage metrics (0–100 range) the result is
|
||||
// capped at 99 so that critical escalation remains reachable when the trigger
|
||||
// is set high (e.g. trigger 95 → critical 99, not 105).
|
||||
func computeCriticalThreshold(trigger float64, metricType string) float64 {
|
||||
critical := trigger + 10
|
||||
if isPercentageMetric(metricType) && critical > 99 {
|
||||
return 99
|
||||
}
|
||||
return critical
|
||||
}
|
||||
|
||||
func buildCanonicalMetricSpec(resourceID, title string, resourceType unifiedresources.ResourceType, metricType string, threshold *HysteresisThreshold) (alertspecs.ResourceAlertSpec, error) {
|
||||
spec := alertspecs.ResourceAlertSpec{
|
||||
ID: canonicalMetricSpecID(resourceID, metricType),
|
||||
|
|
@ -323,7 +346,7 @@ func buildCanonicalMetricSpec(resourceID, title string, resourceType unifiedreso
|
|||
recovery := threshold.Clear
|
||||
spec.MetricThreshold.Recovery = &recovery
|
||||
}
|
||||
critical := threshold.Trigger + 10
|
||||
critical := computeCriticalThreshold(threshold.Trigger, metricType)
|
||||
spec.MetricThreshold.Critical = &critical
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue