Fix unreachable critical escalation for high percentage thresholds

The critical (severity-escalation) threshold was hardcoded as
Trigger + 10 for all metric types. For percentage metrics (cpu,
memory, disk, usage) with high triggers, this made critical
escalation unreachable: a CPU trigger of 95% produced a critical
threshold of 105%, which is impossible for a 0-100% metric.

Fix: add computeCriticalThreshold() helper that caps the critical
threshold at 99 for percentage metrics. Non-percentage metrics
(temperature, diskRead, diskWrite, networkIn, networkOut) keep
the Trigger + 10 offset unchanged.

Applied to all three code paths that derive critical from trigger:
- buildCanonicalMetricSpec (canonical evaluation)
- checkMetric new alert creation (legacy path)
- checkMetric existing alert update (legacy path)
This commit is contained in:
rcourtman 2026-06-26 22:42:02 +01:00
parent 06ea25e5ce
commit e09fe06201
3 changed files with 81 additions and 3 deletions

View file

@ -0,0 +1,55 @@
package alerts
import (
"testing"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts/config"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)
func TestComputeCriticalThreshold_PercentageMetric(t *testing.T) {
tests := []struct {
name string
trigger float64
metricType string
want float64
}{
{name: "cpu low trigger", trigger: 80, metricType: "cpu", want: 90},
{name: "cpu mid trigger", trigger: 85, metricType: "cpu", want: 95},
{name: "cpu high trigger capped", trigger: 95, metricType: "cpu", want: 99},
{name: "cpu 90 capped at boundary", trigger: 90, metricType: "cpu", want: 99},
{name: "memory high trigger capped", trigger: 93, metricType: "memory", want: 99},
{name: "disk high trigger capped", trigger: 91, metricType: "disk", want: 99},
{name: "usage high trigger capped", trigger: 97, metricType: "usage", want: 99},
{name: "temperature not capped", trigger: 95, metricType: "temperature", want: 105},
{name: "diskRead not capped", trigger: 100, metricType: "diskRead", want: 110},
{name: "networkIn not capped", trigger: 50, metricType: "networkIn", want: 60},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := computeCriticalThreshold(tt.trigger, tt.metricType)
if got != tt.want {
t.Errorf("computeCriticalThreshold(%.1f, %q) = %.1f, want %.1f", tt.trigger, tt.metricType, got, tt.want)
}
})
}
}
func TestBuildCanonicalMetricSpec_CriticalCappedForHighTrigger(t *testing.T) {
threshold := &config.HysteresisThreshold{Trigger: 95, Clear: 90}
spec, err := buildCanonicalMetricSpec("res-1", "CPU", unifiedresources.ResourceTypeAgent, "cpu", threshold)
if err != nil {
t.Fatalf("buildCanonicalMetricSpec: %v", err)
}
if spec.MetricThreshold.Critical == nil {
t.Fatal("expected critical threshold to be set")
}
if *spec.MetricThreshold.Critical > 99 {
t.Errorf("critical threshold = %.1f, should be capped at 99 for percentage metric", *spec.MetricThreshold.Critical)
}
if *spec.MetricThreshold.Critical != 99 {
t.Errorf("critical threshold = %.1f, want 99", *spec.MetricThreshold.Critical)
}
}

View file

@ -325,7 +325,7 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
applyCanonicalIdentity(alert, canonicalSpecID, string(alertspecs.AlertSpecKindMetricThreshold))
// Set level based on how much over threshold
if value >= threshold.Trigger+10 {
if value >= computeCriticalThreshold(threshold.Trigger, metricType) {
alert.Level = AlertLevelCritical
}
@ -431,7 +431,7 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
// Update level if needed
oldLevel := existingAlert.Level
if value >= threshold.Trigger+10 {
if value >= computeCriticalThreshold(threshold.Trigger, metricType) {
existingAlert.Level = AlertLevelCritical
} else {
existingAlert.Level = AlertLevelWarning

View file

@ -301,6 +301,29 @@ func unifiedStorageUsageResourceType(typeKey string) bool {
}
}
// isPercentageMetric returns true for metrics whose values are bounded 0100.
// The critical threshold for these must stay below 100 or escalation will never fire.
func isPercentageMetric(metricType string) bool {
switch metricType {
case "cpu", "memory", "disk", "usage":
return true
default:
return false
}
}
// computeCriticalThreshold derives the critical (severity-escalation) threshold
// from the trigger value. For percentage metrics (0100 range) the result is
// capped at 99 so that critical escalation remains reachable when the trigger
// is set high (e.g. trigger 95 → critical 99, not 105).
func computeCriticalThreshold(trigger float64, metricType string) float64 {
critical := trigger + 10
if isPercentageMetric(metricType) && critical > 99 {
return 99
}
return critical
}
func buildCanonicalMetricSpec(resourceID, title string, resourceType unifiedresources.ResourceType, metricType string, threshold *HysteresisThreshold) (alertspecs.ResourceAlertSpec, error) {
spec := alertspecs.ResourceAlertSpec{
ID: canonicalMetricSpecID(resourceID, metricType),
@ -323,7 +346,7 @@ func buildCanonicalMetricSpec(resourceID, title string, resourceType unifiedreso
recovery := threshold.Clear
spec.MetricThreshold.Recovery = &recovery
}
critical := threshold.Trigger + 10
critical := computeCriticalThreshold(threshold.Trigger, metricType)
spec.MetricThreshold.Critical = &critical
}