Pulse/internal/alerts/specs/evaluator_test.go
2026-03-18 16:06:30 +00:00

797 lines
26 KiB
Go

package specs
import (
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)
func boolPtr(v bool) *bool {
return &v
}
func TestEvaluateMetricThresholdTriggerClearAndReevaluation(t *testing.T) {
recovery := 75.0
critical := 90.0
spec := ResourceAlertSpec{
ID: "vm-101-cpu",
ResourceID: "pve1:node1:101",
ResourceType: unifiedresources.ResourceTypeVM,
Kind: AlertSpecKindMetricThreshold,
Severity: AlertSeverityWarning,
MetricThreshold: &MetricThresholdSpec{
Metric: "cpu",
Direction: ThresholdDirectionAbove,
Trigger: 80,
Recovery: &recovery,
Critical: &critical,
},
}
triggered, err := Evaluate(spec, EvaluatorState{}, AlertEvidence{
ObservedAt: time.Date(2026, 3, 10, 9, 0, 0, 0, time.UTC),
MetricThreshold: &MetricThresholdEvidence{
Metric: "cpu",
Direction: ThresholdDirectionAbove,
Observed: 85,
Trigger: 80,
Recovery: &recovery,
Critical: &critical,
},
})
if err != nil {
t.Fatalf("trigger evaluation failed: %v", err)
}
if triggered.State.State != AlertStateFiring {
t.Fatalf("triggered state = %q, want firing", triggered.State.State)
}
if triggered.State.Severity != AlertSeverityWarning {
t.Fatalf("triggered severity = %q, want warning", triggered.State.Severity)
}
if triggered.Transition == nil || triggered.Transition.Kind != EvaluationTransitionActivated {
t.Fatalf("trigger transition = %+v, want activated", triggered.Transition)
}
escalated, err := Evaluate(spec, triggered.State, AlertEvidence{
ObservedAt: time.Date(2026, 3, 10, 9, 0, 3, 0, time.UTC),
MetricThreshold: &MetricThresholdEvidence{
Metric: "cpu",
Direction: ThresholdDirectionAbove,
Observed: 95,
Trigger: 80,
Recovery: &recovery,
Critical: &critical,
},
})
if err != nil {
t.Fatalf("escalation evaluation failed: %v", err)
}
if escalated.State.Severity != AlertSeverityCritical {
t.Fatalf("escalated severity = %q, want critical", escalated.State.Severity)
}
if escalated.Transition == nil || escalated.Transition.Kind != EvaluationTransitionSeverityChanged {
t.Fatalf("escalated transition = %+v, want severity-changed", escalated.Transition)
}
latched, err := Evaluate(spec, escalated.State, AlertEvidence{
ObservedAt: time.Date(2026, 3, 10, 9, 0, 5, 0, time.UTC),
MetricThreshold: &MetricThresholdEvidence{
Metric: "cpu",
Direction: ThresholdDirectionAbove,
Observed: 77,
Trigger: 80,
Recovery: &recovery,
Critical: &critical,
},
})
if err != nil {
t.Fatalf("latched evaluation failed: %v", err)
}
if latched.State.State != AlertStateFiring {
t.Fatalf("latched state = %q, want firing", latched.State.State)
}
if latched.State.Severity != AlertSeverityCritical {
t.Fatalf("latched severity = %q, want critical", latched.State.Severity)
}
if latched.Transition != nil {
t.Fatalf("latched transition = %+v, want nil", latched.Transition)
}
cleared, err := Evaluate(spec, latched.State, AlertEvidence{
ObservedAt: time.Date(2026, 3, 10, 9, 0, 10, 0, time.UTC),
MetricThreshold: &MetricThresholdEvidence{
Metric: "cpu",
Direction: ThresholdDirectionAbove,
Observed: 74,
Trigger: 80,
Recovery: &recovery,
Critical: &critical,
},
})
if err != nil {
t.Fatalf("clear evaluation failed: %v", err)
}
if cleared.State.State != AlertStateClear {
t.Fatalf("cleared state = %q, want clear", cleared.State.State)
}
if cleared.Transition == nil || cleared.Transition.Kind != EvaluationTransitionRecovered {
t.Fatalf("clear transition = %+v, want recovered", cleared.Transition)
}
updated := spec
updated.MetricThreshold.Trigger = 90
updatedCritical := 100.0
updated.MetricThreshold.Critical = &updatedCritical
revaluated, err := Evaluate(updated, triggered.State, AlertEvidence{
ObservedAt: time.Date(2026, 3, 10, 9, 0, 15, 0, time.UTC),
MetricThreshold: &MetricThresholdEvidence{
Metric: "cpu",
Direction: ThresholdDirectionAbove,
Observed: 85,
Trigger: 90,
Recovery: &recovery,
Critical: &updatedCritical,
},
})
if err != nil {
t.Fatalf("reevaluation failed: %v", err)
}
if revaluated.State.State != AlertStateClear {
t.Fatalf("reevaluated state = %q, want clear", revaluated.State.State)
}
if revaluated.Transition == nil || revaluated.Transition.Kind != EvaluationTransitionReevaluated {
t.Fatalf("reevaluated transition = %+v, want reevaluated", revaluated.Transition)
}
}
func TestEvaluateSeverityThresholdEscalationAndRecovery(t *testing.T) {
spec := ResourceAlertSpec{
ID: "pmg-queue-total",
ResourceID: "pmg-1",
ResourceType: unifiedresources.ResourceTypePMG,
Kind: AlertSpecKindSeverityThreshold,
Severity: AlertSeverityWarning,
SeverityThreshold: &SeverityThresholdSpec{
Metric: "queue-total",
Direction: ThresholdDirectionAbove,
Warning: 500,
Critical: 1000,
},
}
warning, err := Evaluate(spec, EvaluatorState{}, AlertEvidence{
ObservedAt: time.Date(2026, 3, 10, 9, 30, 0, 0, time.UTC),
SeverityThreshold: &SeverityThresholdEvidence{
Metric: "queue-total",
Direction: ThresholdDirectionAbove,
Observed: 700,
},
})
if err != nil {
t.Fatalf("warning evaluation failed: %v", err)
}
if warning.State.State != AlertStateFiring || warning.State.Severity != AlertSeverityWarning {
t.Fatalf("warning state = %+v, want firing warning", warning.State)
}
if warning.Transition == nil || warning.Transition.Kind != EvaluationTransitionActivated {
t.Fatalf("warning transition = %+v, want activated", warning.Transition)
}
critical, err := Evaluate(spec, warning.State, AlertEvidence{
ObservedAt: time.Date(2026, 3, 10, 9, 31, 0, 0, time.UTC),
SeverityThreshold: &SeverityThresholdEvidence{
Metric: "queue-total",
Direction: ThresholdDirectionAbove,
Observed: 1200,
},
})
if err != nil {
t.Fatalf("critical evaluation failed: %v", err)
}
if critical.State.Severity != AlertSeverityCritical {
t.Fatalf("critical severity = %q, want critical", critical.State.Severity)
}
if critical.Transition == nil || critical.Transition.Kind != EvaluationTransitionSeverityChanged {
t.Fatalf("critical transition = %+v, want severity-changed", critical.Transition)
}
recovered, err := Evaluate(spec, critical.State, AlertEvidence{
ObservedAt: time.Date(2026, 3, 10, 9, 32, 0, 0, time.UTC),
SeverityThreshold: &SeverityThresholdEvidence{
Metric: "queue-total",
Direction: ThresholdDirectionAbove,
Observed: 200,
},
})
if err != nil {
t.Fatalf("recovery evaluation failed: %v", err)
}
if recovered.State.State != AlertStateClear {
t.Fatalf("recovered state = %q, want clear", recovered.State.State)
}
if recovered.Transition == nil || recovered.Transition.Kind != EvaluationTransitionRecovered {
t.Fatalf("recovered transition = %+v, want recovered", recovered.Transition)
}
}
func TestEvaluateSeverityThresholdHysteresisLatch(t *testing.T) {
recovery := 85.0
spec := ResourceAlertSpec{
ID: "docker:host/container-memory-limit",
ResourceID: "docker:host/container",
ResourceType: unifiedresources.ResourceTypeAppContainer,
Kind: AlertSpecKindSeverityThreshold,
Severity: AlertSeverityWarning,
SeverityThreshold: &SeverityThresholdSpec{
Metric: "memory-limit-percent",
Direction: ThresholdDirectionAbove,
Warning: 90,
Critical: 95,
Recovery: &recovery,
},
}
critical, err := Evaluate(spec, EvaluatorState{}, AlertEvidence{
ObservedAt: time.Date(2026, 3, 10, 9, 40, 0, 0, time.UTC),
SeverityThreshold: &SeverityThresholdEvidence{
Metric: "memory-limit-percent",
Direction: ThresholdDirectionAbove,
Observed: 96,
},
})
if err != nil {
t.Fatalf("critical evaluation failed: %v", err)
}
if critical.State.Severity != AlertSeverityCritical {
t.Fatalf("critical severity = %q, want critical", critical.State.Severity)
}
latched, err := Evaluate(spec, critical.State, AlertEvidence{
ObservedAt: time.Date(2026, 3, 10, 9, 41, 0, 0, time.UTC),
SeverityThreshold: &SeverityThresholdEvidence{
Metric: "memory-limit-percent",
Direction: ThresholdDirectionAbove,
Observed: 88,
},
})
if err != nil {
t.Fatalf("latched evaluation failed: %v", err)
}
if latched.State.State != AlertStateFiring {
t.Fatalf("latched state = %q, want firing", latched.State.State)
}
if latched.State.Severity != AlertSeverityCritical {
t.Fatalf("latched severity = %q, want critical", latched.State.Severity)
}
if latched.Transition != nil {
t.Fatalf("latched transition = %+v, want nil", latched.Transition)
}
cleared, err := Evaluate(spec, latched.State, AlertEvidence{
ObservedAt: time.Date(2026, 3, 10, 9, 42, 0, 0, time.UTC),
SeverityThreshold: &SeverityThresholdEvidence{
Metric: "memory-limit-percent",
Direction: ThresholdDirectionAbove,
Observed: 84,
},
})
if err != nil {
t.Fatalf("clear evaluation failed: %v", err)
}
if cleared.State.State != AlertStateClear {
t.Fatalf("cleared state = %q, want clear", cleared.State.State)
}
if cleared.Transition == nil || cleared.Transition.Kind != EvaluationTransitionRecovered {
t.Fatalf("cleared transition = %+v, want recovered", cleared.Transition)
}
}
func TestEvaluateChangeThresholdAbsoluteAndGrowth(t *testing.T) {
spec := ResourceAlertSpec{
ID: "pmg-quarantine-spam",
ResourceID: "pmg-1",
ResourceType: unifiedresources.ResourceTypePMG,
Kind: AlertSpecKindChangeThreshold,
Severity: AlertSeverityWarning,
ChangeThreshold: &ChangeThresholdSpec{
Metric: "quarantine-spam",
ReferenceWindow: 2 * time.Hour,
WarningCurrent: 2000,
CriticalCurrent: 5000,
WarningDelta: 250,
CriticalDelta: 500,
WarningPercent: 25,
CriticalPercent: 50,
},
}
absolute, err := Evaluate(spec, EvaluatorState{}, AlertEvidence{
ObservedAt: time.Date(2026, 3, 10, 10, 0, 0, 0, time.UTC),
ChangeThreshold: &ChangeThresholdEvidence{
Metric: "quarantine-spam",
Observed: 2500,
},
})
if err != nil {
t.Fatalf("absolute evaluation failed: %v", err)
}
if absolute.State.State != AlertStateFiring || absolute.State.Severity != AlertSeverityWarning {
t.Fatalf("absolute state = %+v, want firing warning", absolute.State)
}
if absolute.State.Reason != "change-threshold-current-warning" {
t.Fatalf("absolute reason = %q, want current warning", absolute.State.Reason)
}
previous := 1000.0
growth, err := Evaluate(spec, EvaluatorState{}, AlertEvidence{
ObservedAt: time.Date(2026, 3, 10, 10, 5, 0, 0, time.UTC),
ChangeThreshold: &ChangeThresholdEvidence{
Metric: "quarantine-spam",
Observed: 1600,
PreviousObserved: &previous,
},
})
if err != nil {
t.Fatalf("growth evaluation failed: %v", err)
}
if growth.State.State != AlertStateFiring || growth.State.Severity != AlertSeverityCritical {
t.Fatalf("growth state = %+v, want firing critical", growth.State)
}
if growth.State.Reason != "change-threshold-growth-critical" {
t.Fatalf("growth reason = %q, want growth critical", growth.State.Reason)
}
}
func TestEvaluateBaselineAnomalyNormalAndQuietSite(t *testing.T) {
spec := ResourceAlertSpec{
ID: "pmg-anomaly-spamIn",
ResourceID: "pmg-1",
ResourceType: unifiedresources.ResourceTypePMG,
Kind: AlertSpecKindBaselineAnomaly,
Severity: AlertSeverityWarning,
BaselineAnomaly: &BaselineAnomalySpec{
Metric: "spamIn",
QuietBaseline: 40,
WarningRatio: 1.8,
CriticalRatio: 2.5,
WarningDelta: 150,
CriticalDelta: 300,
QuietWarningDelta: 60,
QuietCriticalDelta: 120,
},
}
normal, err := Evaluate(spec, EvaluatorState{}, AlertEvidence{
ObservedAt: time.Date(2026, 3, 10, 10, 10, 0, 0, time.UTC),
BaselineAnomaly: &BaselineAnomalyEvidence{
Metric: "spamIn",
Observed: 420,
Baseline: 100,
},
})
if err != nil {
t.Fatalf("normal evaluation failed: %v", err)
}
if normal.State.State != AlertStateFiring || normal.State.Severity != AlertSeverityCritical {
t.Fatalf("normal state = %+v, want firing critical", normal.State)
}
if normal.State.Reason != "baseline-anomaly-critical" {
t.Fatalf("normal reason = %q, want baseline-anomaly-critical", normal.State.Reason)
}
quiet, err := Evaluate(spec, EvaluatorState{}, AlertEvidence{
ObservedAt: time.Date(2026, 3, 10, 10, 11, 0, 0, time.UTC),
BaselineAnomaly: &BaselineAnomalyEvidence{
Metric: "spamIn",
Observed: 140,
Baseline: 10,
},
})
if err != nil {
t.Fatalf("quiet evaluation failed: %v", err)
}
if quiet.State.State != AlertStateFiring || quiet.State.Severity != AlertSeverityCritical {
t.Fatalf("quiet state = %+v, want firing critical", quiet.State)
}
if quiet.State.Reason != "baseline-anomaly-quiet-critical" {
t.Fatalf("quiet reason = %q, want baseline-anomaly-quiet-critical", quiet.State.Reason)
}
}
func TestEvaluateConnectivityConfirmationAndRecovery(t *testing.T) {
spec := ResourceAlertSpec{
ID: "node-pve1-connectivity",
ResourceID: "node/pve-1",
ResourceType: unifiedresources.ResourceTypeAgent,
Kind: AlertSpecKindConnectivity,
Severity: AlertSeverityCritical,
Connectivity: &ConnectivitySpec{
Signal: "heartbeat",
LostAfter: 30 * time.Second,
},
}
first, err := Evaluate(spec, EvaluatorState{}, AlertEvidence{
ObservedAt: time.Date(2026, 3, 10, 10, 0, 0, 0, time.UTC),
Connectivity: &ConnectivityEvidence{
Signal: "heartbeat",
Connected: false,
},
})
if err != nil {
t.Fatalf("first evaluation failed: %v", err)
}
if first.State.State != AlertStatePending || first.State.ConsecutiveMatches != 1 {
t.Fatalf("first state = %+v, want pending with one confirmation", first.State)
}
second, err := Evaluate(spec, first.State, AlertEvidence{
ObservedAt: time.Date(2026, 3, 10, 10, 0, 5, 0, time.UTC),
Connectivity: &ConnectivityEvidence{
Signal: "heartbeat",
Connected: false,
},
})
if err != nil {
t.Fatalf("second evaluation failed: %v", err)
}
if second.State.State != AlertStatePending || second.State.ConsecutiveMatches != 2 {
t.Fatalf("second state = %+v, want pending with two confirmations", second.State)
}
third, err := Evaluate(spec, second.State, AlertEvidence{
ObservedAt: time.Date(2026, 3, 10, 10, 0, 10, 0, time.UTC),
Connectivity: &ConnectivityEvidence{
Signal: "heartbeat",
Connected: false,
},
})
if err != nil {
t.Fatalf("third evaluation failed: %v", err)
}
if third.State.State != AlertStateFiring {
t.Fatalf("third state = %q, want firing", third.State.State)
}
if third.Transition == nil || third.Transition.Kind != EvaluationTransitionActivated {
t.Fatalf("third transition = %+v, want activated", third.Transition)
}
recovered, err := Evaluate(spec, third.State, AlertEvidence{
ObservedAt: time.Date(2026, 3, 10, 10, 0, 15, 0, time.UTC),
Connectivity: &ConnectivityEvidence{
Signal: "heartbeat",
Connected: true,
},
})
if err != nil {
t.Fatalf("recovery evaluation failed: %v", err)
}
if recovered.State.State != AlertStateClear {
t.Fatalf("recovered state = %q, want clear", recovered.State.State)
}
if recovered.Transition == nil || recovered.Transition.Kind != EvaluationTransitionRecovered {
t.Fatalf("recovered transition = %+v, want recovered", recovered.Transition)
}
}
func TestEvaluatePoweredStateSuppressionAndDisable(t *testing.T) {
spec := ResourceAlertSpec{
ID: "vm-101-powered-off",
ResourceID: "pve1:node1:101",
ResourceType: unifiedresources.ResourceTypeVM,
Kind: AlertSpecKindPoweredState,
Severity: AlertSeverityWarning,
SuppressOnConnectivityLoss: true,
PoweredState: &PoweredStateSpec{
Expected: PowerStateOn,
},
}
first, err := Evaluate(spec, EvaluatorState{}, AlertEvidence{
ObservedAt: time.Date(2026, 3, 10, 11, 0, 0, 0, time.UTC),
ParentConnected: boolPtr(true),
PoweredState: &PoweredStateEvidence{
Expected: PowerStateOn,
Observed: PowerStateOff,
},
})
if err != nil {
t.Fatalf("first evaluation failed: %v", err)
}
if first.State.State != AlertStatePending {
t.Fatalf("first state = %q, want pending", first.State.State)
}
active, err := Evaluate(spec, first.State, AlertEvidence{
ObservedAt: time.Date(2026, 3, 10, 11, 0, 5, 0, time.UTC),
ParentConnected: boolPtr(true),
PoweredState: &PoweredStateEvidence{
Expected: PowerStateOn,
Observed: PowerStateOff,
},
})
if err != nil {
t.Fatalf("active evaluation failed: %v", err)
}
if active.State.State != AlertStateFiring {
t.Fatalf("active state = %q, want firing", active.State.State)
}
suppressed, err := Evaluate(spec, active.State, AlertEvidence{
ObservedAt: time.Date(2026, 3, 10, 11, 0, 10, 0, time.UTC),
ParentConnected: boolPtr(false),
PoweredState: &PoweredStateEvidence{
Expected: PowerStateOn,
Observed: PowerStateOff,
},
})
if err != nil {
t.Fatalf("suppressed evaluation failed: %v", err)
}
if suppressed.State.State != AlertStateSuppressed {
t.Fatalf("suppressed state = %q, want suppressed", suppressed.State.State)
}
if suppressed.Transition == nil || suppressed.Transition.Kind != EvaluationTransitionSuppressed {
t.Fatalf("suppressed transition = %+v, want suppressed", suppressed.Transition)
}
disabledSpec := spec
disabledSpec.Disabled = true
disabled, err := Evaluate(disabledSpec, active.State, AlertEvidence{
ObservedAt: time.Date(2026, 3, 10, 11, 0, 15, 0, time.UTC),
ParentConnected: boolPtr(true),
PoweredState: &PoweredStateEvidence{
Expected: PowerStateOn,
Observed: PowerStateOff,
},
})
if err != nil {
t.Fatalf("disabled evaluation failed: %v", err)
}
if disabled.State.State != AlertStateClear {
t.Fatalf("disabled state = %q, want clear", disabled.State.State)
}
if disabled.Transition == nil || disabled.Transition.Kind != EvaluationTransitionDisabled {
t.Fatalf("disabled transition = %+v, want disabled", disabled.Transition)
}
}
func TestEvaluateServiceGapSeverityChanges(t *testing.T) {
spec := ResourceAlertSpec{
ID: "docker-service-gap",
ResourceID: "docker:host-1/service:web",
ResourceType: unifiedresources.ResourceTypeDockerService,
Kind: AlertSpecKindServiceGap,
Severity: AlertSeverityWarning,
ServiceGap: &ServiceGapSpec{
Service: "web",
WarningPercent: 10,
CriticalPercent: 50,
},
}
warning, err := Evaluate(spec, EvaluatorState{}, AlertEvidence{
ObservedAt: time.Date(2026, 3, 10, 12, 0, 0, 0, time.UTC),
ServiceGap: &ServiceGapEvidence{
Service: "web",
Desired: 10,
Running: 8,
},
})
if err != nil {
t.Fatalf("warning evaluation failed: %v", err)
}
if warning.State.State != AlertStateFiring || warning.State.Severity != AlertSeverityWarning {
t.Fatalf("warning state = %+v, want firing warning", warning.State)
}
critical, err := Evaluate(spec, warning.State, AlertEvidence{
ObservedAt: time.Date(2026, 3, 10, 12, 0, 5, 0, time.UTC),
ServiceGap: &ServiceGapEvidence{
Service: "web",
Desired: 10,
Running: 4,
},
})
if err != nil {
t.Fatalf("critical evaluation failed: %v", err)
}
if critical.State.Severity != AlertSeverityCritical {
t.Fatalf("critical severity = %q, want critical", critical.State.Severity)
}
if critical.Transition == nil || critical.Transition.Kind != EvaluationTransitionSeverityChanged {
t.Fatalf("critical transition = %+v, want severity-changed", critical.Transition)
}
recovered, err := Evaluate(spec, critical.State, AlertEvidence{
ObservedAt: time.Date(2026, 3, 10, 12, 0, 10, 0, time.UTC),
ServiceGap: &ServiceGapEvidence{
Service: "web",
Desired: 10,
Running: 10,
},
})
if err != nil {
t.Fatalf("recovery evaluation failed: %v", err)
}
if recovered.State.State != AlertStateClear {
t.Fatalf("recovered state = %q, want clear", recovered.State.State)
}
if recovered.Transition == nil || recovered.Transition.Kind != EvaluationTransitionRecovered {
t.Fatalf("recovered transition = %+v, want recovered", recovered.Transition)
}
}
func TestEvaluateDiscreteStatePreservesStableSpecIdentityAcrossMutableFields(t *testing.T) {
base := ResourceAlertSpec{
ID: "docker-api-runtime-state",
ResourceID: "docker:host-1/container:api",
ResourceType: unifiedresources.ResourceTypeAppContainer,
Kind: AlertSpecKindDiscreteState,
Severity: AlertSeverityWarning,
DiscreteState: &DiscreteStateSpec{
StateKey: "runtime-state",
TriggerStates: []string{"paused"},
},
}
initial, err := Evaluate(base, EvaluatorState{}, AlertEvidence{
ObservedAt: time.Date(2026, 3, 10, 13, 0, 0, 0, time.UTC),
DiscreteState: &DiscreteStateEvidence{
StateKey: "runtime-state",
Observed: "paused",
},
})
if err != nil {
t.Fatalf("initial evaluation failed: %v", err)
}
if initial.State.State != AlertStateFiring || initial.State.Severity != AlertSeverityWarning {
t.Fatalf("initial state = %+v, want firing warning", initial.State)
}
mutated := base
mutated.ConfirmationsRequired = 4
mutated.SuppressOnConnectivityLoss = true
mutated.Severity = AlertSeverityCritical
continued, err := Evaluate(mutated, initial.State, AlertEvidence{
ObservedAt: time.Date(2026, 3, 10, 13, 0, 5, 0, time.UTC),
DiscreteState: &DiscreteStateEvidence{
StateKey: "runtime-state",
Observed: "paused",
},
})
if err != nil {
t.Fatalf("continued evaluation failed: %v", err)
}
if continued.State.SpecID != mutated.ID {
t.Fatalf("continued spec id = %q, want %q", continued.State.SpecID, mutated.ID)
}
if continued.State.State != AlertStateFiring || continued.State.Severity != AlertSeverityCritical {
t.Fatalf("continued state = %+v, want firing critical", continued.State)
}
if continued.Transition == nil || continued.Transition.Kind != EvaluationTransitionSeverityChanged {
t.Fatalf("continued transition = %+v, want severity-changed", continued.Transition)
}
}
func TestEvaluateHealthAssessmentEscalationAndRecovery(t *testing.T) {
spec := ResourceAlertSpec{
ID: "agent:host1/raid:md2-health",
ResourceID: "agent:host1/raid:md2",
ResourceType: unifiedresources.ResourceTypeAgent,
Kind: AlertSpecKindHealthAssessment,
Severity: AlertSeverityWarning,
HealthAssessment: &HealthAssessmentSpec{
Signal: "host-raid",
Codes: []string{"raid_degraded", "raid_rebuilding"},
},
}
warning, err := Evaluate(spec, EvaluatorState{}, AlertEvidence{
ObservedAt: time.Date(2026, 3, 10, 13, 10, 0, 0, time.UTC),
HealthAssessment: &HealthAssessmentEvidence{
Signal: "host-raid",
Severity: AlertSeverityWarning,
Codes: []string{"raid_rebuilding"},
},
})
if err != nil {
t.Fatalf("warning evaluation failed: %v", err)
}
if warning.State.State != AlertStateFiring || warning.State.Severity != AlertSeverityWarning {
t.Fatalf("warning state = %+v, want firing warning", warning.State)
}
critical, err := Evaluate(spec, warning.State, AlertEvidence{
ObservedAt: time.Date(2026, 3, 10, 13, 11, 0, 0, time.UTC),
HealthAssessment: &HealthAssessmentEvidence{
Signal: "host-raid",
Severity: AlertSeverityCritical,
Codes: []string{"raid_degraded"},
},
})
if err != nil {
t.Fatalf("critical evaluation failed: %v", err)
}
if critical.State.Severity != AlertSeverityCritical {
t.Fatalf("critical severity = %q, want critical", critical.State.Severity)
}
if critical.Transition == nil || critical.Transition.Kind != EvaluationTransitionSeverityChanged {
t.Fatalf("critical transition = %+v, want severity-changed", critical.Transition)
}
recovered, err := Evaluate(spec, critical.State, AlertEvidence{
ObservedAt: time.Date(2026, 3, 10, 13, 12, 0, 0, time.UTC),
HealthAssessment: &HealthAssessmentEvidence{
Signal: "host-raid",
},
})
if err != nil {
t.Fatalf("recovery evaluation failed: %v", err)
}
if recovered.State.State != AlertStateClear {
t.Fatalf("recovered state = %q, want clear", recovered.State.State)
}
if recovered.Transition == nil || recovered.Transition.Kind != EvaluationTransitionRecovered {
t.Fatalf("recovered transition = %+v, want recovered", recovered.Transition)
}
}
func TestEvaluatePostureThresholdAgeAndSize(t *testing.T) {
spec := ResourceAlertSpec{
ID: "inst:node:100-snapshot-weekly",
ResourceID: "inst:node:100",
ResourceType: unifiedresources.ResourceTypeVM,
Kind: AlertSpecKindPostureThreshold,
Severity: AlertSeverityWarning,
PostureThreshold: &PostureThresholdSpec{
AgeMetric: "snapshot-age-days",
WarningAge: 7,
CriticalAge: 14,
SizeMetric: "snapshot-size-gib",
WarningSize: 50,
CriticalSize: 100,
},
}
size := 120.0
critical, err := Evaluate(spec, EvaluatorState{}, AlertEvidence{
ObservedAt: time.Date(2026, 3, 10, 13, 20, 0, 0, time.UTC),
PostureThreshold: &PostureThresholdEvidence{
AgeMetric: "snapshot-age-days",
AgeValue: 2,
SizeMetric: "snapshot-size-gib",
SizeValue: &size,
},
})
if err != nil {
t.Fatalf("critical evaluation failed: %v", err)
}
if critical.State.State != AlertStateFiring || critical.State.Severity != AlertSeverityCritical {
t.Fatalf("critical state = %+v, want firing critical", critical.State)
}
if critical.State.Reason != "posture-threshold-size-critical" {
t.Fatalf("critical reason = %q, want size critical", critical.State.Reason)
}
warning, err := Evaluate(spec, EvaluatorState{}, AlertEvidence{
ObservedAt: time.Date(2026, 3, 10, 13, 21, 0, 0, time.UTC),
PostureThreshold: &PostureThresholdEvidence{
AgeMetric: "snapshot-age-days",
AgeValue: 10,
SizeMetric: "snapshot-size-gib",
SizeValue: &[]float64{20}[0],
},
})
if err != nil {
t.Fatalf("warning evaluation failed: %v", err)
}
if warning.State.State != AlertStateFiring || warning.State.Severity != AlertSeverityWarning {
t.Fatalf("warning state = %+v, want firing warning", warning.State)
}
if warning.State.Reason != "posture-threshold-age-warning" {
t.Fatalf("warning reason = %q, want age warning", warning.State.Reason)
}
}