fix(alerts): stabilize noisy lifecycle signals

This commit is contained in:
Pulse Test 2026-08-29 17:32:17 +01:00
parent a506cdde8b
commit d671f05408
11 changed files with 359 additions and 45 deletions

View file

@ -2390,3 +2390,28 @@ against a managed local backend. The browser phase must exercise active-alert
snooze, delivery diagnosis, restart persistence, unsnooze, incident evidence, snooze, delivery diagnosis, restart persistence, unsnooze, incident evidence,
history migration, virtualization, and clear tombstones without replacing the history migration, virtualization, and clear tombstones without replacing the
backend APIs with route mocks. backend APIs with route mocks.
### Noisy gauges use one trust-stable lifecycle
Memory, temperature, and disk-temperature warnings use one continuous
stability window on both lifecycle edges: an unchanged factory configuration
requires five minutes above the trigger to open and five minutes at or below
the recovery threshold to close. A return to the hysteresis band or a renewed
breach resets the recovery run. Critical metric evidence may bypass only this
factory or legacy activation delay; an explicit alert-intent policy remains
authoritative. A metric-specific delay, including zero, remains the simple
operator override and becomes the matching recovery window for these gauges.
Recovery timing uses the reducer's monotonic runtime evidence when supplied,
so wall-clock changes cannot prematurely close an incident. Missing or unknown
observations do not constitute healthy evidence and therefore cannot advance a
recovery run. Restarts conservatively restart an in-progress recovery window
rather than manufacturing elapsed health.
For guest disk capacity, valid per-filesystem evidence owns the alert identity
and suppresses the less actionable guest-wide aggregate. This guarantees one
incident for one full filesystem while retaining the aggregate only when no
usable filesystem evidence exists. OS-managed transient mounts, including
macOS Gatekeeper App Translocation volumes, are removed at the shared
filesystem collection boundary before they can enter capacity inventory or
alert evaluation.

View file

@ -170,6 +170,7 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec
var intent *reducer.DiscreteIntent var intent *reducer.DiscreteIntent
delaySeconds := 0 delaySeconds := 0
effectiveIntent := m.resolveEffectiveIntentPolicyNoLock(spec.ResourceID, resourceType, MetricAlertIntentSignal(metricType)) effectiveIntent := m.resolveEffectiveIntentPolicyNoLock(spec.ResourceID, resourceType, MetricAlertIntentSignal(metricType))
stability := metricStabilityPolicy{}
if effectiveIntent.Explicit { if effectiveIntent.Explicit {
decision := m.evaluateIntentNoLock(spec.ResourceID, resourceType, MetricAlertIntentSignal(metricType), trackingKey, observedAt, triggered, BackupIntentContext{}) decision := m.evaluateIntentNoLock(spec.ResourceID, resourceType, MetricAlertIntentSignal(metricType), trackingKey, observedAt, triggered, BackupIntentContext{})
if decision.StateChanged { if decision.StateChanged {
@ -184,6 +185,9 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec
} else if triggered { } else if triggered {
delaySeconds = m.getTimeThreshold(spec.ResourceID, resourceType, metricType) delaySeconds = m.getTimeThreshold(spec.ResourceID, resourceType, metricType)
} }
if !effectiveIntent.Explicit {
stability = metricStabilityFor(metricType, effectiveIntent.GraceSeconds)
}
evidence := alertspecs.AlertEvidence{ evidence := alertspecs.AlertEvidence{
ObservedAt: observedAt, ObservedAt: observedAt,
@ -210,12 +214,14 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec
RuntimeTickValid: true, RuntimeTickValid: true,
ObservedAt: observedAt, ObservedAt: observedAt,
}, reducer.MetricRule{ }, reducer.MetricRule{
Trigger: spec.MetricThreshold.Trigger, Trigger: spec.MetricThreshold.Trigger,
Clear: clearThreshold, Clear: clearThreshold,
Critical: spec.MetricThreshold.Critical, Critical: spec.MetricThreshold.Critical,
CriticalDisabled: spec.MetricThreshold.Critical == nil, CriticalDisabled: spec.MetricThreshold.Critical == nil,
DelaySeconds: delaySeconds, DelaySeconds: delaySeconds,
Intent: intent, CriticalBypassesDelay: stability.CriticalBypassesDelay,
RecoveryDelaySeconds: stability.RecoveryDelaySeconds,
Intent: intent,
}) })
primary := reducer.EventType("") primary := reducer.EventType("")
if len(events) > 0 { if len(events) > 0 {
@ -243,6 +249,10 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec
"clearThreshold": metricClearThreshold(spec.MetricThreshold, threshold), "clearThreshold": metricClearThreshold(spec.MetricThreshold, threshold),
"monitorOnly": monitorOnly, "monitorOnly": monitorOnly,
} }
if stability.RecoveryDelaySeconds > 0 {
alertMetadata["stabilityWindowSeconds"] = stability.RecoveryDelaySeconds
alertMetadata["criticalBypassesStability"] = stability.CriticalBypassesDelay
}
if unit != "" { if unit != "" {
alertMetadata["unit"] = unit alertMetadata["unit"] = unit
} }

View file

@ -261,40 +261,22 @@ func (m *Manager) CheckGuest(guest any, instanceName string) {
diskReadMetric, diskWriteMetric, networkInMetric, networkOutMetric := guestIORateMetrics(snapshot) diskReadMetric, diskWriteMetric, networkInMetric, networkOutMetric := guestIORateMetrics(snapshot)
diskMetric := &UnifiedResourceMetric{Percent: diskUsage} diskMetric := &UnifiedResourceMetric{Percent: diskUsage}
if len(disks) > 0 { if len(disks) > 0 {
var aggregateTotal, aggregateUsed int64
hasDedicatedDiskOverride := false
seenAggregateDiskKeys := make(map[string]struct{}) seenAggregateDiskKeys := make(map[string]struct{})
for idx, disk := range disks { for idx, disk := range disks {
if disk.Total <= 0 || disk.Usage < 0 { if disk.Total <= 0 || disk.Usage < 0 {
continue continue
} }
label, keySource, sanitizedKey := guestDiskIdentity(disk, idx) _, _, sanitizedKey := guestDiskIdentity(disk, idx)
if _, exists := seenAggregateDiskKeys[sanitizedKey]; exists { if _, exists := seenAggregateDiskKeys[sanitizedKey]; exists {
continue continue
} }
seenAggregateDiskKeys[sanitizedKey] = struct{}{} seenAggregateDiskKeys[sanitizedKey] = struct{}{}
m.mu.RLock()
diskOverride, hasDiskOverride := lookupGuestDiskOverride(m.config.Overrides, guest, guestID, keySource)
m.mu.RUnlock()
if hasDiskOverride && (diskOverride.Disabled || diskOverride.Disk != nil) {
hasDedicatedDiskOverride = true
log.Debug().
Str("guest", name).
Str("diskLabel", label).
Msg("Excluding guest filesystem with a dedicated override from aggregate disk alert")
continue
}
aggregateTotal += disk.Total
aggregateUsed += disk.Used
} }
if hasDedicatedDiskOverride { if len(seenAggregateDiskKeys) > 0 {
if aggregateTotal > 0 { // Filesystem evidence is more actionable than the guest-wide rollup.
diskMetric.Percent = float64(aggregateUsed) / float64(aggregateTotal) * 100 // Evaluating both creates two incidents for the same full filesystem.
} else { diskMetric = nil
diskMetric = nil m.clearAlert(canonicalMetricStateID(guestID, "disk"))
m.clearAlert(canonicalMetricStateID(guestID, "disk"))
}
} }
} }
m.evaluateUnifiedMetrics(&UnifiedResourceInput{ m.evaluateUnifiedMetrics(&UnifiedResourceInput{

View file

@ -11,6 +11,44 @@ import (
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
) )
type metricStabilityPolicy struct {
CriticalBypassesDelay bool
RecoveryDelaySeconds int
}
const (
defaultLegacyMetricDelaySeconds = 5
defaultNoisyGaugeStabilitySeconds = 5 * 60
)
func defaultMetricStabilityDelay(metricType string) int {
switch strings.ToLower(strings.TrimSpace(metricType)) {
case "memory", "temperature", "disktemperature", "disk_temperature":
return defaultNoisyGaugeStabilitySeconds
default:
return 0
}
}
// metricStabilityFor keeps noisy gauges trustworthy without adding a second
// operator-facing policy surface. The configured alert delay is the one
// stability window on both lifecycle edges; critical evidence bypasses only
// the legacy/default activation delay, never an explicit intent policy.
func metricStabilityFor(metricType string, delaySeconds int) metricStabilityPolicy {
if delaySeconds <= 0 {
return metricStabilityPolicy{}
}
switch strings.ToLower(strings.TrimSpace(metricType)) {
case "memory", "temperature", "disktemperature", "disk_temperature":
return metricStabilityPolicy{
CriticalBypassesDelay: true,
RecoveryDelaySeconds: delaySeconds,
}
default:
return metricStabilityPolicy{}
}
}
func isMetricThresholdAlertType(metricType string) bool { func isMetricThresholdAlertType(metricType string) bool {
switch metricType { switch metricType {
case "cpu", "memory", "disk", "diskRead", "diskWrite", "networkIn", "networkOut", "temperature", "usage": case "cpu", "memory", "disk", "diskRead", "diskWrite", "networkIn", "networkOut", "temperature", "usage":
@ -66,23 +104,58 @@ func (m *Manager) getLegacyTimeThresholdWithSource(resourceType, metricType stri
if delay, ok := m.getMetricTimeThreshold(resourceType, metricType); ok { if delay, ok := m.getMetricTimeThreshold(resourceType, metricType); ok {
return delay, "legacy.metricTimeThresholds." + strings.ToLower(strings.TrimSpace(resourceType)) + "." + strings.ToLower(strings.TrimSpace(metricType)), true return delay, "legacy.metricTimeThresholds." + strings.ToLower(strings.TrimSpace(resourceType)) + "." + strings.ToLower(strings.TrimSpace(metricType)), true
} }
if delay, ok := m.getGlobalExactMetricTimeThreshold(metricType); ok {
base, hasTypeSpecific := m.getBaseTimeThreshold(resourceType) return delay, "legacy.metricTimeThresholds.all." + strings.ToLower(strings.TrimSpace(metricType)), true
if !hasTypeSpecific {
if delay, ok := m.getGlobalMetricTimeThreshold(metricType); ok {
return delay, "legacy.metricTimeThresholds.all." + strings.ToLower(strings.TrimSpace(metricType)), true
}
} }
base, hasTypeSpecific := m.getBaseTimeThreshold(resourceType)
if hasTypeSpecific { if hasTypeSpecific {
// The legacy factory delay was five seconds for every metric. Upgrade
// only that unchanged default for noisy gauges; a deliberate type-level
// value remains authoritative, as does any metric-specific override above.
if base == defaultLegacyMetricDelaySeconds {
if delay := defaultMetricStabilityDelay(metricType); delay > 0 {
return delay, "factory.metricStability." + strings.ToLower(strings.TrimSpace(metricType)), true
}
}
return base, "legacy.timeThresholds." + strings.ToLower(strings.TrimSpace(resourceType)), true return base, "legacy.timeThresholds." + strings.ToLower(strings.TrimSpace(resourceType)), true
} }
if delay, ok := m.getGlobalMetricTimeThreshold(metricType); ok {
return delay, "legacy.metricTimeThresholds.all." + strings.ToLower(strings.TrimSpace(metricType)), true
}
if base != 0 { if base != 0 {
if base == defaultLegacyMetricDelaySeconds {
if delay := defaultMetricStabilityDelay(metricType); delay > 0 {
return delay, "factory.metricStability." + strings.ToLower(strings.TrimSpace(metricType)), true
}
}
return base, "legacy.timeThresholds.all", true return base, "legacy.timeThresholds.all", true
} }
return 0, "", false return 0, "", false
} }
// getGlobalExactMetricTimeThreshold returns only an explicit all.<metric>
// override. Broad all.default/all.* fallbacks retain their legacy position
// after resource-type delays and therefore cannot accidentally erase those
// more specific policies.
func (m *Manager) getGlobalExactMetricTimeThreshold(metricType string) (int, bool) {
if len(m.config.MetricTimeThresholds) == 0 {
return 0, false
}
perType, ok := m.config.MetricTimeThresholds["all"]
if !ok || len(perType) == 0 {
return 0, false
}
metricKey := strings.ToLower(strings.TrimSpace(metricType))
if metricKey == "" {
return 0, false
}
delay, ok := perType[metricKey]
return delay, ok
}
// getMetricTimeThreshold returns a metric-specific delay if configured at the resource-type level. // getMetricTimeThreshold returns a metric-specific delay if configured at the resource-type level.
func (m *Manager) getMetricTimeThreshold(resourceType, metricType string) (int, bool) { func (m *Manager) getMetricTimeThreshold(resourceType, metricType string) (int, bool) {
if len(m.config.MetricTimeThresholds) == 0 { if len(m.config.MetricTimeThresholds) == 0 {
@ -278,6 +351,7 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
effectiveIntent := m.resolveEffectiveIntentPolicyNoLock(resourceID, resourceType, MetricAlertIntentSignal(metricType)) effectiveIntent := m.resolveEffectiveIntentPolicyNoLock(resourceID, resourceType, MetricAlertIntentSignal(metricType))
var intent *reducer.DiscreteIntent var intent *reducer.DiscreteIntent
delaySeconds := 0 delaySeconds := 0
stability := metricStabilityPolicy{}
if effectiveIntent.Explicit { if effectiveIntent.Explicit {
decision := m.evaluateIntentNoLock(resourceID, resourceType, MetricAlertIntentSignal(metricType), trackingKey, time.Now(), conditionActive, BackupIntentContext{}) decision := m.evaluateIntentNoLock(resourceID, resourceType, MetricAlertIntentSignal(metricType), trackingKey, time.Now(), conditionActive, BackupIntentContext{})
if decision.StateChanged { if decision.StateChanged {
@ -292,6 +366,9 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
} else if conditionActive { } else if conditionActive {
delaySeconds = m.getTimeThreshold(resourceID, resourceType, metricType) delaySeconds = m.getTimeThreshold(resourceID, resourceType, metricType)
} }
if !effectiveIntent.Explicit {
stability = metricStabilityFor(metricType, effectiveIntent.GraceSeconds)
}
events := m.core.ApplyMetric(reducer.MetricSignal{ events := m.core.ApplyMetric(reducer.MetricSignal{
ResourceID: resourceID, ResourceID: resourceID,
@ -302,10 +379,12 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
RuntimeTickValid: true, RuntimeTickValid: true,
ObservedAt: m.policyNow(), ObservedAt: m.policyNow(),
}, reducer.MetricRule{ }, reducer.MetricRule{
Trigger: threshold.Trigger, Trigger: threshold.Trigger,
Clear: threshold.Clear, Clear: threshold.Clear,
DelaySeconds: delaySeconds, DelaySeconds: delaySeconds,
Intent: intent, CriticalBypassesDelay: stability.CriticalBypassesDelay,
RecoveryDelaySeconds: stability.RecoveryDelaySeconds,
Intent: intent,
}) })
primary := reducer.EventType("") primary := reducer.EventType("")
if len(events) > 0 { if len(events) > 0 {

View file

@ -65,6 +65,15 @@ type MetricRule struct {
// an explicit Intent gate is supplied — the manager's explicit intent // an explicit Intent gate is supplied — the manager's explicit intent
// policies replace the legacy time-threshold delay. // policies replace the legacy time-threshold delay.
DelaySeconds int DelaySeconds int
// CriticalBypassesDelay lets urgent evidence activate immediately while
// warning-level evidence still has to survive the stability window. It is
// intentionally ignored when an explicit Intent gate is present: explicit
// operator policy remains authoritative.
CriticalBypassesDelay bool
// RecoveryDelaySeconds is the continuous time at or below Clear required
// before a firing incident resolves. A value <= 0 preserves immediate
// hysteresis recovery.
RecoveryDelaySeconds int
// Intent is the resolved intent-policy context for this observation // Intent is the resolved intent-policy context for this observation
// (metric.<name> signals); nil means no gate. // (metric.<name> signals); nil means no gate.
Intent *DiscreteIntent Intent *DiscreteIntent
@ -108,9 +117,15 @@ type Incident struct {
// toward DiscreteRule.RecoveryConfirmations; reset by any matching // toward DiscreteRule.RecoveryConfirmations; reset by any matching
// observation. Unused by the metric family. // observation. Unused by the metric family.
RecoveryCount int RecoveryCount int
Acknowledged bool // RecoverySince and its monotonic companions track a continuous healthy
AckUser string // run for metric rules with a recovery stability window.
AckAt time.Time RecoverySince time.Time
RecoveryElapsed time.Duration
RecoveryTicksSupplied bool
LastRecoveryRuntimeTick time.Duration
Acknowledged bool
AckUser string
AckAt time.Time
// Backup-run bookkeeping for the intent gate's backup-offline deferral // Backup-run bookkeeping for the intent gate's backup-offline deferral
// sub-policy (confirmation family only). // sub-policy (confirmation family only).
BackupActive bool BackupActive bool
@ -390,6 +405,7 @@ func (s *State) ApplyMetric(signal MetricSignal, rule MetricRule) []Event {
// recomputes level on every tick, so severity can demote as well as // recomputes level on every tick, so severity can demote as well as
// escalate while firing. // escalate while firing.
if incident != nil && incident.State == StateFiring { if incident != nil && incident.State == StateFiring {
resetMetricRecoveryRun(incident)
previous := incident.Severity previous := incident.Severity
incident.Severity = severityFor(signal.Value, rule, signal.Metric) incident.Severity = severityFor(signal.Value, rule, signal.Metric)
incident.LastValue = signal.Value incident.LastValue = signal.Value
@ -444,7 +460,9 @@ func (s *State) ApplyMetric(signal MetricSignal, rule MetricRule) []Event {
return pendingResult() return pendingResult()
} }
} else if rule.DelaySeconds > 0 { } else if rule.DelaySeconds > 0 {
if signal.ObservedAt.Sub(incident.PendingSince) < time.Duration(rule.DelaySeconds)*time.Second { critical := severityFor(signal.Value, rule, signal.Metric) == SeverityCritical
if !(critical && rule.CriticalBypassesDelay) &&
signal.ObservedAt.Sub(incident.PendingSince) < time.Duration(rule.DelaySeconds)*time.Second {
return pendingResult() return pendingResult()
} }
} }
@ -473,6 +491,31 @@ func (s *State) ApplyMetric(signal MetricSignal, rule MetricRule) []Event {
clear = rule.Trigger clear = rule.Trigger
} }
if signal.Value <= clear { if signal.Value <= clear {
if rule.RecoveryDelaySeconds > 0 {
if incident.RecoverySince.IsZero() {
incident.RecoverySince = signal.ObservedAt
incident.RecoveryTicksSupplied = signal.RuntimeTickValid
incident.LastRecoveryRuntimeTick = signal.RuntimeTick
incident.RecoveryElapsed = 0
incident.LastObservedAt = signal.ObservedAt
return nil
}
if signal.RuntimeTickValid {
if incident.RecoveryTicksSupplied && signal.RuntimeTick >= incident.LastRecoveryRuntimeTick {
incident.RecoveryElapsed += signal.RuntimeTick - incident.LastRecoveryRuntimeTick
}
incident.RecoveryTicksSupplied = true
incident.LastRecoveryRuntimeTick = signal.RuntimeTick
}
elapsed := signal.ObservedAt.Sub(incident.RecoverySince)
if incident.RecoveryTicksSupplied {
elapsed = incident.RecoveryElapsed
}
incident.LastObservedAt = signal.ObservedAt
if elapsed < time.Duration(rule.RecoveryDelaySeconds)*time.Second {
return nil
}
}
severity := incident.Severity severity := incident.Severity
delete(s.incidents, key) delete(s.incidents, key)
s.markAckInactive(key, signal.ObservedAt) s.markAckInactive(key, signal.ObservedAt)
@ -482,9 +525,20 @@ func (s *State) ApplyMetric(signal MetricSignal, rule MetricRule) []Event {
// Hysteresis hold: between clear and trigger the incident stays firing. // Hysteresis hold: between clear and trigger the incident stays firing.
// The manager does not refresh value or last-seen in this band; the // The manager does not refresh value or last-seen in this band; the
// reducer mirrors that so parity diffs stay exact. // reducer mirrors that so parity diffs stay exact.
resetMetricRecoveryRun(incident)
return nil return nil
} }
func resetMetricRecoveryRun(incident *Incident) {
if incident == nil {
return
}
incident.RecoverySince = time.Time{}
incident.RecoveryElapsed = 0
incident.RecoveryTicksSupplied = false
incident.LastRecoveryRuntimeTick = 0
}
// SeedFiringIncident installs a firing incident directly, bypassing the // SeedFiringIncident installs a firing incident directly, bypassing the
// transition functions. Shadow mode uses it to align the reducer with // transition functions. Shadow mode uses it to align the reducer with
// pre-existing manager state (persisted-alert restore, divergence resync); // pre-existing manager state (persisted-alert restore, divergence resync);

View file

@ -132,6 +132,52 @@ func TestDelayFiresWithPendingStartAndDipResets(t *testing.T) {
} }
} }
func TestCriticalCanBypassWarningStabilityDelay(t *testing.T) {
state := NewState()
rule := MetricRule{Trigger: 80, Clear: 75, DelaySeconds: 300, CriticalBypassesDelay: true}
events := state.ApplyMetric(signalAt(95, 0), rule)
if len(events) != 1 || events[0].Type != EventFired || events[0].Severity != SeverityCritical {
t.Fatalf("events = %+v, want immediate critical fire", events)
}
}
func TestCriticalDoesNotBypassExplicitIntent(t *testing.T) {
state := NewState()
rule := MetricRule{
Trigger: 80, Clear: 75, CriticalBypassesDelay: true,
Intent: &DiscreteIntent{Explicit: true, GraceSeconds: 300},
}
events := state.ApplyMetric(signalAt(95, 0), rule)
if len(events) != 1 || events[0].Type != EventPending {
t.Fatalf("events = %+v, want explicit intent to keep critical pending", events)
}
}
func TestMetricRecoveryRequiresContinuousHealthyWindow(t *testing.T) {
state := NewState()
rule := MetricRule{Trigger: 80, Clear: 75, RecoveryDelaySeconds: 120}
state.ApplyMetric(signalAt(90, 0), rule)
if events := state.ApplyMetric(signalAt(70, time.Minute), rule); len(events) != 0 {
t.Fatalf("first healthy observation emitted %+v", events)
}
if events := state.ApplyMetric(signalAt(78, 90*time.Second), rule); len(events) != 0 {
t.Fatalf("hysteresis-band interruption emitted %+v", events)
}
if events := state.ApplyMetric(signalAt(70, 2*time.Minute), rule); len(events) != 0 {
t.Fatalf("restarted healthy run emitted %+v", events)
}
if events := state.ApplyMetric(signalAt(70, 3*time.Minute+59*time.Second), rule); len(events) != 0 {
t.Fatalf("recovery fired before continuous window elapsed: %+v", events)
}
events := state.ApplyMetric(signalAt(70, 4*time.Minute), rule)
if len(events) != 1 || events[0].Type != EventResolved {
t.Fatalf("events = %+v, want recovery after continuous healthy window", events)
}
}
func TestClearFallsBackToTriggerWhenUnset(t *testing.T) { func TestClearFallsBackToTriggerWhenUnset(t *testing.T) {
state := NewState() state := NewState()
rule := MetricRule{Trigger: 80, Clear: 0} rule := MetricRule{Trigger: 80, Clear: 0}

View file

@ -938,6 +938,32 @@ func TestGuestFilesystemThresholdOverrideOwnsMountEvaluation(t *testing.T) {
} }
} }
func TestGuestFilesystemEvidenceSuppressesDuplicateAggregateAlert(t *testing.T) {
m := newTestManager(t)
guestID := BuildGuestKey("pve1", "node1", 109)
m.mu.Lock()
m.config.TimeThresholds = map[string]int{}
m.config.GuestDefaults = ThresholdConfig{Disk: &HysteresisThreshold{Trigger: 80, Clear: 75}}
m.mu.Unlock()
m.CheckGuest(models.VM{
ID: guestID, VMID: 109, Name: "pulse-dev", Node: "node1", Instance: "pve1", Status: "running",
Disk: models.Disk{Usage: 96.5},
Disks: []models.Disk{{Mountpoint: "/", Device: "/dev/vda1", Usage: 96.9, Total: 100, Used: 97, Free: 3}},
}, "pve1")
aggregateID := canonicalMetricStateID(guestID, "disk")
filesystemID := canonicalMetricStateID(guestID+"-disk-dev-vda1", "disk")
m.mu.RLock()
defer m.mu.RUnlock()
if _, exists := testLookupActiveAlert(t, m, aggregateID); exists {
t.Fatalf("aggregate alert %q duplicated filesystem evidence", aggregateID)
}
if _, exists := testLookupActiveAlert(t, m, filesystemID); !exists {
t.Fatalf("actionable filesystem alert %q was not created", filesystemID)
}
}
// Regression: checkMetric stores canonical-identity alerts under the // Regression: checkMetric stores canonical-identity alerts under the
// canonical state key, so hysteresis resolution must not remove only the // canonical state key, so hysteresis resolution must not remove only the
// unregistered legacy "<resourceID>-<metric>" ID. // unregistered legacy "<resourceID>-<metric>" ID.

View file

@ -85,6 +85,59 @@ func TestGetTimeThresholdMetricOverrides(t *testing.T) {
} }
} }
func TestGetTimeThresholdUsesTrustStableNoisyGaugeDefaults(t *testing.T) {
manager := NewManagerWithDataDir(t.TempDir(), WithoutPersistedAlertRestore())
t.Cleanup(manager.Stop)
if got := manager.getTimeThreshold("vm-resource", "VM", "memory"); got != defaultNoisyGaugeStabilitySeconds {
t.Fatalf("default memory stability = %d, want %d", got, defaultNoisyGaugeStabilitySeconds)
}
if got := manager.getTimeThreshold("node-1", "Node", "temperature"); got != defaultNoisyGaugeStabilitySeconds {
t.Fatalf("default temperature stability = %d, want %d", got, defaultNoisyGaugeStabilitySeconds)
}
if got := manager.getTimeThreshold("vm-resource", "VM", "cpu"); got != defaultLegacyMetricDelaySeconds {
t.Fatalf("default CPU delay = %d, want %d", got, defaultLegacyMetricDelaySeconds)
}
manager.mu.Lock()
manager.config.MetricTimeThresholds = map[string]map[string]int{
"all": {"memory": 0},
}
manager.mu.Unlock()
if got := manager.getTimeThreshold("vm-resource", "VM", "memory"); got != 0 {
t.Fatalf("explicit global memory override = %d, want 0", got)
}
}
func TestCheckMetricNoisyWarningWaitsButCriticalFiresImmediately(t *testing.T) {
manager := NewManagerWithDataDir(t.TempDir(), WithoutPersistedAlertRestore())
t.Cleanup(manager.Stop)
threshold := &HysteresisThreshold{Trigger: 85, Clear: 80}
canonicalState := buildCanonicalStateID("vm-resource", "metric-threshold:memory")
manager.checkMetric("vm-resource", "database", "node-1", "pve-1", "VM", "memory", 90, threshold, nil)
manager.mu.RLock()
_, warningActive := manager.getActiveAlertNoLock(canonicalState)
manager.mu.RUnlock()
if warningActive {
t.Fatal("brief warning-level memory evidence opened an alert")
}
manager.checkMetric("vm-resource", "database", "node-1", "pve-1", "VM", "memory", 96, threshold, nil)
manager.mu.RLock()
alert, criticalActive := manager.getActiveAlertNoLock(canonicalState)
manager.mu.RUnlock()
if !criticalActive {
t.Fatal("critical memory evidence did not bypass the factory stability delay")
}
if alert.Level != AlertLevelCritical {
t.Fatalf("critical memory alert level = %q, want %q", alert.Level, AlertLevelCritical)
}
}
func TestCheckMetricUsesPendingStartTime(t *testing.T) { func TestCheckMetricUsesPendingStartTime(t *testing.T) {
manager := NewManager() manager := NewManager()

View file

@ -1559,3 +1559,27 @@ func TestUnifiedStorageForecastKeepsCanonicalMetricSpecAcrossPlatforms(t *testin
t.Fatalf("CanonicalSpecID = %q, want canonical usage spec", alert.CanonicalSpecID) t.Fatalf("CanonicalSpecID = %q, want canonical usage spec", alert.CanonicalSpecID)
} }
} }
func TestUnifiedMetricNoisyWarningWaitsButCriticalFiresImmediately(t *testing.T) {
m := newTestManager(t)
input := &UnifiedResourceInput{
ID: "vm-noisy-memory",
Type: "vm",
Name: "database",
Memory: &UnifiedResourceMetric{Percent: 90},
}
alertID := canonicalMetricStateID(input.ID, "memory")
m.CheckUnifiedResource(input)
assertAlertMissing(t, m, alertID)
input.Memory.Percent = 96
m.CheckUnifiedResource(input)
alert := testRequireActiveAlert(t, m, alertID)
if alert.Level != AlertLevelCritical {
t.Fatalf("critical memory alert level = %q, want %q", alert.Level, AlertLevelCritical)
}
if got := alert.Metadata["stabilityWindowSeconds"]; got != defaultNoisyGaugeStabilitySeconds {
t.Fatalf("stabilityWindowSeconds = %v, want %d", got, defaultNoisyGaugeStabilitySeconds)
}
}

View file

@ -129,6 +129,13 @@ var containerPathPrefixes = []string{
"/mnt/.ix-apps/docker/", // TrueNAS SCALE Docker overlay mounts "/mnt/.ix-apps/docker/", // TrueNAS SCALE Docker overlay mounts
} }
// transientMountPatterns identify OS-managed mounts whose lifetime follows an
// application launch rather than operator-managed storage. They must not
// participate in capacity inventory or alerting.
var transientMountPatterns = []string{
"/AppTranslocation/", // macOS Gatekeeper translocated application image
}
// ShouldSkipFilesystem determines if a filesystem should be excluded from disk // ShouldSkipFilesystem determines if a filesystem should be excluded from disk
// usage aggregation. It checks for read-only filesystems, virtual/pseudo filesystems, // usage aggregation. It checks for read-only filesystems, virtual/pseudo filesystems,
// network mounts, and special system mountpoints. Returns skip=true if the filesystem // network mounts, and special system mountpoints. Returns skip=true if the filesystem
@ -168,6 +175,13 @@ func ShouldSkipFilesystem(fsType, mountpoint string, totalBytes, usedBytes uint6
reasons = append(reasons, "special-mountpoint") reasons = append(reasons, "special-mountpoint")
} }
for _, pattern := range transientMountPatterns {
if strings.Contains(mountpoint, pattern) {
reasons = append(reasons, "transient-mountpoint")
break
}
}
// Windows System Reserved partition // Windows System Reserved partition
if mountpoint == "System Reserved" || strings.Contains(mountpoint, "System Reserved") { if mountpoint == "System Reserved" || strings.Contains(mountpoint, "System Reserved") {
reasons = append(reasons, "special-mountpoint") reasons = append(reasons, "special-mountpoint")

View file

@ -204,6 +204,7 @@ func TestShouldSkipFilesystem(t *testing.T) {
{"macos data companion volume", "apfs", "/System/Volumes/Data", 245107195904, 213573853184, true}, {"macos data companion volume", "apfs", "/System/Volumes/Data", 245107195904, 213573853184, true},
{"macos preboot companion volume", "apfs", "/System/Volumes/Preboot", 245107195904, 213573853184, true}, {"macos preboot companion volume", "apfs", "/System/Volumes/Preboot", 245107195904, 213573853184, true},
{"macos simulator runtime volume", "apfs", "/Library/Developer/CoreSimulator/Volumes/iOS_23E254a", 18058575872, 17593331712, true}, {"macos simulator runtime volume", "apfs", "/Library/Developer/CoreSimulator/Volumes/iOS_23E254a", 18058575872, 17593331712, true},
{"macos app translocation volume", "apfs", "/private/var/folders/xy/random/T/AppTranslocation/2A9C/d/Pulse.app", 245107195904, 221809991680, true},
// Regular filesystems that should NOT be skipped // Regular filesystems that should NOT be skipped
{"ext4 root", "ext4", "/", 100 * 1024 * 1024 * 1024, 50 * 1024 * 1024 * 1024, false}, {"ext4 root", "ext4", "/", 100 * 1024 * 1024 * 1024, 50 * 1024 * 1024 * 1024, false},