mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-19 10:33:29 +00:00
fix(alerts): stabilize noisy lifecycle signals
This commit is contained in:
parent
a506cdde8b
commit
d671f05408
11 changed files with 359 additions and 45 deletions
|
|
@ -2390,3 +2390,28 @@ against a managed local backend. The browser phase must exercise active-alert
|
||||||
snooze, delivery diagnosis, restart persistence, unsnooze, incident evidence,
|
snooze, delivery diagnosis, restart persistence, unsnooze, incident evidence,
|
||||||
history migration, virtualization, and clear tombstones without replacing the
|
history migration, virtualization, and clear tombstones without replacing the
|
||||||
backend APIs with route mocks.
|
backend APIs with route mocks.
|
||||||
|
|
||||||
|
### Noisy gauges use one trust-stable lifecycle
|
||||||
|
|
||||||
|
Memory, temperature, and disk-temperature warnings use one continuous
|
||||||
|
stability window on both lifecycle edges: an unchanged factory configuration
|
||||||
|
requires five minutes above the trigger to open and five minutes at or below
|
||||||
|
the recovery threshold to close. A return to the hysteresis band or a renewed
|
||||||
|
breach resets the recovery run. Critical metric evidence may bypass only this
|
||||||
|
factory or legacy activation delay; an explicit alert-intent policy remains
|
||||||
|
authoritative. A metric-specific delay, including zero, remains the simple
|
||||||
|
operator override and becomes the matching recovery window for these gauges.
|
||||||
|
|
||||||
|
Recovery timing uses the reducer's monotonic runtime evidence when supplied,
|
||||||
|
so wall-clock changes cannot prematurely close an incident. Missing or unknown
|
||||||
|
observations do not constitute healthy evidence and therefore cannot advance a
|
||||||
|
recovery run. Restarts conservatively restart an in-progress recovery window
|
||||||
|
rather than manufacturing elapsed health.
|
||||||
|
|
||||||
|
For guest disk capacity, valid per-filesystem evidence owns the alert identity
|
||||||
|
and suppresses the less actionable guest-wide aggregate. This guarantees one
|
||||||
|
incident for one full filesystem while retaining the aggregate only when no
|
||||||
|
usable filesystem evidence exists. OS-managed transient mounts, including
|
||||||
|
macOS Gatekeeper App Translocation volumes, are removed at the shared
|
||||||
|
filesystem collection boundary before they can enter capacity inventory or
|
||||||
|
alert evaluation.
|
||||||
|
|
|
||||||
|
|
@ -170,6 +170,7 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec
|
||||||
var intent *reducer.DiscreteIntent
|
var intent *reducer.DiscreteIntent
|
||||||
delaySeconds := 0
|
delaySeconds := 0
|
||||||
effectiveIntent := m.resolveEffectiveIntentPolicyNoLock(spec.ResourceID, resourceType, MetricAlertIntentSignal(metricType))
|
effectiveIntent := m.resolveEffectiveIntentPolicyNoLock(spec.ResourceID, resourceType, MetricAlertIntentSignal(metricType))
|
||||||
|
stability := metricStabilityPolicy{}
|
||||||
if effectiveIntent.Explicit {
|
if effectiveIntent.Explicit {
|
||||||
decision := m.evaluateIntentNoLock(spec.ResourceID, resourceType, MetricAlertIntentSignal(metricType), trackingKey, observedAt, triggered, BackupIntentContext{})
|
decision := m.evaluateIntentNoLock(spec.ResourceID, resourceType, MetricAlertIntentSignal(metricType), trackingKey, observedAt, triggered, BackupIntentContext{})
|
||||||
if decision.StateChanged {
|
if decision.StateChanged {
|
||||||
|
|
@ -184,6 +185,9 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec
|
||||||
} else if triggered {
|
} else if triggered {
|
||||||
delaySeconds = m.getTimeThreshold(spec.ResourceID, resourceType, metricType)
|
delaySeconds = m.getTimeThreshold(spec.ResourceID, resourceType, metricType)
|
||||||
}
|
}
|
||||||
|
if !effectiveIntent.Explicit {
|
||||||
|
stability = metricStabilityFor(metricType, effectiveIntent.GraceSeconds)
|
||||||
|
}
|
||||||
|
|
||||||
evidence := alertspecs.AlertEvidence{
|
evidence := alertspecs.AlertEvidence{
|
||||||
ObservedAt: observedAt,
|
ObservedAt: observedAt,
|
||||||
|
|
@ -210,12 +214,14 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec
|
||||||
RuntimeTickValid: true,
|
RuntimeTickValid: true,
|
||||||
ObservedAt: observedAt,
|
ObservedAt: observedAt,
|
||||||
}, reducer.MetricRule{
|
}, reducer.MetricRule{
|
||||||
Trigger: spec.MetricThreshold.Trigger,
|
Trigger: spec.MetricThreshold.Trigger,
|
||||||
Clear: clearThreshold,
|
Clear: clearThreshold,
|
||||||
Critical: spec.MetricThreshold.Critical,
|
Critical: spec.MetricThreshold.Critical,
|
||||||
CriticalDisabled: spec.MetricThreshold.Critical == nil,
|
CriticalDisabled: spec.MetricThreshold.Critical == nil,
|
||||||
DelaySeconds: delaySeconds,
|
DelaySeconds: delaySeconds,
|
||||||
Intent: intent,
|
CriticalBypassesDelay: stability.CriticalBypassesDelay,
|
||||||
|
RecoveryDelaySeconds: stability.RecoveryDelaySeconds,
|
||||||
|
Intent: intent,
|
||||||
})
|
})
|
||||||
primary := reducer.EventType("")
|
primary := reducer.EventType("")
|
||||||
if len(events) > 0 {
|
if len(events) > 0 {
|
||||||
|
|
@ -243,6 +249,10 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec
|
||||||
"clearThreshold": metricClearThreshold(spec.MetricThreshold, threshold),
|
"clearThreshold": metricClearThreshold(spec.MetricThreshold, threshold),
|
||||||
"monitorOnly": monitorOnly,
|
"monitorOnly": monitorOnly,
|
||||||
}
|
}
|
||||||
|
if stability.RecoveryDelaySeconds > 0 {
|
||||||
|
alertMetadata["stabilityWindowSeconds"] = stability.RecoveryDelaySeconds
|
||||||
|
alertMetadata["criticalBypassesStability"] = stability.CriticalBypassesDelay
|
||||||
|
}
|
||||||
if unit != "" {
|
if unit != "" {
|
||||||
alertMetadata["unit"] = unit
|
alertMetadata["unit"] = unit
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -261,40 +261,22 @@ func (m *Manager) CheckGuest(guest any, instanceName string) {
|
||||||
diskReadMetric, diskWriteMetric, networkInMetric, networkOutMetric := guestIORateMetrics(snapshot)
|
diskReadMetric, diskWriteMetric, networkInMetric, networkOutMetric := guestIORateMetrics(snapshot)
|
||||||
diskMetric := &UnifiedResourceMetric{Percent: diskUsage}
|
diskMetric := &UnifiedResourceMetric{Percent: diskUsage}
|
||||||
if len(disks) > 0 {
|
if len(disks) > 0 {
|
||||||
var aggregateTotal, aggregateUsed int64
|
|
||||||
hasDedicatedDiskOverride := false
|
|
||||||
seenAggregateDiskKeys := make(map[string]struct{})
|
seenAggregateDiskKeys := make(map[string]struct{})
|
||||||
for idx, disk := range disks {
|
for idx, disk := range disks {
|
||||||
if disk.Total <= 0 || disk.Usage < 0 {
|
if disk.Total <= 0 || disk.Usage < 0 {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
label, keySource, sanitizedKey := guestDiskIdentity(disk, idx)
|
_, _, sanitizedKey := guestDiskIdentity(disk, idx)
|
||||||
if _, exists := seenAggregateDiskKeys[sanitizedKey]; exists {
|
if _, exists := seenAggregateDiskKeys[sanitizedKey]; exists {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
seenAggregateDiskKeys[sanitizedKey] = struct{}{}
|
seenAggregateDiskKeys[sanitizedKey] = struct{}{}
|
||||||
|
|
||||||
m.mu.RLock()
|
|
||||||
diskOverride, hasDiskOverride := lookupGuestDiskOverride(m.config.Overrides, guest, guestID, keySource)
|
|
||||||
m.mu.RUnlock()
|
|
||||||
if hasDiskOverride && (diskOverride.Disabled || diskOverride.Disk != nil) {
|
|
||||||
hasDedicatedDiskOverride = true
|
|
||||||
log.Debug().
|
|
||||||
Str("guest", name).
|
|
||||||
Str("diskLabel", label).
|
|
||||||
Msg("Excluding guest filesystem with a dedicated override from aggregate disk alert")
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
aggregateTotal += disk.Total
|
|
||||||
aggregateUsed += disk.Used
|
|
||||||
}
|
}
|
||||||
if hasDedicatedDiskOverride {
|
if len(seenAggregateDiskKeys) > 0 {
|
||||||
if aggregateTotal > 0 {
|
// Filesystem evidence is more actionable than the guest-wide rollup.
|
||||||
diskMetric.Percent = float64(aggregateUsed) / float64(aggregateTotal) * 100
|
// Evaluating both creates two incidents for the same full filesystem.
|
||||||
} else {
|
diskMetric = nil
|
||||||
diskMetric = nil
|
m.clearAlert(canonicalMetricStateID(guestID, "disk"))
|
||||||
m.clearAlert(canonicalMetricStateID(guestID, "disk"))
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
m.evaluateUnifiedMetrics(&UnifiedResourceInput{
|
m.evaluateUnifiedMetrics(&UnifiedResourceInput{
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,44 @@ import (
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/rs/zerolog/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type metricStabilityPolicy struct {
|
||||||
|
CriticalBypassesDelay bool
|
||||||
|
RecoveryDelaySeconds int
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
defaultLegacyMetricDelaySeconds = 5
|
||||||
|
defaultNoisyGaugeStabilitySeconds = 5 * 60
|
||||||
|
)
|
||||||
|
|
||||||
|
func defaultMetricStabilityDelay(metricType string) int {
|
||||||
|
switch strings.ToLower(strings.TrimSpace(metricType)) {
|
||||||
|
case "memory", "temperature", "disktemperature", "disk_temperature":
|
||||||
|
return defaultNoisyGaugeStabilitySeconds
|
||||||
|
default:
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// metricStabilityFor keeps noisy gauges trustworthy without adding a second
|
||||||
|
// operator-facing policy surface. The configured alert delay is the one
|
||||||
|
// stability window on both lifecycle edges; critical evidence bypasses only
|
||||||
|
// the legacy/default activation delay, never an explicit intent policy.
|
||||||
|
func metricStabilityFor(metricType string, delaySeconds int) metricStabilityPolicy {
|
||||||
|
if delaySeconds <= 0 {
|
||||||
|
return metricStabilityPolicy{}
|
||||||
|
}
|
||||||
|
switch strings.ToLower(strings.TrimSpace(metricType)) {
|
||||||
|
case "memory", "temperature", "disktemperature", "disk_temperature":
|
||||||
|
return metricStabilityPolicy{
|
||||||
|
CriticalBypassesDelay: true,
|
||||||
|
RecoveryDelaySeconds: delaySeconds,
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
return metricStabilityPolicy{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func isMetricThresholdAlertType(metricType string) bool {
|
func isMetricThresholdAlertType(metricType string) bool {
|
||||||
switch metricType {
|
switch metricType {
|
||||||
case "cpu", "memory", "disk", "diskRead", "diskWrite", "networkIn", "networkOut", "temperature", "usage":
|
case "cpu", "memory", "disk", "diskRead", "diskWrite", "networkIn", "networkOut", "temperature", "usage":
|
||||||
|
|
@ -66,23 +104,58 @@ func (m *Manager) getLegacyTimeThresholdWithSource(resourceType, metricType stri
|
||||||
if delay, ok := m.getMetricTimeThreshold(resourceType, metricType); ok {
|
if delay, ok := m.getMetricTimeThreshold(resourceType, metricType); ok {
|
||||||
return delay, "legacy.metricTimeThresholds." + strings.ToLower(strings.TrimSpace(resourceType)) + "." + strings.ToLower(strings.TrimSpace(metricType)), true
|
return delay, "legacy.metricTimeThresholds." + strings.ToLower(strings.TrimSpace(resourceType)) + "." + strings.ToLower(strings.TrimSpace(metricType)), true
|
||||||
}
|
}
|
||||||
|
if delay, ok := m.getGlobalExactMetricTimeThreshold(metricType); ok {
|
||||||
base, hasTypeSpecific := m.getBaseTimeThreshold(resourceType)
|
return delay, "legacy.metricTimeThresholds.all." + strings.ToLower(strings.TrimSpace(metricType)), true
|
||||||
|
|
||||||
if !hasTypeSpecific {
|
|
||||||
if delay, ok := m.getGlobalMetricTimeThreshold(metricType); ok {
|
|
||||||
return delay, "legacy.metricTimeThresholds.all." + strings.ToLower(strings.TrimSpace(metricType)), true
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
base, hasTypeSpecific := m.getBaseTimeThreshold(resourceType)
|
||||||
if hasTypeSpecific {
|
if hasTypeSpecific {
|
||||||
|
// The legacy factory delay was five seconds for every metric. Upgrade
|
||||||
|
// only that unchanged default for noisy gauges; a deliberate type-level
|
||||||
|
// value remains authoritative, as does any metric-specific override above.
|
||||||
|
if base == defaultLegacyMetricDelaySeconds {
|
||||||
|
if delay := defaultMetricStabilityDelay(metricType); delay > 0 {
|
||||||
|
return delay, "factory.metricStability." + strings.ToLower(strings.TrimSpace(metricType)), true
|
||||||
|
}
|
||||||
|
}
|
||||||
return base, "legacy.timeThresholds." + strings.ToLower(strings.TrimSpace(resourceType)), true
|
return base, "legacy.timeThresholds." + strings.ToLower(strings.TrimSpace(resourceType)), true
|
||||||
}
|
}
|
||||||
|
if delay, ok := m.getGlobalMetricTimeThreshold(metricType); ok {
|
||||||
|
return delay, "legacy.metricTimeThresholds.all." + strings.ToLower(strings.TrimSpace(metricType)), true
|
||||||
|
}
|
||||||
if base != 0 {
|
if base != 0 {
|
||||||
|
if base == defaultLegacyMetricDelaySeconds {
|
||||||
|
if delay := defaultMetricStabilityDelay(metricType); delay > 0 {
|
||||||
|
return delay, "factory.metricStability." + strings.ToLower(strings.TrimSpace(metricType)), true
|
||||||
|
}
|
||||||
|
}
|
||||||
return base, "legacy.timeThresholds.all", true
|
return base, "legacy.timeThresholds.all", true
|
||||||
}
|
}
|
||||||
return 0, "", false
|
return 0, "", false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// getGlobalExactMetricTimeThreshold returns only an explicit all.<metric>
|
||||||
|
// override. Broad all.default/all.* fallbacks retain their legacy position
|
||||||
|
// after resource-type delays and therefore cannot accidentally erase those
|
||||||
|
// more specific policies.
|
||||||
|
func (m *Manager) getGlobalExactMetricTimeThreshold(metricType string) (int, bool) {
|
||||||
|
if len(m.config.MetricTimeThresholds) == 0 {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
|
||||||
|
perType, ok := m.config.MetricTimeThresholds["all"]
|
||||||
|
if !ok || len(perType) == 0 {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
|
||||||
|
metricKey := strings.ToLower(strings.TrimSpace(metricType))
|
||||||
|
if metricKey == "" {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
|
||||||
|
delay, ok := perType[metricKey]
|
||||||
|
return delay, ok
|
||||||
|
}
|
||||||
|
|
||||||
// getMetricTimeThreshold returns a metric-specific delay if configured at the resource-type level.
|
// getMetricTimeThreshold returns a metric-specific delay if configured at the resource-type level.
|
||||||
func (m *Manager) getMetricTimeThreshold(resourceType, metricType string) (int, bool) {
|
func (m *Manager) getMetricTimeThreshold(resourceType, metricType string) (int, bool) {
|
||||||
if len(m.config.MetricTimeThresholds) == 0 {
|
if len(m.config.MetricTimeThresholds) == 0 {
|
||||||
|
|
@ -278,6 +351,7 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
|
||||||
effectiveIntent := m.resolveEffectiveIntentPolicyNoLock(resourceID, resourceType, MetricAlertIntentSignal(metricType))
|
effectiveIntent := m.resolveEffectiveIntentPolicyNoLock(resourceID, resourceType, MetricAlertIntentSignal(metricType))
|
||||||
var intent *reducer.DiscreteIntent
|
var intent *reducer.DiscreteIntent
|
||||||
delaySeconds := 0
|
delaySeconds := 0
|
||||||
|
stability := metricStabilityPolicy{}
|
||||||
if effectiveIntent.Explicit {
|
if effectiveIntent.Explicit {
|
||||||
decision := m.evaluateIntentNoLock(resourceID, resourceType, MetricAlertIntentSignal(metricType), trackingKey, time.Now(), conditionActive, BackupIntentContext{})
|
decision := m.evaluateIntentNoLock(resourceID, resourceType, MetricAlertIntentSignal(metricType), trackingKey, time.Now(), conditionActive, BackupIntentContext{})
|
||||||
if decision.StateChanged {
|
if decision.StateChanged {
|
||||||
|
|
@ -292,6 +366,9 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
|
||||||
} else if conditionActive {
|
} else if conditionActive {
|
||||||
delaySeconds = m.getTimeThreshold(resourceID, resourceType, metricType)
|
delaySeconds = m.getTimeThreshold(resourceID, resourceType, metricType)
|
||||||
}
|
}
|
||||||
|
if !effectiveIntent.Explicit {
|
||||||
|
stability = metricStabilityFor(metricType, effectiveIntent.GraceSeconds)
|
||||||
|
}
|
||||||
|
|
||||||
events := m.core.ApplyMetric(reducer.MetricSignal{
|
events := m.core.ApplyMetric(reducer.MetricSignal{
|
||||||
ResourceID: resourceID,
|
ResourceID: resourceID,
|
||||||
|
|
@ -302,10 +379,12 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
|
||||||
RuntimeTickValid: true,
|
RuntimeTickValid: true,
|
||||||
ObservedAt: m.policyNow(),
|
ObservedAt: m.policyNow(),
|
||||||
}, reducer.MetricRule{
|
}, reducer.MetricRule{
|
||||||
Trigger: threshold.Trigger,
|
Trigger: threshold.Trigger,
|
||||||
Clear: threshold.Clear,
|
Clear: threshold.Clear,
|
||||||
DelaySeconds: delaySeconds,
|
DelaySeconds: delaySeconds,
|
||||||
Intent: intent,
|
CriticalBypassesDelay: stability.CriticalBypassesDelay,
|
||||||
|
RecoveryDelaySeconds: stability.RecoveryDelaySeconds,
|
||||||
|
Intent: intent,
|
||||||
})
|
})
|
||||||
primary := reducer.EventType("")
|
primary := reducer.EventType("")
|
||||||
if len(events) > 0 {
|
if len(events) > 0 {
|
||||||
|
|
|
||||||
|
|
@ -65,6 +65,15 @@ type MetricRule struct {
|
||||||
// an explicit Intent gate is supplied — the manager's explicit intent
|
// an explicit Intent gate is supplied — the manager's explicit intent
|
||||||
// policies replace the legacy time-threshold delay.
|
// policies replace the legacy time-threshold delay.
|
||||||
DelaySeconds int
|
DelaySeconds int
|
||||||
|
// CriticalBypassesDelay lets urgent evidence activate immediately while
|
||||||
|
// warning-level evidence still has to survive the stability window. It is
|
||||||
|
// intentionally ignored when an explicit Intent gate is present: explicit
|
||||||
|
// operator policy remains authoritative.
|
||||||
|
CriticalBypassesDelay bool
|
||||||
|
// RecoveryDelaySeconds is the continuous time at or below Clear required
|
||||||
|
// before a firing incident resolves. A value <= 0 preserves immediate
|
||||||
|
// hysteresis recovery.
|
||||||
|
RecoveryDelaySeconds int
|
||||||
// Intent is the resolved intent-policy context for this observation
|
// Intent is the resolved intent-policy context for this observation
|
||||||
// (metric.<name> signals); nil means no gate.
|
// (metric.<name> signals); nil means no gate.
|
||||||
Intent *DiscreteIntent
|
Intent *DiscreteIntent
|
||||||
|
|
@ -108,9 +117,15 @@ type Incident struct {
|
||||||
// toward DiscreteRule.RecoveryConfirmations; reset by any matching
|
// toward DiscreteRule.RecoveryConfirmations; reset by any matching
|
||||||
// observation. Unused by the metric family.
|
// observation. Unused by the metric family.
|
||||||
RecoveryCount int
|
RecoveryCount int
|
||||||
Acknowledged bool
|
// RecoverySince and its monotonic companions track a continuous healthy
|
||||||
AckUser string
|
// run for metric rules with a recovery stability window.
|
||||||
AckAt time.Time
|
RecoverySince time.Time
|
||||||
|
RecoveryElapsed time.Duration
|
||||||
|
RecoveryTicksSupplied bool
|
||||||
|
LastRecoveryRuntimeTick time.Duration
|
||||||
|
Acknowledged bool
|
||||||
|
AckUser string
|
||||||
|
AckAt time.Time
|
||||||
// Backup-run bookkeeping for the intent gate's backup-offline deferral
|
// Backup-run bookkeeping for the intent gate's backup-offline deferral
|
||||||
// sub-policy (confirmation family only).
|
// sub-policy (confirmation family only).
|
||||||
BackupActive bool
|
BackupActive bool
|
||||||
|
|
@ -390,6 +405,7 @@ func (s *State) ApplyMetric(signal MetricSignal, rule MetricRule) []Event {
|
||||||
// recomputes level on every tick, so severity can demote as well as
|
// recomputes level on every tick, so severity can demote as well as
|
||||||
// escalate while firing.
|
// escalate while firing.
|
||||||
if incident != nil && incident.State == StateFiring {
|
if incident != nil && incident.State == StateFiring {
|
||||||
|
resetMetricRecoveryRun(incident)
|
||||||
previous := incident.Severity
|
previous := incident.Severity
|
||||||
incident.Severity = severityFor(signal.Value, rule, signal.Metric)
|
incident.Severity = severityFor(signal.Value, rule, signal.Metric)
|
||||||
incident.LastValue = signal.Value
|
incident.LastValue = signal.Value
|
||||||
|
|
@ -444,7 +460,9 @@ func (s *State) ApplyMetric(signal MetricSignal, rule MetricRule) []Event {
|
||||||
return pendingResult()
|
return pendingResult()
|
||||||
}
|
}
|
||||||
} else if rule.DelaySeconds > 0 {
|
} else if rule.DelaySeconds > 0 {
|
||||||
if signal.ObservedAt.Sub(incident.PendingSince) < time.Duration(rule.DelaySeconds)*time.Second {
|
critical := severityFor(signal.Value, rule, signal.Metric) == SeverityCritical
|
||||||
|
if !(critical && rule.CriticalBypassesDelay) &&
|
||||||
|
signal.ObservedAt.Sub(incident.PendingSince) < time.Duration(rule.DelaySeconds)*time.Second {
|
||||||
return pendingResult()
|
return pendingResult()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -473,6 +491,31 @@ func (s *State) ApplyMetric(signal MetricSignal, rule MetricRule) []Event {
|
||||||
clear = rule.Trigger
|
clear = rule.Trigger
|
||||||
}
|
}
|
||||||
if signal.Value <= clear {
|
if signal.Value <= clear {
|
||||||
|
if rule.RecoveryDelaySeconds > 0 {
|
||||||
|
if incident.RecoverySince.IsZero() {
|
||||||
|
incident.RecoverySince = signal.ObservedAt
|
||||||
|
incident.RecoveryTicksSupplied = signal.RuntimeTickValid
|
||||||
|
incident.LastRecoveryRuntimeTick = signal.RuntimeTick
|
||||||
|
incident.RecoveryElapsed = 0
|
||||||
|
incident.LastObservedAt = signal.ObservedAt
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if signal.RuntimeTickValid {
|
||||||
|
if incident.RecoveryTicksSupplied && signal.RuntimeTick >= incident.LastRecoveryRuntimeTick {
|
||||||
|
incident.RecoveryElapsed += signal.RuntimeTick - incident.LastRecoveryRuntimeTick
|
||||||
|
}
|
||||||
|
incident.RecoveryTicksSupplied = true
|
||||||
|
incident.LastRecoveryRuntimeTick = signal.RuntimeTick
|
||||||
|
}
|
||||||
|
elapsed := signal.ObservedAt.Sub(incident.RecoverySince)
|
||||||
|
if incident.RecoveryTicksSupplied {
|
||||||
|
elapsed = incident.RecoveryElapsed
|
||||||
|
}
|
||||||
|
incident.LastObservedAt = signal.ObservedAt
|
||||||
|
if elapsed < time.Duration(rule.RecoveryDelaySeconds)*time.Second {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
severity := incident.Severity
|
severity := incident.Severity
|
||||||
delete(s.incidents, key)
|
delete(s.incidents, key)
|
||||||
s.markAckInactive(key, signal.ObservedAt)
|
s.markAckInactive(key, signal.ObservedAt)
|
||||||
|
|
@ -482,9 +525,20 @@ func (s *State) ApplyMetric(signal MetricSignal, rule MetricRule) []Event {
|
||||||
// Hysteresis hold: between clear and trigger the incident stays firing.
|
// Hysteresis hold: between clear and trigger the incident stays firing.
|
||||||
// The manager does not refresh value or last-seen in this band; the
|
// The manager does not refresh value or last-seen in this band; the
|
||||||
// reducer mirrors that so parity diffs stay exact.
|
// reducer mirrors that so parity diffs stay exact.
|
||||||
|
resetMetricRecoveryRun(incident)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func resetMetricRecoveryRun(incident *Incident) {
|
||||||
|
if incident == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
incident.RecoverySince = time.Time{}
|
||||||
|
incident.RecoveryElapsed = 0
|
||||||
|
incident.RecoveryTicksSupplied = false
|
||||||
|
incident.LastRecoveryRuntimeTick = 0
|
||||||
|
}
|
||||||
|
|
||||||
// SeedFiringIncident installs a firing incident directly, bypassing the
|
// SeedFiringIncident installs a firing incident directly, bypassing the
|
||||||
// transition functions. Shadow mode uses it to align the reducer with
|
// transition functions. Shadow mode uses it to align the reducer with
|
||||||
// pre-existing manager state (persisted-alert restore, divergence resync);
|
// pre-existing manager state (persisted-alert restore, divergence resync);
|
||||||
|
|
|
||||||
|
|
@ -132,6 +132,52 @@ func TestDelayFiresWithPendingStartAndDipResets(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCriticalCanBypassWarningStabilityDelay(t *testing.T) {
|
||||||
|
state := NewState()
|
||||||
|
rule := MetricRule{Trigger: 80, Clear: 75, DelaySeconds: 300, CriticalBypassesDelay: true}
|
||||||
|
|
||||||
|
events := state.ApplyMetric(signalAt(95, 0), rule)
|
||||||
|
if len(events) != 1 || events[0].Type != EventFired || events[0].Severity != SeverityCritical {
|
||||||
|
t.Fatalf("events = %+v, want immediate critical fire", events)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCriticalDoesNotBypassExplicitIntent(t *testing.T) {
|
||||||
|
state := NewState()
|
||||||
|
rule := MetricRule{
|
||||||
|
Trigger: 80, Clear: 75, CriticalBypassesDelay: true,
|
||||||
|
Intent: &DiscreteIntent{Explicit: true, GraceSeconds: 300},
|
||||||
|
}
|
||||||
|
|
||||||
|
events := state.ApplyMetric(signalAt(95, 0), rule)
|
||||||
|
if len(events) != 1 || events[0].Type != EventPending {
|
||||||
|
t.Fatalf("events = %+v, want explicit intent to keep critical pending", events)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMetricRecoveryRequiresContinuousHealthyWindow(t *testing.T) {
|
||||||
|
state := NewState()
|
||||||
|
rule := MetricRule{Trigger: 80, Clear: 75, RecoveryDelaySeconds: 120}
|
||||||
|
state.ApplyMetric(signalAt(90, 0), rule)
|
||||||
|
|
||||||
|
if events := state.ApplyMetric(signalAt(70, time.Minute), rule); len(events) != 0 {
|
||||||
|
t.Fatalf("first healthy observation emitted %+v", events)
|
||||||
|
}
|
||||||
|
if events := state.ApplyMetric(signalAt(78, 90*time.Second), rule); len(events) != 0 {
|
||||||
|
t.Fatalf("hysteresis-band interruption emitted %+v", events)
|
||||||
|
}
|
||||||
|
if events := state.ApplyMetric(signalAt(70, 2*time.Minute), rule); len(events) != 0 {
|
||||||
|
t.Fatalf("restarted healthy run emitted %+v", events)
|
||||||
|
}
|
||||||
|
if events := state.ApplyMetric(signalAt(70, 3*time.Minute+59*time.Second), rule); len(events) != 0 {
|
||||||
|
t.Fatalf("recovery fired before continuous window elapsed: %+v", events)
|
||||||
|
}
|
||||||
|
events := state.ApplyMetric(signalAt(70, 4*time.Minute), rule)
|
||||||
|
if len(events) != 1 || events[0].Type != EventResolved {
|
||||||
|
t.Fatalf("events = %+v, want recovery after continuous healthy window", events)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestClearFallsBackToTriggerWhenUnset(t *testing.T) {
|
func TestClearFallsBackToTriggerWhenUnset(t *testing.T) {
|
||||||
state := NewState()
|
state := NewState()
|
||||||
rule := MetricRule{Trigger: 80, Clear: 0}
|
rule := MetricRule{Trigger: 80, Clear: 0}
|
||||||
|
|
|
||||||
|
|
@ -938,6 +938,32 @@ func TestGuestFilesystemThresholdOverrideOwnsMountEvaluation(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestGuestFilesystemEvidenceSuppressesDuplicateAggregateAlert(t *testing.T) {
|
||||||
|
m := newTestManager(t)
|
||||||
|
guestID := BuildGuestKey("pve1", "node1", 109)
|
||||||
|
m.mu.Lock()
|
||||||
|
m.config.TimeThresholds = map[string]int{}
|
||||||
|
m.config.GuestDefaults = ThresholdConfig{Disk: &HysteresisThreshold{Trigger: 80, Clear: 75}}
|
||||||
|
m.mu.Unlock()
|
||||||
|
|
||||||
|
m.CheckGuest(models.VM{
|
||||||
|
ID: guestID, VMID: 109, Name: "pulse-dev", Node: "node1", Instance: "pve1", Status: "running",
|
||||||
|
Disk: models.Disk{Usage: 96.5},
|
||||||
|
Disks: []models.Disk{{Mountpoint: "/", Device: "/dev/vda1", Usage: 96.9, Total: 100, Used: 97, Free: 3}},
|
||||||
|
}, "pve1")
|
||||||
|
|
||||||
|
aggregateID := canonicalMetricStateID(guestID, "disk")
|
||||||
|
filesystemID := canonicalMetricStateID(guestID+"-disk-dev-vda1", "disk")
|
||||||
|
m.mu.RLock()
|
||||||
|
defer m.mu.RUnlock()
|
||||||
|
if _, exists := testLookupActiveAlert(t, m, aggregateID); exists {
|
||||||
|
t.Fatalf("aggregate alert %q duplicated filesystem evidence", aggregateID)
|
||||||
|
}
|
||||||
|
if _, exists := testLookupActiveAlert(t, m, filesystemID); !exists {
|
||||||
|
t.Fatalf("actionable filesystem alert %q was not created", filesystemID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Regression: checkMetric stores canonical-identity alerts under the
|
// Regression: checkMetric stores canonical-identity alerts under the
|
||||||
// canonical state key, so hysteresis resolution must not remove only the
|
// canonical state key, so hysteresis resolution must not remove only the
|
||||||
// unregistered legacy "<resourceID>-<metric>" ID.
|
// unregistered legacy "<resourceID>-<metric>" ID.
|
||||||
|
|
|
||||||
|
|
@ -85,6 +85,59 @@ func TestGetTimeThresholdMetricOverrides(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestGetTimeThresholdUsesTrustStableNoisyGaugeDefaults(t *testing.T) {
|
||||||
|
manager := NewManagerWithDataDir(t.TempDir(), WithoutPersistedAlertRestore())
|
||||||
|
|
||||||
|
t.Cleanup(manager.Stop)
|
||||||
|
|
||||||
|
if got := manager.getTimeThreshold("vm-resource", "VM", "memory"); got != defaultNoisyGaugeStabilitySeconds {
|
||||||
|
t.Fatalf("default memory stability = %d, want %d", got, defaultNoisyGaugeStabilitySeconds)
|
||||||
|
}
|
||||||
|
if got := manager.getTimeThreshold("node-1", "Node", "temperature"); got != defaultNoisyGaugeStabilitySeconds {
|
||||||
|
t.Fatalf("default temperature stability = %d, want %d", got, defaultNoisyGaugeStabilitySeconds)
|
||||||
|
}
|
||||||
|
if got := manager.getTimeThreshold("vm-resource", "VM", "cpu"); got != defaultLegacyMetricDelaySeconds {
|
||||||
|
t.Fatalf("default CPU delay = %d, want %d", got, defaultLegacyMetricDelaySeconds)
|
||||||
|
}
|
||||||
|
|
||||||
|
manager.mu.Lock()
|
||||||
|
manager.config.MetricTimeThresholds = map[string]map[string]int{
|
||||||
|
"all": {"memory": 0},
|
||||||
|
}
|
||||||
|
manager.mu.Unlock()
|
||||||
|
|
||||||
|
if got := manager.getTimeThreshold("vm-resource", "VM", "memory"); got != 0 {
|
||||||
|
t.Fatalf("explicit global memory override = %d, want 0", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCheckMetricNoisyWarningWaitsButCriticalFiresImmediately(t *testing.T) {
|
||||||
|
manager := NewManagerWithDataDir(t.TempDir(), WithoutPersistedAlertRestore())
|
||||||
|
t.Cleanup(manager.Stop)
|
||||||
|
|
||||||
|
threshold := &HysteresisThreshold{Trigger: 85, Clear: 80}
|
||||||
|
canonicalState := buildCanonicalStateID("vm-resource", "metric-threshold:memory")
|
||||||
|
|
||||||
|
manager.checkMetric("vm-resource", "database", "node-1", "pve-1", "VM", "memory", 90, threshold, nil)
|
||||||
|
manager.mu.RLock()
|
||||||
|
_, warningActive := manager.getActiveAlertNoLock(canonicalState)
|
||||||
|
manager.mu.RUnlock()
|
||||||
|
if warningActive {
|
||||||
|
t.Fatal("brief warning-level memory evidence opened an alert")
|
||||||
|
}
|
||||||
|
|
||||||
|
manager.checkMetric("vm-resource", "database", "node-1", "pve-1", "VM", "memory", 96, threshold, nil)
|
||||||
|
manager.mu.RLock()
|
||||||
|
alert, criticalActive := manager.getActiveAlertNoLock(canonicalState)
|
||||||
|
manager.mu.RUnlock()
|
||||||
|
if !criticalActive {
|
||||||
|
t.Fatal("critical memory evidence did not bypass the factory stability delay")
|
||||||
|
}
|
||||||
|
if alert.Level != AlertLevelCritical {
|
||||||
|
t.Fatalf("critical memory alert level = %q, want %q", alert.Level, AlertLevelCritical)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestCheckMetricUsesPendingStartTime(t *testing.T) {
|
func TestCheckMetricUsesPendingStartTime(t *testing.T) {
|
||||||
manager := NewManager()
|
manager := NewManager()
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1559,3 +1559,27 @@ func TestUnifiedStorageForecastKeepsCanonicalMetricSpecAcrossPlatforms(t *testin
|
||||||
t.Fatalf("CanonicalSpecID = %q, want canonical usage spec", alert.CanonicalSpecID)
|
t.Fatalf("CanonicalSpecID = %q, want canonical usage spec", alert.CanonicalSpecID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestUnifiedMetricNoisyWarningWaitsButCriticalFiresImmediately(t *testing.T) {
|
||||||
|
m := newTestManager(t)
|
||||||
|
input := &UnifiedResourceInput{
|
||||||
|
ID: "vm-noisy-memory",
|
||||||
|
Type: "vm",
|
||||||
|
Name: "database",
|
||||||
|
Memory: &UnifiedResourceMetric{Percent: 90},
|
||||||
|
}
|
||||||
|
alertID := canonicalMetricStateID(input.ID, "memory")
|
||||||
|
|
||||||
|
m.CheckUnifiedResource(input)
|
||||||
|
assertAlertMissing(t, m, alertID)
|
||||||
|
|
||||||
|
input.Memory.Percent = 96
|
||||||
|
m.CheckUnifiedResource(input)
|
||||||
|
alert := testRequireActiveAlert(t, m, alertID)
|
||||||
|
if alert.Level != AlertLevelCritical {
|
||||||
|
t.Fatalf("critical memory alert level = %q, want %q", alert.Level, AlertLevelCritical)
|
||||||
|
}
|
||||||
|
if got := alert.Metadata["stabilityWindowSeconds"]; got != defaultNoisyGaugeStabilitySeconds {
|
||||||
|
t.Fatalf("stabilityWindowSeconds = %v, want %d", got, defaultNoisyGaugeStabilitySeconds)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -129,6 +129,13 @@ var containerPathPrefixes = []string{
|
||||||
"/mnt/.ix-apps/docker/", // TrueNAS SCALE Docker overlay mounts
|
"/mnt/.ix-apps/docker/", // TrueNAS SCALE Docker overlay mounts
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// transientMountPatterns identify OS-managed mounts whose lifetime follows an
|
||||||
|
// application launch rather than operator-managed storage. They must not
|
||||||
|
// participate in capacity inventory or alerting.
|
||||||
|
var transientMountPatterns = []string{
|
||||||
|
"/AppTranslocation/", // macOS Gatekeeper translocated application image
|
||||||
|
}
|
||||||
|
|
||||||
// ShouldSkipFilesystem determines if a filesystem should be excluded from disk
|
// ShouldSkipFilesystem determines if a filesystem should be excluded from disk
|
||||||
// usage aggregation. It checks for read-only filesystems, virtual/pseudo filesystems,
|
// usage aggregation. It checks for read-only filesystems, virtual/pseudo filesystems,
|
||||||
// network mounts, and special system mountpoints. Returns skip=true if the filesystem
|
// network mounts, and special system mountpoints. Returns skip=true if the filesystem
|
||||||
|
|
@ -168,6 +175,13 @@ func ShouldSkipFilesystem(fsType, mountpoint string, totalBytes, usedBytes uint6
|
||||||
reasons = append(reasons, "special-mountpoint")
|
reasons = append(reasons, "special-mountpoint")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for _, pattern := range transientMountPatterns {
|
||||||
|
if strings.Contains(mountpoint, pattern) {
|
||||||
|
reasons = append(reasons, "transient-mountpoint")
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Windows System Reserved partition
|
// Windows System Reserved partition
|
||||||
if mountpoint == "System Reserved" || strings.Contains(mountpoint, "System Reserved") {
|
if mountpoint == "System Reserved" || strings.Contains(mountpoint, "System Reserved") {
|
||||||
reasons = append(reasons, "special-mountpoint")
|
reasons = append(reasons, "special-mountpoint")
|
||||||
|
|
|
||||||
|
|
@ -204,6 +204,7 @@ func TestShouldSkipFilesystem(t *testing.T) {
|
||||||
{"macos data companion volume", "apfs", "/System/Volumes/Data", 245107195904, 213573853184, true},
|
{"macos data companion volume", "apfs", "/System/Volumes/Data", 245107195904, 213573853184, true},
|
||||||
{"macos preboot companion volume", "apfs", "/System/Volumes/Preboot", 245107195904, 213573853184, true},
|
{"macos preboot companion volume", "apfs", "/System/Volumes/Preboot", 245107195904, 213573853184, true},
|
||||||
{"macos simulator runtime volume", "apfs", "/Library/Developer/CoreSimulator/Volumes/iOS_23E254a", 18058575872, 17593331712, true},
|
{"macos simulator runtime volume", "apfs", "/Library/Developer/CoreSimulator/Volumes/iOS_23E254a", 18058575872, 17593331712, true},
|
||||||
|
{"macos app translocation volume", "apfs", "/private/var/folders/xy/random/T/AppTranslocation/2A9C/d/Pulse.app", 245107195904, 221809991680, true},
|
||||||
|
|
||||||
// Regular filesystems that should NOT be skipped
|
// Regular filesystems that should NOT be skipped
|
||||||
{"ext4 root", "ext4", "/", 100 * 1024 * 1024 * 1024, 50 * 1024 * 1024 * 1024, false},
|
{"ext4 root", "ext4", "/", 100 * 1024 * 1024 * 1024, 50 * 1024 * 1024 * 1024, false},
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue