mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-07-09 16:00:59 +00:00
Add alert delivery diagnosis endpoint
This commit is contained in:
parent
a801f456d6
commit
597c369a63
10 changed files with 664 additions and 0 deletions
|
|
@ -575,6 +575,7 @@ Alert configuration and history (requires `monitoring:read`/`monitoring:write`).
|
|||
- `PUT /api/alerts/config`
|
||||
- `POST /api/alerts/activate`
|
||||
- `GET /api/alerts/active`
|
||||
- `GET /api/alerts/delivery-diagnosis?alertIdentifier=<alert-id>`
|
||||
- `GET /api/alerts/history`
|
||||
- `DELETE /api/alerts/history`
|
||||
- `GET /api/alerts/incidents`
|
||||
|
|
|
|||
|
|
@ -1513,6 +1513,12 @@ list representation, and masked-echo preservation on update are
|
|||
notifications/API-contract owned and grant no agent install, enrollment,
|
||||
setup-token, or fleet command authority.
|
||||
|
||||
Alert delivery diagnosis on shared `internal/api/alerts.go` is likewise
|
||||
adjacent only: `/api/alerts/delivery-diagnosis` is alerts/API-contract owned
|
||||
read-only notification-policy evidence and must not be interpreted as agent
|
||||
enrollment, agent liveness, install progress, setup-token authority, or fleet
|
||||
repair eligibility.
|
||||
|
||||
The router projection-builder (`internal/api/router.go`) that wires
|
||||
the operator-state provider into the findings runtime now also
|
||||
populates `NeverAutoRemediate` on the projection. The investigation
|
||||
|
|
|
|||
|
|
@ -382,6 +382,11 @@ suppression, monitor-only notification suppression, cooldown decisions, and
|
|||
per-alert rate limiting; future notification-gating changes should extend that
|
||||
policy owner rather than burying new checks inside metric or resource-specific
|
||||
evaluators.
|
||||
The same policy owner also exposes the read-only alert delivery diagnosis
|
||||
projection used by `/api/alerts/delivery-diagnosis`; that projection may explain
|
||||
current gating state, quiet-hours replay timing, cooldown timing, rate-limit
|
||||
counts, and flapping suppression, but it must not dispatch callbacks or mutate
|
||||
flapping/rate-limit tracking maps.
|
||||
The same dispatch policy owns firing-notification evidence on active alerts:
|
||||
any alert that passes notification suppression and enters the fired callback
|
||||
fan-out must carry `LastNotified` before the callback clone is emitted. Resolved
|
||||
|
|
|
|||
|
|
@ -2965,6 +2965,15 @@ a new API state machine, queue contract, or verification-accounting field.
|
|||
|
||||
## Current State
|
||||
|
||||
Alert delivery diagnosis is a read-only monitoring API contract.
|
||||
`GET /api/alerts/delivery-diagnosis?alertIdentifier=<id>` returns the alert
|
||||
manager's current delivery-policy projection for one active alert, including
|
||||
`status` (`would_send`, `deferred`, or `suppressed`), `reason`, canonical
|
||||
`trackingKey`, activation/notification state, cooldown timing, quiet-hours
|
||||
replay timing, recent rate-limit counts, and flapping suppression evidence. The
|
||||
route requires `monitoring:read`, returns `404` for unknown active alerts, and
|
||||
must not send notifications or mutate delivery tracking state.
|
||||
|
||||
Manifest-backed Patrol finding lifecycle schemas are the API source of truth
|
||||
for Assistant provider-tool optionality as well as MCP/API discovery. Legacy
|
||||
Assistant runtime paths may project those schemas into provider-tool JSON, but
|
||||
|
|
|
|||
|
|
@ -1536,6 +1536,12 @@ likewise adjacent only: the webhook `signingSecret` payload field and its
|
|||
masking semantics are notifications/API-contract owned and create no storage,
|
||||
recovery-point, or backup-surface semantics.
|
||||
|
||||
Alert delivery diagnosis on shared `internal/api/alerts.go` is likewise
|
||||
adjacent only: `/api/alerts/delivery-diagnosis` exposes alerts/API-contract
|
||||
read-only notification-policy evidence and creates no storage health,
|
||||
recovery-point, backup verification, restore authorization, or provider
|
||||
coverage semantics.
|
||||
|
||||
Kubernetes pod metadata decoded by `frontend-modern/src/hooks/useUnifiedResources.ts`
|
||||
is shared inventory context for storage/recovery handoffs only; Pod phase,
|
||||
container readiness, owner, image, and restart fields do not become protection
|
||||
|
|
|
|||
|
|
@ -19022,3 +19022,236 @@ func TestDefaultAlertConfigUsesIndependentBackupAlertOrphanedPointer(t *testing.
|
|||
t.Fatal("mutating one default config changed another default config")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnoseAlertDeliveryReady(t *testing.T) {
|
||||
manager, now := newDeliveryDiagnosisManager(t)
|
||||
alert := addDeliveryDiagnosisAlert(manager, &Alert{
|
||||
ID: "alert-ready",
|
||||
Type: "cpu",
|
||||
Level: AlertLevelWarning,
|
||||
ResourceID: "node-1",
|
||||
ResourceName: "node-1",
|
||||
StartTime: now.Add(-10 * time.Minute),
|
||||
})
|
||||
|
||||
diagnosis, ok := manager.DiagnoseAlertDelivery(alert.ID)
|
||||
if !ok {
|
||||
t.Fatal("expected active alert diagnosis")
|
||||
}
|
||||
if diagnosis.Status != AlertDeliveryStatusWouldSend {
|
||||
t.Fatalf("Status = %q, want %q", diagnosis.Status, AlertDeliveryStatusWouldSend)
|
||||
}
|
||||
if diagnosis.Reason != AlertDeliveryReasonReady {
|
||||
t.Fatalf("Reason = %q, want %q", diagnosis.Reason, AlertDeliveryReasonReady)
|
||||
}
|
||||
if diagnosis.TrackingKey == "" {
|
||||
t.Fatal("expected tracking key")
|
||||
}
|
||||
if diagnosis.RecentAlertsInHour != 0 {
|
||||
t.Fatalf("RecentAlertsInHour = %d, want 0", diagnosis.RecentAlertsInHour)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnoseAlertDeliverySuppressesAcknowledged(t *testing.T) {
|
||||
manager, now := newDeliveryDiagnosisManager(t)
|
||||
alert := addDeliveryDiagnosisAlert(manager, &Alert{
|
||||
ID: "alert-acked",
|
||||
Type: "memory",
|
||||
Level: AlertLevelCritical,
|
||||
ResourceID: "node-1",
|
||||
ResourceName: "node-1",
|
||||
StartTime: now.Add(-10 * time.Minute),
|
||||
Acknowledged: true,
|
||||
})
|
||||
|
||||
diagnosis, ok := manager.DiagnoseAlertDelivery(alert.ID)
|
||||
if !ok {
|
||||
t.Fatal("expected active alert diagnosis")
|
||||
}
|
||||
if diagnosis.Status != AlertDeliveryStatusSuppressed {
|
||||
t.Fatalf("Status = %q, want %q", diagnosis.Status, AlertDeliveryStatusSuppressed)
|
||||
}
|
||||
if diagnosis.Reason != AlertDeliveryReasonAcknowledged {
|
||||
t.Fatalf("Reason = %q, want %q", diagnosis.Reason, AlertDeliveryReasonAcknowledged)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnoseAlertDeliveryDefersQuietHours(t *testing.T) {
|
||||
manager, now := newDeliveryDiagnosisManager(t)
|
||||
cfg := manager.GetConfig()
|
||||
cfg.Schedule.QuietHours.Enabled = true
|
||||
cfg.Schedule.QuietHours.Timezone = "UTC"
|
||||
cfg.Schedule.QuietHours.Start = "00:00"
|
||||
cfg.Schedule.QuietHours.End = "23:59"
|
||||
cfg.Schedule.QuietHours.Days = map[string]bool{"monday": true}
|
||||
cfg.Schedule.QuietHours.Suppress.Performance = true
|
||||
manager.UpdateConfig(cfg)
|
||||
manager.now = func() time.Time { return now }
|
||||
|
||||
alert := addDeliveryDiagnosisAlert(manager, &Alert{
|
||||
ID: "alert-quiet",
|
||||
Type: "cpu",
|
||||
Level: AlertLevelCritical,
|
||||
ResourceID: "node-1",
|
||||
ResourceName: "node-1",
|
||||
StartTime: now.Add(-10 * time.Minute),
|
||||
})
|
||||
|
||||
diagnosis, ok := manager.DiagnoseAlertDelivery(alert.ID)
|
||||
if !ok {
|
||||
t.Fatal("expected active alert diagnosis")
|
||||
}
|
||||
if diagnosis.Status != AlertDeliveryStatusDeferred {
|
||||
t.Fatalf("Status = %q, want %q", diagnosis.Status, AlertDeliveryStatusDeferred)
|
||||
}
|
||||
wantReason := AlertDeliveryReasonQuietHours + ":performance"
|
||||
if diagnosis.Reason != wantReason {
|
||||
t.Fatalf("Reason = %q, want %q", diagnosis.Reason, wantReason)
|
||||
}
|
||||
if diagnosis.QuietHoursReplayAt == nil {
|
||||
t.Fatal("expected quiet-hours replay time")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnoseAlertDeliverySuppressesCooldown(t *testing.T) {
|
||||
manager, now := newDeliveryDiagnosisManager(t)
|
||||
lastNotified := now.Add(-2 * time.Minute)
|
||||
alert := addDeliveryDiagnosisAlert(manager, &Alert{
|
||||
ID: "alert-cooldown",
|
||||
Type: "cpu",
|
||||
Level: AlertLevelWarning,
|
||||
ResourceID: "node-1",
|
||||
ResourceName: "node-1",
|
||||
StartTime: now.Add(-10 * time.Minute),
|
||||
LastNotified: &lastNotified,
|
||||
})
|
||||
|
||||
diagnosis, ok := manager.DiagnoseAlertDelivery(alert.ID)
|
||||
if !ok {
|
||||
t.Fatal("expected active alert diagnosis")
|
||||
}
|
||||
if diagnosis.Status != AlertDeliveryStatusSuppressed {
|
||||
t.Fatalf("Status = %q, want %q", diagnosis.Status, AlertDeliveryStatusSuppressed)
|
||||
}
|
||||
if diagnosis.Reason != AlertDeliveryReasonCooldown {
|
||||
t.Fatalf("Reason = %q, want %q", diagnosis.Reason, AlertDeliveryReasonCooldown)
|
||||
}
|
||||
if diagnosis.NextEligibleAt == nil || !diagnosis.NextEligibleAt.Equal(lastNotified.Add(5*time.Minute)) {
|
||||
t.Fatalf("NextEligibleAt = %v, want %v", diagnosis.NextEligibleAt, lastNotified.Add(5*time.Minute))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnoseAlertDeliverySuppressesRateLimitWithoutMutating(t *testing.T) {
|
||||
manager, now := newDeliveryDiagnosisManager(t)
|
||||
cfg := manager.GetConfig()
|
||||
cfg.Schedule.MaxAlertsHour = 1
|
||||
manager.UpdateConfig(cfg)
|
||||
manager.now = func() time.Time { return now }
|
||||
|
||||
alert := addDeliveryDiagnosisAlert(manager, &Alert{
|
||||
ID: "alert-rate-limit",
|
||||
Type: "cpu",
|
||||
Level: AlertLevelWarning,
|
||||
ResourceID: "node-1",
|
||||
ResourceName: "node-1",
|
||||
StartTime: now.Add(-10 * time.Minute),
|
||||
})
|
||||
trackingKey := canonicalTrackingKeyForAlert(alert)
|
||||
manager.mu.Lock()
|
||||
manager.alertRateLimit[trackingKey] = []time.Time{now.Add(-30 * time.Minute)}
|
||||
manager.mu.Unlock()
|
||||
|
||||
diagnosis, ok := manager.DiagnoseAlertDelivery(alert.ID)
|
||||
if !ok {
|
||||
t.Fatal("expected active alert diagnosis")
|
||||
}
|
||||
if diagnosis.Status != AlertDeliveryStatusSuppressed {
|
||||
t.Fatalf("Status = %q, want %q", diagnosis.Status, AlertDeliveryStatusSuppressed)
|
||||
}
|
||||
if diagnosis.Reason != AlertDeliveryReasonRateLimited {
|
||||
t.Fatalf("Reason = %q, want %q", diagnosis.Reason, AlertDeliveryReasonRateLimited)
|
||||
}
|
||||
if diagnosis.RecentAlertsInHour != 1 {
|
||||
t.Fatalf("RecentAlertsInHour = %d, want 1", diagnosis.RecentAlertsInHour)
|
||||
}
|
||||
|
||||
manager.mu.Lock()
|
||||
defer manager.mu.Unlock()
|
||||
if got := len(manager.alertRateLimit[trackingKey]); got != 1 {
|
||||
t.Fatalf("diagnosis mutated alertRateLimit length to %d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnoseAlertDeliverySuppressesFlappingWithoutMutatingRateLimit(t *testing.T) {
|
||||
manager, now := newDeliveryDiagnosisManager(t)
|
||||
alert := addDeliveryDiagnosisAlert(manager, &Alert{
|
||||
ID: "alert-flapping",
|
||||
Type: "cpu",
|
||||
Level: AlertLevelWarning,
|
||||
ResourceID: "node-1",
|
||||
ResourceName: "node-1",
|
||||
StartTime: now.Add(-10 * time.Minute),
|
||||
})
|
||||
trackingKey := canonicalTrackingKeyForAlert(alert)
|
||||
suppressedUntil := now.Add(15 * time.Minute)
|
||||
manager.mu.Lock()
|
||||
manager.flappingHistory[trackingKey] = []time.Time{now.Add(-2 * time.Minute), now.Add(-time.Minute)}
|
||||
manager.flappingActive[trackingKey] = true
|
||||
manager.suppressedUntil[trackingKey] = suppressedUntil
|
||||
manager.alertRateLimit[trackingKey] = []time.Time{now.Add(-30 * time.Minute)}
|
||||
manager.mu.Unlock()
|
||||
|
||||
diagnosis, ok := manager.DiagnoseAlertDelivery(alert.ID)
|
||||
if !ok {
|
||||
t.Fatal("expected active alert diagnosis")
|
||||
}
|
||||
if diagnosis.Status != AlertDeliveryStatusSuppressed {
|
||||
t.Fatalf("Status = %q, want %q", diagnosis.Status, AlertDeliveryStatusSuppressed)
|
||||
}
|
||||
if diagnosis.Reason != AlertDeliveryReasonFlapping {
|
||||
t.Fatalf("Reason = %q, want %q", diagnosis.Reason, AlertDeliveryReasonFlapping)
|
||||
}
|
||||
if diagnosis.SuppressedUntil == nil || !diagnosis.SuppressedUntil.Equal(suppressedUntil) {
|
||||
t.Fatalf("SuppressedUntil = %v, want %v", diagnosis.SuppressedUntil, suppressedUntil)
|
||||
}
|
||||
if diagnosis.RecentAlertsInHour != 1 {
|
||||
t.Fatalf("RecentAlertsInHour = %d, want 1", diagnosis.RecentAlertsInHour)
|
||||
}
|
||||
|
||||
manager.mu.Lock()
|
||||
defer manager.mu.Unlock()
|
||||
if got := len(manager.alertRateLimit[trackingKey]); got != 1 {
|
||||
t.Fatalf("diagnosis mutated alertRateLimit length to %d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func newDeliveryDiagnosisManager(t *testing.T) (*Manager, time.Time) {
|
||||
t.Helper()
|
||||
|
||||
manager := NewManager()
|
||||
t.Cleanup(manager.Stop)
|
||||
|
||||
now := time.Date(2026, 6, 29, 12, 0, 0, 0, time.UTC)
|
||||
manager.now = func() time.Time { return now }
|
||||
|
||||
cfg := manager.GetConfig()
|
||||
cfg.Enabled = true
|
||||
cfg.ActivationState = ActivationActive
|
||||
cfg.Schedule.Cooldown = 5
|
||||
cfg.Schedule.MaxAlertsHour = 10
|
||||
cfg.FlappingEnabled = true
|
||||
cfg.FlappingThreshold = 3
|
||||
cfg.FlappingWindowSeconds = 300
|
||||
cfg.FlappingCooldownMinutes = 15
|
||||
manager.UpdateConfig(cfg)
|
||||
manager.now = func() time.Time { return now }
|
||||
|
||||
return manager, now
|
||||
}
|
||||
|
||||
func addDeliveryDiagnosisAlert(manager *Manager, alert *Alert) *Alert {
|
||||
manager.mu.Lock()
|
||||
defer manager.mu.Unlock()
|
||||
manager.activeAlerts[alert.ID] = alert
|
||||
return alert
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,6 +20,54 @@ const (
|
|||
MetadataQuietHoursReplayAt = "quietHoursReplayAt"
|
||||
)
|
||||
|
||||
const (
|
||||
AlertDeliveryStatusWouldSend = "would_send"
|
||||
AlertDeliveryStatusDeferred = "deferred"
|
||||
AlertDeliveryStatusSuppressed = "suppressed"
|
||||
|
||||
AlertDeliveryReasonReady = "ready"
|
||||
AlertDeliveryReasonAcknowledged = "acknowledged"
|
||||
AlertDeliveryReasonNotificationsDisabled = "notifications_disabled"
|
||||
AlertDeliveryReasonNotificationsInactive = "notifications_inactive"
|
||||
AlertDeliveryReasonSuppressionWindow = "suppression_window"
|
||||
AlertDeliveryReasonFlapping = "flapping"
|
||||
AlertDeliveryReasonRateLimited = "rate_limited"
|
||||
AlertDeliveryReasonCooldown = "cooldown"
|
||||
AlertDeliveryReasonQuietHours = "quiet_hours"
|
||||
AlertDeliveryReasonMonitorOnly = "monitor_only"
|
||||
)
|
||||
|
||||
// AlertDeliveryDiagnosis is a read-only projection of the alert manager's
|
||||
// current delivery policy for one active alert. It is intentionally
|
||||
// non-mutating: it does not dispatch callbacks or advance flapping/rate-limit
|
||||
// counters.
|
||||
type AlertDeliveryDiagnosis struct {
|
||||
AlertIdentifier string `json:"alertIdentifier"`
|
||||
AlertID string `json:"alertId"`
|
||||
TrackingKey string `json:"trackingKey"`
|
||||
Status string `json:"status"`
|
||||
Reason string `json:"reason"`
|
||||
Message string `json:"message"`
|
||||
AlertType string `json:"alertType"`
|
||||
Level AlertLevel `json:"level"`
|
||||
ResourceID string `json:"resourceId,omitempty"`
|
||||
ResourceName string `json:"resourceName,omitempty"`
|
||||
Node string `json:"node,omitempty"`
|
||||
NotificationsEnabled bool `json:"notificationsEnabled"`
|
||||
ActivationState string `json:"activationState"`
|
||||
CooldownMinutes int `json:"cooldownMinutes"`
|
||||
MaxAlertsHour int `json:"maxAlertsHour"`
|
||||
RecentAlertsInHour int `json:"recentAlertsInHour"`
|
||||
FlappingActive bool `json:"flappingActive"`
|
||||
FlappingHistoryInWindow int `json:"flappingHistoryInWindow"`
|
||||
FlappingThreshold int `json:"flappingThreshold"`
|
||||
FlappingWindowSeconds int `json:"flappingWindowSeconds"`
|
||||
LastNotified *time.Time `json:"lastNotified,omitempty"`
|
||||
NextEligibleAt *time.Time `json:"nextEligibleAt,omitempty"`
|
||||
QuietHoursReplayAt *time.Time `json:"quietHoursReplayAt,omitempty"`
|
||||
SuppressedUntil *time.Time `json:"suppressedUntil,omitempty"`
|
||||
}
|
||||
|
||||
// checkFlappingLocked detects alert flapping and returns whether the alert
|
||||
// should be suppressed and whether this call is the first transition into
|
||||
// the flapping state for the current cooldown window. justTransitioned is
|
||||
|
|
@ -568,6 +616,170 @@ func (m *Manager) shouldNotifyAfterCooldown(alert *Alert) bool {
|
|||
return timeSinceLastNotification >= cooldownDuration
|
||||
}
|
||||
|
||||
// DiagnoseAlertDelivery reports the current notification policy outcome for
|
||||
// one active alert without sending a notification or updating any delivery
|
||||
// tracking maps.
|
||||
func (m *Manager) DiagnoseAlertDelivery(alertIdentifier string) (AlertDeliveryDiagnosis, bool) {
|
||||
diagnosis := AlertDeliveryDiagnosis{
|
||||
AlertIdentifier: strings.TrimSpace(alertIdentifier),
|
||||
Status: AlertDeliveryStatusSuppressed,
|
||||
Reason: "not_found",
|
||||
}
|
||||
if m == nil || diagnosis.AlertIdentifier == "" {
|
||||
return diagnosis, false
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
alert, exists := m.getActiveAlertNoLock(diagnosis.AlertIdentifier)
|
||||
if !exists || alert == nil {
|
||||
return diagnosis, false
|
||||
}
|
||||
|
||||
now := m.policyNow()
|
||||
trackingKey := canonicalTrackingKeyForAlert(alert)
|
||||
diagnosis.AlertID = alert.ID
|
||||
diagnosis.TrackingKey = trackingKey
|
||||
diagnosis.AlertType = alert.Type
|
||||
diagnosis.Level = alert.Level
|
||||
diagnosis.ResourceID = alert.ResourceID
|
||||
diagnosis.ResourceName = alert.ResourceName
|
||||
diagnosis.Node = alert.Node
|
||||
diagnosis.NotificationsEnabled = m.config.Enabled && m.config.ActivationState == ActivationActive
|
||||
diagnosis.ActivationState = string(m.config.ActivationState)
|
||||
diagnosis.CooldownMinutes = m.config.Schedule.Cooldown
|
||||
diagnosis.MaxAlertsHour = m.config.Schedule.MaxAlertsHour
|
||||
diagnosis.FlappingThreshold = m.config.FlappingThreshold
|
||||
diagnosis.FlappingWindowSeconds = m.config.FlappingWindowSeconds
|
||||
if alert.LastNotified != nil {
|
||||
lastNotified := *alert.LastNotified
|
||||
diagnosis.LastNotified = &lastNotified
|
||||
}
|
||||
|
||||
diagnosis.FlappingActive, diagnosis.FlappingHistoryInWindow, diagnosis.SuppressedUntil = m.flappingSnapshotLocked(trackingKey, now)
|
||||
diagnosis.RecentAlertsInHour = m.rateLimitCountLocked(trackingKey, now)
|
||||
|
||||
switch {
|
||||
case alert.Acknowledged:
|
||||
diagnosis.setSuppressed(AlertDeliveryReasonAcknowledged, "Alert is acknowledged, so firing notifications are suppressed.")
|
||||
case !m.config.Enabled:
|
||||
diagnosis.setSuppressed(AlertDeliveryReasonNotificationsDisabled, "Alert notifications are disabled in the alert configuration.")
|
||||
case m.config.ActivationState != ActivationActive:
|
||||
diagnosis.setSuppressed(AlertDeliveryReasonNotificationsInactive, "Alert notifications are not active yet.")
|
||||
case diagnosis.SuppressedUntil != nil && diagnosis.SuppressedUntil.After(now):
|
||||
reason := AlertDeliveryReasonSuppressionWindow
|
||||
message := "Alert delivery is inside an active suppression window."
|
||||
if diagnosis.FlappingActive {
|
||||
reason = AlertDeliveryReasonFlapping
|
||||
message = "Alert delivery is suppressed because this alert is flapping."
|
||||
}
|
||||
diagnosis.setSuppressed(reason, message)
|
||||
case m.rateLimitExceededLocked(diagnosis.RecentAlertsInHour):
|
||||
diagnosis.setSuppressed(AlertDeliveryReasonRateLimited, "Alert delivery has reached the configured per-hour rate limit.")
|
||||
case alert.LastNotified != nil && !m.cooldownAllowsDeliveryLocked(alert, now):
|
||||
diagnosis.setSuppressed(AlertDeliveryReasonCooldown, "Alert delivery is waiting for the configured re-notification cooldown.")
|
||||
if next := m.nextCooldownEligibleAtLocked(alert); next != nil {
|
||||
diagnosis.NextEligibleAt = next
|
||||
}
|
||||
case func() bool {
|
||||
suppressed, reason := m.shouldSuppressNotification(alert)
|
||||
if !suppressed {
|
||||
return false
|
||||
}
|
||||
replayAt := m.quietHoursReplayAt()
|
||||
diagnosis.QuietHoursReplayAt = &replayAt
|
||||
if reason != "" {
|
||||
diagnosis.Reason = AlertDeliveryReasonQuietHours + ":" + reason
|
||||
} else {
|
||||
diagnosis.Reason = AlertDeliveryReasonQuietHours
|
||||
}
|
||||
diagnosis.Status = AlertDeliveryStatusDeferred
|
||||
diagnosis.Message = "Alert delivery is deferred by quiet-hours policy and will be replayed later."
|
||||
return true
|
||||
}():
|
||||
case isMonitorOnlyAlert(alert):
|
||||
diagnosis.setSuppressed(AlertDeliveryReasonMonitorOnly, "Alert is marked monitor-only, so it stays visible without notification delivery.")
|
||||
default:
|
||||
diagnosis.Status = AlertDeliveryStatusWouldSend
|
||||
diagnosis.Reason = AlertDeliveryReasonReady
|
||||
diagnosis.Message = "Alert delivery is currently eligible for notification delivery."
|
||||
}
|
||||
|
||||
return diagnosis, true
|
||||
}
|
||||
|
||||
func (d *AlertDeliveryDiagnosis) setSuppressed(reason, message string) {
|
||||
d.Status = AlertDeliveryStatusSuppressed
|
||||
d.Reason = reason
|
||||
d.Message = message
|
||||
}
|
||||
|
||||
func (m *Manager) policyNow() time.Time {
|
||||
if m != nil && m.now != nil {
|
||||
return m.now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
func (m *Manager) flappingSnapshotLocked(trackingKey string, now time.Time) (bool, int, *time.Time) {
|
||||
if trackingKey == "" || !m.config.FlappingEnabled {
|
||||
return false, 0, nil
|
||||
}
|
||||
|
||||
windowDuration := time.Duration(m.config.FlappingWindowSeconds) * time.Second
|
||||
validCount := 0
|
||||
for _, t := range m.flappingHistory[trackingKey] {
|
||||
if now.Sub(t) <= windowDuration {
|
||||
validCount++
|
||||
}
|
||||
}
|
||||
|
||||
var suppressedUntil *time.Time
|
||||
if until, ok := m.suppressedUntil[trackingKey]; ok {
|
||||
u := until
|
||||
suppressedUntil = &u
|
||||
}
|
||||
|
||||
return m.flappingActive[trackingKey], validCount, suppressedUntil
|
||||
}
|
||||
|
||||
func (m *Manager) rateLimitCountLocked(trackingKey string, now time.Time) int {
|
||||
if trackingKey == "" || m.config.Schedule.MaxAlertsHour <= 0 {
|
||||
return 0
|
||||
}
|
||||
cutoff := now.Add(-1 * time.Hour)
|
||||
recentCount := 0
|
||||
for _, t := range m.alertRateLimit[trackingKey] {
|
||||
if t.After(cutoff) {
|
||||
recentCount++
|
||||
}
|
||||
}
|
||||
return recentCount
|
||||
}
|
||||
|
||||
func (m *Manager) rateLimitExceededLocked(recentCount int) bool {
|
||||
return m.config.Schedule.MaxAlertsHour > 0 && recentCount >= m.config.Schedule.MaxAlertsHour
|
||||
}
|
||||
|
||||
func (m *Manager) cooldownAllowsDeliveryLocked(alert *Alert, now time.Time) bool {
|
||||
if alert == nil || alert.LastNotified == nil {
|
||||
return true
|
||||
}
|
||||
if m.config.Schedule.Cooldown <= 0 {
|
||||
return false
|
||||
}
|
||||
return !now.Before(alert.LastNotified.Add(time.Duration(m.config.Schedule.Cooldown) * time.Minute))
|
||||
}
|
||||
|
||||
func (m *Manager) nextCooldownEligibleAtLocked(alert *Alert) *time.Time {
|
||||
if alert == nil || alert.LastNotified == nil || m.config.Schedule.Cooldown <= 0 {
|
||||
return nil
|
||||
}
|
||||
next := alert.LastNotified.Add(time.Duration(m.config.Schedule.Cooldown) * time.Minute)
|
||||
return &next
|
||||
}
|
||||
|
||||
func (m *Manager) allowNotificationByRateLimit(trackingKey string, alert *Alert, reason string) bool {
|
||||
if trackingKey == "" && alert != nil {
|
||||
trackingKey = canonicalTrackingKeyForAlert(alert)
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ type AlertManager interface {
|
|||
GetConfig() alerts.AlertConfig
|
||||
UpdateConfig(alerts.AlertConfig)
|
||||
GetActiveAlerts() []alerts.Alert
|
||||
DiagnoseAlertDelivery(alertIdentifier string) (alerts.AlertDeliveryDiagnosis, bool)
|
||||
NotifyExistingAlert(id string)
|
||||
ClearAlertHistory() error
|
||||
UnacknowledgeAlert(id string) error
|
||||
|
|
@ -293,6 +294,50 @@ func (h *AlertHandlers) GetActiveAlerts(w http.ResponseWriter, r *http.Request)
|
|||
}
|
||||
}
|
||||
|
||||
// GetAlertDeliveryDiagnosis explains current notification delivery policy for
|
||||
// one active alert without sending a notification or mutating delivery state.
|
||||
func (h *AlertHandlers) GetAlertDeliveryDiagnosis(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
|
||||
alertIdentifier := strings.TrimSpace(r.URL.Query().Get("alertIdentifier"))
|
||||
if alertIdentifier == "" {
|
||||
alertIdentifier = strings.TrimSpace(r.URL.Query().Get("id"))
|
||||
}
|
||||
if alertIdentifier == "" {
|
||||
http.Error(w, "alertIdentifier is required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if !validateAlertIdentifier(alertIdentifier) {
|
||||
http.Error(w, "Invalid alert identifier", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
monitor := h.getMonitor(r.Context())
|
||||
if monitor == nil {
|
||||
http.Error(w, "monitor is not initialized", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
manager := monitor.GetAlertManager()
|
||||
if manager == nil {
|
||||
http.Error(w, "alert manager is not initialized", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
diagnosis, exists := manager.DiagnoseAlertDelivery(alertIdentifier)
|
||||
if !exists {
|
||||
http.Error(w, "alert not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
if err := utils.WriteJSONResponse(w, diagnosis); err != nil {
|
||||
log.Error().Err(err).Msg("Failed to write alert delivery diagnosis response")
|
||||
}
|
||||
}
|
||||
|
||||
// GetAlertHistory returns alert history
|
||||
func (h *AlertHandlers) GetAlertHistory(w http.ResponseWriter, r *http.Request) {
|
||||
query := r.URL.Query()
|
||||
|
|
@ -1011,6 +1056,11 @@ func (h *AlertHandlers) HandleAlerts(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
h.GetActiveAlerts(w, r)
|
||||
case path == "delivery-diagnosis" && r.Method == http.MethodGet:
|
||||
if !ensureScope(w, r, config.ScopeMonitoringRead) {
|
||||
return
|
||||
}
|
||||
h.GetAlertDeliveryDiagnosis(w, r)
|
||||
case path == "history" && r.Method == http.MethodGet:
|
||||
if !ensureScope(w, r, config.ScopeMonitoringRead) {
|
||||
return
|
||||
|
|
|
|||
|
|
@ -75,6 +75,11 @@ func (m *MockAlertManager) GetActiveAlerts() []alerts.Alert {
|
|||
return args.Get(0).([]alerts.Alert)
|
||||
}
|
||||
|
||||
func (m *MockAlertManager) DiagnoseAlertDelivery(alertIdentifier string) (alerts.AlertDeliveryDiagnosis, bool) {
|
||||
args := m.Called(alertIdentifier)
|
||||
return args.Get(0).(alerts.AlertDeliveryDiagnosis), args.Bool(1)
|
||||
}
|
||||
|
||||
func (m *MockAlertManager) NotifyExistingAlert(id string) {
|
||||
m.Called(id)
|
||||
}
|
||||
|
|
@ -297,6 +302,69 @@ func TestGetActiveAlerts(t *testing.T) {
|
|||
assert.Equal(t, "a1", resp[0].ID)
|
||||
}
|
||||
|
||||
func TestGetAlertDeliveryDiagnosis(t *testing.T) {
|
||||
mockMonitor := new(MockAlertMonitor)
|
||||
mockManager := new(MockAlertManager)
|
||||
mockMonitor.On("GetAlertManager").Return(mockManager)
|
||||
|
||||
h := NewAlertHandlers(nil, mockMonitor, nil)
|
||||
|
||||
expected := alerts.AlertDeliveryDiagnosis{
|
||||
AlertIdentifier: "canonical:a1",
|
||||
AlertID: "a1",
|
||||
TrackingKey: "node/a1/cpu",
|
||||
Status: alerts.AlertDeliveryStatusDeferred,
|
||||
Reason: alerts.AlertDeliveryReasonQuietHours + ":performance",
|
||||
Message: "Alert delivery is deferred by quiet-hours policy and will be replayed later.",
|
||||
NotificationsEnabled: true,
|
||||
ActivationState: string(alerts.ActivationActive),
|
||||
}
|
||||
mockManager.On("DiagnoseAlertDelivery", "canonical:a1").Return(expected, true)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/alerts/delivery-diagnosis?alertIdentifier=canonical:a1", nil)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
h.GetAlertDeliveryDiagnosis(w, req)
|
||||
|
||||
assert.Equal(t, http.StatusOK, w.Code)
|
||||
var resp alerts.AlertDeliveryDiagnosis
|
||||
if err := json.NewDecoder(w.Body).Decode(&resp); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
assert.Equal(t, expected.AlertIdentifier, resp.AlertIdentifier)
|
||||
assert.Equal(t, expected.Status, resp.Status)
|
||||
assert.Equal(t, expected.Reason, resp.Reason)
|
||||
assert.Equal(t, expected.TrackingKey, resp.TrackingKey)
|
||||
}
|
||||
|
||||
func TestGetAlertDeliveryDiagnosisRejectsMissingIdentifier(t *testing.T) {
|
||||
h := NewAlertHandlers(nil, new(MockAlertMonitor), nil)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/alerts/delivery-diagnosis", nil)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
h.GetAlertDeliveryDiagnosis(w, req)
|
||||
|
||||
assert.Equal(t, http.StatusBadRequest, w.Code)
|
||||
assert.Contains(t, w.Body.String(), "alertIdentifier is required")
|
||||
}
|
||||
|
||||
func TestGetAlertDeliveryDiagnosisNotFound(t *testing.T) {
|
||||
mockMonitor := new(MockAlertMonitor)
|
||||
mockManager := new(MockAlertManager)
|
||||
mockMonitor.On("GetAlertManager").Return(mockManager)
|
||||
mockManager.On("DiagnoseAlertDelivery", "missing").Return(alerts.AlertDeliveryDiagnosis{}, false)
|
||||
|
||||
h := NewAlertHandlers(nil, mockMonitor, nil)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/alerts/delivery-diagnosis?alertIdentifier=missing", nil)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
h.GetAlertDeliveryDiagnosis(w, req)
|
||||
|
||||
assert.Equal(t, http.StatusNotFound, w.Code)
|
||||
}
|
||||
|
||||
func TestValidateAlertIdentifier(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
|
|
@ -629,6 +697,13 @@ func TestHandleAlerts(t *testing.T) {
|
|||
|
||||
routes := []route{
|
||||
{"GET", "/api/alerts/active", func() { mockManager.On("GetActiveAlerts").Return([]alerts.Alert{}).Once() }},
|
||||
{"GET", "/api/alerts/delivery-diagnosis?alertIdentifier=a1", func() {
|
||||
mockManager.On("DiagnoseAlertDelivery", "a1").Return(alerts.AlertDeliveryDiagnosis{
|
||||
AlertIdentifier: "a1",
|
||||
Status: alerts.AlertDeliveryStatusWouldSend,
|
||||
Reason: alerts.AlertDeliveryReasonReady,
|
||||
}, true).Once()
|
||||
}},
|
||||
{"GET", "/api/alerts/history", func() {
|
||||
mockManager.On("GetAlertHistory", mock.MatchedBy(func(int) bool { return true })).Return([]alerts.Alert{}).Once()
|
||||
}},
|
||||
|
|
|
|||
|
|
@ -65,6 +65,73 @@ type resourceContractSnapshot struct {
|
|||
Type string
|
||||
}
|
||||
|
||||
func TestContract_AlertDeliveryDiagnosisRouteIsReadOnlyMonitoringRead(t *testing.T) {
|
||||
source, err := os.ReadFile("alerts.go")
|
||||
if err != nil {
|
||||
t.Fatalf("read alerts.go: %v", err)
|
||||
}
|
||||
src := string(source)
|
||||
|
||||
required := []string{
|
||||
`case path == "delivery-diagnosis" && r.Method == http.MethodGet:`,
|
||||
`ensureScope(w, r, config.ScopeMonitoringRead)`,
|
||||
`h.GetAlertDeliveryDiagnosis(w, r)`,
|
||||
`manager.DiagnoseAlertDelivery(alertIdentifier)`,
|
||||
}
|
||||
for _, snippet := range required {
|
||||
if !strings.Contains(src, snippet) {
|
||||
t.Fatalf("alert delivery diagnosis route contract missing %q", snippet)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestContract_AlertDeliveryDiagnosisPayloadShape(t *testing.T) {
|
||||
replayAt := time.Date(2026, 6, 29, 12, 30, 0, 0, time.UTC)
|
||||
diagnosis := alerts.AlertDeliveryDiagnosis{
|
||||
AlertIdentifier: "canonical:a1",
|
||||
AlertID: "a1",
|
||||
TrackingKey: "node/a1/cpu",
|
||||
Status: alerts.AlertDeliveryStatusDeferred,
|
||||
Reason: alerts.AlertDeliveryReasonQuietHours + ":performance",
|
||||
Message: "Alert delivery is deferred by quiet-hours policy and will be replayed later.",
|
||||
AlertType: "cpu",
|
||||
Level: alerts.AlertLevelCritical,
|
||||
ResourceID: "node-1",
|
||||
ResourceName: "node-1",
|
||||
NotificationsEnabled: true,
|
||||
ActivationState: string(alerts.ActivationActive),
|
||||
CooldownMinutes: 5,
|
||||
MaxAlertsHour: 10,
|
||||
RecentAlertsInHour: 2,
|
||||
FlappingActive: false,
|
||||
FlappingHistoryInWindow: 1,
|
||||
FlappingThreshold: 3,
|
||||
FlappingWindowSeconds: 300,
|
||||
QuietHoursReplayAt: &replayAt,
|
||||
}
|
||||
|
||||
payload, err := json.Marshal(diagnosis)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal alert delivery diagnosis: %v", err)
|
||||
}
|
||||
body := string(payload)
|
||||
for _, field := range []string{
|
||||
`"alertIdentifier":"canonical:a1"`,
|
||||
`"trackingKey":"node/a1/cpu"`,
|
||||
`"status":"deferred"`,
|
||||
`"reason":"quiet_hours:performance"`,
|
||||
`"notificationsEnabled":true`,
|
||||
`"cooldownMinutes":5`,
|
||||
`"recentAlertsInHour":2`,
|
||||
`"flappingHistoryInWindow":1`,
|
||||
`"quietHoursReplayAt":"2026-06-29T12:30:00Z"`,
|
||||
} {
|
||||
if !strings.Contains(body, field) {
|
||||
t.Fatalf("alert delivery diagnosis payload missing %s in %s", field, body)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestContract_PMGInstancesEndpointUsesUnifiedReadStatePayload(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
source := models.PMGInstance{
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue