Add alert delivery diagnosis endpoint

This commit is contained in:
rcourtman 2026-06-29 22:27:01 +01:00
parent a801f456d6
commit 597c369a63
10 changed files with 664 additions and 0 deletions

View file

@ -575,6 +575,7 @@ Alert configuration and history (requires `monitoring:read`/`monitoring:write`).
- `PUT /api/alerts/config`
- `POST /api/alerts/activate`
- `GET /api/alerts/active`
- `GET /api/alerts/delivery-diagnosis?alertIdentifier=<alert-id>`
- `GET /api/alerts/history`
- `DELETE /api/alerts/history`
- `GET /api/alerts/incidents`

View file

@ -1513,6 +1513,12 @@ list representation, and masked-echo preservation on update are
notifications/API-contract owned and grant no agent install, enrollment,
setup-token, or fleet command authority.
Alert delivery diagnosis on shared `internal/api/alerts.go` is likewise
adjacent only: `/api/alerts/delivery-diagnosis` is alerts/API-contract owned
read-only notification-policy evidence and must not be interpreted as agent
enrollment, agent liveness, install progress, setup-token authority, or fleet
repair eligibility.
The router projection-builder (`internal/api/router.go`) that wires
the operator-state provider into the findings runtime now also
populates `NeverAutoRemediate` on the projection. The investigation

View file

@ -382,6 +382,11 @@ suppression, monitor-only notification suppression, cooldown decisions, and
per-alert rate limiting; future notification-gating changes should extend that
policy owner rather than burying new checks inside metric or resource-specific
evaluators.
The same policy owner also exposes the read-only alert delivery diagnosis
projection used by `/api/alerts/delivery-diagnosis`; that projection may explain
current gating state, quiet-hours replay timing, cooldown timing, rate-limit
counts, and flapping suppression, but it must not dispatch callbacks or mutate
flapping/rate-limit tracking maps.
The same dispatch policy owns firing-notification evidence on active alerts:
any alert that passes notification suppression and enters the fired callback
fan-out must carry `LastNotified` before the callback clone is emitted. Resolved

View file

@ -2965,6 +2965,15 @@ a new API state machine, queue contract, or verification-accounting field.
## Current State
Alert delivery diagnosis is a read-only monitoring API contract.
`GET /api/alerts/delivery-diagnosis?alertIdentifier=<id>` returns the alert
manager's current delivery-policy projection for one active alert, including
`status` (`would_send`, `deferred`, or `suppressed`), `reason`, canonical
`trackingKey`, activation/notification state, cooldown timing, quiet-hours
replay timing, recent rate-limit counts, and flapping suppression evidence. The
route requires `monitoring:read`, returns `404` for unknown active alerts, and
must not send notifications or mutate delivery tracking state.
Manifest-backed Patrol finding lifecycle schemas are the API source of truth
for Assistant provider-tool optionality as well as MCP/API discovery. Legacy
Assistant runtime paths may project those schemas into provider-tool JSON, but

View file

@ -1536,6 +1536,12 @@ likewise adjacent only: the webhook `signingSecret` payload field and its
masking semantics are notifications/API-contract owned and create no storage,
recovery-point, or backup-surface semantics.
Alert delivery diagnosis on shared `internal/api/alerts.go` is likewise
adjacent only: `/api/alerts/delivery-diagnosis` exposes alerts/API-contract
read-only notification-policy evidence and creates no storage health,
recovery-point, backup verification, restore authorization, or provider
coverage semantics.
Kubernetes pod metadata decoded by `frontend-modern/src/hooks/useUnifiedResources.ts`
is shared inventory context for storage/recovery handoffs only; Pod phase,
container readiness, owner, image, and restart fields do not become protection

View file

@ -19022,3 +19022,236 @@ func TestDefaultAlertConfigUsesIndependentBackupAlertOrphanedPointer(t *testing.
t.Fatal("mutating one default config changed another default config")
}
}
func TestDiagnoseAlertDeliveryReady(t *testing.T) {
manager, now := newDeliveryDiagnosisManager(t)
alert := addDeliveryDiagnosisAlert(manager, &Alert{
ID: "alert-ready",
Type: "cpu",
Level: AlertLevelWarning,
ResourceID: "node-1",
ResourceName: "node-1",
StartTime: now.Add(-10 * time.Minute),
})
diagnosis, ok := manager.DiagnoseAlertDelivery(alert.ID)
if !ok {
t.Fatal("expected active alert diagnosis")
}
if diagnosis.Status != AlertDeliveryStatusWouldSend {
t.Fatalf("Status = %q, want %q", diagnosis.Status, AlertDeliveryStatusWouldSend)
}
if diagnosis.Reason != AlertDeliveryReasonReady {
t.Fatalf("Reason = %q, want %q", diagnosis.Reason, AlertDeliveryReasonReady)
}
if diagnosis.TrackingKey == "" {
t.Fatal("expected tracking key")
}
if diagnosis.RecentAlertsInHour != 0 {
t.Fatalf("RecentAlertsInHour = %d, want 0", diagnosis.RecentAlertsInHour)
}
}
func TestDiagnoseAlertDeliverySuppressesAcknowledged(t *testing.T) {
manager, now := newDeliveryDiagnosisManager(t)
alert := addDeliveryDiagnosisAlert(manager, &Alert{
ID: "alert-acked",
Type: "memory",
Level: AlertLevelCritical,
ResourceID: "node-1",
ResourceName: "node-1",
StartTime: now.Add(-10 * time.Minute),
Acknowledged: true,
})
diagnosis, ok := manager.DiagnoseAlertDelivery(alert.ID)
if !ok {
t.Fatal("expected active alert diagnosis")
}
if diagnosis.Status != AlertDeliveryStatusSuppressed {
t.Fatalf("Status = %q, want %q", diagnosis.Status, AlertDeliveryStatusSuppressed)
}
if diagnosis.Reason != AlertDeliveryReasonAcknowledged {
t.Fatalf("Reason = %q, want %q", diagnosis.Reason, AlertDeliveryReasonAcknowledged)
}
}
func TestDiagnoseAlertDeliveryDefersQuietHours(t *testing.T) {
manager, now := newDeliveryDiagnosisManager(t)
cfg := manager.GetConfig()
cfg.Schedule.QuietHours.Enabled = true
cfg.Schedule.QuietHours.Timezone = "UTC"
cfg.Schedule.QuietHours.Start = "00:00"
cfg.Schedule.QuietHours.End = "23:59"
cfg.Schedule.QuietHours.Days = map[string]bool{"monday": true}
cfg.Schedule.QuietHours.Suppress.Performance = true
manager.UpdateConfig(cfg)
manager.now = func() time.Time { return now }
alert := addDeliveryDiagnosisAlert(manager, &Alert{
ID: "alert-quiet",
Type: "cpu",
Level: AlertLevelCritical,
ResourceID: "node-1",
ResourceName: "node-1",
StartTime: now.Add(-10 * time.Minute),
})
diagnosis, ok := manager.DiagnoseAlertDelivery(alert.ID)
if !ok {
t.Fatal("expected active alert diagnosis")
}
if diagnosis.Status != AlertDeliveryStatusDeferred {
t.Fatalf("Status = %q, want %q", diagnosis.Status, AlertDeliveryStatusDeferred)
}
wantReason := AlertDeliveryReasonQuietHours + ":performance"
if diagnosis.Reason != wantReason {
t.Fatalf("Reason = %q, want %q", diagnosis.Reason, wantReason)
}
if diagnosis.QuietHoursReplayAt == nil {
t.Fatal("expected quiet-hours replay time")
}
}
func TestDiagnoseAlertDeliverySuppressesCooldown(t *testing.T) {
manager, now := newDeliveryDiagnosisManager(t)
lastNotified := now.Add(-2 * time.Minute)
alert := addDeliveryDiagnosisAlert(manager, &Alert{
ID: "alert-cooldown",
Type: "cpu",
Level: AlertLevelWarning,
ResourceID: "node-1",
ResourceName: "node-1",
StartTime: now.Add(-10 * time.Minute),
LastNotified: &lastNotified,
})
diagnosis, ok := manager.DiagnoseAlertDelivery(alert.ID)
if !ok {
t.Fatal("expected active alert diagnosis")
}
if diagnosis.Status != AlertDeliveryStatusSuppressed {
t.Fatalf("Status = %q, want %q", diagnosis.Status, AlertDeliveryStatusSuppressed)
}
if diagnosis.Reason != AlertDeliveryReasonCooldown {
t.Fatalf("Reason = %q, want %q", diagnosis.Reason, AlertDeliveryReasonCooldown)
}
if diagnosis.NextEligibleAt == nil || !diagnosis.NextEligibleAt.Equal(lastNotified.Add(5*time.Minute)) {
t.Fatalf("NextEligibleAt = %v, want %v", diagnosis.NextEligibleAt, lastNotified.Add(5*time.Minute))
}
}
func TestDiagnoseAlertDeliverySuppressesRateLimitWithoutMutating(t *testing.T) {
manager, now := newDeliveryDiagnosisManager(t)
cfg := manager.GetConfig()
cfg.Schedule.MaxAlertsHour = 1
manager.UpdateConfig(cfg)
manager.now = func() time.Time { return now }
alert := addDeliveryDiagnosisAlert(manager, &Alert{
ID: "alert-rate-limit",
Type: "cpu",
Level: AlertLevelWarning,
ResourceID: "node-1",
ResourceName: "node-1",
StartTime: now.Add(-10 * time.Minute),
})
trackingKey := canonicalTrackingKeyForAlert(alert)
manager.mu.Lock()
manager.alertRateLimit[trackingKey] = []time.Time{now.Add(-30 * time.Minute)}
manager.mu.Unlock()
diagnosis, ok := manager.DiagnoseAlertDelivery(alert.ID)
if !ok {
t.Fatal("expected active alert diagnosis")
}
if diagnosis.Status != AlertDeliveryStatusSuppressed {
t.Fatalf("Status = %q, want %q", diagnosis.Status, AlertDeliveryStatusSuppressed)
}
if diagnosis.Reason != AlertDeliveryReasonRateLimited {
t.Fatalf("Reason = %q, want %q", diagnosis.Reason, AlertDeliveryReasonRateLimited)
}
if diagnosis.RecentAlertsInHour != 1 {
t.Fatalf("RecentAlertsInHour = %d, want 1", diagnosis.RecentAlertsInHour)
}
manager.mu.Lock()
defer manager.mu.Unlock()
if got := len(manager.alertRateLimit[trackingKey]); got != 1 {
t.Fatalf("diagnosis mutated alertRateLimit length to %d, want 1", got)
}
}
func TestDiagnoseAlertDeliverySuppressesFlappingWithoutMutatingRateLimit(t *testing.T) {
manager, now := newDeliveryDiagnosisManager(t)
alert := addDeliveryDiagnosisAlert(manager, &Alert{
ID: "alert-flapping",
Type: "cpu",
Level: AlertLevelWarning,
ResourceID: "node-1",
ResourceName: "node-1",
StartTime: now.Add(-10 * time.Minute),
})
trackingKey := canonicalTrackingKeyForAlert(alert)
suppressedUntil := now.Add(15 * time.Minute)
manager.mu.Lock()
manager.flappingHistory[trackingKey] = []time.Time{now.Add(-2 * time.Minute), now.Add(-time.Minute)}
manager.flappingActive[trackingKey] = true
manager.suppressedUntil[trackingKey] = suppressedUntil
manager.alertRateLimit[trackingKey] = []time.Time{now.Add(-30 * time.Minute)}
manager.mu.Unlock()
diagnosis, ok := manager.DiagnoseAlertDelivery(alert.ID)
if !ok {
t.Fatal("expected active alert diagnosis")
}
if diagnosis.Status != AlertDeliveryStatusSuppressed {
t.Fatalf("Status = %q, want %q", diagnosis.Status, AlertDeliveryStatusSuppressed)
}
if diagnosis.Reason != AlertDeliveryReasonFlapping {
t.Fatalf("Reason = %q, want %q", diagnosis.Reason, AlertDeliveryReasonFlapping)
}
if diagnosis.SuppressedUntil == nil || !diagnosis.SuppressedUntil.Equal(suppressedUntil) {
t.Fatalf("SuppressedUntil = %v, want %v", diagnosis.SuppressedUntil, suppressedUntil)
}
if diagnosis.RecentAlertsInHour != 1 {
t.Fatalf("RecentAlertsInHour = %d, want 1", diagnosis.RecentAlertsInHour)
}
manager.mu.Lock()
defer manager.mu.Unlock()
if got := len(manager.alertRateLimit[trackingKey]); got != 1 {
t.Fatalf("diagnosis mutated alertRateLimit length to %d, want 1", got)
}
}
func newDeliveryDiagnosisManager(t *testing.T) (*Manager, time.Time) {
t.Helper()
manager := NewManager()
t.Cleanup(manager.Stop)
now := time.Date(2026, 6, 29, 12, 0, 0, 0, time.UTC)
manager.now = func() time.Time { return now }
cfg := manager.GetConfig()
cfg.Enabled = true
cfg.ActivationState = ActivationActive
cfg.Schedule.Cooldown = 5
cfg.Schedule.MaxAlertsHour = 10
cfg.FlappingEnabled = true
cfg.FlappingThreshold = 3
cfg.FlappingWindowSeconds = 300
cfg.FlappingCooldownMinutes = 15
manager.UpdateConfig(cfg)
manager.now = func() time.Time { return now }
return manager, now
}
func addDeliveryDiagnosisAlert(manager *Manager, alert *Alert) *Alert {
manager.mu.Lock()
defer manager.mu.Unlock()
manager.activeAlerts[alert.ID] = alert
return alert
}

View file

@ -20,6 +20,54 @@ const (
MetadataQuietHoursReplayAt = "quietHoursReplayAt"
)
const (
AlertDeliveryStatusWouldSend = "would_send"
AlertDeliveryStatusDeferred = "deferred"
AlertDeliveryStatusSuppressed = "suppressed"
AlertDeliveryReasonReady = "ready"
AlertDeliveryReasonAcknowledged = "acknowledged"
AlertDeliveryReasonNotificationsDisabled = "notifications_disabled"
AlertDeliveryReasonNotificationsInactive = "notifications_inactive"
AlertDeliveryReasonSuppressionWindow = "suppression_window"
AlertDeliveryReasonFlapping = "flapping"
AlertDeliveryReasonRateLimited = "rate_limited"
AlertDeliveryReasonCooldown = "cooldown"
AlertDeliveryReasonQuietHours = "quiet_hours"
AlertDeliveryReasonMonitorOnly = "monitor_only"
)
// AlertDeliveryDiagnosis is a read-only projection of the alert manager's
// current delivery policy for one active alert. It is intentionally
// non-mutating: it does not dispatch callbacks or advance flapping/rate-limit
// counters.
type AlertDeliveryDiagnosis struct {
AlertIdentifier string `json:"alertIdentifier"`
AlertID string `json:"alertId"`
TrackingKey string `json:"trackingKey"`
Status string `json:"status"`
Reason string `json:"reason"`
Message string `json:"message"`
AlertType string `json:"alertType"`
Level AlertLevel `json:"level"`
ResourceID string `json:"resourceId,omitempty"`
ResourceName string `json:"resourceName,omitempty"`
Node string `json:"node,omitempty"`
NotificationsEnabled bool `json:"notificationsEnabled"`
ActivationState string `json:"activationState"`
CooldownMinutes int `json:"cooldownMinutes"`
MaxAlertsHour int `json:"maxAlertsHour"`
RecentAlertsInHour int `json:"recentAlertsInHour"`
FlappingActive bool `json:"flappingActive"`
FlappingHistoryInWindow int `json:"flappingHistoryInWindow"`
FlappingThreshold int `json:"flappingThreshold"`
FlappingWindowSeconds int `json:"flappingWindowSeconds"`
LastNotified *time.Time `json:"lastNotified,omitempty"`
NextEligibleAt *time.Time `json:"nextEligibleAt,omitempty"`
QuietHoursReplayAt *time.Time `json:"quietHoursReplayAt,omitempty"`
SuppressedUntil *time.Time `json:"suppressedUntil,omitempty"`
}
// checkFlappingLocked detects alert flapping and returns whether the alert
// should be suppressed and whether this call is the first transition into
// the flapping state for the current cooldown window. justTransitioned is
@ -568,6 +616,170 @@ func (m *Manager) shouldNotifyAfterCooldown(alert *Alert) bool {
return timeSinceLastNotification >= cooldownDuration
}
// DiagnoseAlertDelivery reports the current notification policy outcome for
// one active alert without sending a notification or updating any delivery
// tracking maps.
func (m *Manager) DiagnoseAlertDelivery(alertIdentifier string) (AlertDeliveryDiagnosis, bool) {
diagnosis := AlertDeliveryDiagnosis{
AlertIdentifier: strings.TrimSpace(alertIdentifier),
Status: AlertDeliveryStatusSuppressed,
Reason: "not_found",
}
if m == nil || diagnosis.AlertIdentifier == "" {
return diagnosis, false
}
m.mu.Lock()
defer m.mu.Unlock()
alert, exists := m.getActiveAlertNoLock(diagnosis.AlertIdentifier)
if !exists || alert == nil {
return diagnosis, false
}
now := m.policyNow()
trackingKey := canonicalTrackingKeyForAlert(alert)
diagnosis.AlertID = alert.ID
diagnosis.TrackingKey = trackingKey
diagnosis.AlertType = alert.Type
diagnosis.Level = alert.Level
diagnosis.ResourceID = alert.ResourceID
diagnosis.ResourceName = alert.ResourceName
diagnosis.Node = alert.Node
diagnosis.NotificationsEnabled = m.config.Enabled && m.config.ActivationState == ActivationActive
diagnosis.ActivationState = string(m.config.ActivationState)
diagnosis.CooldownMinutes = m.config.Schedule.Cooldown
diagnosis.MaxAlertsHour = m.config.Schedule.MaxAlertsHour
diagnosis.FlappingThreshold = m.config.FlappingThreshold
diagnosis.FlappingWindowSeconds = m.config.FlappingWindowSeconds
if alert.LastNotified != nil {
lastNotified := *alert.LastNotified
diagnosis.LastNotified = &lastNotified
}
diagnosis.FlappingActive, diagnosis.FlappingHistoryInWindow, diagnosis.SuppressedUntil = m.flappingSnapshotLocked(trackingKey, now)
diagnosis.RecentAlertsInHour = m.rateLimitCountLocked(trackingKey, now)
switch {
case alert.Acknowledged:
diagnosis.setSuppressed(AlertDeliveryReasonAcknowledged, "Alert is acknowledged, so firing notifications are suppressed.")
case !m.config.Enabled:
diagnosis.setSuppressed(AlertDeliveryReasonNotificationsDisabled, "Alert notifications are disabled in the alert configuration.")
case m.config.ActivationState != ActivationActive:
diagnosis.setSuppressed(AlertDeliveryReasonNotificationsInactive, "Alert notifications are not active yet.")
case diagnosis.SuppressedUntil != nil && diagnosis.SuppressedUntil.After(now):
reason := AlertDeliveryReasonSuppressionWindow
message := "Alert delivery is inside an active suppression window."
if diagnosis.FlappingActive {
reason = AlertDeliveryReasonFlapping
message = "Alert delivery is suppressed because this alert is flapping."
}
diagnosis.setSuppressed(reason, message)
case m.rateLimitExceededLocked(diagnosis.RecentAlertsInHour):
diagnosis.setSuppressed(AlertDeliveryReasonRateLimited, "Alert delivery has reached the configured per-hour rate limit.")
case alert.LastNotified != nil && !m.cooldownAllowsDeliveryLocked(alert, now):
diagnosis.setSuppressed(AlertDeliveryReasonCooldown, "Alert delivery is waiting for the configured re-notification cooldown.")
if next := m.nextCooldownEligibleAtLocked(alert); next != nil {
diagnosis.NextEligibleAt = next
}
case func() bool {
suppressed, reason := m.shouldSuppressNotification(alert)
if !suppressed {
return false
}
replayAt := m.quietHoursReplayAt()
diagnosis.QuietHoursReplayAt = &replayAt
if reason != "" {
diagnosis.Reason = AlertDeliveryReasonQuietHours + ":" + reason
} else {
diagnosis.Reason = AlertDeliveryReasonQuietHours
}
diagnosis.Status = AlertDeliveryStatusDeferred
diagnosis.Message = "Alert delivery is deferred by quiet-hours policy and will be replayed later."
return true
}():
case isMonitorOnlyAlert(alert):
diagnosis.setSuppressed(AlertDeliveryReasonMonitorOnly, "Alert is marked monitor-only, so it stays visible without notification delivery.")
default:
diagnosis.Status = AlertDeliveryStatusWouldSend
diagnosis.Reason = AlertDeliveryReasonReady
diagnosis.Message = "Alert delivery is currently eligible for notification delivery."
}
return diagnosis, true
}
func (d *AlertDeliveryDiagnosis) setSuppressed(reason, message string) {
d.Status = AlertDeliveryStatusSuppressed
d.Reason = reason
d.Message = message
}
func (m *Manager) policyNow() time.Time {
if m != nil && m.now != nil {
return m.now()
}
return time.Now()
}
func (m *Manager) flappingSnapshotLocked(trackingKey string, now time.Time) (bool, int, *time.Time) {
if trackingKey == "" || !m.config.FlappingEnabled {
return false, 0, nil
}
windowDuration := time.Duration(m.config.FlappingWindowSeconds) * time.Second
validCount := 0
for _, t := range m.flappingHistory[trackingKey] {
if now.Sub(t) <= windowDuration {
validCount++
}
}
var suppressedUntil *time.Time
if until, ok := m.suppressedUntil[trackingKey]; ok {
u := until
suppressedUntil = &u
}
return m.flappingActive[trackingKey], validCount, suppressedUntil
}
func (m *Manager) rateLimitCountLocked(trackingKey string, now time.Time) int {
if trackingKey == "" || m.config.Schedule.MaxAlertsHour <= 0 {
return 0
}
cutoff := now.Add(-1 * time.Hour)
recentCount := 0
for _, t := range m.alertRateLimit[trackingKey] {
if t.After(cutoff) {
recentCount++
}
}
return recentCount
}
func (m *Manager) rateLimitExceededLocked(recentCount int) bool {
return m.config.Schedule.MaxAlertsHour > 0 && recentCount >= m.config.Schedule.MaxAlertsHour
}
func (m *Manager) cooldownAllowsDeliveryLocked(alert *Alert, now time.Time) bool {
if alert == nil || alert.LastNotified == nil {
return true
}
if m.config.Schedule.Cooldown <= 0 {
return false
}
return !now.Before(alert.LastNotified.Add(time.Duration(m.config.Schedule.Cooldown) * time.Minute))
}
func (m *Manager) nextCooldownEligibleAtLocked(alert *Alert) *time.Time {
if alert == nil || alert.LastNotified == nil || m.config.Schedule.Cooldown <= 0 {
return nil
}
next := alert.LastNotified.Add(time.Duration(m.config.Schedule.Cooldown) * time.Minute)
return &next
}
func (m *Manager) allowNotificationByRateLimit(trackingKey string, alert *Alert, reason string) bool {
if trackingKey == "" && alert != nil {
trackingKey = canonicalTrackingKeyForAlert(alert)

View file

@ -27,6 +27,7 @@ type AlertManager interface {
GetConfig() alerts.AlertConfig
UpdateConfig(alerts.AlertConfig)
GetActiveAlerts() []alerts.Alert
DiagnoseAlertDelivery(alertIdentifier string) (alerts.AlertDeliveryDiagnosis, bool)
NotifyExistingAlert(id string)
ClearAlertHistory() error
UnacknowledgeAlert(id string) error
@ -293,6 +294,50 @@ func (h *AlertHandlers) GetActiveAlerts(w http.ResponseWriter, r *http.Request)
}
}
// GetAlertDeliveryDiagnosis explains current notification delivery policy for
// one active alert without sending a notification or mutating delivery state.
func (h *AlertHandlers) GetAlertDeliveryDiagnosis(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
alertIdentifier := strings.TrimSpace(r.URL.Query().Get("alertIdentifier"))
if alertIdentifier == "" {
alertIdentifier = strings.TrimSpace(r.URL.Query().Get("id"))
}
if alertIdentifier == "" {
http.Error(w, "alertIdentifier is required", http.StatusBadRequest)
return
}
if !validateAlertIdentifier(alertIdentifier) {
http.Error(w, "Invalid alert identifier", http.StatusBadRequest)
return
}
monitor := h.getMonitor(r.Context())
if monitor == nil {
http.Error(w, "monitor is not initialized", http.StatusServiceUnavailable)
return
}
manager := monitor.GetAlertManager()
if manager == nil {
http.Error(w, "alert manager is not initialized", http.StatusServiceUnavailable)
return
}
diagnosis, exists := manager.DiagnoseAlertDelivery(alertIdentifier)
if !exists {
http.Error(w, "alert not found", http.StatusNotFound)
return
}
if err := utils.WriteJSONResponse(w, diagnosis); err != nil {
log.Error().Err(err).Msg("Failed to write alert delivery diagnosis response")
}
}
// GetAlertHistory returns alert history
func (h *AlertHandlers) GetAlertHistory(w http.ResponseWriter, r *http.Request) {
query := r.URL.Query()
@ -1011,6 +1056,11 @@ func (h *AlertHandlers) HandleAlerts(w http.ResponseWriter, r *http.Request) {
return
}
h.GetActiveAlerts(w, r)
case path == "delivery-diagnosis" && r.Method == http.MethodGet:
if !ensureScope(w, r, config.ScopeMonitoringRead) {
return
}
h.GetAlertDeliveryDiagnosis(w, r)
case path == "history" && r.Method == http.MethodGet:
if !ensureScope(w, r, config.ScopeMonitoringRead) {
return

View file

@ -75,6 +75,11 @@ func (m *MockAlertManager) GetActiveAlerts() []alerts.Alert {
return args.Get(0).([]alerts.Alert)
}
func (m *MockAlertManager) DiagnoseAlertDelivery(alertIdentifier string) (alerts.AlertDeliveryDiagnosis, bool) {
args := m.Called(alertIdentifier)
return args.Get(0).(alerts.AlertDeliveryDiagnosis), args.Bool(1)
}
func (m *MockAlertManager) NotifyExistingAlert(id string) {
m.Called(id)
}
@ -297,6 +302,69 @@ func TestGetActiveAlerts(t *testing.T) {
assert.Equal(t, "a1", resp[0].ID)
}
func TestGetAlertDeliveryDiagnosis(t *testing.T) {
mockMonitor := new(MockAlertMonitor)
mockManager := new(MockAlertManager)
mockMonitor.On("GetAlertManager").Return(mockManager)
h := NewAlertHandlers(nil, mockMonitor, nil)
expected := alerts.AlertDeliveryDiagnosis{
AlertIdentifier: "canonical:a1",
AlertID: "a1",
TrackingKey: "node/a1/cpu",
Status: alerts.AlertDeliveryStatusDeferred,
Reason: alerts.AlertDeliveryReasonQuietHours + ":performance",
Message: "Alert delivery is deferred by quiet-hours policy and will be replayed later.",
NotificationsEnabled: true,
ActivationState: string(alerts.ActivationActive),
}
mockManager.On("DiagnoseAlertDelivery", "canonical:a1").Return(expected, true)
req := httptest.NewRequest(http.MethodGet, "/api/alerts/delivery-diagnosis?alertIdentifier=canonical:a1", nil)
w := httptest.NewRecorder()
h.GetAlertDeliveryDiagnosis(w, req)
assert.Equal(t, http.StatusOK, w.Code)
var resp alerts.AlertDeliveryDiagnosis
if err := json.NewDecoder(w.Body).Decode(&resp); err != nil {
t.Fatalf("decode response: %v", err)
}
assert.Equal(t, expected.AlertIdentifier, resp.AlertIdentifier)
assert.Equal(t, expected.Status, resp.Status)
assert.Equal(t, expected.Reason, resp.Reason)
assert.Equal(t, expected.TrackingKey, resp.TrackingKey)
}
func TestGetAlertDeliveryDiagnosisRejectsMissingIdentifier(t *testing.T) {
h := NewAlertHandlers(nil, new(MockAlertMonitor), nil)
req := httptest.NewRequest(http.MethodGet, "/api/alerts/delivery-diagnosis", nil)
w := httptest.NewRecorder()
h.GetAlertDeliveryDiagnosis(w, req)
assert.Equal(t, http.StatusBadRequest, w.Code)
assert.Contains(t, w.Body.String(), "alertIdentifier is required")
}
func TestGetAlertDeliveryDiagnosisNotFound(t *testing.T) {
mockMonitor := new(MockAlertMonitor)
mockManager := new(MockAlertManager)
mockMonitor.On("GetAlertManager").Return(mockManager)
mockManager.On("DiagnoseAlertDelivery", "missing").Return(alerts.AlertDeliveryDiagnosis{}, false)
h := NewAlertHandlers(nil, mockMonitor, nil)
req := httptest.NewRequest(http.MethodGet, "/api/alerts/delivery-diagnosis?alertIdentifier=missing", nil)
w := httptest.NewRecorder()
h.GetAlertDeliveryDiagnosis(w, req)
assert.Equal(t, http.StatusNotFound, w.Code)
}
func TestValidateAlertIdentifier(t *testing.T) {
testCases := []struct {
name string
@ -629,6 +697,13 @@ func TestHandleAlerts(t *testing.T) {
routes := []route{
{"GET", "/api/alerts/active", func() { mockManager.On("GetActiveAlerts").Return([]alerts.Alert{}).Once() }},
{"GET", "/api/alerts/delivery-diagnosis?alertIdentifier=a1", func() {
mockManager.On("DiagnoseAlertDelivery", "a1").Return(alerts.AlertDeliveryDiagnosis{
AlertIdentifier: "a1",
Status: alerts.AlertDeliveryStatusWouldSend,
Reason: alerts.AlertDeliveryReasonReady,
}, true).Once()
}},
{"GET", "/api/alerts/history", func() {
mockManager.On("GetAlertHistory", mock.MatchedBy(func(int) bool { return true })).Return([]alerts.Alert{}).Once()
}},

View file

@ -65,6 +65,73 @@ type resourceContractSnapshot struct {
Type string
}
func TestContract_AlertDeliveryDiagnosisRouteIsReadOnlyMonitoringRead(t *testing.T) {
source, err := os.ReadFile("alerts.go")
if err != nil {
t.Fatalf("read alerts.go: %v", err)
}
src := string(source)
required := []string{
`case path == "delivery-diagnosis" && r.Method == http.MethodGet:`,
`ensureScope(w, r, config.ScopeMonitoringRead)`,
`h.GetAlertDeliveryDiagnosis(w, r)`,
`manager.DiagnoseAlertDelivery(alertIdentifier)`,
}
for _, snippet := range required {
if !strings.Contains(src, snippet) {
t.Fatalf("alert delivery diagnosis route contract missing %q", snippet)
}
}
}
func TestContract_AlertDeliveryDiagnosisPayloadShape(t *testing.T) {
replayAt := time.Date(2026, 6, 29, 12, 30, 0, 0, time.UTC)
diagnosis := alerts.AlertDeliveryDiagnosis{
AlertIdentifier: "canonical:a1",
AlertID: "a1",
TrackingKey: "node/a1/cpu",
Status: alerts.AlertDeliveryStatusDeferred,
Reason: alerts.AlertDeliveryReasonQuietHours + ":performance",
Message: "Alert delivery is deferred by quiet-hours policy and will be replayed later.",
AlertType: "cpu",
Level: alerts.AlertLevelCritical,
ResourceID: "node-1",
ResourceName: "node-1",
NotificationsEnabled: true,
ActivationState: string(alerts.ActivationActive),
CooldownMinutes: 5,
MaxAlertsHour: 10,
RecentAlertsInHour: 2,
FlappingActive: false,
FlappingHistoryInWindow: 1,
FlappingThreshold: 3,
FlappingWindowSeconds: 300,
QuietHoursReplayAt: &replayAt,
}
payload, err := json.Marshal(diagnosis)
if err != nil {
t.Fatalf("marshal alert delivery diagnosis: %v", err)
}
body := string(payload)
for _, field := range []string{
`"alertIdentifier":"canonical:a1"`,
`"trackingKey":"node/a1/cpu"`,
`"status":"deferred"`,
`"reason":"quiet_hours:performance"`,
`"notificationsEnabled":true`,
`"cooldownMinutes":5`,
`"recentAlertsInHour":2`,
`"flappingHistoryInWindow":1`,
`"quietHoursReplayAt":"2026-06-29T12:30:00Z"`,
} {
if !strings.Contains(body, field) {
t.Fatalf("alert delivery diagnosis payload missing %s in %s", field, body)
}
}
}
func TestContract_PMGInstancesEndpointUsesUnifiedReadStatePayload(t *testing.T) {
now := time.Now().UTC()
source := models.PMGInstance{