Preserve queued recovery notifications on alert cancellation (#1350)

This commit is contained in:
rcourtman 2026-03-25 13:18:33 +00:00
parent 2ed1c3b839
commit e46239d8ac
2 changed files with 92 additions and 2 deletions

View file

@ -882,7 +882,7 @@ func (nq *NotificationQueue) CancelByAlertIDs(alertIDs []string) error {
// Query pending/sending notifications
query := `
SELECT id, alerts
SELECT id, type, alerts
FROM notification_queue
WHERE status IN ('pending', 'sending')
`
@ -900,11 +900,15 @@ func (nq *NotificationQueue) CancelByAlertIDs(alertIDs []string) error {
for rows.Next() {
var notifID string
var notifType string
var alertsJSON []byte
if err := rows.Scan(&notifID, &alertsJSON); err != nil {
if err := rows.Scan(&notifID, &notifType, &alertsJSON); err != nil {
log.Error().Err(err).Msg("Failed to scan notification for cancellation")
continue
}
if !queueTypeCancelableOnResolve(notifType) {
continue
}
var alerts []*alerts.Alert
if err := json.Unmarshal(alertsJSON, &alerts); err != nil {
@ -949,3 +953,17 @@ func (nq *NotificationQueue) CancelByAlertIDs(alertIDs []string) error {
return nil
}
func queueTypeCancelableOnResolve(notifType string) bool {
baseType, event := normalizeQueueType(notifType)
if event == eventResolved {
return false
}
switch baseType {
case "email", "webhook", "apprise":
return true
default:
return false
}
}

View file

@ -311,6 +311,78 @@ func TestCancelByAlertIDs_MatchingNotificationCancelled(t *testing.T) {
}
}
func TestCancelByAlertIDs_PreservesResolvedNotifications(t *testing.T) {
tempDir := t.TempDir()
nq, err := NewNotificationQueue(tempDir)
if err != nil {
t.Fatalf("Failed to create notification queue: %v", err)
}
defer nq.Stop()
futureRetry := time.Now().Add(1 * time.Hour)
firing := &QueuedNotification{
ID: "notif-firing",
Type: "webhook",
Status: QueueStatusPending,
MaxAttempts: 3,
Config: []byte(`{}`),
NextRetryAt: &futureRetry,
Alerts: []*alerts.Alert{{ID: "alert-1"}},
}
resolved := &QueuedNotification{
ID: "notif-resolved",
Type: "webhook_resolved",
Status: QueueStatusPending,
MaxAttempts: 3,
Config: []byte(`{}`),
NextRetryAt: &futureRetry,
Alerts: []*alerts.Alert{{ID: "alert-1"}},
}
escalation := &QueuedNotification{
ID: "notif-escalation",
Type: "webhook_escalation",
Status: QueueStatusPending,
MaxAttempts: 3,
Config: []byte(`{}`),
NextRetryAt: &futureRetry,
Alerts: []*alerts.Alert{{ID: "alert-1"}},
}
for _, notif := range []*QueuedNotification{firing, resolved, escalation} {
if err := nq.Enqueue(notif); err != nil {
t.Fatalf("Failed to enqueue %s: %v", notif.ID, err)
}
}
if err := nq.CancelByAlertIDs([]string{"alert-1"}); err != nil {
t.Fatalf("CancelByAlertIDs returned error: %v", err)
}
var statusFiring NotificationQueueStatus
if err := nq.db.QueryRow(`SELECT status FROM notification_queue WHERE id = ?`, firing.ID).Scan(&statusFiring); err != nil {
t.Fatalf("failed to read firing notification status: %v", err)
}
if statusFiring != QueueStatusCancelled {
t.Fatalf("expected firing notification to be cancelled, got %s", statusFiring)
}
var statusResolved NotificationQueueStatus
if err := nq.db.QueryRow(`SELECT status FROM notification_queue WHERE id = ?`, resolved.ID).Scan(&statusResolved); err != nil {
t.Fatalf("failed to read resolved notification status: %v", err)
}
if statusResolved != QueueStatusPending {
t.Fatalf("expected resolved notification to remain pending, got %s", statusResolved)
}
var statusEscalation NotificationQueueStatus
if err := nq.db.QueryRow(`SELECT status FROM notification_queue WHERE id = ?`, escalation.ID).Scan(&statusEscalation); err != nil {
t.Fatalf("failed to read escalation notification status: %v", err)
}
if statusEscalation != QueueStatusCancelled {
t.Fatalf("expected escalation notification to be cancelled, got %s", statusEscalation)
}
}
func TestCancelByAlertIDs_MultipleAlertsPartialMatch(t *testing.T) {
tempDir := t.TempDir()
nq, err := NewNotificationQueue(tempDir)