From e1bc14bd4a079a774b0153e6bfa83a1234e71ad6 Mon Sep 17 00:00:00 2001 From: Pulse Test Date: Fri, 28 Aug 2026 11:27:10 +0100 Subject: [PATCH] Harden alert recovery and qualification --- .../v6/internal/subsystems/alerts.md | 23 +++ .../v6/internal/subsystems/registry.json | 24 ++- internal/alerts/active_persistence.go | 46 +++-- internal/alerts/active_state_bootstrap.go | 166 ++++++++++++++++-- .../alerts/active_state_bootstrap_test.go | 69 ++++++++ .../scripts/run-alert-qualification.mjs | 72 +++++--- .../94-alert-history-real-backend.spec.ts | 98 +++++++++++ 7 files changed, 451 insertions(+), 47 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index 6a88d9dd0..f5447a8de 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -2337,3 +2337,26 @@ The Schedule surface loads the same email, Apprise, and webhook catalog as the Destinations surface. It shows disabled and deleted selections explicitly, prevents an escalation level from becoming destinationless, and explains the critical-repeat stop conditions before save. + +### Recovery authority and operator qualification fail closed + +If a lifecycle append or active-state checkpoint cannot reach SQLite, the +degradation marker is the crash-safe restart authority. The marker is an +atomically replaced, file-synced, directory-synced envelope containing the +complete active-alert snapshot; the JSON mirror remains a compatible recovery +aid but is not required for a new marker. While the marker exists, later saves +refresh that envelope and must not make SQLite authoritative again in the same +process. Bootstrap repairs SQLite from the envelope and only returns authority +after durably removing the marker. Legacy text markers may use a readable JSON +mirror, while malformed or source-less markers defer authority rather than +silently trusting a potentially stale projection. + +`tests/integration/scripts/run-alert-qualification.mjs` is the alerts-owned +operator qualification entrypoint. One run must cover crash-safe active-state +recovery, snooze and escalation policy, destination severity and exact-target +routing, a signed real HTTP webhook, durable delivery receipts and recovery, +dead-man progress and restart-gap reporting, plus the production frontend +against a managed local backend. The browser phase must exercise active-alert +snooze, delivery diagnosis, restart persistence, unsnooze, incident evidence, +history migration, virtualization, and clear tombstones without replacing the +backend APIs with route mocks. diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json index e53ce08a2..fa0976abc 100644 --- a/docs/release-control/v6/internal/subsystems/registry.json +++ b/docs/release-control/v6/internal/subsystems/registry.json @@ -2268,7 +2268,8 @@ "frontend-modern/src/utils/alertThresholdsSectionPresentation.ts", "frontend-modern/src/utils/alertWebhookPresentation.ts", "frontend-modern/src/utils/metricThresholds.ts", - "internal/proxmoxidentity/backup_identity.go" + "internal/proxmoxidentity/backup_identity.go", + "tests/integration/scripts/run-alert-qualification.mjs" ], "verification": { "allow_same_subsystem_tests": true, @@ -2527,6 +2528,27 @@ "frontend-modern/src/utils/__tests__/alertWebhookPresentation.test.ts" ] }, + { + "id": "alert-operator-qualification-runtime", + "label": "alert operator qualification proof", + "match_prefixes": [], + "match_files": [ + "tests/integration/scripts/run-alert-qualification.mjs" + ], + "allow_same_subsystem_tests": false, + "test_prefixes": [], + "exact_files": [ + "internal/alerts/active_state_bootstrap_test.go", + "internal/alerts/snooze_lifecycle_test.go", + "internal/monitoring/deadman_test.go", + "internal/notifications/delivery_receipts_test.go", + "internal/notifications/notifications_additional_test.go", + "internal/notifications/tag_routing_test.go", + "internal/notifications/webhook_signing_test.go", + "tests/integration/tests/93-alert-operator-qualification.spec.ts", + "tests/integration/tests/94-alert-history-real-backend.spec.ts" + ] + }, { "id": "alerts-runtime-support", "label": "alerts runtime support proof", diff --git a/internal/alerts/active_persistence.go b/internal/alerts/active_persistence.go index 14b4c98bd..6ecf4a131 100644 --- a/internal/alerts/active_persistence.go +++ b/internal/alerts/active_persistence.go @@ -56,16 +56,25 @@ func (m *Manager) SaveActiveAlerts() error { for attempt := 0; attempt < maxCheckpointAttempts; attempt++ { failureEpoch := m.activeStateFailureEpoch.Load() store := m.eventLogStore() + if store != nil && m.activeStateDegraded() { + alerts := m.snapshotActiveAlerts() + markerErr, mirrorErr := m.writeDegradedActiveState(alerts, errors.New("SQLite active alert state remains degraded")) + intentErr := m.saveIntentPendingSnapshot() + if err := errors.Join(markerErr, mirrorErr, intentErr); err != nil { + return fmt.Errorf("failed to checkpoint degraded active alert recovery state: %w", err) + } + return nil + } revision := int64(0) if store != nil { var err error revision, err = store.ActiveStateRevision() if err != nil { alerts := m.snapshotActiveAlerts() - if mirrorErr := m.writeActiveAlertsRecoveryMirror(alerts); mirrorErr != nil { - return errors.Join(err, mirrorErr) + markerErr, mirrorErr := m.writeDegradedActiveState(alerts, err) + if recoveryErr := errors.Join(markerErr, mirrorErr); recoveryErr != nil { + return errors.Join(err, recoveryErr) } - m.markActiveStateDegraded(err) return fmt.Errorf("failed to read SQLite active alert revision: %w", err) } } @@ -83,12 +92,18 @@ func (m *Manager) SaveActiveAlerts() error { snapshots, err := activeStateSnapshots(alerts) if err != nil { - m.markActiveStateDegraded(err) + markerErr, degradedMirrorErr := m.writeDegradedActiveState(alerts, err) + if recoveryErr := errors.Join(markerErr, degradedMirrorErr); recoveryErr != nil { + return errors.Join(err, recoveryErr) + } return err } replaced, err := store.ReplaceActiveStateIfRevision(snapshots, revision) if err != nil { - m.markActiveStateDegraded(err) + markerErr, degradedMirrorErr := m.writeDegradedActiveState(alerts, err) + if recoveryErr := errors.Join(markerErr, degradedMirrorErr); recoveryErr != nil { + return errors.Join(err, recoveryErr) + } return fmt.Errorf("failed to checkpoint active alerts in SQLite: %w", err) } if !replaced { @@ -103,7 +118,6 @@ func (m *Manager) SaveActiveAlerts() error { continue } m.activeStateAuthoritative.Store(true) - m.clearActiveStateDegraded() if err := errors.Join(mirrorErr, intentErr); err != nil { return fmt.Errorf("SQLite active alert checkpoint succeeded but recovery persistence failed: %w", err) } @@ -278,13 +292,23 @@ func (m *Manager) checkpointActiveRecoveryAfterDurableFailure(cause error) error } m.recoveryMirrorMu.Lock() alerts := m.activeRecoverySnapshot() - err := m.writeActiveAlertsRecoveryMirrorLocked(alerts) + markerErr := m.writeActiveStateDegradedRecovery(alerts, cause) + mirrorErr := m.writeActiveAlertsRecoveryMirrorLocked(alerts) m.recoveryMirrorMu.Unlock() - if err != nil { - return err + m.activeStateAuthoritative.Store(false) + return errors.Join(markerErr, mirrorErr) +} + +func (m *Manager) writeDegradedActiveState(alerts []*Alert, cause error) (markerErr, mirrorErr error) { + if m == nil { + return nil, nil } - m.markActiveStateDegraded(cause) - return nil + m.activeStateAuthoritative.Store(false) + m.recoveryMirrorMu.Lock() + defer m.recoveryMirrorMu.Unlock() + markerErr = m.writeActiveStateDegradedRecovery(alerts, cause) + mirrorErr = m.writeActiveAlertsRecoveryMirrorLocked(alerts) + return markerErr, mirrorErr } // LoadActiveAlerts restores active alerts from disk. diff --git a/internal/alerts/active_state_bootstrap.go b/internal/alerts/active_state_bootstrap.go index 43d8adea8..d3d36f0b0 100644 --- a/internal/alerts/active_state_bootstrap.go +++ b/internal/alerts/active_state_bootstrap.go @@ -1,17 +1,29 @@ package alerts import ( + "bytes" "encoding/json" "errors" "fmt" "os" "path/filepath" + "time" "github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog" "github.com/rs/zerolog/log" ) -const activeStateDegradedMarker = "active-state-sqlite-degraded" +const ( + activeStateDegradedMarker = "active-state-sqlite-degraded" + activeStateRecoverySchemaVersion = 1 +) + +type activeStateRecoveryEnvelope struct { + SchemaVersion int `json:"schemaVersion"` + RecordedAt time.Time `json:"recordedAt"` + Cause string `json:"cause,omitempty"` + Alerts []*Alert `json:"alerts"` +} // bootstrapActiveState establishes one restart authority. Existing databases // load their projection; pre-projection databases import the already-decoded @@ -36,7 +48,7 @@ func (m *Manager) bootstrapActiveState(store *eventlog.Store) bool { log.Warn().Err(err).Msg("could not inspect active alert recovery mirror") } - if initialized && !m.skipPersistedRestore && (!degraded || !recoveryFileExists || !m.activeRecoveryReadable.Load()) { + if initialized && !m.skipPersistedRestore && !degraded { snapshots, err := store.LoadActiveState() if err != nil { m.markActiveStateDegraded(err) @@ -55,10 +67,35 @@ func (m *Manager) bootstrapActiveState(store *eventlog.Store) bool { return false } m.activeStateAuthoritative.Store(true) - m.clearActiveStateDegraded() return true } + if degraded && !m.skipPersistedRestore { + recovered, selfContained, err := m.loadActiveStateDegradedRecovery() + if err != nil { + log.Error().Err(err).Msg("active alert recovery marker is invalid; SQLite authority initialization deferred") + return false + } + if selfContained { + if err := m.restoreActiveAlertSnapshots(recovered, "crash-safe SQLite recovery marker", true); err != nil { + log.Error().Err(err).Msg("failed to restore active alerts from crash-safe recovery marker") + return false + } + // The self-contained marker is the trusted source, so it can also + // replace a missing or malformed compatibility mirror. Keep the + // marker until both recovery copies and SQLite agree. + m.activeRecoveryWriteBlock.Store(false) + if err := m.writeActiveAlertsRecoveryMirror(recovered); err != nil { + log.Error().Err(err).Msg("failed to repair active alert recovery mirror from crash-safe marker") + return false + } + m.activeRecoveryReadable.Store(true) + } else if !recoveryFileExists || !m.activeRecoveryReadable.Load() { + log.Error().Msg("legacy active alert degradation marker has no readable recovery mirror; SQLite authority initialization deferred") + return false + } + } + // A new database, an explicit clean-room manager, or a prior failed // checkpoint starts from the in-memory recovery state already loaded by the // constructor. Never overwrite a new database from malformed JSON. @@ -77,8 +114,11 @@ func (m *Manager) bootstrapActiveState(store *eventlog.Store) bool { log.Error().Err(err).Msg("failed to initialize SQLite active alert authority") return false } + if err := m.clearActiveStateDegraded(); err != nil { + log.Error().Err(err).Msg("failed to durably clear repaired SQLite active alert degradation marker") + return false + } m.activeStateAuthoritative.Store(true) - m.clearActiveStateDegraded() log.Info().Int("alerts", len(snapshots)).Msg("SQLite active alert authority initialized from recovery state") return true } @@ -128,24 +168,122 @@ func (m *Manager) markActiveStateDegraded(cause error) { return } m.activeStateAuthoritative.Store(false) - if err := os.MkdirAll(m.getAlertsDir(), alertsDirPerm); err != nil { - log.Error().Err(err).Msg("failed to create alerts directory for SQLite degradation marker") - return - } message := "SQLite active alert state requires recovery from active-alerts.json\n" if cause != nil { message += cause.Error() + "\n" } - if err := os.WriteFile(m.activeStateDegradedPath(), []byte(message), alertsFilePerm); err != nil { + m.recoveryMirrorMu.Lock() + _, statErr := os.Stat(m.activeStateDegradedPath()) + var err error + if errors.Is(statErr, os.ErrNotExist) { + err = m.writeActiveStateDegradedMarker([]byte(message)) + } else if statErr != nil { + err = fmt.Errorf("inspect existing SQLite active alert degradation marker: %w", statErr) + } + m.recoveryMirrorMu.Unlock() + if err != nil { log.Error().Err(err).Msg("failed to persist SQLite active alert degradation marker") } } -func (m *Manager) clearActiveStateDegraded() { - if m == nil { - return +func (m *Manager) writeActiveStateDegradedRecovery(alerts []*Alert, cause error) error { + envelope := activeStateRecoveryEnvelope{ + SchemaVersion: activeStateRecoverySchemaVersion, + RecordedAt: time.Now().UTC(), + Alerts: alerts, } - if err := os.Remove(m.activeStateDegradedPath()); err != nil && !errors.Is(err, os.ErrNotExist) { - log.Warn().Err(err).Msg("failed to clear SQLite active alert degradation marker") + if envelope.Alerts == nil { + envelope.Alerts = []*Alert{} } + if cause != nil { + envelope.Cause = cause.Error() + } + data, err := json.Marshal(envelope) + if err != nil { + return fmt.Errorf("marshal SQLite active alert recovery marker: %w", err) + } + return m.writeActiveStateDegradedMarker(data) +} + +func (m *Manager) writeActiveStateDegradedMarker(data []byte) error { + alertsDir := m.getAlertsDir() + if err := os.MkdirAll(alertsDir, alertsDirPerm); err != nil { + return fmt.Errorf("create alerts directory for SQLite degradation marker: %w", err) + } + if err := os.Chmod(alertsDir, alertsDirPerm); err != nil { + return fmt.Errorf("set alerts directory permissions for SQLite degradation marker: %w", err) + } + tmp, err := os.CreateTemp(alertsDir, ".active-state-sqlite-degraded-*.tmp") + if err != nil { + return fmt.Errorf("create SQLite degradation marker temp file: %w", err) + } + tmpName := tmp.Name() + cleanup := true + defer func() { + if cleanup { + _ = os.Remove(tmpName) + } + }() + if _, err := tmp.Write(data); err != nil { + _ = tmp.Close() + return fmt.Errorf("write SQLite degradation marker temp file: %w", err) + } + if err := tmp.Chmod(alertsFilePerm); err != nil { + _ = tmp.Close() + return fmt.Errorf("set SQLite degradation marker permissions: %w", err) + } + if err := tmp.Sync(); err != nil { + _ = tmp.Close() + return fmt.Errorf("sync SQLite degradation marker temp file: %w", err) + } + if err := tmp.Close(); err != nil { + return fmt.Errorf("close SQLite degradation marker temp file: %w", err) + } + if err := replaceActiveAlertsFile(tmpName, m.activeStateDegradedPath()); err != nil { + return fmt.Errorf("replace SQLite degradation marker: %w", err) + } + cleanup = false + if err := syncActiveAlertsDirectory(alertsDir); err != nil { + return fmt.Errorf("sync alerts directory after SQLite degradation marker: %w", err) + } + return nil +} + +func (m *Manager) loadActiveStateDegradedRecovery() ([]*Alert, bool, error) { + data, err := readLimitedRegularFile(m.activeStateDegradedPath(), maxActiveAlertsFileSizeBytes) + if err != nil { + return nil, false, fmt.Errorf("read SQLite active alert recovery marker: %w", err) + } + trimmed := bytes.TrimSpace(data) + if len(trimmed) == 0 || trimmed[0] != '{' { + return nil, false, nil + } + var envelope activeStateRecoveryEnvelope + if err := json.Unmarshal(trimmed, &envelope); err != nil { + return nil, false, fmt.Errorf("decode SQLite active alert recovery marker: %w", err) + } + if envelope.SchemaVersion != activeStateRecoverySchemaVersion { + return nil, false, fmt.Errorf("unsupported SQLite active alert recovery marker schema %d", envelope.SchemaVersion) + } + if envelope.Alerts == nil { + return nil, false, fmt.Errorf("SQLite active alert recovery marker has no alert snapshot") + } + return envelope.Alerts, true, nil +} + +func (m *Manager) clearActiveStateDegraded() error { + if m == nil { + return nil + } + err := os.Remove(m.activeStateDegradedPath()) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("remove SQLite active alert degradation marker: %w", err) + } + if err := syncActiveAlertsDirectory(m.getAlertsDir()); err != nil { + return fmt.Errorf("sync alerts directory after clearing SQLite degradation marker: %w", err) + } + return nil } diff --git a/internal/alerts/active_state_bootstrap_test.go b/internal/alerts/active_state_bootstrap_test.go index f6e471f9f..7e24933cd 100644 --- a/internal/alerts/active_state_bootstrap_test.go +++ b/internal/alerts/active_state_bootstrap_test.go @@ -134,9 +134,44 @@ func TestDurableLifecycleFailureSynchronouslyCheckpointsRecoveryMirror(t *testin if _, err := os.Stat(filepath.Join(dataDir, "alerts", activeStateDegradedMarker)); err != nil { t.Fatalf("degraded marker missing after failed fire: %v", err) } + markerData, err := os.ReadFile(filepath.Join(dataDir, "alerts", activeStateDegradedMarker)) + if err != nil { + t.Fatalf("read degraded recovery marker: %v", err) + } + var envelope activeStateRecoveryEnvelope + if err := json.Unmarshal(markerData, &envelope); err != nil { + t.Fatalf("decode degraded recovery marker: %v", err) + } + if envelope.SchemaVersion != activeStateRecoverySchemaVersion || len(envelope.Alerts) != 1 || envelope.Alerts[0].ID != alert.ID { + t.Fatalf("degraded recovery marker = %+v", envelope) + } m.SetEventLog(nil) m.Stop() + if err := os.WriteFile( + filepath.Join(dataDir, "alerts", "active-alerts.json"), + []byte(`{"truncated":`), + alertsFilePerm, + ); err != nil { + t.Fatalf("corrupt recovery mirror to isolate marker recovery: %v", err) + } + restarted := NewManagerWithDataDir(dataDir) + t.Cleanup(restarted.Stop) + restarted.EnableEventLog() + restarted.mu.RLock() + _, restored := testLookupActiveAlert(t, restarted, alert.ID) + restarted.mu.RUnlock() + if !restored { + t.Fatal("self-contained degraded marker did not restore fired alert over malformed JSON mirror") + } + repairedMirror, err := os.ReadFile(filepath.Join(dataDir, "alerts", "active-alerts.json")) + if err != nil { + t.Fatalf("read repaired recovery mirror: %v", err) + } + var repaired []*Alert + if err := json.Unmarshal(repairedMirror, &repaired); err != nil || len(repaired) != 1 || repaired[0].ID != alert.ID { + t.Fatalf("repaired recovery mirror = %+v, error = %v", repaired, err) + } }) t.Run("resolved before restart", func(t *testing.T) { @@ -167,9 +202,23 @@ func TestDurableLifecycleFailureSynchronouslyCheckpointsRecoveryMirror(t *testin if len(recovered) != 0 { t.Fatalf("failed resolution remained in recovery mirror: %+v", recovered) } + markerData, err := os.ReadFile(filepath.Join(dataDir, "alerts", activeStateDegradedMarker)) + if err != nil { + t.Fatalf("read degraded recovery marker: %v", err) + } + var envelope activeStateRecoveryEnvelope + if err := json.Unmarshal(markerData, &envelope); err != nil { + t.Fatalf("decode degraded recovery marker: %v", err) + } + if envelope.Alerts == nil || len(envelope.Alerts) != 0 { + t.Fatalf("resolved degraded recovery marker = %+v", envelope.Alerts) + } m.SetEventLog(nil) m.Stop() + if err := os.Remove(filepath.Join(dataDir, "alerts", "active-alerts.json")); err != nil { + t.Fatalf("remove recovery mirror to isolate marker recovery: %v", err) + } restarted := NewManagerWithDataDir(dataDir) t.Cleanup(restarted.Stop) restarted.EnableEventLog() @@ -182,6 +231,26 @@ func TestDurableLifecycleFailureSynchronouslyCheckpointsRecoveryMirror(t *testin }) } +func TestDurableLifecycleFailureSurfacesRecoveryMarkerFailure(t *testing.T) { + dataDir := t.TempDir() + m := NewManagerWithDataDir(dataDir) + t.Cleanup(m.Stop) + marker := filepath.Join(dataDir, "alerts", activeStateDegradedMarker) + if err := os.MkdirAll(marker, alertsDirPerm); err != nil { + t.Fatal(err) + } + + alert := durableRestoreAlert(time.Now().UTC()) + m.setActiveRecoveryAlert(alert, alert.ID) + err := m.checkpointActiveRecoveryAfterDurableFailure(os.ErrClosed) + if err == nil { + t.Fatal("checkpoint did not surface an unwritable recovery marker") + } + if _, statErr := os.Stat(filepath.Join(dataDir, "alerts", "active-alerts.json")); statErr != nil { + t.Fatalf("best-effort recovery mirror was not written after marker failure: %v", statErr) + } +} + func TestDegradedMarkerRepairsSQLiteFromRecoveryMirror(t *testing.T) { dataDir := t.TempDir() original := durableRestoreAlert(time.Now().Add(-2 * time.Hour).UTC()) diff --git a/tests/integration/scripts/run-alert-qualification.mjs b/tests/integration/scripts/run-alert-qualification.mjs index ea28e9951..f735dbdc8 100644 --- a/tests/integration/scripts/run-alert-qualification.mjs +++ b/tests/integration/scripts/run-alert-qualification.mjs @@ -74,31 +74,61 @@ await fs.writeFile(activeFixturePath, `${JSON.stringify(activeAlerts, null, 2)}\ let exitCode = 1; try { + const runtimeQualificationPattern = [ + 'TestDurableLifecycleFailureSynchronouslyCheckpointsRecoveryMirror', + 'TestDurableLifecycleFailureSurfacesRecoveryMarkerFailure', + 'TestDegradedMarkerRepairsSQLiteFromRecoveryMirror', + 'TestAlertSnoozeIsDurableAndEnforcedByDeliveryPolicy', + 'TestSnoozeExpiryResumesEscalationWithoutReplayingMissedLevels', + 'TestDeliveryReceiptPersistsAndIsClearedAfterRecovery', + 'TestSendResolvedWebhookHTTP', + 'TestBuildNotificationDeliveryJobsRoutesGroupedAlertsByDestinationSeverity', + 'TestBuildNotificationDeliveryJobsTargetsExactEscalationDestinations', + 'TestWebhookDeliveryCarriesSignatureAndEventID', + 'TestDeadManRunCycleSendsHealthySignalAndPersistsProgress', + 'TestDeadManRestartGapIsReportedExternallyAndRecordedInAlertHistory', + ].join('|'); exitCode = await run( - process.execPath, + 'go', [ - './scripts/run-playwright.mjs', - 'tests/93-alert-operator-qualification.spec.ts', - 'tests/94-alert-history-real-backend.spec.ts', - '--project=chromium', + 'test', + './internal/alerts', + './internal/notifications', + './internal/monitoring', + '-run', + `^(${runtimeQualificationPattern})$`, + '-count=1', ], - { - cwd: integrationRoot, - env: { - ...process.env, - PULSE_E2E_USE_LOCAL_BACKEND: '1', - PULSE_MOCK_MODE: 'false', - PULSE_E2E_ALERT_HISTORY_QUALIFICATION: '1', - PULSE_E2E_ALERT_HISTORY_FIXTURE: historyFixturePath, - PULSE_E2E_ACTIVE_ALERTS_FIXTURE: activeFixturePath, - PULSE_E2E_LOCAL_BACKEND_BINARY: binaryPath, - PULSE_E2E_RUN_ID: runId, - PULSE_E2E_RUNTIME_STATE_PATH: runtimeStatePath, - PULSE_E2E_REPORT_DIR: path.join(runRoot, 'playwright-report'), - PULSE_E2E_RESULTS_DIR: path.join(runRoot, 'test-results'), - }, - }, + { cwd: repoRoot, env: process.env }, ); + + if (exitCode === 0) { + exitCode = await run( + process.execPath, + [ + './scripts/run-playwright.mjs', + 'tests/93-alert-operator-qualification.spec.ts', + 'tests/94-alert-history-real-backend.spec.ts', + '--project=chromium', + ], + { + cwd: integrationRoot, + env: { + ...process.env, + PULSE_E2E_USE_LOCAL_BACKEND: '1', + PULSE_MOCK_MODE: 'false', + PULSE_E2E_ALERT_HISTORY_QUALIFICATION: '1', + PULSE_E2E_ALERT_HISTORY_FIXTURE: historyFixturePath, + PULSE_E2E_ACTIVE_ALERTS_FIXTURE: activeFixturePath, + PULSE_E2E_LOCAL_BACKEND_BINARY: binaryPath, + PULSE_E2E_RUN_ID: runId, + PULSE_E2E_RUNTIME_STATE_PATH: runtimeStatePath, + PULSE_E2E_REPORT_DIR: path.join(runRoot, 'playwright-report'), + PULSE_E2E_RESULTS_DIR: path.join(runRoot, 'test-results'), + }, + }, + ); + } } finally { await fs.rm(fixtureRoot, { recursive: true, force: true }); await fs.rm(runtimeStatePath, { force: true }); diff --git a/tests/integration/tests/94-alert-history-real-backend.spec.ts b/tests/integration/tests/94-alert-history-real-backend.spec.ts index 25bddb6ab..2d74bcbbd 100644 --- a/tests/integration/tests/94-alert-history-real-backend.spec.ts +++ b/tests/integration/tests/94-alert-history-real-backend.spec.ts @@ -61,6 +61,13 @@ type HistoryAlert = { type: string; startTime: string; lastSeen: string; + operationalRecord?: { + state?: string; + suppression?: { + reason?: string; + expiresAt?: string; + }; + }; }; async function readHistory( @@ -74,6 +81,24 @@ async function readHistory( return (await response.json()) as HistoryAlert[]; } +async function readActiveAlert( + page: import('@playwright/test').Page, + alertIdentifier: string, +): Promise { + const response = await apiRequest(page, '/api/alerts/active'); + expect( + response.ok(), + `active alerts API returned ${response.status()}`, + ).toBeTruthy(); + const alerts = (await response.json()) as HistoryAlert[]; + const alert = alerts.find((candidate) => candidate.id === alertIdentifier); + expect( + alert, + `active alert ${alertIdentifier} was not returned`, + ).toBeTruthy(); + return alert!; +} + test.describe('Real backend alert history qualification', () => { test.skip( !qualificationEnabled, @@ -177,6 +202,79 @@ test.describe('Real backend alert history qualification', () => { true, ); + const snoozedUntil = new Date(Date.now() + 60 * 60_000).toISOString(); + const snoozeResponse = await apiRequest(page, '/api/alerts/snooze', { + method: 'POST', + data: { alertIdentifier: ACTIVE_ALERT_ID, until: snoozedUntil }, + }); + expect( + snoozeResponse.ok(), + `snooze API returned ${snoozeResponse.status()}: ${await snoozeResponse.text()}`, + ).toBeTruthy(); + let controlledAlert = await readActiveAlert(page, ACTIVE_ALERT_ID); + expect(controlledAlert.operationalRecord?.state).toBe('suppressed'); + expect(controlledAlert.operationalRecord?.suppression?.reason).toBe( + 'user_snooze', + ); + expect( + Date.parse(controlledAlert.operationalRecord!.suppression!.expiresAt!), + ).toBe(Date.parse(snoozedUntil)); + + const diagnosisResponse = await apiRequest( + page, + `/api/alerts/delivery-diagnosis?alertIdentifier=${encodeURIComponent(ACTIVE_ALERT_ID)}`, + ); + expect( + diagnosisResponse.ok(), + `delivery diagnosis API returned ${diagnosisResponse.status()}`, + ).toBeTruthy(); + const diagnosis = (await diagnosisResponse.json()) as { + alertIdentifier?: string; + status?: string; + reason?: string; + suppressedUntil?: string; + }; + expect(diagnosis).toMatchObject({ + alertIdentifier: ACTIVE_ALERT_ID, + status: 'suppressed', + reason: 'snoozed', + }); + expect(Date.parse(diagnosis.suppressedUntil!)).toBe( + Date.parse(snoozedUntil), + ); + + await restartManagedLocalBackend(); + await ensureAuthenticated(page); + controlledAlert = await readActiveAlert(page, ACTIVE_ALERT_ID); + expect(controlledAlert.operationalRecord?.state).toBe('suppressed'); + expect(controlledAlert.operationalRecord?.suppression?.reason).toBe( + 'user_snooze', + ); + + const unsnoozeResponse = await apiRequest(page, '/api/alerts/unsnooze', { + method: 'POST', + data: { alertIdentifier: ACTIVE_ALERT_ID }, + }); + expect( + unsnoozeResponse.ok(), + `unsnooze API returned ${unsnoozeResponse.status()}: ${await unsnoozeResponse.text()}`, + ).toBeTruthy(); + controlledAlert = await readActiveAlert(page, ACTIVE_ALERT_ID); + expect(controlledAlert.operationalRecord?.state).toBe('open'); + expect(controlledAlert.operationalRecord?.suppression).toBeUndefined(); + + const controlTimelineResponse = await apiRequest( + page, + `/api/alerts/incidents?alertIdentifier=${encodeURIComponent(ACTIVE_ALERT_ID)}&started_at=${encodeURIComponent(activeAlert!.startTime)}`, + ); + expect(controlTimelineResponse.ok()).toBeTruthy(); + const controlTimeline = (await controlTimelineResponse.json()) as { + events?: Array<{ type: string }>; + } | null; + expect(controlTimeline?.events?.map((event) => event.type)).toEqual( + expect.arrayContaining(['alert_snoozed', 'alert_unsnoozed']), + ); + await page.goto('/alerts/overview', { waitUntil: 'domcontentloaded' }); await expect(page.getByText('Active Overlay Node').first()).toBeVisible(); const overviewTimelineRequests: string[] = [];