Harden alert recovery and qualification

docs/release-control/v6/internal/subsystems/alerts.md

@@ -2337,3 +2337,26 @@ The Schedule surface loads the same email, Apprise, and webhook catalog as the
 Destinations surface. It shows disabled and deleted selections explicitly,
 prevents an escalation level from becoming destinationless, and explains the
 critical-repeat stop conditions before save.
+
+### Recovery authority and operator qualification fail closed
+
+If a lifecycle append or active-state checkpoint cannot reach SQLite, the
+degradation marker is the crash-safe restart authority. The marker is an
+atomically replaced, file-synced, directory-synced envelope containing the
+complete active-alert snapshot; the JSON mirror remains a compatible recovery
+aid but is not required for a new marker. While the marker exists, later saves
+refresh that envelope and must not make SQLite authoritative again in the same
+process. Bootstrap repairs SQLite from the envelope and only returns authority
+after durably removing the marker. Legacy text markers may use a readable JSON
+mirror, while malformed or source-less markers defer authority rather than
+silently trusting a potentially stale projection.
+
+`tests/integration/scripts/run-alert-qualification.mjs` is the alerts-owned
+operator qualification entrypoint. One run must cover crash-safe active-state
+recovery, snooze and escalation policy, destination severity and exact-target
+routing, a signed real HTTP webhook, durable delivery receipts and recovery,
+dead-man progress and restart-gap reporting, plus the production frontend
+against a managed local backend. The browser phase must exercise active-alert
+snooze, delivery diagnosis, restart persistence, unsnooze, incident evidence,
+history migration, virtualization, and clear tombstones without replacing the
+backend APIs with route mocks.

docs/release-control/v6/internal/subsystems/registry.json

@@ -2268,7 +2268,8 @@
         "frontend-modern/src/utils/alertThresholdsSectionPresentation.ts",
         "frontend-modern/src/utils/alertWebhookPresentation.ts",
         "frontend-modern/src/utils/metricThresholds.ts",
-        "internal/proxmoxidentity/backup_identity.go"
+        "internal/proxmoxidentity/backup_identity.go",
+        "tests/integration/scripts/run-alert-qualification.mjs"
       ],
       "verification": {
         "allow_same_subsystem_tests": true,
@@ -2527,6 +2528,27 @@
               "frontend-modern/src/utils/__tests__/alertWebhookPresentation.test.ts"
             ]
           },
+          {
+            "id": "alert-operator-qualification-runtime",
+            "label": "alert operator qualification proof",
+            "match_prefixes": [],
+            "match_files": [
+              "tests/integration/scripts/run-alert-qualification.mjs"
+            ],
+            "allow_same_subsystem_tests": false,
+            "test_prefixes": [],
+            "exact_files": [
+              "internal/alerts/active_state_bootstrap_test.go",
+              "internal/alerts/snooze_lifecycle_test.go",
+              "internal/monitoring/deadman_test.go",
+              "internal/notifications/delivery_receipts_test.go",
+              "internal/notifications/notifications_additional_test.go",
+              "internal/notifications/tag_routing_test.go",
+              "internal/notifications/webhook_signing_test.go",
+              "tests/integration/tests/93-alert-operator-qualification.spec.ts",
+              "tests/integration/tests/94-alert-history-real-backend.spec.ts"
+            ]
+          },
           {
             "id": "alerts-runtime-support",
             "label": "alerts runtime support proof",

internal/alerts/active_persistence.go

@@ -56,16 +56,25 @@ func (m *Manager) SaveActiveAlerts() error {
 	for attempt := 0; attempt < maxCheckpointAttempts; attempt++ {
 		failureEpoch := m.activeStateFailureEpoch.Load()
 		store := m.eventLogStore()
+		if store != nil && m.activeStateDegraded() {
+			alerts := m.snapshotActiveAlerts()
+			markerErr, mirrorErr := m.writeDegradedActiveState(alerts, errors.New("SQLite active alert state remains degraded"))
+			intentErr := m.saveIntentPendingSnapshot()
+			if err := errors.Join(markerErr, mirrorErr, intentErr); err != nil {
+				return fmt.Errorf("failed to checkpoint degraded active alert recovery state: %w", err)
+			}
+			return nil
+		}
 		revision := int64(0)
 		if store != nil {
 			var err error
 			revision, err = store.ActiveStateRevision()
 			if err != nil {
 				alerts := m.snapshotActiveAlerts()
-				if mirrorErr := m.writeActiveAlertsRecoveryMirror(alerts); mirrorErr != nil {
-					return errors.Join(err, mirrorErr)
+				markerErr, mirrorErr := m.writeDegradedActiveState(alerts, err)
+				if recoveryErr := errors.Join(markerErr, mirrorErr); recoveryErr != nil {
+					return errors.Join(err, recoveryErr)
 				}
-				m.markActiveStateDegraded(err)
 				return fmt.Errorf("failed to read SQLite active alert revision: %w", err)
 			}
 		}
@@ -83,12 +92,18 @@ func (m *Manager) SaveActiveAlerts() error {
 
 		snapshots, err := activeStateSnapshots(alerts)
 		if err != nil {
-			m.markActiveStateDegraded(err)
+			markerErr, degradedMirrorErr := m.writeDegradedActiveState(alerts, err)
+			if recoveryErr := errors.Join(markerErr, degradedMirrorErr); recoveryErr != nil {
+				return errors.Join(err, recoveryErr)
+			}
 			return err
 		}
 		replaced, err := store.ReplaceActiveStateIfRevision(snapshots, revision)
 		if err != nil {
-			m.markActiveStateDegraded(err)
+			markerErr, degradedMirrorErr := m.writeDegradedActiveState(alerts, err)
+			if recoveryErr := errors.Join(markerErr, degradedMirrorErr); recoveryErr != nil {
+				return errors.Join(err, recoveryErr)
+			}
 			return fmt.Errorf("failed to checkpoint active alerts in SQLite: %w", err)
 		}
 		if !replaced {
@@ -103,7 +118,6 @@ func (m *Manager) SaveActiveAlerts() error {
 			continue
 		}
 		m.activeStateAuthoritative.Store(true)
-		m.clearActiveStateDegraded()
 		if err := errors.Join(mirrorErr, intentErr); err != nil {
 			return fmt.Errorf("SQLite active alert checkpoint succeeded but recovery persistence failed: %w", err)
 		}
@@ -278,13 +292,23 @@ func (m *Manager) checkpointActiveRecoveryAfterDurableFailure(cause error) error
 	}
 	m.recoveryMirrorMu.Lock()
 	alerts := m.activeRecoverySnapshot()
-	err := m.writeActiveAlertsRecoveryMirrorLocked(alerts)
+	markerErr := m.writeActiveStateDegradedRecovery(alerts, cause)
+	mirrorErr := m.writeActiveAlertsRecoveryMirrorLocked(alerts)
 	m.recoveryMirrorMu.Unlock()
-	if err != nil {
-		return err
+	m.activeStateAuthoritative.Store(false)
+	return errors.Join(markerErr, mirrorErr)
+}
+
+func (m *Manager) writeDegradedActiveState(alerts []*Alert, cause error) (markerErr, mirrorErr error) {
+	if m == nil {
+		return nil, nil
 	}
-	m.markActiveStateDegraded(cause)
-	return nil
+	m.activeStateAuthoritative.Store(false)
+	m.recoveryMirrorMu.Lock()
+	defer m.recoveryMirrorMu.Unlock()
+	markerErr = m.writeActiveStateDegradedRecovery(alerts, cause)
+	mirrorErr = m.writeActiveAlertsRecoveryMirrorLocked(alerts)
+	return markerErr, mirrorErr
 }
 
 // LoadActiveAlerts restores active alerts from disk.

internal/alerts/active_state_bootstrap.go

@@ -1,17 +1,29 @@
 package alerts
 
 import (
+	"bytes"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
 	"github.com/rs/zerolog/log"
 )
 
-const activeStateDegradedMarker = "active-state-sqlite-degraded"
+const (
+	activeStateDegradedMarker        = "active-state-sqlite-degraded"
+	activeStateRecoverySchemaVersion = 1
+)
+
+type activeStateRecoveryEnvelope struct {
+	SchemaVersion int       `json:"schemaVersion"`
+	RecordedAt    time.Time `json:"recordedAt"`
+	Cause         string    `json:"cause,omitempty"`
+	Alerts        []*Alert  `json:"alerts"`
+}
 
 // bootstrapActiveState establishes one restart authority. Existing databases
 // load their projection; pre-projection databases import the already-decoded
@@ -36,7 +48,7 @@ func (m *Manager) bootstrapActiveState(store *eventlog.Store) bool {
 		log.Warn().Err(err).Msg("could not inspect active alert recovery mirror")
 	}
 
-	if initialized && !m.skipPersistedRestore && (!degraded || !recoveryFileExists || !m.activeRecoveryReadable.Load()) {
+	if initialized && !m.skipPersistedRestore && !degraded {
 		snapshots, err := store.LoadActiveState()
 		if err != nil {
 			m.markActiveStateDegraded(err)
@@ -55,10 +67,35 @@ func (m *Manager) bootstrapActiveState(store *eventlog.Store) bool {
 			return false
 		}
 		m.activeStateAuthoritative.Store(true)
-		m.clearActiveStateDegraded()
 		return true
 	}
 
+	if degraded && !m.skipPersistedRestore {
+		recovered, selfContained, err := m.loadActiveStateDegradedRecovery()
+		if err != nil {
+			log.Error().Err(err).Msg("active alert recovery marker is invalid; SQLite authority initialization deferred")
+			return false
+		}
+		if selfContained {
+			if err := m.restoreActiveAlertSnapshots(recovered, "crash-safe SQLite recovery marker", true); err != nil {
+				log.Error().Err(err).Msg("failed to restore active alerts from crash-safe recovery marker")
+				return false
+			}
+			// The self-contained marker is the trusted source, so it can also
+			// replace a missing or malformed compatibility mirror. Keep the
+			// marker until both recovery copies and SQLite agree.
+			m.activeRecoveryWriteBlock.Store(false)
+			if err := m.writeActiveAlertsRecoveryMirror(recovered); err != nil {
+				log.Error().Err(err).Msg("failed to repair active alert recovery mirror from crash-safe marker")
+				return false
+			}
+			m.activeRecoveryReadable.Store(true)
+		} else if !recoveryFileExists || !m.activeRecoveryReadable.Load() {
+			log.Error().Msg("legacy active alert degradation marker has no readable recovery mirror; SQLite authority initialization deferred")
+			return false
+		}
+	}
+
 	// A new database, an explicit clean-room manager, or a prior failed
 	// checkpoint starts from the in-memory recovery state already loaded by the
 	// constructor. Never overwrite a new database from malformed JSON.
@@ -77,8 +114,11 @@ func (m *Manager) bootstrapActiveState(store *eventlog.Store) bool {
 		log.Error().Err(err).Msg("failed to initialize SQLite active alert authority")
 		return false
 	}
+	if err := m.clearActiveStateDegraded(); err != nil {
+		log.Error().Err(err).Msg("failed to durably clear repaired SQLite active alert degradation marker")
+		return false
+	}
 	m.activeStateAuthoritative.Store(true)
-	m.clearActiveStateDegraded()
 	log.Info().Int("alerts", len(snapshots)).Msg("SQLite active alert authority initialized from recovery state")
 	return true
 }
@@ -128,24 +168,122 @@ func (m *Manager) markActiveStateDegraded(cause error) {
 		return
 	}
 	m.activeStateAuthoritative.Store(false)
-	if err := os.MkdirAll(m.getAlertsDir(), alertsDirPerm); err != nil {
-		log.Error().Err(err).Msg("failed to create alerts directory for SQLite degradation marker")
-		return
-	}
 	message := "SQLite active alert state requires recovery from active-alerts.json\n"
 	if cause != nil {
 		message += cause.Error() + "\n"
 	}
-	if err := os.WriteFile(m.activeStateDegradedPath(), []byte(message), alertsFilePerm); err != nil {
+	m.recoveryMirrorMu.Lock()
+	_, statErr := os.Stat(m.activeStateDegradedPath())
+	var err error
+	if errors.Is(statErr, os.ErrNotExist) {
+		err = m.writeActiveStateDegradedMarker([]byte(message))
+	} else if statErr != nil {
+		err = fmt.Errorf("inspect existing SQLite active alert degradation marker: %w", statErr)
+	}
+	m.recoveryMirrorMu.Unlock()
+	if err != nil {
 		log.Error().Err(err).Msg("failed to persist SQLite active alert degradation marker")
 	}
 }
 
-func (m *Manager) clearActiveStateDegraded() {
-	if m == nil {
-		return
+func (m *Manager) writeActiveStateDegradedRecovery(alerts []*Alert, cause error) error {
+	envelope := activeStateRecoveryEnvelope{
+		SchemaVersion: activeStateRecoverySchemaVersion,
+		RecordedAt:    time.Now().UTC(),
+		Alerts:        alerts,
 	}
-	if err := os.Remove(m.activeStateDegradedPath()); err != nil && !errors.Is(err, os.ErrNotExist) {
-		log.Warn().Err(err).Msg("failed to clear SQLite active alert degradation marker")
+	if envelope.Alerts == nil {
+		envelope.Alerts = []*Alert{}
 	}
+	if cause != nil {
+		envelope.Cause = cause.Error()
+	}
+	data, err := json.Marshal(envelope)
+	if err != nil {
+		return fmt.Errorf("marshal SQLite active alert recovery marker: %w", err)
+	}
+	return m.writeActiveStateDegradedMarker(data)
+}
+
+func (m *Manager) writeActiveStateDegradedMarker(data []byte) error {
+	alertsDir := m.getAlertsDir()
+	if err := os.MkdirAll(alertsDir, alertsDirPerm); err != nil {
+		return fmt.Errorf("create alerts directory for SQLite degradation marker: %w", err)
+	}
+	if err := os.Chmod(alertsDir, alertsDirPerm); err != nil {
+		return fmt.Errorf("set alerts directory permissions for SQLite degradation marker: %w", err)
+	}
+	tmp, err := os.CreateTemp(alertsDir, ".active-state-sqlite-degraded-*.tmp")
+	if err != nil {
+		return fmt.Errorf("create SQLite degradation marker temp file: %w", err)
+	}
+	tmpName := tmp.Name()
+	cleanup := true
+	defer func() {
+		if cleanup {
+			_ = os.Remove(tmpName)
+		}
+	}()
+	if _, err := tmp.Write(data); err != nil {
+		_ = tmp.Close()
+		return fmt.Errorf("write SQLite degradation marker temp file: %w", err)
+	}
+	if err := tmp.Chmod(alertsFilePerm); err != nil {
+		_ = tmp.Close()
+		return fmt.Errorf("set SQLite degradation marker permissions: %w", err)
+	}
+	if err := tmp.Sync(); err != nil {
+		_ = tmp.Close()
+		return fmt.Errorf("sync SQLite degradation marker temp file: %w", err)
+	}
+	if err := tmp.Close(); err != nil {
+		return fmt.Errorf("close SQLite degradation marker temp file: %w", err)
+	}
+	if err := replaceActiveAlertsFile(tmpName, m.activeStateDegradedPath()); err != nil {
+		return fmt.Errorf("replace SQLite degradation marker: %w", err)
+	}
+	cleanup = false
+	if err := syncActiveAlertsDirectory(alertsDir); err != nil {
+		return fmt.Errorf("sync alerts directory after SQLite degradation marker: %w", err)
+	}
+	return nil
+}
+
+func (m *Manager) loadActiveStateDegradedRecovery() ([]*Alert, bool, error) {
+	data, err := readLimitedRegularFile(m.activeStateDegradedPath(), maxActiveAlertsFileSizeBytes)
+	if err != nil {
+		return nil, false, fmt.Errorf("read SQLite active alert recovery marker: %w", err)
+	}
+	trimmed := bytes.TrimSpace(data)
+	if len(trimmed) == 0 || trimmed[0] != '{' {
+		return nil, false, nil
+	}
+	var envelope activeStateRecoveryEnvelope
+	if err := json.Unmarshal(trimmed, &envelope); err != nil {
+		return nil, false, fmt.Errorf("decode SQLite active alert recovery marker: %w", err)
+	}
+	if envelope.SchemaVersion != activeStateRecoverySchemaVersion {
+		return nil, false, fmt.Errorf("unsupported SQLite active alert recovery marker schema %d", envelope.SchemaVersion)
+	}
+	if envelope.Alerts == nil {
+		return nil, false, fmt.Errorf("SQLite active alert recovery marker has no alert snapshot")
+	}
+	return envelope.Alerts, true, nil
+}
+
+func (m *Manager) clearActiveStateDegraded() error {
+	if m == nil {
+		return nil
+	}
+	err := os.Remove(m.activeStateDegradedPath())
+	if errors.Is(err, os.ErrNotExist) {
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("remove SQLite active alert degradation marker: %w", err)
+	}
+	if err := syncActiveAlertsDirectory(m.getAlertsDir()); err != nil {
+		return fmt.Errorf("sync alerts directory after clearing SQLite degradation marker: %w", err)
+	}
+	return nil
 }

internal/alerts/active_state_bootstrap_test.go

@@ -134,9 +134,44 @@ func TestDurableLifecycleFailureSynchronouslyCheckpointsRecoveryMirror(t *testin
 		if _, err := os.Stat(filepath.Join(dataDir, "alerts", activeStateDegradedMarker)); err != nil {
 			t.Fatalf("degraded marker missing after failed fire: %v", err)
 		}
+		markerData, err := os.ReadFile(filepath.Join(dataDir, "alerts", activeStateDegradedMarker))
+		if err != nil {
+			t.Fatalf("read degraded recovery marker: %v", err)
+		}
+		var envelope activeStateRecoveryEnvelope
+		if err := json.Unmarshal(markerData, &envelope); err != nil {
+			t.Fatalf("decode degraded recovery marker: %v", err)
+		}
+		if envelope.SchemaVersion != activeStateRecoverySchemaVersion || len(envelope.Alerts) != 1 || envelope.Alerts[0].ID != alert.ID {
+			t.Fatalf("degraded recovery marker = %+v", envelope)
+		}
 
 		m.SetEventLog(nil)
 		m.Stop()
+		if err := os.WriteFile(
+			filepath.Join(dataDir, "alerts", "active-alerts.json"),
+			[]byte(`{"truncated":`),
+			alertsFilePerm,
+		); err != nil {
+			t.Fatalf("corrupt recovery mirror to isolate marker recovery: %v", err)
+		}
+		restarted := NewManagerWithDataDir(dataDir)
+		t.Cleanup(restarted.Stop)
+		restarted.EnableEventLog()
+		restarted.mu.RLock()
+		_, restored := testLookupActiveAlert(t, restarted, alert.ID)
+		restarted.mu.RUnlock()
+		if !restored {
+			t.Fatal("self-contained degraded marker did not restore fired alert over malformed JSON mirror")
+		}
+		repairedMirror, err := os.ReadFile(filepath.Join(dataDir, "alerts", "active-alerts.json"))
+		if err != nil {
+			t.Fatalf("read repaired recovery mirror: %v", err)
+		}
+		var repaired []*Alert
+		if err := json.Unmarshal(repairedMirror, &repaired); err != nil || len(repaired) != 1 || repaired[0].ID != alert.ID {
+			t.Fatalf("repaired recovery mirror = %+v, error = %v", repaired, err)
+		}
 	})
 
 	t.Run("resolved before restart", func(t *testing.T) {
@@ -167,9 +202,23 @@ func TestDurableLifecycleFailureSynchronouslyCheckpointsRecoveryMirror(t *testin
 		if len(recovered) != 0 {
 			t.Fatalf("failed resolution remained in recovery mirror: %+v", recovered)
 		}
+		markerData, err := os.ReadFile(filepath.Join(dataDir, "alerts", activeStateDegradedMarker))
+		if err != nil {
+			t.Fatalf("read degraded recovery marker: %v", err)
+		}
+		var envelope activeStateRecoveryEnvelope
+		if err := json.Unmarshal(markerData, &envelope); err != nil {
+			t.Fatalf("decode degraded recovery marker: %v", err)
+		}
+		if envelope.Alerts == nil || len(envelope.Alerts) != 0 {
+			t.Fatalf("resolved degraded recovery marker = %+v", envelope.Alerts)
+		}
 
 		m.SetEventLog(nil)
 		m.Stop()
+		if err := os.Remove(filepath.Join(dataDir, "alerts", "active-alerts.json")); err != nil {
+			t.Fatalf("remove recovery mirror to isolate marker recovery: %v", err)
+		}
 		restarted := NewManagerWithDataDir(dataDir)
 		t.Cleanup(restarted.Stop)
 		restarted.EnableEventLog()
@@ -182,6 +231,26 @@ func TestDurableLifecycleFailureSynchronouslyCheckpointsRecoveryMirror(t *testin
 	})
 }
 
+func TestDurableLifecycleFailureSurfacesRecoveryMarkerFailure(t *testing.T) {
+	dataDir := t.TempDir()
+	m := NewManagerWithDataDir(dataDir)
+	t.Cleanup(m.Stop)
+	marker := filepath.Join(dataDir, "alerts", activeStateDegradedMarker)
+	if err := os.MkdirAll(marker, alertsDirPerm); err != nil {
+		t.Fatal(err)
+	}
+
+	alert := durableRestoreAlert(time.Now().UTC())
+	m.setActiveRecoveryAlert(alert, alert.ID)
+	err := m.checkpointActiveRecoveryAfterDurableFailure(os.ErrClosed)
+	if err == nil {
+		t.Fatal("checkpoint did not surface an unwritable recovery marker")
+	}
+	if _, statErr := os.Stat(filepath.Join(dataDir, "alerts", "active-alerts.json")); statErr != nil {
+		t.Fatalf("best-effort recovery mirror was not written after marker failure: %v", statErr)
+	}
+}
+
 func TestDegradedMarkerRepairsSQLiteFromRecoveryMirror(t *testing.T) {
 	dataDir := t.TempDir()
 	original := durableRestoreAlert(time.Now().Add(-2 * time.Hour).UTC())

tests/integration/scripts/run-alert-qualification.mjs

@@ -74,31 +74,61 @@ await fs.writeFile(activeFixturePath, `${JSON.stringify(activeAlerts, null, 2)}\
 
 let exitCode = 1;
 try {
+  const runtimeQualificationPattern = [
+    'TestDurableLifecycleFailureSynchronouslyCheckpointsRecoveryMirror',
+    'TestDurableLifecycleFailureSurfacesRecoveryMarkerFailure',
+    'TestDegradedMarkerRepairsSQLiteFromRecoveryMirror',
+    'TestAlertSnoozeIsDurableAndEnforcedByDeliveryPolicy',
+    'TestSnoozeExpiryResumesEscalationWithoutReplayingMissedLevels',
+    'TestDeliveryReceiptPersistsAndIsClearedAfterRecovery',
+    'TestSendResolvedWebhookHTTP',
+    'TestBuildNotificationDeliveryJobsRoutesGroupedAlertsByDestinationSeverity',
+    'TestBuildNotificationDeliveryJobsTargetsExactEscalationDestinations',
+    'TestWebhookDeliveryCarriesSignatureAndEventID',
+    'TestDeadManRunCycleSendsHealthySignalAndPersistsProgress',
+    'TestDeadManRestartGapIsReportedExternallyAndRecordedInAlertHistory',
+  ].join('|');
   exitCode = await run(
-    process.execPath,
+    'go',
     [
-      './scripts/run-playwright.mjs',
-      'tests/93-alert-operator-qualification.spec.ts',
-      'tests/94-alert-history-real-backend.spec.ts',
-      '--project=chromium',
+      'test',
+      './internal/alerts',
+      './internal/notifications',
+      './internal/monitoring',
+      '-run',
+      `^(${runtimeQualificationPattern})$`,
+      '-count=1',
     ],
-    {
-      cwd: integrationRoot,
-      env: {
-        ...process.env,
-        PULSE_E2E_USE_LOCAL_BACKEND: '1',
-        PULSE_MOCK_MODE: 'false',
-        PULSE_E2E_ALERT_HISTORY_QUALIFICATION: '1',
-        PULSE_E2E_ALERT_HISTORY_FIXTURE: historyFixturePath,
-        PULSE_E2E_ACTIVE_ALERTS_FIXTURE: activeFixturePath,
-        PULSE_E2E_LOCAL_BACKEND_BINARY: binaryPath,
-        PULSE_E2E_RUN_ID: runId,
-        PULSE_E2E_RUNTIME_STATE_PATH: runtimeStatePath,
-        PULSE_E2E_REPORT_DIR: path.join(runRoot, 'playwright-report'),
-        PULSE_E2E_RESULTS_DIR: path.join(runRoot, 'test-results'),
-      },
-    },
+    { cwd: repoRoot, env: process.env },
   );
+
+  if (exitCode === 0) {
+    exitCode = await run(
+      process.execPath,
+      [
+        './scripts/run-playwright.mjs',
+        'tests/93-alert-operator-qualification.spec.ts',
+        'tests/94-alert-history-real-backend.spec.ts',
+        '--project=chromium',
+      ],
+      {
+        cwd: integrationRoot,
+        env: {
+          ...process.env,
+          PULSE_E2E_USE_LOCAL_BACKEND: '1',
+          PULSE_MOCK_MODE: 'false',
+          PULSE_E2E_ALERT_HISTORY_QUALIFICATION: '1',
+          PULSE_E2E_ALERT_HISTORY_FIXTURE: historyFixturePath,
+          PULSE_E2E_ACTIVE_ALERTS_FIXTURE: activeFixturePath,
+          PULSE_E2E_LOCAL_BACKEND_BINARY: binaryPath,
+          PULSE_E2E_RUN_ID: runId,
+          PULSE_E2E_RUNTIME_STATE_PATH: runtimeStatePath,
+          PULSE_E2E_REPORT_DIR: path.join(runRoot, 'playwright-report'),
+          PULSE_E2E_RESULTS_DIR: path.join(runRoot, 'test-results'),
+        },
+      },
+    );
+  }
 } finally {
   await fs.rm(fixtureRoot, { recursive: true, force: true });
   await fs.rm(runtimeStatePath, { force: true });

tests/integration/tests/94-alert-history-real-backend.spec.ts

@@ -61,6 +61,13 @@ type HistoryAlert = {
   type: string;
   startTime: string;
   lastSeen: string;
+  operationalRecord?: {
+    state?: string;
+    suppression?: {
+      reason?: string;
+      expiresAt?: string;
+    };
+  };
 };
 
 async function readHistory(
@@ -74,6 +81,24 @@ async function readHistory(
   return (await response.json()) as HistoryAlert[];
 }
 
+async function readActiveAlert(
+  page: import('@playwright/test').Page,
+  alertIdentifier: string,
+): Promise<HistoryAlert> {
+  const response = await apiRequest(page, '/api/alerts/active');
+  expect(
+    response.ok(),
+    `active alerts API returned ${response.status()}`,
+  ).toBeTruthy();
+  const alerts = (await response.json()) as HistoryAlert[];
+  const alert = alerts.find((candidate) => candidate.id === alertIdentifier);
+  expect(
+    alert,
+    `active alert ${alertIdentifier} was not returned`,
+  ).toBeTruthy();
+  return alert!;
+}
+
 test.describe('Real backend alert history qualification', () => {
   test.skip(
     !qualificationEnabled,
@@ -177,6 +202,79 @@ test.describe('Real backend alert history qualification', () => {
       true,
     );
 
+    const snoozedUntil = new Date(Date.now() + 60 * 60_000).toISOString();
+    const snoozeResponse = await apiRequest(page, '/api/alerts/snooze', {
+      method: 'POST',
+      data: { alertIdentifier: ACTIVE_ALERT_ID, until: snoozedUntil },
+    });
+    expect(
+      snoozeResponse.ok(),
+      `snooze API returned ${snoozeResponse.status()}: ${await snoozeResponse.text()}`,
+    ).toBeTruthy();
+    let controlledAlert = await readActiveAlert(page, ACTIVE_ALERT_ID);
+    expect(controlledAlert.operationalRecord?.state).toBe('suppressed');
+    expect(controlledAlert.operationalRecord?.suppression?.reason).toBe(
+      'user_snooze',
+    );
+    expect(
+      Date.parse(controlledAlert.operationalRecord!.suppression!.expiresAt!),
+    ).toBe(Date.parse(snoozedUntil));
+
+    const diagnosisResponse = await apiRequest(
+      page,
+      `/api/alerts/delivery-diagnosis?alertIdentifier=${encodeURIComponent(ACTIVE_ALERT_ID)}`,
+    );
+    expect(
+      diagnosisResponse.ok(),
+      `delivery diagnosis API returned ${diagnosisResponse.status()}`,
+    ).toBeTruthy();
+    const diagnosis = (await diagnosisResponse.json()) as {
+      alertIdentifier?: string;
+      status?: string;
+      reason?: string;
+      suppressedUntil?: string;
+    };
+    expect(diagnosis).toMatchObject({
+      alertIdentifier: ACTIVE_ALERT_ID,
+      status: 'suppressed',
+      reason: 'snoozed',
+    });
+    expect(Date.parse(diagnosis.suppressedUntil!)).toBe(
+      Date.parse(snoozedUntil),
+    );
+
+    await restartManagedLocalBackend();
+    await ensureAuthenticated(page);
+    controlledAlert = await readActiveAlert(page, ACTIVE_ALERT_ID);
+    expect(controlledAlert.operationalRecord?.state).toBe('suppressed');
+    expect(controlledAlert.operationalRecord?.suppression?.reason).toBe(
+      'user_snooze',
+    );
+
+    const unsnoozeResponse = await apiRequest(page, '/api/alerts/unsnooze', {
+      method: 'POST',
+      data: { alertIdentifier: ACTIVE_ALERT_ID },
+    });
+    expect(
+      unsnoozeResponse.ok(),
+      `unsnooze API returned ${unsnoozeResponse.status()}: ${await unsnoozeResponse.text()}`,
+    ).toBeTruthy();
+    controlledAlert = await readActiveAlert(page, ACTIVE_ALERT_ID);
+    expect(controlledAlert.operationalRecord?.state).toBe('open');
+    expect(controlledAlert.operationalRecord?.suppression).toBeUndefined();
+
+    const controlTimelineResponse = await apiRequest(
+      page,
+      `/api/alerts/incidents?alertIdentifier=${encodeURIComponent(ACTIVE_ALERT_ID)}&started_at=${encodeURIComponent(activeAlert!.startTime)}`,
+    );
+    expect(controlTimelineResponse.ok()).toBeTruthy();
+    const controlTimeline = (await controlTimelineResponse.json()) as {
+      events?: Array<{ type: string }>;
+    } | null;
+    expect(controlTimeline?.events?.map((event) => event.type)).toEqual(
+      expect.arrayContaining(['alert_snoozed', 'alert_unsnoozed']),
+    );
+
     await page.goto('/alerts/overview', { waitUntil: 'domcontentloaded' });
     await expect(page.getByText('Active Overlay Node').first()).toBeVisible();
     const overviewTimelineRequests: string[] = [];