From ee35d178b5c7dad09877c20bf20acd2aba8f0b51 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Wed, 26 Aug 2026 23:54:12 +0100 Subject: [PATCH] feat(alerts): backup-offline deferral in the reducer intent gate, completing Phase 1 characterization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The last uncharacterized behavior: while Pulse has fresh evidence a Proxmox backup caused an offline state, activation defers — bounded by the max-deferral cap on total condition-active time — and after the backup ends the grace extends to the backup's end plus the post-grace, still capped. The discrete activation path is unified so the gate always operates on a tracked pending incident, and the shadow feed now models the deferral independently from the manager's decision instead of echoing its hold as operator suppression. Unit tests cover deferral, post-grace release, the never-ending-backup cap, and post-grace capping; parity runs the real LoadIntentPolicies/IntentBackup composition with simulated clocks. Fourth consecutive parity slice with no manager defect. Every discrete-family and metric-family behavior is now pinned. --- ...-lifecycle-contract-coverage-2026-08-26.md | 12 +- internal/alerts/canonical_lifecycle.go | 16 +- internal/alerts/reducer/discrete.go | 152 ++++++++------- internal/alerts/reducer/intent_test.go | 73 ++++++++ internal/alerts/reducer/reducer.go | 15 +- internal/alerts/reducer_parity_backup_test.go | 176 ++++++++++++++++++ 6 files changed, 366 insertions(+), 78 deletions(-) create mode 100644 internal/alerts/reducer_parity_backup_test.go diff --git a/docs/release-control/v6/internal/records/alert-lifecycle-contract-coverage-2026-08-26.md b/docs/release-control/v6/internal/records/alert-lifecycle-contract-coverage-2026-08-26.md index 39bf6f9cea..a66081654d 100644 --- a/docs/release-control/v6/internal/records/alert-lifecycle-contract-coverage-2026-08-26.md +++ b/docs/release-control/v6/internal/records/alert-lifecycle-contract-coverage-2026-08-26.md @@ -106,6 +106,12 @@ unhooked mutations). Enabled by the monitoring bootstrap next to the event log. The divergence rate is the go/no-go evidence for each Phase 2 family cutover. -Remaining: the backup-offline deferral sub-policy (the shadow feed maps -its hold to operator suppression, which is behavior-equivalent while -uncharacterized), and Phase 2 itself. +Backup-offline deferral (2026-08-26): characterized — deferral while the +backup runs bounded by the max-deferral cap on total condition-active +time, post-grace from the backup's end still bounded by the cap — with +unit tests and parity against the real LoadIntentPolicies/IntentBackup +composition. The shadow feed now models the deferral independently +instead of echoing the manager's hold. Phase 1 characterization is +complete: every discrete-family and metric-family behavior is pinned. + +Remaining: Phase 2 itself. diff --git a/internal/alerts/canonical_lifecycle.go b/internal/alerts/canonical_lifecycle.go index b6a59743bf..d729f92b5a 100644 --- a/internal/alerts/canonical_lifecycle.go +++ b/internal/alerts/canonical_lifecycle.go @@ -1,6 +1,7 @@ package alerts import ( + "strings" "time" "github.com/rcourtman/pulse-go-rewrite/internal/alerts/reducer" @@ -427,11 +428,20 @@ func (m *Manager) evaluateCanonicalLifecycleAlert(params canonicalLifecycleAlert conditionActive := result.State.State == alertspecs.AlertStatePending || result.State.State == alertspecs.AlertStateFiring decision := m.evaluateIntentNoLock(params.Spec.ResourceID, string(params.Spec.ResourceType), intentSignal, storageKey, params.Evidence.ObservedAt, conditionActive, params.IntentBackup) shadowIntent = &reducer.DiscreteIntent{ - Explicit: decision.Effective.Explicit, - GraceSeconds: decision.Effective.GraceSeconds, - OperatorSuppressed: decision.Suppressed, + Explicit: decision.Effective.Explicit, + GraceSeconds: decision.Effective.GraceSeconds, + // Operator suppression only: the backup-offline deferral is + // modeled independently below so the reducer computes its own + // hold rather than echoing the manager's. + OperatorSuppressed: decision.Suppressed && strings.HasPrefix(decision.Reason, "operator_"), OperatorReason: decision.Reason, } + if backupOffline := decision.Effective.BackupOffline; backupOffline != nil && backupOffline.Enabled && intentSignal == string(AlertIntentSignalOffline) { + shadowIntent.BackupEnabled = true + shadowIntent.BackupActive = params.IntentBackup.Active + shadowIntent.BackupPostGraceSeconds = backupOffline.PostGraceSeconds + shadowIntent.BackupMaxDeferralSeconds = backupOffline.MaxDeferralSeconds + } if decision.StateChanged { m.saveActiveAlertsAsync("lifecycle intent state") } diff --git a/internal/alerts/reducer/discrete.go b/internal/alerts/reducer/discrete.go index 8c3d2511d9..2b8f4c1862 100644 --- a/internal/alerts/reducer/discrete.go +++ b/internal/alerts/reducer/discrete.go @@ -72,13 +72,28 @@ type DiscreteIntent struct { OperatorSuppressed bool // OperatorReason names the suppression, e.g. "operator_maintenance". OperatorReason string + // BackupEnabled arms the backup-offline deferral sub-policy for this + // condition (offline-family signals with an explicit backupOffline + // rule): while a backup is running the activation is deferred, bounded + // by BackupMaxDeferralSeconds of total condition-active time; after + // the backup ends the grace extends to the backup's end point plus + // BackupPostGraceSeconds, still bounded by the cap. + BackupEnabled bool + // BackupActive is the observation-time backup context. + BackupActive bool + BackupPostGraceSeconds int + BackupMaxDeferralSeconds int } // intentHoldsActivation reports whether the gate keeps a would-be -// activation pending. Grace accrues from the condition run's first active -// observation, concurrently with any operator suppression — exactly as the -// manager accrues elapsed time while suppressed. -func intentHoldsActivation(intent *DiscreteIntent, activeSince, observedAt time.Time) bool { +// activation pending, updating the incident's backup-run bookkeeping. +// Grace accrues from the condition run's first active observation, +// concurrently with any operator suppression — exactly as the manager +// accrues elapsed time while suppressed. The backup-offline sub-policy +// mirrors evaluateIntentNoLock: deferral while the backup runs, bounded +// by the max-deferral cap on total condition-active time; post-grace from +// the backup's end, still bounded by the cap. +func intentHoldsActivation(intent *DiscreteIntent, incident *Incident, observedAt time.Time) bool { if intent == nil { return false } @@ -88,7 +103,32 @@ func intentHoldsActivation(intent *DiscreteIntent, activeSince, observedAt time. if !intent.Explicit { return false } - return observedAt.Sub(activeSince) < time.Duration(intent.GraceSeconds)*time.Second + elapsed := observedAt.Sub(incident.PendingSince) + eligible := time.Duration(intent.GraceSeconds) * time.Second + if intent.BackupEnabled { + if intent.BackupActive { + incident.BackupActive = true + incident.BackupEnded = false + } else if incident.BackupActive { + incident.BackupActive = false + incident.BackupEnded = true + incident.BackupEndedElapsed = elapsed + } + deferralCap := time.Duration(intent.BackupMaxDeferralSeconds) * time.Second + if incident.BackupActive && elapsed < deferralCap { + return true + } + if incident.BackupEnded { + post := incident.BackupEndedElapsed + time.Duration(intent.BackupPostGraceSeconds)*time.Second + if post > eligible { + eligible = post + } + } + if eligible > deferralCap { + eligible = deferralCap + } + } + return elapsed < eligible } // recordResolved remembers a resolved occurrence for re-fire restoration. @@ -196,75 +236,53 @@ func (s *State) ApplyDiscrete(signal DiscreteSignal, rule DiscreteRule) []Event return nil } - // Matching while pending: count consecutive matches; fire once the - // requirement is met, with StartedAt at the first matched observation - // (the manager uses FirstMatchedAt as the alert start time). - if incident != nil { - incident.Confirmations++ - incident.Severity = signal.Severity - incident.LastObservedAt = signal.ObservedAt - if incident.Confirmations < required { - return nil - } - incident.Confirmations = required - // The intent gate holds activation only: the incident stays - // pending, counts clamped at the requirement, and the run's first - // active observation remains the eventual start time. - if intentHoldsActivation(rule.Intent, incident.PendingSince, signal.ObservedAt) { - return nil - } - incident.State = StateFiring - incident.StartedAt = incident.PendingSince - s.restoreAck(key, incident, signal.ObservedAt) - if restored, ok := s.consumeRefire(key, signal.ObservedAt); ok { - incident.StartedAt = restored - return []Event{event(EventRefired, incident.Severity)} - } - return []Event{event(EventFired, incident.Severity)} - } - - // First matching observation. - if required <= 1 { - // With the intent gate holding, even a single-confirmation - // condition enters pending: the manager's wrapper forces the - // evaluator's would-be activation back to pending state. - if intentHoldsActivation(rule.Intent, signal.ObservedAt, signal.ObservedAt) { - s.incidents[key] = &Incident{ - ResourceID: signal.ResourceID, - Key: signal.Key, - State: StatePending, - Severity: signal.Severity, - PendingSince: signal.ObservedAt, - Confirmations: 1, - LastObservedAt: signal.ObservedAt, - } - return []Event{event(EventPending, "")} - } - created := &Incident{ + // Matching while pending (or first match): the run enters or advances + // pending, then attempts activation. The intent gate holds activation + // only — the incident stays pending with counts clamped at the + // requirement, and the run's first active observation remains the + // eventual start time (the manager uses FirstMatchedAt as the alert + // start time). + entered := false + if incident == nil { + incident = &Incident{ ResourceID: signal.ResourceID, Key: signal.Key, - State: StateFiring, + State: StatePending, Severity: signal.Severity, - StartedAt: signal.ObservedAt, + PendingSince: signal.ObservedAt, Confirmations: 1, LastObservedAt: signal.ObservedAt, } - s.incidents[key] = created - s.restoreAck(key, created, signal.ObservedAt) - if restored, ok := s.consumeRefire(key, signal.ObservedAt); ok { - created.StartedAt = restored - return []Event{event(EventRefired, signal.Severity)} + s.incidents[key] = incident + entered = true + } else { + incident.Confirmations++ + incident.Severity = signal.Severity + incident.LastObservedAt = signal.ObservedAt + } + + pendingResult := func() []Event { + if entered { + return []Event{event(EventPending, "")} } - return []Event{event(EventFired, signal.Severity)} + return nil } - s.incidents[key] = &Incident{ - ResourceID: signal.ResourceID, - Key: signal.Key, - State: StatePending, - Severity: signal.Severity, - PendingSince: signal.ObservedAt, - Confirmations: 1, - LastObservedAt: signal.ObservedAt, + + if incident.Confirmations < required { + return pendingResult() } - return []Event{event(EventPending, "")} + incident.Confirmations = required + + if intentHoldsActivation(rule.Intent, incident, signal.ObservedAt) { + return pendingResult() + } + + incident.State = StateFiring + incident.StartedAt = incident.PendingSince + s.restoreAck(key, incident, signal.ObservedAt) + if restored, ok := s.consumeRefire(key, signal.ObservedAt); ok { + incident.StartedAt = restored + return []Event{event(EventRefired, incident.Severity)} + } + return []Event{event(EventFired, incident.Severity)} } diff --git a/internal/alerts/reducer/intent_test.go b/internal/alerts/reducer/intent_test.go index 6cf9cf0dcc..ff2d640122 100644 --- a/internal/alerts/reducer/intent_test.go +++ b/internal/alerts/reducer/intent_test.go @@ -113,3 +113,76 @@ func TestIntentGraceComposesWithConfirmations(t *testing.T) { t.Fatalf("StartedAt = %v, want run start %v", incident.StartedAt, t0) } } + +func backupIntent(grace, post, cap int, backupActive bool) *DiscreteIntent { + return &DiscreteIntent{ + Explicit: true, + GraceSeconds: grace, + BackupEnabled: true, + BackupActive: backupActive, + BackupPostGraceSeconds: post, + BackupMaxDeferralSeconds: cap, + } +} + +func TestBackupDeferralHoldsWhileBackupRuns(t *testing.T) { + state := NewState() + rule := func(active bool) DiscreteRule { + return DiscreteRule{Confirmations: 1, Intent: backupIntent(60, 120, 600, active)} + } + + // Offline with a backup running: held well past the plain grace. + state.ApplyDiscrete(discreteSignalAt(true, SeverityCritical, 0), rule(true)) + if events := state.ApplyDiscrete(discreteSignalAt(true, SeverityCritical, 3*time.Minute), rule(true)); len(events) != 0 { + t.Fatalf("backup-deferred observation emitted %+v", events) + } + + // Backup ends at 4m: eligibility extends to 4m + 120s post-grace. + if events := state.ApplyDiscrete(discreteSignalAt(true, SeverityCritical, 4*time.Minute), rule(false)); len(events) != 0 { + t.Fatalf("post-backup grace emitted %+v", events) + } + if events := state.ApplyDiscrete(discreteSignalAt(true, SeverityCritical, 5*time.Minute), rule(false)); len(events) != 0 { + t.Fatalf("still inside post-grace emitted %+v", events) + } + events := state.ApplyDiscrete(discreteSignalAt(true, SeverityCritical, 6*time.Minute), rule(false)) + if len(events) != 1 || events[0].Type != EventFired { + t.Fatalf("events = %+v, want fired at backup end + post-grace", events) + } + incident, _ := state.Incident("node-1", "connectivity") + if !incident.StartedAt.Equal(t0) { + t.Fatalf("StartedAt = %v, want run start %v", incident.StartedAt, t0) + } +} + +func TestBackupDeferralCapBoundsTheHold(t *testing.T) { + state := NewState() + rule := DiscreteRule{Confirmations: 1, Intent: backupIntent(60, 120, 300, true)} + + // Backup never ends: activation at the max-deferral cap (5m). + state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 0), rule) + if events := state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 4*time.Minute), rule); len(events) != 0 { + t.Fatalf("under cap emitted %+v", events) + } + events := state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 5*time.Minute), rule) + if len(events) != 1 || events[0].Type != EventFired { + t.Fatalf("events = %+v, want fired at deferral cap", events) + } +} + +func TestBackupPostGraceCappedByMaxDeferral(t *testing.T) { + state := NewState() + // Backup ends at 4m; post-grace would extend to 4m+180s=7m, but the cap + // is 5m — activation at 5m. + rule := func(active bool) DiscreteRule { + return DiscreteRule{Confirmations: 1, Intent: backupIntent(60, 180, 300, active)} + } + state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 0), rule(true)) + state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 4*time.Minute), rule(false)) + if events := state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 4*time.Minute+30*time.Second), rule(false)); len(events) != 0 { + t.Fatalf("under cap emitted %+v", events) + } + events := state.ApplyDiscrete(discreteSignalAt(true, SeverityWarning, 5*time.Minute), rule(false)) + if len(events) != 1 || events[0].Type != EventFired { + t.Fatalf("events = %+v, want fired at cap despite longer post-grace", events) + } +} diff --git a/internal/alerts/reducer/reducer.go b/internal/alerts/reducer/reducer.go index 83849b2928..60a9ccfbdd 100644 --- a/internal/alerts/reducer/reducer.go +++ b/internal/alerts/reducer/reducer.go @@ -79,11 +79,16 @@ type Incident struct { // RecoveryCount is the consecutive non-matching count while firing, // toward DiscreteRule.RecoveryConfirmations; reset by any matching // observation. Unused by the metric family. - RecoveryCount int - Acknowledged bool - AckUser string - AckAt time.Time - LastObservedAt time.Time + RecoveryCount int + Acknowledged bool + AckUser string + AckAt time.Time + // Backup-run bookkeeping for the intent gate's backup-offline deferral + // sub-policy (confirmation family only). + BackupActive bool + BackupEnded bool + BackupEndedElapsed time.Duration + LastObservedAt time.Time } // EventType enumerates transition events emitted by Apply. diff --git a/internal/alerts/reducer_parity_backup_test.go b/internal/alerts/reducer_parity_backup_test.go new file mode 100644 index 0000000000..6f0293b0da --- /dev/null +++ b/internal/alerts/reducer_parity_backup_test.go @@ -0,0 +1,176 @@ +package alerts + +import ( + "fmt" + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/alerts/reducer" + alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs" + "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" +) + +// Backup-offline deferral parity: while Pulse has fresh evidence a backup +// caused the offline state, activation is deferred, bounded by the +// max-deferral cap on total condition-active time; after the backup ends +// the grace extends to the backup's end plus the post-grace, still capped. +// The manager side runs the real composition (LoadIntentPolicies with a +// backupOffline rule, IntentBackup context per observation) with its time +// seams simulated; the reducer models the deferral independently. + +type backupParityStep struct { + matched bool + backupActive bool + advance time.Duration + + wantFiring bool + wantStartOffset time.Duration // -1 skips +} + +type backupParityScenario struct { + name string + graceSeconds int + postGrace int + maxDeferral int + steps []backupParityStep +} + +const backupParityResourceID = "parity-backup-1" + +func backupParityScenarios() []backupParityScenario { + skip := time.Duration(-1) + return []backupParityScenario{ + { + name: "backup defers activation, post-grace releases it", + graceSeconds: 60, postGrace: 120, maxDeferral: 600, + steps: []backupParityStep{ + {matched: true, backupActive: true, wantFiring: false, wantStartOffset: skip}, + {matched: true, backupActive: true, advance: 3 * time.Minute, wantFiring: false, wantStartOffset: skip}, + {matched: true, backupActive: false, advance: time.Minute, wantFiring: false, wantStartOffset: skip}, + {matched: true, backupActive: false, advance: time.Minute, wantFiring: false, wantStartOffset: skip}, + {matched: true, backupActive: false, advance: time.Minute, wantFiring: true, wantStartOffset: 0}, + }, + }, + { + name: "max deferral caps a backup that never ends", + graceSeconds: 60, postGrace: 120, maxDeferral: 300, + steps: []backupParityStep{ + {matched: true, backupActive: true, wantFiring: false, wantStartOffset: skip}, + {matched: true, backupActive: true, advance: 4 * time.Minute, wantFiring: false, wantStartOffset: skip}, + {matched: true, backupActive: true, advance: time.Minute, wantFiring: true, wantStartOffset: 0}, + }, + }, + } +} + +func TestReducerParityWithManagerBackupDeferral(t *testing.T) { + for _, scenario := range backupParityScenarios() { + scenario := scenario + t.Run(scenario.name, func(t *testing.T) { + epoch := time.Date(2026, 8, 26, 12, 0, 0, 0, time.UTC) + simClock := epoch + + manager := NewManagerWithDataDir(t.TempDir(), WithoutPersistedAlertRestore()) + t.Cleanup(manager.Stop) + manager.mu.Lock() + manager.config.Enabled = true + manager.config.FlappingEnabled = false + manager.now = func() time.Time { return simClock } + manager.intentClock = func() time.Duration { return simClock.Sub(epoch) } + manager.mu.Unlock() + + grace := scenario.graceSeconds + document := NewAlertIntentPolicyDocument() + document.Defaults[string(AlertIntentSignalOffline)] = AlertIntentRule{ + GraceSeconds: &grace, + BackupOffline: &BackupOfflineIntentPolicy{ + Enabled: true, + PostGraceSeconds: scenario.postGrace, + MaxDeferralSeconds: scenario.maxDeferral, + }, + } + if err := manager.LoadIntentPolicies(document); err != nil { + t.Fatalf("LoadIntentPolicies: %v", err) + } + + reducerState := reducer.NewState() + alertID := canonicalConnectivityStateID(backupParityResourceID) + + for i, step := range scenario.steps { + simClock = simClock.Add(step.advance) + + spec, err := buildCanonicalConnectivitySpec( + backupParityResourceID, "parity-backup", unifiedresources.ResourceTypePBS, + AlertLevelCritical, 1, false, + ) + if err != nil { + t.Fatalf("build spec: %v", err) + } + if _, ok := manager.evaluateCanonicalLifecycleAlert(canonicalLifecycleAlertParams{ + Spec: spec, + Evidence: alertspecs.AlertEvidence{ + ObservedAt: simClock, + Connectivity: &alertspecs.ConnectivityEvidence{Signal: "status", Connected: !step.matched}, + }, + Tracking: manager.offlineConfirmations, + TrackingKey: "parity:" + backupParityResourceID, + AlertID: alertID, + AlertType: "offline", + ResourceID: backupParityResourceID, + ResourceName: "parity-backup", + Message: "parity backup condition", + IntentBackup: BackupIntentContext{Active: step.backupActive, ObservedAt: simClock}, + }); !ok { + t.Fatal("evaluateCanonicalLifecycleAlert rejected the parity spec") + } + + reducerState.ApplyDiscrete(reducer.DiscreteSignal{ + ResourceID: backupParityResourceID, + Key: "connectivity", + Matched: step.matched, + Severity: reducer.SeverityCritical, + ObservedAt: simClock, + }, reducer.DiscreteRule{ + Confirmations: 1, + Intent: &reducer.DiscreteIntent{ + Explicit: true, + GraceSeconds: scenario.graceSeconds, + BackupEnabled: true, + BackupActive: step.backupActive, + BackupPostGraceSeconds: scenario.postGrace, + BackupMaxDeferralSeconds: scenario.maxDeferral, + }, + }) + + manager.mu.Lock() + alert, exists := manager.getActiveAlertNoLock(alertID) + manager.mu.Unlock() + managerFiring := exists && alert != nil + incident, ok := reducerState.Incident(backupParityResourceID, "connectivity") + reducerFiring := ok && incident.State == reducer.StateFiring + + label := fmt.Sprintf("step %d (matched=%v backup=%v advance=%s)", i, step.matched, step.backupActive, step.advance) + + if managerFiring != step.wantFiring { + t.Fatalf("%s: manager firing = %v, scenario expects %v (characterization drift)", + label, managerFiring, step.wantFiring) + } + if managerFiring && step.wantStartOffset >= 0 { + wantStart := epoch.Add(step.wantStartOffset) + if !alert.StartTime.Equal(wantStart) { + t.Fatalf("%s: manager start = %v, scenario expects %v", label, alert.StartTime, wantStart) + } + } + + if reducerFiring != managerFiring { + t.Fatalf("%s: PARITY DIVERGENCE — reducer firing = %v, manager = %v", + label, reducerFiring, managerFiring) + } + if managerFiring && step.wantStartOffset >= 0 && !incident.StartedAt.Equal(alert.StartTime) { + t.Fatalf("%s: PARITY DIVERGENCE — reducer start = %v, manager = %v", + label, incident.StartedAt, alert.StartTime) + } + } + }) + } +}