diff --git a/internal/ai/findings_backup_verification.go b/internal/ai/findings_backup_verification.go new file mode 100644 index 000000000..5e8d1f4dc --- /dev/null +++ b/internal/ai/findings_backup_verification.go @@ -0,0 +1,193 @@ +// findings_backup_verification.go emits the "this protected resource hasn't +// had its backup verified recently" signal. The detector reads a +// recovery.ProtectionRollup, decides whether the subject is stale relative +// to its verification window, and builds a backup-category Finding suitable +// for the existing patrol-finding intake path (FindingsStore.Add or +// PatrolService.recordFinding). +// +// Severity is Watch by default and escalates to Warning once the last +// successful backup is itself older than one stale window — that's the +// MVP's "multiple consecutive stale windows" heuristic without depending on +// a separate historical counter. +package ai + +import ( + "fmt" + "strings" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/recovery" +) + +// BackupVerificationStaleFindingKey is the stable, deduped key for the +// backup_verification_stale signal. The finding store keys the dedup set +// on (resourceID, category, key), so this constant must not drift. +const BackupVerificationStaleFindingKey = "backup_verification_stale" + +// backupVerificationStaleWarningMultiplier marks how many staleness windows +// the last successful backup must be older than before the finding escalates +// to Warning severity. +const backupVerificationStaleWarningMultiplier = 2 + +// BuildBackupVerificationStaleFinding inspects a rollup and returns a Finding +// representing the stale-verification signal, or nil if the rollup is not +// stale. The caller is expected to feed the result into the standard patrol +// finding intake (FindingsStore.Add). Returns nil for nil rollups. +func BuildBackupVerificationStaleFinding(rollup *recovery.ProtectionRollup, now time.Time) *Finding { + if rollup == nil || rollup.VerifyIntent != recovery.VerifyIntentStale { + return nil + } + + resourceID := strings.TrimSpace(rollup.SubjectResourceID) + if resourceID == "" { + // Fall back to the rollup key for external refs; the dedup key must + // be stable across patrol runs so the same subject keeps the same ID. + resourceID = strings.TrimSpace(rollup.RollupID) + } + if resourceID == "" { + // Without a stable identifier we cannot dedup; refuse to emit rather + // than risk surfacing duplicate findings. + return nil + } + + resourceName := backupVerificationDisplayLabel(rollup, resourceID) + resourceType := backupVerificationResourceType(rollup) + + severity := FindingSeverityWatch + consecutiveWindows := backupVerificationConsecutiveStaleWindows(rollup, now) + if consecutiveWindows >= backupVerificationStaleWarningMultiplier { + severity = FindingSeverityWarning + } + + id := generateFindingID(resourceID, string(FindingCategoryBackup), BackupVerificationStaleFindingKey) + + finding := &Finding{ + ID: id, + Key: BackupVerificationStaleFindingKey, + Severity: severity, + Category: FindingCategoryBackup, + ResourceID: resourceID, + ResourceName: resourceName, + ResourceType: resourceType, + Title: fmt.Sprintf("Backup not verified recently on %s", resourceName), + Description: fmt.Sprintf( + "A successful backup exists for %s but no verification has landed in the last %s. Restore confidence drops the longer this goes unchecked.", + resourceName, backupVerificationWindowDescription(), + ), + Impact: "If the most recent backup is corrupt or otherwise unrestorable, you will not learn until a real restore is attempted.", + Recommendation: "Run a verification job against the latest backup, or schedule a recurring verify so the staleness window does not lapse again.", + Evidence: backupVerificationEvidence(rollup, now), + Source: "patrol-rule", + } + + return finding +} + +func backupVerificationDisplayLabel(rollup *recovery.ProtectionRollup, fallback string) string { + if rollup.Display != nil { + if label := strings.TrimSpace(rollup.Display.SubjectLabel); label != "" { + return label + } + } + if rollup.SubjectRef != nil { + if name := strings.TrimSpace(rollup.SubjectRef.Name); name != "" { + return name + } + } + return fallback +} + +func backupVerificationResourceType(rollup *recovery.ProtectionRollup) string { + if rollup.Display != nil { + if t := strings.TrimSpace(rollup.Display.SubjectType); t != "" { + return t + } + } + if rollup.SubjectRef != nil { + if t := strings.TrimSpace(rollup.SubjectRef.Type); t != "" { + return t + } + } + return "backup-subject" +} + +// backupVerificationConsecutiveStaleWindows estimates how many staleness +// windows the last successful backup has aged through. A return value >= 2 +// signals the rollup has been stale across multiple windows and the +// emitter should escalate to Warning. +func backupVerificationConsecutiveStaleWindows(rollup *recovery.ProtectionRollup, now time.Time) int { + windowMs := recovery.BackupVerifyStaleWindow.Milliseconds() + if windowMs <= 0 { + return 0 + } + + // Prefer the last successful backup timestamp as the anchor: the rollup + // is stale relative to that point. If there is no success on file, fall + // back to the last verification (which, being outside the window, also + // counts as historical evidence). + var anchorMs int64 + if rollup.LastSuccessAt != nil { + anchorMs = rollup.LastSuccessAt.UTC().UnixMilli() + } else if rollup.LastVerifiedAt != nil { + anchorMs = rollup.LastVerifiedAt.UTC().UnixMilli() + } + if anchorMs <= 0 { + return 0 + } + + ageMs := now.UTC().UnixMilli() - anchorMs + if ageMs <= 0 { + return 0 + } + return int(ageMs / windowMs) +} + +func backupVerificationWindowDescription() string { + days := int(recovery.BackupVerifyStaleWindow / (24 * time.Hour)) + if days <= 0 { + return recovery.BackupVerifyStaleWindow.String() + } + if days == 1 { + return "1 day" + } + return fmt.Sprintf("%d days", days) +} + +func backupVerificationEvidence(rollup *recovery.ProtectionRollup, now time.Time) string { + parts := make([]string, 0, 3) + if rollup.LastSuccessAt != nil { + parts = append(parts, fmt.Sprintf("last successful backup %s ago", durationCoarse(now.Sub(*rollup.LastSuccessAt)))) + } + if rollup.LastVerifiedAt != nil { + parts = append(parts, fmt.Sprintf("last verified %s ago", durationCoarse(now.Sub(*rollup.LastVerifiedAt)))) + } else { + parts = append(parts, "no recorded verification on this rollup") + } + parts = append(parts, fmt.Sprintf("staleness window %s", backupVerificationWindowDescription())) + return strings.Join(parts, "; ") +} + +func durationCoarse(d time.Duration) string { + if d < 0 { + d = -d + } + days := int(d / (24 * time.Hour)) + if days >= 1 { + if days == 1 { + return "1 day" + } + return fmt.Sprintf("%d days", days) + } + hours := int(d / time.Hour) + if hours >= 1 { + if hours == 1 { + return "1 hour" + } + return fmt.Sprintf("%d hours", hours) + } + mins := int(d / time.Minute) + if mins <= 1 { + return "1 minute" + } + return fmt.Sprintf("%d minutes", mins) +} diff --git a/internal/ai/findings_backup_verification_test.go b/internal/ai/findings_backup_verification_test.go new file mode 100644 index 000000000..f16e06e79 --- /dev/null +++ b/internal/ai/findings_backup_verification_test.go @@ -0,0 +1,209 @@ +package ai + +import ( + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/recovery" +) + +func staleRollup(t *testing.T, now time.Time) *recovery.ProtectionRollup { + t.Helper() + // Recent successful backup (within one window), no verification at all. + successAt := now.Add(-2 * 24 * time.Hour) + return &recovery.ProtectionRollup{ + RollupID: "res:vm-100", + SubjectResourceID: "vm-100", + Display: &recovery.RecoveryPointDisplay{ + SubjectLabel: "web-prod", + SubjectType: "proxmox-vm", + }, + LastSuccessAt: &successAt, + LastAttemptAt: &successAt, + LastOutcome: recovery.OutcomeSuccess, + VerifyIntent: recovery.VerifyIntentStale, + } +} + +func TestBuildBackupVerificationStaleFinding_EmitsFindingForStaleRollup(t *testing.T) { + now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC) + rollup := staleRollup(t, now) + + f := BuildBackupVerificationStaleFinding(rollup, now) + if f == nil { + t.Fatal("expected a finding for a stale rollup, got nil") + } + if f.Category != FindingCategoryBackup { + t.Fatalf("Category = %q, want %q", f.Category, FindingCategoryBackup) + } + if f.Key != BackupVerificationStaleFindingKey { + t.Fatalf("Key = %q, want %q", f.Key, BackupVerificationStaleFindingKey) + } + // Default severity is Watch when only a single staleness window has elapsed. + if f.Severity != FindingSeverityWatch { + t.Fatalf("Severity = %q, want %q", f.Severity, FindingSeverityWatch) + } + if f.ResourceID != "vm-100" { + t.Fatalf("ResourceID = %q, want %q", f.ResourceID, "vm-100") + } + if f.ResourceName != "web-prod" { + t.Fatalf("ResourceName = %q, want preferred display label", f.ResourceName) + } + + // ID must be deterministic across two emissions for the same rollup. + second := BuildBackupVerificationStaleFinding(rollup, now) + if second == nil || second.ID != f.ID { + t.Fatalf("expected deterministic finding ID across emissions, got %q vs %q", f.ID, secondID(second)) + } + + // Different resource → different finding ID. + other := staleRollup(t, now) + other.SubjectResourceID = "vm-999" + otherFinding := BuildBackupVerificationStaleFinding(other, now) + if otherFinding == nil || otherFinding.ID == f.ID { + t.Fatalf("expected distinct IDs for distinct resources, got %q vs %q", f.ID, secondID(otherFinding)) + } +} + +func TestBuildBackupVerificationStaleFinding_NilForNonStaleRollup(t *testing.T) { + now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC) + verifiedAt := now.Add(-24 * time.Hour) + rollup := &recovery.ProtectionRollup{ + RollupID: "res:vm-100", + SubjectResourceID: "vm-100", + LastSuccessAt: &verifiedAt, + LastVerifiedAt: &verifiedAt, + LastOutcome: recovery.OutcomeSuccess, + VerifyIntent: recovery.VerifyIntentVerified, + } + + if f := BuildBackupVerificationStaleFinding(rollup, now); f != nil { + t.Fatalf("expected nil for verified rollup, got %#v", f) + } + + rollup.VerifyIntent = recovery.VerifyIntentUnknown + if f := BuildBackupVerificationStaleFinding(rollup, now); f != nil { + t.Fatalf("expected nil for unknown rollup, got %#v", f) + } + + if f := BuildBackupVerificationStaleFinding(nil, now); f != nil { + t.Fatal("expected nil for nil rollup") + } +} + +func TestBuildBackupVerificationStaleFinding_EscalatesAfterMultipleWindows(t *testing.T) { + now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC) + // Last successful backup is 21 days old → 3 staleness windows of 7d. + old := now.Add(-21 * 24 * time.Hour) + rollup := &recovery.ProtectionRollup{ + RollupID: "res:vm-200", + SubjectResourceID: "vm-200", + Display: &recovery.RecoveryPointDisplay{ + SubjectLabel: "db-prod", + SubjectType: "proxmox-vm", + }, + LastSuccessAt: &old, + LastAttemptAt: &old, + LastOutcome: recovery.OutcomeSuccess, + VerifyIntent: recovery.VerifyIntentStale, + } + + f := BuildBackupVerificationStaleFinding(rollup, now) + if f == nil { + t.Fatal("expected finding") + } + if f.Severity != FindingSeverityWarning { + t.Fatalf("Severity = %q, want %q for multi-window stale", f.Severity, FindingSeverityWarning) + } +} + +func TestBuildBackupVerificationStaleFinding_PatrolIntakeDedupes(t *testing.T) { + now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC) + store := NewFindingsStore() + rollup := staleRollup(t, now) + + first := BuildBackupVerificationStaleFinding(rollup, now) + if first == nil { + t.Fatal("expected initial finding") + } + if !store.Add(first) { + t.Fatal("expected first Add to create a new finding") + } + + // Same patrol tick, same rollup. Should dedup on (resourceID, category, key). + second := BuildBackupVerificationStaleFinding(rollup, now.Add(15*time.Minute)) + if second == nil { + t.Fatal("expected second emission for still-stale rollup") + } + if second.ID != first.ID { + t.Fatalf("expected dedup-stable ID, got %q (first) vs %q (second)", first.ID, second.ID) + } + if store.Add(second) { + t.Fatal("expected second Add to update existing (return false), not create new") + } + + stored := store.Get(first.ID) + if stored == nil { + t.Fatal("expected stored finding to exist") + } + if stored.TimesRaised != 2 { + t.Fatalf("TimesRaised = %d, want 2", stored.TimesRaised) + } +} + +func TestBuildBackupVerificationStaleFinding_ResolutionPathFiresOnReverify(t *testing.T) { + now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC) + store := NewFindingsStore() + + rollup := staleRollup(t, now) + finding := BuildBackupVerificationStaleFinding(rollup, now) + if finding == nil { + t.Fatal("expected stale finding") + } + if !store.Add(finding) { + t.Fatal("expected first Add to create") + } + + // Verification reappears on the rollup. The detector returns nil; the + // caller is expected to drive the resolution path. Mirror the lifecycle + // fixtures: Resolve(auto=true) and confirm the auto_resolved event lands. + verifiedAt := now.Add(5 * time.Minute) + rollup.VerifyIntent = recovery.VerifyIntentVerified + rollup.LastVerifiedAt = &verifiedAt + + if reemitted := BuildBackupVerificationStaleFinding(rollup, now.Add(10*time.Minute)); reemitted != nil { + t.Fatalf("expected detector to skip emission once verified, got %#v", reemitted) + } + + if !store.Resolve(finding.ID, true) { + t.Fatal("expected Resolve to succeed") + } + + stored := store.Get(finding.ID) + if stored == nil { + t.Fatal("expected finding to exist after resolve") + } + if stored.ResolvedAt == nil { + t.Fatal("expected ResolvedAt to be set after Resolve") + } + if !stored.AutoResolved { + t.Fatal("expected AutoResolved=true after Resolve(true)") + } + foundAutoResolved := false + for _, e := range stored.Lifecycle { + if e.Type == "auto_resolved" { + foundAutoResolved = true + break + } + } + if !foundAutoResolved { + t.Fatalf("expected auto_resolved lifecycle event; got: %+v", stored.Lifecycle) + } +} + +func secondID(f *Finding) string { + if f == nil { + return "" + } + return f.ID +}