From 6a1f0df61d1b5ce15749975b807a95eca53a6e2b Mon Sep 17 00:00:00 2001 From: rcourtman Date: Wed, 6 May 2026 14:10:41 +0100 Subject: [PATCH] refactor: split alert health assessment runtime Move storage-health reason normalization, ZFS assessment helpers, active health alert value lookup, and canonical health-assessment alert synchronization into internal/alerts/health_assessment.go. Record health_assessment.go as the shared storage-health assessment owner in the alerts subsystem contract and add a reason-code characterization test. Proof: go test ./internal/alerts/... --- .../v6/internal/subsystems/alerts.md | 8 + internal/alerts/alerts.go | 202 ----------------- internal/alerts/health_assessment.go | 213 ++++++++++++++++++ internal/alerts/synology_test.go | 14 ++ 4 files changed, 235 insertions(+), 202 deletions(-) create mode 100644 internal/alerts/health_assessment.go diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index 7c4551277..daab68306 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -65,6 +65,7 @@ operator-facing alert routing behavior for live runtime alerts. 43. `internal/alerts/backup_snapshot.go` 44. `internal/alerts/disk_health.go` 45. `internal/alerts/metric_runtime.go` +46. `internal/alerts/health_assessment.go` ## Shared Boundaries @@ -280,6 +281,13 @@ per-metric delay resolution, legacy metric alert creation/update/clear behavior, metric runtime options, alert key sanitation, and metric delta helpers; future metric-threshold behavior should extend that owner rather than adding shared metric logic back to the central Manager file. +Shared storage-health assessment alerting now lives in +`internal/alerts/health_assessment.go`. That file owns storage-health reason +normalization, ZFS pool/device reason filtering, canonical health-assessment +alert synchronization, and ZFS device assessment construction for host and +storage checkers; future shared health-assessment behavior should extend that +owner rather than reappearing inside resource-specific evaluators or the +central Manager file. Commercial alert handoffs now follow the same shared navigation boundary. `frontend-modern/src/components/Alerts/InvestigateAlertButton.tsx` may resolve the canonical `ai_alerts` destination from the shared license/commercial diff --git a/internal/alerts/alerts.go b/internal/alerts/alerts.go index e4992f97d..071eb6fc7 100644 --- a/internal/alerts/alerts.go +++ b/internal/alerts/alerts.go @@ -6,7 +6,6 @@ import ( "fmt" "os" "path/filepath" - "slices" "strconv" "strings" "sync" @@ -14,8 +13,6 @@ import ( alertconfig "github.com/rcourtman/pulse-go-rewrite/internal/alerts/config" alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs" - "github.com/rcourtman/pulse-go-rewrite/internal/models" - "github.com/rcourtman/pulse-go-rewrite/internal/storagehealth" "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" "github.com/rcourtman/pulse-go-rewrite/internal/utils" "github.com/rs/zerolog/log" @@ -1435,205 +1432,6 @@ func (m *Manager) CheckGuest(guest any, instanceName string) { } } -var ( - zfsPoolAssessmentCodes = []string{ - "zfs_pool_state", - } - zfsPoolErrorAssessmentCodes = []string{ - "zfs_pool_errors", - } - zfsDeviceAssessmentCodes = []string{ - "zfs_device_state", - "zfs_device_errors", - } -) - -type canonicalHealthAssessmentAlertParams struct { - SpecID string - Signal string - Codes []string - Reasons []storagehealth.Reason - AlertID string - AlertType string - SpecResourceID string - ResourceID string - ResourceName string - ResourceType unifiedresources.ResourceType - Node string - Instance string - Metadata map[string]interface{} - Disabled bool - MessageBuilder func(alertspecs.EvaluationResult) (string, float64, float64) -} - -func storageHealthReasonCodes(reasons []storagehealth.Reason) []string { - codes := make([]string, 0, len(reasons)) - seen := make(map[string]struct{}, len(reasons)) - for _, reason := range reasons { - code := strings.TrimSpace(reason.Code) - if code == "" { - continue - } - if _, ok := seen[code]; ok { - continue - } - seen[code] = struct{}{} - codes = append(codes, code) - } - slices.Sort(codes) - return codes -} - -func storageHealthReasonSummaries(reasons []storagehealth.Reason) []string { - summaries := make([]string, 0, len(reasons)) - for _, reason := range reasons { - summary := strings.TrimSpace(reason.Summary) - if summary == "" { - continue - } - summaries = append(summaries, summary) - } - return summaries -} - -func storageHealthAssessmentSeverity(reasons []storagehealth.Reason) alertspecs.AlertSeverity { - severity := alertspecs.AlertSeverity("") - for _, reason := range reasons { - switch reason.Severity { - case storagehealth.RiskCritical: - return alertspecs.AlertSeverityCritical - case storagehealth.RiskWarning: - severity = alertspecs.AlertSeverityWarning - } - } - return severity -} - -func (m *Manager) activeAlertValue(alertID string) (float64, bool) { - m.mu.RLock() - defer m.mu.RUnlock() - - alert, ok := m.getActiveAlertNoLock(alertID) - if !ok || alert == nil { - return 0, false - } - return alert.Value, true -} - -func filterStorageHealthReasonsByCodes(reasons []storagehealth.Reason, codes []string) []storagehealth.Reason { - if len(reasons) == 0 || len(codes) == 0 { - return nil - } - - allowed := make(map[string]struct{}, len(codes)) - for _, code := range codes { - code = strings.TrimSpace(code) - if code == "" { - continue - } - allowed[code] = struct{}{} - } - - filtered := make([]storagehealth.Reason, 0, len(reasons)) - for _, reason := range reasons { - if _, ok := allowed[strings.TrimSpace(reason.Code)]; ok { - filtered = append(filtered, reason) - } - } - return filtered -} - -func zfsDeviceAssessment(device models.ZFSDevice) storagehealth.Assessment { - assessment := storagehealth.Assessment{Level: storagehealth.RiskHealthy} - addReason := func(code string, severity storagehealth.RiskLevel, summary string) { - if strings.TrimSpace(summary) == "" { - return - } - assessment.Reasons = append(assessment.Reasons, storagehealth.Reason{ - Code: code, - Severity: severity, - Summary: summary, - }) - switch severity { - case storagehealth.RiskCritical: - assessment.Level = storagehealth.RiskCritical - case storagehealth.RiskWarning: - if assessment.Level != storagehealth.RiskCritical { - assessment.Level = storagehealth.RiskWarning - } - } - } - - state := strings.ToUpper(strings.TrimSpace(device.State)) - switch state { - case "", "ONLINE", "SPARE": - case "DEGRADED": - addReason("zfs_device_state", storagehealth.RiskWarning, fmt.Sprintf("ZFS device %s is DEGRADED", device.Name)) - default: - addReason("zfs_device_state", storagehealth.RiskCritical, fmt.Sprintf("ZFS device %s is %s", device.Name, state)) - } - - errors := device.ReadErrors + device.WriteErrors + device.ChecksumErrors - if errors > 0 { - addReason( - "zfs_device_errors", - storagehealth.RiskWarning, - fmt.Sprintf("ZFS device %s has errors: %d read, %d write, %d checksum", device.Name, device.ReadErrors, device.WriteErrors, device.ChecksumErrors), - ) - } - - return assessment -} - -func (m *Manager) syncCanonicalHealthAssessmentAlert(params canonicalHealthAssessmentAlertParams) (alertspecs.EvaluationResult, bool) { - if len(params.Reasons) == 0 { - m.clearAlert(buildCanonicalStateID(params.SpecResourceID, params.SpecID)) - return alertspecs.EvaluationResult{}, true - } - - spec, err := buildCanonicalHealthAssessmentSpec( - params.SpecID, - params.SpecResourceID, - params.ResourceName, - params.ResourceType, - params.Signal, - params.Codes, - params.Disabled, - ) - if err != nil { - log.Warn(). - Err(err). - Str("alertID", params.AlertID). - Str("resourceID", params.SpecResourceID). - Msg("Skipping invalid canonical health assessment spec") - return alertspecs.EvaluationResult{}, false - } - - now := time.Now() - return m.evaluateCanonicalStatefulAlert(canonicalStatefulAlertParams{ - Spec: spec, - Evidence: alertspecs.AlertEvidence{ - ObservedAt: now, - HealthAssessment: &alertspecs.HealthAssessmentEvidence{ - Signal: params.Signal, - Severity: storageHealthAssessmentSeverity(params.Reasons), - Codes: storageHealthReasonCodes(params.Reasons), - }, - }, - AlertID: params.AlertID, - AlertType: params.AlertType, - ResourceID: params.ResourceID, - ResourceName: params.ResourceName, - Node: params.Node, - Instance: params.Instance, - Message: strings.Join(storageHealthReasonSummaries(params.Reasons), "; "), - Metadata: cloneMetadata(params.Metadata), - AddToRecent: true, - AddToHistory: true, - MessageBuilder: params.MessageBuilder, - }) -} - func asyncSaveActiveAlerts(reason string, save func() error) { go func() { defer func() { diff --git a/internal/alerts/health_assessment.go b/internal/alerts/health_assessment.go new file mode 100644 index 000000000..852c27efc --- /dev/null +++ b/internal/alerts/health_assessment.go @@ -0,0 +1,213 @@ +package alerts + +import ( + "fmt" + "slices" + "strings" + "time" + + alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs" + "github.com/rcourtman/pulse-go-rewrite/internal/models" + "github.com/rcourtman/pulse-go-rewrite/internal/storagehealth" + "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" + "github.com/rs/zerolog/log" +) + +var ( + zfsPoolAssessmentCodes = []string{ + "zfs_pool_state", + } + zfsPoolErrorAssessmentCodes = []string{ + "zfs_pool_errors", + } + zfsDeviceAssessmentCodes = []string{ + "zfs_device_state", + "zfs_device_errors", + } +) + +type canonicalHealthAssessmentAlertParams struct { + SpecID string + Signal string + Codes []string + Reasons []storagehealth.Reason + AlertID string + AlertType string + SpecResourceID string + ResourceID string + ResourceName string + ResourceType unifiedresources.ResourceType + Node string + Instance string + Metadata map[string]interface{} + Disabled bool + MessageBuilder func(alertspecs.EvaluationResult) (string, float64, float64) +} + +func storageHealthReasonCodes(reasons []storagehealth.Reason) []string { + codes := make([]string, 0, len(reasons)) + seen := make(map[string]struct{}, len(reasons)) + for _, reason := range reasons { + code := strings.TrimSpace(reason.Code) + if code == "" { + continue + } + if _, ok := seen[code]; ok { + continue + } + seen[code] = struct{}{} + codes = append(codes, code) + } + slices.Sort(codes) + return codes +} + +func storageHealthReasonSummaries(reasons []storagehealth.Reason) []string { + summaries := make([]string, 0, len(reasons)) + for _, reason := range reasons { + summary := strings.TrimSpace(reason.Summary) + if summary == "" { + continue + } + summaries = append(summaries, summary) + } + return summaries +} + +func storageHealthAssessmentSeverity(reasons []storagehealth.Reason) alertspecs.AlertSeverity { + severity := alertspecs.AlertSeverity("") + for _, reason := range reasons { + switch reason.Severity { + case storagehealth.RiskCritical: + return alertspecs.AlertSeverityCritical + case storagehealth.RiskWarning: + severity = alertspecs.AlertSeverityWarning + } + } + return severity +} + +func (m *Manager) activeAlertValue(alertID string) (float64, bool) { + m.mu.RLock() + defer m.mu.RUnlock() + + alert, ok := m.getActiveAlertNoLock(alertID) + if !ok || alert == nil { + return 0, false + } + return alert.Value, true +} + +func filterStorageHealthReasonsByCodes(reasons []storagehealth.Reason, codes []string) []storagehealth.Reason { + if len(reasons) == 0 || len(codes) == 0 { + return nil + } + + allowed := make(map[string]struct{}, len(codes)) + for _, code := range codes { + code = strings.TrimSpace(code) + if code == "" { + continue + } + allowed[code] = struct{}{} + } + + filtered := make([]storagehealth.Reason, 0, len(reasons)) + for _, reason := range reasons { + if _, ok := allowed[strings.TrimSpace(reason.Code)]; ok { + filtered = append(filtered, reason) + } + } + return filtered +} + +func zfsDeviceAssessment(device models.ZFSDevice) storagehealth.Assessment { + assessment := storagehealth.Assessment{Level: storagehealth.RiskHealthy} + addReason := func(code string, severity storagehealth.RiskLevel, summary string) { + if strings.TrimSpace(summary) == "" { + return + } + assessment.Reasons = append(assessment.Reasons, storagehealth.Reason{ + Code: code, + Severity: severity, + Summary: summary, + }) + switch severity { + case storagehealth.RiskCritical: + assessment.Level = storagehealth.RiskCritical + case storagehealth.RiskWarning: + if assessment.Level != storagehealth.RiskCritical { + assessment.Level = storagehealth.RiskWarning + } + } + } + + state := strings.ToUpper(strings.TrimSpace(device.State)) + switch state { + case "", "ONLINE", "SPARE": + case "DEGRADED": + addReason("zfs_device_state", storagehealth.RiskWarning, fmt.Sprintf("ZFS device %s is DEGRADED", device.Name)) + default: + addReason("zfs_device_state", storagehealth.RiskCritical, fmt.Sprintf("ZFS device %s is %s", device.Name, state)) + } + + errors := device.ReadErrors + device.WriteErrors + device.ChecksumErrors + if errors > 0 { + addReason( + "zfs_device_errors", + storagehealth.RiskWarning, + fmt.Sprintf("ZFS device %s has errors: %d read, %d write, %d checksum", device.Name, device.ReadErrors, device.WriteErrors, device.ChecksumErrors), + ) + } + + return assessment +} + +func (m *Manager) syncCanonicalHealthAssessmentAlert(params canonicalHealthAssessmentAlertParams) (alertspecs.EvaluationResult, bool) { + if len(params.Reasons) == 0 { + m.clearAlert(buildCanonicalStateID(params.SpecResourceID, params.SpecID)) + return alertspecs.EvaluationResult{}, true + } + + spec, err := buildCanonicalHealthAssessmentSpec( + params.SpecID, + params.SpecResourceID, + params.ResourceName, + params.ResourceType, + params.Signal, + params.Codes, + params.Disabled, + ) + if err != nil { + log.Warn(). + Err(err). + Str("alertID", params.AlertID). + Str("resourceID", params.SpecResourceID). + Msg("Skipping invalid canonical health assessment spec") + return alertspecs.EvaluationResult{}, false + } + + now := time.Now() + return m.evaluateCanonicalStatefulAlert(canonicalStatefulAlertParams{ + Spec: spec, + Evidence: alertspecs.AlertEvidence{ + ObservedAt: now, + HealthAssessment: &alertspecs.HealthAssessmentEvidence{ + Signal: params.Signal, + Severity: storageHealthAssessmentSeverity(params.Reasons), + Codes: storageHealthReasonCodes(params.Reasons), + }, + }, + AlertID: params.AlertID, + AlertType: params.AlertType, + ResourceID: params.ResourceID, + ResourceName: params.ResourceName, + Node: params.Node, + Instance: params.Instance, + Message: strings.Join(storageHealthReasonSummaries(params.Reasons), "; "), + Metadata: cloneMetadata(params.Metadata), + AddToRecent: true, + AddToHistory: true, + MessageBuilder: params.MessageBuilder, + }) +} diff --git a/internal/alerts/synology_test.go b/internal/alerts/synology_test.go index bd1e8bc85..87a2367fc 100644 --- a/internal/alerts/synology_test.go +++ b/internal/alerts/synology_test.go @@ -6,8 +6,22 @@ import ( "time" "github.com/rcourtman/pulse-go-rewrite/internal/models" + "github.com/rcourtman/pulse-go-rewrite/internal/storagehealth" ) +func TestStorageHealthReasonCodesSortsAndDeduplicates(t *testing.T) { + got := storageHealthReasonCodes([]storagehealth.Reason{ + {Code: " zfs_device_errors "}, + {Code: "zfs_device_state"}, + {Code: "zfs_device_errors"}, + {Code: " "}, + }) + + if len(got) != 2 || got[0] != "zfs_device_errors" || got[1] != "zfs_device_state" { + t.Fatalf("storageHealthReasonCodes() = %#v, want sorted unique ZFS codes", got) + } +} + func TestSynologyRAIDSuppression(t *testing.T) { m := newTestManager(t) m.ClearActiveAlerts()