mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-07-09 16:00:59 +00:00
refactor: split alert health assessment runtime
Move storage-health reason normalization, ZFS assessment helpers, active health alert value lookup, and canonical health-assessment alert synchronization into internal/alerts/health_assessment.go. Record health_assessment.go as the shared storage-health assessment owner in the alerts subsystem contract and add a reason-code characterization test. Proof: go test ./internal/alerts/...
This commit is contained in:
rcourtman
parent
2f5aa20122
commit
6a1f0df61d
4 changed files with 235 additions and 202 deletions
|
|
@ -65,6 +65,7 @@ operator-facing alert routing behavior for live runtime alerts.
|
|||
43. `internal/alerts/backup_snapshot.go`
|
||||
44. `internal/alerts/disk_health.go`
|
||||
45. `internal/alerts/metric_runtime.go`
|
||||
46. `internal/alerts/health_assessment.go`
|
||||
|
||||
## Shared Boundaries
|
||||
|
||||
|
|
@ -280,6 +281,13 @@ per-metric delay resolution, legacy metric alert creation/update/clear
|
|||
behavior, metric runtime options, alert key sanitation, and metric delta
|
||||
helpers; future metric-threshold behavior should extend that owner rather than
|
||||
adding shared metric logic back to the central Manager file.
|
||||
Shared storage-health assessment alerting now lives in
|
||||
`internal/alerts/health_assessment.go`. That file owns storage-health reason
|
||||
normalization, ZFS pool/device reason filtering, canonical health-assessment
|
||||
alert synchronization, and ZFS device assessment construction for host and
|
||||
storage checkers; future shared health-assessment behavior should extend that
|
||||
owner rather than reappearing inside resource-specific evaluators or the
|
||||
central Manager file.
|
||||
Commercial alert handoffs now follow the same shared navigation boundary.
|
||||
`frontend-modern/src/components/Alerts/InvestigateAlertButton.tsx` may resolve
|
||||
the canonical `ai_alerts` destination from the shared license/commercial
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ import (
|
|||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
|
@ -14,8 +13,6 @@ import (
|
|||
|
||||
alertconfig "github.com/rcourtman/pulse-go-rewrite/internal/alerts/config"
|
||||
alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/storagehealth"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/utils"
|
||||
"github.com/rs/zerolog/log"
|
||||
|
|
@ -1435,205 +1432,6 @@ func (m *Manager) CheckGuest(guest any, instanceName string) {
|
|||
}
|
||||
}
|
||||
|
||||
var (
|
||||
zfsPoolAssessmentCodes = []string{
|
||||
"zfs_pool_state",
|
||||
}
|
||||
zfsPoolErrorAssessmentCodes = []string{
|
||||
"zfs_pool_errors",
|
||||
}
|
||||
zfsDeviceAssessmentCodes = []string{
|
||||
"zfs_device_state",
|
||||
"zfs_device_errors",
|
||||
}
|
||||
)
|
||||
|
||||
type canonicalHealthAssessmentAlertParams struct {
|
||||
SpecID string
|
||||
Signal string
|
||||
Codes []string
|
||||
Reasons []storagehealth.Reason
|
||||
AlertID string
|
||||
AlertType string
|
||||
SpecResourceID string
|
||||
ResourceID string
|
||||
ResourceName string
|
||||
ResourceType unifiedresources.ResourceType
|
||||
Node string
|
||||
Instance string
|
||||
Metadata map[string]interface{}
|
||||
Disabled bool
|
||||
MessageBuilder func(alertspecs.EvaluationResult) (string, float64, float64)
|
||||
}
|
||||
|
||||
func storageHealthReasonCodes(reasons []storagehealth.Reason) []string {
|
||||
codes := make([]string, 0, len(reasons))
|
||||
seen := make(map[string]struct{}, len(reasons))
|
||||
for _, reason := range reasons {
|
||||
code := strings.TrimSpace(reason.Code)
|
||||
if code == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[code]; ok {
|
||||
continue
|
||||
}
|
||||
seen[code] = struct{}{}
|
||||
codes = append(codes, code)
|
||||
}
|
||||
slices.Sort(codes)
|
||||
return codes
|
||||
}
|
||||
|
||||
func storageHealthReasonSummaries(reasons []storagehealth.Reason) []string {
|
||||
summaries := make([]string, 0, len(reasons))
|
||||
for _, reason := range reasons {
|
||||
summary := strings.TrimSpace(reason.Summary)
|
||||
if summary == "" {
|
||||
continue
|
||||
}
|
||||
summaries = append(summaries, summary)
|
||||
}
|
||||
return summaries
|
||||
}
|
||||
|
||||
func storageHealthAssessmentSeverity(reasons []storagehealth.Reason) alertspecs.AlertSeverity {
|
||||
severity := alertspecs.AlertSeverity("")
|
||||
for _, reason := range reasons {
|
||||
switch reason.Severity {
|
||||
case storagehealth.RiskCritical:
|
||||
return alertspecs.AlertSeverityCritical
|
||||
case storagehealth.RiskWarning:
|
||||
severity = alertspecs.AlertSeverityWarning
|
||||
}
|
||||
}
|
||||
return severity
|
||||
}
|
||||
|
||||
func (m *Manager) activeAlertValue(alertID string) (float64, bool) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
alert, ok := m.getActiveAlertNoLock(alertID)
|
||||
if !ok || alert == nil {
|
||||
return 0, false
|
||||
}
|
||||
return alert.Value, true
|
||||
}
|
||||
|
||||
func filterStorageHealthReasonsByCodes(reasons []storagehealth.Reason, codes []string) []storagehealth.Reason {
|
||||
if len(reasons) == 0 || len(codes) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
allowed := make(map[string]struct{}, len(codes))
|
||||
for _, code := range codes {
|
||||
code = strings.TrimSpace(code)
|
||||
if code == "" {
|
||||
continue
|
||||
}
|
||||
allowed[code] = struct{}{}
|
||||
}
|
||||
|
||||
filtered := make([]storagehealth.Reason, 0, len(reasons))
|
||||
for _, reason := range reasons {
|
||||
if _, ok := allowed[strings.TrimSpace(reason.Code)]; ok {
|
||||
filtered = append(filtered, reason)
|
||||
}
|
||||
}
|
||||
return filtered
|
||||
}
|
||||
|
||||
func zfsDeviceAssessment(device models.ZFSDevice) storagehealth.Assessment {
|
||||
assessment := storagehealth.Assessment{Level: storagehealth.RiskHealthy}
|
||||
addReason := func(code string, severity storagehealth.RiskLevel, summary string) {
|
||||
if strings.TrimSpace(summary) == "" {
|
||||
return
|
||||
}
|
||||
assessment.Reasons = append(assessment.Reasons, storagehealth.Reason{
|
||||
Code: code,
|
||||
Severity: severity,
|
||||
Summary: summary,
|
||||
})
|
||||
switch severity {
|
||||
case storagehealth.RiskCritical:
|
||||
assessment.Level = storagehealth.RiskCritical
|
||||
case storagehealth.RiskWarning:
|
||||
if assessment.Level != storagehealth.RiskCritical {
|
||||
assessment.Level = storagehealth.RiskWarning
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
state := strings.ToUpper(strings.TrimSpace(device.State))
|
||||
switch state {
|
||||
case "", "ONLINE", "SPARE":
|
||||
case "DEGRADED":
|
||||
addReason("zfs_device_state", storagehealth.RiskWarning, fmt.Sprintf("ZFS device %s is DEGRADED", device.Name))
|
||||
default:
|
||||
addReason("zfs_device_state", storagehealth.RiskCritical, fmt.Sprintf("ZFS device %s is %s", device.Name, state))
|
||||
}
|
||||
|
||||
errors := device.ReadErrors + device.WriteErrors + device.ChecksumErrors
|
||||
if errors > 0 {
|
||||
addReason(
|
||||
"zfs_device_errors",
|
||||
storagehealth.RiskWarning,
|
||||
fmt.Sprintf("ZFS device %s has errors: %d read, %d write, %d checksum", device.Name, device.ReadErrors, device.WriteErrors, device.ChecksumErrors),
|
||||
)
|
||||
}
|
||||
|
||||
return assessment
|
||||
}
|
||||
|
||||
func (m *Manager) syncCanonicalHealthAssessmentAlert(params canonicalHealthAssessmentAlertParams) (alertspecs.EvaluationResult, bool) {
|
||||
if len(params.Reasons) == 0 {
|
||||
m.clearAlert(buildCanonicalStateID(params.SpecResourceID, params.SpecID))
|
||||
return alertspecs.EvaluationResult{}, true
|
||||
}
|
||||
|
||||
spec, err := buildCanonicalHealthAssessmentSpec(
|
||||
params.SpecID,
|
||||
params.SpecResourceID,
|
||||
params.ResourceName,
|
||||
params.ResourceType,
|
||||
params.Signal,
|
||||
params.Codes,
|
||||
params.Disabled,
|
||||
)
|
||||
if err != nil {
|
||||
log.Warn().
|
||||
Err(err).
|
||||
Str("alertID", params.AlertID).
|
||||
Str("resourceID", params.SpecResourceID).
|
||||
Msg("Skipping invalid canonical health assessment spec")
|
||||
return alertspecs.EvaluationResult{}, false
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
return m.evaluateCanonicalStatefulAlert(canonicalStatefulAlertParams{
|
||||
Spec: spec,
|
||||
Evidence: alertspecs.AlertEvidence{
|
||||
ObservedAt: now,
|
||||
HealthAssessment: &alertspecs.HealthAssessmentEvidence{
|
||||
Signal: params.Signal,
|
||||
Severity: storageHealthAssessmentSeverity(params.Reasons),
|
||||
Codes: storageHealthReasonCodes(params.Reasons),
|
||||
},
|
||||
},
|
||||
AlertID: params.AlertID,
|
||||
AlertType: params.AlertType,
|
||||
ResourceID: params.ResourceID,
|
||||
ResourceName: params.ResourceName,
|
||||
Node: params.Node,
|
||||
Instance: params.Instance,
|
||||
Message: strings.Join(storageHealthReasonSummaries(params.Reasons), "; "),
|
||||
Metadata: cloneMetadata(params.Metadata),
|
||||
AddToRecent: true,
|
||||
AddToHistory: true,
|
||||
MessageBuilder: params.MessageBuilder,
|
||||
})
|
||||
}
|
||||
|
||||
func asyncSaveActiveAlerts(reason string, save func() error) {
|
||||
go func() {
|
||||
defer func() {
|
||||
|
|
|
|||
213
internal/alerts/health_assessment.go
Normal file
213
internal/alerts/health_assessment.go
Normal file
|
|
@ -0,0 +1,213 @@
|
|||
package alerts
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/storagehealth"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
var (
|
||||
zfsPoolAssessmentCodes = []string{
|
||||
"zfs_pool_state",
|
||||
}
|
||||
zfsPoolErrorAssessmentCodes = []string{
|
||||
"zfs_pool_errors",
|
||||
}
|
||||
zfsDeviceAssessmentCodes = []string{
|
||||
"zfs_device_state",
|
||||
"zfs_device_errors",
|
||||
}
|
||||
)
|
||||
|
||||
type canonicalHealthAssessmentAlertParams struct {
|
||||
SpecID string
|
||||
Signal string
|
||||
Codes []string
|
||||
Reasons []storagehealth.Reason
|
||||
AlertID string
|
||||
AlertType string
|
||||
SpecResourceID string
|
||||
ResourceID string
|
||||
ResourceName string
|
||||
ResourceType unifiedresources.ResourceType
|
||||
Node string
|
||||
Instance string
|
||||
Metadata map[string]interface{}
|
||||
Disabled bool
|
||||
MessageBuilder func(alertspecs.EvaluationResult) (string, float64, float64)
|
||||
}
|
||||
|
||||
func storageHealthReasonCodes(reasons []storagehealth.Reason) []string {
|
||||
codes := make([]string, 0, len(reasons))
|
||||
seen := make(map[string]struct{}, len(reasons))
|
||||
for _, reason := range reasons {
|
||||
code := strings.TrimSpace(reason.Code)
|
||||
if code == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[code]; ok {
|
||||
continue
|
||||
}
|
||||
seen[code] = struct{}{}
|
||||
codes = append(codes, code)
|
||||
}
|
||||
slices.Sort(codes)
|
||||
return codes
|
||||
}
|
||||
|
||||
func storageHealthReasonSummaries(reasons []storagehealth.Reason) []string {
|
||||
summaries := make([]string, 0, len(reasons))
|
||||
for _, reason := range reasons {
|
||||
summary := strings.TrimSpace(reason.Summary)
|
||||
if summary == "" {
|
||||
continue
|
||||
}
|
||||
summaries = append(summaries, summary)
|
||||
}
|
||||
return summaries
|
||||
}
|
||||
|
||||
func storageHealthAssessmentSeverity(reasons []storagehealth.Reason) alertspecs.AlertSeverity {
|
||||
severity := alertspecs.AlertSeverity("")
|
||||
for _, reason := range reasons {
|
||||
switch reason.Severity {
|
||||
case storagehealth.RiskCritical:
|
||||
return alertspecs.AlertSeverityCritical
|
||||
case storagehealth.RiskWarning:
|
||||
severity = alertspecs.AlertSeverityWarning
|
||||
}
|
||||
}
|
||||
return severity
|
||||
}
|
||||
|
||||
func (m *Manager) activeAlertValue(alertID string) (float64, bool) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
alert, ok := m.getActiveAlertNoLock(alertID)
|
||||
if !ok || alert == nil {
|
||||
return 0, false
|
||||
}
|
||||
return alert.Value, true
|
||||
}
|
||||
|
||||
func filterStorageHealthReasonsByCodes(reasons []storagehealth.Reason, codes []string) []storagehealth.Reason {
|
||||
if len(reasons) == 0 || len(codes) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
allowed := make(map[string]struct{}, len(codes))
|
||||
for _, code := range codes {
|
||||
code = strings.TrimSpace(code)
|
||||
if code == "" {
|
||||
continue
|
||||
}
|
||||
allowed[code] = struct{}{}
|
||||
}
|
||||
|
||||
filtered := make([]storagehealth.Reason, 0, len(reasons))
|
||||
for _, reason := range reasons {
|
||||
if _, ok := allowed[strings.TrimSpace(reason.Code)]; ok {
|
||||
filtered = append(filtered, reason)
|
||||
}
|
||||
}
|
||||
return filtered
|
||||
}
|
||||
|
||||
func zfsDeviceAssessment(device models.ZFSDevice) storagehealth.Assessment {
|
||||
assessment := storagehealth.Assessment{Level: storagehealth.RiskHealthy}
|
||||
addReason := func(code string, severity storagehealth.RiskLevel, summary string) {
|
||||
if strings.TrimSpace(summary) == "" {
|
||||
return
|
||||
}
|
||||
assessment.Reasons = append(assessment.Reasons, storagehealth.Reason{
|
||||
Code: code,
|
||||
Severity: severity,
|
||||
Summary: summary,
|
||||
})
|
||||
switch severity {
|
||||
case storagehealth.RiskCritical:
|
||||
assessment.Level = storagehealth.RiskCritical
|
||||
case storagehealth.RiskWarning:
|
||||
if assessment.Level != storagehealth.RiskCritical {
|
||||
assessment.Level = storagehealth.RiskWarning
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
state := strings.ToUpper(strings.TrimSpace(device.State))
|
||||
switch state {
|
||||
case "", "ONLINE", "SPARE":
|
||||
case "DEGRADED":
|
||||
addReason("zfs_device_state", storagehealth.RiskWarning, fmt.Sprintf("ZFS device %s is DEGRADED", device.Name))
|
||||
default:
|
||||
addReason("zfs_device_state", storagehealth.RiskCritical, fmt.Sprintf("ZFS device %s is %s", device.Name, state))
|
||||
}
|
||||
|
||||
errors := device.ReadErrors + device.WriteErrors + device.ChecksumErrors
|
||||
if errors > 0 {
|
||||
addReason(
|
||||
"zfs_device_errors",
|
||||
storagehealth.RiskWarning,
|
||||
fmt.Sprintf("ZFS device %s has errors: %d read, %d write, %d checksum", device.Name, device.ReadErrors, device.WriteErrors, device.ChecksumErrors),
|
||||
)
|
||||
}
|
||||
|
||||
return assessment
|
||||
}
|
||||
|
||||
func (m *Manager) syncCanonicalHealthAssessmentAlert(params canonicalHealthAssessmentAlertParams) (alertspecs.EvaluationResult, bool) {
|
||||
if len(params.Reasons) == 0 {
|
||||
m.clearAlert(buildCanonicalStateID(params.SpecResourceID, params.SpecID))
|
||||
return alertspecs.EvaluationResult{}, true
|
||||
}
|
||||
|
||||
spec, err := buildCanonicalHealthAssessmentSpec(
|
||||
params.SpecID,
|
||||
params.SpecResourceID,
|
||||
params.ResourceName,
|
||||
params.ResourceType,
|
||||
params.Signal,
|
||||
params.Codes,
|
||||
params.Disabled,
|
||||
)
|
||||
if err != nil {
|
||||
log.Warn().
|
||||
Err(err).
|
||||
Str("alertID", params.AlertID).
|
||||
Str("resourceID", params.SpecResourceID).
|
||||
Msg("Skipping invalid canonical health assessment spec")
|
||||
return alertspecs.EvaluationResult{}, false
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
return m.evaluateCanonicalStatefulAlert(canonicalStatefulAlertParams{
|
||||
Spec: spec,
|
||||
Evidence: alertspecs.AlertEvidence{
|
||||
ObservedAt: now,
|
||||
HealthAssessment: &alertspecs.HealthAssessmentEvidence{
|
||||
Signal: params.Signal,
|
||||
Severity: storageHealthAssessmentSeverity(params.Reasons),
|
||||
Codes: storageHealthReasonCodes(params.Reasons),
|
||||
},
|
||||
},
|
||||
AlertID: params.AlertID,
|
||||
AlertType: params.AlertType,
|
||||
ResourceID: params.ResourceID,
|
||||
ResourceName: params.ResourceName,
|
||||
Node: params.Node,
|
||||
Instance: params.Instance,
|
||||
Message: strings.Join(storageHealthReasonSummaries(params.Reasons), "; "),
|
||||
Metadata: cloneMetadata(params.Metadata),
|
||||
AddToRecent: true,
|
||||
AddToHistory: true,
|
||||
MessageBuilder: params.MessageBuilder,
|
||||
})
|
||||
}
|
||||
|
|
@ -6,8 +6,22 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/storagehealth"
|
||||
)
|
||||
|
||||
func TestStorageHealthReasonCodesSortsAndDeduplicates(t *testing.T) {
|
||||
got := storageHealthReasonCodes([]storagehealth.Reason{
|
||||
{Code: " zfs_device_errors "},
|
||||
{Code: "zfs_device_state"},
|
||||
{Code: "zfs_device_errors"},
|
||||
{Code: " "},
|
||||
})
|
||||
|
||||
if len(got) != 2 || got[0] != "zfs_device_errors" || got[1] != "zfs_device_state" {
|
||||
t.Fatalf("storageHealthReasonCodes() = %#v, want sorted unique ZFS codes", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSynologyRAIDSuppression(t *testing.T) {
|
||||
m := newTestManager(t)
|
||||
m.ClearActiveAlerts()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue