diff --git a/docs/AI.md b/docs/AI.md index 005ef1533..c0d4dfbe0 100644 --- a/docs/AI.md +++ b/docs/AI.md @@ -127,7 +127,7 @@ Patrol parses tool outputs for concrete evidence such as backup failures, storag | Signal Type | Trigger | Default Threshold | |------------|---------|-------------------| -| `smart_failure` | SMART health status not OK/PASSED | N/A | +| `smart_failure` | SMART health status not OK/PASSED, or critical SMART counters such as pending sectors, offline uncorrectable sectors, or NVMe media errors | N/A | | `high_cpu` | Average CPU usage | 70% | | `high_memory` | Average memory usage | 80% | | `high_disk` | Storage pool usage | 75% (warning), 95% (critical) | @@ -145,7 +145,7 @@ Thresholds can be configured via alert settings to match user-defined values. | **Storage issues** | Critical | PBS datastore errors, ZFS pool degraded | | **Ceph problems** | Warning/Critical | Degraded OSDs, unhealthy PGs | | **Kubernetes issues** | Warning | Pods stuck in Pending/CrashLoopBackOff | -| **SMART failures** | Critical | Disk health check failed | +| **SMART failures** | Critical | Disk health check failed, pending sectors, offline uncorrectable sectors, or NVMe media errors | | **Alert-triggered investigations** | Pro / Cloud | A fired alert prompts the model to gather surrounding context and explain likely cause | ### What Patrol Ignores (by design) diff --git a/docs/UNIFIED_AGENT.md b/docs/UNIFIED_AGENT.md index a6bd9f1e6..70927b9f1 100644 --- a/docs/UNIFIED_AGENT.md +++ b/docs/UNIFIED_AGENT.md @@ -223,9 +223,9 @@ PULSE_DISK_EXCLUDE=/mnt/backup,*pbs*,/media/external - Prefix: `/mnt/ext*` - matches paths starting with `/mnt/ext` - Contains: `*pbs*` - matches paths containing `pbs` -## S.M.A.R.T. Disk Temperatures +## S.M.A.R.T. Disk Health -The agent can report S.M.A.R.T. disk temperatures when running in Agent mode. This requires: +The agent can report S.M.A.R.T. disk temperatures, health status, identity, and normalized health counters when running in Agent mode. This requires: 1. **smartmontools** installed on the host: ```bash @@ -243,7 +243,7 @@ The agent can report S.M.A.R.T. disk temperatures when running in Agent mode. Th **Notes:** - Disks in standby mode are reported as such (no temperature) to avoid waking them -- S.M.A.R.T. data is collected alongside other host metrics +- S.M.A.R.T. data is collected alongside other host metrics and can enrich the Physical Disks view with temperature, stable disk identity, power-on hours, SSD life, pending sectors, media errors, and related counters - If `smartctl` is not available, S.M.A.R.T. monitoring is silently skipped - **Disk exclusions** (`--disk-exclude` / `PULSE_DISK_EXCLUDE`) also apply to S.M.A.R.T. monitoring. Use patterns like `sda`, `/dev/sdb`, `nvme*`, or `*cache*` to exclude specific block devices. diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.md index 41280b881..87a141ea9 100644 --- a/docs/release-control/v6/internal/subsystems/ai-runtime.md +++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md @@ -1058,6 +1058,14 @@ but status selection must read the current workflow status directly rather than deriving an older display status from `workflowStatusHistory`. 1. Add or change chat runtime, Patrol orchestration, findings generation, or remediation behavior through `internal/ai/` + Patrol deterministic disk-health evidence changes in + `internal/ai/patrol_signals.go` and `internal/ai/patrol_ai.go` must read + SMART health, normalized SMART counters, and canonical physical-disk identity + from storage and metrics tool payloads without treating missing or unknown + SMART health as `PASSED`. Non-zero pending-sector, offline-uncorrectable, + media-error, and related SMART counters must remain model-facing reliability + evidence even when the headline health field is `PASSED`, so Patrol does not + summarize a disk as healthy while deterministic counters indicate disk risk. 2. Add or change canonical AI provider config, provider registry metadata, provider-scoped model selection, or runtime auth/base-URL defaults through `internal/config/ai.go` and `internal/config/ai_providers.go`. Direct providers that speak the shared chat-compatible API must use the registry-backed `AIProviderProtocolOpenAICompatible` path and diff --git a/internal/ai/patrol_ai.go b/internal/ai/patrol_ai.go index b51f85bc8..8ae882fc9 100644 --- a/internal/ai/patrol_ai.go +++ b/internal/ai/patrol_ai.go @@ -1757,6 +1757,7 @@ type patrolPhysicalDiskRow struct { devPath, model string health, status string wearout, temperature int + smartEvidence []string } type patrolPrecomputeNodeSource struct { @@ -2286,17 +2287,18 @@ func patrolPhysicalDiskRows(snap patrolRuntimeState, scopedSet map[string]bool) } rows = append(rows, patrolPhysicalDiskRow{ - id: r.ID, - name: name, - node: strings.TrimSpace(r.ParentName), - diskType: strings.TrimSpace(r.PhysicalDisk.DiskType), - sizeBytes: r.PhysicalDisk.SizeBytes, - devPath: strings.TrimSpace(r.PhysicalDisk.DevPath), - model: strings.TrimSpace(r.PhysicalDisk.Model), - health: health, - status: status, - wearout: r.PhysicalDisk.Wearout, - temperature: r.PhysicalDisk.Temperature, + id: r.ID, + name: name, + node: strings.TrimSpace(r.ParentName), + diskType: strings.TrimSpace(r.PhysicalDisk.DiskType), + sizeBytes: r.PhysicalDisk.SizeBytes, + devPath: strings.TrimSpace(r.PhysicalDisk.DevPath), + model: strings.TrimSpace(r.PhysicalDisk.Model), + health: health, + status: status, + wearout: r.PhysicalDisk.Wearout, + temperature: r.PhysicalDisk.Temperature, + smartEvidence: unifiedPhysicalDiskSMARTIssueParts(r.PhysicalDisk.SMART), }) } return rows @@ -2326,22 +2328,81 @@ func patrolPhysicalDiskRows(snap patrolRuntimeState, scopedSet map[string]bool) } rows = append(rows, patrolPhysicalDiskRow{ - id: d.ID, - name: name, - node: strings.TrimSpace(d.Node), - diskType: strings.TrimSpace(d.Type), - sizeBytes: d.Size, - devPath: strings.TrimSpace(d.DevPath), - model: strings.TrimSpace(d.Model), - health: health, - status: status, - wearout: d.Wearout, - temperature: d.Temperature, + id: d.ID, + name: name, + node: strings.TrimSpace(d.Node), + diskType: strings.TrimSpace(d.Type), + sizeBytes: d.Size, + devPath: strings.TrimSpace(d.DevPath), + model: strings.TrimSpace(d.Model), + health: health, + status: status, + wearout: d.Wearout, + temperature: d.Temperature, + smartEvidence: modelPhysicalDiskSMARTIssueParts(d.SmartAttributes), }) } return rows } +func patrolPhysicalDiskHealthIssue(row patrolPhysicalDiskRow) bool { + health := strings.TrimSpace(row.health) + healthIssue := health != "" && + !strings.EqualFold(health, "PASSED") && + !strings.EqualFold(health, "UNKNOWN") && + !strings.EqualFold(health, "OK") + return healthIssue || + (row.wearout > 0 && row.wearout < 20) || + row.temperature > 55 || + len(row.smartEvidence) > 0 +} + +func patrolPhysicalDiskSummaryIssue(row patrolPhysicalDiskRow) bool { + return patrolPhysicalDiskHealthIssue(row) || strings.ToLower(strings.TrimSpace(row.status)) != "online" +} + +func modelPhysicalDiskSMARTIssueParts(attrs *models.SMARTAttributes) []string { + if attrs == nil { + return nil + } + + var parts []string + appendSMARTInt64Issue(&parts, "reallocated sectors", attrs.ReallocatedSectors) + appendSMARTInt64Issue(&parts, "pending sectors", attrs.PendingSectors) + appendSMARTInt64Issue(&parts, "offline uncorrectable", attrs.OfflineUncorrectable) + appendSMARTInt64Issue(&parts, "UDMA CRC errors", attrs.UDMACRCErrors) + appendSMARTInt64Issue(&parts, "media errors", attrs.MediaErrors) + return parts +} + +func unifiedPhysicalDiskSMARTIssueParts(attrs *unifiedresources.SMARTMeta) []string { + if attrs == nil { + return nil + } + + var parts []string + appendSMARTInt64ValueIssue(&parts, "reallocated sectors", attrs.ReallocatedSectors) + appendSMARTInt64ValueIssue(&parts, "pending sectors", attrs.PendingSectors) + appendSMARTInt64ValueIssue(&parts, "offline uncorrectable", attrs.OfflineUncorrectable) + appendSMARTInt64ValueIssue(&parts, "UDMA CRC errors", attrs.UDMACRCErrors) + appendSMARTInt64ValueIssue(&parts, "media errors", attrs.MediaErrors) + return parts +} + +func appendSMARTInt64Issue(parts *[]string, label string, value *int64) { + if value == nil { + return + } + appendSMARTInt64ValueIssue(parts, label, *value) +} + +func appendSMARTInt64ValueIssue(parts *[]string, label string, value int64) { + if value <= 0 { + return + } + *parts = append(*parts, fmt.Sprintf("%s=%d", label, value)) +} + func patrolPrecomputeNodeSources(snap patrolRuntimeState, scopedSet map[string]bool) []patrolPrecomputeNodeSource { nodeRows := patrolNodeInventoryRows(snap, scopedSet) rows := make([]patrolPrecomputeNodeSource, 0, len(nodeRows)) @@ -2618,7 +2679,7 @@ func (p *PatrolService) seedResourceInventoryState(snap patrolRuntimeState, scop if len(diskRows) > 0 { diskIssues := 0 for _, row := range diskRows { - if (!strings.EqualFold(row.health, "PASSED") && !strings.EqualFold(row.health, "UNKNOWN") && !strings.EqualFold(row.health, "OK") && row.health != "") || row.status != "online" { + if patrolPhysicalDiskSummaryIssue(row) { diskIssues++ } } @@ -2868,12 +2929,24 @@ func (p *PatrolService) seedResourceInventorySummaryState(snap patrolRuntimeStat diskIssues := []string{} for _, row := range diskRows { statusCounts[strings.ToLower(strings.TrimSpace(row.status))]++ - if (!strings.EqualFold(row.health, "PASSED") && !strings.EqualFold(row.health, "UNKNOWN") && !strings.EqualFold(row.health, "OK") && row.health != "") || row.status != "online" { + if patrolPhysicalDiskSummaryIssue(row) { name := row.devPath if name == "" { name = row.name } - diskIssues = append(diskIssues, fmt.Sprintf("%s (%s)", name, row.health)) + issue := strings.TrimSpace(row.health) + if issue == "" { + issue = "UNKNOWN" + } + if len(row.smartEvidence) > 0 { + evidence := strings.Join(row.smartEvidence, ", ") + if strings.EqualFold(issue, "PASSED") || strings.EqualFold(issue, "OK") || strings.EqualFold(issue, "UNKNOWN") { + issue = evidence + } else { + issue = issue + "; " + evidence + } + } + diskIssues = append(diskIssues, fmt.Sprintf("%s (%s)", name, issue)) } } @@ -3109,50 +3182,63 @@ func (p *PatrolService) seedHealthAndAlertsState(snap patrolRuntimeState, scoped } // --- Disk Health --- - diskURP := snap.unifiedResourceProvider - if diskURP != nil { - diskResources := diskURP.GetByType(unifiedresources.ResourceTypePhysicalDisk) - if len(diskResources) > 0 { - hasIssues := false - for _, r := range diskResources { - if r.PhysicalDisk == nil { - continue - } - d := r.PhysicalDisk - if (d.Health != "PASSED" && d.Health != "UNKNOWN" && d.Health != "OK" && d.Health != "") || (d.Wearout > 0 && d.Wearout < 20) || d.Temperature > 55 { - hasIssues = true - break - } + diskRows := patrolPhysicalDiskRows(snap, scopedSet) + if len(diskRows) > 0 { + hasIssues := false + unknownHealth := 0 + for _, row := range diskRows { + health := strings.TrimSpace(row.health) + if health == "" || strings.EqualFold(health, "UNKNOWN") { + unknownHealth++ } - sb.WriteString("# Disk Health\n") - if !hasIssues { - sb.WriteString(fmt.Sprintf("All %d disks healthy (SMART PASSED).\n", len(diskResources))) - } else { - sb.WriteString("| Node | Device | Model | Health | Wearout | Temp |\n") - sb.WriteString("|------|--------|-------|--------|---------|------|\n") - for _, r := range diskResources { - if r.PhysicalDisk == nil { - continue - } - d := r.PhysicalDisk - node := r.ParentName - if node == "" && len(r.Identity.Hostnames) > 0 { - node = r.Identity.Hostnames[0] - } - wearout := "—" - if d.Wearout >= 0 { - wearout = fmt.Sprintf("%d%%", d.Wearout) - } - temp := "—" - if d.Temperature > 0 { - temp = fmt.Sprintf("%d°C", d.Temperature) - } - sb.WriteString(fmt.Sprintf("| %s | %s | %s | %s | %s | %s |\n", - node, d.DevPath, d.Model, d.Health, wearout, temp)) - } + if patrolPhysicalDiskHealthIssue(row) { + hasIssues = true } - sb.WriteString("\n") } + sb.WriteString("# Disk Health\n") + if !hasIssues { + if unknownHealth > 0 { + sb.WriteString(fmt.Sprintf("No disk issues detected across %d disks; SMART health is unknown for %d disk(s).\n", len(diskRows), unknownHealth)) + } else { + sb.WriteString(fmt.Sprintf("All %d disks healthy (SMART PASSED/OK).\n", len(diskRows))) + } + } else { + sb.WriteString("| Node | Device | Model | Health | Wearout | Temp | SMART Evidence |\n") + sb.WriteString("|------|--------|-------|--------|---------|------|----------------|\n") + for _, row := range diskRows { + node := row.node + if node == "" { + node = "—" + } + device := row.devPath + if device == "" { + device = row.name + } + model := row.model + if model == "" { + model = "—" + } + health := row.health + if health == "" { + health = "UNKNOWN" + } + wearout := "—" + if row.wearout >= 0 { + wearout = fmt.Sprintf("%d%%", row.wearout) + } + temp := "—" + if row.temperature > 0 { + temp = fmt.Sprintf("%d°C", row.temperature) + } + smartEvidence := "—" + if len(row.smartEvidence) > 0 { + smartEvidence = strings.Join(row.smartEvidence, ", ") + } + sb.WriteString(fmt.Sprintf("| %s | %s | %s | %s | %s | %s | %s |\n", + node, device, model, health, wearout, temp, smartEvidence)) + } + } + sb.WriteString("\n") } // --- Active Alerts --- diff --git a/internal/ai/patrol_ai_more_test.go b/internal/ai/patrol_ai_more_test.go index e92f6a8b6..53584599d 100644 --- a/internal/ai/patrol_ai_more_test.go +++ b/internal/ai/patrol_ai_more_test.go @@ -873,7 +873,7 @@ func TestSeedHealthAndAlerts_NoIssues(t *testing.T) { } out := ps.seedHealthAndAlertsState(patrolRuntimeStateForTest(ps, state), nil, cfg, now) - if !strings.Contains(out, "All 1 disks healthy") { + if !strings.Contains(out, "All 1 disks healthy (SMART PASSED/OK).") { t.Fatalf("expected healthy disk summary, got: %s", out) } if !strings.Contains(out, "All 2 instances connected") { @@ -881,6 +881,83 @@ func TestSeedHealthAndAlerts_NoIssues(t *testing.T) { } } +func TestSeedHealthAndAlerts_UnknownDiskHealthDoesNotClaimSMARTPassed(t *testing.T) { + ps := NewPatrolService(nil, nil) + cfg := DefaultPatrolConfig() + now := time.Now() + + ps.SetUnifiedResourceProvider(&mockUnifiedResourceProvider{ + getByTypeFunc: func(t unifiedresources.ResourceType) []unifiedresources.Resource { + if t != unifiedresources.ResourceTypePhysicalDisk { + return nil + } + return []unifiedresources.Resource{ + { + Name: "disk", + Type: unifiedresources.ResourceTypePhysicalDisk, + ParentName: "node-1", + Status: unifiedresources.StatusOnline, + PhysicalDisk: &unifiedresources.PhysicalDiskMeta{ + DevPath: "/dev/sda", + Model: "disk", + Health: "UNKNOWN", + Wearout: 100, + Temperature: 40, + }, + }, + } + }, + }) + + out := ps.seedHealthAndAlertsState(patrolRuntimeStateForTest(ps, models.StateSnapshot{}), nil, cfg, now) + if !strings.Contains(out, "No disk issues detected across 1 disks; SMART health is unknown for 1 disk(s).") { + t.Fatalf("expected unknown SMART health summary, got: %s", out) + } + if strings.Contains(out, "SMART PASSED") { + t.Fatalf("unknown SMART health must not be reported as passed, got: %s", out) + } +} + +func TestSeedHealthAndAlerts_SMARTAttributeIssue(t *testing.T) { + ps := NewPatrolService(nil, nil) + cfg := DefaultPatrolConfig() + now := time.Now() + + ps.SetUnifiedResourceProvider(&mockUnifiedResourceProvider{ + getByTypeFunc: func(t unifiedresources.ResourceType) []unifiedresources.Resource { + if t != unifiedresources.ResourceTypePhysicalDisk { + return nil + } + return []unifiedresources.Resource{ + { + Name: "disk", + Type: unifiedresources.ResourceTypePhysicalDisk, + ParentName: "node-1", + Status: unifiedresources.StatusOnline, + PhysicalDisk: &unifiedresources.PhysicalDiskMeta{ + DevPath: "/dev/sda", + Model: "disk", + Health: "PASSED", + Wearout: 100, + Temperature: 40, + SMART: &unifiedresources.SMARTMeta{ + PendingSectors: 2, + }, + }, + }, + } + }, + }) + + out := ps.seedHealthAndAlertsState(patrolRuntimeStateForTest(ps, models.StateSnapshot{}), nil, cfg, now) + if strings.Contains(out, "All 1 disks healthy") { + t.Fatalf("SMART counter evidence must not be summarized as all healthy, got: %s", out) + } + if !strings.Contains(out, "SMART Evidence") || !strings.Contains(out, "pending sectors=2") { + t.Fatalf("expected SMART counter evidence in disk health table, got: %s", out) + } +} + func TestSeedHealthAndAlerts_WithIssues(t *testing.T) { ps := NewPatrolService(nil, nil) cfg := DefaultPatrolConfig() diff --git a/internal/ai/patrol_signals.go b/internal/ai/patrol_signals.go index 44af8085e..f4ef7cfe9 100644 --- a/internal/ai/patrol_signals.go +++ b/internal/ai/patrol_signals.go @@ -142,15 +142,34 @@ func detectSignalsFromToolCall(tc *ToolCallRecord, thresholds SignalThresholds) // --- Storage signals --- +type smartAttributeCounters struct { + ReallocatedSectors *int64 `json:"reallocatedSectors"` + PendingSectors *int64 `json:"pendingSectors"` + OfflineUncorrectable *int64 `json:"offlineUncorrectable"` + UDMACRCErrors *int64 `json:"udmaCrcErrors"` + MediaErrors *int64 `json:"mediaErrors"` +} + +type smartDiskSignalData struct { + ID string `json:"id"` + Device string `json:"device"` + DevPath string `json:"dev_path"` + Health string `json:"health"` + Model string `json:"model"` + Serial string `json:"serial"` + Node string `json:"node"` + Attributes smartAttributeCounters `json:"attributes"` + SmartAttributes smartAttributeCounters `json:"smart_attributes"` + SmartAttributesCamel smartAttributeCounters `json:"smartAttributes"` +} + // storageDiskHealth is the minimal struct for parsing disk_health tool output. type storageDiskHealth struct { - Disks []struct { - Device string `json:"device"` - Health string `json:"health"` - Model string `json:"model"` - Serial string `json:"serial"` - Node string `json:"node"` - } `json:"disks"` + Disks []smartDiskSignalData `json:"disks"` + Hosts []struct { + Hostname string `json:"hostname"` + SMART []smartDiskSignalData `json:"smart"` + } `json:"hosts"` Node string `json:"node"` } @@ -264,37 +283,136 @@ func detectSMARTFailures(tc *ToolCallRecord) []DetectedSignal { node := data.Node for _, disk := range data.Disks { - health := strings.ToUpper(strings.TrimSpace(disk.Health)) - if health == "" || health == "UNKNOWN" || health == "PASSED" || health == "OK" { - continue + if signal, ok := smartDiskSignal(tc, disk, node); ok { + signals = append(signals, signal) } - - diskNode := disk.Node - if diskNode == "" { - diskNode = node + } + for _, host := range data.Hosts { + for _, disk := range host.SMART { + if signal, ok := smartDiskSignal(tc, disk, host.Hostname); ok { + signals = append(signals, signal) + } } - - resourceID := diskNode - if resourceID == "" { - resourceID = disk.Device - } - - signals = append(signals, DetectedSignal{ - SignalType: SignalSMARTFailure, - ResourceID: resourceID, - ResourceName: disk.Device, - ResourceType: "node", - SuggestedSeverity: "critical", - Category: string(FindingCategoryReliability), - Summary: "SMART health check: " + health + " for " + disk.Device, - Evidence: truncateEvidence(tc.Output), - ToolCallID: tc.ID, - }) } return signals } +func smartDiskSignal(tc *ToolCallRecord, disk smartDiskSignalData, fallbackNode string) (DetectedSignal, bool) { + health := strings.ToUpper(strings.TrimSpace(disk.Health)) + healthIssue := health != "" && health != "UNKNOWN" && health != "PASSED" && health != "OK" + + attrParts, hasCriticalAttrs := smartAttributeIssueParts(disk.mergedAttributes()) + if !healthIssue && len(attrParts) == 0 { + return DetectedSignal{}, false + } + + resourceID, resourceName, resourceType := smartDiskResource(disk, fallbackNode) + severity := "warning" + if healthIssue || hasCriticalAttrs { + severity = "critical" + } + + var summary string + switch { + case healthIssue && len(attrParts) > 0: + summary = fmt.Sprintf("SMART health check: %s for %s; counters: %s", health, resourceName, strings.Join(attrParts, ", ")) + case healthIssue: + summary = fmt.Sprintf("SMART health check: %s for %s", health, resourceName) + default: + summary = fmt.Sprintf("SMART counters indicate disk risk for %s: %s", resourceName, strings.Join(attrParts, ", ")) + } + + return DetectedSignal{ + SignalType: SignalSMARTFailure, + ResourceID: resourceID, + ResourceName: resourceName, + ResourceType: resourceType, + SuggestedSeverity: severity, + Category: string(FindingCategoryReliability), + Summary: summary, + Evidence: truncateEvidence(tc.Output), + ToolCallID: tc.ID, + }, true +} + +func (d smartDiskSignalData) mergedAttributes() smartAttributeCounters { + merged := smartAttributeCounters{} + merge := func(attrs smartAttributeCounters) { + if merged.ReallocatedSectors == nil { + merged.ReallocatedSectors = attrs.ReallocatedSectors + } + if merged.PendingSectors == nil { + merged.PendingSectors = attrs.PendingSectors + } + if merged.OfflineUncorrectable == nil { + merged.OfflineUncorrectable = attrs.OfflineUncorrectable + } + if merged.UDMACRCErrors == nil { + merged.UDMACRCErrors = attrs.UDMACRCErrors + } + if merged.MediaErrors == nil { + merged.MediaErrors = attrs.MediaErrors + } + } + merge(d.Attributes) + merge(d.SmartAttributes) + merge(d.SmartAttributesCamel) + return merged +} + +func smartAttributeIssueParts(attrs smartAttributeCounters) ([]string, bool) { + var parts []string + hasCritical := false + appendCounter := func(label string, value *int64, critical bool) { + if value == nil || *value <= 0 { + return + } + parts = append(parts, fmt.Sprintf("%s=%d", label, *value)) + if critical { + hasCritical = true + } + } + + appendCounter("reallocated sectors", attrs.ReallocatedSectors, false) + appendCounter("pending sectors", attrs.PendingSectors, true) + appendCounter("offline uncorrectable", attrs.OfflineUncorrectable, true) + appendCounter("UDMA CRC errors", attrs.UDMACRCErrors, false) + appendCounter("media errors", attrs.MediaErrors, true) + return parts, hasCritical +} + +func smartDiskResource(disk smartDiskSignalData, fallbackNode string) (resourceID, resourceName, resourceType string) { + resourceName = strings.TrimSpace(disk.Device) + if resourceName == "" { + resourceName = strings.TrimSpace(disk.DevPath) + } + if resourceName == "" { + resourceName = strings.TrimSpace(disk.Model) + } + if resourceName == "" { + resourceName = strings.TrimSpace(disk.ID) + } + if resourceName == "" { + resourceName = "disk" + } + + resourceID = strings.TrimSpace(disk.ID) + resourceType = "disk" + if resourceID == "" { + node := strings.TrimSpace(disk.Node) + if node == "" { + node = strings.TrimSpace(fallbackNode) + } + resourceID = node + resourceType = "node" + } + if resourceID == "" { + resourceID = resourceName + } + return resourceID, resourceName, resourceType +} + func detectHighDiskUsage(tc *ToolCallRecord, thresholds SignalThresholds) []DetectedSignal { var signals []DetectedSignal @@ -623,8 +741,15 @@ type metricsPerformance struct { MemPercent float64 `json:"mem_percent"` } +type metricsPhysicalDisks struct { + Disks []smartDiskSignalData `json:"disks"` +} + func detectMetricsSignals(tc *ToolCallRecord, thresholds SignalThresholds) []DetectedSignal { inputType := extractInputType(tc.Input) + if inputType == "disks" { + return detectPhysicalDiskMetricSignals(tc) + } if inputType != "performance" && inputType != "" { return nil } @@ -690,6 +815,28 @@ func detectMetricsSignals(tc *ToolCallRecord, thresholds SignalThresholds) []Det } } + if inputType == "" { + signals = append(signals, detectPhysicalDiskMetricSignals(tc)...) + } + + return signals +} + +func detectPhysicalDiskMetricSignals(tc *ToolCallRecord) []DetectedSignal { + var data metricsPhysicalDisks + if err := json.Unmarshal([]byte(tc.Output), &data); err != nil { + if !tryParseEmbeddedJSON(tc.Output, &data) { + log.Debug().Err(err).Str("tool", tc.ToolName).Msg("patrol_signals: failed to parse physical disks output") + return nil + } + } + + var signals []DetectedSignal + for _, disk := range data.Disks { + if signal, ok := smartDiskSignal(tc, disk, disk.Node); ok { + signals = append(signals, signal) + } + } return signals } diff --git a/internal/ai/patrol_signals_test.go b/internal/ai/patrol_signals_test.go index 4a52979f7..b7fba52bd 100644 --- a/internal/ai/patrol_signals_test.go +++ b/internal/ai/patrol_signals_test.go @@ -2,6 +2,7 @@ package ai import ( "encoding/json" + "strings" "testing" "time" ) @@ -70,6 +71,107 @@ func TestDetectSignals_SMARTPassedNoSignal(t *testing.T) { } } +func TestDetectSignals_SMARTAttributeCountersCreateSignal(t *testing.T) { + output, _ := json.Marshal(map[string]interface{}{ + "node": "pve1", + "disks": []map[string]interface{}{ + { + "id": "physical-disk-1", + "device": "/dev/sdc", + "health": "PASSED", + "attributes": map[string]interface{}{ + "pendingSectors": 2, + "offlineUncorrectable": 1, + }, + }, + }, + }) + + toolCalls := []ToolCallRecord{ + { + ID: "tc-smart-attrs", + ToolName: "pulse_storage", + Input: `{"type":"disk_health","node":"pve1"}`, + Output: string(output), + Success: true, + }, + } + + signals := DetectSignals(toolCalls, DefaultSignalThresholds()) + if len(signals) != 1 { + t.Fatalf("expected 1 SMART counter signal, got %d", len(signals)) + } + s := signals[0] + if s.ResourceID != "physical-disk-1" || s.ResourceType != "disk" { + t.Fatalf("expected physical disk resource identity, got id=%q type=%q", s.ResourceID, s.ResourceType) + } + if s.SuggestedSeverity != "critical" { + t.Fatalf("expected critical severity from pending/offline counters, got %q", s.SuggestedSeverity) + } + if !strings.Contains(s.Summary, "pending sectors=2") || !strings.Contains(s.Summary, "offline uncorrectable=1") { + t.Fatalf("expected SMART counter evidence in summary, got %q", s.Summary) + } +} + +func TestDetectSignals_SMARTHostPayloadAndMetricsDisks(t *testing.T) { + storageOutput, _ := json.Marshal(map[string]interface{}{ + "hosts": []map[string]interface{}{ + { + "hostname": "pve-host", + "smart": []map[string]interface{}{ + { + "device": "/dev/sdd", + "health": "PASSED", + "smartAttributes": map[string]interface{}{ + "udmaCrcErrors": 4, + }, + }, + }, + }, + }, + }) + metricsOutput, _ := json.Marshal(map[string]interface{}{ + "disks": []map[string]interface{}{ + { + "id": "disk-nvme", + "device": "/dev/nvme0n1", + "health": "PASSED", + "smart_attributes": map[string]interface{}{ + "mediaErrors": 1, + }, + }, + }, + }) + + toolCalls := []ToolCallRecord{ + { + ID: "tc-storage-smart", + ToolName: "pulse_storage", + Input: `{"type":"disk_health"}`, + Output: string(storageOutput), + Success: true, + }, + { + ID: "tc-metrics-disks", + ToolName: "pulse_metrics", + Input: `{"type":"disks"}`, + Output: string(metricsOutput), + Success: true, + }, + } + + signals := DetectSignals(toolCalls, DefaultSignalThresholds()) + if len(signals) != 2 { + t.Fatalf("expected 2 SMART counter signals, got %d: %+v", len(signals), signals) + } + if signals[0].ResourceID != "pve-host" || signals[0].SuggestedSeverity != "warning" { + t.Fatalf("expected host fallback warning for CRC-only evidence, got %+v", signals[0]) + } + if signals[1].ResourceID != "disk-nvme" || signals[1].SuggestedSeverity != "critical" { + t.Fatalf("expected disk critical media-error evidence, got %+v", signals[1]) + } +} + func TestDetectSignals_HighCPU(t *testing.T) { output, _ := json.Marshal(map[string]interface{}{ "resources": []map[string]interface{}{ diff --git a/scripts/release_control/ai_runtime_docs_policy_test.py b/scripts/release_control/ai_runtime_docs_policy_test.py index 12fab31df..540adbdb0 100644 --- a/scripts/release_control/ai_runtime_docs_policy_test.py +++ b/scripts/release_control/ai_runtime_docs_policy_test.py @@ -100,6 +100,14 @@ class AIRuntimeDocsPolicyTest(unittest.TestCase): self.assertIn("go run ./cmd/eval -scenario resource-context", content) self.assertIn("EVAL_RESOURCE_CONTEXT_FORBIDDEN", content) self.assertIn("Pulse does not convert them into Pulse-authored findings", content) + self.assertIn( + "critical SMART counters such as pending sectors, offline uncorrectable sectors, or NVMe media errors", + normalized_content, + ) + self.assertIn( + "Disk health check failed, pending sectors, offline uncorrectable sectors, or NVMe media errors", + normalized_content, + ) self.assertNotIn("Anthropic** (API key or OAuth)", content) self.assertNotIn("Anthropic API key or OAuth", content) self.assertNotIn("successful remediations (incident memory)", content)