vrr/Pulse
2
0
Fork 0
mirror of https://github.com/rcourtman/Pulse.git synced 2026-07-09 16:00:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

Improve Patrol SMART disk evidence

Browse source
This commit is contained in:
rcourtman 2026-06-29 18:17:24 +01:00
parent 6b7d0b45f9
commit a801f456d6
8 changed files with 531 additions and 103 deletions
Download patch file Download diff file Expand all files Collapse all files

4
docs/AI.md
View file

@ -127,7 +127,7 @@ Patrol parses tool outputs for concrete evidence such as backup failures, storag
| Signal Type | Trigger | Default Threshold |
|------------|---------|-------------------|
| `smart_failure` | SMART health status not OK/PASSED | N/A |
| `smart_failure` | SMART health status not OK/PASSED, or critical SMART counters such as pending sectors, offline uncorrectable sectors, or NVMe media errors | N/A |
| `high_cpu` | Average CPU usage | 70% |
| `high_memory` | Average memory usage | 80% |
| `high_disk` | Storage pool usage | 75% (warning), 95% (critical) |
@ -145,7 +145,7 @@ Thresholds can be configured via alert settings to match user-defined values.
| **Storage issues** | Critical | PBS datastore errors, ZFS pool degraded |
| **Ceph problems** | Warning/Critical | Degraded OSDs, unhealthy PGs |
| **Kubernetes issues** | Warning | Pods stuck in Pending/CrashLoopBackOff |
| **SMART failures** | Critical | Disk health check failed |
| **SMART failures** | Critical | Disk health check failed, pending sectors, offline uncorrectable sectors, or NVMe media errors |
| **Alert-triggered investigations** | Pro / Cloud | A fired alert prompts the model to gather surrounding context and explain likely cause |
### What Patrol Ignores (by design)

6
docs/UNIFIED_AGENT.md
View file

@ -223,9 +223,9 @@ PULSE_DISK_EXCLUDE=/mnt/backup,*pbs*,/media/external
- Prefix: `/mnt/ext*` - matches paths starting with `/mnt/ext`
- Contains: `*pbs*` - matches paths containing `pbs`
## S.M.A.R.T. Disk Temperatures
## S.M.A.R.T. Disk Health
The agent can report S.M.A.R.T. disk temperatures when running in Agent mode. This requires:
The agent can report S.M.A.R.T. disk temperatures, health status, identity, and normalized health counters when running in Agent mode. This requires:
1. **smartmontools** installed on the host:
```bash
@ -243,7 +243,7 @@ The agent can report S.M.A.R.T. disk temperatures when running in Agent mode. Th
**Notes:**
- Disks in standby mode are reported as such (no temperature) to avoid waking them
- S.M.A.R.T. data is collected alongside other host metrics
- S.M.A.R.T. data is collected alongside other host metrics and can enrich the Physical Disks view with temperature, stable disk identity, power-on hours, SSD life, pending sectors, media errors, and related counters
- If `smartctl` is not available, S.M.A.R.T. monitoring is silently skipped
- **Disk exclusions** (`--disk-exclude` / `PULSE_DISK_EXCLUDE`) also apply to S.M.A.R.T. monitoring.
Use patterns like `sda`, `/dev/sdb`, `nvme*`, or `*cache*` to exclude specific block devices.

8
docs/release-control/v6/internal/subsystems/ai-runtime.md
View file

@ -1058,6 +1058,14 @@ but status selection must read the current workflow status directly rather than
deriving an older display status from `workflowStatusHistory`.
1. Add or change chat runtime, Patrol orchestration, findings generation, or remediation behavior through `internal/ai/`
Patrol deterministic disk-health evidence changes in
`internal/ai/patrol_signals.go` and `internal/ai/patrol_ai.go` must read
SMART health, normalized SMART counters, and canonical physical-disk identity
from storage and metrics tool payloads without treating missing or unknown
SMART health as `PASSED`. Non-zero pending-sector, offline-uncorrectable,
media-error, and related SMART counters must remain model-facing reliability
evidence even when the headline health field is `PASSED`, so Patrol does not
summarize a disk as healthy while deterministic counters indicate disk risk.
2. Add or change canonical AI provider config, provider registry metadata, provider-scoped model selection, or runtime auth/base-URL defaults through `internal/config/ai.go` and `internal/config/ai_providers.go`.
Direct providers that speak the shared chat-compatible API must use the
registry-backed `AIProviderProtocolOpenAICompatible` path and

218
internal/ai/patrol_ai.go
View file

@ -1757,6 +1757,7 @@ type patrolPhysicalDiskRow struct {
devPath, model string
health, status string
wearout, temperature int
smartEvidence []string
}
type patrolPrecomputeNodeSource struct {
@ -2286,17 +2287,18 @@ func patrolPhysicalDiskRows(snap patrolRuntimeState, scopedSet map[string]bool)
}
rows = append(rows, patrolPhysicalDiskRow{
id: r.ID,
name: name,
node: strings.TrimSpace(r.ParentName),
diskType: strings.TrimSpace(r.PhysicalDisk.DiskType),
sizeBytes: r.PhysicalDisk.SizeBytes,
devPath: strings.TrimSpace(r.PhysicalDisk.DevPath),
model: strings.TrimSpace(r.PhysicalDisk.Model),
health: health,
status: status,
wearout: r.PhysicalDisk.Wearout,
temperature: r.PhysicalDisk.Temperature,
id: r.ID,
name: name,
node: strings.TrimSpace(r.ParentName),
diskType: strings.TrimSpace(r.PhysicalDisk.DiskType),
sizeBytes: r.PhysicalDisk.SizeBytes,
devPath: strings.TrimSpace(r.PhysicalDisk.DevPath),
model: strings.TrimSpace(r.PhysicalDisk.Model),
health: health,
status: status,
wearout: r.PhysicalDisk.Wearout,
temperature: r.PhysicalDisk.Temperature,
smartEvidence: unifiedPhysicalDiskSMARTIssueParts(r.PhysicalDisk.SMART),
})
}
return rows
@ -2326,22 +2328,81 @@ func patrolPhysicalDiskRows(snap patrolRuntimeState, scopedSet map[string]bool)
}
rows = append(rows, patrolPhysicalDiskRow{
id: d.ID,
name: name,
node: strings.TrimSpace(d.Node),
diskType: strings.TrimSpace(d.Type),
sizeBytes: d.Size,
devPath: strings.TrimSpace(d.DevPath),
model: strings.TrimSpace(d.Model),
health: health,
status: status,
wearout: d.Wearout,
temperature: d.Temperature,
id: d.ID,
name: name,
node: strings.TrimSpace(d.Node),
diskType: strings.TrimSpace(d.Type),
sizeBytes: d.Size,
devPath: strings.TrimSpace(d.DevPath),
model: strings.TrimSpace(d.Model),
health: health,
status: status,
wearout: d.Wearout,
temperature: d.Temperature,
smartEvidence: modelPhysicalDiskSMARTIssueParts(d.SmartAttributes),
})
}
return rows
}
func patrolPhysicalDiskHealthIssue(row patrolPhysicalDiskRow) bool {
health := strings.TrimSpace(row.health)
healthIssue := health != "" &&
!strings.EqualFold(health, "PASSED") &&
!strings.EqualFold(health, "UNKNOWN") &&
!strings.EqualFold(health, "OK")
return healthIssue ||
(row.wearout > 0 && row.wearout < 20) ||
row.temperature > 55 ||
len(row.smartEvidence) > 0
}
func patrolPhysicalDiskSummaryIssue(row patrolPhysicalDiskRow) bool {
return patrolPhysicalDiskHealthIssue(row) || strings.ToLower(strings.TrimSpace(row.status)) != "online"
}
func modelPhysicalDiskSMARTIssueParts(attrs *models.SMARTAttributes) []string {
if attrs == nil {
return nil
}
var parts []string
appendSMARTInt64Issue(&parts, "reallocated sectors", attrs.ReallocatedSectors)
appendSMARTInt64Issue(&parts, "pending sectors", attrs.PendingSectors)
appendSMARTInt64Issue(&parts, "offline uncorrectable", attrs.OfflineUncorrectable)
appendSMARTInt64Issue(&parts, "UDMA CRC errors", attrs.UDMACRCErrors)
appendSMARTInt64Issue(&parts, "media errors", attrs.MediaErrors)
return parts
}
func unifiedPhysicalDiskSMARTIssueParts(attrs *unifiedresources.SMARTMeta) []string {
if attrs == nil {
return nil
}
var parts []string
appendSMARTInt64ValueIssue(&parts, "reallocated sectors", attrs.ReallocatedSectors)
appendSMARTInt64ValueIssue(&parts, "pending sectors", attrs.PendingSectors)
appendSMARTInt64ValueIssue(&parts, "offline uncorrectable", attrs.OfflineUncorrectable)
appendSMARTInt64ValueIssue(&parts, "UDMA CRC errors", attrs.UDMACRCErrors)
appendSMARTInt64ValueIssue(&parts, "media errors", attrs.MediaErrors)
return parts
}
func appendSMARTInt64Issue(parts *[]string, label string, value *int64) {
if value == nil {
return
}
appendSMARTInt64ValueIssue(parts, label, *value)
}
func appendSMARTInt64ValueIssue(parts *[]string, label string, value int64) {
if value <= 0 {
return
}
*parts = append(*parts, fmt.Sprintf("%s=%d", label, value))
}
func patrolPrecomputeNodeSources(snap patrolRuntimeState, scopedSet map[string]bool) []patrolPrecomputeNodeSource {
nodeRows := patrolNodeInventoryRows(snap, scopedSet)
rows := make([]patrolPrecomputeNodeSource, 0, len(nodeRows))
@ -2618,7 +2679,7 @@ func (p *PatrolService) seedResourceInventoryState(snap patrolRuntimeState, scop
if len(diskRows) > 0 {
diskIssues := 0
for _, row := range diskRows {
if (!strings.EqualFold(row.health, "PASSED") && !strings.EqualFold(row.health, "UNKNOWN") && !strings.EqualFold(row.health, "OK") && row.health != "") || row.status != "online" {
if patrolPhysicalDiskSummaryIssue(row) {
diskIssues++
}
}
@ -2868,12 +2929,24 @@ func (p *PatrolService) seedResourceInventorySummaryState(snap patrolRuntimeStat
diskIssues := []string{}
for _, row := range diskRows {
statusCounts[strings.ToLower(strings.TrimSpace(row.status))]++
if (!strings.EqualFold(row.health, "PASSED") && !strings.EqualFold(row.health, "UNKNOWN") && !strings.EqualFold(row.health, "OK") && row.health != "") || row.status != "online" {
if patrolPhysicalDiskSummaryIssue(row) {
name := row.devPath
if name == "" {
name = row.name
}
diskIssues = append(diskIssues, fmt.Sprintf("%s (%s)", name, row.health))
issue := strings.TrimSpace(row.health)
if issue == "" {
issue = "UNKNOWN"
}
if len(row.smartEvidence) > 0 {
evidence := strings.Join(row.smartEvidence, ", ")
if strings.EqualFold(issue, "PASSED") || strings.EqualFold(issue, "OK") || strings.EqualFold(issue, "UNKNOWN") {
issue = evidence
} else {
issue = issue + "; " + evidence
}
}
diskIssues = append(diskIssues, fmt.Sprintf("%s (%s)", name, issue))
}
}
@ -3109,50 +3182,63 @@ func (p *PatrolService) seedHealthAndAlertsState(snap patrolRuntimeState, scoped
}
// --- Disk Health ---
diskURP := snap.unifiedResourceProvider
if diskURP != nil {
diskResources := diskURP.GetByType(unifiedresources.ResourceTypePhysicalDisk)
if len(diskResources) > 0 {
hasIssues := false
for _, r := range diskResources {
if r.PhysicalDisk == nil {
continue
}
d := r.PhysicalDisk
if (d.Health != "PASSED" && d.Health != "UNKNOWN" && d.Health != "OK" && d.Health != "") || (d.Wearout > 0 && d.Wearout < 20) || d.Temperature > 55 {
hasIssues = true
break
}
diskRows := patrolPhysicalDiskRows(snap, scopedSet)
if len(diskRows) > 0 {
hasIssues := false
unknownHealth := 0
for _, row := range diskRows {
health := strings.TrimSpace(row.health)
if health == "" || strings.EqualFold(health, "UNKNOWN") {
unknownHealth++
}
sb.WriteString("# Disk Health\n")
if !hasIssues {
sb.WriteString(fmt.Sprintf("All %d disks healthy (SMART PASSED).\n", len(diskResources)))
} else {
sb.WriteString("| Node | Device | Model | Health | Wearout | Temp |\n")
sb.WriteString("|------|--------|-------|--------|---------|------|\n")
for _, r := range diskResources {
if r.PhysicalDisk == nil {
continue
}
d := r.PhysicalDisk
node := r.ParentName
if node == "" && len(r.Identity.Hostnames) > 0 {
node = r.Identity.Hostnames[0]
}
wearout := "—"
if d.Wearout >= 0 {
wearout = fmt.Sprintf("%d%%", d.Wearout)
}
temp := "—"
if d.Temperature > 0 {
temp = fmt.Sprintf("%d°C", d.Temperature)
}
sb.WriteString(fmt.Sprintf("| %s | %s | %s | %s | %s | %s |\n",
node, d.DevPath, d.Model, d.Health, wearout, temp))
}
if patrolPhysicalDiskHealthIssue(row) {
hasIssues = true
}
sb.WriteString("\n")
}
sb.WriteString("# Disk Health\n")
if !hasIssues {
if unknownHealth > 0 {
sb.WriteString(fmt.Sprintf("No disk issues detected across %d disks; SMART health is unknown for %d disk(s).\n", len(diskRows), unknownHealth))
} else {
sb.WriteString(fmt.Sprintf("All %d disks healthy (SMART PASSED/OK).\n", len(diskRows)))
}
} else {
sb.WriteString("| Node | Device | Model | Health | Wearout | Temp | SMART Evidence |\n")
sb.WriteString("|------|--------|-------|--------|---------|------|----------------|\n")
for _, row := range diskRows {
node := row.node
if node == "" {
node = "—"
}
device := row.devPath
if device == "" {
device = row.name
}
model := row.model
if model == "" {
model = "—"
}
health := row.health
if health == "" {
health = "UNKNOWN"
}
wearout := "—"
if row.wearout >= 0 {
wearout = fmt.Sprintf("%d%%", row.wearout)
}
temp := "—"
if row.temperature > 0 {
temp = fmt.Sprintf("%d°C", row.temperature)
}
smartEvidence := "—"
if len(row.smartEvidence) > 0 {
smartEvidence = strings.Join(row.smartEvidence, ", ")
}
sb.WriteString(fmt.Sprintf("| %s | %s | %s | %s | %s | %s | %s |\n",
node, device, model, health, wearout, temp, smartEvidence))
}
}
sb.WriteString("\n")
}
// --- Active Alerts ---

79
internal/ai/patrol_ai_more_test.go
View file

@ -873,7 +873,7 @@ func TestSeedHealthAndAlerts_NoIssues(t *testing.T) {
}
out := ps.seedHealthAndAlertsState(patrolRuntimeStateForTest(ps, state), nil, cfg, now)
if !strings.Contains(out, "All 1 disks healthy") {
if !strings.Contains(out, "All 1 disks healthy (SMART PASSED/OK).") {
t.Fatalf("expected healthy disk summary, got: %s", out)
}
if !strings.Contains(out, "All 2 instances connected") {
@ -881,6 +881,83 @@ func TestSeedHealthAndAlerts_NoIssues(t *testing.T) {
}
}
func TestSeedHealthAndAlerts_UnknownDiskHealthDoesNotClaimSMARTPassed(t *testing.T) {
ps := NewPatrolService(nil, nil)
cfg := DefaultPatrolConfig()
now := time.Now()
ps.SetUnifiedResourceProvider(&mockUnifiedResourceProvider{
getByTypeFunc: func(t unifiedresources.ResourceType) []unifiedresources.Resource {
if t != unifiedresources.ResourceTypePhysicalDisk {
return nil
}
return []unifiedresources.Resource{
{
Name: "disk",
Type: unifiedresources.ResourceTypePhysicalDisk,
ParentName: "node-1",
Status: unifiedresources.StatusOnline,
PhysicalDisk: &unifiedresources.PhysicalDiskMeta{
DevPath: "/dev/sda",
Model: "disk",
Health: "UNKNOWN",
Wearout: 100,
Temperature: 40,
},
},
}
},
})
out := ps.seedHealthAndAlertsState(patrolRuntimeStateForTest(ps, models.StateSnapshot{}), nil, cfg, now)
if !strings.Contains(out, "No disk issues detected across 1 disks; SMART health is unknown for 1 disk(s).") {
t.Fatalf("expected unknown SMART health summary, got: %s", out)
}
if strings.Contains(out, "SMART PASSED") {
t.Fatalf("unknown SMART health must not be reported as passed, got: %s", out)
}
}
func TestSeedHealthAndAlerts_SMARTAttributeIssue(t *testing.T) {
ps := NewPatrolService(nil, nil)
cfg := DefaultPatrolConfig()
now := time.Now()
ps.SetUnifiedResourceProvider(&mockUnifiedResourceProvider{
getByTypeFunc: func(t unifiedresources.ResourceType) []unifiedresources.Resource {
if t != unifiedresources.ResourceTypePhysicalDisk {
return nil
}
return []unifiedresources.Resource{
{
Name: "disk",
Type: unifiedresources.ResourceTypePhysicalDisk,
ParentName: "node-1",
Status: unifiedresources.StatusOnline,
PhysicalDisk: &unifiedresources.PhysicalDiskMeta{
DevPath: "/dev/sda",
Model: "disk",
Health: "PASSED",
Wearout: 100,
Temperature: 40,
SMART: &unifiedresources.SMARTMeta{
PendingSectors: 2,
},
},
},
}
},
})
out := ps.seedHealthAndAlertsState(patrolRuntimeStateForTest(ps, models.StateSnapshot{}), nil, cfg, now)
if strings.Contains(out, "All 1 disks healthy") {
t.Fatalf("SMART counter evidence must not be summarized as all healthy, got: %s", out)
}
if !strings.Contains(out, "SMART Evidence") || !strings.Contains(out, "pending sectors=2") {
t.Fatalf("expected SMART counter evidence in disk health table, got: %s", out)
}
}
func TestSeedHealthAndAlerts_WithIssues(t *testing.T) {
ps := NewPatrolService(nil, nil)
cfg := DefaultPatrolConfig()

209
internal/ai/patrol_signals.go
View file

@ -142,15 +142,34 @@ func detectSignalsFromToolCall(tc *ToolCallRecord, thresholds SignalThresholds)
// --- Storage signals ---
type smartAttributeCounters struct {
ReallocatedSectors *int64 `json:"reallocatedSectors"`
PendingSectors *int64 `json:"pendingSectors"`
OfflineUncorrectable *int64 `json:"offlineUncorrectable"`
UDMACRCErrors *int64 `json:"udmaCrcErrors"`
MediaErrors *int64 `json:"mediaErrors"`
}
type smartDiskSignalData struct {
ID string `json:"id"`
Device string `json:"device"`
DevPath string `json:"dev_path"`
Health string `json:"health"`
Model string `json:"model"`
Serial string `json:"serial"`
Node string `json:"node"`
Attributes smartAttributeCounters `json:"attributes"`
SmartAttributes smartAttributeCounters `json:"smart_attributes"`
SmartAttributesCamel smartAttributeCounters `json:"smartAttributes"`
}
// storageDiskHealth is the minimal struct for parsing disk_health tool output.
type storageDiskHealth struct {
Disks []struct {
Device string `json:"device"`
Health string `json:"health"`
Model string `json:"model"`
Serial string `json:"serial"`
Node string `json:"node"`
} `json:"disks"`
Disks []smartDiskSignalData `json:"disks"`
Hosts []struct {
Hostname string `json:"hostname"`
SMART []smartDiskSignalData `json:"smart"`
} `json:"hosts"`
Node string `json:"node"`
}
@ -264,37 +283,136 @@ func detectSMARTFailures(tc *ToolCallRecord) []DetectedSignal {
node := data.Node
for _, disk := range data.Disks {
health := strings.ToUpper(strings.TrimSpace(disk.Health))
if health == "" || health == "UNKNOWN" || health == "PASSED" || health == "OK" {
continue
if signal, ok := smartDiskSignal(tc, disk, node); ok {
signals = append(signals, signal)
}
diskNode := disk.Node
if diskNode == "" {
diskNode = node
}
for _, host := range data.Hosts {
for _, disk := range host.SMART {
if signal, ok := smartDiskSignal(tc, disk, host.Hostname); ok {
signals = append(signals, signal)
}
}
resourceID := diskNode
if resourceID == "" {
resourceID = disk.Device
}
signals = append(signals, DetectedSignal{
SignalType: SignalSMARTFailure,
ResourceID: resourceID,
ResourceName: disk.Device,
ResourceType: "node",
SuggestedSeverity: "critical",
Category: string(FindingCategoryReliability),
Summary: "SMART health check: " + health + " for " + disk.Device,
Evidence: truncateEvidence(tc.Output),
ToolCallID: tc.ID,
})
}
return signals
}
func smartDiskSignal(tc *ToolCallRecord, disk smartDiskSignalData, fallbackNode string) (DetectedSignal, bool) {
health := strings.ToUpper(strings.TrimSpace(disk.Health))
healthIssue := health != "" && health != "UNKNOWN" && health != "PASSED" && health != "OK"
attrParts, hasCriticalAttrs := smartAttributeIssueParts(disk.mergedAttributes())
if !healthIssue && len(attrParts) == 0 {
return DetectedSignal{}, false
}
resourceID, resourceName, resourceType := smartDiskResource(disk, fallbackNode)
severity := "warning"
if healthIssue || hasCriticalAttrs {
severity = "critical"
}
var summary string
switch {
case healthIssue && len(attrParts) > 0:
summary = fmt.Sprintf("SMART health check: %s for %s; counters: %s", health, resourceName, strings.Join(attrParts, ", "))
case healthIssue:
summary = fmt.Sprintf("SMART health check: %s for %s", health, resourceName)
default:
summary = fmt.Sprintf("SMART counters indicate disk risk for %s: %s", resourceName, strings.Join(attrParts, ", "))
}
return DetectedSignal{
SignalType: SignalSMARTFailure,
ResourceID: resourceID,
ResourceName: resourceName,
ResourceType: resourceType,
SuggestedSeverity: severity,
Category: string(FindingCategoryReliability),
Summary: summary,
Evidence: truncateEvidence(tc.Output),
ToolCallID: tc.ID,
}, true
}
func (d smartDiskSignalData) mergedAttributes() smartAttributeCounters {
merged := smartAttributeCounters{}
merge := func(attrs smartAttributeCounters) {
if merged.ReallocatedSectors == nil {
merged.ReallocatedSectors = attrs.ReallocatedSectors
}
if merged.PendingSectors == nil {
merged.PendingSectors = attrs.PendingSectors
}
if merged.OfflineUncorrectable == nil {
merged.OfflineUncorrectable = attrs.OfflineUncorrectable
}
if merged.UDMACRCErrors == nil {
merged.UDMACRCErrors = attrs.UDMACRCErrors
}
if merged.MediaErrors == nil {
merged.MediaErrors = attrs.MediaErrors
}
}
merge(d.Attributes)
merge(d.SmartAttributes)
merge(d.SmartAttributesCamel)
return merged
}
func smartAttributeIssueParts(attrs smartAttributeCounters) ([]string, bool) {
var parts []string
hasCritical := false
appendCounter := func(label string, value *int64, critical bool) {
if value == nil || *value <= 0 {
return
}
parts = append(parts, fmt.Sprintf("%s=%d", label, *value))
if critical {
hasCritical = true
}
}
appendCounter("reallocated sectors", attrs.ReallocatedSectors, false)
appendCounter("pending sectors", attrs.PendingSectors, true)
appendCounter("offline uncorrectable", attrs.OfflineUncorrectable, true)
appendCounter("UDMA CRC errors", attrs.UDMACRCErrors, false)
appendCounter("media errors", attrs.MediaErrors, true)
return parts, hasCritical
}
func smartDiskResource(disk smartDiskSignalData, fallbackNode string) (resourceID, resourceName, resourceType string) {
resourceName = strings.TrimSpace(disk.Device)
if resourceName == "" {
resourceName = strings.TrimSpace(disk.DevPath)
}
if resourceName == "" {
resourceName = strings.TrimSpace(disk.Model)
}
if resourceName == "" {
resourceName = strings.TrimSpace(disk.ID)
}
if resourceName == "" {
resourceName = "disk"
}
resourceID = strings.TrimSpace(disk.ID)
resourceType = "disk"
if resourceID == "" {
node := strings.TrimSpace(disk.Node)
if node == "" {
node = strings.TrimSpace(fallbackNode)
}
resourceID = node
resourceType = "node"
}
if resourceID == "" {
resourceID = resourceName
}
return resourceID, resourceName, resourceType
}
func detectHighDiskUsage(tc *ToolCallRecord, thresholds SignalThresholds) []DetectedSignal {
var signals []DetectedSignal
@ -623,8 +741,15 @@ type metricsPerformance struct {
MemPercent float64 `json:"mem_percent"`
}
type metricsPhysicalDisks struct {
Disks []smartDiskSignalData `json:"disks"`
}
func detectMetricsSignals(tc *ToolCallRecord, thresholds SignalThresholds) []DetectedSignal {
inputType := extractInputType(tc.Input)
if inputType == "disks" {
return detectPhysicalDiskMetricSignals(tc)
}
if inputType != "performance" && inputType != "" {
return nil
}
@ -690,6 +815,28 @@ func detectMetricsSignals(tc *ToolCallRecord, thresholds SignalThresholds) []Det
}
}
if inputType == "" {
signals = append(signals, detectPhysicalDiskMetricSignals(tc)...)
}
return signals
}
func detectPhysicalDiskMetricSignals(tc *ToolCallRecord) []DetectedSignal {
var data metricsPhysicalDisks
if err := json.Unmarshal([]byte(tc.Output), &data); err != nil {
if !tryParseEmbeddedJSON(tc.Output, &data) {
log.Debug().Err(err).Str("tool", tc.ToolName).Msg("patrol_signals: failed to parse physical disks output")
return nil
}
}
var signals []DetectedSignal
for _, disk := range data.Disks {
if signal, ok := smartDiskSignal(tc, disk, disk.Node); ok {
signals = append(signals, signal)
}
}
return signals
}

102
internal/ai/patrol_signals_test.go
View file

@ -2,6 +2,7 @@ package ai
import (
"encoding/json"
"strings"
"testing"
"time"
)
@ -70,6 +71,107 @@ func TestDetectSignals_SMARTPassedNoSignal(t *testing.T) {
}
}
func TestDetectSignals_SMARTAttributeCountersCreateSignal(t *testing.T) {
output, _ := json.Marshal(map[string]interface{}{
"node": "pve1",
"disks": []map[string]interface{}{
{
"id": "physical-disk-1",
"device": "/dev/sdc",
"health": "PASSED",
"attributes": map[string]interface{}{
"pendingSectors": 2,
"offlineUncorrectable": 1,
},
},
},
})
toolCalls := []ToolCallRecord{
{
ID: "tc-smart-attrs",
ToolName: "pulse_storage",
Input: `{"type":"disk_health","node":"pve1"}`,
Output: string(output),
Success: true,
},
}
signals := DetectSignals(toolCalls, DefaultSignalThresholds())
if len(signals) != 1 {
t.Fatalf("expected 1 SMART counter signal, got %d", len(signals))
}
s := signals[0]
if s.ResourceID != "physical-disk-1" || s.ResourceType != "disk" {
t.Fatalf("expected physical disk resource identity, got id=%q type=%q", s.ResourceID, s.ResourceType)
}
if s.SuggestedSeverity != "critical" {
t.Fatalf("expected critical severity from pending/offline counters, got %q", s.SuggestedSeverity)
}
if !strings.Contains(s.Summary, "pending sectors=2") || !strings.Contains(s.Summary, "offline uncorrectable=1") {
t.Fatalf("expected SMART counter evidence in summary, got %q", s.Summary)
}
}
func TestDetectSignals_SMARTHostPayloadAndMetricsDisks(t *testing.T) {
storageOutput, _ := json.Marshal(map[string]interface{}{
"hosts": []map[string]interface{}{
{
"hostname": "pve-host",
"smart": []map[string]interface{}{
{
"device": "/dev/sdd",
"health": "PASSED",
"smartAttributes": map[string]interface{}{
"udmaCrcErrors": 4,
},
},
},
},
},
})
metricsOutput, _ := json.Marshal(map[string]interface{}{
"disks": []map[string]interface{}{
{
"id": "disk-nvme",
"device": "/dev/nvme0n1",
"health": "PASSED",
"smart_attributes": map[string]interface{}{
"mediaErrors": 1,
},
},
},
})
toolCalls := []ToolCallRecord{
{
ID: "tc-storage-smart",
ToolName: "pulse_storage",
Input: `{"type":"disk_health"}`,
Output: string(storageOutput),
Success: true,
},
{
ID: "tc-metrics-disks",
ToolName: "pulse_metrics",
Input: `{"type":"disks"}`,
Output: string(metricsOutput),
Success: true,
},
}
signals := DetectSignals(toolCalls, DefaultSignalThresholds())
if len(signals) != 2 {
t.Fatalf("expected 2 SMART counter signals, got %d: %+v", len(signals), signals)
}
if signals[0].ResourceID != "pve-host" || signals[0].SuggestedSeverity != "warning" {
t.Fatalf("expected host fallback warning for CRC-only evidence, got %+v", signals[0])
}
if signals[1].ResourceID != "disk-nvme" || signals[1].SuggestedSeverity != "critical" {
t.Fatalf("expected disk critical media-error evidence, got %+v", signals[1])
}
}
func TestDetectSignals_HighCPU(t *testing.T) {
output, _ := json.Marshal(map[string]interface{}{
"resources": []map[string]interface{}{

8
scripts/release_control/ai_runtime_docs_policy_test.py
View file

@ -100,6 +100,14 @@ class AIRuntimeDocsPolicyTest(unittest.TestCase):
self.assertIn("go run ./cmd/eval -scenario resource-context", content)
self.assertIn("EVAL_RESOURCE_CONTEXT_FORBIDDEN", content)
self.assertIn("Pulse does not convert them into Pulse-authored findings", content)
self.assertIn(
"critical SMART counters such as pending sectors, offline uncorrectable sectors, or NVMe media errors",
normalized_content,
)
self.assertIn(
"Disk health check failed, pending sectors, offline uncorrectable sectors, or NVMe media errors",
normalized_content,
)
self.assertNotIn("Anthropic** (API key or OAuth)", content)
self.assertNotIn("Anthropic API key or OAuth", content)
self.assertNotIn("successful remediations (incident memory)", content)