mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-07-09 16:00:59 +00:00
Improve Patrol SMART disk evidence
This commit is contained in:
parent
6b7d0b45f9
commit
a801f456d6
8 changed files with 531 additions and 103 deletions
|
|
@ -127,7 +127,7 @@ Patrol parses tool outputs for concrete evidence such as backup failures, storag
|
||||||
|
|
||||||
| Signal Type | Trigger | Default Threshold |
|
| Signal Type | Trigger | Default Threshold |
|
||||||
|------------|---------|-------------------|
|
|------------|---------|-------------------|
|
||||||
| `smart_failure` | SMART health status not OK/PASSED | N/A |
|
| `smart_failure` | SMART health status not OK/PASSED, or critical SMART counters such as pending sectors, offline uncorrectable sectors, or NVMe media errors | N/A |
|
||||||
| `high_cpu` | Average CPU usage | 70% |
|
| `high_cpu` | Average CPU usage | 70% |
|
||||||
| `high_memory` | Average memory usage | 80% |
|
| `high_memory` | Average memory usage | 80% |
|
||||||
| `high_disk` | Storage pool usage | 75% (warning), 95% (critical) |
|
| `high_disk` | Storage pool usage | 75% (warning), 95% (critical) |
|
||||||
|
|
@ -145,7 +145,7 @@ Thresholds can be configured via alert settings to match user-defined values.
|
||||||
| **Storage issues** | Critical | PBS datastore errors, ZFS pool degraded |
|
| **Storage issues** | Critical | PBS datastore errors, ZFS pool degraded |
|
||||||
| **Ceph problems** | Warning/Critical | Degraded OSDs, unhealthy PGs |
|
| **Ceph problems** | Warning/Critical | Degraded OSDs, unhealthy PGs |
|
||||||
| **Kubernetes issues** | Warning | Pods stuck in Pending/CrashLoopBackOff |
|
| **Kubernetes issues** | Warning | Pods stuck in Pending/CrashLoopBackOff |
|
||||||
| **SMART failures** | Critical | Disk health check failed |
|
| **SMART failures** | Critical | Disk health check failed, pending sectors, offline uncorrectable sectors, or NVMe media errors |
|
||||||
| **Alert-triggered investigations** | Pro / Cloud | A fired alert prompts the model to gather surrounding context and explain likely cause |
|
| **Alert-triggered investigations** | Pro / Cloud | A fired alert prompts the model to gather surrounding context and explain likely cause |
|
||||||
|
|
||||||
### What Patrol Ignores (by design)
|
### What Patrol Ignores (by design)
|
||||||
|
|
|
||||||
|
|
@ -223,9 +223,9 @@ PULSE_DISK_EXCLUDE=/mnt/backup,*pbs*,/media/external
|
||||||
- Prefix: `/mnt/ext*` - matches paths starting with `/mnt/ext`
|
- Prefix: `/mnt/ext*` - matches paths starting with `/mnt/ext`
|
||||||
- Contains: `*pbs*` - matches paths containing `pbs`
|
- Contains: `*pbs*` - matches paths containing `pbs`
|
||||||
|
|
||||||
## S.M.A.R.T. Disk Temperatures
|
## S.M.A.R.T. Disk Health
|
||||||
|
|
||||||
The agent can report S.M.A.R.T. disk temperatures when running in Agent mode. This requires:
|
The agent can report S.M.A.R.T. disk temperatures, health status, identity, and normalized health counters when running in Agent mode. This requires:
|
||||||
|
|
||||||
1. **smartmontools** installed on the host:
|
1. **smartmontools** installed on the host:
|
||||||
```bash
|
```bash
|
||||||
|
|
@ -243,7 +243,7 @@ The agent can report S.M.A.R.T. disk temperatures when running in Agent mode. Th
|
||||||
|
|
||||||
**Notes:**
|
**Notes:**
|
||||||
- Disks in standby mode are reported as such (no temperature) to avoid waking them
|
- Disks in standby mode are reported as such (no temperature) to avoid waking them
|
||||||
- S.M.A.R.T. data is collected alongside other host metrics
|
- S.M.A.R.T. data is collected alongside other host metrics and can enrich the Physical Disks view with temperature, stable disk identity, power-on hours, SSD life, pending sectors, media errors, and related counters
|
||||||
- If `smartctl` is not available, S.M.A.R.T. monitoring is silently skipped
|
- If `smartctl` is not available, S.M.A.R.T. monitoring is silently skipped
|
||||||
- **Disk exclusions** (`--disk-exclude` / `PULSE_DISK_EXCLUDE`) also apply to S.M.A.R.T. monitoring.
|
- **Disk exclusions** (`--disk-exclude` / `PULSE_DISK_EXCLUDE`) also apply to S.M.A.R.T. monitoring.
|
||||||
Use patterns like `sda`, `/dev/sdb`, `nvme*`, or `*cache*` to exclude specific block devices.
|
Use patterns like `sda`, `/dev/sdb`, `nvme*`, or `*cache*` to exclude specific block devices.
|
||||||
|
|
|
||||||
|
|
@ -1058,6 +1058,14 @@ but status selection must read the current workflow status directly rather than
|
||||||
deriving an older display status from `workflowStatusHistory`.
|
deriving an older display status from `workflowStatusHistory`.
|
||||||
|
|
||||||
1. Add or change chat runtime, Patrol orchestration, findings generation, or remediation behavior through `internal/ai/`
|
1. Add or change chat runtime, Patrol orchestration, findings generation, or remediation behavior through `internal/ai/`
|
||||||
|
Patrol deterministic disk-health evidence changes in
|
||||||
|
`internal/ai/patrol_signals.go` and `internal/ai/patrol_ai.go` must read
|
||||||
|
SMART health, normalized SMART counters, and canonical physical-disk identity
|
||||||
|
from storage and metrics tool payloads without treating missing or unknown
|
||||||
|
SMART health as `PASSED`. Non-zero pending-sector, offline-uncorrectable,
|
||||||
|
media-error, and related SMART counters must remain model-facing reliability
|
||||||
|
evidence even when the headline health field is `PASSED`, so Patrol does not
|
||||||
|
summarize a disk as healthy while deterministic counters indicate disk risk.
|
||||||
2. Add or change canonical AI provider config, provider registry metadata, provider-scoped model selection, or runtime auth/base-URL defaults through `internal/config/ai.go` and `internal/config/ai_providers.go`.
|
2. Add or change canonical AI provider config, provider registry metadata, provider-scoped model selection, or runtime auth/base-URL defaults through `internal/config/ai.go` and `internal/config/ai_providers.go`.
|
||||||
Direct providers that speak the shared chat-compatible API must use the
|
Direct providers that speak the shared chat-compatible API must use the
|
||||||
registry-backed `AIProviderProtocolOpenAICompatible` path and
|
registry-backed `AIProviderProtocolOpenAICompatible` path and
|
||||||
|
|
|
||||||
|
|
@ -1757,6 +1757,7 @@ type patrolPhysicalDiskRow struct {
|
||||||
devPath, model string
|
devPath, model string
|
||||||
health, status string
|
health, status string
|
||||||
wearout, temperature int
|
wearout, temperature int
|
||||||
|
smartEvidence []string
|
||||||
}
|
}
|
||||||
|
|
||||||
type patrolPrecomputeNodeSource struct {
|
type patrolPrecomputeNodeSource struct {
|
||||||
|
|
@ -2286,17 +2287,18 @@ func patrolPhysicalDiskRows(snap patrolRuntimeState, scopedSet map[string]bool)
|
||||||
}
|
}
|
||||||
|
|
||||||
rows = append(rows, patrolPhysicalDiskRow{
|
rows = append(rows, patrolPhysicalDiskRow{
|
||||||
id: r.ID,
|
id: r.ID,
|
||||||
name: name,
|
name: name,
|
||||||
node: strings.TrimSpace(r.ParentName),
|
node: strings.TrimSpace(r.ParentName),
|
||||||
diskType: strings.TrimSpace(r.PhysicalDisk.DiskType),
|
diskType: strings.TrimSpace(r.PhysicalDisk.DiskType),
|
||||||
sizeBytes: r.PhysicalDisk.SizeBytes,
|
sizeBytes: r.PhysicalDisk.SizeBytes,
|
||||||
devPath: strings.TrimSpace(r.PhysicalDisk.DevPath),
|
devPath: strings.TrimSpace(r.PhysicalDisk.DevPath),
|
||||||
model: strings.TrimSpace(r.PhysicalDisk.Model),
|
model: strings.TrimSpace(r.PhysicalDisk.Model),
|
||||||
health: health,
|
health: health,
|
||||||
status: status,
|
status: status,
|
||||||
wearout: r.PhysicalDisk.Wearout,
|
wearout: r.PhysicalDisk.Wearout,
|
||||||
temperature: r.PhysicalDisk.Temperature,
|
temperature: r.PhysicalDisk.Temperature,
|
||||||
|
smartEvidence: unifiedPhysicalDiskSMARTIssueParts(r.PhysicalDisk.SMART),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return rows
|
return rows
|
||||||
|
|
@ -2326,22 +2328,81 @@ func patrolPhysicalDiskRows(snap patrolRuntimeState, scopedSet map[string]bool)
|
||||||
}
|
}
|
||||||
|
|
||||||
rows = append(rows, patrolPhysicalDiskRow{
|
rows = append(rows, patrolPhysicalDiskRow{
|
||||||
id: d.ID,
|
id: d.ID,
|
||||||
name: name,
|
name: name,
|
||||||
node: strings.TrimSpace(d.Node),
|
node: strings.TrimSpace(d.Node),
|
||||||
diskType: strings.TrimSpace(d.Type),
|
diskType: strings.TrimSpace(d.Type),
|
||||||
sizeBytes: d.Size,
|
sizeBytes: d.Size,
|
||||||
devPath: strings.TrimSpace(d.DevPath),
|
devPath: strings.TrimSpace(d.DevPath),
|
||||||
model: strings.TrimSpace(d.Model),
|
model: strings.TrimSpace(d.Model),
|
||||||
health: health,
|
health: health,
|
||||||
status: status,
|
status: status,
|
||||||
wearout: d.Wearout,
|
wearout: d.Wearout,
|
||||||
temperature: d.Temperature,
|
temperature: d.Temperature,
|
||||||
|
smartEvidence: modelPhysicalDiskSMARTIssueParts(d.SmartAttributes),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return rows
|
return rows
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func patrolPhysicalDiskHealthIssue(row patrolPhysicalDiskRow) bool {
|
||||||
|
health := strings.TrimSpace(row.health)
|
||||||
|
healthIssue := health != "" &&
|
||||||
|
!strings.EqualFold(health, "PASSED") &&
|
||||||
|
!strings.EqualFold(health, "UNKNOWN") &&
|
||||||
|
!strings.EqualFold(health, "OK")
|
||||||
|
return healthIssue ||
|
||||||
|
(row.wearout > 0 && row.wearout < 20) ||
|
||||||
|
row.temperature > 55 ||
|
||||||
|
len(row.smartEvidence) > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func patrolPhysicalDiskSummaryIssue(row patrolPhysicalDiskRow) bool {
|
||||||
|
return patrolPhysicalDiskHealthIssue(row) || strings.ToLower(strings.TrimSpace(row.status)) != "online"
|
||||||
|
}
|
||||||
|
|
||||||
|
func modelPhysicalDiskSMARTIssueParts(attrs *models.SMARTAttributes) []string {
|
||||||
|
if attrs == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var parts []string
|
||||||
|
appendSMARTInt64Issue(&parts, "reallocated sectors", attrs.ReallocatedSectors)
|
||||||
|
appendSMARTInt64Issue(&parts, "pending sectors", attrs.PendingSectors)
|
||||||
|
appendSMARTInt64Issue(&parts, "offline uncorrectable", attrs.OfflineUncorrectable)
|
||||||
|
appendSMARTInt64Issue(&parts, "UDMA CRC errors", attrs.UDMACRCErrors)
|
||||||
|
appendSMARTInt64Issue(&parts, "media errors", attrs.MediaErrors)
|
||||||
|
return parts
|
||||||
|
}
|
||||||
|
|
||||||
|
func unifiedPhysicalDiskSMARTIssueParts(attrs *unifiedresources.SMARTMeta) []string {
|
||||||
|
if attrs == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var parts []string
|
||||||
|
appendSMARTInt64ValueIssue(&parts, "reallocated sectors", attrs.ReallocatedSectors)
|
||||||
|
appendSMARTInt64ValueIssue(&parts, "pending sectors", attrs.PendingSectors)
|
||||||
|
appendSMARTInt64ValueIssue(&parts, "offline uncorrectable", attrs.OfflineUncorrectable)
|
||||||
|
appendSMARTInt64ValueIssue(&parts, "UDMA CRC errors", attrs.UDMACRCErrors)
|
||||||
|
appendSMARTInt64ValueIssue(&parts, "media errors", attrs.MediaErrors)
|
||||||
|
return parts
|
||||||
|
}
|
||||||
|
|
||||||
|
func appendSMARTInt64Issue(parts *[]string, label string, value *int64) {
|
||||||
|
if value == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
appendSMARTInt64ValueIssue(parts, label, *value)
|
||||||
|
}
|
||||||
|
|
||||||
|
func appendSMARTInt64ValueIssue(parts *[]string, label string, value int64) {
|
||||||
|
if value <= 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
*parts = append(*parts, fmt.Sprintf("%s=%d", label, value))
|
||||||
|
}
|
||||||
|
|
||||||
func patrolPrecomputeNodeSources(snap patrolRuntimeState, scopedSet map[string]bool) []patrolPrecomputeNodeSource {
|
func patrolPrecomputeNodeSources(snap patrolRuntimeState, scopedSet map[string]bool) []patrolPrecomputeNodeSource {
|
||||||
nodeRows := patrolNodeInventoryRows(snap, scopedSet)
|
nodeRows := patrolNodeInventoryRows(snap, scopedSet)
|
||||||
rows := make([]patrolPrecomputeNodeSource, 0, len(nodeRows))
|
rows := make([]patrolPrecomputeNodeSource, 0, len(nodeRows))
|
||||||
|
|
@ -2618,7 +2679,7 @@ func (p *PatrolService) seedResourceInventoryState(snap patrolRuntimeState, scop
|
||||||
if len(diskRows) > 0 {
|
if len(diskRows) > 0 {
|
||||||
diskIssues := 0
|
diskIssues := 0
|
||||||
for _, row := range diskRows {
|
for _, row := range diskRows {
|
||||||
if (!strings.EqualFold(row.health, "PASSED") && !strings.EqualFold(row.health, "UNKNOWN") && !strings.EqualFold(row.health, "OK") && row.health != "") || row.status != "online" {
|
if patrolPhysicalDiskSummaryIssue(row) {
|
||||||
diskIssues++
|
diskIssues++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -2868,12 +2929,24 @@ func (p *PatrolService) seedResourceInventorySummaryState(snap patrolRuntimeStat
|
||||||
diskIssues := []string{}
|
diskIssues := []string{}
|
||||||
for _, row := range diskRows {
|
for _, row := range diskRows {
|
||||||
statusCounts[strings.ToLower(strings.TrimSpace(row.status))]++
|
statusCounts[strings.ToLower(strings.TrimSpace(row.status))]++
|
||||||
if (!strings.EqualFold(row.health, "PASSED") && !strings.EqualFold(row.health, "UNKNOWN") && !strings.EqualFold(row.health, "OK") && row.health != "") || row.status != "online" {
|
if patrolPhysicalDiskSummaryIssue(row) {
|
||||||
name := row.devPath
|
name := row.devPath
|
||||||
if name == "" {
|
if name == "" {
|
||||||
name = row.name
|
name = row.name
|
||||||
}
|
}
|
||||||
diskIssues = append(diskIssues, fmt.Sprintf("%s (%s)", name, row.health))
|
issue := strings.TrimSpace(row.health)
|
||||||
|
if issue == "" {
|
||||||
|
issue = "UNKNOWN"
|
||||||
|
}
|
||||||
|
if len(row.smartEvidence) > 0 {
|
||||||
|
evidence := strings.Join(row.smartEvidence, ", ")
|
||||||
|
if strings.EqualFold(issue, "PASSED") || strings.EqualFold(issue, "OK") || strings.EqualFold(issue, "UNKNOWN") {
|
||||||
|
issue = evidence
|
||||||
|
} else {
|
||||||
|
issue = issue + "; " + evidence
|
||||||
|
}
|
||||||
|
}
|
||||||
|
diskIssues = append(diskIssues, fmt.Sprintf("%s (%s)", name, issue))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -3109,50 +3182,63 @@ func (p *PatrolService) seedHealthAndAlertsState(snap patrolRuntimeState, scoped
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- Disk Health ---
|
// --- Disk Health ---
|
||||||
diskURP := snap.unifiedResourceProvider
|
diskRows := patrolPhysicalDiskRows(snap, scopedSet)
|
||||||
if diskURP != nil {
|
if len(diskRows) > 0 {
|
||||||
diskResources := diskURP.GetByType(unifiedresources.ResourceTypePhysicalDisk)
|
hasIssues := false
|
||||||
if len(diskResources) > 0 {
|
unknownHealth := 0
|
||||||
hasIssues := false
|
for _, row := range diskRows {
|
||||||
for _, r := range diskResources {
|
health := strings.TrimSpace(row.health)
|
||||||
if r.PhysicalDisk == nil {
|
if health == "" || strings.EqualFold(health, "UNKNOWN") {
|
||||||
continue
|
unknownHealth++
|
||||||
}
|
|
||||||
d := r.PhysicalDisk
|
|
||||||
if (d.Health != "PASSED" && d.Health != "UNKNOWN" && d.Health != "OK" && d.Health != "") || (d.Wearout > 0 && d.Wearout < 20) || d.Temperature > 55 {
|
|
||||||
hasIssues = true
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
sb.WriteString("# Disk Health\n")
|
if patrolPhysicalDiskHealthIssue(row) {
|
||||||
if !hasIssues {
|
hasIssues = true
|
||||||
sb.WriteString(fmt.Sprintf("All %d disks healthy (SMART PASSED).\n", len(diskResources)))
|
|
||||||
} else {
|
|
||||||
sb.WriteString("| Node | Device | Model | Health | Wearout | Temp |\n")
|
|
||||||
sb.WriteString("|------|--------|-------|--------|---------|------|\n")
|
|
||||||
for _, r := range diskResources {
|
|
||||||
if r.PhysicalDisk == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
d := r.PhysicalDisk
|
|
||||||
node := r.ParentName
|
|
||||||
if node == "" && len(r.Identity.Hostnames) > 0 {
|
|
||||||
node = r.Identity.Hostnames[0]
|
|
||||||
}
|
|
||||||
wearout := "—"
|
|
||||||
if d.Wearout >= 0 {
|
|
||||||
wearout = fmt.Sprintf("%d%%", d.Wearout)
|
|
||||||
}
|
|
||||||
temp := "—"
|
|
||||||
if d.Temperature > 0 {
|
|
||||||
temp = fmt.Sprintf("%d°C", d.Temperature)
|
|
||||||
}
|
|
||||||
sb.WriteString(fmt.Sprintf("| %s | %s | %s | %s | %s | %s |\n",
|
|
||||||
node, d.DevPath, d.Model, d.Health, wearout, temp))
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
sb.WriteString("\n")
|
|
||||||
}
|
}
|
||||||
|
sb.WriteString("# Disk Health\n")
|
||||||
|
if !hasIssues {
|
||||||
|
if unknownHealth > 0 {
|
||||||
|
sb.WriteString(fmt.Sprintf("No disk issues detected across %d disks; SMART health is unknown for %d disk(s).\n", len(diskRows), unknownHealth))
|
||||||
|
} else {
|
||||||
|
sb.WriteString(fmt.Sprintf("All %d disks healthy (SMART PASSED/OK).\n", len(diskRows)))
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
sb.WriteString("| Node | Device | Model | Health | Wearout | Temp | SMART Evidence |\n")
|
||||||
|
sb.WriteString("|------|--------|-------|--------|---------|------|----------------|\n")
|
||||||
|
for _, row := range diskRows {
|
||||||
|
node := row.node
|
||||||
|
if node == "" {
|
||||||
|
node = "—"
|
||||||
|
}
|
||||||
|
device := row.devPath
|
||||||
|
if device == "" {
|
||||||
|
device = row.name
|
||||||
|
}
|
||||||
|
model := row.model
|
||||||
|
if model == "" {
|
||||||
|
model = "—"
|
||||||
|
}
|
||||||
|
health := row.health
|
||||||
|
if health == "" {
|
||||||
|
health = "UNKNOWN"
|
||||||
|
}
|
||||||
|
wearout := "—"
|
||||||
|
if row.wearout >= 0 {
|
||||||
|
wearout = fmt.Sprintf("%d%%", row.wearout)
|
||||||
|
}
|
||||||
|
temp := "—"
|
||||||
|
if row.temperature > 0 {
|
||||||
|
temp = fmt.Sprintf("%d°C", row.temperature)
|
||||||
|
}
|
||||||
|
smartEvidence := "—"
|
||||||
|
if len(row.smartEvidence) > 0 {
|
||||||
|
smartEvidence = strings.Join(row.smartEvidence, ", ")
|
||||||
|
}
|
||||||
|
sb.WriteString(fmt.Sprintf("| %s | %s | %s | %s | %s | %s | %s |\n",
|
||||||
|
node, device, model, health, wearout, temp, smartEvidence))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sb.WriteString("\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- Active Alerts ---
|
// --- Active Alerts ---
|
||||||
|
|
|
||||||
|
|
@ -873,7 +873,7 @@ func TestSeedHealthAndAlerts_NoIssues(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
out := ps.seedHealthAndAlertsState(patrolRuntimeStateForTest(ps, state), nil, cfg, now)
|
out := ps.seedHealthAndAlertsState(patrolRuntimeStateForTest(ps, state), nil, cfg, now)
|
||||||
if !strings.Contains(out, "All 1 disks healthy") {
|
if !strings.Contains(out, "All 1 disks healthy (SMART PASSED/OK).") {
|
||||||
t.Fatalf("expected healthy disk summary, got: %s", out)
|
t.Fatalf("expected healthy disk summary, got: %s", out)
|
||||||
}
|
}
|
||||||
if !strings.Contains(out, "All 2 instances connected") {
|
if !strings.Contains(out, "All 2 instances connected") {
|
||||||
|
|
@ -881,6 +881,83 @@ func TestSeedHealthAndAlerts_NoIssues(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSeedHealthAndAlerts_UnknownDiskHealthDoesNotClaimSMARTPassed(t *testing.T) {
|
||||||
|
ps := NewPatrolService(nil, nil)
|
||||||
|
cfg := DefaultPatrolConfig()
|
||||||
|
now := time.Now()
|
||||||
|
|
||||||
|
ps.SetUnifiedResourceProvider(&mockUnifiedResourceProvider{
|
||||||
|
getByTypeFunc: func(t unifiedresources.ResourceType) []unifiedresources.Resource {
|
||||||
|
if t != unifiedresources.ResourceTypePhysicalDisk {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return []unifiedresources.Resource{
|
||||||
|
{
|
||||||
|
Name: "disk",
|
||||||
|
Type: unifiedresources.ResourceTypePhysicalDisk,
|
||||||
|
ParentName: "node-1",
|
||||||
|
Status: unifiedresources.StatusOnline,
|
||||||
|
PhysicalDisk: &unifiedresources.PhysicalDiskMeta{
|
||||||
|
DevPath: "/dev/sda",
|
||||||
|
Model: "disk",
|
||||||
|
Health: "UNKNOWN",
|
||||||
|
Wearout: 100,
|
||||||
|
Temperature: 40,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
out := ps.seedHealthAndAlertsState(patrolRuntimeStateForTest(ps, models.StateSnapshot{}), nil, cfg, now)
|
||||||
|
if !strings.Contains(out, "No disk issues detected across 1 disks; SMART health is unknown for 1 disk(s).") {
|
||||||
|
t.Fatalf("expected unknown SMART health summary, got: %s", out)
|
||||||
|
}
|
||||||
|
if strings.Contains(out, "SMART PASSED") {
|
||||||
|
t.Fatalf("unknown SMART health must not be reported as passed, got: %s", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSeedHealthAndAlerts_SMARTAttributeIssue(t *testing.T) {
|
||||||
|
ps := NewPatrolService(nil, nil)
|
||||||
|
cfg := DefaultPatrolConfig()
|
||||||
|
now := time.Now()
|
||||||
|
|
||||||
|
ps.SetUnifiedResourceProvider(&mockUnifiedResourceProvider{
|
||||||
|
getByTypeFunc: func(t unifiedresources.ResourceType) []unifiedresources.Resource {
|
||||||
|
if t != unifiedresources.ResourceTypePhysicalDisk {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return []unifiedresources.Resource{
|
||||||
|
{
|
||||||
|
Name: "disk",
|
||||||
|
Type: unifiedresources.ResourceTypePhysicalDisk,
|
||||||
|
ParentName: "node-1",
|
||||||
|
Status: unifiedresources.StatusOnline,
|
||||||
|
PhysicalDisk: &unifiedresources.PhysicalDiskMeta{
|
||||||
|
DevPath: "/dev/sda",
|
||||||
|
Model: "disk",
|
||||||
|
Health: "PASSED",
|
||||||
|
Wearout: 100,
|
||||||
|
Temperature: 40,
|
||||||
|
SMART: &unifiedresources.SMARTMeta{
|
||||||
|
PendingSectors: 2,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
out := ps.seedHealthAndAlertsState(patrolRuntimeStateForTest(ps, models.StateSnapshot{}), nil, cfg, now)
|
||||||
|
if strings.Contains(out, "All 1 disks healthy") {
|
||||||
|
t.Fatalf("SMART counter evidence must not be summarized as all healthy, got: %s", out)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, "SMART Evidence") || !strings.Contains(out, "pending sectors=2") {
|
||||||
|
t.Fatalf("expected SMART counter evidence in disk health table, got: %s", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestSeedHealthAndAlerts_WithIssues(t *testing.T) {
|
func TestSeedHealthAndAlerts_WithIssues(t *testing.T) {
|
||||||
ps := NewPatrolService(nil, nil)
|
ps := NewPatrolService(nil, nil)
|
||||||
cfg := DefaultPatrolConfig()
|
cfg := DefaultPatrolConfig()
|
||||||
|
|
|
||||||
|
|
@ -142,15 +142,34 @@ func detectSignalsFromToolCall(tc *ToolCallRecord, thresholds SignalThresholds)
|
||||||
|
|
||||||
// --- Storage signals ---
|
// --- Storage signals ---
|
||||||
|
|
||||||
|
type smartAttributeCounters struct {
|
||||||
|
ReallocatedSectors *int64 `json:"reallocatedSectors"`
|
||||||
|
PendingSectors *int64 `json:"pendingSectors"`
|
||||||
|
OfflineUncorrectable *int64 `json:"offlineUncorrectable"`
|
||||||
|
UDMACRCErrors *int64 `json:"udmaCrcErrors"`
|
||||||
|
MediaErrors *int64 `json:"mediaErrors"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type smartDiskSignalData struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Device string `json:"device"`
|
||||||
|
DevPath string `json:"dev_path"`
|
||||||
|
Health string `json:"health"`
|
||||||
|
Model string `json:"model"`
|
||||||
|
Serial string `json:"serial"`
|
||||||
|
Node string `json:"node"`
|
||||||
|
Attributes smartAttributeCounters `json:"attributes"`
|
||||||
|
SmartAttributes smartAttributeCounters `json:"smart_attributes"`
|
||||||
|
SmartAttributesCamel smartAttributeCounters `json:"smartAttributes"`
|
||||||
|
}
|
||||||
|
|
||||||
// storageDiskHealth is the minimal struct for parsing disk_health tool output.
|
// storageDiskHealth is the minimal struct for parsing disk_health tool output.
|
||||||
type storageDiskHealth struct {
|
type storageDiskHealth struct {
|
||||||
Disks []struct {
|
Disks []smartDiskSignalData `json:"disks"`
|
||||||
Device string `json:"device"`
|
Hosts []struct {
|
||||||
Health string `json:"health"`
|
Hostname string `json:"hostname"`
|
||||||
Model string `json:"model"`
|
SMART []smartDiskSignalData `json:"smart"`
|
||||||
Serial string `json:"serial"`
|
} `json:"hosts"`
|
||||||
Node string `json:"node"`
|
|
||||||
} `json:"disks"`
|
|
||||||
Node string `json:"node"`
|
Node string `json:"node"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -264,37 +283,136 @@ func detectSMARTFailures(tc *ToolCallRecord) []DetectedSignal {
|
||||||
|
|
||||||
node := data.Node
|
node := data.Node
|
||||||
for _, disk := range data.Disks {
|
for _, disk := range data.Disks {
|
||||||
health := strings.ToUpper(strings.TrimSpace(disk.Health))
|
if signal, ok := smartDiskSignal(tc, disk, node); ok {
|
||||||
if health == "" || health == "UNKNOWN" || health == "PASSED" || health == "OK" {
|
signals = append(signals, signal)
|
||||||
continue
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
diskNode := disk.Node
|
for _, host := range data.Hosts {
|
||||||
if diskNode == "" {
|
for _, disk := range host.SMART {
|
||||||
diskNode = node
|
if signal, ok := smartDiskSignal(tc, disk, host.Hostname); ok {
|
||||||
|
signals = append(signals, signal)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
resourceID := diskNode
|
|
||||||
if resourceID == "" {
|
|
||||||
resourceID = disk.Device
|
|
||||||
}
|
|
||||||
|
|
||||||
signals = append(signals, DetectedSignal{
|
|
||||||
SignalType: SignalSMARTFailure,
|
|
||||||
ResourceID: resourceID,
|
|
||||||
ResourceName: disk.Device,
|
|
||||||
ResourceType: "node",
|
|
||||||
SuggestedSeverity: "critical",
|
|
||||||
Category: string(FindingCategoryReliability),
|
|
||||||
Summary: "SMART health check: " + health + " for " + disk.Device,
|
|
||||||
Evidence: truncateEvidence(tc.Output),
|
|
||||||
ToolCallID: tc.ID,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return signals
|
return signals
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func smartDiskSignal(tc *ToolCallRecord, disk smartDiskSignalData, fallbackNode string) (DetectedSignal, bool) {
|
||||||
|
health := strings.ToUpper(strings.TrimSpace(disk.Health))
|
||||||
|
healthIssue := health != "" && health != "UNKNOWN" && health != "PASSED" && health != "OK"
|
||||||
|
|
||||||
|
attrParts, hasCriticalAttrs := smartAttributeIssueParts(disk.mergedAttributes())
|
||||||
|
if !healthIssue && len(attrParts) == 0 {
|
||||||
|
return DetectedSignal{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
resourceID, resourceName, resourceType := smartDiskResource(disk, fallbackNode)
|
||||||
|
severity := "warning"
|
||||||
|
if healthIssue || hasCriticalAttrs {
|
||||||
|
severity = "critical"
|
||||||
|
}
|
||||||
|
|
||||||
|
var summary string
|
||||||
|
switch {
|
||||||
|
case healthIssue && len(attrParts) > 0:
|
||||||
|
summary = fmt.Sprintf("SMART health check: %s for %s; counters: %s", health, resourceName, strings.Join(attrParts, ", "))
|
||||||
|
case healthIssue:
|
||||||
|
summary = fmt.Sprintf("SMART health check: %s for %s", health, resourceName)
|
||||||
|
default:
|
||||||
|
summary = fmt.Sprintf("SMART counters indicate disk risk for %s: %s", resourceName, strings.Join(attrParts, ", "))
|
||||||
|
}
|
||||||
|
|
||||||
|
return DetectedSignal{
|
||||||
|
SignalType: SignalSMARTFailure,
|
||||||
|
ResourceID: resourceID,
|
||||||
|
ResourceName: resourceName,
|
||||||
|
ResourceType: resourceType,
|
||||||
|
SuggestedSeverity: severity,
|
||||||
|
Category: string(FindingCategoryReliability),
|
||||||
|
Summary: summary,
|
||||||
|
Evidence: truncateEvidence(tc.Output),
|
||||||
|
ToolCallID: tc.ID,
|
||||||
|
}, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d smartDiskSignalData) mergedAttributes() smartAttributeCounters {
|
||||||
|
merged := smartAttributeCounters{}
|
||||||
|
merge := func(attrs smartAttributeCounters) {
|
||||||
|
if merged.ReallocatedSectors == nil {
|
||||||
|
merged.ReallocatedSectors = attrs.ReallocatedSectors
|
||||||
|
}
|
||||||
|
if merged.PendingSectors == nil {
|
||||||
|
merged.PendingSectors = attrs.PendingSectors
|
||||||
|
}
|
||||||
|
if merged.OfflineUncorrectable == nil {
|
||||||
|
merged.OfflineUncorrectable = attrs.OfflineUncorrectable
|
||||||
|
}
|
||||||
|
if merged.UDMACRCErrors == nil {
|
||||||
|
merged.UDMACRCErrors = attrs.UDMACRCErrors
|
||||||
|
}
|
||||||
|
if merged.MediaErrors == nil {
|
||||||
|
merged.MediaErrors = attrs.MediaErrors
|
||||||
|
}
|
||||||
|
}
|
||||||
|
merge(d.Attributes)
|
||||||
|
merge(d.SmartAttributes)
|
||||||
|
merge(d.SmartAttributesCamel)
|
||||||
|
return merged
|
||||||
|
}
|
||||||
|
|
||||||
|
func smartAttributeIssueParts(attrs smartAttributeCounters) ([]string, bool) {
|
||||||
|
var parts []string
|
||||||
|
hasCritical := false
|
||||||
|
appendCounter := func(label string, value *int64, critical bool) {
|
||||||
|
if value == nil || *value <= 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
parts = append(parts, fmt.Sprintf("%s=%d", label, *value))
|
||||||
|
if critical {
|
||||||
|
hasCritical = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
appendCounter("reallocated sectors", attrs.ReallocatedSectors, false)
|
||||||
|
appendCounter("pending sectors", attrs.PendingSectors, true)
|
||||||
|
appendCounter("offline uncorrectable", attrs.OfflineUncorrectable, true)
|
||||||
|
appendCounter("UDMA CRC errors", attrs.UDMACRCErrors, false)
|
||||||
|
appendCounter("media errors", attrs.MediaErrors, true)
|
||||||
|
return parts, hasCritical
|
||||||
|
}
|
||||||
|
|
||||||
|
func smartDiskResource(disk smartDiskSignalData, fallbackNode string) (resourceID, resourceName, resourceType string) {
|
||||||
|
resourceName = strings.TrimSpace(disk.Device)
|
||||||
|
if resourceName == "" {
|
||||||
|
resourceName = strings.TrimSpace(disk.DevPath)
|
||||||
|
}
|
||||||
|
if resourceName == "" {
|
||||||
|
resourceName = strings.TrimSpace(disk.Model)
|
||||||
|
}
|
||||||
|
if resourceName == "" {
|
||||||
|
resourceName = strings.TrimSpace(disk.ID)
|
||||||
|
}
|
||||||
|
if resourceName == "" {
|
||||||
|
resourceName = "disk"
|
||||||
|
}
|
||||||
|
|
||||||
|
resourceID = strings.TrimSpace(disk.ID)
|
||||||
|
resourceType = "disk"
|
||||||
|
if resourceID == "" {
|
||||||
|
node := strings.TrimSpace(disk.Node)
|
||||||
|
if node == "" {
|
||||||
|
node = strings.TrimSpace(fallbackNode)
|
||||||
|
}
|
||||||
|
resourceID = node
|
||||||
|
resourceType = "node"
|
||||||
|
}
|
||||||
|
if resourceID == "" {
|
||||||
|
resourceID = resourceName
|
||||||
|
}
|
||||||
|
return resourceID, resourceName, resourceType
|
||||||
|
}
|
||||||
|
|
||||||
func detectHighDiskUsage(tc *ToolCallRecord, thresholds SignalThresholds) []DetectedSignal {
|
func detectHighDiskUsage(tc *ToolCallRecord, thresholds SignalThresholds) []DetectedSignal {
|
||||||
var signals []DetectedSignal
|
var signals []DetectedSignal
|
||||||
|
|
||||||
|
|
@ -623,8 +741,15 @@ type metricsPerformance struct {
|
||||||
MemPercent float64 `json:"mem_percent"`
|
MemPercent float64 `json:"mem_percent"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type metricsPhysicalDisks struct {
|
||||||
|
Disks []smartDiskSignalData `json:"disks"`
|
||||||
|
}
|
||||||
|
|
||||||
func detectMetricsSignals(tc *ToolCallRecord, thresholds SignalThresholds) []DetectedSignal {
|
func detectMetricsSignals(tc *ToolCallRecord, thresholds SignalThresholds) []DetectedSignal {
|
||||||
inputType := extractInputType(tc.Input)
|
inputType := extractInputType(tc.Input)
|
||||||
|
if inputType == "disks" {
|
||||||
|
return detectPhysicalDiskMetricSignals(tc)
|
||||||
|
}
|
||||||
if inputType != "performance" && inputType != "" {
|
if inputType != "performance" && inputType != "" {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
@ -690,6 +815,28 @@ func detectMetricsSignals(tc *ToolCallRecord, thresholds SignalThresholds) []Det
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if inputType == "" {
|
||||||
|
signals = append(signals, detectPhysicalDiskMetricSignals(tc)...)
|
||||||
|
}
|
||||||
|
|
||||||
|
return signals
|
||||||
|
}
|
||||||
|
|
||||||
|
func detectPhysicalDiskMetricSignals(tc *ToolCallRecord) []DetectedSignal {
|
||||||
|
var data metricsPhysicalDisks
|
||||||
|
if err := json.Unmarshal([]byte(tc.Output), &data); err != nil {
|
||||||
|
if !tryParseEmbeddedJSON(tc.Output, &data) {
|
||||||
|
log.Debug().Err(err).Str("tool", tc.ToolName).Msg("patrol_signals: failed to parse physical disks output")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var signals []DetectedSignal
|
||||||
|
for _, disk := range data.Disks {
|
||||||
|
if signal, ok := smartDiskSignal(tc, disk, disk.Node); ok {
|
||||||
|
signals = append(signals, signal)
|
||||||
|
}
|
||||||
|
}
|
||||||
return signals
|
return signals
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@ package ai
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
@ -70,6 +71,107 @@ func TestDetectSignals_SMARTPassedNoSignal(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestDetectSignals_SMARTAttributeCountersCreateSignal(t *testing.T) {
|
||||||
|
output, _ := json.Marshal(map[string]interface{}{
|
||||||
|
"node": "pve1",
|
||||||
|
"disks": []map[string]interface{}{
|
||||||
|
{
|
||||||
|
"id": "physical-disk-1",
|
||||||
|
"device": "/dev/sdc",
|
||||||
|
"health": "PASSED",
|
||||||
|
"attributes": map[string]interface{}{
|
||||||
|
"pendingSectors": 2,
|
||||||
|
"offlineUncorrectable": 1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
toolCalls := []ToolCallRecord{
|
||||||
|
{
|
||||||
|
ID: "tc-smart-attrs",
|
||||||
|
ToolName: "pulse_storage",
|
||||||
|
Input: `{"type":"disk_health","node":"pve1"}`,
|
||||||
|
Output: string(output),
|
||||||
|
Success: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
signals := DetectSignals(toolCalls, DefaultSignalThresholds())
|
||||||
|
if len(signals) != 1 {
|
||||||
|
t.Fatalf("expected 1 SMART counter signal, got %d", len(signals))
|
||||||
|
}
|
||||||
|
s := signals[0]
|
||||||
|
if s.ResourceID != "physical-disk-1" || s.ResourceType != "disk" {
|
||||||
|
t.Fatalf("expected physical disk resource identity, got id=%q type=%q", s.ResourceID, s.ResourceType)
|
||||||
|
}
|
||||||
|
if s.SuggestedSeverity != "critical" {
|
||||||
|
t.Fatalf("expected critical severity from pending/offline counters, got %q", s.SuggestedSeverity)
|
||||||
|
}
|
||||||
|
if !strings.Contains(s.Summary, "pending sectors=2") || !strings.Contains(s.Summary, "offline uncorrectable=1") {
|
||||||
|
t.Fatalf("expected SMART counter evidence in summary, got %q", s.Summary)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDetectSignals_SMARTHostPayloadAndMetricsDisks(t *testing.T) {
|
||||||
|
storageOutput, _ := json.Marshal(map[string]interface{}{
|
||||||
|
"hosts": []map[string]interface{}{
|
||||||
|
{
|
||||||
|
"hostname": "pve-host",
|
||||||
|
"smart": []map[string]interface{}{
|
||||||
|
{
|
||||||
|
"device": "/dev/sdd",
|
||||||
|
"health": "PASSED",
|
||||||
|
"smartAttributes": map[string]interface{}{
|
||||||
|
"udmaCrcErrors": 4,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
metricsOutput, _ := json.Marshal(map[string]interface{}{
|
||||||
|
"disks": []map[string]interface{}{
|
||||||
|
{
|
||||||
|
"id": "disk-nvme",
|
||||||
|
"device": "/dev/nvme0n1",
|
||||||
|
"health": "PASSED",
|
||||||
|
"smart_attributes": map[string]interface{}{
|
||||||
|
"mediaErrors": 1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
toolCalls := []ToolCallRecord{
|
||||||
|
{
|
||||||
|
ID: "tc-storage-smart",
|
||||||
|
ToolName: "pulse_storage",
|
||||||
|
Input: `{"type":"disk_health"}`,
|
||||||
|
Output: string(storageOutput),
|
||||||
|
Success: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ID: "tc-metrics-disks",
|
||||||
|
ToolName: "pulse_metrics",
|
||||||
|
Input: `{"type":"disks"}`,
|
||||||
|
Output: string(metricsOutput),
|
||||||
|
Success: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
signals := DetectSignals(toolCalls, DefaultSignalThresholds())
|
||||||
|
if len(signals) != 2 {
|
||||||
|
t.Fatalf("expected 2 SMART counter signals, got %d: %+v", len(signals), signals)
|
||||||
|
}
|
||||||
|
if signals[0].ResourceID != "pve-host" || signals[0].SuggestedSeverity != "warning" {
|
||||||
|
t.Fatalf("expected host fallback warning for CRC-only evidence, got %+v", signals[0])
|
||||||
|
}
|
||||||
|
if signals[1].ResourceID != "disk-nvme" || signals[1].SuggestedSeverity != "critical" {
|
||||||
|
t.Fatalf("expected disk critical media-error evidence, got %+v", signals[1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestDetectSignals_HighCPU(t *testing.T) {
|
func TestDetectSignals_HighCPU(t *testing.T) {
|
||||||
output, _ := json.Marshal(map[string]interface{}{
|
output, _ := json.Marshal(map[string]interface{}{
|
||||||
"resources": []map[string]interface{}{
|
"resources": []map[string]interface{}{
|
||||||
|
|
|
||||||
|
|
@ -100,6 +100,14 @@ class AIRuntimeDocsPolicyTest(unittest.TestCase):
|
||||||
self.assertIn("go run ./cmd/eval -scenario resource-context", content)
|
self.assertIn("go run ./cmd/eval -scenario resource-context", content)
|
||||||
self.assertIn("EVAL_RESOURCE_CONTEXT_FORBIDDEN", content)
|
self.assertIn("EVAL_RESOURCE_CONTEXT_FORBIDDEN", content)
|
||||||
self.assertIn("Pulse does not convert them into Pulse-authored findings", content)
|
self.assertIn("Pulse does not convert them into Pulse-authored findings", content)
|
||||||
|
self.assertIn(
|
||||||
|
"critical SMART counters such as pending sectors, offline uncorrectable sectors, or NVMe media errors",
|
||||||
|
normalized_content,
|
||||||
|
)
|
||||||
|
self.assertIn(
|
||||||
|
"Disk health check failed, pending sectors, offline uncorrectable sectors, or NVMe media errors",
|
||||||
|
normalized_content,
|
||||||
|
)
|
||||||
self.assertNotIn("Anthropic** (API key or OAuth)", content)
|
self.assertNotIn("Anthropic** (API key or OAuth)", content)
|
||||||
self.assertNotIn("Anthropic API key or OAuth", content)
|
self.assertNotIn("Anthropic API key or OAuth", content)
|
||||||
self.assertNotIn("successful remediations (incident memory)", content)
|
self.assertNotIn("successful remediations (incident memory)", content)
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue