From c8dc45577cc9010b9caf7e54050cae0d699d0593 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Mon, 29 Jun 2026 22:54:19 +0100 Subject: [PATCH] Emit TrueNAS pool status incidents --- .../v6/internal/subsystems/monitoring.md | 6 +- internal/truenas/provider.go | 62 ++++++++++++- internal/truenas/provider_test.go | 90 +++++++++++++++++++ 3 files changed, 155 insertions(+), 3 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index 25d451e7f..3b65936ff 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -243,7 +243,11 @@ truth for live infrastructure data. TrueNAS storage and alert inventory follow native query methods first: pools use `pool.query`, datasets use `pool.dataset.query`, disks use `disk.query` with pool join options, and alerts use `alert.list`, with - legacy REST allowed only as compatibility fallback. + legacy REST allowed only as compatibility fallback. Unhealthy pool state + from `pool.query` must emit a provider-native `zfs_pool_state` incident on + the canonical pool resource when `alert.list` does not already provide a + warning or critical pool alert for that same pool, so pool degradation does + not depend on the TrueNAS appliance's own email or alert-delivery setup. TrueNAS VMs and network shares follow the same provider-owned inventory boundary: `vm.query` data publishes native `TrueNASData.VM` on canonical `vm` resources, while SMB/NFS share data from `sharing.smb.query` and diff --git a/internal/truenas/provider.go b/internal/truenas/provider.go index a31c0fe2a..0a74418e5 100644 --- a/internal/truenas/provider.go +++ b/internal/truenas/provider.go @@ -456,7 +456,7 @@ func truenasRecordsFromSnapshot(snapshot *FixtureSnapshot, now func() time.Time) systemAssessment := assessSystemStorage(snapshot) systemRisk := unifiedresources.StorageRiskFromAssessment(systemAssessment) _, protectionReduced, rebuildInProgress, protectionSummary, rebuildSummary := unifiedresources.StorageRiskSemantics(systemRisk) - systemIncidents, poolIncidents, diskIncidents := buildIncidentAssignments(snapshot) + systemIncidents, poolIncidents, diskIncidents := buildIncidentAssignments(snapshot, collectedAt) records := make([]unifiedresources.IngestRecord, 0, 1+len(snapshot.Pools)+len(snapshot.Datasets)+len(snapshot.Apps)+len(snapshot.VMs)+len(snapshot.Shares)+len(snapshot.Disks)) totalCapacity, totalUsed := aggregatePoolUsage(snapshot.Pools) @@ -997,7 +997,7 @@ func enrichAppStatsFromPreviousSnapshot(current *FixtureSnapshot, previous *Fixt } } -func buildIncidentAssignments(snapshot *FixtureSnapshot) ([]unifiedresources.ResourceIncident, map[string][]unifiedresources.ResourceIncident, map[string][]unifiedresources.ResourceIncident) { +func buildIncidentAssignments(snapshot *FixtureSnapshot, observedAt time.Time) ([]unifiedresources.ResourceIncident, map[string][]unifiedresources.ResourceIncident, map[string][]unifiedresources.ResourceIncident) { systemIncidents := make([]unifiedresources.ResourceIncident, 0) poolIncidents := make(map[string][]unifiedresources.ResourceIncident) diskIncidents := make(map[string][]unifiedresources.ResourceIncident) @@ -1015,6 +1015,7 @@ func buildIncidentAssignments(snapshot *FixtureSnapshot) ([]unifiedresources.Res diskPools[diskName] = poolName } + nativePoolAlertSeen := make(map[string]struct{}, len(snapshot.Alerts)) for _, alert := range snapshot.Alerts { if alert.Dismissed { continue @@ -1027,6 +1028,9 @@ func buildIncidentAssignments(snapshot *FixtureSnapshot) ([]unifiedresources.Res if poolName := poolNameFromAlert(alert); poolName != "" { poolIncidents[poolName] = append(poolIncidents[poolName], incident) + if incident.Severity == storagehealth.RiskWarning || incident.Severity == storagehealth.RiskCritical { + nativePoolAlertSeen[poolName] = struct{}{} + } } if diskName := diskNameFromAlert(alert); diskName != "" { @@ -1037,6 +1041,22 @@ func buildIncidentAssignments(snapshot *FixtureSnapshot) ([]unifiedresources.Res } } + for _, pool := range snapshot.Pools { + poolName := strings.TrimSpace(pool.Name) + if poolName == "" { + continue + } + if _, exists := nativePoolAlertSeen[poolName]; exists { + continue + } + incident, ok := incidentFromPoolStatus(pool, observedAt) + if !ok { + continue + } + systemIncidents = append(systemIncidents, incident) + poolIncidents[poolName] = append(poolIncidents[poolName], incident) + } + return systemIncidents, poolIncidents, diskIncidents } @@ -1071,6 +1091,44 @@ func incidentFromAlert(alert Alert) (unifiedresources.ResourceIncident, bool) { }, true } +func incidentFromPoolStatus(pool Pool, observedAt time.Time) (unifiedresources.ResourceIncident, bool) { + assessment := assessPool(pool) + if assessment.Level != storagehealth.RiskWarning && assessment.Level != storagehealth.RiskCritical { + return unifiedresources.ResourceIncident{}, false + } + reason, ok := primaryIncidentReason(assessment, "zfs_pool_state") + if !ok { + return unifiedresources.ResourceIncident{}, false + } + + poolName := strings.TrimSpace(pool.Name) + if poolName == "" { + return unifiedresources.ResourceIncident{}, false + } + return unifiedresources.ResourceIncident{ + Provider: "truenas", + NativeID: "pool:" + poolName + ":state", + Code: reason.Code, + Severity: reason.Severity, + Source: "pool.query", + Summary: reason.Summary, + StartedAt: observedAt, + }, true +} + +func primaryIncidentReason(assessment storagehealth.Assessment, code string) (storagehealth.Reason, bool) { + code = strings.TrimSpace(code) + for _, reason := range assessment.Reasons { + if strings.TrimSpace(reason.Code) == code { + return reason, true + } + } + if len(assessment.Reasons) == 0 { + return storagehealth.Reason{}, false + } + return assessment.Reasons[0], true +} + func severityFromAlertLevel(level string) (storagehealth.RiskLevel, bool) { switch strings.ToUpper(strings.TrimSpace(level)) { case "CRITICAL", "ERROR", "ALERT": diff --git a/internal/truenas/provider_test.go b/internal/truenas/provider_test.go index d0607aac2..4f8780067 100644 --- a/internal/truenas/provider_test.go +++ b/internal/truenas/provider_test.go @@ -214,6 +214,96 @@ func TestProviderRefreshUpdatesLastSnapshot(t *testing.T) { } } +func TestProviderRecordsSynthesizesIncidentFromUnhealthyPoolStatus(t *testing.T) { + collectedAt := time.Date(2026, 6, 29, 12, 0, 0, 0, time.UTC) + records := FixtureRecords(FixtureSnapshot{ + CollectedAt: collectedAt, + System: SystemInfo{ + Hostname: "truenas-main", + Healthy: true, + }, + Pools: []Pool{{ + ID: "pool-tank", + Name: "tank", + Status: "DEGRADED", + TotalBytes: 1000, + UsedBytes: 400, + FreeBytes: 600, + }}, + }) + + record := requirePoolRecord(t, records, "tank") + if record.Resource.Status != unifiedresources.StatusWarning { + t.Fatalf("pool status = %q, want warning", record.Resource.Status) + } + if len(record.Resource.Incidents) != 1 { + t.Fatalf("expected one synthesized pool incident, got %+v", record.Resource.Incidents) + } + incident := record.Resource.Incidents[0] + if incident.Code != "zfs_pool_state" { + t.Fatalf("incident code = %q, want zfs_pool_state", incident.Code) + } + if incident.NativeID != "pool:tank:state" { + t.Fatalf("incident native id = %q, want pool:tank:state", incident.NativeID) + } + if incident.Source != "pool.query" { + t.Fatalf("incident source = %q, want pool.query", incident.Source) + } + if incident.Severity != storagehealth.RiskWarning { + t.Fatalf("incident severity = %q, want warning", incident.Severity) + } + if incident.Summary != "ZFS pool tank is DEGRADED" { + t.Fatalf("incident summary = %q", incident.Summary) + } + if !incident.StartedAt.Equal(collectedAt) { + t.Fatalf("incident started at = %s, want %s", incident.StartedAt, collectedAt) + } +} + +func TestProviderRecordsDoesNotDuplicateNativePoolAlert(t *testing.T) { + alertTime := time.Date(2026, 6, 29, 11, 30, 0, 0, time.UTC) + records := FixtureRecords(FixtureSnapshot{ + CollectedAt: time.Date(2026, 6, 29, 12, 0, 0, 0, time.UTC), + System: SystemInfo{ + Hostname: "truenas-main", + Healthy: true, + }, + Pools: []Pool{{ + ID: "pool-tank", + Name: "tank", + Status: "DEGRADED", + TotalBytes: 1000, + UsedBytes: 400, + FreeBytes: 600, + }}, + Alerts: []Alert{{ + ID: "native-pool-alert", + Level: "WARNING", + Source: "VolumeStatus", + Message: "Pool tank is DEGRADED: one member is faulted.", + Datetime: alertTime, + }}, + }) + + record := requirePoolRecord(t, records, "tank") + if len(record.Resource.Incidents) != 1 { + t.Fatalf("expected one native pool incident, got %+v", record.Resource.Incidents) + } + incident := record.Resource.Incidents[0] + if incident.Code != "truenas_volume_status" { + t.Fatalf("incident code = %q, want truenas_volume_status", incident.Code) + } + if incident.NativeID != "native-pool-alert" { + t.Fatalf("incident native id = %q, want native-pool-alert", incident.NativeID) + } + if incident.Source != "VolumeStatus" { + t.Fatalf("incident source = %q, want VolumeStatus", incident.Source) + } + if !incident.StartedAt.Equal(alertTime) { + t.Fatalf("incident started at = %s, want %s", incident.StartedAt, alertTime) + } +} + func TestProviderControlAppUsesFetcherAndRefreshesSnapshot(t *testing.T) { fixtures := DefaultFixtures() for i := range fixtures.Apps {