Emit TrueNAS pool status incidents

This commit is contained in:
rcourtman 2026-06-29 22:54:19 +01:00
parent 4eac3b535c
commit c8dc45577c
3 changed files with 155 additions and 3 deletions

View file

@ -243,7 +243,11 @@ truth for live infrastructure data.
TrueNAS storage and alert inventory follow native query methods first:
pools use `pool.query`, datasets use `pool.dataset.query`, disks use
`disk.query` with pool join options, and alerts use `alert.list`, with
legacy REST allowed only as compatibility fallback.
legacy REST allowed only as compatibility fallback. Unhealthy pool state
from `pool.query` must emit a provider-native `zfs_pool_state` incident on
the canonical pool resource when `alert.list` does not already provide a
warning or critical pool alert for that same pool, so pool degradation does
not depend on the TrueNAS appliance's own email or alert-delivery setup.
TrueNAS VMs and network shares follow the same provider-owned inventory
boundary: `vm.query` data publishes native `TrueNASData.VM` on canonical
`vm` resources, while SMB/NFS share data from `sharing.smb.query` and

View file

@ -456,7 +456,7 @@ func truenasRecordsFromSnapshot(snapshot *FixtureSnapshot, now func() time.Time)
systemAssessment := assessSystemStorage(snapshot)
systemRisk := unifiedresources.StorageRiskFromAssessment(systemAssessment)
_, protectionReduced, rebuildInProgress, protectionSummary, rebuildSummary := unifiedresources.StorageRiskSemantics(systemRisk)
systemIncidents, poolIncidents, diskIncidents := buildIncidentAssignments(snapshot)
systemIncidents, poolIncidents, diskIncidents := buildIncidentAssignments(snapshot, collectedAt)
records := make([]unifiedresources.IngestRecord, 0, 1+len(snapshot.Pools)+len(snapshot.Datasets)+len(snapshot.Apps)+len(snapshot.VMs)+len(snapshot.Shares)+len(snapshot.Disks))
totalCapacity, totalUsed := aggregatePoolUsage(snapshot.Pools)
@ -997,7 +997,7 @@ func enrichAppStatsFromPreviousSnapshot(current *FixtureSnapshot, previous *Fixt
}
}
func buildIncidentAssignments(snapshot *FixtureSnapshot) ([]unifiedresources.ResourceIncident, map[string][]unifiedresources.ResourceIncident, map[string][]unifiedresources.ResourceIncident) {
func buildIncidentAssignments(snapshot *FixtureSnapshot, observedAt time.Time) ([]unifiedresources.ResourceIncident, map[string][]unifiedresources.ResourceIncident, map[string][]unifiedresources.ResourceIncident) {
systemIncidents := make([]unifiedresources.ResourceIncident, 0)
poolIncidents := make(map[string][]unifiedresources.ResourceIncident)
diskIncidents := make(map[string][]unifiedresources.ResourceIncident)
@ -1015,6 +1015,7 @@ func buildIncidentAssignments(snapshot *FixtureSnapshot) ([]unifiedresources.Res
diskPools[diskName] = poolName
}
nativePoolAlertSeen := make(map[string]struct{}, len(snapshot.Alerts))
for _, alert := range snapshot.Alerts {
if alert.Dismissed {
continue
@ -1027,6 +1028,9 @@ func buildIncidentAssignments(snapshot *FixtureSnapshot) ([]unifiedresources.Res
if poolName := poolNameFromAlert(alert); poolName != "" {
poolIncidents[poolName] = append(poolIncidents[poolName], incident)
if incident.Severity == storagehealth.RiskWarning || incident.Severity == storagehealth.RiskCritical {
nativePoolAlertSeen[poolName] = struct{}{}
}
}
if diskName := diskNameFromAlert(alert); diskName != "" {
@ -1037,6 +1041,22 @@ func buildIncidentAssignments(snapshot *FixtureSnapshot) ([]unifiedresources.Res
}
}
for _, pool := range snapshot.Pools {
poolName := strings.TrimSpace(pool.Name)
if poolName == "" {
continue
}
if _, exists := nativePoolAlertSeen[poolName]; exists {
continue
}
incident, ok := incidentFromPoolStatus(pool, observedAt)
if !ok {
continue
}
systemIncidents = append(systemIncidents, incident)
poolIncidents[poolName] = append(poolIncidents[poolName], incident)
}
return systemIncidents, poolIncidents, diskIncidents
}
@ -1071,6 +1091,44 @@ func incidentFromAlert(alert Alert) (unifiedresources.ResourceIncident, bool) {
}, true
}
func incidentFromPoolStatus(pool Pool, observedAt time.Time) (unifiedresources.ResourceIncident, bool) {
assessment := assessPool(pool)
if assessment.Level != storagehealth.RiskWarning && assessment.Level != storagehealth.RiskCritical {
return unifiedresources.ResourceIncident{}, false
}
reason, ok := primaryIncidentReason(assessment, "zfs_pool_state")
if !ok {
return unifiedresources.ResourceIncident{}, false
}
poolName := strings.TrimSpace(pool.Name)
if poolName == "" {
return unifiedresources.ResourceIncident{}, false
}
return unifiedresources.ResourceIncident{
Provider: "truenas",
NativeID: "pool:" + poolName + ":state",
Code: reason.Code,
Severity: reason.Severity,
Source: "pool.query",
Summary: reason.Summary,
StartedAt: observedAt,
}, true
}
func primaryIncidentReason(assessment storagehealth.Assessment, code string) (storagehealth.Reason, bool) {
code = strings.TrimSpace(code)
for _, reason := range assessment.Reasons {
if strings.TrimSpace(reason.Code) == code {
return reason, true
}
}
if len(assessment.Reasons) == 0 {
return storagehealth.Reason{}, false
}
return assessment.Reasons[0], true
}
func severityFromAlertLevel(level string) (storagehealth.RiskLevel, bool) {
switch strings.ToUpper(strings.TrimSpace(level)) {
case "CRITICAL", "ERROR", "ALERT":

View file

@ -214,6 +214,96 @@ func TestProviderRefreshUpdatesLastSnapshot(t *testing.T) {
}
}
func TestProviderRecordsSynthesizesIncidentFromUnhealthyPoolStatus(t *testing.T) {
collectedAt := time.Date(2026, 6, 29, 12, 0, 0, 0, time.UTC)
records := FixtureRecords(FixtureSnapshot{
CollectedAt: collectedAt,
System: SystemInfo{
Hostname: "truenas-main",
Healthy: true,
},
Pools: []Pool{{
ID: "pool-tank",
Name: "tank",
Status: "DEGRADED",
TotalBytes: 1000,
UsedBytes: 400,
FreeBytes: 600,
}},
})
record := requirePoolRecord(t, records, "tank")
if record.Resource.Status != unifiedresources.StatusWarning {
t.Fatalf("pool status = %q, want warning", record.Resource.Status)
}
if len(record.Resource.Incidents) != 1 {
t.Fatalf("expected one synthesized pool incident, got %+v", record.Resource.Incidents)
}
incident := record.Resource.Incidents[0]
if incident.Code != "zfs_pool_state" {
t.Fatalf("incident code = %q, want zfs_pool_state", incident.Code)
}
if incident.NativeID != "pool:tank:state" {
t.Fatalf("incident native id = %q, want pool:tank:state", incident.NativeID)
}
if incident.Source != "pool.query" {
t.Fatalf("incident source = %q, want pool.query", incident.Source)
}
if incident.Severity != storagehealth.RiskWarning {
t.Fatalf("incident severity = %q, want warning", incident.Severity)
}
if incident.Summary != "ZFS pool tank is DEGRADED" {
t.Fatalf("incident summary = %q", incident.Summary)
}
if !incident.StartedAt.Equal(collectedAt) {
t.Fatalf("incident started at = %s, want %s", incident.StartedAt, collectedAt)
}
}
func TestProviderRecordsDoesNotDuplicateNativePoolAlert(t *testing.T) {
alertTime := time.Date(2026, 6, 29, 11, 30, 0, 0, time.UTC)
records := FixtureRecords(FixtureSnapshot{
CollectedAt: time.Date(2026, 6, 29, 12, 0, 0, 0, time.UTC),
System: SystemInfo{
Hostname: "truenas-main",
Healthy: true,
},
Pools: []Pool{{
ID: "pool-tank",
Name: "tank",
Status: "DEGRADED",
TotalBytes: 1000,
UsedBytes: 400,
FreeBytes: 600,
}},
Alerts: []Alert{{
ID: "native-pool-alert",
Level: "WARNING",
Source: "VolumeStatus",
Message: "Pool tank is DEGRADED: one member is faulted.",
Datetime: alertTime,
}},
})
record := requirePoolRecord(t, records, "tank")
if len(record.Resource.Incidents) != 1 {
t.Fatalf("expected one native pool incident, got %+v", record.Resource.Incidents)
}
incident := record.Resource.Incidents[0]
if incident.Code != "truenas_volume_status" {
t.Fatalf("incident code = %q, want truenas_volume_status", incident.Code)
}
if incident.NativeID != "native-pool-alert" {
t.Fatalf("incident native id = %q, want native-pool-alert", incident.NativeID)
}
if incident.Source != "VolumeStatus" {
t.Fatalf("incident source = %q, want VolumeStatus", incident.Source)
}
if !incident.StartedAt.Equal(alertTime) {
t.Fatalf("incident started at = %s, want %s", incident.StartedAt, alertTime)
}
}
func TestProviderControlAppUsesFetcherAndRefreshesSnapshot(t *testing.T) {
fixtures := DefaultFixtures()
for i := range fixtures.Apps {