mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-07-09 16:00:59 +00:00
Emit TrueNAS pool status incidents
This commit is contained in:
parent
4eac3b535c
commit
c8dc45577c
3 changed files with 155 additions and 3 deletions
|
|
@ -243,7 +243,11 @@ truth for live infrastructure data.
|
|||
TrueNAS storage and alert inventory follow native query methods first:
|
||||
pools use `pool.query`, datasets use `pool.dataset.query`, disks use
|
||||
`disk.query` with pool join options, and alerts use `alert.list`, with
|
||||
legacy REST allowed only as compatibility fallback.
|
||||
legacy REST allowed only as compatibility fallback. Unhealthy pool state
|
||||
from `pool.query` must emit a provider-native `zfs_pool_state` incident on
|
||||
the canonical pool resource when `alert.list` does not already provide a
|
||||
warning or critical pool alert for that same pool, so pool degradation does
|
||||
not depend on the TrueNAS appliance's own email or alert-delivery setup.
|
||||
TrueNAS VMs and network shares follow the same provider-owned inventory
|
||||
boundary: `vm.query` data publishes native `TrueNASData.VM` on canonical
|
||||
`vm` resources, while SMB/NFS share data from `sharing.smb.query` and
|
||||
|
|
|
|||
|
|
@ -456,7 +456,7 @@ func truenasRecordsFromSnapshot(snapshot *FixtureSnapshot, now func() time.Time)
|
|||
systemAssessment := assessSystemStorage(snapshot)
|
||||
systemRisk := unifiedresources.StorageRiskFromAssessment(systemAssessment)
|
||||
_, protectionReduced, rebuildInProgress, protectionSummary, rebuildSummary := unifiedresources.StorageRiskSemantics(systemRisk)
|
||||
systemIncidents, poolIncidents, diskIncidents := buildIncidentAssignments(snapshot)
|
||||
systemIncidents, poolIncidents, diskIncidents := buildIncidentAssignments(snapshot, collectedAt)
|
||||
records := make([]unifiedresources.IngestRecord, 0, 1+len(snapshot.Pools)+len(snapshot.Datasets)+len(snapshot.Apps)+len(snapshot.VMs)+len(snapshot.Shares)+len(snapshot.Disks))
|
||||
|
||||
totalCapacity, totalUsed := aggregatePoolUsage(snapshot.Pools)
|
||||
|
|
@ -997,7 +997,7 @@ func enrichAppStatsFromPreviousSnapshot(current *FixtureSnapshot, previous *Fixt
|
|||
}
|
||||
}
|
||||
|
||||
func buildIncidentAssignments(snapshot *FixtureSnapshot) ([]unifiedresources.ResourceIncident, map[string][]unifiedresources.ResourceIncident, map[string][]unifiedresources.ResourceIncident) {
|
||||
func buildIncidentAssignments(snapshot *FixtureSnapshot, observedAt time.Time) ([]unifiedresources.ResourceIncident, map[string][]unifiedresources.ResourceIncident, map[string][]unifiedresources.ResourceIncident) {
|
||||
systemIncidents := make([]unifiedresources.ResourceIncident, 0)
|
||||
poolIncidents := make(map[string][]unifiedresources.ResourceIncident)
|
||||
diskIncidents := make(map[string][]unifiedresources.ResourceIncident)
|
||||
|
|
@ -1015,6 +1015,7 @@ func buildIncidentAssignments(snapshot *FixtureSnapshot) ([]unifiedresources.Res
|
|||
diskPools[diskName] = poolName
|
||||
}
|
||||
|
||||
nativePoolAlertSeen := make(map[string]struct{}, len(snapshot.Alerts))
|
||||
for _, alert := range snapshot.Alerts {
|
||||
if alert.Dismissed {
|
||||
continue
|
||||
|
|
@ -1027,6 +1028,9 @@ func buildIncidentAssignments(snapshot *FixtureSnapshot) ([]unifiedresources.Res
|
|||
|
||||
if poolName := poolNameFromAlert(alert); poolName != "" {
|
||||
poolIncidents[poolName] = append(poolIncidents[poolName], incident)
|
||||
if incident.Severity == storagehealth.RiskWarning || incident.Severity == storagehealth.RiskCritical {
|
||||
nativePoolAlertSeen[poolName] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
if diskName := diskNameFromAlert(alert); diskName != "" {
|
||||
|
|
@ -1037,6 +1041,22 @@ func buildIncidentAssignments(snapshot *FixtureSnapshot) ([]unifiedresources.Res
|
|||
}
|
||||
}
|
||||
|
||||
for _, pool := range snapshot.Pools {
|
||||
poolName := strings.TrimSpace(pool.Name)
|
||||
if poolName == "" {
|
||||
continue
|
||||
}
|
||||
if _, exists := nativePoolAlertSeen[poolName]; exists {
|
||||
continue
|
||||
}
|
||||
incident, ok := incidentFromPoolStatus(pool, observedAt)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
systemIncidents = append(systemIncidents, incident)
|
||||
poolIncidents[poolName] = append(poolIncidents[poolName], incident)
|
||||
}
|
||||
|
||||
return systemIncidents, poolIncidents, diskIncidents
|
||||
}
|
||||
|
||||
|
|
@ -1071,6 +1091,44 @@ func incidentFromAlert(alert Alert) (unifiedresources.ResourceIncident, bool) {
|
|||
}, true
|
||||
}
|
||||
|
||||
func incidentFromPoolStatus(pool Pool, observedAt time.Time) (unifiedresources.ResourceIncident, bool) {
|
||||
assessment := assessPool(pool)
|
||||
if assessment.Level != storagehealth.RiskWarning && assessment.Level != storagehealth.RiskCritical {
|
||||
return unifiedresources.ResourceIncident{}, false
|
||||
}
|
||||
reason, ok := primaryIncidentReason(assessment, "zfs_pool_state")
|
||||
if !ok {
|
||||
return unifiedresources.ResourceIncident{}, false
|
||||
}
|
||||
|
||||
poolName := strings.TrimSpace(pool.Name)
|
||||
if poolName == "" {
|
||||
return unifiedresources.ResourceIncident{}, false
|
||||
}
|
||||
return unifiedresources.ResourceIncident{
|
||||
Provider: "truenas",
|
||||
NativeID: "pool:" + poolName + ":state",
|
||||
Code: reason.Code,
|
||||
Severity: reason.Severity,
|
||||
Source: "pool.query",
|
||||
Summary: reason.Summary,
|
||||
StartedAt: observedAt,
|
||||
}, true
|
||||
}
|
||||
|
||||
func primaryIncidentReason(assessment storagehealth.Assessment, code string) (storagehealth.Reason, bool) {
|
||||
code = strings.TrimSpace(code)
|
||||
for _, reason := range assessment.Reasons {
|
||||
if strings.TrimSpace(reason.Code) == code {
|
||||
return reason, true
|
||||
}
|
||||
}
|
||||
if len(assessment.Reasons) == 0 {
|
||||
return storagehealth.Reason{}, false
|
||||
}
|
||||
return assessment.Reasons[0], true
|
||||
}
|
||||
|
||||
func severityFromAlertLevel(level string) (storagehealth.RiskLevel, bool) {
|
||||
switch strings.ToUpper(strings.TrimSpace(level)) {
|
||||
case "CRITICAL", "ERROR", "ALERT":
|
||||
|
|
|
|||
|
|
@ -214,6 +214,96 @@ func TestProviderRefreshUpdatesLastSnapshot(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestProviderRecordsSynthesizesIncidentFromUnhealthyPoolStatus(t *testing.T) {
|
||||
collectedAt := time.Date(2026, 6, 29, 12, 0, 0, 0, time.UTC)
|
||||
records := FixtureRecords(FixtureSnapshot{
|
||||
CollectedAt: collectedAt,
|
||||
System: SystemInfo{
|
||||
Hostname: "truenas-main",
|
||||
Healthy: true,
|
||||
},
|
||||
Pools: []Pool{{
|
||||
ID: "pool-tank",
|
||||
Name: "tank",
|
||||
Status: "DEGRADED",
|
||||
TotalBytes: 1000,
|
||||
UsedBytes: 400,
|
||||
FreeBytes: 600,
|
||||
}},
|
||||
})
|
||||
|
||||
record := requirePoolRecord(t, records, "tank")
|
||||
if record.Resource.Status != unifiedresources.StatusWarning {
|
||||
t.Fatalf("pool status = %q, want warning", record.Resource.Status)
|
||||
}
|
||||
if len(record.Resource.Incidents) != 1 {
|
||||
t.Fatalf("expected one synthesized pool incident, got %+v", record.Resource.Incidents)
|
||||
}
|
||||
incident := record.Resource.Incidents[0]
|
||||
if incident.Code != "zfs_pool_state" {
|
||||
t.Fatalf("incident code = %q, want zfs_pool_state", incident.Code)
|
||||
}
|
||||
if incident.NativeID != "pool:tank:state" {
|
||||
t.Fatalf("incident native id = %q, want pool:tank:state", incident.NativeID)
|
||||
}
|
||||
if incident.Source != "pool.query" {
|
||||
t.Fatalf("incident source = %q, want pool.query", incident.Source)
|
||||
}
|
||||
if incident.Severity != storagehealth.RiskWarning {
|
||||
t.Fatalf("incident severity = %q, want warning", incident.Severity)
|
||||
}
|
||||
if incident.Summary != "ZFS pool tank is DEGRADED" {
|
||||
t.Fatalf("incident summary = %q", incident.Summary)
|
||||
}
|
||||
if !incident.StartedAt.Equal(collectedAt) {
|
||||
t.Fatalf("incident started at = %s, want %s", incident.StartedAt, collectedAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderRecordsDoesNotDuplicateNativePoolAlert(t *testing.T) {
|
||||
alertTime := time.Date(2026, 6, 29, 11, 30, 0, 0, time.UTC)
|
||||
records := FixtureRecords(FixtureSnapshot{
|
||||
CollectedAt: time.Date(2026, 6, 29, 12, 0, 0, 0, time.UTC),
|
||||
System: SystemInfo{
|
||||
Hostname: "truenas-main",
|
||||
Healthy: true,
|
||||
},
|
||||
Pools: []Pool{{
|
||||
ID: "pool-tank",
|
||||
Name: "tank",
|
||||
Status: "DEGRADED",
|
||||
TotalBytes: 1000,
|
||||
UsedBytes: 400,
|
||||
FreeBytes: 600,
|
||||
}},
|
||||
Alerts: []Alert{{
|
||||
ID: "native-pool-alert",
|
||||
Level: "WARNING",
|
||||
Source: "VolumeStatus",
|
||||
Message: "Pool tank is DEGRADED: one member is faulted.",
|
||||
Datetime: alertTime,
|
||||
}},
|
||||
})
|
||||
|
||||
record := requirePoolRecord(t, records, "tank")
|
||||
if len(record.Resource.Incidents) != 1 {
|
||||
t.Fatalf("expected one native pool incident, got %+v", record.Resource.Incidents)
|
||||
}
|
||||
incident := record.Resource.Incidents[0]
|
||||
if incident.Code != "truenas_volume_status" {
|
||||
t.Fatalf("incident code = %q, want truenas_volume_status", incident.Code)
|
||||
}
|
||||
if incident.NativeID != "native-pool-alert" {
|
||||
t.Fatalf("incident native id = %q, want native-pool-alert", incident.NativeID)
|
||||
}
|
||||
if incident.Source != "VolumeStatus" {
|
||||
t.Fatalf("incident source = %q, want VolumeStatus", incident.Source)
|
||||
}
|
||||
if !incident.StartedAt.Equal(alertTime) {
|
||||
t.Fatalf("incident started at = %s, want %s", incident.StartedAt, alertTime)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderControlAppUsesFetcherAndRefreshesSnapshot(t *testing.T) {
|
||||
fixtures := DefaultFixtures()
|
||||
for i := range fixtures.Apps {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue