Run ceph pool alerts for agent-sourced clusters
Some checks failed
Build and Test / Secret Scan (push) Has been cancelled
Build and Test / Frontend & Backend (push) Has been cancelled
Core E2E Tests / Playwright Core E2E (push) Has been cancelled

Fixes #1341

When a Pulse host-agent reports Ceph data, ApplyHostReport upserts the
cluster into state but only the Proxmox-API polling path ran
cephPoolAlertStorageTargets. Users with agent-reported Ceph (instance
prefix "agent:hostname") saved per-pool overrides under
agent-prefixed IDs that the alert manager never evaluated. The
threshold appeared to save (and showed Custom in the UI), but the
polling cycle was checking a different storage ID, so the alert
silently stayed dormant. Run CheckStorage for each pool right after
the agent upsert so the override key actually drives evaluation.
This commit is contained in:
rcourtman 2026-05-28 14:11:22 +01:00
parent 2c46c6c2db
commit ce1607694e
2 changed files with 87 additions and 0 deletions

View file

@ -3659,6 +3659,17 @@ func (m *Monitor) ApplyHostReport(report agentshost.Report, tokenRecord *config.
Str("health", cephCluster.Health).
Int("osds", cephCluster.NumOSDs).
Msg("Updated Ceph cluster from host agent")
// #1341: the agent-reported Ceph cluster used to land in state but
// never get evaluated for pool alerts. Only the Proxmox-API polling
// path ran cephPoolAlertStorageTargets, so overrides set against
// agent-prefixed pool IDs (e.g. agent:hostname-ceph-pool-foo) were
// dormant. Run the alert check here so agent-sourced pools fire.
if m.alertManager != nil {
for _, storage := range cephPoolAlertStorageTargets(cephCluster) {
m.alertManager.CheckStorage(storage)
}
}
}
if m.alertManager != nil {

View file

@ -1160,3 +1160,79 @@ func TestApplyHostReport_FallbackIdentifier(t *testing.T) {
t.Errorf("expected hostname 'fallback-host', got %q", host.Hostname)
}
}
// Regression for #1341: when Ceph is reported by a Pulse host-agent (not the
// Proxmox API), the agent-sourced cluster used to land in state but the
// pool alert evaluation only ran in the Proxmox-API polling path. Per-pool
// overrides keyed against the agent-prefixed pool ID (e.g.
// `agent:hostname-ceph-pool-name`) were silently dormant. Verify the
// agent path now fires CheckStorage for each pool so overrides match.
func TestApplyHostReportFiresCephPoolAlertsForAgentSourcedCluster(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
rateTracker: NewRateTracker(),
}
t.Cleanup(func() { monitor.alertManager.Stop() })
overrideTrigger := alerts.HysteresisThreshold{Trigger: 50, Clear: 45}
cfg := monitor.alertManager.GetConfig()
cfg.MinimumDelta = 0
if cfg.TimeThresholds == nil {
cfg.TimeThresholds = make(map[string]int)
}
cfg.TimeThresholds["storage"] = 0
cfg.StorageDefault = alerts.HysteresisThreshold{Trigger: 95, Clear: 90}
cfg.Overrides = map[string]alerts.ThresholdConfig{
"agent:pve5-ceph-pool-data_replication": {Usage: &overrideTrigger},
}
monitor.alertManager.UpdateConfig(cfg)
report := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "agent-ceph",
Version: "1.0.0",
IntervalSeconds: 30,
},
Host: agentshost.HostInfo{
ID: "pve5-host",
Hostname: "pve5",
Platform: "linux",
},
Timestamp: time.Now().UTC(),
Ceph: &agentshost.CephCluster{
FSID: "ceph-fsid-1341",
Health: agentshost.CephHealth{Status: "HEALTH_OK"},
Pools: []agentshost.CephPool{
{
ID: 2,
Name: "data_replication",
BytesUsed: 611,
BytesAvailable: 389,
PercentUsed: 61.1,
},
},
},
}
if _, err := monitor.ApplyHostReport(report, nil); err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
active := monitor.alertManager.GetActiveAlerts()
found := false
for _, alert := range active {
if alert.ID == "agent:pve5-ceph-pool-data_replication-usage" {
found = true
if alert.Threshold != 50 {
t.Fatalf("Ceph pool alert fired but threshold = %.1f, want 50 (override)", alert.Threshold)
}
break
}
}
if !found {
t.Fatalf("expected agent-sourced Ceph pool alert to fire under override at 61.1%% usage; got %d active alerts", len(active))
}
}