mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-07-10 08:23:23 +00:00
Run ceph pool alerts for agent-sourced clusters
Fixes #1341 When a Pulse host-agent reports Ceph data, ApplyHostReport upserts the cluster into state but only the Proxmox-API polling path ran cephPoolAlertStorageTargets. Users with agent-reported Ceph (instance prefix "agent:hostname") saved per-pool overrides under agent-prefixed IDs that the alert manager never evaluated. The threshold appeared to save (and showed Custom in the UI), but the polling cycle was checking a different storage ID, so the alert silently stayed dormant. Run CheckStorage for each pool right after the agent upsert so the override key actually drives evaluation.
This commit is contained in:
parent
2c46c6c2db
commit
ce1607694e
2 changed files with 87 additions and 0 deletions
|
|
@ -3659,6 +3659,17 @@ func (m *Monitor) ApplyHostReport(report agentshost.Report, tokenRecord *config.
|
|||
Str("health", cephCluster.Health).
|
||||
Int("osds", cephCluster.NumOSDs).
|
||||
Msg("Updated Ceph cluster from host agent")
|
||||
|
||||
// #1341: the agent-reported Ceph cluster used to land in state but
|
||||
// never get evaluated for pool alerts. Only the Proxmox-API polling
|
||||
// path ran cephPoolAlertStorageTargets, so overrides set against
|
||||
// agent-prefixed pool IDs (e.g. agent:hostname-ceph-pool-foo) were
|
||||
// dormant. Run the alert check here so agent-sourced pools fire.
|
||||
if m.alertManager != nil {
|
||||
for _, storage := range cephPoolAlertStorageTargets(cephCluster) {
|
||||
m.alertManager.CheckStorage(storage)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if m.alertManager != nil {
|
||||
|
|
|
|||
|
|
@ -1160,3 +1160,79 @@ func TestApplyHostReport_FallbackIdentifier(t *testing.T) {
|
|||
t.Errorf("expected hostname 'fallback-host', got %q", host.Hostname)
|
||||
}
|
||||
}
|
||||
|
||||
// Regression for #1341: when Ceph is reported by a Pulse host-agent (not the
|
||||
// Proxmox API), the agent-sourced cluster used to land in state but the
|
||||
// pool alert evaluation only ran in the Proxmox-API polling path. Per-pool
|
||||
// overrides keyed against the agent-prefixed pool ID (e.g.
|
||||
// `agent:hostname-ceph-pool-name`) were silently dormant. Verify the
|
||||
// agent path now fires CheckStorage for each pool so overrides match.
|
||||
func TestApplyHostReportFiresCephPoolAlertsForAgentSourcedCluster(t *testing.T) {
|
||||
monitor := &Monitor{
|
||||
state: models.NewState(),
|
||||
alertManager: alerts.NewManager(),
|
||||
hostTokenBindings: make(map[string]string),
|
||||
config: &config.Config{},
|
||||
rateTracker: NewRateTracker(),
|
||||
}
|
||||
t.Cleanup(func() { monitor.alertManager.Stop() })
|
||||
|
||||
overrideTrigger := alerts.HysteresisThreshold{Trigger: 50, Clear: 45}
|
||||
cfg := monitor.alertManager.GetConfig()
|
||||
cfg.MinimumDelta = 0
|
||||
if cfg.TimeThresholds == nil {
|
||||
cfg.TimeThresholds = make(map[string]int)
|
||||
}
|
||||
cfg.TimeThresholds["storage"] = 0
|
||||
cfg.StorageDefault = alerts.HysteresisThreshold{Trigger: 95, Clear: 90}
|
||||
cfg.Overrides = map[string]alerts.ThresholdConfig{
|
||||
"agent:pve5-ceph-pool-data_replication": {Usage: &overrideTrigger},
|
||||
}
|
||||
monitor.alertManager.UpdateConfig(cfg)
|
||||
|
||||
report := agentshost.Report{
|
||||
Agent: agentshost.AgentInfo{
|
||||
ID: "agent-ceph",
|
||||
Version: "1.0.0",
|
||||
IntervalSeconds: 30,
|
||||
},
|
||||
Host: agentshost.HostInfo{
|
||||
ID: "pve5-host",
|
||||
Hostname: "pve5",
|
||||
Platform: "linux",
|
||||
},
|
||||
Timestamp: time.Now().UTC(),
|
||||
Ceph: &agentshost.CephCluster{
|
||||
FSID: "ceph-fsid-1341",
|
||||
Health: agentshost.CephHealth{Status: "HEALTH_OK"},
|
||||
Pools: []agentshost.CephPool{
|
||||
{
|
||||
ID: 2,
|
||||
Name: "data_replication",
|
||||
BytesUsed: 611,
|
||||
BytesAvailable: 389,
|
||||
PercentUsed: 61.1,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
if _, err := monitor.ApplyHostReport(report, nil); err != nil {
|
||||
t.Fatalf("ApplyHostReport: %v", err)
|
||||
}
|
||||
|
||||
active := monitor.alertManager.GetActiveAlerts()
|
||||
found := false
|
||||
for _, alert := range active {
|
||||
if alert.ID == "agent:pve5-ceph-pool-data_replication-usage" {
|
||||
found = true
|
||||
if alert.Threshold != 50 {
|
||||
t.Fatalf("Ceph pool alert fired but threshold = %.1f, want 50 (override)", alert.Threshold)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("expected agent-sourced Ceph pool alert to fire under override at 61.1%% usage; got %d active alerts", len(active))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue