Fix Ceph pool alert threshold targets

Refs #1341
This commit is contained in:
rcourtman 2026-05-24 21:35:46 +01:00
parent 2dffa40379
commit 6f3bea32ff
9 changed files with 491 additions and 9 deletions

View file

@ -24,6 +24,7 @@ import type {
Host,
Alert,
Storage,
CephCluster,
PBSInstance,
PMGInstance,
DockerHost,
@ -43,6 +44,7 @@ import { ResourceTable, Resource, GroupHeaderMeta } from './ResourceTable';
import { useAlertsActivation } from '@/stores/alertsActivation';
import { logger } from '@/utils/logger';
import { formatTemperature } from '@/utils/temperature';
import { buildThresholdStorageResources } from '@/utils/cephThresholdStorage';
type OverrideType =
| 'guest'
| 'node'
@ -167,6 +169,7 @@ interface ThresholdsTableProps {
nodes: Node[];
hosts: Host[];
storage: Storage[];
cephClusters?: CephCluster[];
dockerHosts: DockerHost[];
pbsInstances?: PBSInstance[]; // PBS instances from state
pmgInstances?: PMGInstance[]; // PMG instances from state
@ -1715,8 +1718,12 @@ export function ThresholdsTable(props: ThresholdsTableProps) {
const search = searchTerm().toLowerCase();
const overridesMap = new Map((props.overrides() ?? []).map((o) => [o.id, o]));
const storageResources = buildThresholdStorageResources(
props.storage ?? [],
props.cephClusters ?? [],
);
const storageDevices = (props.storage ?? []).map((storage) => {
const storageDevices = storageResources.map((storage) => {
const override = overridesMap.get(storage.id);
// Storage only has usage threshold
@ -1801,7 +1808,7 @@ export function ThresholdsTable(props: ThresholdsTableProps) {
{
key: 'storage' as const,
label: 'Storage',
total: props.storage?.length ?? 0,
total: storageWithOverrides().length,
overrides: countOverrides(storageWithOverrides()),
tab: 'proxmox' as const,
},
@ -3253,7 +3260,7 @@ export function ThresholdsTable(props: ThresholdsTableProps) {
<CollapsibleSection
id="storage"
title="Storage Devices"
resourceCount={props.storage.length}
resourceCount={storageWithOverrides().length}
collapsed={isCollapsed('storage')}
onToggle={() => toggleSection('storage')}
icon={<HardDrive class="w-5 h-5" />}

View file

@ -100,6 +100,7 @@ const baseProps = () => ({
nodes: [],
hosts: [],
storage: [],
cephClusters: [],
dockerHosts: [],
pbsInstances: [],
pmgInstances: [],
@ -302,6 +303,50 @@ describe('ThresholdsTable Resource Rendering', () => {
expect(screen.getByTestId('resource-name-guest1')).toHaveTextContent('vm1');
});
it('renders Ceph pools as storage threshold resources', async () => {
setPathname('/alerts/thresholds/proxmox');
render(() => <ThresholdsTable
{...(baseProps() as any)}
cephClusters={[
{
id: 'ceph-fsid',
instance: 'inst1',
name: 'Ceph',
health: 'HEALTH_OK',
totalBytes: 1000,
usedBytes: 910,
availableBytes: 90,
usagePercent: 91,
numMons: 3,
numMgrs: 1,
numOsds: 3,
numOsdsUp: 3,
numOsdsIn: 3,
numPGs: 64,
lastUpdated: 1,
pools: [
{
id: 2,
name: 'data_replication',
storedBytes: 910,
availableBytes: 90,
objects: 42,
percentUsed: 91,
},
],
},
]}
/>);
await waitFor(() => {
expect(screen.getByTestId('section-Storage Devices')).toBeInTheDocument();
});
expect(screen.getByTestId('resource-row-inst1-ceph-pool-data_replication')).toBeInTheDocument();
expect(screen.getByTestId('resource-name-inst1-ceph-pool-data_replication')).toHaveTextContent('data_replication');
});
});
describe('ThresholdsTable Metric Formatting', () => {

View file

@ -26,6 +26,7 @@ import type { Alert, Incident, IncidentEvent, State, VM, Container, DockerHost,
import { useNavigate, useLocation } from '@solidjs/router';
import { useAlertsActivation } from '@/stores/alertsActivation';
import { logger } from '@/utils/logger';
import { buildThresholdStorageResources } from '@/utils/cephThresholdStorage';
import LayoutDashboard from 'lucide-solid/icons/layout-dashboard';
import History from 'lucide-solid/icons/history';
import Gauge from 'lucide-solid/icons/gauge';
@ -716,12 +717,15 @@ export function Alerts() {
const [rawOverridesConfig, setRawOverridesConfig] = createSignal<
Record<string, RawOverrideConfig>
>({}); // Store raw config
const thresholdStorageResources = createMemo(() =>
buildThresholdStorageResources(state.storage || [], state.cephClusters || []),
);
createEffect(() => {
const currentRawOverrides = rawOverridesConfig();
const normalized = normalizeRawOverrideConfigKeys(
currentRawOverrides,
state.storage || [],
thresholdStorageResources(),
[...(state.vms || []), ...(state.containers || [])],
);
if (JSON.stringify(normalized) !== JSON.stringify(currentRawOverrides)) {
@ -976,7 +980,7 @@ export function Alerts() {
});
} else {
// Check if it's a storage device
const storage = (state.storage || []).find((s) => s.id === key);
const storage = thresholdStorageResources().find((s) => s.id === key);
if (storage) {
overridesList.push({
id: key,
@ -1341,7 +1345,7 @@ export function Alerts() {
setRawOverridesConfig(
normalizeRawOverrideConfigKeys(
config.overrides || {},
state.storage || [],
thresholdStorageResources(),
[...(state.vms || []), ...(state.containers || [])],
),
);
@ -3115,6 +3119,7 @@ function ThresholdsTab(props: ThresholdsTabProps) {
nodes={props.state.nodes || []}
hosts={props.hosts}
storage={props.state.storage || []}
cephClusters={props.state.cephClusters || []}
dockerHosts={props.state.dockerHosts || []}
pbsInstances={props.state.pbs || []}
pmgInstances={props.state.pmg || []}
@ -4742,7 +4747,10 @@ function HistoryTab(props: {
if (node) return 'Node';
// Check storage
const storage = state.storage?.find((s) => s.name === resourceName || s.id === resourceName);
const storage = buildThresholdStorageResources(
state.storage || [],
state.cephClusters || [],
).find((s) => s.name === resourceName || s.id === resourceName);
if (storage) return 'Storage';
// Docker hosts

View file

@ -0,0 +1,117 @@
import { describe, expect, it } from 'vitest';
import {
buildCephPoolThresholdStorage,
buildThresholdStorageResources,
cephPoolStorageId,
} from '../cephThresholdStorage';
import type { CephCluster, Storage } from '@/types/api';
describe('ceph threshold storage helpers', () => {
it('uses the same stable Ceph pool storage id shape as the backend', () => {
expect(cephPoolStorageId('inst1', { id: 2, name: 'data_replication' })).toBe(
'inst1-ceph-pool-data_replication',
);
expect(cephPoolStorageId('Main Cluster', { id: 3, name: 'pool/slash' })).toBe(
'Main-Cluster-ceph-pool-pool-slash',
);
});
it('projects Ceph pools into storage threshold resources', () => {
const clusters: CephCluster[] = [
{
id: 'cluster-1',
instance: 'inst1',
name: 'Ceph',
health: 'HEALTH_OK',
totalBytes: 1000,
usedBytes: 910,
availableBytes: 90,
usagePercent: 91,
numMons: 3,
numMgrs: 1,
numOsds: 3,
numOsdsUp: 3,
numOsdsIn: 3,
numPGs: 64,
lastUpdated: 1,
pools: [
{
id: 2,
name: 'data_replication',
storedBytes: 910,
availableBytes: 90,
objects: 42,
percentUsed: 91,
},
],
},
];
expect(buildCephPoolThresholdStorage(clusters)).toMatchObject([
{
id: 'inst1-ceph-pool-data_replication',
name: 'data_replication',
node: 'cluster',
instance: 'inst1',
type: 'ceph-pool',
status: 'available',
usage: 91,
shared: true,
},
]);
});
it('deduplicates Ceph pool resources against existing storage resources', () => {
const storage: Storage[] = [
{
id: 'inst1-ceph-pool-data_replication',
name: 'data_replication',
node: 'cluster',
instance: 'inst1',
type: 'rbd',
status: 'available',
total: 1000,
used: 910,
free: 90,
usage: 91,
content: 'images',
shared: true,
enabled: true,
active: true,
},
];
const clusters: CephCluster[] = [
{
id: 'cluster-1',
instance: 'inst1',
name: 'Ceph',
health: 'HEALTH_OK',
totalBytes: 1000,
usedBytes: 910,
availableBytes: 90,
usagePercent: 91,
numMons: 3,
numMgrs: 1,
numOsds: 3,
numOsdsUp: 3,
numOsdsIn: 3,
numPGs: 64,
lastUpdated: 1,
pools: [
{
id: 2,
name: 'data_replication',
storedBytes: 1,
availableBytes: 1,
objects: 1,
percentUsed: 50,
},
],
},
];
expect(buildThresholdStorageResources(storage, clusters)).toHaveLength(1);
});
});

View file

@ -0,0 +1,71 @@
import type { CephCluster, CephPool, Storage } from '@/types/api';
export const sanitizeCephPoolStorageComponent = (value: string): string => {
const trimmed = value.trim();
if (!trimmed) return '';
return trimmed
.replace(/[^A-Za-z0-9_.:-]+/g, '-')
.replace(/-{2,}/g, '-')
.replace(/^-|-$/g, '');
};
export const cephPoolStorageId = (
instanceName: string,
pool: Pick<CephPool, 'id' | 'name'>,
): string => {
const instance = sanitizeCephPoolStorageComponent(instanceName) || 'ceph';
const poolName = sanitizeCephPoolStorageComponent(pool.name || '') || `pool-${pool.id}`;
return `${instance}-ceph-pool-${poolName}`;
};
export const buildCephPoolThresholdStorage = (cephClusters: CephCluster[] = []): Storage[] => {
const targets: Storage[] = [];
cephClusters.forEach((cluster) => {
const instance = cluster.instance?.trim() || cluster.id || 'ceph';
(cluster.pools || []).forEach((pool) => {
const name = pool.name?.trim() || `pool-${pool.id}`;
const used = pool.storedBytes || 0;
const free = pool.availableBytes || 0;
const total = used + free;
const usage = pool.percentUsed > 0 ? pool.percentUsed : total > 0 ? (used / total) * 100 : 0;
targets.push({
id: cephPoolStorageId(instance, pool),
name,
node: 'cluster',
instance,
type: 'ceph-pool',
status: 'available',
total,
used,
free,
usage,
content: 'ceph',
shared: true,
enabled: true,
active: true,
pool: name,
});
});
});
return targets;
};
export const buildThresholdStorageResources = (
storage: Storage[] = [],
cephClusters: CephCluster[] = [],
): Storage[] => {
const resources = [...storage];
const seen = new Set(resources.map((entry) => entry.id));
buildCephPoolThresholdStorage(cephClusters).forEach((entry) => {
if (seen.has(entry.id)) return;
seen.add(entry.id);
resources.push(entry);
});
return resources;
};

View file

@ -118,6 +118,113 @@ func hydrateCephStorageUsageFromDF(storage []models.Storage, df *proxmox.CephDF)
return updated
}
func sanitizeCephPoolStorageComponent(value string) string {
value = strings.TrimSpace(value)
if value == "" {
return ""
}
var builder strings.Builder
lastDash := false
for _, r := range value {
allowed := (r >= 'a' && r <= 'z') ||
(r >= 'A' && r <= 'Z') ||
(r >= '0' && r <= '9') ||
r == '_' ||
r == '.' ||
r == ':' ||
r == '-'
if allowed {
builder.WriteRune(r)
lastDash = false
continue
}
if !lastDash {
builder.WriteByte('-')
lastDash = true
}
}
result := builder.String()
for strings.Contains(result, "--") {
result = strings.ReplaceAll(result, "--", "-")
}
return strings.Trim(result, "-")
}
func cephPoolStorageID(instanceName string, pool models.CephPool) string {
instance := sanitizeCephPoolStorageComponent(instanceName)
if instance == "" {
instance = "ceph"
}
poolName := sanitizeCephPoolStorageComponent(pool.Name)
if poolName == "" {
poolName = fmt.Sprintf("pool-%d", pool.ID)
}
return fmt.Sprintf("%s-ceph-pool-%s", instance, poolName)
}
func cephPoolAlertStorageTargets(cluster models.CephCluster) []models.Storage {
if len(cluster.Pools) == 0 {
return nil
}
instanceName := strings.TrimSpace(cluster.Instance)
if instanceName == "" {
instanceName = cluster.ID
}
targets := make([]models.Storage, 0, len(cluster.Pools))
for _, pool := range cluster.Pools {
name := strings.TrimSpace(pool.Name)
if name == "" {
name = fmt.Sprintf("pool-%d", pool.ID)
}
used := pool.StoredBytes
free := pool.AvailableBytes
total := used + free
usage := pool.PercentUsed
if usage <= 0 && total > 0 {
usage = safePercentage(float64(used), float64(total))
}
targets = append(targets, models.Storage{
ID: cephPoolStorageID(instanceName, pool),
Name: name,
Node: "cluster",
Instance: instanceName,
Type: "ceph-pool",
Status: "available",
Pool: name,
Total: total,
Used: used,
Free: free,
Usage: usage,
Content: "ceph",
Shared: true,
Enabled: true,
Active: true,
})
}
return targets
}
func cephPoolAlertStorageTargetsForInstance(state models.StateSnapshot, instanceName string) []models.Storage {
instanceName = strings.TrimSpace(instanceName)
targets := make([]models.Storage, 0)
for _, cluster := range state.CephClusters {
if instanceName != "" && strings.TrimSpace(cluster.Instance) != instanceName {
continue
}
targets = append(targets, cephPoolAlertStorageTargets(cluster)...)
}
return targets
}
// pollCephCluster gathers Ceph cluster information when Ceph-backed storage is detected.
func (m *Monitor) pollCephCluster(ctx context.Context, instanceName string, client PVEClientInterface, cephDetected bool) {
if !cephDetected {

View file

@ -12175,6 +12175,15 @@ func (m *Monitor) checkMockAlerts() {
Msg("Checking storage for alerts")
m.alertManager.CheckStorage(storage)
}
for _, cluster := range state.CephClusters {
for _, storage := range cephPoolAlertStorageTargets(cluster) {
log.Debug().
Str("name", storage.Name).
Float64("usage", storage.Usage).
Msg("Checking Ceph pool storage for alerts")
m.alertManager.CheckStorage(storage)
}
}
// Check alerts for PBS instances
log.Info().Int("pbsCount", len(state.PBSInstances)).Msg("Checking PBS alerts")

View file

@ -1978,19 +1978,35 @@ func (m *Monitor) pollStorageWithNodes(ctx context.Context, instanceName string,
var cephDF *proxmox.CephDF
cephFetchAttempted := false
cephFetchFailed := false
var cephPoolCheckTargets []models.Storage
var cephPoolSyncTargets []models.Storage
var cephClusterForState *models.CephCluster
if (instanceCfg == nil || !instanceCfg.DisableCeph) && cephDetected {
cephFetchAttempted = true
var err error
cephStatus, cephDF, err = fetchCephClusterData(ctx, instanceName, client)
if err != nil {
cephFetchFailed = true
cephPoolSyncTargets = cephPoolAlertStorageTargetsForInstance(m.state.GetSnapshot(), instanceName)
} else if cephStatus != nil {
hydrateCephStorageUsageFromDF(allStorage, cephDF)
cluster := buildCephClusterModel(instanceName, cephStatus, cephDF)
if cluster.ID == "" {
cluster.ID = instanceName
}
cephClusterForState = &cluster
cephPoolCheckTargets = cephPoolAlertStorageTargets(cluster)
cephPoolSyncTargets = cephPoolCheckTargets
}
}
storageCheckTargets := append([]models.Storage{}, allStorage...)
storageCheckTargets = append(storageCheckTargets, cephPoolCheckTargets...)
storageSyncTargets := append([]models.Storage{}, allStorage...)
storageSyncTargets = append(storageSyncTargets, cephPoolSyncTargets...)
// Record metrics and check alerts for all storage devices
for _, storage := range allStorage {
for _, storage := range storageCheckTargets {
if m.metricsHistory != nil {
timestamp := time.Now()
m.metricsHistory.AddStorageMetric(storage.ID, "usage", storage.Usage, timestamp)
@ -2012,7 +2028,10 @@ func (m *Monitor) pollStorageWithNodes(ctx context.Context, instanceName string,
}
}
if m.alertManager != nil {
m.alertManager.SyncStorageAlertsForInstance(storageInstanceName, allStorage)
m.alertManager.SyncStorageAlertsForInstance(storageInstanceName, storageSyncTargets)
if instanceName != storageInstanceName {
m.alertManager.SyncStorageAlertsForInstance(instanceName, storageSyncTargets)
}
}
// Update state with all storage
@ -2027,6 +2046,8 @@ func (m *Monitor) pollStorageWithNodes(ctx context.Context, instanceName string,
// Preserve previous Ceph state when the refresh fails.
case cephFetchAttempted && cephStatus == nil:
m.state.UpdateCephClustersForInstance(instanceName, []models.CephCluster{})
case cephClusterForState != nil:
m.state.UpdateCephClustersForInstance(instanceName, []models.CephCluster{*cephClusterForState})
default:
cluster := buildCephClusterModel(instanceName, cephStatus, cephDF)
if cluster.ID == "" {

View file

@ -541,6 +541,103 @@ func TestPollStorageWithNodesOptimizedHydratesSharedCephStorageFromDF(t *testing
}
}
func TestPollStorageWithNodesOptimizedChecksCephPoolAlerts(t *testing.T) {
t.Setenv("PULSE_DATA_DIR", t.TempDir())
monitor := &Monitor{
state: &models.State{},
metricsHistory: NewMetricsHistory(16, time.Hour),
alertManager: alerts.NewManager(),
}
t.Cleanup(func() {
monitor.alertManager.Stop()
})
cfg := monitor.alertManager.GetConfig()
cfg.MinimumDelta = 0
if cfg.TimeThresholds == nil {
cfg.TimeThresholds = make(map[string]int)
}
cfg.TimeThresholds["storage"] = 0
cfg.StorageDefault = alerts.HysteresisThreshold{Trigger: 80, Clear: 70}
monitor.alertManager.UpdateConfig(cfg)
cephStorage := proxmox.Storage{
Storage: "ceph-shared",
Type: "rbd",
Content: "images,rootdir",
Nodes: "pve1,pve2",
Pool: "ceph-pool",
}
client := &fakeStorageClient{
allStorage: []proxmox.Storage{cephStorage},
storageByNode: map[string][]proxmox.Storage{
"pve1": {},
"pve2": {},
},
cephStatus: &proxmox.CephStatus{
FSID: "ceph-fsid",
PGMap: proxmox.CephPGMap{
BytesTotal: 5000,
BytesUsed: 4500,
BytesAvail: 500,
},
},
cephDF: &proxmox.CephDF{
Data: proxmox.CephDFData{
Pools: []proxmox.CephDFPool{
{
ID: 2,
Name: "data_replication",
Stats: proxmox.CephDFPoolStat{
BytesUsed: 910,
MaxAvail: 90,
PercentUsed: 91,
},
},
},
},
},
}
nodes := []proxmox.Node{
{Node: "pve1", Status: "online"},
{Node: "pve2", Status: "online"},
}
monitor.pollStorageWithNodes(context.Background(), "inst1", client, nodes)
for _, storage := range monitor.state.Storage {
if storage.ID == "inst1-ceph-pool-data_replication" {
t.Fatal("Ceph pool alert target should not be inserted into the main storage inventory")
}
}
metrics := monitor.metricsHistory.GetAllStorageMetrics("inst1-ceph-pool-data_replication", time.Minute)
if len(metrics["usage"]) != 1 {
t.Fatalf("expected one Ceph pool usage metric entry, got %d", len(metrics["usage"]))
}
if diff := math.Abs(metrics["usage"][0].Value - 91); diff > 0.001 {
t.Fatalf("expected Ceph pool usage metric 91, diff %.4f", diff)
}
alerts := monitor.alertManager.GetActiveAlerts()
found := false
for _, alert := range alerts {
if alert.ID == "inst1-ceph-pool-data_replication-usage" {
found = true
if alert.ResourceName != "data_replication" {
t.Fatalf("Ceph pool alert resource name = %q, want data_replication", alert.ResourceName)
}
break
}
}
if !found {
t.Fatal("expected Ceph pool usage alert to be active")
}
}
func TestPollStorageWithNodesOptimizedClearsStaleStorageAlertsWhenIdentityChanges(t *testing.T) {
t.Setenv("PULSE_DATA_DIR", t.TempDir())