Fix host-relative memory trends

This commit is contained in:
rcourtman 2026-08-02 19:07:31 +01:00
parent c8ff52339f
commit f92bb3b527
20 changed files with 278 additions and 62 deletions

View file

@ -305,6 +305,10 @@ Proxmox legacy model CPU fields have the inverse compatibility shape:
`models.Node.CPU`, `models.VM.CPU`, and `models.Container.CPU` remain 0..1
ratios, but `/api/metrics-store/history` live fallback points must convert
those ratios to 0..100 chart percentages before serialization.
Proxmox VM and system-container chart payloads add a `memoryused` byte series
beside the existing guest-relative `memory` percentage series. Consumers that
present memory against a host denominator must derive that trend from
`memoryused`; they must not relabel the percentage series.
Candidate import plans for PVE/PBS/PMG onboarding are API-contract consumers,
not local UI estimates. Probe and Discovery candidates must build their

View file

@ -477,9 +477,12 @@ changes.
trimmed connection ID with defaults applied" active-connection policy. A
new platform poller of this family must reuse both rather than copying
the TrueNAS/VMware loop, and traditional PVE guest polling records
per-guest series through the canonical `recordGuestMetric` helper in
per-guest series through the canonical `recordGuestMetric` helper in
`internal/monitoring/monitor_pve_guest_helpers.go` instead of inline
metric writes.
metric writes. Proxmox guest memory history stores both the canonical
guest-relative `memory` percentage and raw `memoryused` bytes so API
consumers can apply alternate capacity denominators without reconstructing
bytes from a mutable guest allocation.
Discovery config and configured-host IP resolution must stay off the
monitor lock. `internal/monitoring/monitor_discovery_helpers.go` exposes
the canonical `discoveryConfigSnapshot()` that discovery providers consume,

View file

@ -604,9 +604,12 @@ change may globally weaken the Task 03 lifecycle-state idempotency invariant.
Direct used-versus-total comparisons may provide a comparison label and a
separate severity percentage. This lets Proxmox guest rows draw used bytes
against the host total while retaining guest-allocation threshold meaning.
Host-relative mode must not relabel guest-relative history as host history;
until raw-byte history supports the alternate denominator, the memory cell
stays on the current comparison bar when the rest of the table uses trends.
Host-relative mode must not relabel guest-relative history as host history.
Proxmox guest history records raw `memoryused` byte samples alongside the
guest-relative percentage, and the workload table derives every host-share
trend point from those bytes and the bounded parent-node total. If raw-byte
history is unavailable, the trend remains empty rather than substituting the
guest-relative percentage.
22. Extend metric-bar width, label-fit logic, and resize-observer runtime through `frontend-modern/src/components/Workloads/metricBarModel.ts` and `frontend-modern/src/components/Workloads/useMetricBarState.ts` rather than rebuilding metric-local state and threshold mapping inside `frontend-modern/src/components/Workloads/MetricBar.tsx`
23. Extend enhanced CPU bar usage/anomaly presentation and tooltip runtime through `frontend-modern/src/components/Workloads/enhancedCpuBarModel.ts` and `frontend-modern/src/components/Workloads/useEnhancedCPUBarState.ts` rather than rebuilding tooltip-local state and CPU-threshold formatting inside `frontend-modern/src/components/Workloads/EnhancedCPUBar.tsx`
Workload and unified-resource metric bars may accept resolved alert display

View file

@ -25,6 +25,7 @@ export interface AggregatedMetricPoint {
export interface ChartData {
cpu?: MetricPoint[];
memory?: MetricPoint[];
memoryused?: MetricPoint[];
disk?: MetricPoint[];
diskread?: MetricPoint[];
diskwrite?: MetricPoint[];

View file

@ -31,7 +31,10 @@ import { getWorkloadGuestDiskStatusMessage } from '@/utils/workloadGuestPresenta
import { ResourceNameWithWebInterfaceLink } from '@/components/shared/WebInterfaceLink';
import type { GuestRowProps } from './guestRowModel';
import { useGuestRowState } from './useGuestRowState';
import type { WorkloadTableMetric } from './workloadMetricHistoryModel';
import type {
WorkloadMetricHistoryReader,
WorkloadTableMetric,
} from './workloadMetricHistoryModel';
type Guest = WorkloadGuest;
@ -120,21 +123,21 @@ export function GuestRow(props: GuestRowProps) {
const noun = context.count === 1 ? 'container' : 'containers';
return `${context.count} nested ${context.label} ${noun}. Open row for details.`;
});
const normalizeMetricPercent = (value: number | null | undefined): number => {
const clampMetricPercent = (value: number | null | undefined): number => {
if (typeof value !== 'number' || !Number.isFinite(value)) return 0;
return value <= 1 ? Math.max(0, value * 100) : Math.max(0, value);
return Math.max(0, Math.min(100, value));
};
const formatMetricPercent = (value: number | null | undefined): string =>
`${Math.round(normalizeMetricPercent(value))}%`;
`${Math.round(clampMetricPercent(value))}%`;
const usagePercent = (
used?: number | null,
total?: number | null,
fallbackUsage?: number | null,
): number => {
if (typeof used === 'number' && typeof total === 'number' && total > 0) {
return normalizeMetricPercent((used / total) * 100);
return Math.max(0, Math.min(100, (used / total) * 100));
}
return normalizeMetricPercent(fallbackUsage);
return clampMetricPercent(fallbackUsage);
};
const renderMetricSparkline = (
metric: WorkloadTableMetric,
@ -143,9 +146,10 @@ export function GuestRow(props: GuestRowProps) {
unit = '%',
valueLabelMode: 'inline' | 'tooltip' | 'hidden' = 'inline',
formatValue?: (value: number) => string,
seriesOptions?: Parameters<WorkloadMetricHistoryReader['getGuestMetricSeries']>[2],
) => (
<MetricMiniSparkline
series={props.metricHistory?.getGuestMetricSeries(props.guest, metric) ?? []}
series={props.metricHistory?.getGuestMetricSeries(props.guest, metric, seriesOptions) ?? []}
valueLabel={valueLabel}
valueLabelMode={valueLabelMode}
title={title}
@ -372,7 +376,7 @@ export function GuestRow(props: GuestRowProps) {
<Show when={isColVisible('memory')}>
<td class="px-1.5 sm:px-2 py-0.5 align-middle" data-workload-col="memory">
<Show
when={isSparklineMode() && !isHostMemoryBasis()}
when={isSparklineMode()}
fallback={
<div title={isHostMemoryBasis() ? undefined : (memoryTooltip() ?? undefined)}>
<StackedMemoryBar
@ -409,19 +413,25 @@ export function GuestRow(props: GuestRowProps) {
>
{renderMetricSparkline(
'memory',
props.guest.memory?.usageUnavailable
memoryDisplayUnavailable()
? 'N/A'
: formatMetricPercent(
usagePercent(
props.guest.memory?.used,
props.guest.memory?.total,
props.guest.memory?.usage,
memoryDisplayTotal(),
isHostMemoryBasis() ? undefined : props.guest.memory?.usage,
),
),
`${props.guest.name} memory history`,
isHostMemoryBasis()
? `${props.guest.name} host memory share history`
: `${props.guest.name} memory history`,
'%',
'inline',
formatMetricPercent,
{
memoryDisplayBasis: props.memoryDisplayBasis,
parentMemoryTotal: props.parentMemoryTotal,
},
)}
</Show>
</td>

View file

@ -272,7 +272,20 @@ describe('GuestRow', () => {
expect(screen.getByTestId('memory-bar')).toBeTruthy();
});
it('compares guest memory use with the parent host total when requested', () => {
it('renders raw-byte memory history against the parent host total when requested', () => {
const getGuestMetricSeries = vi.fn(
(_guest: WorkloadGuest, _metric: string, _options?: unknown) => [
{
id: 'memory',
label: 'Host memory share',
color: '#f59e0b',
points: [
{ timestamp: 1, value: 10 },
{ timestamp: 2, value: 12.5 },
],
},
],
);
renderGuestRow({
guest: makeGuest({ memory: makeMemory({ used: 2 * 1024 ** 3, total: 4 * 1024 ** 3 }) }),
visibleColumnIds: ['name', 'memory'],
@ -280,21 +293,22 @@ describe('GuestRow', () => {
parentMemoryTotal: 16 * 1024 ** 3,
parentNodeName: 'pve-01',
metricDisplayMode: 'sparklines',
metricHistory: {
getGuestMetricSeries,
getNodeMetricSeries: () => [],
},
});
expect(screen.getByTestId('memory-bar')).toHaveAttribute(
'data-total',
String(16 * 1024 ** 3),
expect(screen.getByTestId('metric-mini-sparkline')).toHaveAttribute(
'title',
'test-vm host memory share history',
);
expect(screen.getByTestId('memory-bar')).toHaveAttribute(
'data-comparison-total-label',
'Host total',
);
expect(screen.getByTestId('memory-bar')).toHaveAttribute(
'data-tooltip-title',
'pve-01 memory share',
);
expect(screen.queryByTestId('metric-mini-sparkline')).toBeNull();
expect(screen.getByText('13%')).toBeTruthy();
expect(screen.queryByTestId('memory-bar')).toBeNull();
expect(getGuestMetricSeries).toHaveBeenCalledWith(expect.any(Object), 'memory', {
memoryDisplayBasis: 'host',
parentMemoryTotal: 16 * 1024 ** 3,
});
});
it('marks host-relative memory unavailable when the parent total is missing', () => {
@ -307,6 +321,21 @@ describe('GuestRow', () => {
expect(screen.getByTestId('memory-bar')).toHaveAttribute('data-unavailable', 'true');
});
it('does not inflate a sub-one-percent host memory share', () => {
renderGuestRow({
guest: makeGuest({
memory: makeMemory({ used: 128 * 1024 ** 2, total: 4 * 1024 ** 3 }),
}),
visibleColumnIds: ['name', 'memory'],
memoryDisplayBasis: 'host',
parentMemoryTotal: 16 * 1024 ** 3,
metricDisplayMode: 'sparklines',
});
expect(screen.getByText('1%')).toBeTruthy();
expect(screen.queryByText('78%')).toBeNull();
});
it('renders disk bar when disk usage is available', () => {
renderGuestRow({ guest: makeGuest() });
expect(screen.getByTestId('disk-bar')).toBeTruthy();

View file

@ -83,6 +83,40 @@ describe('workloadMetricHistoryModel', () => {
expect(series[0].points[1].value).toBe(20);
});
it('derives host-relative memory history from raw used bytes', () => {
const gib = 1024 ** 3;
const series = getMetricSparklineSeriesFromChartData(
{
memory: [
{ timestamp: 1, value: 50 },
{ timestamp: 2, value: 75 },
],
memoryused: [
{ timestamp: 1, value: 2 * gib },
{ timestamp: 2, value: 4 * gib },
],
},
'memory',
{ memoryDisplayBasis: 'host', parentMemoryTotal: 16 * gib },
);
expect(series[0].label).toBe('Host memory share');
expect(series[0].points).toEqual([
{ timestamp: 1, value: 12.5 },
{ timestamp: 2, value: 25 },
]);
});
it('does not relabel guest-relative percentages when raw host-relative history is absent', () => {
const series = getMetricSparklineSeriesFromChartData(
{ memory: [{ timestamp: 1, value: 75 }] },
'memory',
{ memoryDisplayBasis: 'host', parentMemoryTotal: 16 * 1024 ** 3 },
);
expect(series[0].points).toEqual([]);
});
it('builds bounded mini sparkline paths', () => {
const scale = getMetricMiniSparklineScale(
[

View file

@ -2,7 +2,6 @@ import { createMemo, type Accessor } from 'solid-js';
import type { WorkloadChartsResponse } from '@/api/charts';
import type { Node } from '@/types/api';
import type { WorkloadGuest } from '@/types/workloads';
import { createNonSuspendingQuery } from '@/hooks/createNonSuspendingQuery';
import {
fetchInfrastructureSummaryAndCache,
@ -94,13 +93,17 @@ export function useWorkloadTableMetricHistory(
pollMs: WORKLOAD_TABLE_HISTORY_POLL_MS,
});
const getGuestMetricSeries = (guest: WorkloadGuest, metric: WorkloadTableMetric) => {
const getGuestMetricSeries: WorkloadMetricHistoryReader['getGuestMetricSeries'] = (
guest,
metric,
seriesOptions,
) => {
const response = workloadHistory.value();
const chartData = findChartDataForCandidates(getWorkloadChartKeyCandidates(guest), [
response.data,
response.dockerData,
]);
return getMetricSparklineSeriesFromChartData(chartData, metric);
return getMetricSparklineSeriesFromChartData(chartData, metric, seriesOptions);
};
const getNodeMetricSeries = (node: Node, metric: WorkloadTableMetric) => {

View file

@ -13,10 +13,16 @@ export interface WorkloadMetricSparklineSeries {
points: MetricPoint[];
}
export interface WorkloadGuestMetricSeriesOptions {
memoryDisplayBasis?: 'guest' | 'host';
parentMemoryTotal?: number;
}
export interface WorkloadMetricHistoryReader {
getGuestMetricSeries: (
guest: WorkloadGuest,
metric: WorkloadTableMetric,
options?: WorkloadGuestMetricSeriesOptions,
) => WorkloadMetricSparklineSeries[];
getNodeMetricSeries: (node: Node, metric: WorkloadTableMetric) => WorkloadMetricSparklineSeries[];
}
@ -199,6 +205,7 @@ const sanitizeMetricPoints = (
export const getMetricSparklineSeriesFromChartData = (
chartData: ChartData | undefined,
metric: WorkloadTableMetric,
options?: WorkloadGuestMetricSeriesOptions,
): WorkloadMetricSparklineSeries[] => {
if (!chartData) return [];
@ -213,6 +220,24 @@ export const getMetricSparklineSeriesFromChartData = (
},
];
case 'memory':
if (options?.memoryDisplayBasis === 'host') {
const hostTotal = options.parentMemoryTotal;
const points =
typeof hostTotal === 'number' && Number.isFinite(hostTotal) && hostTotal > 0
? chartData.memoryused?.map((point) => ({
timestamp: point.timestamp,
value: (point.value / hostTotal) * 100,
}))
: undefined;
return [
{
id: 'memory',
label: 'Host memory share',
color: MEMORY_COLOR,
points: sanitizeMetricPoints(points, clampPercent),
},
];
}
return [
{
id: 'memory',

View file

@ -8,6 +8,7 @@ import {
persistWorkloadsSummaryCache,
readInMemoryWorkloadsSummaryCache,
readWorkloadsSummaryCache,
WORKLOADS_SUMMARY_CACHE_VERSION,
workloadsSummaryCacheScopeKey,
} from '@/utils/workloadsSummaryCache';
@ -41,7 +42,7 @@ vi.mock('@/utils/apiClient', () => ({
// the org-switch cache-invalidation handler by emitting the event.
const ORG_FROM_GETORGID = 'test-org';
const VERSION = 6;
const VERSION = WORKLOADS_SUMMARY_CACHE_VERSION;
const PREFIX = 'pulse.workloadsSummaryCharts.';
// Mirrors the module's private `resolveOrgScope` (with the mocked getOrgID) for
@ -69,6 +70,7 @@ const makeResponse = (cpu?: MetricPoint[]): WorkloadChartsResponse => ({
w1: {
cpu: cpu ?? [metric(1), metric(2)],
memory: [metric(3), metric(4)],
memoryused: [metric(15), metric(16)],
disk: [metric(5), metric(6)],
diskread: [metric(7), metric(8)],
diskwrite: [metric(9), metric(10)],

View file

@ -8,7 +8,7 @@ import { getOrgID } from '@/utils/apiClient';
import { normalizeOrgScope } from '@/utils/orgScope';
import { eventBus } from '@/stores/events';
export const WORKLOADS_SUMMARY_CACHE_VERSION = 6;
export const WORKLOADS_SUMMARY_CACHE_VERSION = 7;
const WORKLOADS_SUMMARY_CACHE_PREFIX = 'pulse.workloadsSummaryCharts.';
const WORKLOADS_SUMMARY_CACHE_MAX_AGE_MS = 5 * 60_000;
@ -18,7 +18,7 @@ const MAX_IN_MEMORY_WORKLOAD_ENTRIES = 20;
type CachedChartData = Pick<
ChartData,
'cpu' | 'memory' | 'disk' | 'diskread' | 'diskwrite' | 'netin' | 'netout'
'cpu' | 'memory' | 'memoryused' | 'disk' | 'diskread' | 'diskwrite' | 'netin' | 'netout'
>;
type CachedWorkloadsSummary = {
@ -88,6 +88,7 @@ const trimPoints = <T>(points: T[] | undefined, max: number): T[] => {
const toCachedChartData = (data: ChartData): CachedChartData => ({
cpu: trimPoints(data.cpu, WORKLOADS_SUMMARY_CACHE_MAX_POINTS_PER_SERIES),
memory: trimPoints(data.memory, WORKLOADS_SUMMARY_CACHE_MAX_POINTS_PER_SERIES),
memoryused: trimPoints(data.memoryused, WORKLOADS_SUMMARY_CACHE_MAX_POINTS_PER_SERIES),
disk: trimPoints(data.disk, WORKLOADS_SUMMARY_CACHE_MAX_POINTS_PER_SERIES),
diskread: trimPoints(data.diskread, WORKLOADS_SUMMARY_CACHE_MAX_POINTS_PER_SERIES),
diskwrite: trimPoints(data.diskwrite, WORKLOADS_SUMMARY_CACHE_MAX_POINTS_PER_SERIES),

View file

@ -6847,6 +6847,7 @@ type guestChartSourceView interface {
SourceID() string
CPUPercent() float64
MemoryPercent() float64
MemoryUsed() int64
DiskPercent() float64
NetIn() float64
NetOut() float64
@ -6874,7 +6875,7 @@ func collectGuestChartData[V guestChartSourceView](
requests = append(requests, monitoring.GuestChartRequest{InMemoryKey: id, SQLResourceID: id})
}
}
batch := monitor.GetGuestMetricsForChartBatch(storeType, requests, duration, infrastructureSummaryMetricOrder...)
batch := monitor.GetGuestMetricsForChartBatch(storeType, requests, duration, guestSparklineMetricOrder...)
for _, g := range guests {
if g == zero {
continue
@ -6890,6 +6891,7 @@ func collectGuestChartData[V guestChartSourceView](
if len(chartData[id]["cpu"]) == 0 {
chartData[id]["cpu"] = []MetricPoint{{Timestamp: currentTime, Value: g.CPUPercent()}}
chartData[id]["memory"] = []MetricPoint{{Timestamp: currentTime, Value: g.MemoryPercent()}}
chartData[id]["memoryused"] = []MetricPoint{{Timestamp: currentTime, Value: float64(g.MemoryUsed())}}
chartData[id]["disk"] = []MetricPoint{{Timestamp: currentTime, Value: g.DiskPercent()}}
chartData[id]["netin"] = []MetricPoint{{Timestamp: currentTime, Value: g.NetIn()}}
chartData[id]["netout"] = []MetricPoint{{Timestamp: currentTime, Value: g.NetOut()}}
@ -6922,13 +6924,14 @@ func fillChartSeriesFromBatch(dst VMChartData, batchMetrics map[string][]monitor
}
var sparklineMetrics = map[string]bool{
"cpu": true,
"memory": true,
"disk": true,
"diskread": true,
"diskwrite": true,
"netin": true,
"netout": true,
"cpu": true,
"memory": true,
"memoryused": true,
"disk": true,
"diskread": true,
"diskwrite": true,
"netin": true,
"netout": true,
}
var infrastructureSummaryMetricOrder = []string{
@ -6941,6 +6944,17 @@ var infrastructureSummaryMetricOrder = []string{
"netout",
}
var guestSparklineMetricOrder = []string{
"cpu",
"memory",
"memoryused",
"disk",
"diskread",
"diskwrite",
"netin",
"netout",
}
var workloadSummaryMetricOrder = []string{
"cpu",
"memory",
@ -7471,11 +7485,11 @@ func (r *Router) buildWorkloadChartsResponse(
workloadChartsBatchWG.Add(4)
go func() {
defer workloadChartsBatchWG.Done()
vmBatchMetrics = monitor.GetGuestMetricsForChartBatch("vm", vmRequests, duration, infrastructureSummaryMetricOrder...)
vmBatchMetrics = monitor.GetGuestMetricsForChartBatch("vm", vmRequests, duration, guestSparklineMetricOrder...)
}()
go func() {
defer workloadChartsBatchWG.Done()
containerBatchMetrics = monitor.GetGuestMetricsForChartBatch("container", containerRequests, duration, infrastructureSummaryMetricOrder...)
containerBatchMetrics = monitor.GetGuestMetricsForChartBatch("container", containerRequests, duration, guestSparklineMetricOrder...)
}()
go func() {
defer workloadChartsBatchWG.Done()
@ -7496,6 +7510,7 @@ func (r *Router) buildWorkloadChartsResponse(
if len(series["cpu"]) == 0 {
series["cpu"] = []MetricPoint{{Timestamp: currentTime, Value: vm.CPUPercent()}}
series["memory"] = []MetricPoint{{Timestamp: currentTime, Value: vm.MemoryPercent()}}
series["memoryused"] = []MetricPoint{{Timestamp: currentTime, Value: float64(vm.MemoryUsed())}}
series["disk"] = []MetricPoint{{Timestamp: currentTime, Value: vm.DiskPercent()}}
series["netin"] = []MetricPoint{{Timestamp: currentTime, Value: vm.NetIn()}}
series["netout"] = []MetricPoint{{Timestamp: currentTime, Value: vm.NetOut()}}
@ -7512,6 +7527,7 @@ func (r *Router) buildWorkloadChartsResponse(
if len(series["cpu"]) == 0 {
series["cpu"] = []MetricPoint{{Timestamp: currentTime, Value: ct.CPUPercent()}}
series["memory"] = []MetricPoint{{Timestamp: currentTime, Value: ct.MemoryPercent()}}
series["memoryused"] = []MetricPoint{{Timestamp: currentTime, Value: float64(ct.MemoryUsed())}}
series["disk"] = []MetricPoint{{Timestamp: currentTime, Value: ct.DiskPercent()}}
series["netin"] = []MetricPoint{{Timestamp: currentTime, Value: ct.NetIn()}}
series["netout"] = []MetricPoint{{Timestamp: currentTime, Value: ct.NetOut()}}

View file

@ -466,11 +466,14 @@ func TestHandleCharts_StatsDebugMetadata(t *testing.T) {
}
// With no history in the test monitor, handleCharts falls back to synthetic points:
// guests: cpu/memory/disk/netin/netout (5 — diskread/diskwrite excluded from sparkline payloads)
// guests: cpu/memory/memoryused/disk/netin/netout (6 — diskread/diskwrite excluded from sparkline payloads)
// nodes: cpu/memory/disk (3)
// storage: disk (1)
if decoded.Stats.PointCounts.Guests != 5 {
t.Fatalf("expected stats.pointCounts.guests=5, got %d", decoded.Stats.PointCounts.Guests)
if decoded.Stats.PointCounts.Guests != 6 {
t.Fatalf("expected stats.pointCounts.guests=6, got %d", decoded.Stats.PointCounts.Guests)
}
if points := decoded.ChartData["vm-1"]["memoryused"]; len(points) != 1 || points[0].Value != 0 {
t.Fatalf("expected additive raw memory-used fallback point, got %+v", points)
}
if decoded.Stats.PointCounts.Nodes != 3 {
t.Fatalf("expected stats.pointCounts.nodes=3, got %d", decoded.Stats.PointCounts.Nodes)
@ -1342,7 +1345,7 @@ func TestHandleWorkloadCharts_UnknownNodeFilterFallsBackToGlobalScope(t *testing
Node: "pve-1",
Instance: "pve",
CPU: 0.3,
Memory: models.Memory{Usage: 42},
Memory: models.Memory{Total: 100, Used: 42, Free: 58, Usage: 42},
Disk: models.Disk{Usage: 55},
}}
@ -1363,6 +1366,11 @@ func TestHandleWorkloadCharts_UnknownNodeFilterFallsBackToGlobalScope(t *testing
if len(decoded.ChartData) == 0 {
t.Fatalf("expected fallback to global scope when node filter is stale")
}
for _, series := range decoded.ChartData {
if points := series["memoryused"]; len(points) != 1 || points[0].Value != 42 {
t.Fatalf("expected raw memory-used workload fallback point, got %+v", points)
}
}
}
func TestHandleWorkloadsSummaryCharts_UnknownNodeFilterFallsBackToGlobalScope(t *testing.T) {

View file

@ -94,7 +94,7 @@ func TestIssue1613HistoryWritesValidZeroAndSkipsUnknownRates(t *testing.T) {
now := time.Now()
diskRead, diskWrite, networkIn, networkOut := guestHistoryRates(0, 0, 0, 0, models.IORateValidity{Explicit: true})
monitor.recordGuestMetric("vm", "unknown", 0, 0, 0, diskRead, diskWrite, networkIn, networkOut, now)
monitor.recordGuestMetric("vm", "unknown", 0, 0, 0, 0, diskRead, diskWrite, networkIn, networkOut, now)
if points := monitor.metricsHistory.GetGuestMetrics("unknown", "diskread", time.Hour); len(points) != 0 {
t.Fatalf("unknown disk read rate was written to history: %+v", points)
}
@ -106,7 +106,7 @@ func TestIssue1613HistoryWritesValidZeroAndSkipsUnknownRates(t *testing.T) {
NetworkIn: true,
NetworkOut: true,
})
monitor.recordGuestMetric("vm", "idle", 0, 0, 0, diskRead, diskWrite, networkIn, networkOut, now)
monitor.recordGuestMetric("vm", "idle", 0, 0, 0, 0, diskRead, diskWrite, networkIn, networkOut, now)
points := monitor.metricsHistory.GetGuestMetrics("idle", "diskread", time.Hour)
if len(points) != 1 || points[0].Value != 0 {
t.Fatalf("valid idle disk read rate was not written as zero: %+v", points)

View file

@ -881,6 +881,7 @@ func TestRecordGuestMetric_SkipsNativeWritesInMockMode(t *testing.T) {
"test:node1:101",
42,
48,
1024,
51,
1024,
512,
@ -905,8 +906,8 @@ func TestRecordGuestMetricPreservesHistoryWhenLiveMemoryIsUnavailable(t *testing
defer mon.notificationMgr.Stop()
now := time.Now().UTC()
mon.recordGuestMetric("vm", "test:node1:101", 20, 76, 40, -1, -1, -1, -1, now.Add(-time.Minute))
mon.recordGuestMetric("vm", "test:node1:101", 25, -1, 42, -1, -1, -1, -1, now)
mon.recordGuestMetric("vm", "test:node1:101", 20, 76, 1024, 40, -1, -1, -1, -1, now.Add(-time.Minute))
mon.recordGuestMetric("vm", "test:node1:101", 25, -1, -1, 42, -1, -1, -1, -1, now)
points := mon.metricsHistory.GetGuestMetrics("test:node1:101", "memory", time.Hour)
if len(points) != 1 {
@ -915,6 +916,10 @@ func TestRecordGuestMetricPreservesHistoryWhenLiveMemoryIsUnavailable(t *testing
if points[0].Value != 76 {
t.Fatalf("memory history value = %.2f, want 76", points[0].Value)
}
usedPoints := mon.metricsHistory.GetGuestMetrics("test:node1:101", "memoryused", time.Hour)
if len(usedPoints) != 1 || usedPoints[0].Value != 1024 {
t.Fatalf("memory-used history = %+v, want one trusted 1024-byte point", usedPoints)
}
}
func TestHandleClusterContainerResourceMemoryTrustCharacterization(t *testing.T) {

View file

@ -16,6 +16,7 @@ type MetricPoint = models.MetricPoint
type GuestMetrics struct {
CPU []MetricPoint `json:"cpu"`
Memory []MetricPoint `json:"memory"`
MemoryUsed []MetricPoint `json:"memoryused"`
Disk []MetricPoint `json:"disk"`
GPU []MetricPoint `json:"gpu"`
GPUMemory []MetricPoint `json:"gpu_memory"`
@ -115,6 +116,7 @@ func cloneGuestMetrics(metrics *GuestMetrics) *GuestMetrics {
return &GuestMetrics{
CPU: append([]MetricPoint(nil), metrics.CPU...),
Memory: append([]MetricPoint(nil), metrics.Memory...),
MemoryUsed: append([]MetricPoint(nil), metrics.MemoryUsed...),
Disk: append([]MetricPoint(nil), metrics.Disk...),
GPU: append([]MetricPoint(nil), metrics.GPU...),
GPUMemory: append([]MetricPoint(nil), metrics.GPUMemory...),
@ -157,6 +159,8 @@ func (mh *MetricsHistory) AddGuestMetric(guestID string, metricType string, valu
metrics.CPU = mh.appendMetric(metrics.CPU, point)
case "memory":
metrics.Memory = mh.appendMetric(metrics.Memory, point)
case "memoryused":
metrics.MemoryUsed = mh.appendMetric(metrics.MemoryUsed, point)
case "disk":
metrics.Disk = mh.appendMetric(metrics.Disk, point)
case "gpu":
@ -197,6 +201,8 @@ func (mh *MetricsHistory) AddNodeMetric(nodeID string, metricType string, value
metrics.CPU = mh.appendMetric(metrics.CPU, point)
case "memory":
metrics.Memory = mh.appendMetric(metrics.Memory, point)
case "memoryused":
metrics.MemoryUsed = mh.appendMetric(metrics.MemoryUsed, point)
case "disk":
metrics.Disk = mh.appendMetric(metrics.Disk, point)
case "netin":
@ -291,6 +297,8 @@ func (mh *MetricsHistory) addGuestMetricSeries(guestID, metricType string, value
metrics.CPU = mh.appendMetricSeries(metrics.CPU, values, timestamps)
case "memory":
metrics.Memory = mh.appendMetricSeries(metrics.Memory, values, timestamps)
case "memoryused":
metrics.MemoryUsed = mh.appendMetricSeries(metrics.MemoryUsed, values, timestamps)
case "disk":
metrics.Disk = mh.appendMetricSeries(metrics.Disk, values, timestamps)
case "gpu":
@ -325,6 +333,8 @@ func (mh *MetricsHistory) addNodeMetricSeries(nodeID, metricType string, values
metrics.CPU = mh.appendMetricSeries(metrics.CPU, values, timestamps)
case "memory":
metrics.Memory = mh.appendMetricSeries(metrics.Memory, values, timestamps)
case "memoryused":
metrics.MemoryUsed = mh.appendMetricSeries(metrics.MemoryUsed, values, timestamps)
case "disk":
metrics.Disk = mh.appendMetricSeries(metrics.Disk, values, timestamps)
case "netin":
@ -394,6 +404,8 @@ func (mh *MetricsHistory) GetGuestMetrics(guestID string, metricType string, dur
data = metrics.CPU
case "memory":
data = metrics.Memory
case "memoryused":
data = metrics.MemoryUsed
case "disk":
data = metrics.Disk
case "gpu":
@ -445,6 +457,8 @@ func (mh *MetricsHistory) GetNodeMetrics(nodeID string, metricType string, durat
data = metrics.CPU
case "memory":
data = metrics.Memory
case "memoryused":
data = metrics.MemoryUsed
case "disk":
data = metrics.Disk
case "netin":
@ -509,6 +523,8 @@ func guestMetricSeries(metrics *GuestMetrics, metricType string) []MetricPoint {
return metrics.CPU
case "memory":
return metrics.Memory
case "memoryused":
return metrics.MemoryUsed
case "disk":
return metrics.Disk
case "gpu":
@ -538,7 +554,7 @@ func guestMetricsCoverageSpan(metrics *GuestMetrics, metricTypes []string, cutof
}
if len(metricTypes) == 0 {
metricTypes = []string{"cpu", "memory", "disk", "gpu", "gpu_memory", "gpu_temperature", "diskread", "diskwrite", "netin", "netout", "temperature"}
metricTypes = []string{"cpu", "memory", "memoryused", "disk", "gpu", "gpu_memory", "gpu_temperature", "diskread", "diskwrite", "netin", "netout", "temperature"}
}
var best time.Duration
@ -601,6 +617,7 @@ func (mh *MetricsHistory) GetAllGuestMetrics(guestID string, duration time.Durat
result["cpu"] = filterMetricsByTime(metrics.CPU, cutoffTime)
result["memory"] = filterMetricsByTime(metrics.Memory, cutoffTime)
result["memoryused"] = filterMetricsByTime(metrics.MemoryUsed, cutoffTime)
result["disk"] = filterMetricsByTime(metrics.Disk, cutoffTime)
result["gpu"] = filterMetricsByTime(metrics.GPU, cutoffTime)
result["gpu_memory"] = filterMetricsByTime(metrics.GPUMemory, cutoffTime)

View file

@ -628,6 +628,18 @@ func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.Fi
}
}
}
seedStoreGuestMemoryUsedSeries := func(resourceType, resourceID string, memoryTotal float64) {
if ms == nil || strings.TrimSpace(resourceID) == "" || memoryTotal <= 0 {
return
}
for _, plan := range mockStoreSeedPlans {
coverageKey := metrics.NormalizedSeriesKey(resourceType, resourceID, "memoryused")
for _, ts := range seedStoreGapTimestamps(plan, coverageKey) {
percent := clampFloat(sampler.SampleMetric(resourceType, resourceID, "memory", ts), 0, 100)
queueStorePoint(resourceType, resourceID, "memoryused", memoryTotal*(percent/100), ts, plan.tier)
}
}
}
recordStorageTimeline := func(storageID string, currentTotal float64) {
if strings.TrimSpace(storageID) == "" || currentTotal <= 0 || numPoints == 0 {
return
@ -737,6 +749,22 @@ func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.Fi
}
seedStoreSeries(storeType, storeID, storeMetrics...)
}
recordGuestMemoryUsed := func(metricIDs []string, storeType, storeID string, memoryTotal float64) {
if len(metricIDs) == 0 || strings.TrimSpace(storeID) == "" || memoryTotal <= 0 {
return
}
percentSeries := canonicalMetricSeriesWithSampler(sampler, storeType, storeID, "memory", seedTimestamps)
usedSeries := make([]float64, len(percentSeries))
for i, percent := range percentSeries {
usedSeries[i] = memoryTotal * (clampFloat(percent, 0, 100) / 100)
}
for _, metricID := range metricIDs {
if id := strings.TrimSpace(metricID); id != "" {
mh.addGuestMetricSeries(id, "memoryused", usedSeries, seedTimestamps)
}
}
seedStoreGuestMemoryUsedSeries(storeType, storeID, memoryTotal)
}
log.Debug().Int("count", len(state.Nodes)).Msg("mock seeding: processing nodes")
for _, node := range state.Nodes {
@ -752,6 +780,7 @@ func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.Fi
log.Debug().Int("total", len(state.VMs)).Int("running", runningVMs).Msg("mock seeding: processing VMs (including stopped)")
for _, vm := range state.VMs {
recordGuest([]string{vm.ID}, "vm", vm.ID, true, true, true)
recordGuestMemoryUsed([]string{vm.ID}, "vm", vm.ID, float64(vm.Memory.Total))
}
runningContainers := 0
@ -763,6 +792,7 @@ func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.Fi
log.Debug().Int("total", len(state.Containers)).Int("running", runningContainers).Msg("mock seeding: processing containers (including stopped)")
for _, ct := range state.Containers {
recordGuest([]string{ct.ID}, "container", ct.ID, true, true, true)
recordGuestMemoryUsed([]string{ct.ID}, "container", ct.ID, float64(ct.Memory.Total))
}
k8sNodeCount := 0
@ -1306,17 +1336,20 @@ func recordVMwareFixturesMetrics(mh *MetricsHistory, ms *metrics.Store, sampler
type guestMetricSource interface {
GetID() string
GetStatus() string
GetMemoryTotal() int64
}
type vmAdapter struct{ *models.VM }
func (v vmAdapter) GetID() string { return v.VM.ID }
func (v vmAdapter) GetStatus() string { return v.VM.Status }
func (v vmAdapter) GetID() string { return v.VM.ID }
func (v vmAdapter) GetStatus() string { return v.VM.Status }
func (v vmAdapter) GetMemoryTotal() int64 { return v.VM.Memory.Total }
type containerAdapter struct{ *models.Container }
func (c containerAdapter) GetID() string { return c.Container.ID }
func (c containerAdapter) GetStatus() string { return c.Container.Status }
func (c containerAdapter) GetID() string { return c.Container.ID }
func (c containerAdapter) GetStatus() string { return c.Container.Status }
func (c containerAdapter) GetMemoryTotal() int64 { return c.Container.Memory.Total }
func recordGuestMetrics[T guestMetricSource](mh *MetricsHistory, ms *metrics.Store, sampler mock.MetricSampler, guests []T, prefix string, ts time.Time) {
for _, guest := range guests {
@ -1327,6 +1360,7 @@ func recordGuestMetrics[T guestMetricSource](mh *MetricsHistory, ms *metrics.Sto
id := guest.GetID()
cpu := sampler.SampleMetric(prefix, id, "cpu", ts)
memory := sampler.SampleMetric(prefix, id, "memory", ts)
memoryUsed := float64(guest.GetMemoryTotal()) * (clampFloat(memory, 0, 100) / 100)
disk := sampler.SampleMetric(prefix, id, "disk", ts)
diskread := sampler.SampleMetric(prefix, id, "diskread", ts)
diskwrite := sampler.SampleMetric(prefix, id, "diskwrite", ts)
@ -1335,6 +1369,9 @@ func recordGuestMetrics[T guestMetricSource](mh *MetricsHistory, ms *metrics.Sto
mh.AddGuestMetric(id, "cpu", cpu, ts)
mh.AddGuestMetric(id, "memory", memory, ts)
if guest.GetMemoryTotal() > 0 {
mh.AddGuestMetric(id, "memoryused", memoryUsed, ts)
}
mh.AddGuestMetric(id, "disk", disk, ts)
mh.AddGuestMetric(id, "diskread", diskread, ts)
mh.AddGuestMetric(id, "diskwrite", diskwrite, ts)
@ -1344,6 +1381,9 @@ func recordGuestMetrics[T guestMetricSource](mh *MetricsHistory, ms *metrics.Sto
if ms != nil {
ms.Write(prefix, id, "cpu", cpu, ts)
ms.Write(prefix, id, "memory", memory, ts)
if guest.GetMemoryTotal() > 0 {
ms.Write(prefix, id, "memoryused", memoryUsed, ts)
}
ms.Write(prefix, id, "disk", disk, ts)
ms.Write(prefix, id, "diskread", diskread, ts)
ms.Write(prefix, id, "diskwrite", diskwrite, ts)

View file

@ -391,6 +391,7 @@ func (m *Monitor) collectContainersWithNodes(ctx context.Context, instanceName s
ct.ID,
unifiedresources.ProxmoxGuestCPUPercent(ct.CPU),
historyMemoryUsage(ct.Memory),
historyMemoryUsed(ct.Memory),
ct.Disk.Usage,
diskRead,
diskWrite,

View file

@ -141,6 +141,7 @@ func (m *Monitor) collectVMsWithNodes(ctx context.Context, instanceName string,
vm.ID,
unifiedresources.ProxmoxGuestCPUPercent(vm.CPU),
historyMemoryUsage(vm.Memory),
historyMemoryUsed(vm.Memory),
vm.Disk.Usage,
diskRead,
diskWrite,

View file

@ -134,7 +134,7 @@ func (m *Monitor) recordGuestMetrics(allVMs []models.VM, allContainers []models.
}
if vm.Status == "running" {
diskRead, diskWrite, networkIn, networkOut := guestHistoryRates(vm.DiskRead, vm.DiskWrite, vm.NetworkIn, vm.NetworkOut, vm.IORateValidity)
m.recordGuestMetric("vm", vm.ID, unifiedresources.ProxmoxGuestCPUPercent(vm.CPU), historyMemoryUsage(vm.Memory), vm.Disk.Usage, diskRead, diskWrite, networkIn, networkOut, now)
m.recordGuestMetric("vm", vm.ID, unifiedresources.ProxmoxGuestCPUPercent(vm.CPU), historyMemoryUsage(vm.Memory), historyMemoryUsed(vm.Memory), vm.Disk.Usage, diskRead, diskWrite, networkIn, networkOut, now)
}
}
for _, ct := range allContainers {
@ -143,7 +143,7 @@ func (m *Monitor) recordGuestMetrics(allVMs []models.VM, allContainers []models.
}
if ct.Status == "running" {
diskRead, diskWrite, networkIn, networkOut := guestHistoryRates(ct.DiskRead, ct.DiskWrite, ct.NetworkIn, ct.NetworkOut, ct.IORateValidity)
m.recordGuestMetric("container", ct.ID, unifiedresources.ProxmoxGuestCPUPercent(ct.CPU), historyMemoryUsage(ct.Memory), ct.Disk.Usage, diskRead, diskWrite, networkIn, networkOut, now)
m.recordGuestMetric("container", ct.ID, unifiedresources.ProxmoxGuestCPUPercent(ct.CPU), historyMemoryUsage(ct.Memory), historyMemoryUsed(ct.Memory), ct.Disk.Usage, diskRead, diskWrite, networkIn, networkOut, now)
}
}
}
@ -172,11 +172,18 @@ func historyMemoryUsage(memory models.Memory) float64 {
return memory.Usage
}
func historyMemoryUsed(memory models.Memory) float64 {
if !memory.HasKnownUsage() {
return -1
}
return float64(memory.Used)
}
// recordGuestMetric records metrics for a single guest (VM or container) to both
// the in-memory metrics history and the persistent metrics store.
func (m *Monitor) recordGuestMetric(
resourceType, resourceID string,
cpu, memory, diskUsage float64,
cpu, memory, memoryUsed, diskUsage float64,
diskRead, diskWrite, networkIn, networkOut int64,
now time.Time,
) {
@ -189,6 +196,9 @@ func (m *Monitor) recordGuestMetric(
if memory >= 0 {
m.metricsHistory.AddGuestMetric(resourceID, "memory", memory, now)
}
if memoryUsed >= 0 {
m.metricsHistory.AddGuestMetric(resourceID, "memoryused", memoryUsed, now)
}
if diskUsage >= 0 {
m.metricsHistory.AddGuestMetric(resourceID, "disk", diskUsage, now)
}
@ -211,6 +221,9 @@ func (m *Monitor) recordGuestMetric(
if memory >= 0 {
m.metricsStore.Write(resourceType, resourceID, "memory", memory, now)
}
if memoryUsed >= 0 {
m.metricsStore.Write(resourceType, resourceID, "memoryused", memoryUsed, now)
}
if diskUsage >= 0 {
m.metricsStore.Write(resourceType, resourceID, "disk", diskUsage, now)
}