From 6c82a3c47c52ebdd781d61389a43d258f2b05deb Mon Sep 17 00:00:00 2001 From: "courtmanr@gmail.com" Date: Thu, 30 Jul 2026 16:58:52 +0100 Subject: [PATCH] feat(monitoring): promote GPU load to first-class metrics --- .../v6/internal/subsystems/agent-lifecycle.md | 9 ++- .../subsystems/frontend-primitives.md | 18 +++++ .../v6/internal/subsystems/monitoring.md | 12 ++- .../internal/subsystems/unified-resources.md | 10 ++- .../Infrastructure/ResourceDetailDrawer.tsx | 1 + ...rMetricsHistoryModel.branchcov0712.test.ts | 44 +++++++++++ ...resourceDetailDrawerMetricsHistoryModel.ts | 71 ++++++++++++++++++ .../useResourceDetailDrawerDerivedState.ts | 3 + .../standalone/AgentsMachinesTable.tsx | 27 ++++++- .../__tests__/AgentsMachinesTable.test.tsx | 28 +++++++ .../__tests__/agentMachineTableModel.test.ts | 40 ++++++++++ .../standalone/agentMachineTableModel.ts | 59 ++++++++++++++- internal/monitoring/metrics_history.go | 71 +++++++++++++----- internal/monitoring/metrics_history_test.go | 74 ++++++++++++++++--- internal/monitoring/monitor_agents.go | 53 +++++++++++++ .../monitoring/monitor_host_agents_test.go | 28 +++++++ 16 files changed, 509 insertions(+), 39 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index 1f2da0cff4..8a4bc660c9 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -1493,9 +1493,12 @@ the intentionally sparse public response. query as a best-effort fallback when `lm-sensors` is unavailable. The report must keep typed GPU readings in `sensors.gpu` while mapping only direct `temperature.gpu` readings into existing `sensors.temperatureCelsius` - `gpu_nvidia_` keys for compatibility; it must not infer lifecycle - health, command authority, enrollment state, or GPU workload/process - inventory from `nvidia-smi` output. + `gpu_nvidia_` keys for compatibility. Authenticated server ingest may + reduce those typed samples into bounded host-level GPU utilization, VRAM + pressure, and GPU temperature history on the existing agent identity, but + that monitoring projection does not change the report or command protocol. + Neither side may infer lifecycle health, command authority, enrollment + state, or GPU workload/process inventory from `nvidia-smi` output. Runtime RAID collection uses `/proc/mdstat` as the canonical discovery baseline for Linux md arrays. `mdadm --detail` may enrich level, state, member, UUID, and rebuild fields when available, but missing or failing diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index 262de7bc69..443bb1bf37 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -5530,3 +5530,21 @@ renders the canonical dataset name and formatted used, available, referenced, and mountpoint values. Empty dataset collections add no new panel, route, or navigation state. The presentation mapper owns byte formatting and missing mountpoint fallback so components do not reinterpret provider data. + +### Host GPU telemetry reuses shared metric and history primitives + +The standalone machine table may expose typed host GPU utilization as a +toggleable, sortable `metric-bar` column, but it must render through the shared +`MetricBar` primitive and use the unified resource's existing metric key. The +row model owns validation, maximum-per-host selection, and the per-device +inspection title; the component must not invent vendor-specific probes or a +parallel GPU table. + +The shared resource drawer keeps `GuestDrawerHistory` as the sole history +renderer. Resources with typed GPU sensors may extend its normal groups with +core utilization, VRAM pressure, and GPU temperature series and provide current +typed readings as initial fallbacks. Resources without GPU sensors retain the +default history groups unchanged. Rendered table proof belongs in +`frontend-modern/src/features/standalone/__tests__/AgentsMachinesTable.test.tsx`; +drawer grouping and fallback proof belongs in +`frontend-modern/src/components/Infrastructure/__tests__/resourceDetailDrawerMetricsHistoryModel.branchcov0712.test.ts`. diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index b21fe3eed9..a3fbbd63bb 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -1270,9 +1270,15 @@ exists. Host-agent GPU sensor summaries follow that same descriptive-host-telemetry path. Monitoring must preserve typed GPU id, name, temperature, utilization, and VRAM readings from agent reports through models, read-state projection, and -frontend conversion, while still using only real Celsius readings for -`agent.temperature` or `metric=temperature` and without promoting GPU workload -or process inventory into monitoring state. +frontend conversion. When typed readings are present, authenticated host-report +ingest records bounded maximum-per-host `gpu`, `gpu_memory`, and +`gpu_temperature` samples in both in-memory guest history and the persisted +`agent` metrics-store stream. Utilization and VRAM pressure are percentages; +VRAM pressure is derived only from non-negative used bytes and a positive total, +and temperatures must be positive Celsius values no greater than 150. Missing +or invalid fields omit only their own series. The aggregate remains attached to +the existing agent resource and does not promote GPU workload or process +inventory into monitoring state. Host-agent power sensor summaries follow the same descriptive-host-telemetry path. Monitoring must preserve `sensors.powerWatts` readings from agent reports through models, read-state projection, and frontend conversion without diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index ab03d1609b..3718dd95ea 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -464,9 +464,13 @@ history target. Typed GPU host telemetry follows the same canonical sensor route. `HostSensorMeta.gpu` may carry GPU id, name, temperature, utilization, and VRAM readings through clone, merge, transport, and frontend decode paths, but those -readings remain descriptive host context. Unified-resource consumers must not -promote GPU sensors into separate hardware resources, lifecycle state, workload -identity, or history targets beyond direct numeric temperature compatibility. +readings remain descriptive host context. Standalone machine consumers may +present the maximum valid utilization as a sortable shared metric-bar column, +include per-device utilization, VRAM, and temperature in inspection text, and +add `gpu`, `gpu_memory`, and `gpu_temperature` series to the existing agent +drawer history target. Those projections must stay on the host resource; +consumers must not promote GPU sensors into separate hardware resources, +lifecycle state, workload identity, or GPU-specific history identities. Host-agent wattage follows that same canonical sensor route. `HostSensorMeta.powerWatts` may carry named power readings through clone, merge, transport, and frontend decode paths, but those readings remain diff --git a/frontend-modern/src/components/Infrastructure/ResourceDetailDrawer.tsx b/frontend-modern/src/components/Infrastructure/ResourceDetailDrawer.tsx index 2731e0e47f..c3dcc3dbee 100644 --- a/frontend-modern/src/components/Infrastructure/ResourceDetailDrawer.tsx +++ b/frontend-modern/src/components/Infrastructure/ResourceDetailDrawer.tsx @@ -196,6 +196,7 @@ const DrawerContent: Component = (props) => { target={target()} range={drawer.metricsHistoryRange()} fallbackMetrics={drawer.metricsHistoryFallbackMetrics()} + groups={drawer.metricsHistoryGroups()} /> )} diff --git a/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailDrawerMetricsHistoryModel.branchcov0712.test.ts b/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailDrawerMetricsHistoryModel.branchcov0712.test.ts index 2709d00951..105c344aee 100644 --- a/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailDrawerMetricsHistoryModel.branchcov0712.test.ts +++ b/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailDrawerMetricsHistoryModel.branchcov0712.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest'; import { getResourceMetricsHistoryFallbackMetrics, + getResourceMetricsHistoryGroups, getResourceMetricsHistoryTarget, } from '@/components/Infrastructure/resourceDetailDrawerMetricsHistoryModel'; import type { MetricsHistoryTargetResourceType, Resource } from '@/types/resource'; @@ -128,6 +129,9 @@ describe('getResourceMetricsHistoryFallbackMetrics branch coverage', () => { netout: 2000, diskread: 3000, diskwrite: 4000, + gpu: undefined, + gpu_memory: undefined, + gpu_temperature: undefined, }); }); @@ -143,6 +147,9 @@ describe('getResourceMetricsHistoryFallbackMetrics branch coverage', () => { netout: undefined, diskread: undefined, diskwrite: undefined, + gpu: undefined, + gpu_memory: undefined, + gpu_temperature: undefined, }); }); @@ -207,9 +214,46 @@ describe('getResourceMetricsHistoryFallbackMetrics branch coverage', () => { netout: undefined, diskread: undefined, diskwrite: undefined, + gpu: undefined, + gpu_memory: undefined, + gpu_temperature: undefined, }); }); + it('returns bounded aggregate GPU fallbacks and GPU history groups', () => { + const resource = baseResource({ + agent: { + sensors: { + gpu: [ + { + id: '0', + name: 'NVIDIA A6000', + utilizationPercent: 80, + memoryUsedBytes: 24, + memoryTotalBytes: 48, + temperatureCelsius: 70, + }, + { + id: '1', + utilizationPercent: 101, + memoryUsedBytes: 7, + memoryTotalBytes: 8, + temperatureCelsius: 151, + }, + ], + }, + }, + }); + + const metrics = getResourceMetricsHistoryFallbackMetrics(resource); + expect(metrics.gpu).toBe(80); + expect(metrics.gpu_memory).toBe(87.5); + expect(metrics.gpu_temperature).toBe(70); + expect(getResourceMetricsHistoryGroups(resource).map((group) => group.id)).toEqual( + expect.arrayContaining(['gpu-utilization', 'gpu-thermal']), + ); + }); + it('emits the memory ternary FALSE arm as undefined when memory is explicitly absent but disk is present', () => { // Mixes a present disk (TRUE arm) with an absent memory (FALSE arm) to prove // the two ternaries are evaluated independently. diff --git a/frontend-modern/src/components/Infrastructure/resourceDetailDrawerMetricsHistoryModel.ts b/frontend-modern/src/components/Infrastructure/resourceDetailDrawerMetricsHistoryModel.ts index 7291771371..3dd204daf8 100644 --- a/frontend-modern/src/components/Infrastructure/resourceDetailDrawerMetricsHistoryModel.ts +++ b/frontend-modern/src/components/Infrastructure/resourceDetailDrawerMetricsHistoryModel.ts @@ -1,5 +1,10 @@ import type { ResourceType as MetricsHistoryResourceType } from '@/api/charts'; import type { GuestDrawerHistoryTarget } from '@/components/Workloads/guestDrawerModel'; +import { + GUEST_DRAWER_HISTORY_GROUPS, + type GuestDrawerHistoryGroupConfig, +} from '@/components/Workloads/guestDrawerModel'; +import type { HostGPUSensor } from '@/types/api'; import type { Resource } from '@/types/resource'; import { getDiskPercent, getMemoryPercent } from '@/types/resource'; import { asTrimmedString } from '@/utils/stringUtils'; @@ -7,6 +12,69 @@ import { asTrimmedString } from '@/utils/stringUtils'; const finiteMetric = (value: number | undefined): number | undefined => typeof value === 'number' && Number.isFinite(value) ? value : undefined; +const maxFiniteGPUValue = ( + resource: Resource, + select: (gpu: HostGPUSensor) => number | undefined, +): number | undefined => { + let maximum: number | undefined; + for (const gpu of resource.agent?.sensors?.gpu ?? []) { + const value = finiteMetric(select(gpu)); + if (value === undefined) continue; + maximum = maximum === undefined ? value : Math.max(maximum, value); + } + return maximum; +}; + +export const getResourceGPUUtilizationPercent = (resource: Resource): number | undefined => + maxFiniteGPUValue(resource, (gpu) => { + const utilization = finiteMetric(gpu.utilizationPercent); + return utilization !== undefined && utilization >= 0 && utilization <= 100 + ? utilization + : undefined; + }); + +export const getResourceGPUMemoryPercent = (resource: Resource): number | undefined => + maxFiniteGPUValue(resource, (gpu) => { + const used = finiteMetric(gpu.memoryUsedBytes); + const total = finiteMetric(gpu.memoryTotalBytes); + if (used === undefined || total === undefined || used < 0 || total <= 0) return undefined; + return Math.min(100, Math.max(0, (used / total) * 100)); + }); + +export const getResourceGPUTemperatureCelsius = (resource: Resource): number | undefined => + maxFiniteGPUValue(resource, (gpu) => { + const temperature = finiteMetric(gpu.temperatureCelsius); + return temperature !== undefined && temperature > 0 && temperature <= 150 + ? temperature + : undefined; + }); + +const GPU_HISTORY_GROUPS: GuestDrawerHistoryGroupConfig[] = [ + ...GUEST_DRAWER_HISTORY_GROUPS, + { + id: 'gpu-utilization', + label: 'GPU Utilization', + unit: '%', + series: [ + { metric: 'gpu', label: 'Core', unit: '%', color: '#06b6d4' }, + { metric: 'gpu_memory', label: 'VRAM', unit: '%', color: '#6366f1' }, + ], + }, + { + id: 'gpu-thermal', + label: 'GPU Thermal', + unit: 'C', + series: [{ metric: 'gpu_temperature', label: 'GPU', unit: 'C', color: '#ef4444' }], + }, +]; + +export const getResourceMetricsHistoryGroups = ( + resource: Resource, +): GuestDrawerHistoryGroupConfig[] => + (resource.agent?.sensors?.gpu?.length ?? 0) > 0 + ? GPU_HISTORY_GROUPS + : GUEST_DRAWER_HISTORY_GROUPS; + export const getResourceMetricsHistoryTarget = ( resource: Resource, ): GuestDrawerHistoryTarget | null => { @@ -46,5 +114,8 @@ export const getResourceMetricsHistoryFallbackMetrics = ( netout: finiteMetric(resource.network?.txBytes), diskread: finiteMetric(resource.diskIO?.readRate), diskwrite: finiteMetric(resource.diskIO?.writeRate), + gpu: getResourceGPUUtilizationPercent(resource), + gpu_memory: getResourceGPUMemoryPercent(resource), + gpu_temperature: getResourceGPUTemperatureCelsius(resource), }; }; diff --git a/frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDerivedState.ts b/frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDerivedState.ts index 75a5b7e3f7..eab0ad17b6 100644 --- a/frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDerivedState.ts +++ b/frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDerivedState.ts @@ -61,6 +61,7 @@ import { } from './resourceDetailDrawerOperationalModel'; import { getResourceMetricsHistoryFallbackMetrics, + getResourceMetricsHistoryGroups, getResourceMetricsHistoryTarget, resourceSupportsMetricsHistory, } from './resourceDetailDrawerMetricsHistoryModel'; @@ -344,6 +345,7 @@ export const useResourceDetailDrawerDerivedState = ( const metricsHistoryFallbackMetrics = createMemo(() => getResourceMetricsHistoryFallbackMetrics(resource), ); + const metricsHistoryGroups = createMemo(() => getResourceMetricsHistoryGroups(resource)); const hasMetricsHistory = createMemo(() => resourceSupportsMetricsHistory(resource)); const hasDiscoveryTab = createMemo( () => isPulseAgentPlatformResource(resource) && Boolean(discoveryConfig()), @@ -474,6 +476,7 @@ export const useResourceDetailDrawerDerivedState = ( serviceDetailsSummary, metricsHistoryTarget, metricsHistoryFallbackMetrics, + metricsHistoryGroups, hasMetricsHistory, hasDiscoveryTab, hasVMwareDetails, diff --git a/frontend-modern/src/features/standalone/AgentsMachinesTable.tsx b/frontend-modern/src/features/standalone/AgentsMachinesTable.tsx index b6e6237a44..1e73f4dd1a 100644 --- a/frontend-modern/src/features/standalone/AgentsMachinesTable.tsx +++ b/frontend-modern/src/features/standalone/AgentsMachinesTable.tsx @@ -15,6 +15,7 @@ import { Portal } from 'solid-js/web'; import { AgentMetadataAPI, type AgentMetadata } from '@/api/agentMetadata'; import { MonitoringAPI } from '@/api/monitoring'; import { EnhancedCPUBar } from '@/components/Workloads/EnhancedCPUBar'; +import { MetricBar } from '@/components/Workloads/MetricBar'; import { StackedDiskBar } from '@/components/Workloads/StackedDiskBar'; import { StackedMemoryBar } from '@/components/Workloads/StackedMemoryBar'; import { ActionIconButton } from '@/components/shared/Button'; @@ -78,6 +79,8 @@ import { getWorkloadsGuestNetworkEmptyState } from '@/utils/workloadGuestPresent import { AGENT_MACHINE_COLUMNS, getAgentMachineCpuPercent, + getAgentMachineGPUTitle, + getAgentMachineGPUUtilizationPercent, getAgentMachineDiskIODetails, getAgentMachineDiskIOTotal, getAgentMachineIpValues, @@ -1057,6 +1060,8 @@ const machineColumnWidthClass = (columnId: AgentMachineColumnId): string => { case 'memory': case 'disk': return 'w-[20%] md:w-[8%]'; + case 'gpu': + return 'hidden md:table-cell md:w-[8%]'; case 'network': case 'diskio': return 'hidden lg:table-cell lg:w-[9%]'; @@ -1179,7 +1184,7 @@ export const AgentsMachinesTable: Component<{ const [sortKey, setSortKey] = createSignal('name'); const [sortDirection, setSortDirection] = createSignal<'asc' | 'desc'>('asc'); const columnVisibility = useColumnVisibility( - 'pulse:standalone:machines:columns:v3', + 'pulse:standalone:machines:columns:v4', AGENT_MACHINE_COLUMNS, [], undefined, @@ -1468,6 +1473,8 @@ export const AgentsMachinesTable: Component<{ const diskThresholds = () => alertsActivation.getMetricThresholds('agent', 'disk', alertResourceIds()); const cpuPercent = () => getAgentMachineCpuPercent(machine); + const gpuUtilization = () => getAgentMachineGPUUtilizationPercent(machine); + const gpuTitle = () => getAgentMachineGPUTitle(machine); const cpuCores = () => cpuCoresFor(machine); const cpuLoadAverage = () => cpuLoadAverageFor(machine); const memoryUsed = () => memoryUsedFor(machine); @@ -1646,6 +1653,24 @@ export const AgentsMachinesTable: Component<{ /> + + + + + + + diff --git a/frontend-modern/src/features/standalone/__tests__/AgentsMachinesTable.test.tsx b/frontend-modern/src/features/standalone/__tests__/AgentsMachinesTable.test.tsx index cbabd55c44..852289fde5 100644 --- a/frontend-modern/src/features/standalone/__tests__/AgentsMachinesTable.test.tsx +++ b/frontend-modern/src/features/standalone/__tests__/AgentsMachinesTable.test.tsx @@ -59,6 +59,17 @@ vi.mock('@/components/Workloads/EnhancedCPUBar', () => ({ ), })); +vi.mock('@/components/Workloads/MetricBar', () => ({ + MetricBar: (props: { value: number; label: string; resourceId?: string }) => ( +
+ ), +})); + vi.mock('@/components/Workloads/StackedMemoryBar', () => ({ StackedMemoryBar: (props: { used: number; @@ -195,6 +206,18 @@ describe('AgentsMachinesTable', () => { rebuildPercent: 0, }, ], + sensors: { + gpu: [ + { + id: '0', + name: 'NVIDIA RTX A6000', + utilizationPercent: 67, + memoryUsedBytes: 12 * 1024 ** 3, + memoryTotalBytes: 48 * 1024 ** 3, + temperatureCelsius: 71, + }, + ], + }, }, }), ]} @@ -212,8 +235,13 @@ describe('AgentsMachinesTable', () => { expect(screen.getByRole('button', { name: 'Sort by Net I/O' })).toBeInTheDocument(); expect(screen.getByRole('button', { name: 'Sort by Disk I/O' })).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'Sort by GPU' })).toBeInTheDocument(); expect(screen.getByText('1.00 KB/s')).toBeInTheDocument(); expect(screen.getByText('8.00 KB/s')).toBeInTheDocument(); + expect(screen.getByTestId('agent-machine-gpu-bar')).toHaveAttribute('data-value', '67'); + expect( + screen.getByTitle('NVIDIA RTX A6000: 67% load, 12.0 GB / 48.0 GB VRAM, 71 °C'), + ).toBeInTheDocument(); expect(screen.queryByRole('button', { name: 'Sort by IP' })).not.toBeInTheDocument(); expect(screen.getByText('192.168.0.10')).toBeInTheDocument(); diff --git a/frontend-modern/src/features/standalone/__tests__/agentMachineTableModel.test.ts b/frontend-modern/src/features/standalone/__tests__/agentMachineTableModel.test.ts index c4b393e528..4ae78a3fb7 100644 --- a/frontend-modern/src/features/standalone/__tests__/agentMachineTableModel.test.ts +++ b/frontend-modern/src/features/standalone/__tests__/agentMachineTableModel.test.ts @@ -3,6 +3,8 @@ import type { Resource } from '@/types/resource'; import { getAgentMachineDiskPercent, getAgentMachineDiskIODetails, + getAgentMachineGPUTitle, + getAgentMachineGPUUtilizationPercent, getAgentMachineNetworkInterfaceDetails, getAgentMachineRaidArrayDetails, getAgentMachineTemperatureCelsius, @@ -27,6 +29,44 @@ const resource = (overrides: Partial): Resource => }) as Resource; describe('agentMachineTableModel', () => { + it('summarizes and sorts typed GPU telemetry', () => { + const gpuMachine = resource({ + id: 'gpu-host', + agent: { + sensors: { + gpu: [ + { + id: '0', + name: 'NVIDIA RTX A6000', + utilizationPercent: 7, + memoryUsedBytes: 2 * 1024 ** 3, + memoryTotalBytes: 48 * 1024 ** 3, + temperatureCelsius: 63, + }, + { id: '1', utilizationPercent: 81 }, + { id: 'invalid', utilizationPercent: 101 }, + ], + }, + }, + }); + const cpuOnlyMachine = resource({ id: 'cpu-only' }); + + expect(getAgentMachineGPUUtilizationPercent(gpuMachine)).toBe(81); + expect(getAgentMachineGPUUtilizationPercent(cpuOnlyMachine)).toBeUndefined(); + expect(getAgentMachineGPUTitle(gpuMachine)).toContain( + 'NVIDIA RTX A6000: 7% load, 2.00 GB / 48.0 GB VRAM, 63 °C', + ); + expect( + sortAgentMachines( + [cpuOnlyMachine, gpuMachine], + 'gpu', + 'desc', + () => '', + () => '', + ).map((machine) => machine.id), + ).toEqual(['gpu-host', 'cpu-only']); + }); + it('normalizes agent disk I/O device counters for table inspection', () => { const details = getAgentMachineDiskIODetails( resource({ diff --git a/frontend-modern/src/features/standalone/agentMachineTableModel.ts b/frontend-modern/src/features/standalone/agentMachineTableModel.ts index 8e42d60cf4..1ba11593c6 100644 --- a/frontend-modern/src/features/standalone/agentMachineTableModel.ts +++ b/frontend-modern/src/features/standalone/agentMachineTableModel.ts @@ -2,7 +2,7 @@ import type { ColumnDef } from '@/hooks/useColumnVisibility'; import type { HostDiskIO, HostRAIDArray, HostRAIDDevice } from '@/types/api'; import type { Resource } from '@/types/resource'; import { getPlatformTableFiniteMetric } from '@/features/platformPage/sharedPlatformPage'; -import { normalizeDiskArray } from '@/utils/format'; +import { formatBytes, normalizeDiskArray } from '@/utils/format'; import { asTrimmedString } from '@/utils/stringUtils'; export type AgentMachineColumnId = @@ -10,6 +10,7 @@ export type AgentMachineColumnId = | 'system' | 'agent' | 'cpu' + | 'gpu' | 'memory' | 'disk' | 'network' @@ -28,6 +29,7 @@ export type AgentMachineSortKey = | 'system' | 'agent' | 'cpu' + | 'gpu' | 'memory' | 'disk' | 'network' @@ -52,6 +54,7 @@ export const AGENT_MACHINE_COLUMNS: AgentMachineColumn[] = [ { id: 'system', label: 'System', kind: 'text', sortKey: 'system', toggleable: true }, { id: 'agent', label: 'Agent', kind: 'text', sortKey: 'agent', toggleable: true }, { id: 'cpu', label: 'CPU', kind: 'metric-bar', sortKey: 'cpu' }, + { id: 'gpu', label: 'GPU', kind: 'metric-bar', sortKey: 'gpu', toggleable: true }, { id: 'memory', label: 'Memory', kind: 'metric-bar', sortKey: 'memory' }, { id: 'disk', label: 'Disk', kind: 'metric-bar', sortKey: 'disk' }, { @@ -134,6 +137,7 @@ export const AGENT_MACHINE_COLUMNS: AgentMachineColumn[] = [ const AGENT_MACHINE_SORT_DESC_DEFAULTS = new Set([ 'cpu', + 'gpu', 'memory', 'disk', 'network', @@ -308,6 +312,44 @@ const getMetricPercent = (metric: Resource['cpu'] | undefined): number | undefin export const getAgentMachineCpuPercent = (machine: Resource): number | undefined => getMetricPercent(machine.cpu); +export const getAgentMachineGPUUtilizationPercent = (machine: Resource): number | undefined => { + const utilization = (machine.agent?.sensors?.gpu ?? []) + .map((gpu) => getPlatformTableFiniteMetric(gpu.utilizationPercent)) + .filter((value): value is number => value !== undefined && value >= 0 && value <= 100); + + return utilization.length > 0 ? Math.max(...utilization) : undefined; +}; + +export const getAgentMachineGPUTitle = (machine: Resource): string => { + const rows = (machine.agent?.sensors?.gpu ?? []).flatMap((gpu, index) => { + const name = asTrimmedString(gpu.name) ?? asTrimmedString(gpu.id) ?? `GPU ${index}`; + const utilization = getPlatformTableFiniteMetric(gpu.utilizationPercent); + const memoryUsed = getPlatformTableFiniteMetric(gpu.memoryUsedBytes); + const memoryTotal = getPlatformTableFiniteMetric(gpu.memoryTotalBytes); + const temperature = getPlatformTableFiniteMetric(gpu.temperatureCelsius); + const details: string[] = []; + + if (utilization !== undefined && utilization >= 0 && utilization <= 100) { + details.push(`${Math.round(utilization)}% load`); + } + if ( + memoryUsed !== undefined && + memoryUsed >= 0 && + memoryTotal !== undefined && + memoryTotal > 0 + ) { + details.push(`${formatBytes(memoryUsed)} / ${formatBytes(memoryTotal)} VRAM`); + } + if (temperature !== undefined && temperature > 0 && temperature <= 150) { + details.push(`${Math.round(temperature)} °C`); + } + + return details.length > 0 ? [`${name}: ${details.join(', ')}`] : []; + }); + + return rows.join('\n'); +}; + export const getAgentMachineMemoryPercent = (machine: Resource): number | undefined => { if (!machine.memory && machine.agent?.memory?.usageUnavailable === true) { return undefined; @@ -390,6 +432,14 @@ const sensorSearchValues = (machine: Resource): unknown[] => [ ...Object.keys(machine.agent?.sensors?.temperatureCelsius ?? {}), ...Object.keys(machine.agent?.sensors?.fanRpm ?? {}), ...Object.keys(machine.agent?.sensors?.additional ?? {}), + ...(machine.agent?.sensors?.gpu ?? []).flatMap((gpu) => [ + gpu.id, + gpu.name, + gpu.temperatureCelsius, + gpu.utilizationPercent, + gpu.memoryUsedBytes, + gpu.memoryTotalBytes, + ]), ...(machine.agent?.sensors?.smart ?? []).flatMap((disk) => [ disk.device, disk.model, @@ -841,6 +891,13 @@ export const sortAgentMachines = ( direction, ); break; + case 'gpu': + result = compareNullableNumber( + getAgentMachineGPUUtilizationPercent(left), + getAgentMachineGPUUtilizationPercent(right), + direction, + ); + break; case 'memory': result = compareNullableNumber( getAgentMachineMemoryPercent(left), diff --git a/internal/monitoring/metrics_history.go b/internal/monitoring/metrics_history.go index fe23616e92..08f688c372 100644 --- a/internal/monitoring/metrics_history.go +++ b/internal/monitoring/metrics_history.go @@ -14,14 +14,17 @@ type MetricPoint = models.MetricPoint // GuestMetrics holds historical metrics for a single guest type GuestMetrics struct { - CPU []MetricPoint `json:"cpu"` - Memory []MetricPoint `json:"memory"` - Disk []MetricPoint `json:"disk"` - DiskRead []MetricPoint `json:"diskread"` - DiskWrite []MetricPoint `json:"diskwrite"` - NetworkIn []MetricPoint `json:"netin"` - NetworkOut []MetricPoint `json:"netout"` - Temperature []MetricPoint `json:"temperature"` + CPU []MetricPoint `json:"cpu"` + Memory []MetricPoint `json:"memory"` + Disk []MetricPoint `json:"disk"` + GPU []MetricPoint `json:"gpu"` + GPUMemory []MetricPoint `json:"gpu_memory"` + GPUTemperature []MetricPoint `json:"gpu_temperature"` + DiskRead []MetricPoint `json:"diskread"` + DiskWrite []MetricPoint `json:"diskwrite"` + NetworkIn []MetricPoint `json:"netin"` + NetworkOut []MetricPoint `json:"netout"` + Temperature []MetricPoint `json:"temperature"` } // StorageMetrics holds historical metrics for a single storage @@ -110,14 +113,17 @@ func cloneGuestMetrics(metrics *GuestMetrics) *GuestMetrics { return nil } return &GuestMetrics{ - CPU: append([]MetricPoint(nil), metrics.CPU...), - Memory: append([]MetricPoint(nil), metrics.Memory...), - Disk: append([]MetricPoint(nil), metrics.Disk...), - DiskRead: append([]MetricPoint(nil), metrics.DiskRead...), - DiskWrite: append([]MetricPoint(nil), metrics.DiskWrite...), - NetworkIn: append([]MetricPoint(nil), metrics.NetworkIn...), - NetworkOut: append([]MetricPoint(nil), metrics.NetworkOut...), - Temperature: append([]MetricPoint(nil), metrics.Temperature...), + CPU: append([]MetricPoint(nil), metrics.CPU...), + Memory: append([]MetricPoint(nil), metrics.Memory...), + Disk: append([]MetricPoint(nil), metrics.Disk...), + GPU: append([]MetricPoint(nil), metrics.GPU...), + GPUMemory: append([]MetricPoint(nil), metrics.GPUMemory...), + GPUTemperature: append([]MetricPoint(nil), metrics.GPUTemperature...), + DiskRead: append([]MetricPoint(nil), metrics.DiskRead...), + DiskWrite: append([]MetricPoint(nil), metrics.DiskWrite...), + NetworkIn: append([]MetricPoint(nil), metrics.NetworkIn...), + NetworkOut: append([]MetricPoint(nil), metrics.NetworkOut...), + Temperature: append([]MetricPoint(nil), metrics.Temperature...), } } @@ -153,6 +159,12 @@ func (mh *MetricsHistory) AddGuestMetric(guestID string, metricType string, valu metrics.Memory = mh.appendMetric(metrics.Memory, point) case "disk": metrics.Disk = mh.appendMetric(metrics.Disk, point) + case "gpu": + metrics.GPU = mh.appendMetric(metrics.GPU, point) + case "gpu_memory": + metrics.GPUMemory = mh.appendMetric(metrics.GPUMemory, point) + case "gpu_temperature": + metrics.GPUTemperature = mh.appendMetric(metrics.GPUTemperature, point) case "diskread": metrics.DiskRead = mh.appendMetric(metrics.DiskRead, point) case "diskwrite": @@ -281,6 +293,12 @@ func (mh *MetricsHistory) addGuestMetricSeries(guestID, metricType string, value metrics.Memory = mh.appendMetricSeries(metrics.Memory, values, timestamps) case "disk": metrics.Disk = mh.appendMetricSeries(metrics.Disk, values, timestamps) + case "gpu": + metrics.GPU = mh.appendMetricSeries(metrics.GPU, values, timestamps) + case "gpu_memory": + metrics.GPUMemory = mh.appendMetricSeries(metrics.GPUMemory, values, timestamps) + case "gpu_temperature": + metrics.GPUTemperature = mh.appendMetricSeries(metrics.GPUTemperature, values, timestamps) case "diskread": metrics.DiskRead = mh.appendMetricSeries(metrics.DiskRead, values, timestamps) case "diskwrite": @@ -378,6 +396,12 @@ func (mh *MetricsHistory) GetGuestMetrics(guestID string, metricType string, dur data = metrics.Memory case "disk": data = metrics.Disk + case "gpu": + data = metrics.GPU + case "gpu_memory": + data = metrics.GPUMemory + case "gpu_temperature": + data = metrics.GPUTemperature case "diskread": data = metrics.DiskRead case "diskwrite": @@ -487,6 +511,12 @@ func guestMetricSeries(metrics *GuestMetrics, metricType string) []MetricPoint { return metrics.Memory case "disk": return metrics.Disk + case "gpu": + return metrics.GPU + case "gpu_memory": + return metrics.GPUMemory + case "gpu_temperature": + return metrics.GPUTemperature case "diskread": return metrics.DiskRead case "diskwrite": @@ -508,7 +538,7 @@ func guestMetricsCoverageSpan(metrics *GuestMetrics, metricTypes []string, cutof } if len(metricTypes) == 0 { - metricTypes = []string{"cpu", "memory", "disk", "diskread", "diskwrite", "netin", "netout", "temperature"} + metricTypes = []string{"cpu", "memory", "disk", "gpu", "gpu_memory", "gpu_temperature", "diskread", "diskwrite", "netin", "netout", "temperature"} } var best time.Duration @@ -572,6 +602,9 @@ func (mh *MetricsHistory) GetAllGuestMetrics(guestID string, duration time.Durat result["cpu"] = filterMetricsByTime(metrics.CPU, cutoffTime) result["memory"] = filterMetricsByTime(metrics.Memory, cutoffTime) result["disk"] = filterMetricsByTime(metrics.Disk, cutoffTime) + result["gpu"] = filterMetricsByTime(metrics.GPU, cutoffTime) + result["gpu_memory"] = filterMetricsByTime(metrics.GPUMemory, cutoffTime) + result["gpu_temperature"] = filterMetricsByTime(metrics.GPUTemperature, cutoffTime) result["diskread"] = filterMetricsByTime(metrics.DiskRead, cutoffTime) result["diskwrite"] = filterMetricsByTime(metrics.DiskWrite, cutoffTime) result["netin"] = filterMetricsByTime(metrics.NetworkIn, cutoffTime) @@ -693,6 +726,9 @@ func (mh *MetricsHistory) Cleanup() { metrics.CPU = mh.cleanupMetrics(metrics.CPU, cutoffTime) metrics.Memory = mh.cleanupMetrics(metrics.Memory, cutoffTime) metrics.Disk = mh.cleanupMetrics(metrics.Disk, cutoffTime) + metrics.GPU = mh.cleanupMetrics(metrics.GPU, cutoffTime) + metrics.GPUMemory = mh.cleanupMetrics(metrics.GPUMemory, cutoffTime) + metrics.GPUTemperature = mh.cleanupMetrics(metrics.GPUTemperature, cutoffTime) metrics.DiskRead = mh.cleanupMetrics(metrics.DiskRead, cutoffTime) metrics.DiskWrite = mh.cleanupMetrics(metrics.DiskWrite, cutoffTime) metrics.NetworkIn = mh.cleanupMetrics(metrics.NetworkIn, cutoffTime) @@ -701,6 +737,7 @@ func (mh *MetricsHistory) Cleanup() { // If all slices are empty, remove the map entry entirely to free memory if len(metrics.CPU) == 0 && len(metrics.Memory) == 0 && len(metrics.Disk) == 0 && + len(metrics.GPU) == 0 && len(metrics.GPUMemory) == 0 && len(metrics.GPUTemperature) == 0 && len(metrics.DiskRead) == 0 && len(metrics.DiskWrite) == 0 && len(metrics.NetworkIn) == 0 && len(metrics.NetworkOut) == 0 && len(metrics.Temperature) == 0 { diff --git a/internal/monitoring/metrics_history_test.go b/internal/monitoring/metrics_history_test.go index 28d0cad113..3954ceaa90 100644 --- a/internal/monitoring/metrics_history_test.go +++ b/internal/monitoring/metrics_history_test.go @@ -326,6 +326,9 @@ func TestAddGuestMetric(t *testing.T) { value float64 }{ {name: "cpu metric", guestID: "vm-100", metricType: "cpu", value: 75.5}, + {name: "gpu metric", guestID: "vm-100", metricType: "gpu", value: 65.5}, + {name: "gpu memory metric", guestID: "vm-100", metricType: "gpu_memory", value: 42.0}, + {name: "gpu temperature metric", guestID: "vm-100", metricType: "gpu_temperature", value: 71.0}, {name: "memory metric", guestID: "vm-100", metricType: "memory", value: 80.0}, {name: "disk metric", guestID: "ct-101", metricType: "disk", value: 50.0}, {name: "diskread metric", guestID: "vm-102", metricType: "diskread", value: 100.5}, @@ -354,6 +357,12 @@ func TestAddGuestMetric(t *testing.T) { switch tt.metricType { case "cpu": slice = metrics.CPU + case "gpu": + slice = metrics.GPU + case "gpu_memory": + slice = metrics.GPUMemory + case "gpu_temperature": + slice = metrics.GPUTemperature case "memory": slice = metrics.Memory case "disk": @@ -368,7 +377,8 @@ func TestAddGuestMetric(t *testing.T) { slice = metrics.NetworkOut default: // Unknown types should not populate any slice - if len(metrics.CPU)+len(metrics.Memory)+len(metrics.Disk)+ + if len(metrics.CPU)+len(metrics.GPU)+len(metrics.GPUMemory)+ + len(metrics.GPUTemperature)+len(metrics.Memory)+len(metrics.Disk)+ len(metrics.DiskRead)+len(metrics.DiskWrite)+ len(metrics.NetworkIn)+len(metrics.NetworkOut) != 0 { t.Error("unknown metric type populated a slice") @@ -596,6 +606,9 @@ func TestGetGuestMetrics_AllMetricTypes(t *testing.T) { value float64 }{ {name: "cpu metric", metricType: "cpu", value: 50.5}, + {name: "gpu metric", metricType: "gpu", value: 80.5}, + {name: "gpu memory metric", metricType: "gpu_memory", value: 62.5}, + {name: "gpu temperature metric", metricType: "gpu_temperature", value: 74.0}, {name: "memory metric", metricType: "memory", value: 70.0}, {name: "disk metric", metricType: "disk", value: 45.0}, {name: "diskread metric", metricType: "diskread", value: 1024.0}, @@ -745,17 +758,31 @@ func TestGetGuestMetrics_EmptyMetricsData(t *testing.T) { // Directly populate an empty guest metrics entry mh.mu.Lock() mh.guestMetrics["vm-empty"] = &GuestMetrics{ - CPU: []MetricPoint{}, - Memory: []MetricPoint{}, - Disk: []MetricPoint{}, - DiskRead: []MetricPoint{}, - DiskWrite: []MetricPoint{}, - NetworkIn: []MetricPoint{}, - NetworkOut: []MetricPoint{}, + CPU: []MetricPoint{}, + GPU: []MetricPoint{}, + GPUMemory: []MetricPoint{}, + GPUTemperature: []MetricPoint{}, + Memory: []MetricPoint{}, + Disk: []MetricPoint{}, + DiskRead: []MetricPoint{}, + DiskWrite: []MetricPoint{}, + NetworkIn: []MetricPoint{}, + NetworkOut: []MetricPoint{}, } mh.mu.Unlock() - metricTypes := []string{"cpu", "memory", "disk", "diskread", "diskwrite", "netin", "netout"} + metricTypes := []string{ + "cpu", + "gpu", + "gpu_memory", + "gpu_temperature", + "memory", + "disk", + "diskread", + "diskwrite", + "netin", + "netout", + } for _, metricType := range metricTypes { t.Run(metricType, func(t *testing.T) { @@ -995,6 +1022,9 @@ func TestGetAllGuestMetrics(t *testing.T) { // Add test data for multiple metric types mh.AddGuestMetric("vm-100", "cpu", 50.0, now.Add(-30*time.Minute)) + mh.AddGuestMetric("vm-100", "gpu", 80.0, now.Add(-29*time.Minute)) + mh.AddGuestMetric("vm-100", "gpu_memory", 62.5, now.Add(-28*time.Minute)) + mh.AddGuestMetric("vm-100", "gpu_temperature", 72.0, now.Add(-27*time.Minute)) mh.AddGuestMetric("vm-100", "memory", 70.0, now.Add(-25*time.Minute)) mh.AddGuestMetric("vm-100", "disk", 40.0, now.Add(-20*time.Minute)) mh.AddGuestMetric("vm-100", "diskread", 100.0, now.Add(-15*time.Minute)) @@ -1012,7 +1042,18 @@ func TestGetAllGuestMetrics(t *testing.T) { name: "get all metrics within hour", guestID: "vm-100", duration: time.Hour, - wantKeys: []string{"cpu", "memory", "disk", "diskread", "diskwrite", "netin", "netout"}, + wantKeys: []string{ + "cpu", + "gpu", + "gpu_memory", + "gpu_temperature", + "memory", + "disk", + "diskread", + "diskwrite", + "netin", + "netout", + }, }, { name: "nonexistent guest", @@ -1024,7 +1065,18 @@ func TestGetAllGuestMetrics(t *testing.T) { name: "short duration filters out old", guestID: "vm-100", duration: 10 * time.Minute, - wantKeys: []string{"cpu", "memory", "disk", "diskread", "diskwrite", "netin", "netout"}, // keys exist but may be empty + wantKeys: []string{ + "cpu", + "gpu", + "gpu_memory", + "gpu_temperature", + "memory", + "disk", + "diskread", + "diskwrite", + "netin", + "netout", + }, // keys exist but may be empty }, } diff --git a/internal/monitoring/monitor_agents.go b/internal/monitoring/monitor_agents.go index 89c8b17cb0..869635a079 100644 --- a/internal/monitoring/monitor_agents.go +++ b/internal/monitoring/monitor_agents.go @@ -2949,6 +2949,7 @@ func (m *Monitor) ApplyHostReport(report agentshost.Report, tokenRecord *config. hostDiskPercent = host.Disks[0].Usage } hostTemperature := hostPrimaryTemperatureCelsius(host.Sensors) + hostGPUUtilization, hostGPUMemory, hostGPUTemperature := hostGPUMetrics(host.Sensors) if !shouldSkipNativeMockStateMetricWrites() { if m.metricsHistory != nil { @@ -2960,6 +2961,15 @@ func (m *Monitor) ApplyHostReport(report agentshost.Report, tokenRecord *config. if hostTemperature != nil { m.metricsHistory.AddGuestMetric(hostMetricKey, "temperature", *hostTemperature, now) } + if hostGPUUtilization != nil { + m.metricsHistory.AddGuestMetric(hostMetricKey, "gpu", *hostGPUUtilization, now) + } + if hostGPUMemory != nil { + m.metricsHistory.AddGuestMetric(hostMetricKey, "gpu_memory", *hostGPUMemory, now) + } + if hostGPUTemperature != nil { + m.metricsHistory.AddGuestMetric(hostMetricKey, "gpu_temperature", *hostGPUTemperature, now) + } if netInRate >= 0 { m.metricsHistory.AddGuestMetric(hostMetricKey, "netin", netInRate, now) @@ -2986,6 +2996,15 @@ func (m *Monitor) ApplyHostReport(report agentshost.Report, tokenRecord *config. if hostTemperature != nil { m.metricsStore.Write("agent", host.ID, "temperature", *hostTemperature, now) } + if hostGPUUtilization != nil { + m.metricsStore.Write("agent", host.ID, "gpu", *hostGPUUtilization, now) + } + if hostGPUMemory != nil { + m.metricsStore.Write("agent", host.ID, "gpu_memory", *hostGPUMemory, now) + } + if hostGPUTemperature != nil { + m.metricsStore.Write("agent", host.ID, "gpu_temperature", *hostGPUTemperature, now) + } m.writeHostSMARTMetrics(host, now) if netInRate >= 0 { m.metricsStore.Write("agent", host.ID, "netin", netInRate, now) @@ -3225,6 +3244,40 @@ func hostPrimaryTemperatureCelsius(sensors models.HostSensorSummary) *float64 { return &best } +func hostGPUMetrics(sensors models.HostSensorSummary) ( + utilizationPercent *float64, + memoryPercent *float64, + temperatureCelsius *float64, +) { + for _, gpu := range sensors.GPU { + if gpu.UtilizationPercent != nil { + value := *gpu.UtilizationPercent + if value >= 0 && value <= 100 && + (utilizationPercent == nil || value > *utilizationPercent) { + copy := value + utilizationPercent = © + } + } + if gpu.MemoryUsedBytes != nil && gpu.MemoryTotalBytes != nil && + *gpu.MemoryUsedBytes >= 0 && *gpu.MemoryTotalBytes > 0 { + value := float64(*gpu.MemoryUsedBytes) / float64(*gpu.MemoryTotalBytes) * 100 + if value <= 100 && (memoryPercent == nil || value > *memoryPercent) { + copy := value + memoryPercent = © + } + } + if gpu.TemperatureCelsius != nil { + value := *gpu.TemperatureCelsius + if value > 0 && value <= 150 && + (temperatureCelsius == nil || value > *temperatureCelsius) { + copy := value + temperatureCelsius = © + } + } + } + return utilizationPercent, memoryPercent, temperatureCelsius +} + func nodePrimaryTemperatureCelsius(temperature *models.Temperature) *float64 { if temperature == nil || !temperature.Available { return nil diff --git a/internal/monitoring/monitor_host_agents_test.go b/internal/monitoring/monitor_host_agents_test.go index ea0b225ab8..dbec358f3f 100644 --- a/internal/monitoring/monitor_host_agents_test.go +++ b/internal/monitoring/monitor_host_agents_test.go @@ -2610,12 +2610,22 @@ func TestApplyHostReportPersistsAgentTemperatureMetric(t *testing.T) { } func TestApplyHostReportPreservesGPUSensorSummary(t *testing.T) { + storeCfg := metrics.DefaultConfig(t.TempDir()) + storeCfg.WriteBufferSize = 1 + store, err := metrics.NewStore(storeCfg) + if err != nil { + t.Fatalf("NewStore: %v", err) + } + defer store.Close() + monitor := &Monitor{ state: models.NewState(), alertManager: alerts.NewManager(), hostTokenBindings: make(map[string]string), config: &config.Config{}, rateTracker: NewRateTracker(), + metricsHistory: NewMetricsHistory(10, time.Hour), + metricsStore: store, } t.Cleanup(func() { monitor.alertManager.Stop() }) @@ -2657,6 +2667,7 @@ func TestApplyHostReportPreservesGPUSensorSummary(t *testing.T) { if err != nil { t.Fatalf("ApplyHostReport: %v", err) } + store.Flush() if len(host.Sensors.GPU) != 1 { t.Fatalf("host GPU sensors = %+v, want one sensor", host.Sensors.GPU) } @@ -2677,6 +2688,23 @@ func TestApplyHostReportPreservesGPUSensorSummary(t *testing.T) { t.Fatalf("legacy GPU temperature compatibility = %+v, want %.1f", host.Sensors.TemperatureCelsius, temperature) } + expectedMetrics := map[string]float64{ + "gpu": utilization, + "gpu_memory": (float64(usedBytes) / float64(totalBytes)) * 100, + "gpu_temperature": temperature, + } + history := monitor.GetGuestMetrics("agent:"+host.ID, time.Hour) + for metric, expected := range expectedMetrics { + storePoints := waitForStoredMetric(t, store, "agent", host.ID, metric) + if got := storePoints[len(storePoints)-1].Value; got != expected { + t.Fatalf("persisted %s = %f, want %f", metric, got, expected) + } + historyPoints := history[metric] + if len(historyPoints) == 0 || historyPoints[len(historyPoints)-1].Value != expected { + t.Fatalf("in-memory %s = %+v, want %f", metric, historyPoints, expected) + } + } + projected := hostSensorsFromReadStateView(&unifiedresources.HostSensorMeta{ GPU: []unifiedresources.HostGPUSensor{ {