diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index 27765057c..508bd3019 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -876,14 +876,15 @@ surface and no new `internal/api/` lifecycle handler. it must report Darwin `pmset` thermal and performance pressure as `sensors.thermalState` instead of inventing Celsius readings from unavailable Apple silicon sensor values. - Linux NVIDIA GPU temperature telemetry belongs in that same host-agent - sensor contract: the runtime may supplement `lm-sensors` with a bounded - `nvidia-smi` temperature query and may use that query as a best-effort - fallback when `lm-sensors` is unavailable. The report must map only direct + Linux NVIDIA GPU telemetry belongs in that same host-agent sensor contract: + the runtime may supplement `lm-sensors` with a bounded `nvidia-smi` query + for direct GPU temperature, utilization, and VRAM readings, and may use that + query as a best-effort fallback when `lm-sensors` is unavailable. The report + must keep typed GPU readings in `sensors.gpu` while mapping only direct `temperature.gpu` readings into existing `sensors.temperatureCelsius` - `gpu_nvidia_` keys; it must not infer lifecycle health, command - authority, enrollment state, or GPU workload/process inventory from - `nvidia-smi` output. + `gpu_nvidia_` keys for compatibility; it must not infer lifecycle + health, command authority, enrollment state, or GPU workload/process + inventory from `nvidia-smi` output. Runtime RAID collection uses `/proc/mdstat` as the canonical discovery baseline for Linux md arrays. `mdadm --detail` may enrich level, state, member, UUID, and rebuild fields when available, but missing or failing diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index 4bd714b86..7dc50040e 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -3071,6 +3071,12 @@ macOS that expose thermal pressure but not stable Celsius sensor readings. API payloads must keep that state separate from `temperatureCelsius`; clients may display pressure and throttling limits, but must not synthesize a temperature metric or table value from pressure-only payloads. +Host sensor summaries may also carry typed GPU readings in `gpu`: id, name, +temperature, utilization, used VRAM, and total VRAM. Backend API payloads and +`frontend-modern/src/types/api.ts` must preserve that optional collection +without making it a required compatibility field, and clients must keep it as +descriptive host telemetry rather than a separate resource or process/workload +inventory contract. `aicontracts.Finding` (the shape Patrol hands the investigation orchestrator) carries optional `OperatorContext` and diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index 3b65936ff..edf48dd96 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -604,6 +604,12 @@ the shared model conversion helpers must preserve `sensors.thermalState` through ingest, read-state projection, and frontend conversion, while leaving `agent.temperature` and `metric=temperature` unset unless a real Celsius value exists. +Host-agent GPU sensor summaries follow that same descriptive-host-telemetry +path. Monitoring must preserve typed GPU id, name, temperature, utilization, +and VRAM readings from agent reports through models, read-state projection, and +frontend conversion, while still using only real Celsius readings for +`agent.temperature` or `metric=temperature` and without promoting GPU workload +or process inventory into monitoring state. That same monitoring owner also owns canonical unified-resource publication on `/api/state` and the websocket `state.resources` hydrate path. Monitoring must publish those resources from the same canonical unified snapshot that diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md index 3916b8a08..621f4f836 100644 --- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md +++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md @@ -802,6 +802,11 @@ platform tables may show a short pressure status such as `Nominal` or exists, while `resourceDetailMappers.ts` owns the fuller thermals-card rows for pressure source and throttling limits. The table fallback must stay row-local and must not add history reads, polling work, or a second thermal query path. +GPU utilization and VRAM rows in `resourceDetailMappers.ts` follow the same +drawer-only rule. The mapper may render typed GPU sensor values already present +on the selected resource payload, but it must not add GPU-specific history +reads, per-row polling, browser-side `nvidia-smi` assumptions, or table-wide +aggregation work. The Proxmox node drawer overview should follow the existing guest drawer compact detail-card pattern and expose node-specific context such as platform, kernel, hardware, raw capacity, telemetry, and thermal facts rather than diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index 5b7f6eee6..3f09119f0 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -1973,6 +1973,11 @@ The same boundary applies to host-agent `thermalState`: macOS pressure and throttling limits may appear as host context, but storage and recovery must not reinterpret pressure state as disk temperature, pool risk, backup freshness, or a storage-owned thermal timeline. +That same boundary applies to typed GPU host sensor metadata carried through +`internal/unifiedresources/types.go`: GPU temperature, utilization, and VRAM +readings may appear as descriptive host context, but storage and recovery must +not reinterpret those values as disk cache, storage-tier health, backup +freshness, restore evidence, or protection readiness. Storage and recovery still consume the shared unified-resource contract, but they do not own the timeline store itself. The canonical resource-change diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index 1475a567b..5b2db41c4 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -321,6 +321,12 @@ temperature source. Unified-resource adapters must carry may render that pressure as compact status text only when no numeric temperature exists, and must not convert pressure into a Celsius metric or history target. +Typed GPU host telemetry follows the same canonical sensor route. +`HostSensorMeta.gpu` may carry GPU id, name, temperature, utilization, and VRAM +readings through clone, merge, transport, and frontend decode paths, but those +readings remain descriptive host context. Unified-resource consumers must not +promote GPU sensors into separate hardware resources, lifecycle state, workload +identity, or history targets beyond direct numeric temperature compatibility. Metric bar fallbacks follow that split as well: unified-resource consumers own which CPU or memory value is selected, plus any source-specific fallback reason such as outdated standalone agent telemetry, while platform tables must use diff --git a/frontend-modern/src/components/Infrastructure/__tests__/UnifiedResourceTable.performance.contract.test.tsx b/frontend-modern/src/components/Infrastructure/__tests__/UnifiedResourceTable.performance.contract.test.tsx index 2ae668bf8..f537ca73f 100644 --- a/frontend-modern/src/components/Infrastructure/__tests__/UnifiedResourceTable.performance.contract.test.tsx +++ b/frontend-modern/src/components/Infrastructure/__tests__/UnifiedResourceTable.performance.contract.test.tsx @@ -594,6 +594,40 @@ describe('UnifiedResourceTable performance contract', () => { expect(resourceDetailDiscoveryModelSource).toContain('export const toDiscoveryConfig'); }); + it('keeps typed GPU detail rows bounded to the selected resource payload', () => { + expect( + buildTemperatureRows({ + temperatureCelsius: { + cpu_package: 48, + gpu_nvidia_0: 63, + }, + gpu: [ + { + id: '0', + name: 'NVIDIA RTX A6000', + temperatureCelsius: 63, + utilizationPercent: 0, + memoryUsedBytes: 2 * 1024 * 1024 * 1024, + memoryTotalBytes: 48 * 1024 * 1024 * 1024, + }, + ], + }), + ).toEqual([ + { + label: 'GPU 0', + value: 'NVIDIA RTX A6000 · 63°C · 0% · 2.00 GB / 48.0 GB', + valueTitle: 'NVIDIA RTX A6000 · 63°C · 0% · 2.00 GB / 48.0 GB', + }, + { + label: 'Package', + value: '48°C', + valueTitle: '48.0°C', + }, + ]); + expect(resourceDetailMappersSource).not.toContain('nvidia-smi'); + expect(resourceDetailMappersSource).not.toContain('fetch('); + }); + it('keeps hot-path table state and windowing in the shared table state owner', () => { expect(unifiedResourceTableSource).toContain('useUnifiedResourceTableState'); expect(unifiedResourceTableSource).toContain('UnifiedResourceHostTableCard'); diff --git a/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts b/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts index b27d99983..78e9967f3 100644 --- a/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts +++ b/frontend-modern/src/components/Infrastructure/__tests__/resourceDetailMappers.test.ts @@ -122,6 +122,38 @@ describe('resourceDetailMappers', () => { }, ]); }); + + it('surfaces typed GPU utilization and memory readings', () => { + const rows = buildTemperatureRows({ + temperatureCelsius: { + gpu_nvidia_0: 63, + cpu_package: 41, + }, + gpu: [ + { + id: '0', + name: 'NVIDIA RTX A6000', + temperatureCelsius: 63, + utilizationPercent: 0, + memoryUsedBytes: 2 * 1024 * 1024 * 1024, + memoryTotalBytes: 48 * 1024 * 1024 * 1024, + }, + ], + }); + + expect(rows).toEqual([ + { + label: 'GPU 0', + value: 'NVIDIA RTX A6000 · 63°C · 0% · 2.00 GB / 48.0 GB', + valueTitle: 'NVIDIA RTX A6000 · 63°C · 0% · 2.00 GB / 48.0 GB', + }, + { + label: 'Package', + value: '41°C', + valueTitle: '41.0°C', + }, + ]); + }); }); describe('toNodeFromProxmox', () => { diff --git a/frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts b/frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts index 8d68d724e..1ebac8ef4 100644 --- a/frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts +++ b/frontend-modern/src/components/Infrastructure/resourceDetailMappers.ts @@ -15,6 +15,7 @@ import type { } from '@/types/api'; import type { Resource, ResourceMetric, ResourceVMwareMeta } from '@/types/resource'; import { formatTemperature } from '@/utils/temperature'; +import { formatBytes } from '@/utils/format'; import { getActionableAgentIdFromResource } from '@/utils/agentResources'; import { getPreferredInfrastructureDisplayName, @@ -324,9 +325,49 @@ export const formatSensorName = (name: string) => { return titleCaseDelimitedLabel(clean); }; +const formatGPUStatsLabel = (id: string | undefined, index: number): string => { + const trimmed = (id || '').trim(); + return trimmed ? `GPU ${trimmed}` : `GPU ${index + 1}`; +}; + +const formatGPUStatsValue = (gpu: NonNullable[number]) => { + const parts: string[] = []; + if (gpu.name) parts.push(gpu.name); + if (typeof gpu.temperatureCelsius === 'number' && Number.isFinite(gpu.temperatureCelsius)) { + parts.push(formatTemperature(gpu.temperatureCelsius)); + } + if (typeof gpu.utilizationPercent === 'number' && Number.isFinite(gpu.utilizationPercent)) { + parts.push(`${Math.round(Math.max(0, gpu.utilizationPercent))}%`); + } + if ( + typeof gpu.memoryTotalBytes === 'number' && + Number.isFinite(gpu.memoryTotalBytes) && + gpu.memoryTotalBytes > 0 + ) { + const used = + typeof gpu.memoryUsedBytes === 'number' && Number.isFinite(gpu.memoryUsedBytes) + ? Math.max(0, gpu.memoryUsedBytes) + : 0; + parts.push(`${formatBytes(used)} / ${formatBytes(gpu.memoryTotalBytes)}`); + } + return parts.join(' · '); +}; + +const buildTypedGPUTemperatureKeys = (gpus?: HostSensorSummary['gpu']) => { + const keys = new Set(); + gpus?.forEach((gpu) => { + const id = (gpu.id || '').trim(); + if (!id) return; + if (typeof gpu.temperatureCelsius !== 'number' || !Number.isFinite(gpu.temperatureCelsius)) return; + keys.add(`gpu_nvidia_${id}`); + }); + return keys; +}; + export const buildTemperatureRows = (sensors?: HostSensorSummary) => { const rows: { label: string; value: string; valueTitle?: string }[] = []; const thermalState = sensors?.thermalState; + const typedGPUTemperatureKeys = buildTypedGPUTemperatureKeys(sensors?.gpu); if (thermalState?.pressure) { rows.push({ label: 'Thermal pressure', @@ -349,9 +390,23 @@ export const buildTemperatureRows = (sensors?: HostSensorSummary) => { }); }); } + const gpus = sensors?.gpu; + if (gpus) { + gpus.forEach((gpu, index) => { + const value = formatGPUStatsValue(gpu); + if (!value) return; + rows.push({ + label: formatGPUStatsLabel(gpu.id, index), + value, + valueTitle: value, + }); + }); + } const temps = sensors?.temperatureCelsius; if (temps) { - const entries = Object.entries(temps).sort(([a], [b]) => a.localeCompare(b)); + const entries = Object.entries(temps) + .filter(([name]) => !typedGPUTemperatureKeys.has(name)) + .sort(([a], [b]) => a.localeCompare(b)); entries.forEach(([name, temp]) => { rows.push({ label: formatSensorName(name), diff --git a/frontend-modern/src/types/api.ts b/frontend-modern/src/types/api.ts index 399e0672c..f2e709d79 100644 --- a/frontend-modern/src/types/api.ts +++ b/frontend-modern/src/types/api.ts @@ -484,10 +484,20 @@ export interface HostSensorSummary { temperatureCelsius?: Record; fanRpm?: Record; additional?: Record; + gpu?: HostGPUSensor[]; thermalState?: HostThermalState; smart?: HostDiskSMART[]; // S.M.A.R.T. disk data } +export interface HostGPUSensor { + id?: string; + name?: string; + temperatureCelsius?: number; + utilizationPercent?: number; + memoryUsedBytes?: number; + memoryTotalBytes?: number; +} + export interface HostThermalState { source?: string; pressure?: 'nominal' | 'constrained' | 'unknown' | string; diff --git a/internal/hostagent/agent_metrics_test.go b/internal/hostagent/agent_metrics_test.go index 8e58cd256..ee1f7c45b 100644 --- a/internal/hostagent/agent_metrics_test.go +++ b/internal/hostagent/agent_metrics_test.go @@ -480,10 +480,10 @@ func TestBuildReportIncludesNVIDIASMITemperaturesWhenLMSensorsUnavailable(t *tes if name != "/usr/bin/nvidia-smi" { t.Fatalf("command name = %q, want /usr/bin/nvidia-smi", name) } - if len(arg) != 2 || arg[0] != "--query-gpu=index,name,temperature.gpu" || arg[1] != "--format=csv,noheader,nounits" { - t.Fatalf("command args = %#v, want NVIDIA temperature query", arg) + if len(arg) != 2 || arg[0] != "--query-gpu=index,name,temperature.gpu,utilization.gpu,memory.used,memory.total" || arg[1] != "--format=csv,noheader,nounits" { + t.Fatalf("command args = %#v, want NVIDIA stats query", arg) } - return "0, NVIDIA GeForce RTX 4090, 63\n", nil + return "0, NVIDIA GeForce RTX 4090, 63, 42, 8192, 24576\n", nil }, } @@ -505,6 +505,15 @@ func TestBuildReportIncludesNVIDIASMITemperaturesWhenLMSensorsUnavailable(t *tes if report.Sensors.TemperatureCelsius["gpu_nvidia_0"] != 63 { t.Fatalf("NVIDIA GPU temp = %v, want 63", report.Sensors.TemperatureCelsius["gpu_nvidia_0"]) } + if len(report.Sensors.GPU) != 1 { + t.Fatalf("GPU stats = %d, want 1: %+v", len(report.Sensors.GPU), report.Sensors.GPU) + } + if report.Sensors.GPU[0].UtilizationPercent == nil || *report.Sensors.GPU[0].UtilizationPercent != 42 { + t.Fatalf("GPU utilization = %#v, want 42", report.Sensors.GPU[0].UtilizationPercent) + } + if report.Sensors.GPU[0].MemoryTotalBytes == nil || *report.Sensors.GPU[0].MemoryTotalBytes != 24576*1024*1024 { + t.Fatalf("GPU memory total = %#v, want 24576 MiB", report.Sensors.GPU[0].MemoryTotalBytes) + } } func TestBuildReportUsesResolvedNASOSIdentity(t *testing.T) { diff --git a/internal/hostagent/agent_sensors_test.go b/internal/hostagent/agent_sensors_test.go index 2b71892ed..5643a843c 100644 --- a/internal/hostagent/agent_sensors_test.go +++ b/internal/hostagent/agent_sensors_test.go @@ -77,10 +77,10 @@ func TestAgent_collectTemperatures_MergesNVIDIASMITemperatures(t *testing.T) { if name != "/usr/bin/nvidia-smi" { t.Fatalf("command name = %q, want /usr/bin/nvidia-smi", name) } - if len(arg) != 2 || arg[0] != "--query-gpu=index,name,temperature.gpu" || arg[1] != "--format=csv,noheader,nounits" { - t.Fatalf("command args = %#v, want NVIDIA temperature query", arg) + if len(arg) != 2 || arg[0] != "--query-gpu=index,name,temperature.gpu,utilization.gpu,memory.used,memory.total" || arg[1] != "--format=csv,noheader,nounits" { + t.Fatalf("command args = %#v, want NVIDIA stats query", arg) } - return "0, NVIDIA GeForce RTX 4090, 61\n", nil + return "0, NVIDIA GeForce RTX 4090, 61, 7, 4096, 24576\n", nil }, } @@ -93,6 +93,15 @@ func TestAgent_collectTemperatures_MergesNVIDIASMITemperatures(t *testing.T) { if got.TemperatureCelsius["gpu_nvidia_0"] != 61 { t.Fatalf("NVIDIA GPU temp = %v, want 61", got.TemperatureCelsius["gpu_nvidia_0"]) } + if len(got.GPU) != 1 { + t.Fatalf("GPU stats = %d, want 1: %+v", len(got.GPU), got.GPU) + } + if got.GPU[0].UtilizationPercent == nil || *got.GPU[0].UtilizationPercent != 7 { + t.Fatalf("GPU utilization = %#v, want 7", got.GPU[0].UtilizationPercent) + } + if got.GPU[0].MemoryUsedBytes == nil || *got.GPU[0].MemoryUsedBytes != 4096*1024*1024 { + t.Fatalf("GPU memory used = %#v, want 4096 MiB", got.GPU[0].MemoryUsedBytes) + } } func TestAgent_collectTemperatures_UsesNVIDIASMIWhenLMSensorsUnavailable(t *testing.T) { @@ -106,7 +115,7 @@ func TestAgent_collectTemperatures_UsesNVIDIASMIWhenLMSensorsUnavailable(t *test return "/usr/bin/nvidia-smi", nil }, commandCombinedOutputFn: func(context.Context, string, ...string) (string, error) { - return "0, NVIDIA RTX A6000, 58\n", nil + return "0, NVIDIA RTX A6000, 58, 0, 0, 49152\n", nil }, } @@ -119,6 +128,12 @@ func TestAgent_collectTemperatures_UsesNVIDIASMIWhenLMSensorsUnavailable(t *test if got.TemperatureCelsius["gpu_nvidia_0"] != 58 { t.Fatalf("NVIDIA GPU temp = %v, want 58", got.TemperatureCelsius["gpu_nvidia_0"]) } + if got.GPU[0].UtilizationPercent == nil || *got.GPU[0].UtilizationPercent != 0 { + t.Fatalf("GPU utilization = %#v, want 0", got.GPU[0].UtilizationPercent) + } + if got.GPU[0].MemoryUsedBytes == nil || *got.GPU[0].MemoryUsedBytes != 0 { + t.Fatalf("GPU memory used = %#v, want 0", got.GPU[0].MemoryUsedBytes) + } } func TestAgent_collectTemperatures_BestEffortFailuresReturnEmpty(t *testing.T) { @@ -143,10 +158,10 @@ func TestAgent_collectTemperatures_BestEffortFailuresReturnEmpty(t *testing.T) { } func TestParseNVIDIASMITemperatures(t *testing.T) { - input := `0, NVIDIA GeForce RTX 4090, 47 -1, "NVIDIA, RTX A6000", N/A -2, NVIDIA Tesla T4, 58 C -0000:65:00.0, NVIDIA H100, 64` + input := `0, NVIDIA GeForce RTX 4090, 47, 11, 2048, 24576 +1, "NVIDIA, RTX A6000", N/A, N/A, N/A, N/A +2, NVIDIA Tesla T4, 58 C, 0 %, 0 MiB, 15360 MiB +0000:65:00.0, NVIDIA H100, 64, 95, 70000, 81920` got, err := parseNVIDIASMITemperatures(input) if err != nil { @@ -168,6 +183,40 @@ func TestParseNVIDIASMITemperatures(t *testing.T) { } } +func TestParseNVIDIASMIStats(t *testing.T) { + input := `0, NVIDIA GeForce RTX 4090, 47, 11, 2048, 24576 +1, NVIDIA RTX A6000, 0, 0, 0, 49152` + + got, err := parseNVIDIASMIStats(input) + if err != nil { + t.Fatalf("parseNVIDIASMIStats returned error: %v", err) + } + if len(got) != 2 { + t.Fatalf("GPU stats = %d, want 2: %+v", len(got), got) + } + if got[0].ID != "0" || got[0].Name != "NVIDIA GeForce RTX 4090" { + t.Fatalf("unexpected first GPU identity: %+v", got[0]) + } + if got[0].TemperatureCelsius == nil || *got[0].TemperatureCelsius != 47 { + t.Fatalf("first GPU temp = %#v, want 47", got[0].TemperatureCelsius) + } + if got[0].UtilizationPercent == nil || *got[0].UtilizationPercent != 11 { + t.Fatalf("first GPU utilization = %#v, want 11", got[0].UtilizationPercent) + } + if got[0].MemoryUsedBytes == nil || *got[0].MemoryUsedBytes != 2048*1024*1024 { + t.Fatalf("first GPU memory used = %#v, want 2048 MiB", got[0].MemoryUsedBytes) + } + if got[1].TemperatureCelsius != nil { + t.Fatalf("second GPU temp = %#v, want nil for zero temperature", got[1].TemperatureCelsius) + } + if got[1].UtilizationPercent == nil || *got[1].UtilizationPercent != 0 { + t.Fatalf("second GPU utilization = %#v, want 0", got[1].UtilizationPercent) + } + if got[1].MemoryUsedBytes == nil || *got[1].MemoryUsedBytes != 0 { + t.Fatalf("second GPU memory used = %#v, want 0", got[1].MemoryUsedBytes) + } +} + func TestParseNVIDIASMITemperatures_NoUsableRows(t *testing.T) { got, err := parseNVIDIASMITemperatures("0, NVIDIA RTX, N/A\nbad row\n") if err != nil { diff --git a/internal/hostagent/nvidia_smi.go b/internal/hostagent/nvidia_smi.go index 52c750dfd..648ae84f4 100644 --- a/internal/hostagent/nvidia_smi.go +++ b/internal/hostagent/nvidia_smi.go @@ -5,6 +5,7 @@ import ( "encoding/csv" "errors" "fmt" + "math" "os" "os/exec" "strconv" @@ -14,41 +15,70 @@ import ( agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host" ) -const nvidiaSMITemperatureTimeout = 5 * time.Second +const ( + nvidiaSMIQueryTimeout = 5 * time.Second + bytesPerMiB = 1024 * 1024 +) -var nvidiaSMITemperatureArgs = []string{ - "--query-gpu=index,name,temperature.gpu", +var nvidiaSMIStatsArgs = []string{ + "--query-gpu=index,name,temperature.gpu,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits", } func (a *Agent) collectNVIDIATemperatureSensors(ctx context.Context) agentshost.Sensors { var result agentshost.Sensors - a.mergeNVIDIATemperatures(ctx, &result) - if len(result.TemperatureCelsius) == 0 { + a.mergeNVIDIASMIStats(ctx, &result) + if len(result.TemperatureCelsius) == 0 && len(result.GPU) == 0 { return agentshost.Sensors{} } return result } func (a *Agent) mergeNVIDIATemperatures(ctx context.Context, result *agentshost.Sensors) { - temps, err := a.queryNVIDIASMITemperatures(ctx) + a.mergeNVIDIASMIStats(ctx, result) +} + +func (a *Agent) mergeNVIDIASMIStats(ctx context.Context, result *agentshost.Sensors) { + gpus, err := a.queryNVIDIASMIStats(ctx) if err != nil { - a.logger.Debug().Err(err).Msg("Failed to collect NVIDIA GPU temperatures") + a.logger.Debug().Err(err).Msg("Failed to collect NVIDIA GPU stats") return } - if len(temps) == 0 { + if len(gpus) == 0 { return } - if result.TemperatureCelsius == nil { - result.TemperatureCelsius = make(map[string]float64, len(temps)) - } - for key, temp := range temps { - result.TemperatureCelsius[key] = temp + result.GPU = mergeNVIDIAGPUStats(result.GPU, gpus) + for _, gpu := range gpus { + if gpu.TemperatureCelsius == nil || *gpu.TemperatureCelsius <= 0 { + continue + } + if result.TemperatureCelsius == nil { + result.TemperatureCelsius = make(map[string]float64, len(gpus)) + } + result.TemperatureCelsius["gpu_nvidia_"+gpu.ID] = *gpu.TemperatureCelsius } } func (a *Agent) queryNVIDIASMITemperatures(ctx context.Context) (map[string]float64, error) { + gpus, err := a.queryNVIDIASMIStats(ctx) + if err != nil { + return nil, err + } + temps := make(map[string]float64) + for _, gpu := range gpus { + if gpu.TemperatureCelsius == nil || *gpu.TemperatureCelsius <= 0 { + continue + } + temps["gpu_nvidia_"+gpu.ID] = *gpu.TemperatureCelsius + } + if len(temps) == 0 { + return nil, nil + } + return temps, nil +} + +func (a *Agent) queryNVIDIASMIStats(ctx context.Context) ([]agentshost.GPUSensor, error) { path, err := a.collector.LookPath("nvidia-smi") if err != nil { if errors.Is(err, exec.ErrNotFound) || os.IsNotExist(err) { @@ -57,22 +87,42 @@ func (a *Agent) queryNVIDIASMITemperatures(ctx context.Context) (map[string]floa return nil, fmt.Errorf("locate nvidia-smi: %w", err) } - cmdCtx, cancel := context.WithTimeout(ctx, nvidiaSMITemperatureTimeout) + cmdCtx, cancel := context.WithTimeout(ctx, nvidiaSMIQueryTimeout) defer cancel() - output, err := a.collector.CommandCombinedOutput(cmdCtx, path, nvidiaSMITemperatureArgs...) + output, err := a.collector.CommandCombinedOutput(cmdCtx, path, nvidiaSMIStatsArgs...) if err != nil { - return nil, fmt.Errorf("execute nvidia-smi temperature query: %w", err) + return nil, fmt.Errorf("execute nvidia-smi stats query: %w", err) } - temps, err := parseNVIDIASMITemperatures(output) + gpus, err := parseNVIDIASMIStats(output) if err != nil { - return nil, fmt.Errorf("parse nvidia-smi temperature output: %w", err) + return nil, fmt.Errorf("parse nvidia-smi stats output: %w", err) + } + return gpus, nil +} + +func parseNVIDIASMITemperatures(output string) (map[string]float64, error) { + gpus, err := parseNVIDIASMIStats(output) + if err != nil { + return nil, err + } + + temps := make(map[string]float64) + for _, gpu := range gpus { + if gpu.TemperatureCelsius == nil || *gpu.TemperatureCelsius <= 0 { + continue + } + temps["gpu_nvidia_"+gpu.ID] = *gpu.TemperatureCelsius + } + + if len(temps) == 0 { + return nil, nil } return temps, nil } -func parseNVIDIASMITemperatures(output string) (map[string]float64, error) { +func parseNVIDIASMIStats(output string) ([]agentshost.GPUSensor, error) { output = strings.TrimSpace(output) if output == "" { return nil, nil @@ -87,32 +137,61 @@ func parseNVIDIASMITemperatures(output string) (map[string]float64, error) { return nil, err } - temps := make(map[string]float64) + gpus := make([]agentshost.GPUSensor, 0, len(records)) + usedIDs := make(map[string]struct{}, len(records)) for row, record := range records { if len(record) < 3 { continue } - temp, ok := parseNVIDIASMITemperatureValue(record[2]) - if !ok { + id := normalizeNVIDIASMIIndex(record[0], row) + if _, exists := usedIDs[id]; exists { + id = strconv.Itoa(row) + } + usedIDs[id] = struct{}{} + + gpu := agentshost.GPUSensor{ + ID: id, + Name: strings.TrimSpace(record[1]), + } + if temp, ok := parseNVIDIASMINumber(record[2], false); ok { + gpu.TemperatureCelsius = &temp + } + if len(record) > 3 { + if utilization, ok := parseNVIDIASMINumber(record[3], true); ok { + gpu.UtilizationPercent = &utilization + } + } + if len(record) > 4 { + if memoryUsedMiB, ok := parseNVIDIASMINumber(record[4], true); ok { + memoryUsedBytes := miBToBytes(memoryUsedMiB) + gpu.MemoryUsedBytes = &memoryUsedBytes + } + } + if len(record) > 5 { + if memoryTotalMiB, ok := parseNVIDIASMINumber(record[5], false); ok { + memoryTotalBytes := miBToBytes(memoryTotalMiB) + gpu.MemoryTotalBytes = &memoryTotalBytes + } + } + + if gpu.TemperatureCelsius == nil && gpu.UtilizationPercent == nil && gpu.MemoryUsedBytes == nil && gpu.MemoryTotalBytes == nil { continue } - - index := normalizeNVIDIASMIIndex(record[0], row) - key := "gpu_nvidia_" + index - if _, exists := temps[key]; exists { - key = fmt.Sprintf("gpu_nvidia_%d", row) - } - temps[key] = temp + gpus = append(gpus, gpu) } - if len(temps) == 0 { + if len(gpus) == 0 { return nil, nil } - return temps, nil + return gpus, nil } func parseNVIDIASMITemperatureValue(value string) (float64, bool) { + return parseNVIDIASMINumber(value, false) +} + +func parseNVIDIASMINumber(value string, allowZero bool) (float64, bool) { value = strings.TrimSpace(value) if value == "" { return 0, false @@ -127,13 +206,48 @@ func parseNVIDIASMITemperatureValue(value string) (float64, bool) { if len(fields) > 0 { value = fields[0] } + value = strings.TrimSuffix(value, "%") value = strings.TrimSuffix(strings.TrimSuffix(value, "C"), "c") + value = strings.TrimSuffix(strings.TrimSuffix(value, "MiB"), "Mib") - temp, err := strconv.ParseFloat(value, 64) - if err != nil || temp <= 0 { + parsed, err := strconv.ParseFloat(value, 64) + if err != nil { return 0, false } - return temp, true + if parsed < 0 || (!allowZero && parsed == 0) { + return 0, false + } + return parsed, true +} + +func miBToBytes(value float64) int64 { + return int64(math.Round(value * bytesPerMiB)) +} + +func mergeNVIDIAGPUStats(existing []agentshost.GPUSensor, incoming []agentshost.GPUSensor) []agentshost.GPUSensor { + if len(existing) == 0 { + return append([]agentshost.GPUSensor(nil), incoming...) + } + + result := append([]agentshost.GPUSensor(nil), existing...) + indexByID := make(map[string]int, len(result)) + for i, gpu := range result { + if gpu.ID == "" { + continue + } + indexByID[gpu.ID] = i + } + for _, gpu := range incoming { + if gpu.ID != "" { + if i, ok := indexByID[gpu.ID]; ok { + result[i] = gpu + continue + } + indexByID[gpu.ID] = len(result) + } + result = append(result, gpu) + } + return result } func normalizeNVIDIASMIIndex(value string, fallback int) string { diff --git a/internal/models/converters.go b/internal/models/converters.go index 8c8aef0ce..3ada62a8e 100644 --- a/internal/models/converters.go +++ b/internal/models/converters.go @@ -839,7 +839,7 @@ func (s DockerSwarmInfo) ToFrontend() DockerSwarmFrontend { } func hostSensorSummaryToFrontend(src HostSensorSummary) *HostSensorSummaryFrontend { - if len(src.TemperatureCelsius) == 0 && len(src.FanRPM) == 0 && len(src.Additional) == 0 && src.ThermalState == nil && len(src.SMART) == 0 { + if len(src.TemperatureCelsius) == 0 && len(src.FanRPM) == 0 && len(src.Additional) == 0 && len(src.GPU) == 0 && src.ThermalState == nil && len(src.SMART) == 0 { return nil } @@ -853,6 +853,19 @@ func hostSensorSummaryToFrontend(src HostSensorSummary) *HostSensorSummaryFronte if len(src.Additional) > 0 { dest.Additional = copyStringFloatMap(src.Additional) } + if len(src.GPU) > 0 { + dest.GPU = make([]HostGPUSensorFrontend, len(src.GPU)) + for i, gpu := range src.GPU { + dest.GPU[i] = HostGPUSensorFrontend{ + ID: gpu.ID, + Name: gpu.Name, + TemperatureCelsius: cloneFloat64Ptr(gpu.TemperatureCelsius), + UtilizationPercent: cloneFloat64Ptr(gpu.UtilizationPercent), + MemoryUsedBytes: cloneInt64Ptr(gpu.MemoryUsedBytes), + MemoryTotalBytes: cloneInt64Ptr(gpu.MemoryTotalBytes), + } + } + } if src.ThermalState != nil { dest.ThermalState = copyHostThermalState(src.ThermalState) } diff --git a/internal/models/deepcopy.go b/internal/models/deepcopy.go index 7cb596647..1d0db9418 100644 --- a/internal/models/deepcopy.go +++ b/internal/models/deepcopy.go @@ -247,11 +247,30 @@ func cloneHostSensorSummary(src HostSensorSummary) HostSensorSummary { TemperatureCelsius: cloneStringFloat64Map(src.TemperatureCelsius), FanRPM: cloneStringFloat64Map(src.FanRPM), Additional: cloneStringFloat64Map(src.Additional), + GPU: cloneHostGPUSensors(src.GPU), ThermalState: cloneHostThermalState(src.ThermalState), SMART: cloneHostDiskSMART(src.SMART), }.NormalizeCollections() } +func cloneHostGPUSensors(src []HostGPUSensor) []HostGPUSensor { + if len(src) == 0 { + return nil + } + dest := make([]HostGPUSensor, len(src)) + for i, gpu := range src { + dest[i] = HostGPUSensor{ + ID: gpu.ID, + Name: gpu.Name, + TemperatureCelsius: cloneFloat64Ptr(gpu.TemperatureCelsius), + UtilizationPercent: cloneFloat64Ptr(gpu.UtilizationPercent), + MemoryUsedBytes: cloneInt64Ptr(gpu.MemoryUsedBytes), + MemoryTotalBytes: cloneInt64Ptr(gpu.MemoryTotalBytes), + } + } + return dest +} + func cloneHostThermalState(src *HostThermalState) *HostThermalState { if src == nil { return nil diff --git a/internal/models/models.go b/internal/models/models.go index 85f80703d..0fd587dda 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -354,10 +354,20 @@ type HostSensorSummary struct { TemperatureCelsius map[string]float64 `json:"temperatureCelsius,omitempty"` FanRPM map[string]float64 `json:"fanRpm,omitempty"` Additional map[string]float64 `json:"additional,omitempty"` + GPU []HostGPUSensor `json:"gpu,omitempty"` ThermalState *HostThermalState `json:"thermalState,omitempty"` SMART []HostDiskSMART `json:"smart,omitempty"` // S.M.A.R.T. disk data } +type HostGPUSensor struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + TemperatureCelsius *float64 `json:"temperatureCelsius,omitempty"` + UtilizationPercent *float64 `json:"utilizationPercent,omitempty"` + MemoryUsedBytes *int64 `json:"memoryUsedBytes,omitempty"` + MemoryTotalBytes *int64 `json:"memoryTotalBytes,omitempty"` +} + type HostThermalState struct { Source string `json:"source,omitempty"` Pressure string `json:"pressure,omitempty"` @@ -377,6 +387,9 @@ func (s HostSensorSummary) NormalizeCollections() HostSensorSummary { if s.Additional == nil { s.Additional = map[string]float64{} } + if s.GPU == nil { + s.GPU = []HostGPUSensor{} + } if s.SMART == nil { s.SMART = []HostDiskSMART{} } diff --git a/internal/models/models_frontend.go b/internal/models/models_frontend.go index 4438bdccf..e820e6f93 100644 --- a/internal/models/models_frontend.go +++ b/internal/models/models_frontend.go @@ -750,10 +750,20 @@ type HostSensorSummaryFrontend struct { TemperatureCelsius map[string]float64 `json:"temperatureCelsius"` FanRPM map[string]float64 `json:"fanRpm"` Additional map[string]float64 `json:"additional"` + GPU []HostGPUSensorFrontend `json:"gpu"` ThermalState *HostThermalState `json:"thermalState,omitempty"` SMART []HostDiskSMARTFrontend `json:"smart"` // S.M.A.R.T. disk data } +type HostGPUSensorFrontend struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + TemperatureCelsius *float64 `json:"temperatureCelsius,omitempty"` + UtilizationPercent *float64 `json:"utilizationPercent,omitempty"` + MemoryUsedBytes *int64 `json:"memoryUsedBytes,omitempty"` + MemoryTotalBytes *int64 `json:"memoryTotalBytes,omitempty"` +} + func (s HostSensorSummaryFrontend) NormalizeCollections() HostSensorSummaryFrontend { if s.TemperatureCelsius == nil { s.TemperatureCelsius = map[string]float64{} @@ -764,6 +774,9 @@ func (s HostSensorSummaryFrontend) NormalizeCollections() HostSensorSummaryFront if s.Additional == nil { s.Additional = map[string]float64{} } + if s.GPU == nil { + s.GPU = []HostGPUSensorFrontend{} + } if s.SMART == nil { s.SMART = []HostDiskSMARTFrontend{} } diff --git a/internal/monitoring/host_agent_temps.go b/internal/monitoring/host_agent_temps.go index cd2de2902..da86e7e60 100644 --- a/internal/monitoring/host_agent_temps.go +++ b/internal/monitoring/host_agent_temps.go @@ -121,10 +121,29 @@ func convertUnifiedHostSensorsToTemperature(sensors *unifiedresources.HostSensor TemperatureCelsius: cloneStringFloatMap(sensors.TemperatureCelsius), FanRPM: cloneStringFloatMap(sensors.FanRPM), Additional: cloneStringFloatMap(sensors.Additional), + GPU: convertUnifiedHostGPU(sensors.GPU), SMART: convertUnifiedHostSMART(sensors.SMART), }, lastSeen) } +func convertUnifiedHostGPU(gpus []unifiedresources.HostGPUSensor) []models.HostGPUSensor { + if len(gpus) == 0 { + return nil + } + result := make([]models.HostGPUSensor, len(gpus)) + for i, gpu := range gpus { + result[i] = models.HostGPUSensor{ + ID: gpu.ID, + Name: gpu.Name, + TemperatureCelsius: cloneFloat64Ptr(gpu.TemperatureCelsius), + UtilizationPercent: cloneFloat64Ptr(gpu.UtilizationPercent), + MemoryUsedBytes: cloneInt64Ptr(gpu.MemoryUsedBytes), + MemoryTotalBytes: cloneInt64Ptr(gpu.MemoryTotalBytes), + } + } + return result +} + func convertUnifiedHostSMART(smart []unifiedresources.HostSMARTMeta) []models.HostDiskSMART { if len(smart) == 0 { return nil diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go index 68528235a..53147043d 100644 --- a/internal/monitoring/monitor.go +++ b/internal/monitoring/monitor.go @@ -3383,6 +3383,19 @@ func hostSensorsFromReadStateView(sensors *unifiedresources.HostSensorMeta) mode out.Additional[k] = v } } + if len(sensors.GPU) > 0 { + out.GPU = make([]models.HostGPUSensor, len(sensors.GPU)) + for i, gpu := range sensors.GPU { + out.GPU[i] = models.HostGPUSensor{ + ID: gpu.ID, + Name: gpu.Name, + TemperatureCelsius: cloneFloat64Ptr(gpu.TemperatureCelsius), + UtilizationPercent: cloneFloat64Ptr(gpu.UtilizationPercent), + MemoryUsedBytes: cloneInt64Ptr(gpu.MemoryUsedBytes), + MemoryTotalBytes: cloneInt64Ptr(gpu.MemoryTotalBytes), + } + } + } if sensors.ThermalState != nil { out.ThermalState = hostThermalStateFromReadStateView(sensors.ThermalState) } diff --git a/internal/monitoring/monitor_agents.go b/internal/monitoring/monitor_agents.go index a4e372a86..e53474fb5 100644 --- a/internal/monitoring/monitor_agents.go +++ b/internal/monitoring/monitor_agents.go @@ -1937,6 +1937,7 @@ func (m *Monitor) ApplyHostReport(report agentshost.Report, tokenRecord *config. TemperatureCelsius: cloneStringFloatMap(report.Sensors.TemperatureCelsius), FanRPM: cloneStringFloatMap(report.Sensors.FanRPM), Additional: cloneStringFloatMap(report.Sensors.Additional), + GPU: convertAgentGPUToModels(report.Sensors.GPU), ThermalState: convertAgentThermalStateToModels(report.Sensors.ThermalState), SMART: convertAgentSMARTToModels(report.Sensors.SMART), }, diff --git a/internal/monitoring/monitor_helpers.go b/internal/monitoring/monitor_helpers.go index 2b33ece14..7461d0917 100644 --- a/internal/monitoring/monitor_helpers.go +++ b/internal/monitoring/monitor_helpers.go @@ -216,6 +216,22 @@ func cloneIntPtr(src *int) *int { return &out } +func cloneFloat64Ptr(src *float64) *float64 { + if src == nil { + return nil + } + out := *src + return &out +} + +func cloneInt64Ptr(src *int64) *int64 { + if src == nil { + return nil + } + out := *src + return &out +} + func cloneStringMap(src map[string]string) map[string]string { if len(src) == 0 { return nil @@ -614,6 +630,24 @@ func convertAgentSMARTToModels(smart []agentshost.DiskSMART) []models.HostDiskSM return result } +func convertAgentGPUToModels(gpus []agentshost.GPUSensor) []models.HostGPUSensor { + if len(gpus) == 0 { + return nil + } + result := make([]models.HostGPUSensor, len(gpus)) + for i, gpu := range gpus { + result[i] = models.HostGPUSensor{ + ID: gpu.ID, + Name: gpu.Name, + TemperatureCelsius: cloneFloat64Ptr(gpu.TemperatureCelsius), + UtilizationPercent: cloneFloat64Ptr(gpu.UtilizationPercent), + MemoryUsedBytes: cloneInt64Ptr(gpu.MemoryUsedBytes), + MemoryTotalBytes: cloneInt64Ptr(gpu.MemoryTotalBytes), + } + } + return result +} + func convertAgentThermalStateToModels(src *agentshost.ThermalState) *models.HostThermalState { if src == nil { return nil diff --git a/internal/monitoring/monitor_host_agents_test.go b/internal/monitoring/monitor_host_agents_test.go index 2c021c04c..b227953fe 100644 --- a/internal/monitoring/monitor_host_agents_test.go +++ b/internal/monitoring/monitor_host_agents_test.go @@ -11,6 +11,7 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/mock" "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/storagehealth" + "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker" agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host" "github.com/rcourtman/pulse-go-rewrite/pkg/metrics" @@ -1670,6 +1671,91 @@ func TestApplyHostReportPersistsAgentTemperatureMetric(t *testing.T) { } } +func TestApplyHostReportPreservesGPUSensorSummary(t *testing.T) { + monitor := &Monitor{ + state: models.NewState(), + alertManager: alerts.NewManager(), + hostTokenBindings: make(map[string]string), + config: &config.Config{}, + rateTracker: NewRateTracker(), + } + t.Cleanup(func() { monitor.alertManager.Stop() }) + + temperature := 63.0 + utilization := 7.0 + usedBytes := int64(2 * 1024 * 1024 * 1024) + totalBytes := int64(48 * 1024 * 1024 * 1024) + report := agentshost.Report{ + Agent: agentshost.AgentInfo{ + ID: "agent-gpu", + Version: "1.0.0", + IntervalSeconds: 30, + }, + Host: agentshost.HostInfo{ + ID: "machine-gpu", + Hostname: "gpu-node", + MachineID: "machine-gpu", + }, + Metrics: agentshost.Metrics{ + Memory: agentshost.MemoryMetric{TotalBytes: 1024, UsedBytes: 512, FreeBytes: 512, Usage: 50}, + }, + Sensors: agentshost.Sensors{ + TemperatureCelsius: map[string]float64{"gpu_nvidia_0": temperature}, + GPU: []agentshost.GPUSensor{ + { + ID: "0", + Name: "NVIDIA RTX A6000", + TemperatureCelsius: &temperature, + UtilizationPercent: &utilization, + MemoryUsedBytes: &usedBytes, + MemoryTotalBytes: &totalBytes, + }, + }, + }, + Timestamp: time.Now().UTC(), + } + + host, err := monitor.ApplyHostReport(report, nil) + if err != nil { + t.Fatalf("ApplyHostReport: %v", err) + } + if len(host.Sensors.GPU) != 1 { + t.Fatalf("host GPU sensors = %+v, want one sensor", host.Sensors.GPU) + } + gpu := host.Sensors.GPU[0] + if gpu.ID != "0" || gpu.Name != "NVIDIA RTX A6000" { + t.Fatalf("unexpected GPU identity: %+v", gpu) + } + if gpu.TemperatureCelsius == nil || *gpu.TemperatureCelsius != temperature { + t.Fatalf("GPU temperature = %#v, want %.1f", gpu.TemperatureCelsius, temperature) + } + if gpu.UtilizationPercent == nil || *gpu.UtilizationPercent != utilization { + t.Fatalf("GPU utilization = %#v, want %.1f", gpu.UtilizationPercent, utilization) + } + if gpu.MemoryUsedBytes == nil || *gpu.MemoryUsedBytes != usedBytes { + t.Fatalf("GPU used memory = %#v, want %d", gpu.MemoryUsedBytes, usedBytes) + } + if host.Sensors.TemperatureCelsius["gpu_nvidia_0"] != temperature { + t.Fatalf("legacy GPU temperature compatibility = %+v, want %.1f", host.Sensors.TemperatureCelsius, temperature) + } + + projected := hostSensorsFromReadStateView(&unifiedresources.HostSensorMeta{ + GPU: []unifiedresources.HostGPUSensor{ + { + ID: "0", + Name: "NVIDIA RTX A6000", + TemperatureCelsius: &temperature, + UtilizationPercent: &utilization, + MemoryUsedBytes: &usedBytes, + MemoryTotalBytes: &totalBytes, + }, + }, + }) + if len(projected.GPU) != 1 || projected.GPU[0].MemoryTotalBytes == nil || *projected.GPU[0].MemoryTotalBytes != totalBytes { + t.Fatalf("read-state GPU projection = %+v, want total VRAM %d", projected.GPU, totalBytes) + } +} + func TestApplyHostReportPersistsSMARTMetricsForAgentDisksWithFallbackID(t *testing.T) { t.Helper() diff --git a/internal/unifiedresources/adapters.go b/internal/unifiedresources/adapters.go index 2f56b5ed6..677bde24c 100644 --- a/internal/unifiedresources/adapters.go +++ b/internal/unifiedresources/adapters.go @@ -168,7 +168,7 @@ func resourceFromHost(host models.Host) (Resource, ResourceIdentity) { storageAssessments := make([]storagehealth.Assessment, 0, len(host.RAID)+1) // Populate sensors - if len(host.Sensors.TemperatureCelsius) > 0 || len(host.Sensors.FanRPM) > 0 || len(host.Sensors.Additional) > 0 || host.Sensors.ThermalState != nil || len(host.Sensors.SMART) > 0 { + if len(host.Sensors.TemperatureCelsius) > 0 || len(host.Sensors.FanRPM) > 0 || len(host.Sensors.Additional) > 0 || len(host.Sensors.GPU) > 0 || host.Sensors.ThermalState != nil || len(host.Sensors.SMART) > 0 { sensorMeta := &HostSensorMeta{} if len(host.Sensors.TemperatureCelsius) > 0 { sensorMeta.TemperatureCelsius = make(map[string]float64, len(host.Sensors.TemperatureCelsius)) @@ -188,6 +188,19 @@ func resourceFromHost(host models.Host) (Resource, ResourceIdentity) { sensorMeta.Additional[k] = v } } + if len(host.Sensors.GPU) > 0 { + sensorMeta.GPU = make([]HostGPUSensor, len(host.Sensors.GPU)) + for i, gpu := range host.Sensors.GPU { + sensorMeta.GPU[i] = HostGPUSensor{ + ID: gpu.ID, + Name: gpu.Name, + TemperatureCelsius: cloneFloat64Ptr(gpu.TemperatureCelsius), + UtilizationPercent: cloneFloat64Ptr(gpu.UtilizationPercent), + MemoryUsedBytes: cloneInt64Ptr(gpu.MemoryUsedBytes), + MemoryTotalBytes: cloneInt64Ptr(gpu.MemoryTotalBytes), + } + } + } if host.Sensors.ThermalState != nil { sensorMeta.ThermalState = hostThermalStateToMeta(host.Sensors.ThermalState) } diff --git a/internal/unifiedresources/adapters_test.go b/internal/unifiedresources/adapters_test.go index 9ab0c90dc..e20f4e7fe 100644 --- a/internal/unifiedresources/adapters_test.go +++ b/internal/unifiedresources/adapters_test.go @@ -555,6 +555,53 @@ func TestResourceFromHostProjectsPressureOnlyThermalState(t *testing.T) { } } +func TestResourceFromHostProjectsTypedGPUSensors(t *testing.T) { + temperature := 63.0 + utilization := 18.0 + usedBytes := int64(3 * 1024 * 1024 * 1024) + totalBytes := int64(24 * 1024 * 1024 * 1024) + host := models.Host{ + ID: "gpu-host", + Hostname: "gpu-node", + Platform: "linux", + Status: "online", + Sensors: models.HostSensorSummary{ + TemperatureCelsius: map[string]float64{"gpu_nvidia_0": temperature}, + GPU: []models.HostGPUSensor{ + { + ID: "0", + Name: "NVIDIA RTX A6000", + TemperatureCelsius: &temperature, + UtilizationPercent: &utilization, + MemoryUsedBytes: &usedBytes, + MemoryTotalBytes: &totalBytes, + }, + }, + }, + } + + resource, _ := resourceFromHost(host) + if resource.Agent == nil || resource.Agent.Sensors == nil { + t.Fatalf("expected agent sensor payload, got %+v", resource.Agent) + } + if len(resource.Agent.Sensors.GPU) != 1 { + t.Fatalf("GPU sensors = %+v, want one sensor", resource.Agent.Sensors.GPU) + } + gpu := resource.Agent.Sensors.GPU[0] + if gpu.ID != "0" || gpu.Name != "NVIDIA RTX A6000" { + t.Fatalf("unexpected GPU identity: %+v", gpu) + } + if gpu.UtilizationPercent == nil || *gpu.UtilizationPercent != utilization { + t.Fatalf("GPU utilization = %#v, want %.1f", gpu.UtilizationPercent, utilization) + } + if gpu.MemoryTotalBytes == nil || *gpu.MemoryTotalBytes != totalBytes { + t.Fatalf("GPU memory total = %#v, want %d", gpu.MemoryTotalBytes, totalBytes) + } + if resource.Agent.Sensors.TemperatureCelsius["gpu_nvidia_0"] != temperature { + t.Fatalf("legacy GPU temperature = %+v, want %.1f", resource.Agent.Sensors.TemperatureCelsius, temperature) + } +} + func TestResourceFromVMPreservesProxmoxPool(t *testing.T) { vm := models.VM{ ID: "cluster-a:pve-a:101", diff --git a/internal/unifiedresources/clone.go b/internal/unifiedresources/clone.go index 2301ed6dc..b23786908 100644 --- a/internal/unifiedresources/clone.go +++ b/internal/unifiedresources/clone.go @@ -689,11 +689,30 @@ func cloneHostSensorMeta(in *HostSensorMeta) *HostSensorMeta { out.TemperatureCelsius = cloneStringFloat64Map(in.TemperatureCelsius) out.FanRPM = cloneStringFloat64Map(in.FanRPM) out.Additional = cloneStringFloat64Map(in.Additional) + out.GPU = cloneHostGPUSensors(in.GPU) out.ThermalState = cloneHostThermalState(in.ThermalState) out.SMART = cloneHostSMARTMetaSlice(in.SMART) return &out } +func cloneHostGPUSensors(in []HostGPUSensor) []HostGPUSensor { + if len(in) == 0 { + return nil + } + out := make([]HostGPUSensor, len(in)) + for i, gpu := range in { + out[i] = HostGPUSensor{ + ID: gpu.ID, + Name: gpu.Name, + TemperatureCelsius: cloneFloat64Ptr(gpu.TemperatureCelsius), + UtilizationPercent: cloneFloat64Ptr(gpu.UtilizationPercent), + MemoryUsedBytes: cloneInt64Ptr(gpu.MemoryUsedBytes), + MemoryTotalBytes: cloneInt64Ptr(gpu.MemoryTotalBytes), + } + } + return out +} + func cloneHostThermalState(in *HostThermalState) *HostThermalState { if in == nil { return nil diff --git a/internal/unifiedresources/code_standards_test.go b/internal/unifiedresources/code_standards_test.go index 657d33f3f..1ef35d292 100644 --- a/internal/unifiedresources/code_standards_test.go +++ b/internal/unifiedresources/code_standards_test.go @@ -2179,6 +2179,52 @@ func TestBroadcastStateUsesSharedCanonicalResourceContract(t *testing.T) { } } +func TestCloneHostSensorMetaKeepsGPUSensorsIsolated(t *testing.T) { + temperature := 63.0 + utilization := 42.0 + usedBytes := int64(4 * 1024 * 1024 * 1024) + totalBytes := int64(16 * 1024 * 1024 * 1024) + source := &HostSensorMeta{ + GPU: []HostGPUSensor{ + { + ID: "0", + Name: "NVIDIA RTX A6000", + TemperatureCelsius: &temperature, + UtilizationPercent: &utilization, + MemoryUsedBytes: &usedBytes, + MemoryTotalBytes: &totalBytes, + }, + }, + } + + clone := cloneHostSensorMeta(source) + if clone == nil || len(clone.GPU) != 1 { + t.Fatalf("GPU clone = %+v, want one sensor", clone) + } + temperature = 10 + utilization = 1 + usedBytes = 1 + totalBytes = 2 + source.GPU[0].Name = "mutated" + + gpu := clone.GPU[0] + if gpu.Name != "NVIDIA RTX A6000" { + t.Fatalf("GPU name clone = %q, want original name", gpu.Name) + } + if gpu.TemperatureCelsius == nil || *gpu.TemperatureCelsius != 63 { + t.Fatalf("GPU temperature clone = %#v, want 63", gpu.TemperatureCelsius) + } + if gpu.UtilizationPercent == nil || *gpu.UtilizationPercent != 42 { + t.Fatalf("GPU utilization clone = %#v, want 42", gpu.UtilizationPercent) + } + if gpu.MemoryUsedBytes == nil || *gpu.MemoryUsedBytes != int64(4*1024*1024*1024) { + t.Fatalf("GPU used memory clone = %#v, want 4 GiB", gpu.MemoryUsedBytes) + } + if gpu.MemoryTotalBytes == nil || *gpu.MemoryTotalBytes != int64(16*1024*1024*1024) { + t.Fatalf("GPU total memory clone = %#v, want 16 GiB", gpu.MemoryTotalBytes) + } +} + func TestCloneVMwareDataKeepsNestedRuntimeDetailsIsolated(t *testing.T) { repoRoot := filepath.Join("..", "..") clonePath := filepath.Join(repoRoot, "internal", "unifiedresources", "clone.go") diff --git a/internal/unifiedresources/types.go b/internal/unifiedresources/types.go index 35d4d0643..a1ebb8276 100644 --- a/internal/unifiedresources/types.go +++ b/internal/unifiedresources/types.go @@ -535,10 +535,20 @@ type HostSensorMeta struct { TemperatureCelsius map[string]float64 `json:"temperatureCelsius,omitempty"` FanRPM map[string]float64 `json:"fanRpm,omitempty"` Additional map[string]float64 `json:"additional,omitempty"` + GPU []HostGPUSensor `json:"gpu,omitempty"` ThermalState *HostThermalState `json:"thermalState,omitempty"` SMART []HostSMARTMeta `json:"smart,omitempty"` } +type HostGPUSensor struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + TemperatureCelsius *float64 `json:"temperatureCelsius,omitempty"` + UtilizationPercent *float64 `json:"utilizationPercent,omitempty"` + MemoryUsedBytes *int64 `json:"memoryUsedBytes,omitempty"` + MemoryTotalBytes *int64 `json:"memoryTotalBytes,omitempty"` +} + type HostThermalState struct { Source string `json:"source,omitempty"` Pressure string `json:"pressure,omitempty"` diff --git a/pkg/agents/host/report.go b/pkg/agents/host/report.go index 7173613a8..8b5ae61b7 100644 --- a/pkg/agents/host/report.go +++ b/pkg/agents/host/report.go @@ -119,10 +119,21 @@ type Sensors struct { FanRPM map[string]float64 `json:"fanRpm,omitempty"` PowerWatts map[string]float64 `json:"powerWatts,omitempty"` // Power consumption (e.g., cpu_package, dram) Additional map[string]float64 `json:"additional,omitempty"` + GPU []GPUSensor `json:"gpu,omitempty"` ThermalState *ThermalState `json:"thermalState,omitempty"` SMART []DiskSMART `json:"smart,omitempty"` // S.M.A.R.T. disk data } +// GPUSensor captures direct GPU telemetry reported by a local host agent. +type GPUSensor struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + TemperatureCelsius *float64 `json:"temperatureCelsius,omitempty"` + UtilizationPercent *float64 `json:"utilizationPercent,omitempty"` + MemoryUsedBytes *int64 `json:"memoryUsedBytes,omitempty"` + MemoryTotalBytes *int64 `json:"memoryTotalBytes,omitempty"` +} + // ThermalState captures OS-level thermal pressure that is not a direct // Celsius temperature reading. macOS exposes this kind of signal more // reliably than raw sensor temperatures. diff --git a/pkg/agents/host/report_test.go b/pkg/agents/host/report_test.go index f5846afc1..44c71ee84 100644 --- a/pkg/agents/host/report_test.go +++ b/pkg/agents/host/report_test.go @@ -6,6 +6,14 @@ import ( "time" ) +func float64Ptr(value float64) *float64 { + return &value +} + +func int64Ptr(value int64) *int64 { + return &value +} + func TestReport_JSONMarshal(t *testing.T) { report := Report{ Agent: AgentInfo{ @@ -197,6 +205,15 @@ func TestSensors_Fields(t *testing.T) { Additional: map[string]float64{ "voltage": 12.0, }, + GPU: []GPUSensor{ + { + ID: "0", + Name: "NVIDIA RTX A6000", + UtilizationPercent: float64Ptr(42), + MemoryUsedBytes: int64Ptr(8 * 1024 * 1024 * 1024), + MemoryTotalBytes: int64Ptr(48 * 1024 * 1024 * 1024), + }, + }, ThermalState: &ThermalState{ Source: "pmset", Pressure: "nominal", @@ -214,6 +231,12 @@ func TestSensors_Fields(t *testing.T) { if sensors.FanRPM["cpu_fan"] != 1200.0 { t.Errorf("cpu_fan RPM = %f, want 1200.0", sensors.FanRPM["cpu_fan"]) } + if len(sensors.GPU) != 1 { + t.Fatalf("GPU count = %d, want 1", len(sensors.GPU)) + } + if sensors.GPU[0].UtilizationPercent == nil || *sensors.GPU[0].UtilizationPercent != 42 { + t.Fatalf("GPU utilization = %#v, want 42", sensors.GPU[0].UtilizationPercent) + } if sensors.ThermalState == nil || sensors.ThermalState.Pressure != "nominal" { t.Fatalf("ThermalState = %#v, want nominal state", sensors.ThermalState) }