From fe3c0f3ee3069498274e58ace9ba1d5f135fb957 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Tue, 30 Jun 2026 09:11:18 +0100 Subject: [PATCH] Collect NVIDIA GPU temperatures --- .../v6/internal/subsystems/agent-lifecycle.md | 8 + internal/hostagent/agent.go | 7 +- internal/hostagent/agent_metrics_test.go | 58 +++++++ internal/hostagent/agent_sensors_test.go | 121 +++++++++++++ internal/hostagent/nvidia_smi.go | 164 ++++++++++++++++++ 5 files changed, 355 insertions(+), 3 deletions(-) create mode 100644 internal/hostagent/nvidia_smi.go diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index 700d3061a..27765057c 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -876,6 +876,14 @@ surface and no new `internal/api/` lifecycle handler. it must report Darwin `pmset` thermal and performance pressure as `sensors.thermalState` instead of inventing Celsius readings from unavailable Apple silicon sensor values. + Linux NVIDIA GPU temperature telemetry belongs in that same host-agent + sensor contract: the runtime may supplement `lm-sensors` with a bounded + `nvidia-smi` temperature query and may use that query as a best-effort + fallback when `lm-sensors` is unavailable. The report must map only direct + `temperature.gpu` readings into existing `sensors.temperatureCelsius` + `gpu_nvidia_` keys; it must not infer lifecycle health, command + authority, enrollment state, or GPU workload/process inventory from + `nvidia-smi` output. Runtime RAID collection uses `/proc/mdstat` as the canonical discovery baseline for Linux md arrays. `mdadm --detail` may enrich level, state, member, UUID, and rebuild fields when available, but missing or failing diff --git a/internal/hostagent/agent.go b/internal/hostagent/agent.go index fd74b07f1..3b1cfc21a 100644 --- a/internal/hostagent/agent.go +++ b/internal/hostagent/agent.go @@ -1038,22 +1038,23 @@ func (a *Agent) collectTemperatures(ctx context.Context) agentshost.Sensors { jsonOutput, err := a.collector.SensorsLocal(ctx) if err != nil { a.logger.Debug().Err(err).Msg("Failed to collect sensor data (lm-sensors may not be installed)") - return agentshost.Sensors{} + return a.collectNVIDIATemperatureSensors(ctx) } // Parse the sensor output tempData, err := a.collector.SensorsParse(jsonOutput) if err != nil { a.logger.Debug().Err(err).Msg("Failed to parse sensor data") - return agentshost.Sensors{} + return a.collectNVIDIATemperatureSensors(ctx) } if !tempData.Available { a.logger.Debug().Msg("No temperature sensors available on this system") - return agentshost.Sensors{} + return a.collectNVIDIATemperatureSensors(ctx) } result := convertTemperatureDataToSensors(tempData) + a.mergeNVIDIATemperatures(ctx, &result) // Collect power consumption data (Intel RAPL, etc.) if powerData, err := a.collector.SensorsPower(ctx); err == nil && powerData != nil && powerData.Available { diff --git a/internal/hostagent/agent_metrics_test.go b/internal/hostagent/agent_metrics_test.go index 30cfd9ec1..8e58cd256 100644 --- a/internal/hostagent/agent_metrics_test.go +++ b/internal/hostagent/agent_metrics_test.go @@ -449,6 +449,64 @@ func TestBuildReportIncludesDarwinThermalState(t *testing.T) { } } +func TestBuildReportIncludesNVIDIASMITemperaturesWhenLMSensorsUnavailable(t *testing.T) { + mc := &mockCollector{ + goos: "linux", + nowFn: func() time.Time { return time.Date(2026, 6, 30, 8, 0, 0, 0, time.UTC) }, + hostInfoFn: func(ctx context.Context) (*gohost.InfoStat, error) { + return &gohost.InfoStat{ + Hostname: "gpu-node", + OS: "linux", + Platform: "ubuntu", + HostID: "gpu-node-id", + }, nil + }, + hostUptimeFn: func(context.Context) (uint64, error) { + return 7200, nil + }, + metricsFn: func(context.Context, []string) (hostmetrics.Snapshot, error) { + return hostmetrics.Snapshot{}, nil + }, + sensorsLocalFn: func(context.Context) (string, error) { + return "", errors.New("lm-sensors unavailable") + }, + lookPathFn: func(file string) (string, error) { + if file == "nvidia-smi" { + return "/usr/bin/nvidia-smi", nil + } + return "", os.ErrNotExist + }, + commandCombinedOutputFn: func(_ context.Context, name string, arg ...string) (string, error) { + if name != "/usr/bin/nvidia-smi" { + t.Fatalf("command name = %q, want /usr/bin/nvidia-smi", name) + } + if len(arg) != 2 || arg[0] != "--query-gpu=index,name,temperature.gpu" || arg[1] != "--format=csv,noheader,nounits" { + t.Fatalf("command args = %#v, want NVIDIA temperature query", arg) + } + return "0, NVIDIA GeForce RTX 4090, 63\n", nil + }, + } + + agent, err := New(Config{ + AgentID: "gpu-agent", + APIToken: "token", + LogLevel: -1, + Collector: mc, + }) + if err != nil { + t.Fatalf("New() failed: %v", err) + } + + report, err := agent.buildReport(context.Background()) + if err != nil { + t.Fatalf("buildReport failed: %v", err) + } + + if report.Sensors.TemperatureCelsius["gpu_nvidia_0"] != 63 { + t.Fatalf("NVIDIA GPU temp = %v, want 63", report.Sensors.TemperatureCelsius["gpu_nvidia_0"]) + } +} + func TestBuildReportUsesResolvedNASOSIdentity(t *testing.T) { fixedTime := time.Date(2026, time.April, 15, 12, 0, 0, 0, time.UTC) diff --git a/internal/hostagent/agent_sensors_test.go b/internal/hostagent/agent_sensors_test.go index 15148ed13..2b71892ed 100644 --- a/internal/hostagent/agent_sensors_test.go +++ b/internal/hostagent/agent_sensors_test.go @@ -3,6 +3,8 @@ package hostagent import ( "context" "errors" + "os" + "os/exec" "testing" "github.com/rcourtman/pulse-go-rewrite/internal/sensors" @@ -55,6 +57,70 @@ func TestAgent_collectTemperatures_MapsKeys(t *testing.T) { } } +func TestAgent_collectTemperatures_MergesNVIDIASMITemperatures(t *testing.T) { + mc := &mockCollector{ + goos: "linux", + sensorsLocalFn: func(context.Context) (string, error) { return "{}", nil }, + sensorsParseFn: func(string) (*sensors.TemperatureData, error) { + return &sensors.TemperatureData{ + Available: true, + CPUPackage: 55.5, + }, nil + }, + lookPathFn: func(file string) (string, error) { + if file != "nvidia-smi" { + t.Fatalf("look path file = %q, want nvidia-smi", file) + } + return "/usr/bin/nvidia-smi", nil + }, + commandCombinedOutputFn: func(_ context.Context, name string, arg ...string) (string, error) { + if name != "/usr/bin/nvidia-smi" { + t.Fatalf("command name = %q, want /usr/bin/nvidia-smi", name) + } + if len(arg) != 2 || arg[0] != "--query-gpu=index,name,temperature.gpu" || arg[1] != "--format=csv,noheader,nounits" { + t.Fatalf("command args = %#v, want NVIDIA temperature query", arg) + } + return "0, NVIDIA GeForce RTX 4090, 61\n", nil + }, + } + + a := &Agent{logger: zerolog.Nop(), collector: mc} + + got := a.collectTemperatures(context.Background()) + if got.TemperatureCelsius["cpu_package"] != 55.5 { + t.Fatalf("cpu package temp = %v, want 55.5", got.TemperatureCelsius["cpu_package"]) + } + if got.TemperatureCelsius["gpu_nvidia_0"] != 61 { + t.Fatalf("NVIDIA GPU temp = %v, want 61", got.TemperatureCelsius["gpu_nvidia_0"]) + } +} + +func TestAgent_collectTemperatures_UsesNVIDIASMIWhenLMSensorsUnavailable(t *testing.T) { + mc := &mockCollector{ + goos: "linux", + sensorsLocalFn: func(context.Context) (string, error) { return "", errors.New("no sensors") }, + lookPathFn: func(file string) (string, error) { + if file != "nvidia-smi" { + t.Fatalf("look path file = %q, want nvidia-smi", file) + } + return "/usr/bin/nvidia-smi", nil + }, + commandCombinedOutputFn: func(context.Context, string, ...string) (string, error) { + return "0, NVIDIA RTX A6000, 58\n", nil + }, + } + + a := &Agent{logger: zerolog.Nop(), collector: mc} + + got := a.collectTemperatures(context.Background()) + if len(got.TemperatureCelsius) != 1 { + t.Fatalf("temperature keys = %d, want 1; got %v", len(got.TemperatureCelsius), got.TemperatureCelsius) + } + if got.TemperatureCelsius["gpu_nvidia_0"] != 58 { + t.Fatalf("NVIDIA GPU temp = %v, want 58", got.TemperatureCelsius["gpu_nvidia_0"]) + } +} + func TestAgent_collectTemperatures_BestEffortFailuresReturnEmpty(t *testing.T) { mc := &mockCollector{goos: "linux"} a := &Agent{logger: zerolog.Nop(), collector: mc} @@ -76,6 +142,61 @@ func TestAgent_collectTemperatures_BestEffortFailuresReturnEmpty(t *testing.T) { } } +func TestParseNVIDIASMITemperatures(t *testing.T) { + input := `0, NVIDIA GeForce RTX 4090, 47 +1, "NVIDIA, RTX A6000", N/A +2, NVIDIA Tesla T4, 58 C +0000:65:00.0, NVIDIA H100, 64` + + got, err := parseNVIDIASMITemperatures(input) + if err != nil { + t.Fatalf("parseNVIDIASMITemperatures returned error: %v", err) + } + + want := map[string]float64{ + "gpu_nvidia_0": 47, + "gpu_nvidia_2": 58, + "gpu_nvidia_0000_65_00_0": 64, + } + if len(got) != len(want) { + t.Fatalf("temperature keys = %d, want %d; got %v", len(got), len(want), got) + } + for key, wantTemp := range want { + if gotTemp := got[key]; gotTemp != wantTemp { + t.Fatalf("%s = %v, want %v", key, gotTemp, wantTemp) + } + } +} + +func TestParseNVIDIASMITemperatures_NoUsableRows(t *testing.T) { + got, err := parseNVIDIASMITemperatures("0, NVIDIA RTX, N/A\nbad row\n") + if err != nil { + t.Fatalf("parseNVIDIASMITemperatures returned error: %v", err) + } + if got != nil { + t.Fatalf("expected nil temps, got %v", got) + } +} + +func TestAgent_queryNVIDIASMITemperatures_CommandNotInstalled(t *testing.T) { + for _, lookPathErr := range []error{os.ErrNotExist, exec.ErrNotFound} { + mc := &mockCollector{ + lookPathFn: func(string) (string, error) { + return "", lookPathErr + }, + } + a := &Agent{logger: zerolog.Nop(), collector: mc} + + got, err := a.queryNVIDIASMITemperatures(context.Background()) + if err != nil { + t.Fatalf("queryNVIDIASMITemperatures returned error for %v: %v", lookPathErr, err) + } + if got != nil { + t.Fatalf("expected nil temps for %v, got %v", lookPathErr, got) + } + } +} + func TestAgent_collectTemperatures_SkipsUnsupportedOS(t *testing.T) { mc := &mockCollector{goos: "windows"} a := &Agent{logger: zerolog.Nop(), collector: mc} diff --git a/internal/hostagent/nvidia_smi.go b/internal/hostagent/nvidia_smi.go new file mode 100644 index 000000000..52c750dfd --- /dev/null +++ b/internal/hostagent/nvidia_smi.go @@ -0,0 +1,164 @@ +package hostagent + +import ( + "context" + "encoding/csv" + "errors" + "fmt" + "os" + "os/exec" + "strconv" + "strings" + "time" + + agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host" +) + +const nvidiaSMITemperatureTimeout = 5 * time.Second + +var nvidiaSMITemperatureArgs = []string{ + "--query-gpu=index,name,temperature.gpu", + "--format=csv,noheader,nounits", +} + +func (a *Agent) collectNVIDIATemperatureSensors(ctx context.Context) agentshost.Sensors { + var result agentshost.Sensors + a.mergeNVIDIATemperatures(ctx, &result) + if len(result.TemperatureCelsius) == 0 { + return agentshost.Sensors{} + } + return result +} + +func (a *Agent) mergeNVIDIATemperatures(ctx context.Context, result *agentshost.Sensors) { + temps, err := a.queryNVIDIASMITemperatures(ctx) + if err != nil { + a.logger.Debug().Err(err).Msg("Failed to collect NVIDIA GPU temperatures") + return + } + if len(temps) == 0 { + return + } + + if result.TemperatureCelsius == nil { + result.TemperatureCelsius = make(map[string]float64, len(temps)) + } + for key, temp := range temps { + result.TemperatureCelsius[key] = temp + } +} + +func (a *Agent) queryNVIDIASMITemperatures(ctx context.Context) (map[string]float64, error) { + path, err := a.collector.LookPath("nvidia-smi") + if err != nil { + if errors.Is(err, exec.ErrNotFound) || os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("locate nvidia-smi: %w", err) + } + + cmdCtx, cancel := context.WithTimeout(ctx, nvidiaSMITemperatureTimeout) + defer cancel() + + output, err := a.collector.CommandCombinedOutput(cmdCtx, path, nvidiaSMITemperatureArgs...) + if err != nil { + return nil, fmt.Errorf("execute nvidia-smi temperature query: %w", err) + } + + temps, err := parseNVIDIASMITemperatures(output) + if err != nil { + return nil, fmt.Errorf("parse nvidia-smi temperature output: %w", err) + } + return temps, nil +} + +func parseNVIDIASMITemperatures(output string) (map[string]float64, error) { + output = strings.TrimSpace(output) + if output == "" { + return nil, nil + } + + reader := csv.NewReader(strings.NewReader(output)) + reader.FieldsPerRecord = -1 + reader.TrimLeadingSpace = true + + records, err := reader.ReadAll() + if err != nil { + return nil, err + } + + temps := make(map[string]float64) + for row, record := range records { + if len(record) < 3 { + continue + } + + temp, ok := parseNVIDIASMITemperatureValue(record[2]) + if !ok { + continue + } + + index := normalizeNVIDIASMIIndex(record[0], row) + key := "gpu_nvidia_" + index + if _, exists := temps[key]; exists { + key = fmt.Sprintf("gpu_nvidia_%d", row) + } + temps[key] = temp + } + + if len(temps) == 0 { + return nil, nil + } + return temps, nil +} + +func parseNVIDIASMITemperatureValue(value string) (float64, bool) { + value = strings.TrimSpace(value) + if value == "" { + return 0, false + } + + lower := strings.ToLower(value) + if lower == "n/a" || lower == "na" || strings.Contains(lower, "not supported") || strings.Contains(lower, "unknown") { + return 0, false + } + + fields := strings.Fields(value) + if len(fields) > 0 { + value = fields[0] + } + value = strings.TrimSuffix(strings.TrimSuffix(value, "C"), "c") + + temp, err := strconv.ParseFloat(value, 64) + if err != nil || temp <= 0 { + return 0, false + } + return temp, true +} + +func normalizeNVIDIASMIIndex(value string, fallback int) string { + value = strings.ToLower(strings.TrimSpace(value)) + if value == "" { + return strconv.Itoa(fallback) + } + + var builder strings.Builder + lastUnderscore := false + for _, r := range value { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') { + builder.WriteRune(r) + lastUnderscore = false + continue + } + if builder.Len() > 0 && !lastUnderscore { + builder.WriteByte('_') + lastUnderscore = true + } + } + + normalized := strings.Trim(builder.String(), "_") + if normalized == "" { + return strconv.Itoa(fallback) + } + return normalized +}