Collect NVIDIA GPU temperatures

docs/release-control/v6/internal/subsystems/agent-lifecycle.md

@@ -876,6 +876,14 @@ surface and no new `internal/api/` lifecycle handler.
    it must report Darwin `pmset` thermal and performance pressure as
    `sensors.thermalState` instead of inventing Celsius readings from unavailable
    Apple silicon sensor values.
+   Linux NVIDIA GPU temperature telemetry belongs in that same host-agent
+   sensor contract: the runtime may supplement `lm-sensors` with a bounded
+   `nvidia-smi` temperature query and may use that query as a best-effort
+   fallback when `lm-sensors` is unavailable. The report must map only direct
+   `temperature.gpu` readings into existing `sensors.temperatureCelsius`
+   `gpu_nvidia_<index>` keys; it must not infer lifecycle health, command
+   authority, enrollment state, or GPU workload/process inventory from
+   `nvidia-smi` output.
    Runtime RAID collection uses `/proc/mdstat` as the canonical discovery
    baseline for Linux md arrays. `mdadm --detail` may enrich level, state,
    member, UUID, and rebuild fields when available, but missing or failing

internal/hostagent/agent.go

@@ -1038,22 +1038,23 @@ func (a *Agent) collectTemperatures(ctx context.Context) agentshost.Sensors {
 	jsonOutput, err := a.collector.SensorsLocal(ctx)
 	if err != nil {
 		a.logger.Debug().Err(err).Msg("Failed to collect sensor data (lm-sensors may not be installed)")
-		return agentshost.Sensors{}
+		return a.collectNVIDIATemperatureSensors(ctx)
 	}
 
 	// Parse the sensor output
 	tempData, err := a.collector.SensorsParse(jsonOutput)
 	if err != nil {
 		a.logger.Debug().Err(err).Msg("Failed to parse sensor data")
-		return agentshost.Sensors{}
+		return a.collectNVIDIATemperatureSensors(ctx)
 	}
 
 	if !tempData.Available {
 		a.logger.Debug().Msg("No temperature sensors available on this system")
-		return agentshost.Sensors{}
+		return a.collectNVIDIATemperatureSensors(ctx)
 	}
 
 	result := convertTemperatureDataToSensors(tempData)
+	a.mergeNVIDIATemperatures(ctx, &result)
 
 	// Collect power consumption data (Intel RAPL, etc.)
 	if powerData, err := a.collector.SensorsPower(ctx); err == nil && powerData != nil && powerData.Available {

internal/hostagent/agent_metrics_test.go

@@ -449,6 +449,64 @@ func TestBuildReportIncludesDarwinThermalState(t *testing.T) {
 	}
 }
 
+func TestBuildReportIncludesNVIDIASMITemperaturesWhenLMSensorsUnavailable(t *testing.T) {
+	mc := &mockCollector{
+		goos:  "linux",
+		nowFn: func() time.Time { return time.Date(2026, 6, 30, 8, 0, 0, 0, time.UTC) },
+		hostInfoFn: func(ctx context.Context) (*gohost.InfoStat, error) {
+			return &gohost.InfoStat{
+				Hostname: "gpu-node",
+				OS:       "linux",
+				Platform: "ubuntu",
+				HostID:   "gpu-node-id",
+			}, nil
+		},
+		hostUptimeFn: func(context.Context) (uint64, error) {
+			return 7200, nil
+		},
+		metricsFn: func(context.Context, []string) (hostmetrics.Snapshot, error) {
+			return hostmetrics.Snapshot{}, nil
+		},
+		sensorsLocalFn: func(context.Context) (string, error) {
+			return "", errors.New("lm-sensors unavailable")
+		},
+		lookPathFn: func(file string) (string, error) {
+			if file == "nvidia-smi" {
+				return "/usr/bin/nvidia-smi", nil
+			}
+			return "", os.ErrNotExist
+		},
+		commandCombinedOutputFn: func(_ context.Context, name string, arg ...string) (string, error) {
+			if name != "/usr/bin/nvidia-smi" {
+				t.Fatalf("command name = %q, want /usr/bin/nvidia-smi", name)
+			}
+			if len(arg) != 2 || arg[0] != "--query-gpu=index,name,temperature.gpu" || arg[1] != "--format=csv,noheader,nounits" {
+				t.Fatalf("command args = %#v, want NVIDIA temperature query", arg)
+			}
+			return "0, NVIDIA GeForce RTX 4090, 63\n", nil
+		},
+	}
+
+	agent, err := New(Config{
+		AgentID:   "gpu-agent",
+		APIToken:  "token",
+		LogLevel:  -1,
+		Collector: mc,
+	})
+	if err != nil {
+		t.Fatalf("New() failed: %v", err)
+	}
+
+	report, err := agent.buildReport(context.Background())
+	if err != nil {
+		t.Fatalf("buildReport failed: %v", err)
+	}
+
+	if report.Sensors.TemperatureCelsius["gpu_nvidia_0"] != 63 {
+		t.Fatalf("NVIDIA GPU temp = %v, want 63", report.Sensors.TemperatureCelsius["gpu_nvidia_0"])
+	}
+}
+
 func TestBuildReportUsesResolvedNASOSIdentity(t *testing.T) {
 	fixedTime := time.Date(2026, time.April, 15, 12, 0, 0, 0, time.UTC)
 

internal/hostagent/agent_sensors_test.go

@@ -3,6 +3,8 @@ package hostagent
 import (
 	"context"
 	"errors"
+	"os"
+	"os/exec"
 	"testing"
 
 	"github.com/rcourtman/pulse-go-rewrite/internal/sensors"
@@ -55,6 +57,70 @@ func TestAgent_collectTemperatures_MapsKeys(t *testing.T) {
 	}
 }
 
+func TestAgent_collectTemperatures_MergesNVIDIASMITemperatures(t *testing.T) {
+	mc := &mockCollector{
+		goos:           "linux",
+		sensorsLocalFn: func(context.Context) (string, error) { return "{}", nil },
+		sensorsParseFn: func(string) (*sensors.TemperatureData, error) {
+			return &sensors.TemperatureData{
+				Available:  true,
+				CPUPackage: 55.5,
+			}, nil
+		},
+		lookPathFn: func(file string) (string, error) {
+			if file != "nvidia-smi" {
+				t.Fatalf("look path file = %q, want nvidia-smi", file)
+			}
+			return "/usr/bin/nvidia-smi", nil
+		},
+		commandCombinedOutputFn: func(_ context.Context, name string, arg ...string) (string, error) {
+			if name != "/usr/bin/nvidia-smi" {
+				t.Fatalf("command name = %q, want /usr/bin/nvidia-smi", name)
+			}
+			if len(arg) != 2 || arg[0] != "--query-gpu=index,name,temperature.gpu" || arg[1] != "--format=csv,noheader,nounits" {
+				t.Fatalf("command args = %#v, want NVIDIA temperature query", arg)
+			}
+			return "0, NVIDIA GeForce RTX 4090, 61\n", nil
+		},
+	}
+
+	a := &Agent{logger: zerolog.Nop(), collector: mc}
+
+	got := a.collectTemperatures(context.Background())
+	if got.TemperatureCelsius["cpu_package"] != 55.5 {
+		t.Fatalf("cpu package temp = %v, want 55.5", got.TemperatureCelsius["cpu_package"])
+	}
+	if got.TemperatureCelsius["gpu_nvidia_0"] != 61 {
+		t.Fatalf("NVIDIA GPU temp = %v, want 61", got.TemperatureCelsius["gpu_nvidia_0"])
+	}
+}
+
+func TestAgent_collectTemperatures_UsesNVIDIASMIWhenLMSensorsUnavailable(t *testing.T) {
+	mc := &mockCollector{
+		goos:           "linux",
+		sensorsLocalFn: func(context.Context) (string, error) { return "", errors.New("no sensors") },
+		lookPathFn: func(file string) (string, error) {
+			if file != "nvidia-smi" {
+				t.Fatalf("look path file = %q, want nvidia-smi", file)
+			}
+			return "/usr/bin/nvidia-smi", nil
+		},
+		commandCombinedOutputFn: func(context.Context, string, ...string) (string, error) {
+			return "0, NVIDIA RTX A6000, 58\n", nil
+		},
+	}
+
+	a := &Agent{logger: zerolog.Nop(), collector: mc}
+
+	got := a.collectTemperatures(context.Background())
+	if len(got.TemperatureCelsius) != 1 {
+		t.Fatalf("temperature keys = %d, want 1; got %v", len(got.TemperatureCelsius), got.TemperatureCelsius)
+	}
+	if got.TemperatureCelsius["gpu_nvidia_0"] != 58 {
+		t.Fatalf("NVIDIA GPU temp = %v, want 58", got.TemperatureCelsius["gpu_nvidia_0"])
+	}
+}
+
 func TestAgent_collectTemperatures_BestEffortFailuresReturnEmpty(t *testing.T) {
 	mc := &mockCollector{goos: "linux"}
 	a := &Agent{logger: zerolog.Nop(), collector: mc}
@@ -76,6 +142,61 @@ func TestAgent_collectTemperatures_BestEffortFailuresReturnEmpty(t *testing.T) {
 	}
 }
 
+func TestParseNVIDIASMITemperatures(t *testing.T) {
+	input := `0, NVIDIA GeForce RTX 4090, 47
+1, "NVIDIA, RTX A6000", N/A
+2, NVIDIA Tesla T4, 58 C
+0000:65:00.0, NVIDIA H100, 64`
+
+	got, err := parseNVIDIASMITemperatures(input)
+	if err != nil {
+		t.Fatalf("parseNVIDIASMITemperatures returned error: %v", err)
+	}
+
+	want := map[string]float64{
+		"gpu_nvidia_0":            47,
+		"gpu_nvidia_2":            58,
+		"gpu_nvidia_0000_65_00_0": 64,
+	}
+	if len(got) != len(want) {
+		t.Fatalf("temperature keys = %d, want %d; got %v", len(got), len(want), got)
+	}
+	for key, wantTemp := range want {
+		if gotTemp := got[key]; gotTemp != wantTemp {
+			t.Fatalf("%s = %v, want %v", key, gotTemp, wantTemp)
+		}
+	}
+}
+
+func TestParseNVIDIASMITemperatures_NoUsableRows(t *testing.T) {
+	got, err := parseNVIDIASMITemperatures("0, NVIDIA RTX, N/A\nbad row\n")
+	if err != nil {
+		t.Fatalf("parseNVIDIASMITemperatures returned error: %v", err)
+	}
+	if got != nil {
+		t.Fatalf("expected nil temps, got %v", got)
+	}
+}
+
+func TestAgent_queryNVIDIASMITemperatures_CommandNotInstalled(t *testing.T) {
+	for _, lookPathErr := range []error{os.ErrNotExist, exec.ErrNotFound} {
+		mc := &mockCollector{
+			lookPathFn: func(string) (string, error) {
+				return "", lookPathErr
+			},
+		}
+		a := &Agent{logger: zerolog.Nop(), collector: mc}
+
+		got, err := a.queryNVIDIASMITemperatures(context.Background())
+		if err != nil {
+			t.Fatalf("queryNVIDIASMITemperatures returned error for %v: %v", lookPathErr, err)
+		}
+		if got != nil {
+			t.Fatalf("expected nil temps for %v, got %v", lookPathErr, got)
+		}
+	}
+}
+
 func TestAgent_collectTemperatures_SkipsUnsupportedOS(t *testing.T) {
 	mc := &mockCollector{goos: "windows"}
 	a := &Agent{logger: zerolog.Nop(), collector: mc}

internal/hostagent/nvidia_smi.go

@@ -0,0 +1,164 @@
+package hostagent
+
+import (
+	"context"
+	"encoding/csv"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+	"time"
+
+	agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host"
+)
+
+const nvidiaSMITemperatureTimeout = 5 * time.Second
+
+var nvidiaSMITemperatureArgs = []string{
+	"--query-gpu=index,name,temperature.gpu",
+	"--format=csv,noheader,nounits",
+}
+
+func (a *Agent) collectNVIDIATemperatureSensors(ctx context.Context) agentshost.Sensors {
+	var result agentshost.Sensors
+	a.mergeNVIDIATemperatures(ctx, &result)
+	if len(result.TemperatureCelsius) == 0 {
+		return agentshost.Sensors{}
+	}
+	return result
+}
+
+func (a *Agent) mergeNVIDIATemperatures(ctx context.Context, result *agentshost.Sensors) {
+	temps, err := a.queryNVIDIASMITemperatures(ctx)
+	if err != nil {
+		a.logger.Debug().Err(err).Msg("Failed to collect NVIDIA GPU temperatures")
+		return
+	}
+	if len(temps) == 0 {
+		return
+	}
+
+	if result.TemperatureCelsius == nil {
+		result.TemperatureCelsius = make(map[string]float64, len(temps))
+	}
+	for key, temp := range temps {
+		result.TemperatureCelsius[key] = temp
+	}
+}
+
+func (a *Agent) queryNVIDIASMITemperatures(ctx context.Context) (map[string]float64, error) {
+	path, err := a.collector.LookPath("nvidia-smi")
+	if err != nil {
+		if errors.Is(err, exec.ErrNotFound) || os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("locate nvidia-smi: %w", err)
+	}
+
+	cmdCtx, cancel := context.WithTimeout(ctx, nvidiaSMITemperatureTimeout)
+	defer cancel()
+
+	output, err := a.collector.CommandCombinedOutput(cmdCtx, path, nvidiaSMITemperatureArgs...)
+	if err != nil {
+		return nil, fmt.Errorf("execute nvidia-smi temperature query: %w", err)
+	}
+
+	temps, err := parseNVIDIASMITemperatures(output)
+	if err != nil {
+		return nil, fmt.Errorf("parse nvidia-smi temperature output: %w", err)
+	}
+	return temps, nil
+}
+
+func parseNVIDIASMITemperatures(output string) (map[string]float64, error) {
+	output = strings.TrimSpace(output)
+	if output == "" {
+		return nil, nil
+	}
+
+	reader := csv.NewReader(strings.NewReader(output))
+	reader.FieldsPerRecord = -1
+	reader.TrimLeadingSpace = true
+
+	records, err := reader.ReadAll()
+	if err != nil {
+		return nil, err
+	}
+
+	temps := make(map[string]float64)
+	for row, record := range records {
+		if len(record) < 3 {
+			continue
+		}
+
+		temp, ok := parseNVIDIASMITemperatureValue(record[2])
+		if !ok {
+			continue
+		}
+
+		index := normalizeNVIDIASMIIndex(record[0], row)
+		key := "gpu_nvidia_" + index
+		if _, exists := temps[key]; exists {
+			key = fmt.Sprintf("gpu_nvidia_%d", row)
+		}
+		temps[key] = temp
+	}
+
+	if len(temps) == 0 {
+		return nil, nil
+	}
+	return temps, nil
+}
+
+func parseNVIDIASMITemperatureValue(value string) (float64, bool) {
+	value = strings.TrimSpace(value)
+	if value == "" {
+		return 0, false
+	}
+
+	lower := strings.ToLower(value)
+	if lower == "n/a" || lower == "na" || strings.Contains(lower, "not supported") || strings.Contains(lower, "unknown") {
+		return 0, false
+	}
+
+	fields := strings.Fields(value)
+	if len(fields) > 0 {
+		value = fields[0]
+	}
+	value = strings.TrimSuffix(strings.TrimSuffix(value, "C"), "c")
+
+	temp, err := strconv.ParseFloat(value, 64)
+	if err != nil || temp <= 0 {
+		return 0, false
+	}
+	return temp, true
+}
+
+func normalizeNVIDIASMIIndex(value string, fallback int) string {
+	value = strings.ToLower(strings.TrimSpace(value))
+	if value == "" {
+		return strconv.Itoa(fallback)
+	}
+
+	var builder strings.Builder
+	lastUnderscore := false
+	for _, r := range value {
+		if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
+			builder.WriteRune(r)
+			lastUnderscore = false
+			continue
+		}
+		if builder.Len() > 0 && !lastUnderscore {
+			builder.WriteByte('_')
+			lastUnderscore = true
+		}
+	}
+
+	normalized := strings.Trim(builder.String(), "_")
+	if normalized == "" {
+		return strconv.Itoa(fallback)
+	}
+	return normalized
+}