From 2518e09d6775ae9916c52dba16b52d88a0e7beaa Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sat, 18 Apr 2026 07:03:42 +0300
Subject: [PATCH] refactor(monitoring): store pod:tensor_saturation as unitless
 ratio

Align pod:tensor_saturation:avg5m with namespace:tensor_active:avg and
DCGM's native 0..1 range by dropping the * 100 from the recording rule
and multiplying at display time in gpu-efficiency.json. Also scope
pod:util_per_watt:avg5m with avg by (Hostname, gpu, UUID, namespace,
pod) so the series mirrors pod:tensor_saturation's grouping and stays
usable in topk queries.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 dashboards/gpu/gpu-efficiency.json               |  4 ++--
 .../alerts/gpu-recording.rules.yaml              | 16 ++++++++++------
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/dashboards/gpu/gpu-efficiency.json b/dashboards/gpu/gpu-efficiency.json
index 07000f71..dddb2001 100644
--- a/dashboards/gpu/gpu-efficiency.json
+++ b/dashboards/gpu/gpu-efficiency.json
@@ -35,7 +35,7 @@
       "id": 2,
       "targets": [
         {
-          "expr": "avg(pod:tensor_saturation:avg5m{namespace=~\"$namespace\"})",
+          "expr": "avg(pod:tensor_saturation:avg5m{namespace=~\"$namespace\"}) * 100",
           "refId": "A"
         }
       ],
@@ -363,7 +363,7 @@
       "id": 21,
       "targets": [
         {
-          "expr": "topk(20, pod:tensor_saturation:avg5m)",
+          "expr": "topk(20, pod:tensor_saturation:avg5m * 100)",
           "instant": true,
           "range": false,
           "format": "table",
diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
index a24951f5..a6ecb20a 100644
--- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
+++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
@@ -64,20 +64,24 @@ spec:
       rules:
         # Tensor hardware saturation — the honest "am I using the GPU" metric
         # for AI/LLM workloads. Unlike NVML, idle tensor cores are visible.
+        # Stored as a 0..1 ratio to stay consistent with namespace:tensor_active:avg
+        # and DCGM's native units. Consumers multiply by 100 at display time.
         - record: pod:tensor_saturation:avg5m
           expr: |
             avg by (Hostname, gpu, UUID, namespace, pod) (
               avg_over_time(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m])
-            ) * 100
+            )
 
         # Power efficiency — utilization per watt, reveals unoptimized clients.
         - record: pod:util_per_watt:avg5m
           expr: |
-            avg_over_time(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m])
-            /
-            clamp_min(
-              avg_over_time(DCGM_FI_DEV_POWER_USAGE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]),
-              1
+            avg by (Hostname, gpu, UUID, namespace, pod) (
+              avg_over_time(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m])
+              /
+              clamp_min(
+                avg_over_time(DCGM_FI_DEV_POWER_USAGE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]),
+                1
+              )
             )
 
         # Fraction of time power-throttled (TDP cap) — 1.0 = fully throttled.