{ "uid": "gpu-efficiency", "title": "GPU Efficiency Score", "description": "Tensor saturation, util/watt and throttling — reveals inefficient GPU workloads", "tags": [ "gpu", "efficiency", "finops" ], "timezone": "browser", "editable": true, "graphTooltip": 1, "time": { "from": "now-1h", "to": "now" }, "fiscalYearStartMonth": 0, "schemaVersion": 42, "panels": [ { "type": "row", "collapsed": false, "title": "Overall efficiency metrics", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 1, "panels": [] }, { "type": "stat", "id": 2, "targets": [ { "expr": "avg(gpu:tensor_saturation:avg5m) * 100", "refId": "A" } ], "title": "Avg Tensor Saturation", "description": "Mean tensor core saturation across all GPUs. \u003c10% means GPUs are used inefficiently (workloads could move to CPU or optimize their code). Cluster-wide — DCGM metrics cannot be attributed to workload namespaces.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 5, "w": 8, "x": 0, "y": 1 }, "repeatDirection": "h", "options": { "graphMode": "area", "colorMode": "background", "justifyMode": "auto", "textMode": "value", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" ] }, "percentChangeColorMode": "standard", "orientation": "" }, "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100, "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" }, { "value": 10, "color": "orange" }, { "value": 30, "color": "yellow" }, { "value": 60, "color": "green" } ] } }, "overrides": [] } }, { "type": "stat", "id": 3, "targets": [ { "expr": "avg(gpu:util_per_watt:avg5m)", "refId": "A" } ], "title": "Avg Utilization per Watt", "description": "NVML utilization % per watt across all GPUs. Higher value = more efficient workload. Cluster-wide — DCGM metrics cannot be attributed to workload namespaces.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 5, "w": 8, "x": 8, "y": 1 }, "repeatDirection": "h", "options": { "graphMode": "area", "colorMode": "background", "justifyMode": "auto", "textMode": "value", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" ] }, "percentChangeColorMode": "standard", "orientation": "" }, "fieldConfig": { "defaults": { "unit": "none", "decimals": 3, "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" }, { "value": 0.5, "color": "yellow" }, { "value": 1, "color": "green" } ] } }, "overrides": [] } }, { "type": "stat", "id": 4, "targets": [ { "expr": "avg(gpu:power_throttle_fraction:rate5m)", "refId": "A" } ], "title": "Avg Power Throttling", "description": "Fraction of time GPUs hit the TDP cap and lose performance. \u003e5% means tenants underutilize billed FLOPS. Cluster-wide — rule aggregates by (Hostname, gpu, UUID) and drops namespace label.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 5, "w": 8, "x": 16, "y": 1 }, "repeatDirection": "h", "options": { "graphMode": "area", "colorMode": "background", "justifyMode": "auto", "textMode": "value", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" ] }, "percentChangeColorMode": "standard", "orientation": "" }, "fieldConfig": { "defaults": { "unit": "percentunit", "decimals": 2, "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "green" }, { "value": 0.05, "color": "yellow" }, { "value": 0.2, "color": "red" } ] } }, "overrides": [] } }, { "type": "row", "collapsed": false, "title": "NVML vs Tensor (mismatch detector)", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 }, "id": 10, "panels": [] }, { "type": "timeseries", "id": 11, "targets": [ { "expr": "DCGM_FI_DEV_GPU_UTIL", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } ], "title": "NVML GPU Utilization", "description": "Classic utilization metric. Shows activity of any engine (SM, copy, encoder). Cluster-wide — DCGM namespace is the exporter's own namespace, not the workload namespace.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 7 }, "repeatDirection": "h", "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": false, "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "" } }, "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "fillOpacity": 10, "showPoints": "never" } }, "overrides": [] } }, { "type": "timeseries", "id": 12, "targets": [ { "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE * 100", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } ], "title": "Tensor Pipe Active", "description": "Real tensor core load (HMMA). For AI/LLM inference it should be ≥20%, otherwise the workload is unoptimized. Cluster-wide — DCGM namespace is the exporter's own namespace, not the workload namespace.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 7 }, "repeatDirection": "h", "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": false, "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "" } }, "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "fillOpacity": 10, "showPoints": "never" } }, "overrides": [] } }, { "type": "row", "collapsed": false, "title": "Per-GPU ranking", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 15 }, "id": 20, "panels": [] }, { "type": "table", "id": 21, "targets": [ { "expr": "topk(20, gpu:tensor_saturation:avg5m * 100)", "instant": true, "range": false, "format": "table", "refId": "A" } ], "title": "Tensor Saturation per GPU (5m avg)", "description": "Which GPUs are exercising tensor cores and which are not. Sorted descending. Grouped by Hostname/gpu/UUID — DCGM metrics cannot be attributed to workload namespaces.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, "repeatDirection": "h", "transformations": [ { "id": "organize", "options": { "excludeByName": { "DCGM_FI_DRIVER_VERSION": true, "Time": true, "__name__": true, "cluster": true, "container": true, "device": true, "endpoint": true, "gpu_driver_version": true, "instance": true, "job": true, "modelName": true, "namespace": true, "pci_bus_id": true, "pod": true, "prometheus": true, "service": true, "tenant": true, "tier": true, "uid": true, "unit": true }, "indexByName": { "Hostname": 0, "UUID": 2, "Value": 3, "gpu": 1 }, "renameByName": { "Hostname": "Node", "UUID": "UUID", "Value": "Saturation", "gpu": "GPU" } } } ], "options": { "frameIndex": 0, "showHeader": true, "showTypeIcons": false, "sortBy": [ { "displayName": "Saturation", "desc": true } ], "footer": { "show": false, "reducer": [] }, "cellHeight": "sm" }, "fieldConfig": { "defaults": {}, "overrides": [ { "matcher": { "id": "byName", "options": "Saturation" }, "properties": [ { "id": "unit", "value": "percent" }, { "id": "min", "value": 0 }, { "id": "max", "value": 100 }, { "id": "custom.cellOptions", "value": { "mode": "gradient", "type": "gauge" } }, { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "red" }, { "color": "orange", "value": 10 }, { "color": "yellow", "value": 30 }, { "color": "green", "value": 60 } ] } } ] } ] } }, { "type": "table", "id": 22, "targets": [ { "expr": "topk(20, gpu:util_per_watt:avg5m)", "instant": true, "range": false, "format": "table", "refId": "A" } ], "title": "Utilization / Watt per GPU (5m avg)", "description": "How efficiently each GPU spends watts. Low value = poor optimization. Grouped by Hostname/gpu/UUID — DCGM metrics cannot be attributed to workload namespaces.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, "repeatDirection": "h", "transformations": [ { "id": "organize", "options": { "excludeByName": { "DCGM_FI_DRIVER_VERSION": true, "Time": true, "__name__": true, "cluster": true, "container": true, "device": true, "endpoint": true, "gpu_driver_version": true, "instance": true, "job": true, "modelName": true, "namespace": true, "pci_bus_id": true, "pod": true, "prometheus": true, "service": true, "tenant": true, "tier": true, "uid": true, "unit": true }, "indexByName": { "Hostname": 0, "UUID": 2, "Value": 3, "gpu": 1 }, "renameByName": { "Hostname": "Node", "UUID": "UUID", "Value": "Util/Watt", "gpu": "GPU" } } } ], "options": { "frameIndex": 0, "showHeader": true, "showTypeIcons": false, "sortBy": [ { "displayName": "Util/Watt", "desc": true } ], "footer": { "show": false, "reducer": [] }, "cellHeight": "sm" }, "fieldConfig": { "defaults": {}, "overrides": [ { "matcher": { "id": "byName", "options": "Util/Watt" }, "properties": [ { "id": "unit", "value": "none" }, { "id": "decimals", "value": 3 }, { "id": "min", "value": 0 }, { "id": "max", "value": 1.5 }, { "id": "custom.cellOptions", "value": { "mode": "gradient", "type": "gauge" } }, { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "red" }, { "color": "yellow", "value": 0.5 }, { "color": "green", "value": 1 } ] } } ] } ] } }, { "type": "row", "collapsed": false, "title": "Throttling", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 }, "id": 30, "panels": [] }, { "type": "timeseries", "id": 31, "targets": [ { "expr": "gpu:power_throttle_fraction:rate5m", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } ], "title": "Power throttle fraction per GPU", "description": "Fraction of time the GPU was power-throttled. 1.0 = always throttled. Cluster-wide — rule aggregates by (Hostname, gpu, UUID) and drops namespace label.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 25 }, "repeatDirection": "h", "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": false, "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "" } }, "fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1, "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "green" }, { "value": 0.05, "color": "yellow" }, { "value": 0.2, "color": "red" } ] }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "fillOpacity": 25, "showPoints": "never" } }, "overrides": [] } }, { "type": "timeseries", "id": 32, "targets": [ { "expr": "gpu:thermal_throttle_fraction:rate5m", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } ], "title": "Thermal throttle fraction per GPU", "description": "Fraction of time the GPU was thermal-throttled. Cluster-wide — rule aggregates by (Hostname, gpu, UUID) and drops namespace label.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 25 }, "repeatDirection": "h", "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": false, "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "" } }, "fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1, "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "green" }, { "value": 0.05, "color": "yellow" }, { "value": 0.2, "color": "red" } ] }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "fillOpacity": 25, "showPoints": "never" } }, "overrides": [] } } ], "templating": { "list": [ { "type": "datasource", "name": "ds_prometheus", "label": "Prometheus", "skipUrlSync": false, "query": "prometheus", "current": { "selected": false, "text": "default", "value": "default" }, "multi": false, "allowCustomValue": true, "includeAll": false, "regex": "", "auto": false, "auto_min": "10s", "auto_count": 30 } ] }, "annotations": {} }