bb51c88f78
Address review feedback from coderabbitai on dashboards/gpu/gpu-efficiency.json:839 Signed-off-by: Arsolitt <arsolitt@gmail.com>
819 lines
20 KiB
JSON
819 lines
20 KiB
JSON
{
|
|
"uid": "gpu-efficiency",
|
|
"title": "GPU Efficiency Score",
|
|
"description": "Tensor saturation, util/watt and throttling — reveals inefficient GPU workloads",
|
|
"tags": [
|
|
"gpu",
|
|
"efficiency",
|
|
"finops"
|
|
],
|
|
"timezone": "browser",
|
|
"editable": true,
|
|
"graphTooltip": 1,
|
|
"time": {
|
|
"from": "now-1h",
|
|
"to": "now"
|
|
},
|
|
"fiscalYearStartMonth": 0,
|
|
"schemaVersion": 42,
|
|
"panels": [
|
|
{
|
|
"type": "row",
|
|
"collapsed": false,
|
|
"title": "Overall efficiency metrics",
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"id": 1,
|
|
"panels": []
|
|
},
|
|
{
|
|
"type": "stat",
|
|
"id": 2,
|
|
"targets": [
|
|
{
|
|
"expr": "avg(gpu:tensor_saturation:avg5m) * 100",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Avg Tensor Saturation",
|
|
"description": "Mean tensor core saturation across all GPUs. \u003c10% means GPUs are used inefficiently (workloads could move to CPU or optimize their code). Cluster-wide — DCGM metrics cannot be attributed to workload namespaces.",
|
|
"transparent": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds_prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 1
|
|
},
|
|
"repeatDirection": "h",
|
|
"options": {
|
|
"graphMode": "area",
|
|
"colorMode": "background",
|
|
"justifyMode": "auto",
|
|
"textMode": "value",
|
|
"wideLayout": true,
|
|
"showPercentChange": false,
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"percentChangeColorMode": "standard",
|
|
"orientation": ""
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percent",
|
|
"min": 0,
|
|
"max": 100,
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"value": null,
|
|
"color": "red"
|
|
},
|
|
{
|
|
"value": 10,
|
|
"color": "orange"
|
|
},
|
|
{
|
|
"value": 30,
|
|
"color": "yellow"
|
|
},
|
|
{
|
|
"value": 60,
|
|
"color": "green"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"type": "stat",
|
|
"id": 3,
|
|
"targets": [
|
|
{
|
|
"expr": "avg(gpu:util_per_watt:avg5m)",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Avg Utilization per Watt",
|
|
"description": "NVML utilization % per watt across all GPUs. Higher value = more efficient workload. Cluster-wide — DCGM metrics cannot be attributed to workload namespaces.",
|
|
"transparent": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds_prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 1
|
|
},
|
|
"repeatDirection": "h",
|
|
"options": {
|
|
"graphMode": "area",
|
|
"colorMode": "background",
|
|
"justifyMode": "auto",
|
|
"textMode": "value",
|
|
"wideLayout": true,
|
|
"showPercentChange": false,
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"percentChangeColorMode": "standard",
|
|
"orientation": ""
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "none",
|
|
"decimals": 3,
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"value": null,
|
|
"color": "red"
|
|
},
|
|
{
|
|
"value": 0.5,
|
|
"color": "yellow"
|
|
},
|
|
{
|
|
"value": 1,
|
|
"color": "green"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"type": "stat",
|
|
"id": 4,
|
|
"targets": [
|
|
{
|
|
"expr": "avg(gpu:power_throttle_fraction:rate5m)",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Avg Power Throttling",
|
|
"description": "Fraction of time GPUs hit the TDP cap and lose performance. \u003e5% means tenants underutilize billed FLOPS. Cluster-wide — rule aggregates by (Hostname, gpu, UUID) and drops namespace label.",
|
|
"transparent": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds_prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 1
|
|
},
|
|
"repeatDirection": "h",
|
|
"options": {
|
|
"graphMode": "area",
|
|
"colorMode": "background",
|
|
"justifyMode": "auto",
|
|
"textMode": "value",
|
|
"wideLayout": true,
|
|
"showPercentChange": false,
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"percentChangeColorMode": "standard",
|
|
"orientation": ""
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit",
|
|
"decimals": 2,
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"value": null,
|
|
"color": "green"
|
|
},
|
|
{
|
|
"value": 0.05,
|
|
"color": "yellow"
|
|
},
|
|
{
|
|
"value": 0.2,
|
|
"color": "red"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"type": "row",
|
|
"collapsed": false,
|
|
"title": "NVML vs Tensor (mismatch detector)",
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 6
|
|
},
|
|
"id": 10,
|
|
"panels": []
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"id": 11,
|
|
"targets": [
|
|
{
|
|
"expr": "DCGM_FI_DEV_GPU_UTIL",
|
|
"legendFormat": "{{Hostname}}/{{gpu}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "NVML GPU Utilization",
|
|
"description": "Classic utilization metric. Shows activity of any engine (SM, copy, encoder). Cluster-wide — DCGM namespace is the exporter's own namespace, not the workload namespace.",
|
|
"transparent": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds_prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 7
|
|
},
|
|
"repeatDirection": "h",
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": false,
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": ""
|
|
}
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percent",
|
|
"min": 0,
|
|
"max": 100,
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "linear",
|
|
"fillOpacity": 10,
|
|
"showPoints": "never"
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"id": 12,
|
|
"targets": [
|
|
{
|
|
"expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE * 100",
|
|
"legendFormat": "{{Hostname}}/{{gpu}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Tensor Pipe Active",
|
|
"description": "Real tensor core load (HMMA). For AI/LLM inference it should be ≥20%, otherwise the workload is unoptimized. Cluster-wide — DCGM namespace is the exporter's own namespace, not the workload namespace.",
|
|
"transparent": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds_prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 7
|
|
},
|
|
"repeatDirection": "h",
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": false,
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": ""
|
|
}
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percent",
|
|
"min": 0,
|
|
"max": 100,
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "linear",
|
|
"fillOpacity": 10,
|
|
"showPoints": "never"
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"type": "row",
|
|
"collapsed": false,
|
|
"title": "Per-GPU ranking",
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 15
|
|
},
|
|
"id": 20,
|
|
"panels": []
|
|
},
|
|
{
|
|
"type": "table",
|
|
"id": 21,
|
|
"targets": [
|
|
{
|
|
"expr": "topk(20, gpu:tensor_saturation:avg5m * 100)",
|
|
"instant": true,
|
|
"range": false,
|
|
"format": "table",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Tensor Saturation per GPU (5m avg)",
|
|
"description": "Which GPUs are exercising tensor cores and which are not. Sorted descending. Grouped by Hostname/gpu/UUID — DCGM metrics cannot be attributed to workload namespaces.",
|
|
"transparent": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds_prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 16
|
|
},
|
|
"repeatDirection": "h",
|
|
"transformations": [
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {
|
|
"DCGM_FI_DRIVER_VERSION": true,
|
|
"Time": true,
|
|
"__name__": true,
|
|
"cluster": true,
|
|
"container": true,
|
|
"device": true,
|
|
"endpoint": true,
|
|
"gpu_driver_version": true,
|
|
"instance": true,
|
|
"job": true,
|
|
"modelName": true,
|
|
"namespace": true,
|
|
"pci_bus_id": true,
|
|
"pod": true,
|
|
"prometheus": true,
|
|
"service": true,
|
|
"tenant": true,
|
|
"tier": true,
|
|
"uid": true,
|
|
"unit": true
|
|
},
|
|
"indexByName": {
|
|
"Hostname": 0,
|
|
"UUID": 2,
|
|
"Value": 3,
|
|
"gpu": 1
|
|
},
|
|
"renameByName": {
|
|
"Hostname": "Node",
|
|
"UUID": "UUID",
|
|
"Value": "Saturation",
|
|
"gpu": "GPU"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"options": {
|
|
"frameIndex": 0,
|
|
"showHeader": true,
|
|
"showTypeIcons": false,
|
|
"sortBy": [
|
|
{
|
|
"displayName": "Saturation",
|
|
"desc": true
|
|
}
|
|
],
|
|
"footer": {
|
|
"show": false,
|
|
"reducer": []
|
|
},
|
|
"cellHeight": "sm"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "Saturation"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "unit",
|
|
"value": "percent"
|
|
},
|
|
{
|
|
"id": "min",
|
|
"value": 0
|
|
},
|
|
{
|
|
"id": "max",
|
|
"value": 100
|
|
},
|
|
{
|
|
"id": "custom.cellOptions",
|
|
"value": {
|
|
"mode": "gradient",
|
|
"type": "gauge"
|
|
}
|
|
},
|
|
{
|
|
"id": "thresholds",
|
|
"value": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "red"
|
|
},
|
|
{
|
|
"color": "orange",
|
|
"value": 10
|
|
},
|
|
{
|
|
"color": "yellow",
|
|
"value": 30
|
|
},
|
|
{
|
|
"color": "green",
|
|
"value": 60
|
|
}
|
|
]
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"type": "table",
|
|
"id": 22,
|
|
"targets": [
|
|
{
|
|
"expr": "topk(20, gpu:util_per_watt:avg5m)",
|
|
"instant": true,
|
|
"range": false,
|
|
"format": "table",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Utilization / Watt per GPU (5m avg)",
|
|
"description": "How efficiently each GPU spends watts. Low value = poor optimization. Grouped by Hostname/gpu/UUID — DCGM metrics cannot be attributed to workload namespaces.",
|
|
"transparent": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds_prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 16
|
|
},
|
|
"repeatDirection": "h",
|
|
"transformations": [
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {
|
|
"DCGM_FI_DRIVER_VERSION": true,
|
|
"Time": true,
|
|
"__name__": true,
|
|
"cluster": true,
|
|
"container": true,
|
|
"device": true,
|
|
"endpoint": true,
|
|
"gpu_driver_version": true,
|
|
"instance": true,
|
|
"job": true,
|
|
"modelName": true,
|
|
"namespace": true,
|
|
"pci_bus_id": true,
|
|
"pod": true,
|
|
"prometheus": true,
|
|
"service": true,
|
|
"tenant": true,
|
|
"tier": true,
|
|
"uid": true,
|
|
"unit": true
|
|
},
|
|
"indexByName": {
|
|
"Hostname": 0,
|
|
"UUID": 2,
|
|
"Value": 3,
|
|
"gpu": 1
|
|
},
|
|
"renameByName": {
|
|
"Hostname": "Node",
|
|
"UUID": "UUID",
|
|
"Value": "Util/Watt",
|
|
"gpu": "GPU"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"options": {
|
|
"frameIndex": 0,
|
|
"showHeader": true,
|
|
"showTypeIcons": false,
|
|
"sortBy": [
|
|
{
|
|
"displayName": "Util/Watt",
|
|
"desc": true
|
|
}
|
|
],
|
|
"footer": {
|
|
"show": false,
|
|
"reducer": []
|
|
},
|
|
"cellHeight": "sm"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "Util/Watt"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "unit",
|
|
"value": "none"
|
|
},
|
|
{
|
|
"id": "decimals",
|
|
"value": 3
|
|
},
|
|
{
|
|
"id": "min",
|
|
"value": 0
|
|
},
|
|
{
|
|
"id": "max",
|
|
"value": 1.5
|
|
},
|
|
{
|
|
"id": "custom.cellOptions",
|
|
"value": {
|
|
"mode": "gradient",
|
|
"type": "gauge"
|
|
}
|
|
},
|
|
{
|
|
"id": "thresholds",
|
|
"value": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "red"
|
|
},
|
|
{
|
|
"color": "yellow",
|
|
"value": 0.5
|
|
},
|
|
{
|
|
"color": "green",
|
|
"value": 1
|
|
}
|
|
]
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"type": "row",
|
|
"collapsed": false,
|
|
"title": "Throttling",
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 24
|
|
},
|
|
"id": 30,
|
|
"panels": []
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"id": 31,
|
|
"targets": [
|
|
{
|
|
"expr": "gpu:power_throttle_fraction:rate5m",
|
|
"legendFormat": "{{Hostname}}/{{gpu}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Power throttle fraction per GPU",
|
|
"description": "Fraction of time the GPU was power-throttled. 1.0 = always throttled. Cluster-wide — rule aggregates by (Hostname, gpu, UUID) and drops namespace label.",
|
|
"transparent": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds_prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 25
|
|
},
|
|
"repeatDirection": "h",
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": false,
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": ""
|
|
}
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit",
|
|
"min": 0,
|
|
"max": 1,
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"value": null,
|
|
"color": "green"
|
|
},
|
|
{
|
|
"value": 0.05,
|
|
"color": "yellow"
|
|
},
|
|
{
|
|
"value": 0.2,
|
|
"color": "red"
|
|
}
|
|
]
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "linear",
|
|
"fillOpacity": 25,
|
|
"showPoints": "never"
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"id": 32,
|
|
"targets": [
|
|
{
|
|
"expr": "gpu:thermal_throttle_fraction:rate5m",
|
|
"legendFormat": "{{Hostname}}/{{gpu}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Thermal throttle fraction per GPU",
|
|
"description": "Fraction of time the GPU was thermal-throttled. Cluster-wide — rule aggregates by (Hostname, gpu, UUID) and drops namespace label.",
|
|
"transparent": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds_prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 25
|
|
},
|
|
"repeatDirection": "h",
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": false,
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": ""
|
|
}
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit",
|
|
"min": 0,
|
|
"max": 1,
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"value": null,
|
|
"color": "green"
|
|
},
|
|
{
|
|
"value": 0.05,
|
|
"color": "yellow"
|
|
},
|
|
{
|
|
"value": 0.2,
|
|
"color": "red"
|
|
}
|
|
]
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "linear",
|
|
"fillOpacity": 25,
|
|
"showPoints": "never"
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
}
|
|
],
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"type": "datasource",
|
|
"name": "ds_prometheus",
|
|
"label": "Prometheus",
|
|
"skipUrlSync": false,
|
|
"query": "prometheus",
|
|
"current": {
|
|
"selected": false,
|
|
"text": "default",
|
|
"value": "default"
|
|
},
|
|
"multi": false,
|
|
"allowCustomValue": true,
|
|
"includeAll": false,
|
|
"regex": "",
|
|
"auto": false,
|
|
"auto_min": "10s",
|
|
"auto_count": 30
|
|
}
|
|
]
|
|
},
|
|
"annotations": {}
|
|
}
|