1157 lines
26 KiB
JSON
1157 lines
26 KiB
JSON
{
|
|
"uid": "gpu-fleet",
|
|
"title": "GPU Fleet Overview",
|
|
"description": "Cluster-wide GPU inventory, capacity, utilization and health — admin view across the whole fleet.",
|
|
"tags": [
|
|
"gpu",
|
|
"fleet",
|
|
"admin"
|
|
],
|
|
"timezone": "browser",
|
|
"editable": true,
|
|
"graphTooltip": 1,
|
|
"time": {
|
|
"from": "now-6h",
|
|
"to": "now"
|
|
},
|
|
"fiscalYearStartMonth": 0,
|
|
"schemaVersion": 42,
|
|
"panels": [
|
|
{
|
|
"type": "row",
|
|
"collapsed": false,
|
|
"title": "Cluster snapshot",
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"id": 1,
|
|
"panels": []
|
|
},
|
|
{
|
|
"type": "stat",
|
|
"id": 2,
|
|
"targets": [
|
|
{
|
|
"expr": "cluster:gpu_count:total",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Total GPUs",
|
|
"description": "Physical GPUs discovered by DCGM across all nodes.",
|
|
"transparent": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds_prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 4,
|
|
"x": 0,
|
|
"y": 1
|
|
},
|
|
"repeatDirection": "h",
|
|
"options": {
|
|
"graphMode": "none",
|
|
"colorMode": "value",
|
|
"justifyMode": "auto",
|
|
"textMode": "value",
|
|
"wideLayout": true,
|
|
"showPercentChange": false,
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"percentChangeColorMode": "standard",
|
|
"orientation": ""
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"value": null,
|
|
"color": "blue"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"type": "stat",
|
|
"id": 3,
|
|
"targets": [
|
|
{
|
|
"expr": "cluster:gpu_count:allocated",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Allocated",
|
|
"description": "Sum of kube GPU requests across all pods. Billable view — includes Pending pods.",
|
|
"transparent": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds_prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 4,
|
|
"x": 4,
|
|
"y": 1
|
|
},
|
|
"repeatDirection": "h",
|
|
"options": {
|
|
"graphMode": "none",
|
|
"colorMode": "background",
|
|
"justifyMode": "auto",
|
|
"textMode": "value",
|
|
"wideLayout": true,
|
|
"showPercentChange": false,
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"percentChangeColorMode": "standard",
|
|
"orientation": ""
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"value": null,
|
|
"color": "blue"
|
|
},
|
|
{
|
|
"value": 1,
|
|
"color": "green"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"type": "stat",
|
|
"id": 4,
|
|
"targets": [
|
|
{
|
|
"expr": "cluster:gpu_count:free",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Free",
|
|
"description": "Unallocated capacity available for new tenants.",
|
|
"transparent": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds_prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 4,
|
|
"x": 8,
|
|
"y": 1
|
|
},
|
|
"repeatDirection": "h",
|
|
"options": {
|
|
"graphMode": "none",
|
|
"colorMode": "background",
|
|
"justifyMode": "auto",
|
|
"textMode": "value",
|
|
"wideLayout": true,
|
|
"showPercentChange": false,
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"percentChangeColorMode": "standard",
|
|
"orientation": ""
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"value": null,
|
|
"color": "red"
|
|
},
|
|
{
|
|
"value": 1,
|
|
"color": "yellow"
|
|
},
|
|
{
|
|
"value": 2,
|
|
"color": "green"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"type": "stat",
|
|
"id": 5,
|
|
"targets": [
|
|
{
|
|
"expr": "count(count by (Hostname) (DCGM_FI_DEV_GPU_UTIL))",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Nodes with GPU",
|
|
"description": "Nodes reporting at least one GPU to DCGM. Counted via DCGM_FI_DEV_GPU_UTIL because kube-state-metrics does not expose node labels by default.",
|
|
"transparent": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds_prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 4,
|
|
"x": 12,
|
|
"y": 1
|
|
},
|
|
"repeatDirection": "h",
|
|
"options": {
|
|
"graphMode": "none",
|
|
"colorMode": "value",
|
|
"justifyMode": "auto",
|
|
"textMode": "value",
|
|
"wideLayout": true,
|
|
"showPercentChange": false,
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"percentChangeColorMode": "standard",
|
|
"orientation": ""
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"value": null,
|
|
"color": "blue"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"type": "stat",
|
|
"id": 6,
|
|
"targets": [
|
|
{
|
|
"expr": "cluster:gpu_util:avg",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Average utilization",
|
|
"description": "Legacy NVML utilization across all tenant GPUs.",
|
|
"transparent": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds_prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 4,
|
|
"x": 16,
|
|
"y": 1
|
|
},
|
|
"repeatDirection": "h",
|
|
"options": {
|
|
"graphMode": "area",
|
|
"colorMode": "value",
|
|
"justifyMode": "auto",
|
|
"textMode": "value",
|
|
"wideLayout": true,
|
|
"showPercentChange": false,
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"percentChangeColorMode": "standard",
|
|
"orientation": ""
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percent",
|
|
"min": 0,
|
|
"max": 100,
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"value": null,
|
|
"color": "blue"
|
|
},
|
|
{
|
|
"value": 30,
|
|
"color": "green"
|
|
},
|
|
{
|
|
"value": 80,
|
|
"color": "orange"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"type": "stat",
|
|
"id": 7,
|
|
"targets": [
|
|
{
|
|
"expr": "cluster:gpu_power_watts:sum",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Total power",
|
|
"description": "Live power draw summed across the fleet.",
|
|
"transparent": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds_prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 4,
|
|
"x": 20,
|
|
"y": 1
|
|
},
|
|
"repeatDirection": "h",
|
|
"options": {
|
|
"graphMode": "area",
|
|
"colorMode": "value",
|
|
"justifyMode": "auto",
|
|
"textMode": "value",
|
|
"wideLayout": true,
|
|
"showPercentChange": false,
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"percentChangeColorMode": "standard",
|
|
"orientation": ""
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "watt",
|
|
"decimals": 0,
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"value": null,
|
|
"color": "green"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"type": "row",
|
|
"collapsed": false,
|
|
"title": "Per-node GPU activity",
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 5
|
|
},
|
|
"id": 10,
|
|
"panels": []
|
|
},
|
|
{
|
|
"type": "bargauge",
|
|
"id": 11,
|
|
"targets": [
|
|
{
|
|
"expr": "avg by (Hostname) (DCGM_FI_DEV_GPU_UTIL)",
|
|
"legendFormat": "{{Hostname}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "GPU utilization per node",
|
|
"description": "Average NVML utilization per node — spot idle or saturated hosts at a glance.",
|
|
"transparent": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds_prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 6
|
|
},
|
|
"repeatDirection": "h",
|
|
"options": {
|
|
"displayMode": "gradient",
|
|
"valueMode": "color",
|
|
"namePlacement": "auto",
|
|
"showUnfilled": true,
|
|
"sizing": "auto",
|
|
"minVizWidth": 8,
|
|
"minVizHeight": 16,
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false,
|
|
"calcs": []
|
|
},
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"maxVizHeight": 300,
|
|
"orientation": "horizontal"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percent",
|
|
"min": 0,
|
|
"max": 100,
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"value": null,
|
|
"color": "blue"
|
|
},
|
|
{
|
|
"value": 30,
|
|
"color": "green"
|
|
},
|
|
{
|
|
"value": 80,
|
|
"color": "orange"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"type": "bargauge",
|
|
"id": 12,
|
|
"targets": [
|
|
{
|
|
"expr": "sum by (Hostname) (DCGM_FI_DEV_POWER_USAGE) / clamp_min(sum by (Hostname) (DCGM_FI_DEV_POWER_MGMT_LIMIT), 1) * 100",
|
|
"legendFormat": "{{Hostname}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Power draw per node (% of TDP)",
|
|
"description": "GPU power usage as a fraction of the node's combined TDP cap. Works across GPU SKUs without per-model thresholds. Requires DCGM_FI_DEV_POWER_MGMT_LIMIT from custom DCGM CSV (see packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml).",
|
|
"transparent": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds_prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 6
|
|
},
|
|
"repeatDirection": "h",
|
|
"options": {
|
|
"displayMode": "gradient",
|
|
"valueMode": "color",
|
|
"namePlacement": "auto",
|
|
"showUnfilled": true,
|
|
"sizing": "auto",
|
|
"minVizWidth": 8,
|
|
"minVizHeight": 16,
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false,
|
|
"calcs": []
|
|
},
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"maxVizHeight": 300,
|
|
"orientation": "horizontal"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percent",
|
|
"min": 0,
|
|
"max": 100,
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"value": null,
|
|
"color": "green"
|
|
},
|
|
{
|
|
"value": 60,
|
|
"color": "yellow"
|
|
},
|
|
{
|
|
"value": 80,
|
|
"color": "orange"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"type": "row",
|
|
"collapsed": false,
|
|
"title": "Usage trends",
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 14
|
|
},
|
|
"id": 20,
|
|
"panels": []
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"id": 21,
|
|
"targets": [
|
|
{
|
|
"expr": "avg by (Hostname) (DCGM_FI_DEV_GPU_UTIL)",
|
|
"legendFormat": "{{Hostname}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "GPU utilization per node (trend)",
|
|
"description": "Per-node NVML utilization over time — shows shifting workload across hosts.",
|
|
"transparent": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds_prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 15
|
|
},
|
|
"repeatDirection": "h",
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": false,
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": ""
|
|
}
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percent",
|
|
"min": 0,
|
|
"max": 100,
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "linear",
|
|
"fillOpacity": 20,
|
|
"showPoints": "never"
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"id": 22,
|
|
"targets": [
|
|
{
|
|
"expr": "sum by (Hostname) (DCGM_FI_DEV_POWER_USAGE)",
|
|
"legendFormat": "{{Hostname}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Power draw per node (trend)",
|
|
"description": "Per-node GPU power consumption over time.",
|
|
"transparent": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds_prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 15
|
|
},
|
|
"repeatDirection": "h",
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": false,
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": ""
|
|
}
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "watt",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "linear",
|
|
"fillOpacity": 20,
|
|
"showPoints": "never"
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"type": "row",
|
|
"collapsed": false,
|
|
"title": "Health",
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 23
|
|
},
|
|
"id": 30,
|
|
"panels": []
|
|
},
|
|
{
|
|
"type": "stat",
|
|
"id": 31,
|
|
"targets": [
|
|
{
|
|
"expr": "count((max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS)) \u003e 0) or vector(0)",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "GPUs with XID errors",
|
|
"description": "Count of GPUs reporting a non-zero XID code. XID = NVIDIA hardware/driver error — any non-zero value needs investigation.",
|
|
"transparent": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds_prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 0,
|
|
"y": 24
|
|
},
|
|
"repeatDirection": "h",
|
|
"options": {
|
|
"graphMode": "none",
|
|
"colorMode": "background",
|
|
"justifyMode": "auto",
|
|
"textMode": "value",
|
|
"wideLayout": true,
|
|
"showPercentChange": false,
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"percentChangeColorMode": "standard",
|
|
"orientation": ""
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"value": null,
|
|
"color": "green"
|
|
},
|
|
{
|
|
"value": 1,
|
|
"color": "red"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"type": "stat",
|
|
"id": 32,
|
|
"targets": [
|
|
{
|
|
"expr": "avg(gpu:power_throttle_fraction:rate5m)",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Power throttling (cluster avg)",
|
|
"description": "Fraction of time the fleet spent power-throttled. \u003e5% = billed FLOPS are being underdelivered.",
|
|
"transparent": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds_prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 6,
|
|
"y": 24
|
|
},
|
|
"repeatDirection": "h",
|
|
"options": {
|
|
"graphMode": "none",
|
|
"colorMode": "background",
|
|
"justifyMode": "auto",
|
|
"textMode": "value",
|
|
"wideLayout": true,
|
|
"showPercentChange": false,
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"percentChangeColorMode": "standard",
|
|
"orientation": ""
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit",
|
|
"decimals": 2,
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"value": null,
|
|
"color": "green"
|
|
},
|
|
{
|
|
"value": 0.05,
|
|
"color": "yellow"
|
|
},
|
|
{
|
|
"value": 0.2,
|
|
"color": "red"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"type": "stat",
|
|
"id": 33,
|
|
"targets": [
|
|
{
|
|
"expr": "avg(gpu:thermal_throttle_fraction:rate5m)",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Thermal throttling (cluster avg)",
|
|
"description": "Fraction of time the fleet spent thermal-throttled. Indicates cooling/datacenter issues.",
|
|
"transparent": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds_prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 12,
|
|
"y": 24
|
|
},
|
|
"repeatDirection": "h",
|
|
"options": {
|
|
"graphMode": "none",
|
|
"colorMode": "background",
|
|
"justifyMode": "auto",
|
|
"textMode": "value",
|
|
"wideLayout": true,
|
|
"showPercentChange": false,
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"percentChangeColorMode": "standard",
|
|
"orientation": ""
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit",
|
|
"decimals": 2,
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"value": null,
|
|
"color": "green"
|
|
},
|
|
{
|
|
"value": 0.05,
|
|
"color": "yellow"
|
|
},
|
|
{
|
|
"value": 0.2,
|
|
"color": "red"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"type": "stat",
|
|
"id": 34,
|
|
"targets": [
|
|
{
|
|
"expr": "max(DCGM_FI_DEV_GPU_TEMP)",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Max GPU temperature",
|
|
"description": "Hottest GPU in the fleet right now. A10 sustained target \u003c83°C.",
|
|
"transparent": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds_prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 18,
|
|
"y": 24
|
|
},
|
|
"repeatDirection": "h",
|
|
"options": {
|
|
"graphMode": "none",
|
|
"colorMode": "background",
|
|
"justifyMode": "auto",
|
|
"textMode": "value",
|
|
"wideLayout": true,
|
|
"showPercentChange": false,
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"percentChangeColorMode": "standard",
|
|
"orientation": ""
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "celsius",
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"value": null,
|
|
"color": "green"
|
|
},
|
|
{
|
|
"value": 75,
|
|
"color": "yellow"
|
|
},
|
|
{
|
|
"value": 85,
|
|
"color": "red"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"id": 35,
|
|
"targets": [
|
|
{
|
|
"expr": "DCGM_FI_DEV_GPU_TEMP",
|
|
"legendFormat": "{{Hostname}}/{{gpu}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Temperature per GPU",
|
|
"description": "Per-GPU temperature trace. Sustained red zone = cooling issue or aggressive workload.",
|
|
"transparent": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds_prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 29
|
|
},
|
|
"repeatDirection": "h",
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": false,
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": ""
|
|
}
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "celsius",
|
|
"min": 20,
|
|
"max": 100,
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"value": null,
|
|
"color": "green"
|
|
},
|
|
{
|
|
"value": 75,
|
|
"color": "yellow"
|
|
},
|
|
{
|
|
"value": 85,
|
|
"color": "red"
|
|
}
|
|
]
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "linear",
|
|
"fillOpacity": 10,
|
|
"showPoints": "never"
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"id": 36,
|
|
"targets": [
|
|
{
|
|
"expr": "gpu:power_throttle_fraction:rate5m",
|
|
"legendFormat": "power {{Hostname}}/{{gpu}}",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "gpu:thermal_throttle_fraction:rate5m",
|
|
"legendFormat": "thermal {{Hostname}}/{{gpu}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Throttle fraction per GPU",
|
|
"description": "Red = power throttle, blue = thermal throttle. Both on same chart to compare causes.",
|
|
"transparent": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds_prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 29
|
|
},
|
|
"repeatDirection": "h",
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": false,
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": ""
|
|
}
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit",
|
|
"min": 0,
|
|
"max": 1,
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "linear",
|
|
"fillOpacity": 25,
|
|
"showPoints": "never"
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"type": "row",
|
|
"collapsed": false,
|
|
"title": "Hardware inventory",
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 37
|
|
},
|
|
"id": 40,
|
|
"panels": []
|
|
},
|
|
{
|
|
"type": "table",
|
|
"id": 41,
|
|
"targets": [
|
|
{
|
|
"expr": "group by (Hostname, modelName, gpu, UUID) (DCGM_FI_DEV_GPU_UTIL)",
|
|
"instant": true,
|
|
"range": false,
|
|
"format": "table",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "GPU inventory",
|
|
"description": "One row per physical GPU. Model / index / UUID sourced from DCGM labels.",
|
|
"transparent": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds_prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 38
|
|
},
|
|
"repeatDirection": "h",
|
|
"transformations": [
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {
|
|
"Time": true,
|
|
"Value": true,
|
|
"__name__": true,
|
|
"cluster": true,
|
|
"container": true,
|
|
"device": true,
|
|
"endpoint": true,
|
|
"instance": true,
|
|
"job": true,
|
|
"pci_bus_id": true,
|
|
"prometheus": true,
|
|
"service": true,
|
|
"tenant": true,
|
|
"tier": true,
|
|
"uid": true,
|
|
"unit": true
|
|
},
|
|
"indexByName": {
|
|
"Hostname": 0,
|
|
"UUID": 3,
|
|
"gpu": 2,
|
|
"modelName": 1
|
|
},
|
|
"renameByName": {
|
|
"Hostname": "Node",
|
|
"UUID": "UUID",
|
|
"gpu": "Index",
|
|
"modelName": "GPU Model"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"options": {
|
|
"frameIndex": 0,
|
|
"showHeader": true,
|
|
"showTypeIcons": false,
|
|
"footer": {
|
|
"show": false,
|
|
"reducer": []
|
|
},
|
|
"cellHeight": "sm"
|
|
}
|
|
}
|
|
],
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"type": "datasource",
|
|
"name": "ds_prometheus",
|
|
"label": "Prometheus",
|
|
"skipUrlSync": false,
|
|
"query": "prometheus",
|
|
"current": {
|
|
"selected": false,
|
|
"text": "default",
|
|
"value": "default"
|
|
},
|
|
"multi": false,
|
|
"allowCustomValue": true,
|
|
"includeAll": false,
|
|
"regex": "",
|
|
"auto": false,
|
|
"auto_min": "10s",
|
|
"auto_count": 30
|
|
}
|
|
]
|
|
},
|
|
"annotations": {}
|
|
}
|