{ "uid": "gpu-performance", "title": "GPU Performance", "tags": [ "gpu", "dcgm" ], "timezone": "browser", "editable": true, "graphTooltip": 1, "time": { "from": "now-1h", "to": "now" }, "fiscalYearStartMonth": 0, "schemaVersion": 42, "panels": [ { "type": "row", "collapsed": false, "title": "Overview", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 1, "panels": [] }, { "type": "stat", "id": 2, "targets": [ { "expr": "cluster:gpu_count:total", "refId": "A" } ], "title": "Total GPUs", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, "repeatDirection": "h", "options": { "graphMode": "none", "colorMode": "value", "justifyMode": "auto", "textMode": "value", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" ] }, "percentChangeColorMode": "standard", "orientation": "" }, "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "blue" } ] } }, "overrides": [] } }, { "type": "stat", "id": 3, "targets": [ { "expr": "cluster:gpu_count:allocated", "refId": "A" } ], "title": "Allocated", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 }, "repeatDirection": "h", "options": { "graphMode": "none", "colorMode": "value", "justifyMode": "auto", "textMode": "value", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" ] }, "percentChangeColorMode": "standard", "orientation": "" }, "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "green" }, { "value": 1, "color": "yellow" } ] } }, "overrides": [] } }, { "type": "stat", "id": 4, "targets": [ { "expr": "cluster:gpu_util:avg", "refId": "A" } ], "title": "Average utilization", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 }, "repeatDirection": "h", "options": { "graphMode": "area", "colorMode": "value", "justifyMode": "auto", "textMode": "value", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" ] }, "percentChangeColorMode": "standard", "orientation": "" }, "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100, "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "blue" }, { "value": 30, "color": "green" }, { "value": 80, "color": "orange" } ] } }, "overrides": [] } }, { "type": "stat", "id": 5, "targets": [ { "expr": "cluster:gpu_power_watts:sum", "refId": "A" } ], "title": "Power draw", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 }, "repeatDirection": "h", "options": { "graphMode": "area", "colorMode": "value", "justifyMode": "auto", "textMode": "value", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" ] }, "percentChangeColorMode": "standard", "orientation": "" }, "fieldConfig": { "defaults": { "unit": "watt", "decimals": 0, "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "green" } ] } }, "overrides": [] } }, { "type": "row", "collapsed": false, "title": "Utilization", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, "id": 10, "panels": [] }, { "type": "timeseries", "id": 11, "targets": [ { "expr": "DCGM_FI_DEV_GPU_UTIL{Hostname=~\"$Hostname\"}", "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", "refId": "A" } ], "title": "GPU Utilization (NVML)", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, "repeatDirection": "h", "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": false, "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "" } }, "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "fillOpacity": 10, "showPoints": "never", "spanNulls": false } }, "overrides": [] } }, { "type": "timeseries", "id": 12, "targets": [ { "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{Hostname=~\"$Hostname\"} * 100", "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", "refId": "A" } ], "title": "Tensor Pipe Active (realistic load for LLM/AI)", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, "repeatDirection": "h", "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": false, "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "" } }, "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "fillOpacity": 10, "showPoints": "never", "spanNulls": false } }, "overrides": [] } }, { "type": "timeseries", "id": 13, "targets": [ { "expr": "DCGM_FI_PROF_GR_ENGINE_ACTIVE{Hostname=~\"$Hostname\"} * 100", "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", "refId": "A" } ], "title": "Graphics Engine Active", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 14 }, "repeatDirection": "h", "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": false, "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "" } }, "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "fillOpacity": 10, "showPoints": "never", "spanNulls": false } }, "overrides": [] } }, { "type": "timeseries", "id": 14, "targets": [ { "expr": "DCGM_FI_DEV_MEM_COPY_UTIL{Hostname=~\"$Hostname\"}", "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", "refId": "A" } ], "title": "Memory Copy Utilization", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 14 }, "repeatDirection": "h", "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": false, "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "" } }, "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "fillOpacity": 10, "showPoints": "never", "spanNulls": false } }, "overrides": [] } }, { "type": "row", "collapsed": false, "title": "Memory", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 }, "id": 20, "panels": [] }, { "type": "timeseries", "id": 21, "targets": [ { "expr": "DCGM_FI_DEV_FB_USED{Hostname=~\"$Hostname\"} * 1048576", "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", "refId": "A" } ], "title": "VRAM Used", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 23 }, "repeatDirection": "h", "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": false, "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "" } }, "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "drawStyle": "line", "lineInterpolation": "linear", "fillOpacity": 25, "showPoints": "never" } }, "overrides": [] } }, { "type": "timeseries", "id": 22, "targets": [ { "expr": "DCGM_FI_DEV_FB_FREE{Hostname=~\"$Hostname\"} * 1048576", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } ], "title": "VRAM Free", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 23 }, "repeatDirection": "h", "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": false, "calcs": [ "mean", "min" ] }, "tooltip": { "mode": "multi", "sort": "" } }, "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "drawStyle": "line", "lineInterpolation": "linear", "fillOpacity": 10, "showPoints": "never" } }, "overrides": [] } }, { "type": "row", "collapsed": false, "title": "Power \u0026 Temperature", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 31 }, "id": 30, "panels": [] }, { "type": "timeseries", "id": 31, "targets": [ { "expr": "DCGM_FI_DEV_POWER_USAGE{Hostname=~\"$Hostname\"}", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } ], "title": "Power Usage per GPU", "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 }, "repeatDirection": "h", "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": false, "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "" } }, "fieldConfig": { "defaults": { "unit": "watt", "custom": { "drawStyle": "line", "lineInterpolation": "linear", "fillOpacity": 10, "showPoints": "never" } }, "overrides": [] } }, { "type": "timeseries", "id": 32, "targets": [ { "expr": "DCGM_FI_DEV_GPU_TEMP{Hostname=~\"$Hostname\"}", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } ], "title": "GPU Temperature", "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 32 }, "repeatDirection": "h", "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": false, "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "" } }, "fieldConfig": { "defaults": { "unit": "celsius", "min": 20, "max": 100, "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "green" }, { "value": 75, "color": "yellow" }, { "value": 85, "color": "red" } ] }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "fillOpacity": 10, "showPoints": "never" } }, "overrides": [] } }, { "type": "row", "collapsed": false, "title": "Health", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 40 }, "id": 40, "panels": [] }, { "type": "stat", "id": 41, "targets": [ { "expr": "max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS{Hostname=~\"$Hostname\"})", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } ], "title": "XID errors (latest)", "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 5, "w": 8, "x": 0, "y": 41 }, "repeatDirection": "h", "options": { "graphMode": "none", "colorMode": "background", "justifyMode": "auto", "textMode": "value_and_name", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" ] }, "percentChangeColorMode": "standard", "orientation": "" }, "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "green" }, { "value": 1, "color": "red" } ] } }, "overrides": [] } }, { "type": "timeseries", "id": 42, "targets": [ { "expr": "rate(DCGM_FI_DEV_POWER_VIOLATION{Hostname=~\"$Hostname\"}[5m])", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } ], "title": "Power Violation (µs/s throttled due to power)", "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 5, "w": 8, "x": 8, "y": 41 }, "repeatDirection": "h", "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": false, "calcs": [] }, "tooltip": { "mode": "multi", "sort": "" } }, "fieldConfig": { "defaults": { "unit": "µs", "custom": { "drawStyle": "line", "lineInterpolation": "linear", "fillOpacity": 10, "showPoints": "never" } }, "overrides": [] } }, { "type": "timeseries", "id": 43, "targets": [ { "expr": "rate(DCGM_FI_DEV_THERMAL_VIOLATION{Hostname=~\"$Hostname\"}[5m])", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } ], "title": "Thermal Violation (µs/s throttled due to thermals)", "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 5, "w": 8, "x": 16, "y": 41 }, "repeatDirection": "h", "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": false, "calcs": [] }, "tooltip": { "mode": "multi", "sort": "" } }, "fieldConfig": { "defaults": { "unit": "µs", "custom": { "drawStyle": "line", "lineInterpolation": "linear", "fillOpacity": 10, "showPoints": "never" } }, "overrides": [] } } ], "templating": { "list": [ { "type": "datasource", "name": "ds_prometheus", "label": "Prometheus", "skipUrlSync": false, "query": "prometheus", "current": { "selected": false, "text": "default", "value": "default" }, "multi": false, "allowCustomValue": true, "includeAll": false, "regex": "", "auto": false, "auto_min": "10s", "auto_count": 30 }, { "type": "query", "name": "Hostname", "label": "Host", "skipUrlSync": false, "query": "label_values(DCGM_FI_DEV_GPU_UTIL, Hostname)", "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "current": { "selected": true, "text": "All", "value": "$__all" }, "multi": true, "allowCustomValue": true, "refresh": 2, "sort": 1, "includeAll": true, "auto": false, "auto_min": "10s", "auto_count": 30 } ] }, "annotations": {} }