vrr/cozystack
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
cozystack/dashboards/gpu/gpu-tenants.json
Arsolitt b2a8cca3bb
fix(monitoring): filter zero-valued series from active tenants count 
Address review feedback from coderabbitai on dashboards/gpu/gpu-tenants.json:39

Signed-off-by: Arsolitt <arsolitt@gmail.com>
2026-04-28 12:15:32 +03:00

1098 lines
26 KiB
JSON
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

{
"uid": "gpu-tenants",
"title": "GPU Tenant Usage",
"description": "Per-tenant GPU consumption: live allocation, utilization, tensor saturation, power, and 24h GPU-hours / kWh — admin view across all namespaces.",
"tags": [
"gpu",
"tenants",
"admin"
],
"timezone": "browser",
"editable": true,
"graphTooltip": 1,
"time": {
"from": "now-24h",
"to": "now"
},
"fiscalYearStartMonth": 0,
"schemaVersion": 42,
"panels": [
{
"type": "row",
"collapsed": false,
"title": "Tenant fleet snapshot",
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 1,
"panels": []
},
{
"type": "stat",
"id": 2,
"targets": [
{
"expr": "count(namespace:gpu_count:allocated{namespace=~\"$namespace\"} \u003e 0)",
"refId": "A"
}
],
"title": "Active tenants",
"description": "Namespaces with at least one GPU request (Pending + Running).",
"transparent": false,
"datasource": {
"type": "prometheus",
"uid": "$ds_prometheus"
},
"gridPos": {
"h": 4,
"w": 4,
"x": 0,
"y": 1
},
"repeatDirection": "h",
"options": {
"graphMode": "none",
"colorMode": "value",
"justifyMode": "auto",
"textMode": "value",
"wideLayout": true,
"showPercentChange": false,
"reduceOptions": {
"calcs": [
"lastNotNull"
]
},
"percentChangeColorMode": "standard",
"orientation": ""
},
"fieldConfig": {
"defaults": {
"unit": "short",
"thresholds": {
"mode": "absolute",
"steps": [
{
"value": null,
"color": "blue"
}
]
}
},
"overrides": []
}
},
{
"type": "stat",
"id": 3,
"targets": [
{
"expr": "sum(namespace:gpu_count:allocated{namespace=~\"$namespace\"})",
"refId": "A"
}
],
"title": "Allocated GPUs",
"description": "Sum of kube GPU requests across tenants. Billable view — includes Pending pods.",
"transparent": false,
"datasource": {
"type": "prometheus",
"uid": "$ds_prometheus"
},
"gridPos": {
"h": 4,
"w": 4,
"x": 4,
"y": 1
},
"repeatDirection": "h",
"options": {
"graphMode": "none",
"colorMode": "background",
"justifyMode": "auto",
"textMode": "value",
"wideLayout": true,
"showPercentChange": false,
"reduceOptions": {
"calcs": [
"lastNotNull"
]
},
"percentChangeColorMode": "standard",
"orientation": ""
},
"fieldConfig": {
"defaults": {
"unit": "short",
"thresholds": {
"mode": "absolute",
"steps": [
{
"value": null,
"color": "blue"
},
{
"value": 1,
"color": "green"
}
]
}
},
"overrides": []
}
},
{
"type": "stat",
"id": 4,
"targets": [
{
"expr": "cluster:gpu_util:avg",
"refId": "A"
}
],
"title": "Avg cluster GPU util",
"description": "Average NVML utilization across all GPUs in the cluster. Cluster-wide — DCGM hardware metrics cannot be attributed to workload namespaces.",
"transparent": false,
"datasource": {
"type": "prometheus",
"uid": "$ds_prometheus"
},
"gridPos": {
"h": 4,
"w": 4,
"x": 8,
"y": 1
},
"repeatDirection": "h",
"options": {
"graphMode": "area",
"colorMode": "value",
"justifyMode": "auto",
"textMode": "value",
"wideLayout": true,
"showPercentChange": false,
"reduceOptions": {
"calcs": [
"lastNotNull"
]
},
"percentChangeColorMode": "standard",
"orientation": ""
},
"fieldConfig": {
"defaults": {
"unit": "percent",
"min": 0,
"max": 100,
"thresholds": {
"mode": "absolute",
"steps": [
{
"value": null,
"color": "blue"
},
{
"value": 30,
"color": "green"
},
{
"value": 80,
"color": "orange"
}
]
}
},
"overrides": []
}
},
{
"type": "stat",
"id": 5,
"targets": [
{
"expr": "cluster:gpu_power_watts:sum",
"refId": "A"
}
],
"title": "Total GPU power",
"description": "Live power draw summed across all GPUs in the cluster. Cluster-wide — DCGM hardware metrics cannot be attributed to workload namespaces.",
"transparent": false,
"datasource": {
"type": "prometheus",
"uid": "$ds_prometheus"
},
"gridPos": {
"h": 4,
"w": 4,
"x": 12,
"y": 1
},
"repeatDirection": "h",
"options": {
"graphMode": "area",
"colorMode": "value",
"justifyMode": "auto",
"textMode": "value",
"wideLayout": true,
"showPercentChange": false,
"reduceOptions": {
"calcs": [
"lastNotNull"
]
},
"percentChangeColorMode": "standard",
"orientation": ""
},
"fieldConfig": {
"defaults": {
"unit": "watt",
"decimals": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"value": null,
"color": "green"
}
]
}
},
"overrides": []
}
},
{
"type": "stat",
"id": 6,
"targets": [
{
"expr": "sum(sum_over_time(node:power_watts:sum[24h:1m])) / 60 / 1000",
"refId": "A"
}
],
"title": "Energy (last 24h)",
"description": "Integrated cluster-wide GPU power draw over 24h. Cluster-wide — DCGM hardware metrics cannot be attributed to workload namespaces.",
"transparent": false,
"datasource": {
"type": "prometheus",
"uid": "$ds_prometheus"
},
"gridPos": {
"h": 4,
"w": 4,
"x": 16,
"y": 1
},
"repeatDirection": "h",
"options": {
"graphMode": "none",
"colorMode": "value",
"justifyMode": "auto",
"textMode": "value",
"wideLayout": true,
"showPercentChange": false,
"reduceOptions": {
"calcs": [
"lastNotNull"
]
},
"percentChangeColorMode": "standard",
"orientation": ""
},
"fieldConfig": {
"defaults": {
"unit": "kwatth",
"decimals": 1,
"thresholds": {
"mode": "absolute",
"steps": [
{
"value": null,
"color": "green"
}
]
}
},
"overrides": []
}
},
{
"type": "stat",
"id": 7,
"targets": [
{
"expr": "sum(sum_over_time(namespace:gpu_count:allocated{namespace=~\"$namespace\"}[24h:1m])) / 60",
"refId": "A"
}
],
"title": "GPU-hours (last 24h)",
"description": "Sum of (GPU count × time requested) across all tenants over the last 24h. Integrated from kube requests, so Pending pods count.",
"transparent": false,
"datasource": {
"type": "prometheus",
"uid": "$ds_prometheus"
},
"gridPos": {
"h": 4,
"w": 4,
"x": 20,
"y": 1
},
"repeatDirection": "h",
"options": {
"graphMode": "none",
"colorMode": "value",
"justifyMode": "auto",
"textMode": "value",
"wideLayout": true,
"showPercentChange": false,
"reduceOptions": {
"calcs": [
"lastNotNull"
]
},
"percentChangeColorMode": "standard",
"orientation": ""
},
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 1,
"thresholds": {
"mode": "absolute",
"steps": [
{
"value": null,
"color": "blue"
}
]
}
},
"overrides": []
}
},
{
"type": "row",
"collapsed": false,
"title": "Current allocation",
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 5
},
"id": 10,
"panels": []
},
{
"type": "bargauge",
"id": 11,
"targets": [
{
"expr": "namespace:gpu_count:allocated{namespace=~\"$namespace\"}",
"legendFormat": "{{namespace}}",
"refId": "A"
}
],
"title": "GPUs per namespace",
"description": "Kube GPU requests per tenant (billable). Includes Pending pods.",
"transparent": false,
"datasource": {
"type": "prometheus",
"uid": "$ds_prometheus"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 6
},
"repeatDirection": "h",
"options": {
"displayMode": "gradient",
"valueMode": "color",
"namePlacement": "auto",
"showUnfilled": true,
"sizing": "auto",
"minVizWidth": 8,
"minVizHeight": 16,
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": false,
"calcs": []
},
"reduceOptions": {
"calcs": [
"lastNotNull"
]
},
"maxVizHeight": 300,
"orientation": "horizontal"
},
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"value": null,
"color": "green"
}
]
}
},
"overrides": []
}
},
{
"type": "bargauge",
"id": 12,
"targets": [
{
"expr": "node:fb_used_bytes:sum",
"legendFormat": "{{Hostname}}",
"refId": "A"
}
],
"title": "VRAM used per node",
"description": "FB memory actively used per node. DCGM hardware metrics cannot be attributed to workload namespaces — shown per-node instead.",
"transparent": false,
"datasource": {
"type": "prometheus",
"uid": "$ds_prometheus"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 6
},
"repeatDirection": "h",
"options": {
"displayMode": "gradient",
"valueMode": "color",
"namePlacement": "auto",
"showUnfilled": true,
"sizing": "auto",
"minVizWidth": 8,
"minVizHeight": 16,
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": false,
"calcs": []
},
"reduceOptions": {
"calcs": [
"lastNotNull"
]
},
"maxVizHeight": 300,
"orientation": "horizontal"
},
"fieldConfig": {
"defaults": {
"unit": "bytes",
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"value": null,
"color": "green"
}
]
}
},
"overrides": []
}
},
{
"type": "row",
"collapsed": false,
"title": "Utilization (per-node)",
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 14
},
"id": 20,
"panels": []
},
{
"type": "timeseries",
"id": 21,
"targets": [
{
"expr": "node:gpu_util:avg",
"legendFormat": "{{Hostname}}",
"refId": "A"
}
],
"title": "NVML utilization per node",
"description": "Per-node NVML utilization. Stacked to show fleet-wide pressure over time. DCGM hardware metrics cannot be attributed to workload namespaces — shown per-node instead.",
"transparent": false,
"datasource": {
"type": "prometheus",
"uid": "$ds_prometheus"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 15
},
"repeatDirection": "h",
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"showLegend": false,
"calcs": [
"mean",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": ""
}
},
"fieldConfig": {
"defaults": {
"unit": "percent",
"min": 0,
"custom": {
"drawStyle": "line",
"lineInterpolation": "linear",
"fillOpacity": 40,
"showPoints": "never",
"stacking": {
"mode": "normal"
}
}
},
"overrides": []
}
},
{
"type": "timeseries",
"id": 22,
"targets": [
{
"expr": "node:tensor_active:avg * 100",
"legendFormat": "{{Hostname}}",
"refId": "A"
}
],
"title": "Tensor saturation per node",
"description": "Real tensor core load per node. Useful to spot nodes with idle tensor cores. DCGM hardware metrics cannot be attributed to workload namespaces — shown per-node instead.",
"transparent": false,
"datasource": {
"type": "prometheus",
"uid": "$ds_prometheus"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 15
},
"repeatDirection": "h",
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"showLegend": false,
"calcs": [
"mean",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": ""
}
},
"fieldConfig": {
"defaults": {
"unit": "percent",
"min": 0,
"custom": {
"drawStyle": "line",
"lineInterpolation": "linear",
"fillOpacity": 40,
"showPoints": "never",
"stacking": {
"mode": "normal"
}
}
},
"overrides": []
}
},
{
"type": "row",
"collapsed": false,
"title": "Power \u0026 allocation trend",
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 23
},
"id": 30,
"panels": []
},
{
"type": "timeseries",
"id": 31,
"targets": [
{
"expr": "node:power_watts:sum",
"legendFormat": "{{Hostname}}",
"refId": "A"
}
],
"title": "Power draw per node",
"description": "Per-node GPU power draw. Stacked to see cluster-wide energy burn over time. DCGM hardware metrics cannot be attributed to workload namespaces — shown per-node instead.",
"transparent": false,
"datasource": {
"type": "prometheus",
"uid": "$ds_prometheus"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 24
},
"repeatDirection": "h",
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"showLegend": false,
"calcs": [
"mean",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": ""
}
},
"fieldConfig": {
"defaults": {
"unit": "watt",
"min": 0,
"custom": {
"drawStyle": "line",
"lineInterpolation": "linear",
"fillOpacity": 40,
"showPoints": "never",
"stacking": {
"mode": "normal"
}
}
},
"overrides": []
}
},
{
"type": "timeseries",
"id": 32,
"targets": [
{
"expr": "namespace:gpu_count:allocated{namespace=~\"$namespace\"}",
"legendFormat": "{{namespace}}",
"refId": "A"
}
],
"title": "Allocated GPUs per namespace",
"description": "Kube-requested GPU count per tenant over time. Step-after interpolation reflects discrete allocation changes.",
"transparent": false,
"datasource": {
"type": "prometheus",
"uid": "$ds_prometheus"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 24
},
"repeatDirection": "h",
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"showLegend": false,
"calcs": [
"mean",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": ""
}
},
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0,
"custom": {
"drawStyle": "line",
"lineInterpolation": "stepAfter",
"fillOpacity": 40,
"showPoints": "never",
"stacking": {
"mode": "normal"
}
}
},
"overrides": []
}
},
{
"type": "row",
"collapsed": false,
"title": "Top consumers (live)",
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 32
},
"id": 40,
"panels": []
},
{
"type": "table",
"id": 41,
"targets": [
{
"expr": "namespace:gpu_count:allocated{namespace=~\"$namespace\"}",
"instant": true,
"range": false,
"format": "table",
"refId": "gpus"
}
],
"title": "Top consumers (allocation)",
"description": "Live ranking of tenants by GPU allocation. Hardware metrics (util, tensor, power, VRAM) cannot be attributed to workload namespaces — see Fleet and Performance dashboards for per-node/per-GPU hardware views.",
"transparent": false,
"datasource": {
"type": "prometheus",
"uid": "$ds_prometheus"
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 33
},
"repeatDirection": "h",
"transformations": [
{
"id": "organize",
"options": {
"excludeByName": {
"Time": true,
"Time 1": true,
"Time 2": true,
"Time 3": true,
"Time 4": true,
"Time 5": true,
"__name__": true,
"__name__ 1": true,
"__name__ 2": true,
"__name__ 3": true,
"__name__ 4": true,
"__name__ 5": true,
"cluster": true,
"cluster 1": true,
"cluster 2": true,
"cluster 3": true,
"cluster 4": true,
"cluster 5": true,
"endpoint": true,
"endpoint 1": true,
"endpoint 2": true,
"endpoint 3": true,
"endpoint 4": true,
"endpoint 5": true,
"instance": true,
"instance 1": true,
"instance 2": true,
"instance 3": true,
"instance 4": true,
"instance 5": true,
"job": true,
"job 1": true,
"job 2": true,
"job 3": true,
"job 4": true,
"job 5": true,
"prometheus": true,
"prometheus 1": true,
"prometheus 2": true,
"prometheus 3": true,
"prometheus 4": true,
"prometheus 5": true,
"service": true,
"service 1": true,
"service 2": true,
"service 3": true,
"service 4": true,
"service 5": true,
"tenant": true,
"tenant 1": true,
"tenant 2": true,
"tenant 3": true,
"tenant 4": true,
"tenant 5": true,
"tier": true,
"tier 1": true,
"tier 2": true,
"tier 3": true,
"tier 4": true,
"tier 5": true,
"uid": true,
"uid 1": true,
"uid 2": true,
"uid 3": true,
"uid 4": true,
"uid 5": true,
"unit": true,
"unit 1": true,
"unit 2": true,
"unit 3": true,
"unit 4": true,
"unit 5": true
},
"indexByName": {
"Value #gpus": 1,
"namespace": 0
},
"renameByName": {
"Value #gpus": "GPUs",
"namespace": "Namespace"
}
}
}
],
"options": {
"frameIndex": 0,
"showHeader": true,
"showTypeIcons": false,
"sortBy": [
{
"displayName": "GPUs",
"desc": true
}
],
"footer": {
"show": false,
"reducer": []
},
"cellHeight": "sm"
}
},
{
"type": "row",
"collapsed": false,
"title": "Historical usage (last 24h)",
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 43
},
"id": 50,
"panels": []
},
{
"type": "bargauge",
"id": 51,
"targets": [
{
"expr": "sum_over_time(namespace:gpu_count:allocated{namespace=~\"$namespace\"}[24h:1m]) / 60",
"legendFormat": "{{namespace}}",
"refId": "A"
}
],
"title": "GPU-hours per namespace (last 24h)",
"description": "Billable GPU-hours per tenant over the last 24h window.",
"transparent": false,
"datasource": {
"type": "prometheus",
"uid": "$ds_prometheus"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 44
},
"repeatDirection": "h",
"options": {
"displayMode": "gradient",
"valueMode": "color",
"namePlacement": "auto",
"showUnfilled": true,
"sizing": "auto",
"minVizWidth": 8,
"minVizHeight": 16,
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": false,
"calcs": []
},
"reduceOptions": {
"calcs": [
"lastNotNull"
]
},
"maxVizHeight": 300,
"orientation": "horizontal"
},
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 1,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"value": null,
"color": "blue"
}
]
}
},
"overrides": []
}
},
{
"type": "bargauge",
"id": 52,
"targets": [
{
"expr": "sum_over_time(node:power_watts:sum[24h:1m]) / 60 / 1000",
"legendFormat": "{{Hostname}}",
"refId": "A"
}
],
"title": "Energy per node (last 24h)",
"description": "kWh consumed per node over the last 24h. Integrated from 1-minute power samples. DCGM hardware metrics cannot be attributed to workload namespaces — shown per-node instead.",
"transparent": false,
"datasource": {
"type": "prometheus",
"uid": "$ds_prometheus"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 44
},
"repeatDirection": "h",
"options": {
"displayMode": "gradient",
"valueMode": "color",
"namePlacement": "auto",
"showUnfilled": true,
"sizing": "auto",
"minVizWidth": 8,
"minVizHeight": 16,
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": false,
"calcs": []
},
"reduceOptions": {
"calcs": [
"lastNotNull"
]
},
"maxVizHeight": 300,
"orientation": "horizontal"
},
"fieldConfig": {
"defaults": {
"unit": "kwatth",
"decimals": 2,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"value": null,
"color": "green"
}
]
}
},
"overrides": []
}
}
],
"templating": {
"list": [
{
"type": "datasource",
"name": "ds_prometheus",
"label": "Prometheus",
"skipUrlSync": false,
"query": "prometheus",
"current": {
"selected": false,
"text": "default",
"value": "default"
},
"multi": false,
"allowCustomValue": true,
"includeAll": false,
"regex": "",
"auto": false,
"auto_min": "10s",
"auto_count": 30
},
{
"type": "query",
"name": "namespace",
"label": "Namespace",
"skipUrlSync": false,
"query": "label_values(namespace:gpu_count:allocated, namespace)",
"datasource": {
"type": "prometheus",
"uid": "$ds_prometheus"
},
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"multi": true,
"allowCustomValue": true,
"refresh": 2,
"sort": 1,
"includeAll": true,
"auto": false,
"auto_min": "10s",
"auto_count": 30
}
]
},
"annotations": {}
}
Reference in a new issue View git blame Copy permalink