From cb3e0adf6438a2520bf7a943c798da7de8104c42 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Fri, 17 Apr 2026 16:55:42 +0300 Subject: [PATCH 001/130] feat(monitoring): add GPU recording rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add VMRule with recording rules for DCGM metrics at three levels: - gpu.recording.30s: per-GPU aggregates over 30s windows - gpu.recording.cluster.1m: cluster-wide totals for overview panels - gpu.recording.namespace.1m: per-namespace aggregates for tenant reporting and GPU-hour calculations The rules are safe to ship on clusters without DCGM — they evaluate to empty series when no matching metrics are scraped. Used by dashboards/gpu/gpu-performance.json. Assisted-By: Claude Signed-off-by: Arsolitt --- .../alerts/gpu-recording.rules.yaml | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml new file mode 100644 index 00000000..871a46fc --- /dev/null +++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml @@ -0,0 +1,57 @@ +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMRule +metadata: + name: alerts-gpu-recording.rules +spec: + groups: + - name: gpu.recording.30s + interval: 30s + rules: + - record: gpu:util:avg + expr: avg by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_DEV_GPU_UTIL) + - record: gpu:mem_copy_util:avg + expr: avg by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_DEV_MEM_COPY_UTIL) + - record: gpu:fb_used_bytes:max + expr: max by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_DEV_FB_USED) * 1048576 + - record: gpu:fb_free_bytes:max + expr: max by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_DEV_FB_FREE) * 1048576 + - record: gpu:power_watts:avg + expr: avg by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_DEV_POWER_USAGE) + - record: gpu:temp_celsius:max + expr: max by (Hostname, gpu, UUID, modelName) (DCGM_FI_DEV_GPU_TEMP) + - record: gpu:tensor_active:avg + expr: avg by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_PROF_PIPE_TENSOR_ACTIVE) + - record: gpu:gr_engine_active:avg + expr: avg by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_PROF_GR_ENGINE_ACTIVE) + + - name: gpu.recording.cluster.1m + interval: 1m + rules: + - record: cluster:gpu_count:total + expr: count(group by (UUID) (DCGM_FI_DEV_GPU_UTIL)) + - record: cluster:gpu_count:allocated + expr: count(group by (UUID) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"})) + - record: cluster:gpu_count:free + expr: cluster:gpu_count:total - (cluster:gpu_count:allocated or vector(0)) + - record: cluster:gpu_util:avg + expr: avg(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}) + - record: cluster:gpu_power_watts:sum + expr: sum(DCGM_FI_DEV_POWER_USAGE) + + - name: gpu.recording.namespace.1m + interval: 1m + rules: + - record: namespace:gpu_count:sum + expr: count by (namespace) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}) + - record: namespace:gpu_util:avg + expr: avg by (namespace) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}) + - record: namespace:tensor_active:avg + expr: avg by (namespace) (DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace!="", namespace!~"cozy-.*|kube-.*"}) + - record: namespace:fb_used_bytes:sum + expr: sum by (namespace) (DCGM_FI_DEV_FB_USED{namespace!="", namespace!~"cozy-.*|kube-.*"}) * 1048576 + - record: namespace:power_watts:sum + expr: sum by (namespace) (DCGM_FI_DEV_POWER_USAGE{namespace!="", namespace!~"cozy-.*|kube-.*"}) + - record: namespace:energy_joules:sum + expr: sum by (namespace) (DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION{namespace!="", namespace!~"cozy-.*|kube-.*"}) / 1000 + - record: namespace:gpu_allocated_count:gauge + expr: count by (namespace) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}) From d1d19e9978f965b65ed910b3b6486469e2d07f8d Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Fri, 17 Apr 2026 16:55:51 +0300 Subject: [PATCH 002/130] feat(monitoring): add GPU performance Grafana dashboard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the gpu/gpu-performance dashboard and register it in the infra dashboard list. The dashboard provides: - Cluster overview: total/allocated GPUs, average utilization, aggregate power draw. - Utilization: GPU util (NVML), tensor pipe active (realistic load for LLM/AI workloads), graphics engine active, memory copy util. - Memory: VRAM used/free per GPU. - Power and temperature per GPU. - Health: XID errors, power and thermal throttling. The dashboard relies on DCGM_FI_* metrics plus the cluster:gpu_* and namespace:gpu_* recording rules added to monitoring-agents. The JSON follows the cozystack convention — Prometheus data source is selected via the $ds_prometheus template variable. Assisted-By: Claude Signed-off-by: Arsolitt --- dashboards/gpu/gpu-performance.json | 935 ++++++++++++++++++ .../system/monitoring/dashboards-infra.list | 1 + 2 files changed, 936 insertions(+) create mode 100644 dashboards/gpu/gpu-performance.json diff --git a/dashboards/gpu/gpu-performance.json b/dashboards/gpu/gpu-performance.json new file mode 100644 index 00000000..0b458d4e --- /dev/null +++ b/dashboards/gpu/gpu-performance.json @@ -0,0 +1,935 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "schemaVersion": 39, + "tags": [ + "gpu", + "dcgm" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "hide": 0, + "includeAll": false, + "label": "Prometheus", + "multi": false, + "name": "ds_prometheus", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "definition": "label_values(DCGM_FI_DEV_GPU_UTIL, Hostname)", + "hide": 0, + "includeAll": true, + "label": "Host", + "multi": true, + "name": "Hostname", + "options": [], + "query": { + "query": "label_values(DCGM_FI_DEV_GPU_UTIL, Hostname)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "definition": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)", + "hide": 0, + "includeAll": true, + "label": "Namespace", + "multi": true, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "GPU Performance", + "uid": "gpu-performance", + "weekStart": "", + "panels": [ + { + "type": "row", + "title": "Overview", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "collapsed": false, + "id": 1, + "panels": [] + }, + { + "type": "stat", + "title": "Total GPUs", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 2, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "targets": [ + { + "expr": "cluster:gpu_count:total", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + } + }, + { + "type": "stat", + "title": "Allocated", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 3, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "targets": [ + { + "expr": "cluster:gpu_count:allocated", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + } + }, + { + "type": "stat", + "title": "Average utilization", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 4, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "targets": [ + { + "expr": "cluster:gpu_util:avg", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "green", + "value": 30 + }, + { + "color": "orange", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + } + }, + { + "type": "stat", + "title": "Power draw", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 5, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "targets": [ + { + "expr": "cluster:gpu_power_watts:sum", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "watt", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + } + }, + { + "type": "row", + "title": "Utilization", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "collapsed": false, + "id": 10, + "panels": [] + }, + { + "type": "timeseries", + "title": "GPU Utilization (NVML)", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 11, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "targets": [ + { + "expr": "DCGM_FI_DEV_GPU_UTIL{Hostname=~\"$Hostname\", namespace=~\"$namespace\"}", + "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "timeseries", + "title": "Tensor Pipe Active (realistic load for LLM/AI)", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 12, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "targets": [ + { + "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 100", + "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "timeseries", + "title": "Graphics Engine Active", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 13, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "targets": [ + { + "expr": "DCGM_FI_PROF_GR_ENGINE_ACTIVE{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 100", + "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "timeseries", + "title": "Memory Copy Utilization", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 14, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "targets": [ + { + "expr": "DCGM_FI_DEV_MEM_COPY_UTIL{Hostname=~\"$Hostname\", namespace=~\"$namespace\"}", + "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "row", + "title": "Memory", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "collapsed": false, + "id": 20, + "panels": [] + }, + { + "type": "timeseries", + "title": "VRAM Used", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 21, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "targets": [ + { + "expr": "DCGM_FI_DEV_FB_USED{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 1048576", + "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 25, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "timeseries", + "title": "VRAM Free", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 22, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "targets": [ + { + "expr": "DCGM_FI_DEV_FB_FREE{Hostname=~\"$Hostname\"} * 1048576", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "min" + ] + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "row", + "title": "Power & Temperature", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "collapsed": false, + "id": 30, + "panels": [] + }, + { + "type": "timeseries", + "title": "Power Usage per GPU", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 31, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "targets": [ + { + "expr": "DCGM_FI_DEV_POWER_USAGE{Hostname=~\"$Hostname\"}", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "watt", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "timeseries", + "title": "GPU Temperature", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 32, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "targets": [ + { + "expr": "DCGM_FI_DEV_GPU_TEMP{Hostname=~\"$Hostname\"}", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "celsius", + "min": 20, + "max": 100, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 75 + }, + { + "color": "red", + "value": 85 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "row", + "title": "Health", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "collapsed": false, + "id": 40, + "panels": [] + }, + { + "type": "stat", + "title": "XID errors (latest)", + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 41 + }, + "id": 41, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "targets": [ + { + "expr": "max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS{Hostname=~\"$Hostname\"})", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + } + }, + { + "type": "timeseries", + "title": "Power Violation (\u00b5s/s throttled due to power)", + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 41 + }, + "id": 42, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "targets": [ + { + "expr": "rate(DCGM_FI_DEV_POWER_VIOLATION{Hostname=~\"$Hostname\"}[5m])", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "\u00b5s", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "timeseries", + "title": "Thermal Violation (\u00b5s/s throttled due to thermals)", + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 41 + }, + "id": 43, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "targets": [ + { + "expr": "rate(DCGM_FI_DEV_THERMAL_VIOLATION{Hostname=~\"$Hostname\"}[5m])", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "\u00b5s", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + } + ] +} \ No newline at end of file diff --git a/packages/system/monitoring/dashboards-infra.list b/packages/system/monitoring/dashboards-infra.list index b5b9f52a..581f9a05 100644 --- a/packages/system/monitoring/dashboards-infra.list +++ b/packages/system/monitoring/dashboards-infra.list @@ -32,3 +32,4 @@ hubble/overview hubble/dns-namespace hubble/l7-http-metrics hubble/network-overview +gpu/gpu-performance From 5d6654c6f4eb63447d91bbf448e6116317228931 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Fri, 17 Apr 2026 16:56:03 +0300 Subject: [PATCH 003/130] docs(gpu-operator): add native-pod Talos reference manifests Add reference manifests (not templates) under packages/system/gpu-operator/examples/ documenting one working configuration for running CUDA workloads directly in pods on a Talos cluster, with DCGM metrics that drive the gpu/gpu-performance dashboard. - values-native-talos.yaml: Cozystack Package values that disable the sandbox path, enable the device plugin, and wire DCGM to the custom metrics ConfigMap. - dcgm-custom-metrics.yaml: ConfigMap extending the default DCGM CSV with profiling, ECC, throttling and energy counters used by the dashboard and recording rules. - nvidia-driver-compat.yaml: DaemonSet that stages libnvidia-ml.so.1 and nvidia-smi from the Talos glibc tree into a location the gpu-operator validator inspects. Workaround for NVIDIA/gpu-operator#1687. - README.md: explains why these are shipped as references rather than first-class templates (sandbox vs native is a deployment choice), and how the pieces connect. The out-of-the-box values-talos.yaml still targets the sandbox (VFIO passthrough) scenario. Operators who want native pod GPU workloads can start from these references. Assisted-By: Claude Signed-off-by: Arsolitt --- .../system/gpu-operator/examples/README.md | 69 ++++++++++++++++ .../examples/dcgm-custom-metrics.yaml | 82 +++++++++++++++++++ .../examples/nvidia-driver-compat.yaml | 72 ++++++++++++++++ .../examples/values-native-talos.yaml | 45 ++++++++++ 4 files changed, 268 insertions(+) create mode 100644 packages/system/gpu-operator/examples/README.md create mode 100644 packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml create mode 100644 packages/system/gpu-operator/examples/nvidia-driver-compat.yaml create mode 100644 packages/system/gpu-operator/examples/values-native-talos.yaml diff --git a/packages/system/gpu-operator/examples/README.md b/packages/system/gpu-operator/examples/README.md new file mode 100644 index 00000000..dac3703e --- /dev/null +++ b/packages/system/gpu-operator/examples/README.md @@ -0,0 +1,69 @@ +# GPU operator — native pod workload on Talos (reference) + +The files in this directory are **not** templates. They are reference +artifacts that document one working configuration for running GPU +workloads directly in pods on a Talos-based Cozystack cluster, together +with the DCGM metrics needed by the `gpu/gpu-performance` Grafana +dashboard. + +The out-of-the-box `values-talos.yaml` for this package targets the +sandbox (VFIO passthrough to KubeVirt VMs) scenario. The files here +illustrate an alternative — running CUDA workloads in regular pods with +the NVIDIA device plugin — and the workarounds it currently requires on +Talos. + +## Files + +- [`values-native-talos.yaml`](./values-native-talos.yaml) — Cozystack + `Package` values that disable sandbox workloads, enable the device + plugin, point `hostPaths.driverInstallDir` at the staging location + used by the compat DaemonSet, and wire DCGM to the custom metrics + ConfigMap. +- [`dcgm-custom-metrics.yaml`](./dcgm-custom-metrics.yaml) — `ConfigMap` + with a DCGM metrics CSV that adds profiling, ECC, throttling and + energy counters on top of the upstream defaults. Required by the + recording rules in `packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml` + and by several panels in the `gpu/gpu-performance` dashboard. +- [`nvidia-driver-compat.yaml`](./nvidia-driver-compat.yaml) — DaemonSet + that stages `libnvidia-ml.so.1` and `nvidia-smi` from the Talos glibc + tree into a path where the NVIDIA GPU Operator validator expects + them. See the "Why the compat DaemonSet exists" section below. + +## Why these are reference, not templates + +Shipping these as first-class templates would silently impose +assumptions that do not hold for every user: + +- Whether the NVIDIA Talos system extension is installed on the nodes. +- Whether GPUs are exposed directly to pods or passed through to VMs. +- The exact path the installed driver ends up at (depends on the + extension version and Talos release). + +The sandbox-oriented `values-talos.yaml` remains the default. Operators +who want native pod GPU workloads can start from this directory and +adapt as needed. + +## Why the compat DaemonSet exists + +The NVIDIA GPU Operator validator checks for `libnvidia-ml.so.1` and +`bin/nvidia-smi` in the path given by `hostPaths.driverInstallDir`. +Talos installs them under `/usr/local/glibc/usr/lib/` and +`/usr/local/bin/`, which the validator does not look at. Until upstream +addresses [NVIDIA/gpu-operator#1687][1], the DaemonSet copies those +files into a directory the validator does inspect and creates the +`.driver-ctr-ready` flag file so the validator proceeds. + +[1]: https://github.com/NVIDIA/gpu-operator/issues/1687 + +## How the dashboard and recording rules fit in + +- `dashboards/gpu/gpu-performance.json` expects `DCGM_FI_*` metrics, + including profiling series (`DCGM_FI_PROF_PIPE_TENSOR_ACTIVE`, + `DCGM_FI_PROF_GR_ENGINE_ACTIVE`) and throttling counters + (`DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION`). + These are only emitted when DCGM Exporter is started with the custom + CSV in `dcgm-custom-metrics.yaml`. +- `packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml` + precomputes cluster-wide and per-namespace aggregations used by the + overview panels of the dashboard. The rules are safe to ship on any + cluster — they evaluate to empty series when DCGM is not scraped. diff --git a/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml b/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml new file mode 100644 index 00000000..14b89b5c --- /dev/null +++ b/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml @@ -0,0 +1,82 @@ +# Custom DCGM Exporter metrics CSV. Referenced from +# examples/values-native-talos.yaml via dcgmExporter.config.name. +# +# Extends the upstream default set with profiling counters, ECC, page +# retirement, row remap, energy and throttling violations — everything +# the gpu/gpu-performance dashboard and the GPU recording rules in +# monitoring-agents expect. +apiVersion: v1 +kind: ConfigMap +metadata: + name: dcgm-custom-metrics + namespace: cozy-gpu-operator +data: + dcgm-metrics.csv: | + # Format + # If line starts with a '#' it is considered a comment + # DCGM FIELD, Prometheus metric type, help message + + # Clocks + DCGM_FI_DEV_SM_CLOCK, gauge, SM clock frequency (in MHz). + DCGM_FI_DEV_MEM_CLOCK, gauge, Memory clock frequency (in MHz). + + # Temperature + DCGM_FI_DEV_MEMORY_TEMP, gauge, Memory temperature (in C). + DCGM_FI_DEV_GPU_TEMP, gauge, GPU temperature (in C). + + # Power + DCGM_FI_DEV_POWER_USAGE, gauge, Power draw (in W). + DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION, counter, Total energy consumption since boot (in mJ). + + # PCIE + DCGM_FI_DEV_PCIE_REPLAY_COUNTER, counter, Total number of PCIe retries. + + # Utilization (the sample period varies depending on the product) + DCGM_FI_DEV_GPU_UTIL, gauge, GPU utilization (in %). + DCGM_FI_DEV_MEM_COPY_UTIL, gauge, Memory utilization (in %). + DCGM_FI_DEV_ENC_UTIL, gauge, Encoder utilization (in %). + DCGM_FI_DEV_DEC_UTIL, gauge, Decoder utilization (in %). + + # Errors and violations + DCGM_FI_DEV_XID_ERRORS, gauge, Value of the last XID error encountered. + + # Memory usage + DCGM_FI_DEV_FB_FREE, gauge, Framebuffer memory free (in MiB). + DCGM_FI_DEV_FB_USED, gauge, Framebuffer memory used (in MiB). + DCGM_FI_DEV_FB_RESERVED, gauge, Framebuffer memory reserved (in MiB). + + # ECC (supported on datacenter-class GPUs such as A10) + DCGM_FI_DEV_ECC_SBE_VOL_TOTAL, counter, Total number of single-bit volatile ECC errors. + DCGM_FI_DEV_ECC_DBE_VOL_TOTAL, counter, Total number of double-bit volatile ECC errors. + DCGM_FI_DEV_ECC_SBE_AGG_TOTAL, counter, Total number of single-bit persistent ECC errors. + DCGM_FI_DEV_ECC_DBE_AGG_TOTAL, counter, Total number of double-bit persistent ECC errors. + + # Retired pages + DCGM_FI_DEV_RETIRED_SBE, counter, Total number of retired pages due to single-bit errors. + DCGM_FI_DEV_RETIRED_DBE, counter, Total number of retired pages due to double-bit errors. + DCGM_FI_DEV_RETIRED_PENDING, counter, Total number of pages pending retirement. + + # Row remapping (not applicable to GDDR6 but left for datacenter GPUs) + DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for uncorrectable errors. + DCGM_FI_DEV_CORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for correctable errors. + DCGM_FI_DEV_ROW_REMAP_FAILURE, gauge, Whether remapping of rows has failed. + + # Throttle / violation counters (crucial for SLA and tenant monitoring) + DCGM_FI_DEV_POWER_VIOLATION, counter, Throttling duration due to power constraints (in us). + DCGM_FI_DEV_THERMAL_VIOLATION, counter, Throttling duration due to thermal constraints (in us). + DCGM_FI_DEV_SYNC_BOOST_VIOLATION, counter, Throttling duration due to sync-boost constraints (in us). + DCGM_FI_DEV_BOARD_LIMIT_VIOLATION, counter, Throttling duration due to board limit constraints (in us). + DCGM_FI_DEV_LOW_UTIL_VIOLATION, counter, Throttling duration due to low utilization (in us). + DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (in us). + + # NVLink — enable only for GPUs that actually have NVLink (A10 has none). + # DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL, counter, Total number of NVLink bandwidth counters for all lanes. + + # DCP (profiling) metrics — supported from Ampere onwards. + DCGM_FI_PROF_GR_ENGINE_ACTIVE, gauge, Ratio of time the graphics engine is active. + DCGM_FI_PROF_SM_ACTIVE, gauge, The ratio of cycles an SM has at least 1 warp assigned. + DCGM_FI_PROF_SM_OCCUPANCY, gauge, The ratio of number of warps resident on an SM. + DCGM_FI_PROF_PIPE_TENSOR_ACTIVE, gauge, Ratio of cycles the tensor (HMMA) pipe is active. + DCGM_FI_PROF_DRAM_ACTIVE, gauge, Ratio of cycles the device memory interface is active sending or receiving data. + DCGM_FI_PROF_PCIE_TX_BYTES, counter, The number of bytes of active PCIe tx (transmit) data including both header and payload. + DCGM_FI_PROF_PCIE_RX_BYTES, counter, The number of bytes of active PCIe rx (read) data including both header and payload. diff --git a/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml b/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml new file mode 100644 index 00000000..e4718e0a --- /dev/null +++ b/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml @@ -0,0 +1,72 @@ +# Workaround for https://github.com/NVIDIA/gpu-operator/issues/1687 +# +# On Talos, the NVIDIA system extension installs driver files under +# /usr/local/glibc/usr/lib/ and /usr/local/bin/. The GPU Operator +# validator only looks for libnvidia-ml.so.1 and bin/nvidia-smi under +# the path configured as hostPaths.driverInstallDir. This DaemonSet +# stages the required files into /var/nvidia-driver on each node and +# creates the .driver-ctr-ready flag so the validator proceeds. +# +# Paired with examples/values-native-talos.yaml, which sets +# hostPaths.driverInstallDir to /var/nvidia-driver. +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: nvidia-driver-compat + namespace: cozy-gpu-operator + labels: + app: nvidia-driver-compat +spec: + selector: + matchLabels: + app: nvidia-driver-compat + template: + metadata: + labels: + app: nvidia-driver-compat + spec: + hostPID: true + priorityClassName: system-node-critical + tolerations: + - operator: Exists + initContainers: + - name: create-driver-tree + image: busybox:1.37 + command: + - sh + - -c + - | + set -e + GLIBC_LIB="/host/usr/local/glibc/usr/lib" + SRC_BIN="/host/usr/local/bin" + DST="/host/var/nvidia-driver" + + mkdir -p "$DST/bin" + + if [ -f "$GLIBC_LIB/libnvidia-ml.so.1" ]; then + cp -f "$GLIBC_LIB/libnvidia-ml.so.1" "$DST/libnvidia-ml.so.1" + echo "Copied libnvidia-ml.so.1" + fi + + if [ -f "$SRC_BIN/nvidia-smi" ]; then + cp -f "$SRC_BIN/nvidia-smi" "$DST/bin/nvidia-smi" + chmod +x "$DST/bin/nvidia-smi" + echo "Copied nvidia-smi" + fi + + mkdir -p /host/run/nvidia/validations + touch /host/run/nvidia/validations/.driver-ctr-ready + echo "Created driver-ctr-ready flag" + echo "Done" + securityContext: + privileged: true + volumeMounts: + - name: host-root + mountPath: /host + containers: + - name: pause + image: registry.k8s.io/pause:3.10 + volumes: + - name: host-root + hostPath: + path: / diff --git a/packages/system/gpu-operator/examples/values-native-talos.yaml b/packages/system/gpu-operator/examples/values-native-talos.yaml new file mode 100644 index 00000000..86e436c4 --- /dev/null +++ b/packages/system/gpu-operator/examples/values-native-talos.yaml @@ -0,0 +1,45 @@ +# Cozystack Package values for running GPU workloads natively in pods +# on Talos. This is the counterpart to values-talos.yaml, which targets +# the sandbox (VFIO passthrough) scenario. +# +# Prerequisites: +# - NVIDIA Talos system extension installed on GPU nodes. +# - examples/nvidia-driver-compat.yaml deployed to stage driver files +# where the gpu-operator validator looks for them. +# - examples/dcgm-custom-metrics.yaml applied so DCGM Exporter exports +# the full set of metrics used by the dashboard and recording rules. +apiVersion: cozystack.io/v1alpha1 +kind: Package +metadata: + name: cozystack.gpu-operator +spec: + variant: default + components: + gpu-operator: + values: + gpu-operator: + # The compat DaemonSet stages libnvidia-ml.so.1 and nvidia-smi + # under /var/nvidia-driver. Point the validator at that path. + hostPaths: + driverInstallDir: "/var/nvidia-driver" + # Disable the sandbox path — workloads run in pods, not VMs. + sandboxWorkloads: + enabled: false + vfioManager: + enabled: false + # The Talos extension provides the driver and runtime hooks, + # so the operator's own toolkit and driver components must be + # switched off to avoid conflicting installations. + toolkit: + enabled: false + devicePlugin: + enabled: true + # Export full set of DCGM metrics using the ConfigMap in + # examples/dcgm-custom-metrics.yaml. + dcgmExporter: + serviceMonitor: + enabled: true + interval: "15s" + honorLabels: true + config: + name: dcgm-custom-metrics From c1b9a06a3647c48df5a8e25327c38c05425504c8 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Fri, 17 Apr 2026 17:08:18 +0300 Subject: [PATCH 004/130] =?UTF-8?q?feat(monitoring):=20expand=20GPU=20dash?= =?UTF-8?q?boards=20=E2=80=94=20efficiency=20and=20quotas?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Revise gpu-performance and add two new dashboards, registered in dashboards-infra.list: - gpu-efficiency (GPU Efficiency Score) — utilization vs. capacity and workload efficiency signals. - gpu-quotas (GPU Quotas & Allocation) — per-namespace requested vs. used GPUs for tenant capacity planning. All three dashboards use the $ds_prometheus template variable, per the project convention. Assisted-By: Claude Signed-off-by: Arsolitt --- dashboards/gpu/gpu-efficiency.json | 1131 ++++++++++++ dashboards/gpu/gpu-performance.json | 1523 +++++++++++------ dashboards/gpu/gpu-quotas.json | 822 +++++++++ .../system/monitoring/dashboards-infra.list | 2 + 4 files changed, 2956 insertions(+), 522 deletions(-) create mode 100644 dashboards/gpu/gpu-efficiency.json create mode 100644 dashboards/gpu/gpu-quotas.json diff --git a/dashboards/gpu/gpu-efficiency.json b/dashboards/gpu/gpu-efficiency.json new file mode 100644 index 00000000..cc2bca6c --- /dev/null +++ b/dashboards/gpu/gpu-efficiency.json @@ -0,0 +1,1131 @@ +{ + "__inputs": [ + { + "name": "DS_VM-SHORTTERM", + "label": "vm-shortterm", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.4.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Tensor saturation, util/watt and throttling \u2014 reveals inefficient GPU workloads", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "\u041e\u0431\u0449\u0438\u0435 \u043c\u0435\u0442\u0440\u0438\u043a\u0438 \u044d\u0444\u0444\u0435\u043a\u0442\u0438\u0432\u043d\u043e\u0441\u0442\u0438", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "\u0421\u0440\u0435\u0434\u043d\u044f\u044f \u0437\u0430\u0433\u0440\u0443\u0437\u043a\u0430 \u0442\u0435\u043d\u0437\u043e\u0440\u043d\u044b\u0445 \u044f\u0434\u0435\u0440. <10% \u2014 GPU \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u044e\u0442\u0441\u044f \u043d\u0435\u044d\u0444\u0444\u0435\u043a\u0442\u0438\u0432\u043d\u043e (\u043a\u043b\u0438\u0435\u043d\u0442\u044b \u043c\u043e\u0433\u043b\u0438 \u0431\u044b \u043f\u0435\u0440\u0435\u0435\u0445\u0430\u0442\u044c \u043d\u0430 CPU \u0438\u043b\u0438 \u043e\u043f\u0442\u0438\u043c\u0438\u0437\u0438\u0440\u043e\u0432\u0430\u0442\u044c \u043a\u043e\u0434).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 10 + }, + { + "color": "yellow", + "value": 30 + }, + { + "color": "green", + "value": 60 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "avg(pod:tensor_saturation:avg5m)", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + } + } + ], + "title": "Cluster Tensor Saturation", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + }, + "description": "NVML utilization % \u043d\u0430 \u043a\u0430\u0436\u0434\u044b\u0439 \u0432\u0430\u0442\u0442. \u0412\u044b\u0441\u043e\u043a\u043e\u0435 \u0437\u043d\u0430\u0447\u0435\u043d\u0438\u0435 = \u044d\u0444\u0444\u0435\u043a\u0442\u0438\u0432\u043d\u0430\u044f \u043d\u0430\u0433\u0440\u0443\u0437\u043a\u0430.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 3, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "avg(pod:util_per_watt:avg5m)", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + } + } + ], + "title": "Avg Utilization per Watt", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "\u0414\u043e\u043b\u044f \u0432\u0440\u0435\u043c\u0435\u043d\u0438, \u043a\u043e\u0433\u0434\u0430 GPU \u0443\u043f\u0438\u0440\u0430\u044e\u0442\u0441\u044f \u0432 TDP \u0438 \u0442\u0435\u0440\u044f\u044e\u0442 \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c. >5% \u2014 \u043a\u043b\u0438\u0435\u043d\u0442\u044b \u043d\u0435\u0434\u043e\u043f\u043e\u043b\u0443\u0447\u0430\u044e\u0442 FLOPS.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "red", + "value": 20 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "avg(gpu:power_throttle_fraction:rate5m) * 100", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + } + } + ], + "title": "Avg Power Throttling", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 10, + "panels": [], + "title": "NVML vs Tensor (\u043a\u0442\u043e \u043e\u0431\u043c\u0430\u043d\u044b\u0432\u0430\u0435\u0442\u0441\u044f)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + }, + "description": "\u041a\u043b\u0430\u0441\u0441\u0438\u0447\u0435\u0441\u043a\u0430\u044f \u043c\u0435\u0442\u0440\u0438\u043a\u0430 \u0443\u0442\u0438\u043b\u0438\u0437\u0430\u0446\u0438\u0438. \u041f\u043e\u043a\u0430\u0437\u044b\u0432\u0430\u0435\u0442 \u0437\u0430\u043d\u044f\u0442\u043e\u0441\u0442\u044c \u043b\u044e\u0431\u043e\u0433\u043e \u0434\u0432\u0438\u0436\u043a\u0430 (SM, copy, encoder).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "DCGM_FI_DEV_GPU_UTIL{namespace=~\"$namespace\", namespace!=\"\"}", + "legendFormat": "{{namespace}}/{{pod}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + } + } + ], + "title": "NVML GPU Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "\u0420\u0435\u0430\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u0433\u0440\u0443\u0437\u043a\u0430 \u0442\u0435\u043d\u0437\u043e\u0440\u043d\u044b\u0445 \u044f\u0434\u0435\u0440 (HMMA). \u0414\u043b\u044f AI/LLM \u0438\u043d\u0444\u0435\u0440\u0435\u043d\u0441\u0430 \u0434\u043e\u043b\u0436\u043d\u0430 \u0431\u044b\u0442\u044c \u226520%, \u0438\u043d\u0430\u0447\u0435 workload \u043d\u0435 \u043e\u043f\u0442\u0438\u043c\u0438\u0437\u0438\u0440\u043e\u0432\u0430\u043d.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace=~\"$namespace\", namespace!=\"\"} * 100", + "legendFormat": "{{namespace}}/{{pod}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + } + } + ], + "title": "Tensor Pipe Active", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 20, + "panels": [], + "title": "Per-pod \u0440\u0435\u0439\u0442\u0438\u043d\u0433", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + }, + "description": "\u041a\u0442\u043e \u0436\u043c\u0451\u0442 \u0442\u0435\u043d\u0437\u043e\u0440\u043d\u044b\u0435 \u044f\u0434\u0440\u0430, \u0430 \u043a\u0442\u043e \u043d\u0435\u0442. \u0421\u043e\u0440\u0442\u0438\u0440\u043e\u0432\u043a\u0430 \u043e\u0442 \u043c\u0430\u043a\u0441\u0438\u043c\u0443\u043c\u0430.", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Saturation" + }, + "properties": [ + { + "id": "unit", + "value": "percent" + }, + { + "id": "min", + "value": 0 + }, + { + "id": "max", + "value": 100 + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 10 + }, + { + "color": "yellow", + "value": 30 + }, + { + "color": "green", + "value": 60 + } + ] + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 21, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Saturation" + } + ] + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "topk(20, pod:tensor_saturation:avg5m)", + "format": "table", + "instant": true, + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + } + } + ], + "title": "Tensor Saturation per pod (5m avg)", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Hostname": true, + "Time": true, + "UUID": true, + "__name__": true, + "cluster": true, + "container": true, + "endpoint": true, + "gpu": true, + "instance": true, + "job": true, + "modelName": true, + "prometheus": true, + "service": true, + "tenant": true, + "tier": true, + "uid": true, + "unit": true + }, + "indexByName": { + "Value": 2, + "namespace": 0, + "pod": 1 + }, + "renameByName": { + "Value": "Saturation", + "namespace": "Namespace", + "pod": "Pod" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "\u041d\u0430\u0441\u043a\u043e\u043b\u044c\u043a\u043e \u044d\u0444\u0444\u0435\u043a\u0442\u0438\u0432\u043d\u043e \u043a\u0430\u0436\u0434\u044b\u0439 \u043f\u043e\u0434 \u0442\u0440\u0430\u0442\u0438\u0442 \u0432\u0430\u0442\u0442\u044b. \u041d\u0438\u0437\u043a\u043e\u0435 \u0437\u043d\u0430\u0447\u0435\u043d\u0438\u0435 = \u043f\u043b\u043e\u0445\u0430\u044f \u043e\u043f\u0442\u0438\u043c\u0438\u0437\u0430\u0446\u0438\u044f.", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Util/Watt" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "decimals", + "value": 3 + }, + { + "id": "min", + "value": 0 + }, + { + "id": "max", + "value": 1.5 + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "green", + "value": 1 + } + ] + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 22, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Util/Watt" + } + ] + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "topk(20, pod:util_per_watt:avg5m)", + "format": "table", + "instant": true, + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + } + } + ], + "title": "Utilization / Watt per pod (5m avg)", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Hostname": true, + "Time": true, + "UUID": true, + "__name__": true, + "cluster": true, + "container": true, + "endpoint": true, + "gpu": true, + "instance": true, + "job": true, + "modelName": true, + "prometheus": true, + "service": true, + "tenant": true, + "tier": true, + "uid": true, + "unit": true + }, + "indexByName": { + "Value": 2, + "namespace": 0, + "pod": 1 + }, + "renameByName": { + "Value": "Util/Watt", + "namespace": "Namespace", + "pod": "Pod" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 30, + "panels": [], + "title": "Throttling", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + }, + "description": "\u0414\u043e\u043b\u044f \u0432\u0440\u0435\u043c\u0435\u043d\u0438, \u043a\u043e\u0433\u0434\u0430 GPU \u0434\u0440\u043e\u0441\u0441\u0435\u043b\u0438\u0440\u043e\u0432\u0430\u043d\u0430 \u043f\u043e \u043f\u0438\u0442\u0430\u043d\u0438\u044e. 1.0 = \u043f\u043e\u0441\u0442\u043e\u044f\u043d\u043d\u043e.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.05 + }, + { + "color": "red", + "value": 0.2 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 31, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "gpu:power_throttle_fraction:rate5m", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + } + } + ], + "title": "Power throttle fraction per GPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "\u0414\u043e\u043b\u044f \u0432\u0440\u0435\u043c\u0435\u043d\u0438, \u043a\u043e\u0433\u0434\u0430 GPU \u0434\u0440\u043e\u0441\u0441\u0435\u043b\u0438\u0440\u043e\u0432\u0430\u043d\u0430 \u043f\u043e \u0442\u0435\u043c\u043f\u0435\u0440\u0430\u0442\u0443\u0440\u0435.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.05 + }, + { + "color": "red", + "value": 0.2 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 32, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "gpu:thermal_throttle_fraction:rate5m", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + } + } + ], + "title": "Thermal throttle fraction per GPU", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 40, + "tags": [ + "gpu", + "efficiency", + "finops" + ], + "templating": { + "list": [ + { + "current": {}, + "includeAll": false, + "label": "Datasource", + "name": "ds_prometheus", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "vm-.*", + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + }, + "definition": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)", + "includeAll": true, + "label": "Namespace", + "multi": true, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "GPU Efficiency Score", + "uid": "gpu-efficiency", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/dashboards/gpu/gpu-performance.json b/dashboards/gpu/gpu-performance.json index 0b458d4e..906d6a11 100644 --- a/dashboards/gpu/gpu-performance.json +++ b/dashboards/gpu/gpu-performance.json @@ -1,120 +1,100 @@ { + "__inputs": [ + { + "name": "DS_VM-SHORTTERM", + "label": "vm-shortterm", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.4.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], "annotations": { - "list": [] + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": null, "links": [], - "liveNow": false, - "schemaVersion": 39, - "tags": [ - "gpu", - "dcgm" - ], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "default", - "value": "default" - }, - "hide": 0, - "includeAll": false, - "label": "Prometheus", - "multi": false, - "name": "ds_prometheus", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "definition": "label_values(DCGM_FI_DEV_GPU_UTIL, Hostname)", - "hide": 0, - "includeAll": true, - "label": "Host", - "multi": true, - "name": "Hostname", - "options": [], - "query": { - "query": "label_values(DCGM_FI_DEV_GPU_UTIL, Hostname)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "definition": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)", - "hide": 0, - "includeAll": true, - "label": "Namespace", - "multi": true, - "name": "namespace", - "options": [], - "query": { - "query": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "GPU Performance", - "uid": "gpu-performance", - "weekStart": "", "panels": [ { - "type": "row", - "title": "Overview", + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, - "collapsed": false, "id": 1, - "panels": [] + "panels": [], + "title": "Overview", + "type": "row" }, { - "type": "stat", - "title": "Total GPUs", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, "gridPos": { "h": 4, "w": 6, @@ -122,36 +102,12 @@ "y": 1 }, "id": 2, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "targets": [ - { - "expr": "cluster:gpu_count:total", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue" - } - ] - } - }, - "overrides": [] - }, "options": { "colorMode": "value", "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -159,35 +115,35 @@ "fields": "", "values": false }, - "textMode": "value" - } - }, - { - "type": "stat", - "title": "Allocated", - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 1 - }, - "id": 3, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" + "showPercentChange": false, + "textMode": "value", + "wideLayout": true }, + "pluginVersion": "11.4.0", "targets": [ { - "expr": "cluster:gpu_count:allocated", - "refId": "A" + "expr": "cluster:gpu_count:total", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + } } ], + "title": "Total GPUs", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + }, "fieldConfig": { "defaults": { - "unit": "short", "color": { "mode": "thresholds" }, + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ @@ -200,13 +156,24 @@ "value": 1 } ] - } + }, + "unit": "short" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 3, "options": { "colorMode": "value", "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -214,37 +181,37 @@ "fields": "", "values": false }, - "textMode": "value" - } + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "cluster:gpu_count:allocated", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + } + } + ], + "title": "Allocated", + "type": "stat" }, { - "type": "stat", - "title": "Average utilization", - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 1 - }, - "id": 4, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, - "targets": [ - { - "expr": "cluster:gpu_util:avg", - "refId": "A" - } - ], "fieldConfig": { "defaults": { - "unit": "percent", - "min": 0, - "max": 100, "color": { "mode": "thresholds" }, + "mappings": [], + "max": 100, + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -261,13 +228,24 @@ "value": 80 } ] - } + }, + "unit": "percent" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 4, "options": { "colorMode": "value", "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -275,12 +253,49 @@ "fields": "", "values": false }, - "textMode": "value" - } + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "cluster:gpu_util:avg", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + } + } + ], + "title": "Average utilization", + "type": "stat" }, { - "type": "stat", - "title": "Power draw", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "watt" + }, + "overrides": [] + }, "gridPos": { "h": 4, "w": 6, @@ -288,37 +303,12 @@ "y": 1 }, "id": 5, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "targets": [ - { - "expr": "cluster:gpu_power_watts:sum", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "unit": "watt", - "decimals": 0, - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, "options": { "colorMode": "value", "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -326,25 +316,100 @@ "fields": "", "values": false }, - "textMode": "value" - } + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "cluster:gpu_power_watts:sum", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + } + } + ], + "title": "Power draw", + "type": "stat" }, { - "type": "row", - "title": "Utilization", + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, - "collapsed": false, "id": 10, - "panels": [] + "panels": [], + "title": "Utilization", + "type": "row" }, { - "type": "timeseries", - "title": "GPU Utilization (NVML)", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, "gridPos": { "h": 8, "w": 12, @@ -352,49 +417,99 @@ "y": 6 }, "id": 11, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, + "pluginVersion": "11.4.0", "targets": [ { "expr": "DCGM_FI_DEV_GPU_UTIL{Hostname=~\"$Hostname\", namespace=~\"$namespace\"}", "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", - "refId": "A" + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + } } ], + "title": "GPU Utilization (NVML)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + }, "fieldConfig": { "defaults": { - "unit": "percent", - "min": 0, - "max": 100, + "color": { + "mode": "palette-classic" + }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", - "lineInterpolation": "linear", "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", - "spanNulls": false - } + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" }, "overrides": [] }, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi" - } - } - }, - { - "type": "timeseries", - "title": "Tensor Pipe Active (realistic load for LLM/AI)", "gridPos": { "h": 8, "w": 12, @@ -402,49 +517,99 @@ "y": 6 }, "id": 12, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, + "pluginVersion": "11.4.0", "targets": [ { "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 100", "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", - "refId": "A" + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + } } ], + "title": "Tensor Pipe Active (realistic load for LLM/AI)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, "fieldConfig": { "defaults": { - "unit": "percent", - "min": 0, - "max": 100, + "color": { + "mode": "palette-classic" + }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", - "lineInterpolation": "linear", "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", - "spanNulls": false - } + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" }, "overrides": [] }, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi" - } - } - }, - { - "type": "timeseries", - "title": "Graphics Engine Active", "gridPos": { "h": 8, "w": 12, @@ -452,49 +617,99 @@ "y": 14 }, "id": 13, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, + "pluginVersion": "11.4.0", "targets": [ { "expr": "DCGM_FI_PROF_GR_ENGINE_ACTIVE{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 100", "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", - "refId": "A" + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + } } ], + "title": "Graphics Engine Active", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + }, "fieldConfig": { "defaults": { - "unit": "percent", - "min": 0, - "max": 100, + "color": { + "mode": "palette-classic" + }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", - "lineInterpolation": "linear", "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", - "spanNulls": false - } + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" }, "overrides": [] }, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi" - } - } - }, - { - "type": "timeseries", - "title": "Memory Copy Utilization", "gridPos": { "h": 8, "w": 12, @@ -502,62 +717,110 @@ "y": 14 }, "id": 14, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, + "pluginVersion": "11.4.0", "targets": [ { "expr": "DCGM_FI_DEV_MEM_COPY_UTIL{Hostname=~\"$Hostname\", namespace=~\"$namespace\"}", "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", - "refId": "A" + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + } } ], - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi" - } - } + "title": "Memory Copy Utilization", + "type": "timeseries" }, { - "type": "row", - "title": "Memory", + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 }, - "collapsed": false, "id": 20, - "panels": [] + "panels": [], + "title": "Memory", + "type": "row" }, { - "type": "timeseries", - "title": "VRAM Used", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, "gridPos": { "h": 8, "w": 12, @@ -565,49 +828,97 @@ "y": 23 }, "id": 21, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, + "pluginVersion": "11.4.0", "targets": [ { "expr": "DCGM_FI_DEV_FB_USED{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 1048576", "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", - "refId": "A" + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + } } ], + "title": "VRAM Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + }, "fieldConfig": { "defaults": { - "unit": "bytes", + "color": { + "mode": "palette-classic" + }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, "lineInterpolation": "linear", - "fillOpacity": 25, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", + "spanNulls": false, "stacking": { + "group": "A", "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } - } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" }, "overrides": [] }, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi" - } - } - }, - { - "type": "timeseries", - "title": "VRAM Free", "gridPos": { "h": 8, "w": 12, @@ -615,59 +926,110 @@ "y": 23 }, "id": 22, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" + "options": { + "legend": { + "calcs": [ + "mean", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, + "pluginVersion": "11.4.0", "targets": [ { "expr": "DCGM_FI_DEV_FB_FREE{Hostname=~\"$Hostname\"} * 1048576", "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + } } ], - "fieldConfig": { - "defaults": { - "unit": "bytes", - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "min" - ] - }, - "tooltip": { - "mode": "multi" - } - } + "title": "VRAM Free", + "type": "timeseries" }, { - "type": "row", - "title": "Power & Temperature", + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 31 }, - "collapsed": false, "id": 30, - "panels": [] + "panels": [], + "title": "Power & Temperature", + "type": "row" }, { - "type": "timeseries", - "title": "Power Usage per GPU", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "watt" + }, + "overrides": [] + }, "gridPos": { "h": 8, "w": 12, @@ -675,80 +1037,88 @@ "y": 32 }, "id": 31, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, + "pluginVersion": "11.4.0", "targets": [ { "expr": "DCGM_FI_DEV_POWER_USAGE{Hostname=~\"$Hostname\"}", "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + } } ], - "fieldConfig": { - "defaults": { - "unit": "watt", - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi" - } - } + "title": "Power Usage per GPU", + "type": "timeseries" }, { - "type": "timeseries", - "title": "GPU Temperature", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 32, "datasource": { "type": "prometheus", - "uid": "$ds_prometheus" + "uid": "${DS_VM-SHORTTERM}" }, - "targets": [ - { - "expr": "DCGM_FI_DEV_GPU_TEMP{Hostname=~\"$Hostname\"}", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" - } - ], "fieldConfig": { "defaults": { - "unit": "celsius", - "min": 20, - "max": 100, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never" + "color": { + "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 20, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "yellow", @@ -759,61 +1129,68 @@ "value": 85 } ] - } + }, + "unit": "celsius" }, "overrides": [] }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 32, "options": { "legend": { - "displayMode": "table", - "placement": "bottom", "calcs": [ "mean", "max" - ] + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true }, "tooltip": { - "mode": "multi" + "mode": "multi", + "sort": "none" } - } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "DCGM_FI_DEV_GPU_TEMP{Hostname=~\"$Hostname\"}", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + } + } + ], + "title": "GPU Temperature", + "type": "timeseries" }, { - "type": "row", - "title": "Health", + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 40 }, - "collapsed": false, "id": 40, - "panels": [] + "panels": [], + "title": "Health", + "type": "row" }, { - "type": "stat", - "title": "XID errors (latest)", - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 41 - }, - "id": 41, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, - "targets": [ - { - "expr": "max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS{Hostname=~\"$Hostname\"})", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" - } - ], "fieldConfig": { "defaults": { - "unit": "short", "color": { "mode": "thresholds" }, @@ -828,10 +1205,18 @@ "value": 1 } ] - } + }, + "unit": "short" }, "overrides": [] }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 41 + }, + "id": 41, "options": { "colorMode": "background", "graphMode": "none", @@ -843,11 +1228,38 @@ "values": false }, "textMode": "value_and_name" - } + }, + "targets": [ + { + "expr": "max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS{Hostname=~\"$Hostname\"})", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + } + } + ], + "title": "XID errors (latest)", + "type": "stat" }, { - "type": "timeseries", - "title": "Power Violation (\u00b5s/s throttled due to power)", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "linear", + "showPoints": "never" + }, + "unit": "µs" + }, + "overrides": [] + }, "gridPos": { "h": 5, "w": 8, @@ -855,29 +1267,6 @@ "y": 41 }, "id": 42, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "targets": [ - { - "expr": "rate(DCGM_FI_DEV_POWER_VIOLATION{Hostname=~\"$Hostname\"}[5m])", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "unit": "\u00b5s", - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [] - }, "options": { "legend": { "displayMode": "list", @@ -886,11 +1275,38 @@ "tooltip": { "mode": "multi" } - } + }, + "targets": [ + { + "expr": "rate(DCGM_FI_DEV_POWER_VIOLATION{Hostname=~\"$Hostname\"}[5m])", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + } + } + ], + "title": "Power Violation (µs/s throttled due to power)", + "type": "timeseries" }, { - "type": "timeseries", - "title": "Thermal Violation (\u00b5s/s throttled due to thermals)", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "linear", + "showPoints": "never" + }, + "unit": "µs" + }, + "overrides": [] + }, "gridPos": { "h": 5, "w": 8, @@ -898,29 +1314,6 @@ "y": 41 }, "id": 43, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "targets": [ - { - "expr": "rate(DCGM_FI_DEV_THERMAL_VIOLATION{Hostname=~\"$Hostname\"}[5m])", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "unit": "\u00b5s", - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [] - }, "options": { "legend": { "displayMode": "list", @@ -929,7 +1322,93 @@ "tooltip": { "mode": "multi" } - } + }, + "targets": [ + { + "expr": "rate(DCGM_FI_DEV_THERMAL_VIOLATION{Hostname=~\"$Hostname\"}[5m])", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + } + } + ], + "title": "Thermal Violation (µs/s throttled due to thermals)", + "type": "timeseries" } - ] -} \ No newline at end of file + ], + "refresh": "", + "schemaVersion": 40, + "tags": [ + "gpu", + "dcgm" + ], + "templating": { + "list": [ + { + "current": {}, + "includeAll": false, + "label": "Prometheus", + "name": "ds_prometheus", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + }, + "definition": "label_values(DCGM_FI_DEV_GPU_UTIL, Hostname)", + "includeAll": true, + "label": "Host", + "multi": true, + "name": "Hostname", + "options": [], + "query": { + "query": "label_values(DCGM_FI_DEV_GPU_UTIL, Hostname)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "definition": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)", + "includeAll": true, + "label": "Namespace", + "multi": true, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "GPU Performance", + "uid": "gpu-performance", + "version": 2, + "weekStart": "" +} diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json new file mode 100644 index 00000000..28f2c892 --- /dev/null +++ b/dashboards/gpu/gpu-quotas.json @@ -0,0 +1,822 @@ +{ + "__inputs": [ + { + "name": "DS_VM-SHORTTERM", + "label": "vm-shortterm", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "bargauge", + "name": "Bar gauge", + "version": "" + }, + { + "type": "panel", + "id": "gauge", + "name": "Gauge", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.4.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Allocation overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"})", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + } + } + ], + "title": "GPU allocatable", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"})", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + } + } + ], + "title": "GPU requested", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "green", + "value": 40 + }, + { + "color": "yellow", + "value": 80 + }, + { + "color": "red", + "value": 95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 4, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}) / sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"}) * 100", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + } + } + ], + "title": "Allocation ratio", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "count((kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"} > 0) * on(namespace, pod) group_left() (kube_pod_status_phase{phase=\"Pending\"} == 1)) or vector(0)", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + } + } + ], + "title": "Pending pods (GPU)", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 10, + "panels": [], + "title": "Per namespace", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 8, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 11, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "sum by (namespace) (kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"})", + "legendFormat": "{{namespace}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + } + } + ], + "title": "GPU requested per namespace", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Allocatable (total)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"})", + "legendFormat": "Allocatable (total)", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + } + }, + { + "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"})", + "legendFormat": "Requested", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + } + } + ], + "title": "GPU allocated over time", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 20, + "panels": [], + "title": "Pods with GPU", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "color-background" + } + }, + { + "id": "mappings", + "value": [ + { + "options": { + "Failed": { + "color": "red", + "index": 2 + }, + "Pending": { + "color": "orange", + "index": 1 + }, + "Running": { + "color": "green", + "index": 0 + } + }, + "type": "value" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 21, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left(phase) (kube_pod_status_phase{phase=~\"Running|Pending|Failed|Succeeded\"} == 1)", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "requested", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + } + }, + { + "expr": "kube_pod_container_resource_limits{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"}", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "limits", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + } + } + ], + "title": "Pods requesting GPU", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "pod", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "__name__": true, + "__name__ 1": true, + "__name__ 2": true, + "cluster": true, + "cluster 1": true, + "cluster 2": true, + "container 2": true, + "endpoint": true, + "endpoint 1": true, + "endpoint 2": true, + "instance": true, + "instance 1": true, + "instance 2": true, + "job": true, + "job 1": true, + "job 2": true, + "namespace 2": true, + "node 2": true, + "prometheus": true, + "prometheus 1": true, + "prometheus 2": true, + "resource": true, + "resource 1": true, + "resource 2": true, + "service": true, + "service 1": true, + "service 2": true, + "tenant": true, + "tenant 1": true, + "tenant 2": true, + "tier": true, + "tier 1": true, + "tier 2": true, + "uid": true, + "uid 1": true, + "uid 2": true, + "unit": true, + "unit 1": true, + "unit 2": true + }, + "indexByName": { + "Value #limits": 5, + "Value #requested": 4, + "container 1": 2, + "namespace 1": 0, + "node 1": 3, + "phase": 6, + "pod": 1 + }, + "renameByName": { + "Value #limits": "Limit", + "Value #requested": "Req", + "container 1": "Container", + "namespace 1": "Namespace", + "node 1": "Node", + "phase": "Status" + } + } + } + ], + "type": "table" + } + ], + "refresh": "", + "schemaVersion": 40, + "tags": [ + "gpu", + "quotas" + ], + "templating": { + "list": [ + { + "current": {}, + "includeAll": false, + "label": "Prometheus", + "name": "ds_prometheus", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + }, + "definition": "label_values(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}, namespace)", + "includeAll": true, + "label": "Namespace", + "multi": true, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}, namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "GPU Quotas & Allocation", + "uid": "gpu-quotas", + "version": 4, + "weekStart": "" +} diff --git a/packages/system/monitoring/dashboards-infra.list b/packages/system/monitoring/dashboards-infra.list index 581f9a05..b402fa68 100644 --- a/packages/system/monitoring/dashboards-infra.list +++ b/packages/system/monitoring/dashboards-infra.list @@ -33,3 +33,5 @@ hubble/dns-namespace hubble/l7-http-metrics hubble/network-overview gpu/gpu-performance +gpu/gpu-efficiency +gpu/gpu-quotas From 7e5f3a7f12ae6ff744032c30eb0e7b05071986a2 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Fri, 17 Apr 2026 17:45:06 +0300 Subject: [PATCH 005/130] refactor(monitoring): clean up GPU dashboards Strip Grafana export boilerplate (__inputs, __elements, __requires, default annotations, embedded datasource inputs) and tighten panel layouts across the three GPU dashboards. All three continue to use the $ds_prometheus template variable. Assisted-By: Claude Signed-off-by: Arsolitt --- dashboards/gpu/gpu-efficiency.json | 1186 ++++++++------------ dashboards/gpu/gpu-performance.json | 1574 ++++++++++----------------- dashboards/gpu/gpu-quotas.json | 876 ++++++--------- 3 files changed, 1358 insertions(+), 2278 deletions(-) diff --git a/dashboards/gpu/gpu-efficiency.json b/dashboards/gpu/gpu-efficiency.json index cc2bca6c..1a305b4b 100644 --- a/dashboards/gpu/gpu-efficiency.json +++ b/dashboards/gpu/gpu-efficiency.json @@ -1,72 +1,26 @@ { - "__inputs": [ - { - "name": "DS_VM-SHORTTERM", - "label": "vm-shortterm", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } + "uid": "gpu-efficiency", + "title": "GPU Efficiency Score", + "description": "Tensor saturation, util/watt and throttling — reveals inefficient GPU workloads", + "tags": [ + "gpu", + "efficiency", + "finops" ], - "__elements": {}, - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "11.4.0" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "stat", - "name": "Stat", - "version": "" - }, - { - "type": "panel", - "id": "table", - "name": "Table", - "version": "" - }, - { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "Tensor saturation, util/watt and throttling \u2014 reveals inefficient GPU workloads", + "timezone": "browser", "editable": true, - "fiscalYearStartMonth": 0, "graphTooltip": 1, - "id": null, - "links": [], + "time": { + "from": "now-1h", + "to": "now" + }, + "fiscalYearStartMonth": 0, + "schemaVersion": 42, "panels": [ { + "type": "row", "collapsed": false, + "title": "Overall efficiency metrics", "gridPos": { "h": 1, "w": 24, @@ -74,48 +28,23 @@ "y": 0 }, "id": 1, - "panels": [], - "title": "\u041e\u0431\u0449\u0438\u0435 \u043c\u0435\u0442\u0440\u0438\u043a\u0438 \u044d\u0444\u0444\u0435\u043a\u0442\u0438\u0432\u043d\u043e\u0441\u0442\u0438", - "type": "row" + "panels": [] }, { + "type": "stat", + "id": 2, + "targets": [ + { + "expr": "avg(pod:tensor_saturation:avg5m)", + "refId": "A" + } + ], + "title": "Cluster Tensor Saturation", + "description": "Mean tensor core saturation. \u003c10% means GPUs are used inefficiently (tenants could move to CPU or optimize their code).", + "transparent": false, "datasource": { "type": "prometheus", - "uid": "${ds_prometheus}" - }, - "description": "\u0421\u0440\u0435\u0434\u043d\u044f\u044f \u0437\u0430\u0433\u0440\u0443\u0437\u043a\u0430 \u0442\u0435\u043d\u0437\u043e\u0440\u043d\u044b\u0445 \u044f\u0434\u0435\u0440. <10% \u2014 GPU \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u044e\u0442\u0441\u044f \u043d\u0435\u044d\u0444\u0444\u0435\u043a\u0442\u0438\u0432\u043d\u043e (\u043a\u043b\u0438\u0435\u043d\u0442\u044b \u043c\u043e\u0433\u043b\u0438 \u0431\u044b \u043f\u0435\u0440\u0435\u0435\u0445\u0430\u0442\u044c \u043d\u0430 CPU \u0438\u043b\u0438 \u043e\u043f\u0442\u0438\u043c\u0438\u0437\u0438\u0440\u043e\u0432\u0430\u0442\u044c \u043a\u043e\u0434).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "orange", - "value": 10 - }, - { - "color": "yellow", - "value": 30 - }, - { - "color": "green", - "value": 60 - } - ] - }, - "unit": "percent" - }, - "overrides": [] + "uid": "$ds_prometheus" }, "gridPos": { "h": 5, @@ -123,71 +52,67 @@ "x": 0, "y": 1 }, - "id": 2, + "repeatDirection": "h", "options": { - "colorMode": "background", "graphMode": "area", + "colorMode": "background", "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" - ], - "fields": "", - "values": false + ] }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": true + "percentChangeColorMode": "standard", + "orientation": "" }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "avg(pod:tensor_saturation:avg5m)", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${ds_prometheus}" - } - } - ], - "title": "Cluster Tensor Saturation", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - }, - "description": "NVML utilization % \u043d\u0430 \u043a\u0430\u0436\u0434\u044b\u0439 \u0432\u0430\u0442\u0442. \u0412\u044b\u0441\u043e\u043a\u043e\u0435 \u0437\u043d\u0430\u0447\u0435\u043d\u0438\u0435 = \u044d\u0444\u0444\u0435\u043a\u0442\u0438\u0432\u043d\u0430\u044f \u043d\u0430\u0433\u0440\u0443\u0437\u043a\u0430.", "fieldConfig": { "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 3, - "mappings": [], + "unit": "percent", + "min": 0, + "max": 100, "thresholds": { "mode": "absolute", "steps": [ { - "color": "red", - "value": null + "value": null, + "color": "red" }, { - "color": "yellow", - "value": 0.5 + "value": 10, + "color": "orange" }, { - "color": "green", - "value": 1 + "value": 30, + "color": "yellow" + }, + { + "value": 60, + "color": "green" } ] - }, - "unit": "none" + } }, "overrides": [] + } + }, + { + "type": "stat", + "id": 3, + "targets": [ + { + "expr": "avg(pod:util_per_watt:avg5m)", + "refId": "A" + } + ], + "title": "Avg Utilization per Watt", + "description": "NVML utilization % per watt. Higher value = more efficient workload.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 5, @@ -195,73 +120,62 @@ "x": 8, "y": 1 }, - "id": 3, + "repeatDirection": "h", "options": { - "colorMode": "background", "graphMode": "area", + "colorMode": "background", "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" - ], - "fields": "", - "values": false + ] }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": true + "percentChangeColorMode": "standard", + "orientation": "" }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "avg(pod:util_per_watt:avg5m)", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - } - } - ], - "title": "Avg Utilization per Watt", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${ds_prometheus}" - }, - "description": "\u0414\u043e\u043b\u044f \u0432\u0440\u0435\u043c\u0435\u043d\u0438, \u043a\u043e\u0433\u0434\u0430 GPU \u0443\u043f\u0438\u0440\u0430\u044e\u0442\u0441\u044f \u0432 TDP \u0438 \u0442\u0435\u0440\u044f\u044e\u0442 \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c. >5% \u2014 \u043a\u043b\u0438\u0435\u043d\u0442\u044b \u043d\u0435\u0434\u043e\u043f\u043e\u043b\u0443\u0447\u0430\u044e\u0442 FLOPS.", "fieldConfig": { "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 2, - "mappings": [], - "max": 100, - "min": 0, + "unit": "none", + "decimals": 3, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "value": null, + "color": "red" }, { - "color": "yellow", - "value": 5 + "value": 0.5, + "color": "yellow" }, { - "color": "red", - "value": 20 + "value": 1, + "color": "green" } ] - }, - "unit": "percent" + } }, "overrides": [] + } + }, + { + "type": "stat", + "id": 4, + "targets": [ + { + "expr": "avg(gpu:power_throttle_fraction:rate5m)", + "refId": "A" + } + ], + "title": "Avg Power Throttling", + "description": "Fraction of time GPUs hit the TDP cap and lose performance. \u003e5% means tenants underutilize billed FLOPS.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 5, @@ -269,40 +183,51 @@ "x": 16, "y": 1 }, - "id": 4, + "repeatDirection": "h", "options": { - "colorMode": "background", "graphMode": "area", + "colorMode": "background", "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" - ], - "fields": "", - "values": false + ] }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": true + "percentChangeColorMode": "standard", + "orientation": "" }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "avg(gpu:power_throttle_fraction:rate5m) * 100", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${ds_prometheus}" + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 0.05, + "color": "yellow" + }, + { + "value": 0.2, + "color": "red" + } + ] } - } - ], - "title": "Avg Power Throttling", - "type": "stat" + }, + "overrides": [] + } }, { + "type": "row", "collapsed": false, + "title": "NVML vs Tensor (mismatch detector)", "gridPos": { "h": 1, "w": 24, @@ -310,73 +235,24 @@ "y": 6 }, "id": 10, - "panels": [], - "title": "NVML vs Tensor (\u043a\u0442\u043e \u043e\u0431\u043c\u0430\u043d\u044b\u0432\u0430\u0435\u0442\u0441\u044f)", - "type": "row" + "panels": [] }, { + "type": "timeseries", + "id": 11, + "targets": [ + { + "expr": "DCGM_FI_DEV_GPU_UTIL{namespace=~\"$namespace\", namespace!=\"\"}", + "legendFormat": "{{namespace}}/{{pod}}", + "refId": "A" + } + ], + "title": "NVML GPU Utilization", + "description": "Classic utilization metric. Shows activity of any engine (SM, copy, encoder).", + "transparent": false, "datasource": { "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - }, - "description": "\u041a\u043b\u0430\u0441\u0441\u0438\u0447\u0435\u0441\u043a\u0430\u044f \u043c\u0435\u0442\u0440\u0438\u043a\u0430 \u0443\u0442\u0438\u043b\u0438\u0437\u0430\u0446\u0438\u0438. \u041f\u043e\u043a\u0430\u0437\u044b\u0432\u0430\u0435\u0442 \u0437\u0430\u043d\u044f\u0442\u043e\u0441\u0442\u044c \u043b\u044e\u0431\u043e\u0433\u043e \u0434\u0432\u0438\u0436\u043a\u0430 (SM, copy, encoder).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] + "uid": "$ds_prometheus" }, "gridPos": { "h": 8, @@ -384,100 +260,53 @@ "x": 0, "y": 7 }, - "id": 11, + "repeatDirection": "h", "options": { "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, "calcs": [ "mean", "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + ] }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "" } }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "DCGM_FI_DEV_GPU_UTIL{namespace=~\"$namespace\", namespace!=\"\"}", - "legendFormat": "{{namespace}}/{{pod}}", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - } - } - ], - "title": "NVML GPU Utilization", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${ds_prometheus}" - }, - "description": "\u0420\u0435\u0430\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u0433\u0440\u0443\u0437\u043a\u0430 \u0442\u0435\u043d\u0437\u043e\u0440\u043d\u044b\u0445 \u044f\u0434\u0435\u0440 (HMMA). \u0414\u043b\u044f AI/LLM \u0438\u043d\u0444\u0435\u0440\u0435\u043d\u0441\u0430 \u0434\u043e\u043b\u0436\u043d\u0430 \u0431\u044b\u0442\u044c \u226520%, \u0438\u043d\u0430\u0447\u0435 workload \u043d\u0435 \u043e\u043f\u0442\u0438\u043c\u0438\u0437\u0438\u0440\u043e\u0432\u0430\u043d.", "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 100, + "unit": "percent", "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" + "max": 100, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never" + } }, "overrides": [] + } + }, + { + "type": "timeseries", + "id": 12, + "targets": [ + { + "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace=~\"$namespace\", namespace!=\"\"} * 100", + "legendFormat": "{{namespace}}/{{pod}}", + "refId": "A" + } + ], + "title": "Tensor Pipe Active", + "description": "Real tensor core load (HMMA). For AI/LLM inference it should be ≥20%, otherwise the workload is unoptimized.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 8, @@ -485,39 +314,41 @@ "x": 12, "y": 7 }, - "id": 12, + "repeatDirection": "h", "options": { "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, "calcs": [ "mean", "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + ] }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "" } }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace=~\"$namespace\", namespace!=\"\"} * 100", - "legendFormat": "{{namespace}}/{{pod}}", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${ds_prometheus}" + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never" } - } - ], - "title": "Tensor Pipe Active", - "type": "timeseries" + }, + "overrides": [] + } }, { + "type": "row", "collapsed": false, + "title": "Per-pod ranking", "gridPos": { "h": 1, "w": 24, @@ -525,40 +356,92 @@ "y": 15 }, "id": 20, - "panels": [], - "title": "Per-pod \u0440\u0435\u0439\u0442\u0438\u043d\u0433", - "type": "row" + "panels": [] }, { + "type": "table", + "id": 21, + "targets": [ + { + "expr": "topk(20, pod:tensor_saturation:avg5m)", + "instant": true, + "range": false, + "format": "table", + "refId": "A" + } + ], + "title": "Tensor Saturation per pod (5m avg)", + "description": "Who is exercising tensor cores and who is not. Sorted descending.", + "transparent": false, "datasource": { "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" + "uid": "$ds_prometheus" }, - "description": "\u041a\u0442\u043e \u0436\u043c\u0451\u0442 \u0442\u0435\u043d\u0437\u043e\u0440\u043d\u044b\u0435 \u044f\u0434\u0440\u0430, \u0430 \u043a\u0442\u043e \u043d\u0435\u0442. \u0421\u043e\u0440\u0442\u0438\u0440\u043e\u0432\u043a\u0430 \u043e\u0442 \u043c\u0430\u043a\u0441\u0438\u043c\u0443\u043c\u0430.", - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "repeatDirection": "h", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "DCGM_FI_DRIVER_VERSION": true, + "Hostname": true, + "Time": true, + "UUID": true, + "__name__": true, + "cluster": true, + "container": true, + "device": true, + "endpoint": true, + "gpu": true, + "gpu_driver_version": true, + "instance": true, + "job": true, + "modelName": true, + "pci_bus_id": true, + "prometheus": true, + "service": true, + "tenant": true, + "tier": true, + "uid": true, + "unit": true }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "indexByName": { + "Value": 2, + "namespace": 0, + "pod": 1 + }, + "renameByName": { + "Value": "Saturation", + "namespace": "Namespace", + "pod": "Pod" + } } + } + ], + "options": { + "frameIndex": 0, + "showHeader": true, + "showTypeIcons": false, + "sortBy": [ + { + "displayName": "Saturation", + "desc": true + } + ], + "footer": { + "show": false, + "reducer": [] }, + "cellHeight": "sm" + }, + "fieldConfig": { + "defaults": {}, "overrides": [ { "matcher": { @@ -591,8 +474,7 @@ "mode": "absolute", "steps": [ { - "color": "red", - "value": null + "color": "red" }, { "color": "orange", @@ -612,62 +494,54 @@ ] } ] + } + }, + { + "type": "table", + "id": 22, + "targets": [ + { + "expr": "topk(20, pod:util_per_watt:avg5m)", + "instant": true, + "range": false, + "format": "table", + "refId": "A" + } + ], + "title": "Utilization / Watt per pod (5m avg)", + "description": "How efficiently each pod spends watts. Low value = poor optimization.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 8, "w": 12, - "x": 0, + "x": 12, "y": 16 }, - "id": 21, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Saturation" - } - ] - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "topk(20, pod:tensor_saturation:avg5m)", - "format": "table", - "instant": true, - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - } - } - ], - "title": "Tensor Saturation per pod (5m avg)", + "repeatDirection": "h", "transformations": [ { "id": "organize", "options": { "excludeByName": { + "DCGM_FI_DRIVER_VERSION": true, "Hostname": true, "Time": true, "UUID": true, "__name__": true, "cluster": true, "container": true, + "device": true, "endpoint": true, "gpu": true, + "gpu_driver_version": true, "instance": true, "job": true, "modelName": true, + "pci_bus_id": true, "prometheus": true, "service": true, "tenant": true, @@ -681,45 +555,31 @@ "pod": 1 }, "renameByName": { - "Value": "Saturation", + "Value": "Util/Watt", "namespace": "Namespace", "pod": "Pod" } } } ], - "type": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${ds_prometheus}" - }, - "description": "\u041d\u0430\u0441\u043a\u043e\u043b\u044c\u043a\u043e \u044d\u0444\u0444\u0435\u043a\u0442\u0438\u0432\u043d\u043e \u043a\u0430\u0436\u0434\u044b\u0439 \u043f\u043e\u0434 \u0442\u0440\u0430\u0442\u0438\u0442 \u0432\u0430\u0442\u0442\u044b. \u041d\u0438\u0437\u043a\u043e\u0435 \u0437\u043d\u0430\u0447\u0435\u043d\u0438\u0435 = \u043f\u043b\u043e\u0445\u0430\u044f \u043e\u043f\u0442\u0438\u043c\u0438\u0437\u0430\u0446\u0438\u044f.", - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "options": { + "frameIndex": 0, + "showHeader": true, + "showTypeIcons": false, + "sortBy": [ + { + "displayName": "Util/Watt", + "desc": true } + ], + "footer": { + "show": false, + "reducer": [] }, + "cellHeight": "sm" + }, + "fieldConfig": { + "defaults": {}, "overrides": [ { "matcher": { @@ -756,8 +616,7 @@ "mode": "absolute", "steps": [ { - "color": "red", - "value": null + "color": "red" }, { "color": "yellow", @@ -773,86 +632,12 @@ ] } ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 22, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Util/Watt" - } - ] - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "topk(20, pod:util_per_watt:avg5m)", - "format": "table", - "instant": true, - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${ds_prometheus}" - } - } - ], - "title": "Utilization / Watt per pod (5m avg)", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "Hostname": true, - "Time": true, - "UUID": true, - "__name__": true, - "cluster": true, - "container": true, - "endpoint": true, - "gpu": true, - "instance": true, - "job": true, - "modelName": true, - "prometheus": true, - "service": true, - "tenant": true, - "tier": true, - "uid": true, - "unit": true - }, - "indexByName": { - "Value": 2, - "namespace": 0, - "pod": 1 - }, - "renameByName": { - "Value": "Util/Watt", - "namespace": "Namespace", - "pod": "Pod" - } - } - } - ], - "type": "table" + } }, { + "type": "row", "collapsed": false, + "title": "Throttling", "gridPos": { "h": 1, "w": 24, @@ -860,77 +645,24 @@ "y": 24 }, "id": 30, - "panels": [], - "title": "Throttling", - "type": "row" + "panels": [] }, { + "type": "timeseries", + "id": 31, + "targets": [ + { + "expr": "gpu:power_throttle_fraction:rate5m", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A" + } + ], + "title": "Power throttle fraction per GPU", + "description": "Fraction of time the GPU was power-throttled. 1.0 = always throttled.", + "transparent": false, "datasource": { "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - }, - "description": "\u0414\u043e\u043b\u044f \u0432\u0440\u0435\u043c\u0435\u043d\u0438, \u043a\u043e\u0433\u0434\u0430 GPU \u0434\u0440\u043e\u0441\u0441\u0435\u043b\u0438\u0440\u043e\u0432\u0430\u043d\u0430 \u043f\u043e \u043f\u0438\u0442\u0430\u043d\u0438\u044e. 1.0 = \u043f\u043e\u0441\u0442\u043e\u044f\u043d\u043d\u043e.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 25, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.05 - }, - { - "color": "red", - "value": 0.2 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] + "uid": "$ds_prometheus" }, "gridPos": { "h": 8, @@ -938,104 +670,70 @@ "x": 0, "y": 25 }, - "id": 31, + "repeatDirection": "h", "options": { "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, "calcs": [ "mean", "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + ] }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "" } }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "gpu:power_throttle_fraction:rate5m", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - } - } - ], - "title": "Power throttle fraction per GPU", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${ds_prometheus}" - }, - "description": "\u0414\u043e\u043b\u044f \u0432\u0440\u0435\u043c\u0435\u043d\u0438, \u043a\u043e\u0433\u0434\u0430 GPU \u0434\u0440\u043e\u0441\u0441\u0435\u043b\u0438\u0440\u043e\u0432\u0430\u043d\u0430 \u043f\u043e \u0442\u0435\u043c\u043f\u0435\u0440\u0430\u0442\u0443\u0440\u0435.", "fieldConfig": { "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 25, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 1, + "unit": "percentunit", "min": 0, + "max": 1, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "value": null, + "color": "green" }, { - "color": "yellow", - "value": 0.05 + "value": 0.05, + "color": "yellow" }, { - "color": "red", - "value": 0.2 + "value": 0.2, + "color": "red" } ] }, - "unit": "percentunit" + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 25, + "showPoints": "never" + } }, "overrides": [] + } + }, + { + "type": "timeseries", + "id": 32, + "targets": [ + { + "expr": "gpu:thermal_throttle_fraction:rate5m", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A" + } + ], + "title": "Thermal throttle fraction per GPU", + "description": "Fraction of time the GPU was thermal-throttled.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 8, @@ -1043,89 +741,101 @@ "x": 12, "y": 25 }, - "id": 32, + "repeatDirection": "h", "options": { "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, "calcs": [ "mean", "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + ] }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "" } }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "gpu:thermal_throttle_fraction:rate5m", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${ds_prometheus}" + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 0.05, + "color": "yellow" + }, + { + "value": 0.2, + "color": "red" + } + ] + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 25, + "showPoints": "never" } - } - ], - "title": "Thermal throttle fraction per GPU", - "type": "timeseries" + }, + "overrides": [] + } } ], - "refresh": "", - "schemaVersion": 40, - "tags": [ - "gpu", - "efficiency", - "finops" - ], "templating": { "list": [ { - "current": {}, - "includeAll": false, - "label": "Datasource", + "type": "datasource", "name": "ds_prometheus", - "options": [], + "label": "Prometheus", + "skipUrlSync": false, "query": "prometheus", - "refresh": 1, - "regex": "vm-.*", - "type": "datasource" + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "multi": false, + "allowCustomValue": true, + "includeAll": false, + "regex": "", + "auto": false, + "auto_min": "10s", + "auto_count": 30 }, { - "current": {}, + "type": "query", + "name": "namespace", + "label": "Namespace", + "skipUrlSync": false, + "query": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)", "datasource": { "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" + "uid": "$ds_prometheus" + }, + "current": { + "selected": true, + "text": "All", + "value": "$__all" }, - "definition": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)", - "includeAll": true, - "label": "Namespace", "multi": true, - "name": "namespace", - "options": [], - "query": { - "query": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)", - "refId": "StandardVariableQuery" - }, + "allowCustomValue": true, "refresh": 2, - "regex": "", "sort": 1, - "type": "query" + "includeAll": true, + "auto": false, + "auto_min": "10s", + "auto_count": 30 } ] }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "GPU Efficiency Score", - "uid": "gpu-efficiency", - "version": 2, - "weekStart": "" + "annotations": {} } \ No newline at end of file diff --git a/dashboards/gpu/gpu-performance.json b/dashboards/gpu/gpu-performance.json index 906d6a11..b0f26958 100644 --- a/dashboards/gpu/gpu-performance.json +++ b/dashboards/gpu/gpu-performance.json @@ -1,65 +1,24 @@ { - "__inputs": [ - { - "name": "DS_VM-SHORTTERM", - "label": "vm-shortterm", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } + "uid": "gpu-performance", + "title": "GPU Performance", + "tags": [ + "gpu", + "dcgm" ], - "__elements": {}, - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "11.4.0" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "stat", - "name": "Stat", - "version": "" - }, - { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, + "timezone": "browser", "editable": true, - "fiscalYearStartMonth": 0, "graphTooltip": 1, - "id": null, - "links": [], + "time": { + "from": "now-1h", + "to": "now" + }, + "fiscalYearStartMonth": 0, + "schemaVersion": 42, "panels": [ { + "type": "row", "collapsed": false, + "title": "Overview", "gridPos": { "h": 1, "w": 24, @@ -67,99 +26,75 @@ "y": 0 }, "id": 1, - "panels": [], - "title": "Overview", - "type": "row" + "panels": [] }, { + "type": "stat", + "id": 2, + "targets": [ + { + "expr": "cluster:gpu_count:total", + "refId": "A" + } + ], + "title": "Total GPUs", + "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, - "id": 2, + "repeatDirection": "h", "options": { - "colorMode": "value", "graphMode": "none", + "colorMode": "value", "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" - ], - "fields": "", - "values": false + ] }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "cluster:gpu_count:total", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - } - } - ], - "title": "Total GPUs", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" + "percentChangeColorMode": "standard", + "orientation": "" }, "fieldConfig": { "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], + "unit": "short", "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 + "value": null, + "color": "blue" } ] - }, - "unit": "short" + } }, "overrides": [] + } + }, + { + "type": "stat", + "id": 3, + "targets": [ + { + "expr": "cluster:gpu_count:allocated", + "refId": "A" + } + ], + "title": "Allocated", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 4, @@ -167,71 +102,56 @@ "x": 6, "y": 1 }, - "id": 3, + "repeatDirection": "h", "options": { - "colorMode": "value", "graphMode": "none", + "colorMode": "value", "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" - ], - "fields": "", - "values": false + ] }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "cluster:gpu_count:allocated", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - } - } - ], - "title": "Allocated", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" + "percentChangeColorMode": "standard", + "orientation": "" }, "fieldConfig": { "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "max": 100, - "min": 0, + "unit": "short", "thresholds": { "mode": "absolute", "steps": [ { - "color": "blue", - "value": null + "value": null, + "color": "green" }, { - "color": "green", - "value": 30 - }, - { - "color": "orange", - "value": 80 + "value": 1, + "color": "yellow" } ] - }, - "unit": "percent" + } }, "overrides": [] + } + }, + { + "type": "stat", + "id": 4, + "targets": [ + { + "expr": "cluster:gpu_util:avg", + "refId": "A" + } + ], + "title": "Average utilization", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 4, @@ -239,62 +159,62 @@ "x": 12, "y": 1 }, - "id": 4, + "repeatDirection": "h", "options": { - "colorMode": "value", "graphMode": "area", + "colorMode": "value", "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" - ], - "fields": "", - "values": false + ] }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "cluster:gpu_util:avg", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - } - } - ], - "title": "Average utilization", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" + "percentChangeColorMode": "standard", + "orientation": "" }, "fieldConfig": { "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 0, - "mappings": [], + "unit": "percent", + "min": 0, + "max": 100, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "value": null, + "color": "blue" + }, + { + "value": 30, + "color": "green" + }, + { + "value": 80, + "color": "orange" } ] - }, - "unit": "watt" + } }, "overrides": [] + } + }, + { + "type": "stat", + "id": 5, + "targets": [ + { + "expr": "cluster:gpu_power_watts:sum", + "refId": "A" + } + ], + "title": "Power draw", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 4, @@ -302,40 +222,43 @@ "x": 18, "y": 1 }, - "id": 5, + "repeatDirection": "h", "options": { - "colorMode": "value", "graphMode": "area", + "colorMode": "value", "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" - ], - "fields": "", - "values": false + ] }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": true + "percentChangeColorMode": "standard", + "orientation": "" }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "cluster:gpu_power_watts:sum", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" + "fieldConfig": { + "defaults": { + "unit": "watt", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + } + ] } - } - ], - "title": "Power draw", - "type": "stat" + }, + "overrides": [] + } }, { + "type": "row", "collapsed": false, + "title": "Utilization", "gridPos": { "h": 1, "w": 24, @@ -343,172 +266,77 @@ "y": 5 }, "id": 10, - "panels": [], - "title": "Utilization", - "type": "row" + "panels": [] }, { - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 6 - }, + "type": "timeseries", "id": 11, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", "targets": [ { "expr": "DCGM_FI_DEV_GPU_UTIL{Hostname=~\"$Hostname\", namespace=~\"$namespace\"}", "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - } + "refId": "A" } ], "title": "GPU Utilization (NVML)", - "type": "timeseries" - }, - { + "transparent": false, "datasource": { "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "repeatDirection": "h", + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "" + } }, "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 100, + "unit": "percent", "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" + "max": 100, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false + } }, "overrides": [] + } + }, + { + "type": "timeseries", + "id": 12, + "targets": [ + { + "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 100", + "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", + "refId": "A" + } + ], + "title": "Tensor Pipe Active (realistic load for LLM/AI)", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 8, @@ -516,99 +344,53 @@ "x": 12, "y": 6 }, - "id": 12, + "repeatDirection": "h", "options": { "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, "calcs": [ "mean", "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + ] }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "" } }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 100", - "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - } - } - ], - "title": "Tensor Pipe Active (realistic load for LLM/AI)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 100, + "unit": "percent", "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" + "max": 100, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false + } }, "overrides": [] + } + }, + { + "type": "timeseries", + "id": 13, + "targets": [ + { + "expr": "DCGM_FI_PROF_GR_ENGINE_ACTIVE{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 100", + "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", + "refId": "A" + } + ], + "title": "Graphics Engine Active", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 8, @@ -616,99 +398,53 @@ "x": 0, "y": 14 }, - "id": 13, + "repeatDirection": "h", "options": { "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, "calcs": [ "mean", "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + ] }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "" } }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "DCGM_FI_PROF_GR_ENGINE_ACTIVE{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 100", - "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - } - } - ], - "title": "Graphics Engine Active", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - }, "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 100, + "unit": "percent", "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" + "max": 100, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false + } }, "overrides": [] + } + }, + { + "type": "timeseries", + "id": 14, + "targets": [ + { + "expr": "DCGM_FI_DEV_MEM_COPY_UTIL{Hostname=~\"$Hostname\", namespace=~\"$namespace\"}", + "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", + "refId": "A" + } + ], + "title": "Memory Copy Utilization", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 8, @@ -716,39 +452,42 @@ "x": 12, "y": 14 }, - "id": 14, + "repeatDirection": "h", "options": { "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, "calcs": [ "mean", "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + ] }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "" } }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "DCGM_FI_DEV_MEM_COPY_UTIL{Hostname=~\"$Hostname\", namespace=~\"$namespace\"}", - "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false } - } - ], - "title": "Memory Copy Utilization", - "type": "timeseries" + }, + "overrides": [] + } }, { + "type": "row", "collapsed": false, + "title": "Memory", "gridPos": { "h": 1, "w": 24, @@ -756,168 +495,74 @@ "y": 22 }, "id": 20, - "panels": [], - "title": "Memory", - "type": "row" + "panels": [] }, { + "type": "timeseries", + "id": 21, + "targets": [ + { + "expr": "DCGM_FI_DEV_FB_USED{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 1048576", + "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", + "refId": "A" + } + ], + "title": "VRAM Used", + "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 25, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 23 }, - "id": 21, + "repeatDirection": "h", "options": { "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, "calcs": [ "mean", "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + ] }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "" } }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "DCGM_FI_DEV_FB_USED{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 1048576", - "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - } - } - ], - "title": "VRAM Used", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - }, "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, + "unit": "bytes", "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" + "fillOpacity": 25, + "showPoints": "never" + } }, "overrides": [] + } + }, + { + "type": "timeseries", + "id": 22, + "targets": [ + { + "expr": "DCGM_FI_DEV_FB_FREE{Hostname=~\"$Hostname\"} * 1048576", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A" + } + ], + "title": "VRAM Free", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 8, @@ -925,39 +570,39 @@ "x": 12, "y": 23 }, - "id": 22, + "repeatDirection": "h", "options": { "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, "calcs": [ "mean", "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + ] }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "" } }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "DCGM_FI_DEV_FB_FREE{Hostname=~\"$Hostname\"} * 1048576", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never" } - } - ], - "title": "VRAM Free", - "type": "timeseries" + }, + "overrides": [] + } }, { + "type": "row", "collapsed": false, + "title": "Power \u0026 Temperature", "gridPos": { "h": 1, "w": 24, @@ -965,174 +610,74 @@ "y": 31 }, "id": 30, - "panels": [], - "title": "Power & Temperature", - "type": "row" + "panels": [] }, { + "type": "timeseries", + "id": 31, + "targets": [ + { + "expr": "DCGM_FI_DEV_POWER_USAGE{Hostname=~\"$Hostname\"}", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A" + } + ], + "title": "Power Usage per GPU", + "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "watt" - }, - "overrides": [] - }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 }, - "id": 31, + "repeatDirection": "h", "options": { "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, "calcs": [ "mean", "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + ] }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "" } }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "DCGM_FI_DEV_POWER_USAGE{Hostname=~\"$Hostname\"}", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - } - } - ], - "title": "Power Usage per GPU", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - }, "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, + "unit": "watt", "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 100, - "min": 20, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 75 - }, - { - "color": "red", - "value": 85 - } - ] - }, - "unit": "celsius" + "fillOpacity": 10, + "showPoints": "never" + } }, "overrides": [] + } + }, + { + "type": "timeseries", + "id": 32, + "targets": [ + { + "expr": "DCGM_FI_DEV_GPU_TEMP{Hostname=~\"$Hostname\"}", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A" + } + ], + "title": "GPU Temperature", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 8, @@ -1140,39 +685,58 @@ "x": 12, "y": 32 }, - "id": 32, + "repeatDirection": "h", "options": { "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, "calcs": [ "mean", "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + ] }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "" } }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "DCGM_FI_DEV_GPU_TEMP{Hostname=~\"$Hostname\"}", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" + "fieldConfig": { + "defaults": { + "unit": "celsius", + "min": 20, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 75, + "color": "yellow" + }, + { + "value": 85, + "color": "red" + } + ] + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never" } - } - ], - "title": "GPU Temperature", - "type": "timeseries" + }, + "overrides": [] + } }, { + "type": "row", "collapsed": false, + "title": "Health", "gridPos": { "h": 1, "w": 24, @@ -1180,85 +744,81 @@ "y": 40 }, "id": 40, - "panels": [], - "title": "Health", - "type": "row" + "panels": [] }, { + "type": "stat", + "id": 41, + "targets": [ + { + "expr": "max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS{Hostname=~\"$Hostname\"})", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A" + } + ], + "title": "XID errors (latest)", + "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 1 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, "gridPos": { "h": 5, "w": 8, "x": 0, "y": 41 }, - "id": 41, + "repeatDirection": "h", "options": { - "colorMode": "background", "graphMode": "none", + "colorMode": "background", + "justifyMode": "auto", + "textMode": "value_and_name", + "wideLayout": true, + "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" - ], - "fields": "", - "values": false + ] }, - "textMode": "value_and_name" - }, - "targets": [ - { - "expr": "max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS{Hostname=~\"$Hostname\"})", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - } - } - ], - "title": "XID errors (latest)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" + "percentChangeColorMode": "standard", + "orientation": "" }, "fieldConfig": { "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineInterpolation": "linear", - "showPoints": "never" - }, - "unit": "µs" + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 1, + "color": "red" + } + ] + } }, "overrides": [] + } + }, + { + "type": "timeseries", + "id": 42, + "targets": [ + { + "expr": "rate(DCGM_FI_DEV_POWER_VIOLATION{Hostname=~\"$Hostname\"}[5m])", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A" + } + ], + "title": "Power Violation (µs/s throttled due to power)", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 5, @@ -1266,46 +826,47 @@ "x": 8, "y": 41 }, - "id": 42, + "repeatDirection": "h", "options": { "legend": { "displayMode": "list", - "placement": "bottom" + "placement": "bottom", + "showLegend": false, + "calcs": [] }, "tooltip": { - "mode": "multi" + "mode": "multi", + "sort": "" } }, - "targets": [ - { - "expr": "rate(DCGM_FI_DEV_POWER_VIOLATION{Hostname=~\"$Hostname\"}[5m])", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - } - } - ], - "title": "Power Violation (µs/s throttled due to power)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, "fieldConfig": { "defaults": { + "unit": "µs", "custom": { "drawStyle": "line", - "fillOpacity": 10, "lineInterpolation": "linear", + "fillOpacity": 10, "showPoints": "never" - }, - "unit": "µs" + } }, "overrides": [] + } + }, + { + "type": "timeseries", + "id": 43, + "targets": [ + { + "expr": "rate(DCGM_FI_DEV_THERMAL_VIOLATION{Hostname=~\"$Hostname\"}[5m])", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A" + } + ], + "title": "Thermal Violation (µs/s throttled due to thermals)", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 5, @@ -1313,102 +874,103 @@ "x": 16, "y": 41 }, - "id": 43, + "repeatDirection": "h", "options": { "legend": { "displayMode": "list", - "placement": "bottom" + "placement": "bottom", + "showLegend": false, + "calcs": [] }, "tooltip": { - "mode": "multi" + "mode": "multi", + "sort": "" } }, - "targets": [ - { - "expr": "rate(DCGM_FI_DEV_THERMAL_VIOLATION{Hostname=~\"$Hostname\"}[5m])", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" + "fieldConfig": { + "defaults": { + "unit": "µs", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never" } - } - ], - "title": "Thermal Violation (µs/s throttled due to thermals)", - "type": "timeseries" + }, + "overrides": [] + } } ], - "refresh": "", - "schemaVersion": 40, - "tags": [ - "gpu", - "dcgm" - ], "templating": { "list": [ { - "current": {}, - "includeAll": false, - "label": "Prometheus", + "type": "datasource", "name": "ds_prometheus", - "options": [], + "label": "Prometheus", + "skipUrlSync": false, "query": "prometheus", - "refresh": 1, + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "multi": false, + "allowCustomValue": true, + "includeAll": false, "regex": "", - "type": "datasource" + "auto": false, + "auto_min": "10s", + "auto_count": 30 }, { - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - }, - "definition": "label_values(DCGM_FI_DEV_GPU_UTIL, Hostname)", - "includeAll": true, - "label": "Host", - "multi": true, + "type": "query", "name": "Hostname", - "options": [], - "query": { - "query": "label_values(DCGM_FI_DEV_GPU_UTIL, Hostname)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": {}, + "label": "Host", + "skipUrlSync": false, + "query": "label_values(DCGM_FI_DEV_GPU_UTIL, Hostname)", "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, - "definition": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)", - "includeAll": true, - "label": "Namespace", - "multi": true, - "name": "namespace", - "options": [], - "query": { - "query": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)", - "refId": "StandardVariableQuery" + "current": { + "selected": true, + "text": "All", + "value": "$__all" }, + "multi": true, + "allowCustomValue": true, "refresh": 2, - "regex": "", "sort": 1, - "type": "query" + "includeAll": true, + "auto": false, + "auto_min": "10s", + "auto_count": 30 + }, + { + "type": "query", + "name": "namespace", + "label": "Namespace", + "skipUrlSync": false, + "query": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "multi": true, + "allowCustomValue": true, + "refresh": 2, + "sort": 1, + "includeAll": true, + "auto": false, + "auto_min": "10s", + "auto_count": 30 } ] }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "GPU Performance", - "uid": "gpu-performance", - "version": 2, - "weekStart": "" + "annotations": {} } diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json index 28f2c892..840065f3 100644 --- a/dashboards/gpu/gpu-quotas.json +++ b/dashboards/gpu/gpu-quotas.json @@ -1,83 +1,24 @@ { - "__inputs": [ - { - "name": "DS_VM-SHORTTERM", - "label": "vm-shortterm", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } + "uid": "gpu-quotas", + "title": "GPU Quotas \u0026 Allocation", + "tags": [ + "gpu", + "quotas" ], - "__elements": {}, - "__requires": [ - { - "type": "panel", - "id": "bargauge", - "name": "Bar gauge", - "version": "" - }, - { - "type": "panel", - "id": "gauge", - "name": "Gauge", - "version": "" - }, - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "11.4.0" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "stat", - "name": "Stat", - "version": "" - }, - { - "type": "panel", - "id": "table", - "name": "Table", - "version": "" - }, - { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, + "timezone": "browser", "editable": true, - "fiscalYearStartMonth": 0, "graphTooltip": 1, - "id": null, - "links": [], + "time": { + "from": "now-6h", + "to": "now" + }, + "fiscalYearStartMonth": 0, + "schemaVersion": 42, "panels": [ { + "type": "row", "collapsed": false, + "title": "Allocation overview", "gridPos": { "h": 1, "w": 24, @@ -85,99 +26,75 @@ "y": 0 }, "id": 1, - "panels": [], - "title": "Allocation overview", - "type": "row" + "panels": [] }, { + "type": "stat", + "id": 2, + "targets": [ + { + "expr": "sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"})", + "refId": "A" + } + ], + "title": "GPU allocatable", + "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, - "id": 2, + "repeatDirection": "h", "options": { - "colorMode": "value", "graphMode": "none", + "colorMode": "value", "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" - ], - "fields": "", - "values": false + ] }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"})", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - } - } - ], - "title": "GPU allocatable", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" + "percentChangeColorMode": "standard", + "orientation": "" }, "fieldConfig": { "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], + "unit": "short", "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 + "value": null, + "color": "blue" } ] - }, - "unit": "short" + } }, "overrides": [] + } + }, + { + "type": "stat", + "id": 3, + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"})", + "refId": "A" + } + ], + "title": "GPU requested", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 4, @@ -185,75 +102,56 @@ "x": 6, "y": 1 }, - "id": 3, + "repeatDirection": "h", "options": { - "colorMode": "value", "graphMode": "none", + "colorMode": "value", "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" - ], - "fields": "", - "values": false + ] }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"})", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - } - } - ], - "title": "GPU requested", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" + "percentChangeColorMode": "standard", + "orientation": "" }, "fieldConfig": { "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "max": 100, - "min": 0, + "unit": "short", "thresholds": { "mode": "absolute", "steps": [ { - "color": "blue", - "value": null + "value": null, + "color": "green" }, { - "color": "green", - "value": 40 - }, - { - "color": "yellow", - "value": 80 - }, - { - "color": "red", - "value": 95 + "value": 1, + "color": "yellow" } ] - }, - "unit": "percent" + } }, "overrides": [] + } + }, + { + "type": "gauge", + "id": 4, + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}) / sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"}) * 100", + "refId": "A" + } + ], + "title": "Allocation ratio", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 4, @@ -261,63 +159,64 @@ "x": 12, "y": 1 }, - "id": 4, + "repeatDirection": "h", "options": { - "minVizHeight": 75, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", "minVizWidth": 75, - "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" - ], - "fields": "", - "values": false + ] }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}) / sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"}) * 100", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - } - } - ], - "title": "Allocation ratio", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" + "minVizHeight": 75, + "orientation": "" }, "fieldConfig": { "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], + "unit": "percent", + "min": 0, + "max": 100, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "value": null, + "color": "blue" }, { - "color": "red", - "value": 1 + "value": 40, + "color": "green" + }, + { + "value": 80, + "color": "yellow" + }, + { + "value": 95, + "color": "red" } ] - }, - "unit": "short" + } }, "overrides": [] + } + }, + { + "type": "stat", + "id": 5, + "targets": [ + { + "expr": "count((kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"} \u003e 0) * on(namespace, pod) group_left() (kube_pod_status_phase{phase=\"Pending\"} == 1)) or vector(0)", + "refId": "A" + } + ], + "title": "Pending pods (GPU)", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 4, @@ -325,40 +224,46 @@ "x": 18, "y": 1 }, - "id": 5, + "repeatDirection": "h", "options": { - "colorMode": "background", "graphMode": "none", + "colorMode": "background", "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" - ], - "fields": "", - "values": false + ] }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": true + "percentChangeColorMode": "standard", + "orientation": "" }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "count((kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"} > 0) * on(namespace, pod) group_left() (kube_pod_status_phase{phase=\"Pending\"} == 1)) or vector(0)", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 1, + "color": "red" + } + ] } - } - ], - "title": "Pending pods (GPU)", - "type": "stat" + }, + "overrides": [] + } }, { + "type": "row", "collapsed": false, + "title": "Per namespace", "gridPos": { "h": 1, "w": 24, @@ -366,140 +271,120 @@ "y": 5 }, "id": 10, - "panels": [], - "title": "Per namespace", - "type": "row" + "panels": [] }, { + "type": "bargauge", + "id": 11, + "targets": [ + { + "expr": "sum by (namespace) (kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"})", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "title": "GPU requested per namespace", + "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "max": 8, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, - "id": 11, + "repeatDirection": "h", "options": { "displayMode": "gradient", + "valueMode": "color", + "namePlacement": "auto", + "showUnfilled": true, + "sizing": "auto", + "minVizWidth": 8, + "minVizHeight": 16, "legend": { - "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": false + "showLegend": false, + "calcs": [] }, - "maxVizHeight": 300, - "minVizHeight": 16, - "minVizWidth": 8, - "namePlacement": "auto", - "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" - ], - "fields": "", - "values": false + ] }, - "showUnfilled": true, - "sizing": "auto", - "valueMode": "color" - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "sum by (namespace) (kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"})", - "legendFormat": "{{namespace}}", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - } - } - ], - "title": "GPU requested per namespace", - "type": "bargauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" + "maxVizHeight": 300, + "orientation": "horizontal" }, "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "stepAfter", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], + "unit": "short", + "min": 0, + "max": 8, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 + "value": null, + "color": "green" } ] - }, - "unit": "short" + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "id": 12, + "targets": [ + { + "expr": "sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"})", + "legendFormat": "Allocatable (total)", + "refId": "A" + }, + { + "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"})", + "legendFormat": "Requested", + "refId": "B" + } + ], + "title": "GPU allocated over time", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "repeatDirection": "h", + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineInterpolation": "stepAfter", + "fillOpacity": 10, + "showPoints": "never" + } }, "overrides": [ { @@ -518,52 +403,12 @@ ] } ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 6 - }, - "id": 12, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"})", - "legendFormat": "Allocatable (total)", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - } - }, - { - "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"})", - "legendFormat": "Requested", - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - } - } - ], - "title": "GPU allocated over time", - "type": "timeseries" + } }, { + "type": "row", "collapsed": false, + "title": "Pods with GPU", "gridPos": { "h": 1, "w": 24, @@ -571,124 +416,40 @@ "y": 14 }, "id": 20, - "panels": [], - "title": "Pods with GPU", - "type": "row" + "panels": [] }, { + "type": "table", + "id": 21, + "targets": [ + { + "expr": "kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left(phase) (kube_pod_status_phase{phase=~\"Running|Pending|Failed|Succeeded\"} == 1)", + "instant": true, + "range": false, + "format": "table", + "refId": "requested" + }, + { + "expr": "kube_pod_container_resource_limits{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"}", + "instant": true, + "range": false, + "format": "table", + "refId": "limits" + } + ], + "title": "Pods requesting GPU", + "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Status" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "mode": "basic", - "type": "color-background" - } - }, - { - "id": "mappings", - "value": [ - { - "options": { - "Failed": { - "color": "red", - "index": 2 - }, - "Pending": { - "color": "orange", - "index": 1 - }, - "Running": { - "color": "green", - "index": 0 - } - }, - "type": "value" - } - ] - } - ] - } - ] - }, "gridPos": { "h": 10, "w": 24, "x": 0, "y": 15 }, - "id": 21, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left(phase) (kube_pod_status_phase{phase=~\"Running|Pending|Failed|Succeeded\"} == 1)", - "format": "table", - "instant": true, - "legendFormat": "", - "refId": "requested", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - } - }, - { - "expr": "kube_pod_container_resource_limits{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"}", - "format": "table", - "instant": true, - "legendFormat": "", - "refId": "limits", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - } - } - ], - "title": "Pods requesting GPU", + "repeatDirection": "h", "transformations": [ { "id": "joinByField", @@ -764,59 +525,106 @@ } } ], - "type": "table" + "options": { + "frameIndex": 0, + "showHeader": true, + "showTypeIcons": false, + "footer": { + "show": false, + "reducer": [] + }, + "cellHeight": "sm" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "color-background" + } + }, + { + "id": "mappings", + "value": [ + { + "options": { + "Failed": { + "color": "red", + "index": 2 + }, + "Pending": { + "color": "orange", + "index": 1 + }, + "Running": { + "color": "green", + "index": 0 + } + }, + "type": "value" + } + ] + } + ] + } + ] + } } ], - "refresh": "", - "schemaVersion": 40, - "tags": [ - "gpu", - "quotas" - ], "templating": { "list": [ { - "current": {}, - "includeAll": false, - "label": "Prometheus", + "type": "datasource", "name": "ds_prometheus", - "options": [], + "label": "Prometheus", + "skipUrlSync": false, "query": "prometheus", - "refresh": 1, + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "multi": false, + "allowCustomValue": true, + "includeAll": false, "regex": "", - "type": "datasource" + "auto": false, + "auto_min": "10s", + "auto_count": 30 }, { - "current": {}, + "type": "query", + "name": "namespace", + "label": "Namespace", + "skipUrlSync": false, + "query": "label_values(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}, namespace)", "datasource": { "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" + "uid": "$ds_prometheus" + }, + "current": { + "selected": true, + "text": "All", + "value": "$__all" }, - "definition": "label_values(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}, namespace)", - "includeAll": true, - "label": "Namespace", "multi": true, - "name": "namespace", - "options": [], - "query": { - "query": "label_values(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}, namespace)", - "refId": "StandardVariableQuery" - }, + "allowCustomValue": true, "refresh": 2, - "regex": "", "sort": 1, - "type": "query" + "includeAll": true, + "auto": false, + "auto_min": "10s", + "auto_count": 30 } ] }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "GPU Quotas & Allocation", - "uid": "gpu-quotas", - "version": 4, - "weekStart": "" -} + "annotations": {} +} \ No newline at end of file From 11f7d3589b3284fca96d0df21a02a5e967dadce7 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 05:47:33 +0300 Subject: [PATCH 006/130] chore: ignore CLAUDE.local.md Signed-off-by: Arsolitt --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 61003de5..64470222 100644 --- a/.gitignore +++ b/.gitignore @@ -84,3 +84,4 @@ tmp/ # build revision marker (generated by make image-packages) packages/core/platform/.build-revision .claude/ +CLAUDE.local.md From 4f8cef47bf307d7e23eac6536a1e7153a940d283 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 05:54:21 +0300 Subject: [PATCH 007/130] fix(monitoring): restore trailing newline in GPU dashboards Signed-off-by: Arsolitt --- dashboards/gpu/gpu-efficiency.json | 2 +- dashboards/gpu/gpu-quotas.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboards/gpu/gpu-efficiency.json b/dashboards/gpu/gpu-efficiency.json index 1a305b4b..596961e0 100644 --- a/dashboards/gpu/gpu-efficiency.json +++ b/dashboards/gpu/gpu-efficiency.json @@ -838,4 +838,4 @@ ] }, "annotations": {} -} \ No newline at end of file +} diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json index 840065f3..60c628f6 100644 --- a/dashboards/gpu/gpu-quotas.json +++ b/dashboards/gpu/gpu-quotas.json @@ -627,4 +627,4 @@ ] }, "annotations": {} -} \ No newline at end of file +} From 4e8731b58882edab23d79c99b5b6b49dda96c572 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 06:02:17 +0300 Subject: [PATCH 008/130] refactor(monitoring): rework GPU recording rules - Drop gpu.recording.30s group: per-GPU 30s aggregates had no consumers in tracked dashboards, only burned cardinality. - Drop namespace:gpu_allocated_count:gauge: identical expression to namespace:gpu_count:sum under a different name. - Reground :allocated on kube_pod_container_resource_requests so it reflects what tenants requested (Pending+Running) rather than what DCGM currently sees. namespace:gpu_count:sum stays DCGM-based and represents actually-running pods; the gap between the two is the signal admins want. - Add namespace:gpu_count:allocated as the per-namespace counterpart. Signed-off-by: Arsolitt --- .../alerts/gpu-recording.rules.yaml | 82 +++++++++++++------ 1 file changed, 59 insertions(+), 23 deletions(-) diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml index 871a46fc..a54860b7 100644 --- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml +++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml @@ -4,33 +4,17 @@ metadata: name: alerts-gpu-recording.rules spec: groups: - - name: gpu.recording.30s - interval: 30s - rules: - - record: gpu:util:avg - expr: avg by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_DEV_GPU_UTIL) - - record: gpu:mem_copy_util:avg - expr: avg by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_DEV_MEM_COPY_UTIL) - - record: gpu:fb_used_bytes:max - expr: max by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_DEV_FB_USED) * 1048576 - - record: gpu:fb_free_bytes:max - expr: max by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_DEV_FB_FREE) * 1048576 - - record: gpu:power_watts:avg - expr: avg by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_DEV_POWER_USAGE) - - record: gpu:temp_celsius:max - expr: max by (Hostname, gpu, UUID, modelName) (DCGM_FI_DEV_GPU_TEMP) - - record: gpu:tensor_active:avg - expr: avg by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_PROF_PIPE_TENSOR_ACTIVE) - - record: gpu:gr_engine_active:avg - expr: avg by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_PROF_GR_ENGINE_ACTIVE) - - name: gpu.recording.cluster.1m interval: 1m rules: - record: cluster:gpu_count:total expr: count(group by (UUID) (DCGM_FI_DEV_GPU_UTIL)) + # Kube-allocated GPU count: GPUs requested by *all* pods regardless of + # phase (Pending+Running). Source of truth for "what tenants asked for" + # — used for capacity planning and billing. Includes system pods so it + # stays consistent with cluster:gpu_count:total when computing :free. - record: cluster:gpu_count:allocated - expr: count(group by (UUID) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"})) + expr: sum(kube_pod_container_resource_requests{resource="nvidia_com_gpu"}) - record: cluster:gpu_count:free expr: cluster:gpu_count:total - (cluster:gpu_count:allocated or vector(0)) - record: cluster:gpu_util:avg @@ -41,8 +25,16 @@ spec: - name: gpu.recording.namespace.1m interval: 1m rules: + # DCGM-visible GPU count per namespace — counts GPUs that are actually + # running a tenant pod right now (driver loaded, scheduler placed it). + # Differs from :allocated when pods are Pending or stuck. - record: namespace:gpu_count:sum expr: count by (namespace) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}) + # Kube-requested GPU count per namespace — billable view, includes + # Pending pods. Use this for GPU-hour reporting via + # sum_over_time(...[1h:1m])/60. + - record: namespace:gpu_count:allocated + expr: sum by (namespace) (kube_pod_container_resource_requests{resource="nvidia_com_gpu"}) - record: namespace:gpu_util:avg expr: avg by (namespace) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}) - record: namespace:tensor_active:avg @@ -53,5 +45,49 @@ spec: expr: sum by (namespace) (DCGM_FI_DEV_POWER_USAGE{namespace!="", namespace!~"cozy-.*|kube-.*"}) - record: namespace:energy_joules:sum expr: sum by (namespace) (DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION{namespace!="", namespace!~"cozy-.*|kube-.*"}) / 1000 - - record: namespace:gpu_allocated_count:gauge - expr: count by (namespace) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}) + + - name: gpu.recording.efficiency.1m + interval: 1m + rules: + # Tensor hardware saturation — the honest "am I using the GPU" metric + # for AI/LLM workloads. Unlike NVML, idle tensor cores are visible. + - record: pod:tensor_saturation:avg5m + expr: | + avg by (Hostname, gpu, UUID, namespace, pod) ( + avg_over_time(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]) + ) * 100 + + # NVML vs tensor gap — ratio <10% means NVML lies: user thinks GPU + # is busy but specialized hardware is idle (cheap tenant signal). + - record: pod:tensor_to_nvml_ratio:avg5m + expr: | + ( + avg_over_time(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]) * 100 + ) + / + clamp_min( + avg_over_time(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]), + 1 + ) + + # Power efficiency — utilization per watt, reveals unoptimized clients. + - record: pod:util_per_watt:avg5m + expr: | + avg_over_time(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]) + / + clamp_min( + avg_over_time(DCGM_FI_DEV_POWER_USAGE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]), + 1 + ) + + # Fraction of time power-throttled (TDP cap) — 1.0 = fully throttled. + # DCGM_FI_DEV_*_VIOLATION is documented as µs but on A10/DCGM 3.x the + # counter grows in nanoseconds in practice — divide by 1e9 to get a + # 0..1 fraction (verified empirically when /1e6 yielded >100× reality). + # clamp_max protects against rate() artefacts at counter resets. + - record: gpu:power_throttle_fraction:rate5m + expr: clamp_max(rate(DCGM_FI_DEV_POWER_VIOLATION[5m]) / 1e9, 1) + + # Fraction of time thermal-throttled. + - record: gpu:thermal_throttle_fraction:rate5m + expr: clamp_max(rate(DCGM_FI_DEV_THERMAL_VIOLATION[5m]) / 1e9, 1) From 0e20159bd999795fd92f9b2099529bbe9b4de040 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 06:09:44 +0300 Subject: [PATCH 009/130] fix(gpu-operator): scope compat DaemonSet to GPU nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restrict the nvidia-driver-compat DaemonSet to nodes labelled nvidia.com/gpu.present=true (NFD/GPU Operator label). Without the nodeSelector it was scheduling onto every node — control-plane and CPU-only workers included — burning a privileged pod slot per host for no benefit. Add resource requests and limits to the init and pause containers so the DaemonSet stays within control-plane budget on small clusters. Signed-off-by: Arsolitt --- .../examples/nvidia-driver-compat.yaml | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml b/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml index e4718e0a..63e07c4f 100644 --- a/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml +++ b/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml @@ -27,6 +27,14 @@ spec: spec: hostPID: true priorityClassName: system-node-critical + # Restrict to GPU nodes only. Without this the DaemonSet schedules onto + # every node (including control-plane and CPU-only workers) and burns + # resources on hosts where the compat tree is meaningless. + # The label is published by Node Feature Discovery / GPU Operator's NFD + # subchart; if NFD is disabled, replace this with whatever label your + # cluster uses to mark GPU hosts. + nodeSelector: + nvidia.com/gpu.present: "true" tolerations: - operator: Exists initContainers: @@ -60,12 +68,26 @@ spec: echo "Done" securityContext: privileged: true + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 100m + memory: 64Mi volumeMounts: - name: host-root mountPath: /host containers: - name: pause image: registry.k8s.io/pause:3.10 + resources: + requests: + cpu: 10m + memory: 8Mi + limits: + cpu: 50m + memory: 16Mi volumes: - name: host-root hostPath: From 38c8a37cb585ff235c7d90d0824a457e7da93095 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 06:15:58 +0300 Subject: [PATCH 010/130] docs(gpu-operator): clarify minimum required DCGM metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous wording implied that the entire custom DCGM CSV was required by the recording rules. In fact only the profiling counters (DCGM_FI_PROF_*) need to be added on top of the upstream defaults — everything else the rules consume is already in default-counters.csv. Add a Verification status block flagging that the minimum-set claim is derived from the DCGM Exporter version pinned in the currently shipped gpu-operator package and must be re-checked when that package moves to a newer release. Signed-off-by: Arsolitt --- .../system/gpu-operator/examples/README.md | 30 +++++++++++++++++-- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/packages/system/gpu-operator/examples/README.md b/packages/system/gpu-operator/examples/README.md index dac3703e..11c2bb09 100644 --- a/packages/system/gpu-operator/examples/README.md +++ b/packages/system/gpu-operator/examples/README.md @@ -21,9 +21,16 @@ Talos. ConfigMap. - [`dcgm-custom-metrics.yaml`](./dcgm-custom-metrics.yaml) — `ConfigMap` with a DCGM metrics CSV that adds profiling, ECC, throttling and - energy counters on top of the upstream defaults. Required by the - recording rules in `packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml` - and by several panels in the `gpu/gpu-performance` dashboard. + energy counters on top of the upstream defaults. The CSV is the + superset needed for full dashboard coverage; the **recording rules + themselves** only require the profiling subset + (`DCGM_FI_PROF_PIPE_TENSOR_ACTIVE`, `DCGM_FI_PROF_GR_ENGINE_ACTIVE`) + on top of the upstream `default-counters.csv` — every other DCGM + series the rules consume (utilization, FB used/free, power, + temperature, energy) is already in the default set. The + `gpu/gpu-performance` dashboard additionally needs the throttle + counters (`DCGM_FI_DEV_POWER_VIOLATION`, + `DCGM_FI_DEV_THERMAL_VIOLATION`), which are not in the default set. - [`nvidia-driver-compat.yaml`](./nvidia-driver-compat.yaml) — DaemonSet that stages `libnvidia-ml.so.1` and `nvidia-smi` from the Talos glibc tree into a path where the NVIDIA GPU Operator validator expects @@ -55,6 +62,23 @@ files into a directory the validator does inspect and creates the [1]: https://github.com/NVIDIA/gpu-operator/issues/1687 +## Verification status + +> **Pending verification on an updated GPU Operator release.** +> +> The minimum-CSV claim above (only `DCGM_FI_PROF_*` is needed beyond +> the upstream default counters) is derived by cross-referencing +> `gpu-recording.rules.yaml` against the DCGM Exporter +> [`default-counters.csv`][default-csv] for the version pinned in the +> currently shipped `gpu-operator` package. The package in this branch +> is **not** the latest GPU Operator release; once we move to a newer +> version, the claim must be re-checked because the upstream default +> set occasionally adds or removes counters between releases. Until +> then, treat the CSV in `dcgm-custom-metrics.yaml` as a known-good +> superset rather than a minimal config. + +[default-csv]: https://github.com/NVIDIA/dcgm-exporter/blob/main/etc/default-counters.csv + ## How the dashboard and recording rules fit in - `dashboards/gpu/gpu-performance.json` expects `DCGM_FI_*` metrics, From 49c1d7e7abeddebf6047a52b6de7e13f53c60ba5 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 06:22:31 +0300 Subject: [PATCH 011/130] test(monitoring): cross-check GPU dashboards against recording rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Catch dangling references at PR time: every recording-rule name used inside a tracked GPU dashboard must exist in packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml. The first iteration of gpu-efficiency.json shipped panels keyed on pod:tensor_saturation:avg5m without the rule defined; the test fails on exactly that class of bug. Scoped to dashboards listed under gpu/* in dashboards-infra.list, so untracked drafts stay out of scope until they are registered. Reverse direction (rule defined but unused) is intentionally NOT enforced — some rules exist for ad-hoc PromQL or upcoming dashboards. Auto-discovered by make bats-unit-tests via hack/cozytest.sh. Signed-off-by: Arsolitt --- hack/check-gpu-recording-rules.bats | 114 ++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 hack/check-gpu-recording-rules.bats diff --git a/hack/check-gpu-recording-rules.bats b/hack/check-gpu-recording-rules.bats new file mode 100644 index 00000000..87e7c8f1 --- /dev/null +++ b/hack/check-gpu-recording-rules.bats @@ -0,0 +1,114 @@ +#!/usr/bin/env bats +# ----------------------------------------------------------------------------- +# Cross-validation between GPU recording rules and the dashboards that consume +# them. Catches: +# +# 1. dangling references — a dashboard query mentions a recording rule name +# that doesn't exist in gpu-recording.rules.yaml. This is the bug the +# pre-merge review caught: gpu-efficiency.json shipped panels keyed on +# pod:tensor_saturation:avg5m without the rule being defined, so the +# panel showed "No data" everywhere. +# +# 2. typos in rule names — same bug class, manifested as a single-character +# difference between rule and reference. +# +# Scope: only dashboards listed in packages/system/monitoring/dashboards-infra.list +# under the "gpu/" prefix (i.e. shipped to production), not every JSON file in +# dashboards/gpu/. Untracked drafts stay out of scope on purpose — adding one +# to dashboards-infra.list is what brings it under the test. +# +# Reverse direction (rule defined but never consumed) is intentionally NOT +# enforced: some rules exist for ad-hoc PromQL or upcoming dashboards. Treat +# unused rules as an editorial concern, not a regression. +# +# Title syntax constraints from cozytest.sh's awk parser: +# - Titles delimited by ASCII double quotes; embedded quotes truncate. +# - Only [A-Za-z0-9] from the title survives into the function name; titles +# differing only in punctuation collapse to the same function. +# +# Run with: hack/cozytest.sh hack/check-gpu-recording-rules.bats +# ----------------------------------------------------------------------------- + +REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" +RULES_FILE="$REPO_ROOT/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml" +DASHBOARDS_LIST="$REPO_ROOT/packages/system/monitoring/dashboards-infra.list" +DASHBOARDS_DIR="$REPO_ROOT/dashboards" + +# Extract the set of "- record: NAME" entries from the rules YAML. +# Outputs one rule name per line, sorted and deduplicated. +extract_rules() { + awk '/^[[:space:]]*-[[:space:]]*record:[[:space:]]/ { + sub(/^[[:space:]]*-[[:space:]]*record:[[:space:]]*/, "") + sub(/[[:space:]]*$/, "") + print + }' "$RULES_FILE" | sort -u +} + +# Extract the set of recording-rule references from a dashboard JSON. +# A recording-rule reference is matched by the pattern +# :(:)+ +# where each is [a-z0-9_]. Raw DCGM metrics (DCGM_FI_*), +# kube-state-metrics (kube_*) and similar uppercase / single-word metric +# names do not match because the leading segment must be lowercase and the +# whole expression must contain at least two ':' characters. +extract_refs() { + json_file=$1 + grep -hoE '[a-z][a-z0-9_]*:[a-z0-9_]+:[a-z0-9_]+' "$json_file" | sort -u +} + +# Resolve "gpu/foo" -> "$DASHBOARDS_DIR/gpu/foo.json" +list_tracked_gpu_dashboards() { + awk '/^gpu\// { print $0 ".json" }' "$DASHBOARDS_LIST" +} + +@test "every recording rule reference in tracked GPU dashboards has a matching record" { + TMP=$(mktemp -d) + trap 'rm -rf "$TMP"' EXIT + + extract_rules > "$TMP/rules.txt" + [ -s "$TMP/rules.txt" ] || { echo "no recording rules extracted from $RULES_FILE" >&2; exit 1; } + + list_tracked_gpu_dashboards > "$TMP/dashboards.txt" + [ -s "$TMP/dashboards.txt" ] || { echo "no gpu/* dashboards listed in $DASHBOARDS_LIST" >&2; exit 1; } + + failed=0 + while IFS= read -r dashboard_rel; do + dashboard="$DASHBOARDS_DIR/$dashboard_rel" + if [ ! -f "$dashboard" ]; then + echo "ERROR: dashboard listed but file missing: $dashboard" >&2 + failed=1 + continue + fi + + extract_refs "$dashboard" > "$TMP/refs.txt" + # comm -23: lines unique to refs.txt (referenced but not defined) + # Both inputs must be sorted; extract_* helpers already sort. + comm -23 "$TMP/refs.txt" "$TMP/rules.txt" > "$TMP/missing.txt" + if [ -s "$TMP/missing.txt" ]; then + echo "ERROR: $dashboard_rel references undefined recording rules:" >&2 + sed 's/^/ - /' "$TMP/missing.txt" >&2 + failed=1 + fi + done < "$TMP/dashboards.txt" + + [ "$failed" -eq 0 ] +} + +@test "every tracked GPU dashboard listed in dashboards-infra.list exists on disk" { + TMP=$(mktemp -d) + trap 'rm -rf "$TMP"' EXIT + + list_tracked_gpu_dashboards > "$TMP/dashboards.txt" + [ -s "$TMP/dashboards.txt" ] || { echo "no gpu/* dashboards listed in $DASHBOARDS_LIST" >&2; exit 1; } + + failed=0 + while IFS= read -r dashboard_rel; do + dashboard="$DASHBOARDS_DIR/$dashboard_rel" + if [ ! -f "$dashboard" ]; then + echo "ERROR: $dashboard_rel listed in dashboards-infra.list but $dashboard does not exist" >&2 + failed=1 + fi + done < "$TMP/dashboards.txt" + + [ "$failed" -eq 0 ] +} From dbed4992b08392c28bac8f0b5f917619b88cec65 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 06:28:49 +0300 Subject: [PATCH 012/130] fix(monitoring): exclude system namespaces from namespace:gpu_count:allocated MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Align namespace:gpu_count:allocated with every other namespace:* rule by filtering out cozy-*/kube-*. All other per-namespace rules (gpu_util, tensor_active, fb_used_bytes, power_watts) already exclude system namespaces, so the label set produced by :allocated diverged from them — any dashboard variable or join that reads across these rules could end up with a different namespace list depending on which rule supplied the :allocated column. Trade-off: per-namespace GPU accounting for system workloads is no longer available through this rule. If it's ever needed, add a dedicated system:gpu_count:allocated rather than widening this one — the "billable tenant view" invariant is what the filter is protecting. Cluster-level cluster:gpu_count:allocated intentionally keeps system pods so it stays aligned with cluster:gpu_count:total and cluster:gpu_count:free remains meaningful. As a consequence, sum(namespace:gpu_count:allocated) no longer equals cluster:gpu_count:allocated; the delta is system-pod GPU usage, which is fine for the cluster-admin view. Signed-off-by: Arsolitt --- .../alerts/gpu-recording.rules.yaml | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml index a54860b7..3a31f5a2 100644 --- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml +++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml @@ -33,8 +33,27 @@ spec: # Kube-requested GPU count per namespace — billable view, includes # Pending pods. Use this for GPU-hour reporting via # sum_over_time(...[1h:1m])/60. + # + # Filters cozy-*/kube-* so the namespace set here matches every + # other namespace:* rule below. This keeps the $namespace variable + # in the tenant dashboard consistent with panels that read filtered + # rules — otherwise picking a system ns from the dropdown leaves + # half the panels empty while others show values. + # + # Trade-off: per-namespace GPU accounting for system workloads is + # no longer available here. If it's ever needed (e.g. operator + # pods pinning tenant GPUs), add a separate rule like + # system:gpu_count:allocated rather than widening this one — the + # "billable tenant view" invariant is relied on by tenants.json + # and keeping it narrow protects consumers from the asymmetry + # that triggered this filter in the first place. + # + # Note: sum(namespace:gpu_count:allocated) no longer equals + # cluster:gpu_count:allocated — the cluster-level rule intentionally + # keeps system pods to stay aligned with cluster:gpu_count:total + # when computing :free. - record: namespace:gpu_count:allocated - expr: sum by (namespace) (kube_pod_container_resource_requests{resource="nvidia_com_gpu"}) + expr: sum by (namespace) (kube_pod_container_resource_requests{resource="nvidia_com_gpu", namespace!="", namespace!~"cozy-.*|kube-.*"}) - record: namespace:gpu_util:avg expr: avg by (namespace) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}) - record: namespace:tensor_active:avg From 6d9066f0748744d78f823e3137f72b5e912069e9 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 06:35:12 +0300 Subject: [PATCH 013/130] feat(monitoring): add GPU fleet and tenants dashboards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - gpu-fleet: cluster-wide admin view — inventory, capacity (total / allocated / free), per-node utilization and power, throttling, temperatures, XID errors. - gpu-tenants: per-namespace view — live allocation, utilization, tensor saturation, power, and 24h GPU-hours / kWh integrations for billing inputs. Register both under gpu/* in dashboards-infra.list so they ship as GrafanaDashboard CRs and fall under the bats cross-check introduced earlier on this branch. Update examples/README to spell out which DCGM counters each of the five gpu/* dashboards actually needs on top of the upstream default CSV — gpu-performance needs profiling and throttling counters, gpu-efficiency needs profiling, gpu-tenants needs only DCGM_FI_PROF_PIPE_TENSOR_ACTIVE for its tensor panel, and gpu-fleet and gpu-quotas work on the default counter set alone. Signed-off-by: Arsolitt --- dashboards/gpu/gpu-fleet.json | 1157 +++++++++++++++ dashboards/gpu/gpu-tenants.json | 1236 +++++++++++++++++ .../system/gpu-operator/examples/README.md | 74 +- .../system/monitoring/dashboards-infra.list | 2 + 4 files changed, 2436 insertions(+), 33 deletions(-) create mode 100644 dashboards/gpu/gpu-fleet.json create mode 100644 dashboards/gpu/gpu-tenants.json diff --git a/dashboards/gpu/gpu-fleet.json b/dashboards/gpu/gpu-fleet.json new file mode 100644 index 00000000..af5ad2fa --- /dev/null +++ b/dashboards/gpu/gpu-fleet.json @@ -0,0 +1,1157 @@ +{ + "uid": "gpu-fleet", + "title": "GPU Fleet Overview", + "description": "Cluster-wide GPU inventory, capacity, utilization and health — admin view across the whole fleet.", + "tags": [ + "gpu", + "fleet", + "admin" + ], + "timezone": "browser", + "editable": true, + "graphTooltip": 1, + "time": { + "from": "now-6h", + "to": "now" + }, + "fiscalYearStartMonth": 0, + "schemaVersion": 42, + "panels": [ + { + "type": "row", + "collapsed": false, + "title": "Cluster snapshot", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [] + }, + { + "type": "stat", + "id": 2, + "targets": [ + { + "expr": "cluster:gpu_count:total", + "refId": "A" + } + ], + "title": "Total GPUs", + "description": "Physical GPUs discovered by DCGM across all nodes.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "repeatDirection": "h", + "options": { + "graphMode": "none", + "colorMode": "value", + "justifyMode": "auto", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "percentChangeColorMode": "standard", + "orientation": "" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "blue" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "id": 3, + "targets": [ + { + "expr": "cluster:gpu_count:allocated", + "refId": "A" + } + ], + "title": "Allocated", + "description": "Sum of kube GPU requests across all pods. Billable view — includes Pending pods.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "repeatDirection": "h", + "options": { + "graphMode": "none", + "colorMode": "background", + "justifyMode": "auto", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "percentChangeColorMode": "standard", + "orientation": "" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "blue" + }, + { + "value": 1, + "color": "green" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "id": 4, + "targets": [ + { + "expr": "cluster:gpu_count:free", + "refId": "A" + } + ], + "title": "Free", + "description": "Unallocated capacity available for new tenants.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, + "repeatDirection": "h", + "options": { + "graphMode": "none", + "colorMode": "background", + "justifyMode": "auto", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "percentChangeColorMode": "standard", + "orientation": "" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "red" + }, + { + "value": 1, + "color": "yellow" + }, + { + "value": 2, + "color": "green" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "id": 5, + "targets": [ + { + "expr": "count(kube_node_labels{label_nvidia_com_gpu_present=\"true\"})", + "refId": "A" + } + ], + "title": "Nodes with GPU", + "description": "Kubernetes nodes advertising nvidia.com/gpu.present=true.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 1 + }, + "repeatDirection": "h", + "options": { + "graphMode": "none", + "colorMode": "value", + "justifyMode": "auto", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "percentChangeColorMode": "standard", + "orientation": "" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "blue" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "id": 6, + "targets": [ + { + "expr": "cluster:gpu_util:avg", + "refId": "A" + } + ], + "title": "Average utilization", + "description": "NVML utilization across all tenant GPUs (engine-active %).", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 + }, + "repeatDirection": "h", + "options": { + "graphMode": "area", + "colorMode": "value", + "justifyMode": "auto", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "percentChangeColorMode": "standard", + "orientation": "" + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "blue" + }, + { + "value": 30, + "color": "green" + }, + { + "value": 80, + "color": "orange" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "id": 7, + "targets": [ + { + "expr": "cluster:gpu_power_watts:sum", + "refId": "A" + } + ], + "title": "Total power", + "description": "Live power draw summed across the fleet.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 1 + }, + "repeatDirection": "h", + "options": { + "graphMode": "area", + "colorMode": "value", + "justifyMode": "auto", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "percentChangeColorMode": "standard", + "orientation": "" + }, + "fieldConfig": { + "defaults": { + "unit": "watt", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "row", + "collapsed": false, + "title": "Per-node GPU activity", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 10, + "panels": [] + }, + { + "type": "bargauge", + "id": 11, + "targets": [ + { + "expr": "avg by (Hostname) (DCGM_FI_DEV_GPU_UTIL)", + "legendFormat": "{{Hostname}}", + "refId": "A" + } + ], + "title": "GPU utilization per node", + "description": "Average NVML utilization per node — spot idle or saturated hosts at a glance.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "repeatDirection": "h", + "options": { + "displayMode": "gradient", + "valueMode": "color", + "namePlacement": "auto", + "showUnfilled": true, + "sizing": "auto", + "minVizWidth": 8, + "minVizHeight": 16, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false, + "calcs": [] + }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "maxVizHeight": 300, + "orientation": "horizontal" + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "blue" + }, + { + "value": 30, + "color": "green" + }, + { + "value": 80, + "color": "orange" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "bargauge", + "id": 12, + "targets": [ + { + "expr": "sum by (Hostname) (DCGM_FI_DEV_POWER_USAGE)", + "legendFormat": "{{Hostname}}", + "refId": "A" + } + ], + "title": "Power draw per node", + "description": "Sum of GPU power usage per node in watts.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "repeatDirection": "h", + "options": { + "displayMode": "gradient", + "valueMode": "color", + "namePlacement": "auto", + "showUnfilled": true, + "sizing": "auto", + "minVizWidth": 8, + "minVizHeight": 16, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false, + "calcs": [] + }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "maxVizHeight": 300, + "orientation": "horizontal" + }, + "fieldConfig": { + "defaults": { + "unit": "watt", + "min": 0, + "max": 200, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 120, + "color": "yellow" + }, + { + "value": 160, + "color": "orange" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "row", + "collapsed": false, + "title": "Usage trends", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 20, + "panels": [] + }, + { + "type": "timeseries", + "id": 21, + "targets": [ + { + "expr": "avg by (Hostname) (DCGM_FI_DEV_GPU_UTIL)", + "legendFormat": "{{Hostname}}", + "refId": "A" + } + ], + "title": "GPU utilization per node (trend)", + "description": "Per-node NVML utilization over time — shows shifting workload across hosts.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "repeatDirection": "h", + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "" + } + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 20, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "id": 22, + "targets": [ + { + "expr": "sum by (Hostname) (DCGM_FI_DEV_POWER_USAGE)", + "legendFormat": "{{Hostname}}", + "refId": "A" + } + ], + "title": "Power draw per node (trend)", + "description": "Per-node GPU power consumption over time.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "repeatDirection": "h", + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "" + } + }, + "fieldConfig": { + "defaults": { + "unit": "watt", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 20, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "row", + "collapsed": false, + "title": "Health", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 30, + "panels": [] + }, + { + "type": "stat", + "id": 31, + "targets": [ + { + "expr": "count((max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS)) \u003e 0) or vector(0)", + "refId": "A" + } + ], + "title": "GPUs with XID errors", + "description": "Count of GPUs reporting a non-zero XID code. XID = NVIDIA hardware/driver error — any non-zero value needs investigation.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 24 + }, + "repeatDirection": "h", + "options": { + "graphMode": "none", + "colorMode": "background", + "justifyMode": "auto", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "percentChangeColorMode": "standard", + "orientation": "" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 1, + "color": "red" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "id": 32, + "targets": [ + { + "expr": "avg(gpu:power_throttle_fraction:rate5m)", + "refId": "A" + } + ], + "title": "Power throttling (cluster avg)", + "description": "Fraction of time the fleet spent power-throttled. \u003e5% = billed FLOPS are being underdelivered.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 24 + }, + "repeatDirection": "h", + "options": { + "graphMode": "none", + "colorMode": "background", + "justifyMode": "auto", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "percentChangeColorMode": "standard", + "orientation": "" + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 0.05, + "color": "yellow" + }, + { + "value": 0.2, + "color": "red" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "id": 33, + "targets": [ + { + "expr": "avg(gpu:thermal_throttle_fraction:rate5m)", + "refId": "A" + } + ], + "title": "Thermal throttling (cluster avg)", + "description": "Fraction of time the fleet spent thermal-throttled. Indicates cooling/datacenter issues.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 24 + }, + "repeatDirection": "h", + "options": { + "graphMode": "none", + "colorMode": "background", + "justifyMode": "auto", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "percentChangeColorMode": "standard", + "orientation": "" + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 0.05, + "color": "yellow" + }, + { + "value": 0.2, + "color": "red" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "id": 34, + "targets": [ + { + "expr": "max(DCGM_FI_DEV_GPU_TEMP)", + "refId": "A" + } + ], + "title": "Max GPU temperature", + "description": "Hottest GPU in the fleet right now. A10 sustained target \u003c83°C.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 24 + }, + "repeatDirection": "h", + "options": { + "graphMode": "none", + "colorMode": "background", + "justifyMode": "auto", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "percentChangeColorMode": "standard", + "orientation": "" + }, + "fieldConfig": { + "defaults": { + "unit": "celsius", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 75, + "color": "yellow" + }, + { + "value": 85, + "color": "red" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "id": 35, + "targets": [ + { + "expr": "DCGM_FI_DEV_GPU_TEMP", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A" + } + ], + "title": "Temperature per GPU", + "description": "Per-GPU temperature trace. Sustained red zone = cooling issue or aggressive workload.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 29 + }, + "repeatDirection": "h", + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "" + } + }, + "fieldConfig": { + "defaults": { + "unit": "celsius", + "min": 20, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 75, + "color": "yellow" + }, + { + "value": 85, + "color": "red" + } + ] + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "id": 36, + "targets": [ + { + "expr": "gpu:power_throttle_fraction:rate5m", + "legendFormat": "power {{Hostname}}/{{gpu}}", + "refId": "A" + }, + { + "expr": "gpu:thermal_throttle_fraction:rate5m", + "legendFormat": "thermal {{Hostname}}/{{gpu}}", + "refId": "B" + } + ], + "title": "Throttle fraction per GPU", + "description": "Red = power throttle, blue = thermal throttle. Both on same chart to compare causes.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 29 + }, + "repeatDirection": "h", + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "" + } + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 25, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "row", + "collapsed": false, + "title": "Hardware inventory", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 40, + "panels": [] + }, + { + "type": "table", + "id": 41, + "targets": [ + { + "expr": "group by (Hostname, modelName, gpu, UUID) (DCGM_FI_DEV_GPU_UTIL)", + "instant": true, + "range": false, + "format": "table", + "refId": "A" + } + ], + "title": "GPU inventory", + "description": "One row per physical GPU. Model / index / UUID sourced from DCGM labels.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 38 + }, + "repeatDirection": "h", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "__name__": true, + "cluster": true, + "container": true, + "device": true, + "endpoint": true, + "instance": true, + "job": true, + "pci_bus_id": true, + "prometheus": true, + "service": true, + "tenant": true, + "tier": true, + "uid": true, + "unit": true + }, + "indexByName": { + "Hostname": 0, + "UUID": 3, + "gpu": 2, + "modelName": 1 + }, + "renameByName": { + "Hostname": "Node", + "UUID": "UUID", + "gpu": "Index", + "modelName": "GPU Model" + } + } + } + ], + "options": { + "frameIndex": 0, + "showHeader": true, + "showTypeIcons": false, + "footer": { + "show": false, + "reducer": [] + }, + "cellHeight": "sm" + } + } + ], + "templating": { + "list": [ + { + "type": "datasource", + "name": "ds_prometheus", + "label": "Prometheus", + "skipUrlSync": false, + "query": "prometheus", + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "multi": false, + "allowCustomValue": true, + "includeAll": false, + "regex": "", + "auto": false, + "auto_min": "10s", + "auto_count": 30 + } + ] + }, + "annotations": {} +} diff --git a/dashboards/gpu/gpu-tenants.json b/dashboards/gpu/gpu-tenants.json new file mode 100644 index 00000000..d0265c0d --- /dev/null +++ b/dashboards/gpu/gpu-tenants.json @@ -0,0 +1,1236 @@ +{ + "uid": "gpu-tenants", + "title": "GPU Tenant Usage", + "description": "Per-tenant GPU consumption: live allocation, utilization, tensor saturation, power, and 24h GPU-hours / kWh — admin view across all namespaces.", + "tags": [ + "gpu", + "tenants", + "admin" + ], + "timezone": "browser", + "editable": true, + "graphTooltip": 1, + "time": { + "from": "now-24h", + "to": "now" + }, + "fiscalYearStartMonth": 0, + "schemaVersion": 42, + "panels": [ + { + "type": "row", + "collapsed": false, + "title": "Tenant fleet snapshot", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [] + }, + { + "type": "stat", + "id": 2, + "targets": [ + { + "expr": "count(namespace:gpu_count:allocated{namespace=~\"$namespace\"})", + "refId": "A" + } + ], + "title": "Active tenants", + "description": "Namespaces with at least one GPU request (Pending + Running).", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "repeatDirection": "h", + "options": { + "graphMode": "none", + "colorMode": "value", + "justifyMode": "auto", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "percentChangeColorMode": "standard", + "orientation": "" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "blue" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "id": 3, + "targets": [ + { + "expr": "sum(namespace:gpu_count:allocated{namespace=~\"$namespace\"})", + "refId": "A" + } + ], + "title": "Allocated GPUs", + "description": "Sum of kube GPU requests across tenants. Billable view — includes Pending pods.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "repeatDirection": "h", + "options": { + "graphMode": "none", + "colorMode": "background", + "justifyMode": "auto", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "percentChangeColorMode": "standard", + "orientation": "" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "blue" + }, + { + "value": 1, + "color": "green" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "id": 4, + "targets": [ + { + "expr": "avg(namespace:gpu_util:avg{namespace=~\"$namespace\"})", + "refId": "A" + } + ], + "title": "Avg tenant util", + "description": "Average NVML utilization across all tenant GPUs.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, + "repeatDirection": "h", + "options": { + "graphMode": "area", + "colorMode": "value", + "justifyMode": "auto", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "percentChangeColorMode": "standard", + "orientation": "" + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "blue" + }, + { + "value": 30, + "color": "green" + }, + { + "value": 80, + "color": "orange" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "id": 5, + "targets": [ + { + "expr": "sum(namespace:power_watts:sum{namespace=~\"$namespace\"})", + "refId": "A" + } + ], + "title": "Total tenant power", + "description": "Live power draw summed across tenant workloads.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 1 + }, + "repeatDirection": "h", + "options": { + "graphMode": "area", + "colorMode": "value", + "justifyMode": "auto", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "percentChangeColorMode": "standard", + "orientation": "" + }, + "fieldConfig": { + "defaults": { + "unit": "watt", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "id": 6, + "targets": [ + { + "expr": "sum(sum_over_time(namespace:power_watts:sum{namespace=~\"$namespace\"}[24h:1m])) / 60 / 1000", + "refId": "A" + } + ], + "title": "Energy (last 24h)", + "description": "Integrated power draw over 24h. Feed into billing/chargeback as kWh.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 + }, + "repeatDirection": "h", + "options": { + "graphMode": "none", + "colorMode": "value", + "justifyMode": "auto", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "percentChangeColorMode": "standard", + "orientation": "" + }, + "fieldConfig": { + "defaults": { + "unit": "kwatth", + "decimals": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "id": 7, + "targets": [ + { + "expr": "sum(sum_over_time(namespace:gpu_count:allocated{namespace=~\"$namespace\"}[24h:1m])) / 60", + "refId": "A" + } + ], + "title": "GPU-hours (last 24h)", + "description": "Sum of (GPU count × time requested) across all tenants over the last 24h. Integrated from kube requests, so Pending pods count.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 1 + }, + "repeatDirection": "h", + "options": { + "graphMode": "none", + "colorMode": "value", + "justifyMode": "auto", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "percentChangeColorMode": "standard", + "orientation": "" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "blue" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "row", + "collapsed": false, + "title": "Current allocation", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 10, + "panels": [] + }, + { + "type": "bargauge", + "id": 11, + "targets": [ + { + "expr": "namespace:gpu_count:allocated{namespace=~\"$namespace\"}", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "title": "GPUs per namespace", + "description": "Kube GPU requests per tenant (billable). Includes Pending pods.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "repeatDirection": "h", + "options": { + "displayMode": "gradient", + "valueMode": "color", + "namePlacement": "auto", + "showUnfilled": true, + "sizing": "auto", + "minVizWidth": 8, + "minVizHeight": 16, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false, + "calcs": [] + }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "maxVizHeight": 300, + "orientation": "horizontal" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "bargauge", + "id": 12, + "targets": [ + { + "expr": "namespace:fb_used_bytes:sum{namespace=~\"$namespace\"}", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "title": "VRAM used per namespace", + "description": "FB memory actively used by tenant workloads.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "repeatDirection": "h", + "options": { + "displayMode": "gradient", + "valueMode": "color", + "namePlacement": "auto", + "showUnfilled": true, + "sizing": "auto", + "minVizWidth": 8, + "minVizHeight": 16, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false, + "calcs": [] + }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "maxVizHeight": 300, + "orientation": "horizontal" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "row", + "collapsed": false, + "title": "Utilization", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 20, + "panels": [] + }, + { + "type": "timeseries", + "id": 21, + "targets": [ + { + "expr": "namespace:gpu_util:avg{namespace=~\"$namespace\"}", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "title": "NVML utilization per namespace", + "description": "Per-tenant NVML utilization. Stacked to show fleet-wide pressure over time.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "repeatDirection": "h", + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "" + } + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 40, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "id": 22, + "targets": [ + { + "expr": "namespace:tensor_active:avg{namespace=~\"$namespace\"} * 100", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "title": "Tensor saturation per namespace", + "description": "Real tensor core load per tenant. Useful to find AI tenants that claim GPUs but leave tensor cores idle.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "repeatDirection": "h", + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "" + } + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 40, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + } + }, + { + "type": "row", + "collapsed": false, + "title": "Power \u0026 allocation trend", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 30, + "panels": [] + }, + { + "type": "timeseries", + "id": 31, + "targets": [ + { + "expr": "namespace:power_watts:sum{namespace=~\"$namespace\"}", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "title": "Power draw per namespace", + "description": "Per-tenant power draw. Stacked to see cluster-wide energy burn over time.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "repeatDirection": "h", + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "" + } + }, + "fieldConfig": { + "defaults": { + "unit": "watt", + "min": 0, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 40, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "id": 32, + "targets": [ + { + "expr": "namespace:gpu_count:allocated{namespace=~\"$namespace\"}", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "title": "Allocated GPUs per namespace", + "description": "Kube-requested GPU count per tenant over time. Step-after interpolation reflects discrete allocation changes.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "repeatDirection": "h", + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "custom": { + "drawStyle": "line", + "lineInterpolation": "stepAfter", + "fillOpacity": 40, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + } + }, + { + "type": "row", + "collapsed": false, + "title": "Top consumers (live)", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 40, + "panels": [] + }, + { + "type": "table", + "id": 41, + "targets": [ + { + "expr": "namespace:gpu_count:allocated{namespace=~\"$namespace\"}", + "instant": true, + "range": false, + "format": "table", + "refId": "gpus" + }, + { + "expr": "namespace:gpu_util:avg{namespace=~\"$namespace\"}", + "instant": true, + "range": false, + "format": "table", + "refId": "util" + }, + { + "expr": "namespace:tensor_active:avg{namespace=~\"$namespace\"} * 100", + "instant": true, + "range": false, + "format": "table", + "refId": "tensor" + }, + { + "expr": "namespace:power_watts:sum{namespace=~\"$namespace\"}", + "instant": true, + "range": false, + "format": "table", + "refId": "power" + }, + { + "expr": "namespace:fb_used_bytes:sum{namespace=~\"$namespace\"}", + "instant": true, + "range": false, + "format": "table", + "refId": "vram" + } + ], + "title": "Top consumers", + "description": "Live ranking of tenants by power draw. Columns joined from namespace recording rules.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 33 + }, + "repeatDirection": "h", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "namespace", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Time 5": true, + "__name__": true, + "__name__ 1": true, + "__name__ 2": true, + "__name__ 3": true, + "__name__ 4": true, + "__name__ 5": true, + "cluster": true, + "cluster 1": true, + "cluster 2": true, + "cluster 3": true, + "cluster 4": true, + "cluster 5": true, + "endpoint": true, + "endpoint 1": true, + "endpoint 2": true, + "endpoint 3": true, + "endpoint 4": true, + "endpoint 5": true, + "instance": true, + "instance 1": true, + "instance 2": true, + "instance 3": true, + "instance 4": true, + "instance 5": true, + "job": true, + "job 1": true, + "job 2": true, + "job 3": true, + "job 4": true, + "job 5": true, + "prometheus": true, + "prometheus 1": true, + "prometheus 2": true, + "prometheus 3": true, + "prometheus 4": true, + "prometheus 5": true, + "service": true, + "service 1": true, + "service 2": true, + "service 3": true, + "service 4": true, + "service 5": true, + "tenant": true, + "tenant 1": true, + "tenant 2": true, + "tenant 3": true, + "tenant 4": true, + "tenant 5": true, + "tier": true, + "tier 1": true, + "tier 2": true, + "tier 3": true, + "tier 4": true, + "tier 5": true, + "uid": true, + "uid 1": true, + "uid 2": true, + "uid 3": true, + "uid 4": true, + "uid 5": true, + "unit": true, + "unit 1": true, + "unit 2": true, + "unit 3": true, + "unit 4": true, + "unit 5": true + }, + "indexByName": { + "Value #gpus": 1, + "Value #power": 4, + "Value #tensor": 3, + "Value #util": 2, + "Value #vram": 5, + "namespace": 0 + }, + "renameByName": { + "Value #gpus": "GPUs", + "Value #power": "Power", + "Value #tensor": "Tensor %", + "Value #util": "Util %", + "Value #vram": "VRAM", + "namespace": "Namespace" + } + } + } + ], + "options": { + "frameIndex": 0, + "showHeader": true, + "showTypeIcons": false, + "sortBy": [ + { + "displayName": "Power", + "desc": true + } + ], + "footer": { + "show": false, + "reducer": [] + }, + "cellHeight": "sm" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Util %" + }, + "properties": [ + { + "id": "unit", + "value": "percent" + }, + { + "id": "min", + "value": 0 + }, + { + "id": "max", + "value": 100 + }, + { + "id": "decimals", + "value": 1 + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Tensor %" + }, + "properties": [ + { + "id": "unit", + "value": "percent" + }, + { + "id": "min", + "value": 0 + }, + { + "id": "max", + "value": 100 + }, + { + "id": "decimals", + "value": 1 + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Power" + }, + "properties": [ + { + "id": "unit", + "value": "watt" + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VRAM" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + } + }, + { + "type": "row", + "collapsed": false, + "title": "Historical usage (last 24h)", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 43 + }, + "id": 50, + "panels": [] + }, + { + "type": "bargauge", + "id": 51, + "targets": [ + { + "expr": "sum_over_time(namespace:gpu_count:allocated{namespace=~\"$namespace\"}[24h:1m]) / 60", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "title": "GPU-hours per namespace (last 24h)", + "description": "Billable GPU-hours per tenant over the last 24h window.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 44 + }, + "repeatDirection": "h", + "options": { + "displayMode": "gradient", + "valueMode": "color", + "namePlacement": "auto", + "showUnfilled": true, + "sizing": "auto", + "minVizWidth": 8, + "minVizHeight": 16, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false, + "calcs": [] + }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "maxVizHeight": 300, + "orientation": "horizontal" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "blue" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "bargauge", + "id": 52, + "targets": [ + { + "expr": "sum_over_time(namespace:power_watts:sum{namespace=~\"$namespace\"}[24h:1m]) / 60 / 1000", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "title": "Energy per namespace (last 24h)", + "description": "kWh consumed per tenant over the last 24h. Integrated from 1-minute power samples.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 44 + }, + "repeatDirection": "h", + "options": { + "displayMode": "gradient", + "valueMode": "color", + "namePlacement": "auto", + "showUnfilled": true, + "sizing": "auto", + "minVizWidth": 8, + "minVizHeight": 16, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false, + "calcs": [] + }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "maxVizHeight": 300, + "orientation": "horizontal" + }, + "fieldConfig": { + "defaults": { + "unit": "kwatth", + "decimals": 2, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + } + ] + } + }, + "overrides": [] + } + } + ], + "templating": { + "list": [ + { + "type": "datasource", + "name": "ds_prometheus", + "label": "Prometheus", + "skipUrlSync": false, + "query": "prometheus", + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "multi": false, + "allowCustomValue": true, + "includeAll": false, + "regex": "", + "auto": false, + "auto_min": "10s", + "auto_count": 30 + }, + { + "type": "query", + "name": "namespace", + "label": "Namespace", + "skipUrlSync": false, + "query": "label_values(namespace:gpu_count:allocated, namespace)", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "multi": true, + "allowCustomValue": true, + "refresh": 2, + "sort": 1, + "includeAll": true, + "auto": false, + "auto_min": "10s", + "auto_count": 30 + } + ] + }, + "annotations": {} +} diff --git a/packages/system/gpu-operator/examples/README.md b/packages/system/gpu-operator/examples/README.md index 11c2bb09..5374bffc 100644 --- a/packages/system/gpu-operator/examples/README.md +++ b/packages/system/gpu-operator/examples/README.md @@ -21,16 +21,10 @@ Talos. ConfigMap. - [`dcgm-custom-metrics.yaml`](./dcgm-custom-metrics.yaml) — `ConfigMap` with a DCGM metrics CSV that adds profiling, ECC, throttling and - energy counters on top of the upstream defaults. The CSV is the - superset needed for full dashboard coverage; the **recording rules - themselves** only require the profiling subset - (`DCGM_FI_PROF_PIPE_TENSOR_ACTIVE`, `DCGM_FI_PROF_GR_ENGINE_ACTIVE`) - on top of the upstream `default-counters.csv` — every other DCGM - series the rules consume (utilization, FB used/free, power, - temperature, energy) is already in the default set. The - `gpu/gpu-performance` dashboard additionally needs the throttle - counters (`DCGM_FI_DEV_POWER_VIOLATION`, - `DCGM_FI_DEV_THERMAL_VIOLATION`), which are not in the default set. + energy counters on top of the upstream defaults. The CSV is a + superset needed for full coverage of the `gpu/gpu-performance` + dashboard. Which parts are actually required depends on which + dashboards you ship — see the table below. - [`nvidia-driver-compat.yaml`](./nvidia-driver-compat.yaml) — DaemonSet that stages `libnvidia-ml.so.1` and `nvidia-smi` from the Talos glibc tree into a path where the NVIDIA GPU Operator validator expects @@ -62,32 +56,46 @@ files into a directory the validator does inspect and creates the [1]: https://github.com/NVIDIA/gpu-operator/issues/1687 +## Dashboards and what DCGM metrics they need + +Five GPU dashboards live under `gpu/*` in +`packages/system/monitoring/dashboards-infra.list`. All of them share +`packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml` as +their source of aggregated series. The recording rules are safe to +ship on any cluster — they evaluate to empty series when DCGM is not +scraped, or when optional counters are missing. + +What each dashboard needs on top of the upstream DCGM Exporter +[`default-counters.csv`][default-csv]: + +| Dashboard | Scope | Needs beyond defaults | +| ----------------- | ---------------------------------- | -------------------------------------------------------------------------------------------- | +| `gpu-performance` | Per-node, per-GPU deep dive | `DCGM_FI_PROF_PIPE_TENSOR_ACTIVE`, `DCGM_FI_PROF_GR_ENGINE_ACTIVE`, `DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION` | +| `gpu-efficiency` | Per-workload util vs tensor active | `DCGM_FI_PROF_PIPE_TENSOR_ACTIVE` | +| `gpu-fleet` | Cluster-wide admin inventory | nothing (works on default counters) | +| `gpu-quotas` | Kube-quota vs live usage | nothing (kube-state-metrics + default counters) | +| `gpu-tenants` | Per-namespace tenant view | `DCGM_FI_PROF_PIPE_TENSOR_ACTIVE` for the tensor-saturation panel; other panels work on defaults | + +The throttling counters (`DCGM_FI_DEV_POWER_VIOLATION`, +`DCGM_FI_DEV_THERMAL_VIOLATION`) are only required by `gpu-performance`. +The profiling counters (`DCGM_FI_PROF_*`) are required by +`gpu-performance` and `gpu-efficiency`, and unlock the tensor panel in +`gpu-tenants`. Everything else the recording rules consume — +utilization, FB used/free, power, temperature, energy — is already in +the default counter set. + ## Verification status > **Pending verification on an updated GPU Operator release.** > -> The minimum-CSV claim above (only `DCGM_FI_PROF_*` is needed beyond -> the upstream default counters) is derived by cross-referencing -> `gpu-recording.rules.yaml` against the DCGM Exporter -> [`default-counters.csv`][default-csv] for the version pinned in the -> currently shipped `gpu-operator` package. The package in this branch -> is **not** the latest GPU Operator release; once we move to a newer -> version, the claim must be re-checked because the upstream default -> set occasionally adds or removes counters between releases. Until -> then, treat the CSV in `dcgm-custom-metrics.yaml` as a known-good -> superset rather than a minimal config. +> The minimum-CSV claims above are derived by cross-referencing +> `gpu-recording.rules.yaml` and each dashboard against the DCGM +> Exporter [`default-counters.csv`][default-csv] for the version pinned +> in the currently shipped `gpu-operator` package. The package in this +> branch is **not** the latest GPU Operator release; once we move to a +> newer version, the claims must be re-checked because the upstream +> default set occasionally adds or removes counters between releases. +> Until then, treat the CSV in `dcgm-custom-metrics.yaml` as a +> known-good superset rather than a minimal config. [default-csv]: https://github.com/NVIDIA/dcgm-exporter/blob/main/etc/default-counters.csv - -## How the dashboard and recording rules fit in - -- `dashboards/gpu/gpu-performance.json` expects `DCGM_FI_*` metrics, - including profiling series (`DCGM_FI_PROF_PIPE_TENSOR_ACTIVE`, - `DCGM_FI_PROF_GR_ENGINE_ACTIVE`) and throttling counters - (`DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION`). - These are only emitted when DCGM Exporter is started with the custom - CSV in `dcgm-custom-metrics.yaml`. -- `packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml` - precomputes cluster-wide and per-namespace aggregations used by the - overview panels of the dashboard. The rules are safe to ship on any - cluster — they evaluate to empty series when DCGM is not scraped. diff --git a/packages/system/monitoring/dashboards-infra.list b/packages/system/monitoring/dashboards-infra.list index b402fa68..4c7e494d 100644 --- a/packages/system/monitoring/dashboards-infra.list +++ b/packages/system/monitoring/dashboards-infra.list @@ -35,3 +35,5 @@ hubble/network-overview gpu/gpu-performance gpu/gpu-efficiency gpu/gpu-quotas +gpu/gpu-fleet +gpu/gpu-tenants From 7eb9fe8ade53e450962e71548129d776c8fa7e6d Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 06:41:45 +0300 Subject: [PATCH 014/130] refactor(monitoring): drop unused GPU recording rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - namespace:gpu_count:sum — never consumed by any tracked dashboard; the billable view is already covered by namespace:gpu_count:allocated, and the admin view by cluster:gpu_count:allocated. - namespace:energy_joules:sum — no panel integrates joules; kWh readings on the tenant dashboard compute their own integrations from namespace:power_watts:sum. - pod:tensor_to_nvml_ratio:avg5m — interesting tenant signal in theory, but not wired into any panel and carrying it just burns cardinality on large fleets. Signed-off-by: Arsolitt --- .../alerts/gpu-recording.rules.yaml | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml index 3a31f5a2..28d6a45e 100644 --- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml +++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml @@ -25,11 +25,6 @@ spec: - name: gpu.recording.namespace.1m interval: 1m rules: - # DCGM-visible GPU count per namespace — counts GPUs that are actually - # running a tenant pod right now (driver loaded, scheduler placed it). - # Differs from :allocated when pods are Pending or stuck. - - record: namespace:gpu_count:sum - expr: count by (namespace) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}) # Kube-requested GPU count per namespace — billable view, includes # Pending pods. Use this for GPU-hour reporting via # sum_over_time(...[1h:1m])/60. @@ -62,8 +57,6 @@ spec: expr: sum by (namespace) (DCGM_FI_DEV_FB_USED{namespace!="", namespace!~"cozy-.*|kube-.*"}) * 1048576 - record: namespace:power_watts:sum expr: sum by (namespace) (DCGM_FI_DEV_POWER_USAGE{namespace!="", namespace!~"cozy-.*|kube-.*"}) - - record: namespace:energy_joules:sum - expr: sum by (namespace) (DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION{namespace!="", namespace!~"cozy-.*|kube-.*"}) / 1000 - name: gpu.recording.efficiency.1m interval: 1m @@ -76,19 +69,6 @@ spec: avg_over_time(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]) ) * 100 - # NVML vs tensor gap — ratio <10% means NVML lies: user thinks GPU - # is busy but specialized hardware is idle (cheap tenant signal). - - record: pod:tensor_to_nvml_ratio:avg5m - expr: | - ( - avg_over_time(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]) * 100 - ) - / - clamp_min( - avg_over_time(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]), - 1 - ) - # Power efficiency — utilization per watt, reveals unoptimized clients. - record: pod:util_per_watt:avg5m expr: | From 2fa4b3e31cd9777bdc88743272b064f076429c87 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 06:48:33 +0300 Subject: [PATCH 015/130] refactor(monitoring): tighten GPU dashboard queries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - gpu-efficiency: scope Tensor Saturation, Util-per-Watt and Power Throttle stats to the $namespace selector. Cluster-wide means were misleading when a user had narrowed the dashboard to specific tenants — the headline numbers lied relative to the panels below. - gpu-fleet: show per-node power draw as % of combined TDP cap (DCGM_FI_DEV_POWER_MGMT_LIMIT) instead of raw watts. Thresholds (60 / 80 %) generalize across GPU SKUs without per-model tuning. - gpu-quotas: read cluster:gpu_count:allocated from the recording rules instead of recomputing sum(kube_pod_container_resource_requests) inline. Keeps the dashboard aligned with the canonical definition in gpu-recording.rules.yaml so the two can't drift. Signed-off-by: Arsolitt --- dashboards/gpu/gpu-efficiency.json | 12 ++++++------ dashboards/gpu/gpu-fleet.json | 14 +++++++------- dashboards/gpu/gpu-quotas.json | 4 ++-- .../alerts/gpu-recording.rules.yaml | 1 + 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/dashboards/gpu/gpu-efficiency.json b/dashboards/gpu/gpu-efficiency.json index 596961e0..07000f71 100644 --- a/dashboards/gpu/gpu-efficiency.json +++ b/dashboards/gpu/gpu-efficiency.json @@ -35,12 +35,12 @@ "id": 2, "targets": [ { - "expr": "avg(pod:tensor_saturation:avg5m)", + "expr": "avg(pod:tensor_saturation:avg5m{namespace=~\"$namespace\"})", "refId": "A" } ], - "title": "Cluster Tensor Saturation", - "description": "Mean tensor core saturation. \u003c10% means GPUs are used inefficiently (tenants could move to CPU or optimize their code).", + "title": "Avg Tensor Saturation", + "description": "Mean tensor core saturation across selected namespaces. \u003c10% means GPUs are used inefficiently (tenants could move to CPU or optimize their code).", "transparent": false, "datasource": { "type": "prometheus", @@ -103,12 +103,12 @@ "id": 3, "targets": [ { - "expr": "avg(pod:util_per_watt:avg5m)", + "expr": "avg(pod:util_per_watt:avg5m{namespace=~\"$namespace\"})", "refId": "A" } ], "title": "Avg Utilization per Watt", - "description": "NVML utilization % per watt. Higher value = more efficient workload.", + "description": "NVML utilization % per watt across selected namespaces. Higher value = more efficient workload.", "transparent": false, "datasource": { "type": "prometheus", @@ -166,7 +166,7 @@ "id": 4, "targets": [ { - "expr": "avg(gpu:power_throttle_fraction:rate5m)", + "expr": "avg(gpu:power_throttle_fraction:rate5m{namespace=~\"$namespace\"})", "refId": "A" } ], diff --git a/dashboards/gpu/gpu-fleet.json b/dashboards/gpu/gpu-fleet.json index af5ad2fa..4cd5b4b5 100644 --- a/dashboards/gpu/gpu-fleet.json +++ b/dashboards/gpu/gpu-fleet.json @@ -467,13 +467,13 @@ "id": 12, "targets": [ { - "expr": "sum by (Hostname) (DCGM_FI_DEV_POWER_USAGE)", + "expr": "sum by (Hostname) (DCGM_FI_DEV_POWER_USAGE) / sum by (Hostname) (DCGM_FI_DEV_POWER_MGMT_LIMIT) * 100", "legendFormat": "{{Hostname}}", "refId": "A" } ], - "title": "Power draw per node", - "description": "Sum of GPU power usage per node in watts.", + "title": "Power draw per node (% of TDP)", + "description": "GPU power usage as a fraction of the node's combined TDP cap. Works across GPU SKUs without per-model thresholds.", "transparent": false, "datasource": { "type": "prometheus", @@ -510,9 +510,9 @@ }, "fieldConfig": { "defaults": { - "unit": "watt", + "unit": "percent", "min": 0, - "max": 200, + "max": 100, "thresholds": { "mode": "absolute", "steps": [ @@ -521,11 +521,11 @@ "color": "green" }, { - "value": 120, + "value": 60, "color": "yellow" }, { - "value": 160, + "value": 80, "color": "orange" } ] diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json index 60c628f6..3bc01a83 100644 --- a/dashboards/gpu/gpu-quotas.json +++ b/dashboards/gpu/gpu-quotas.json @@ -86,7 +86,7 @@ "id": 3, "targets": [ { - "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"})", + "expr": "cluster:gpu_count:allocated", "refId": "A" } ], @@ -143,7 +143,7 @@ "id": 4, "targets": [ { - "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}) / sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"}) * 100", + "expr": "cluster:gpu_count:allocated / sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"}) * 100", "refId": "A" } ], diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml index 28d6a45e..a24951f5 100644 --- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml +++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml @@ -2,6 +2,7 @@ apiVersion: operator.victoriametrics.com/v1beta1 kind: VMRule metadata: name: alerts-gpu-recording.rules + namespace: cozy-monitoring spec: groups: - name: gpu.recording.cluster.1m From ccfec2ef62684b854508294ab82e379bffcad771 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 06:55:17 +0300 Subject: [PATCH 016/130] fix(monitoring): close DCGM coverage gap for gpu-fleet TDP panel gpu-fleet.json references DCGM_FI_DEV_POWER_MGMT_LIMIT for its "TDP vs draw" panel, but the custom DCGM Exporter CSV did not declare it, so the panel silently rendered "No data" on clusters using that config. Declare the counter, fix the dashboards table in the gpu-operator examples README, and add a bats test that cross-checks every DCGM_FI_* reference in tracked dashboards and recording rules against the union of the upstream default set (snapshotted under hack/) and the project's custom CSV. Signed-off-by: Arsolitt --- hack/check-gpu-recording-rules.bats | 92 +++++++++++++++- hack/dcgm-default-counters.csv | 104 ++++++++++++++++++ .../system/gpu-operator/examples/README.md | 50 ++++----- .../examples/dcgm-custom-metrics.yaml | 1 + 4 files changed, 220 insertions(+), 27 deletions(-) create mode 100644 hack/dcgm-default-counters.csv diff --git a/hack/check-gpu-recording-rules.bats b/hack/check-gpu-recording-rules.bats index 87e7c8f1..4981959a 100644 --- a/hack/check-gpu-recording-rules.bats +++ b/hack/check-gpu-recording-rules.bats @@ -1,7 +1,7 @@ #!/usr/bin/env bats # ----------------------------------------------------------------------------- -# Cross-validation between GPU recording rules and the dashboards that consume -# them. Catches: +# Cross-validation between GPU recording rules, the dashboards that consume +# them, and the DCGM Exporter metric set the cluster actually scrapes. Catches: # # 1. dangling references — a dashboard query mentions a recording rule name # that doesn't exist in gpu-recording.rules.yaml. This is the bug the @@ -12,6 +12,13 @@ # 2. typos in rule names — same bug class, manifested as a single-character # difference between rule and reference. # +# 3. undeclared DCGM metrics — a dashboard query or recording rule mentions +# a DCGM_FI_* metric that is neither in the upstream default CSV nor in +# the project's custom CSV (dcgm-custom-metrics.yaml), meaning DCGM +# Exporter would never emit it and the panel silently shows "No data". +# Example regression: gpu-fleet.json shipped a TDP panel referencing +# DCGM_FI_DEV_POWER_MGMT_LIMIT before the custom CSV declared it. +# # Scope: only dashboards listed in packages/system/monitoring/dashboards-infra.list # under the "gpu/" prefix (i.e. shipped to production), not every JSON file in # dashboards/gpu/. Untracked drafts stay out of scope on purpose — adding one @@ -33,6 +40,8 @@ REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" RULES_FILE="$REPO_ROOT/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml" DASHBOARDS_LIST="$REPO_ROOT/packages/system/monitoring/dashboards-infra.list" DASHBOARDS_DIR="$REPO_ROOT/dashboards" +DCGM_DEFAULT_CSV="$REPO_ROOT/hack/dcgm-default-counters.csv" +DCGM_CUSTOM_CSV="$REPO_ROOT/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml" # Extract the set of "- record: NAME" entries from the rules YAML. # Outputs one rule name per line, sorted and deduplicated. @@ -61,6 +70,37 @@ list_tracked_gpu_dashboards() { awk '/^gpu\// { print $0 ".json" }' "$DASHBOARDS_LIST" } +# Extract the set of DCGM_FI_* metric names declared in a CSV file. Handles +# both the upstream-style default CSV (unindented) and the ConfigMap-style +# custom CSV (YAML-indented). A declaration line starts — after any leading +# whitespace — with "DCGM_FI_," ; comment lines begin with "#" and are +# skipped. Uses POSIX awk's match()+RSTART/RLENGTH so no GNU extensions +# are required. +extract_csv_metrics() { + file=$1 + awk ' + { + line = $0 + sub(/^[[:space:]]+/, "", line) + if (line ~ /^#/) next + if (match(line, /^DCGM_FI_[A-Z0-9_]+/)) { + print substr(line, RSTART, RLENGTH) + } + } + ' "$file" | sort -u +} + +# Extract the set of DCGM_FI_* metric references from a text file (dashboard +# JSON or rules YAML). A DCGM metric name has at least two underscore-delimited +# segments after the "DCGM_FI_" prefix (e.g. DCGM_FI_DEV_GPU_UTIL, DCGM_FI_PROF_ +# PIPE_TENSOR_ACTIVE, DCGM_FI_DRIVER_VERSION). Requiring two segments keeps +# the matcher from latching onto glob stubs like "DCGM_FI_DEV_*_VIOLATION" that +# appear in comments. +extract_dcgm_refs() { + file=$1 + grep -hoE 'DCGM_FI_[A-Z0-9]+(_[A-Z0-9]+)+' "$file" | sort -u +} + @test "every recording rule reference in tracked GPU dashboards has a matching record" { TMP=$(mktemp -d) trap 'rm -rf "$TMP"' EXIT @@ -94,6 +134,54 @@ list_tracked_gpu_dashboards() { [ "$failed" -eq 0 ] } +@test "every DCGM metric referenced in tracked dashboards and rules is declared" { + TMP=$(mktemp -d) + trap 'rm -rf "$TMP"' EXIT + + [ -f "$DCGM_DEFAULT_CSV" ] || { echo "missing $DCGM_DEFAULT_CSV" >&2; exit 1; } + [ -f "$DCGM_CUSTOM_CSV" ] || { echo "missing $DCGM_CUSTOM_CSV" >&2; exit 1; } + + { + extract_csv_metrics "$DCGM_DEFAULT_CSV" + extract_csv_metrics "$DCGM_CUSTOM_CSV" + } | sort -u > "$TMP/declared.txt" + [ -s "$TMP/declared.txt" ] || { echo "no DCGM metrics extracted from CSVs" >&2; exit 1; } + + list_tracked_gpu_dashboards > "$TMP/dashboards.txt" + [ -s "$TMP/dashboards.txt" ] || { echo "no gpu/* dashboards listed in $DASHBOARDS_LIST" >&2; exit 1; } + + failed=0 + + # Dashboard coverage — every dashboard's DCGM references must resolve. + while IFS= read -r dashboard_rel; do + dashboard="$DASHBOARDS_DIR/$dashboard_rel" + [ -f "$dashboard" ] || continue # handled by the existence test + extract_dcgm_refs "$dashboard" > "$TMP/refs.txt" + [ -s "$TMP/refs.txt" ] || continue # dashboard relies entirely on recording rules + comm -23 "$TMP/refs.txt" "$TMP/declared.txt" > "$TMP/missing.txt" + if [ -s "$TMP/missing.txt" ]; then + echo "ERROR: $dashboard_rel references DCGM metrics not declared in any CSV:" >&2 + sed 's/^/ - /' "$TMP/missing.txt" >&2 + failed=1 + fi + done < "$TMP/dashboards.txt" + + # Rules coverage — recording rules consume DCGM directly, so their set + # must be declared too, otherwise derived series on every dashboard + # collapse to empty. + extract_dcgm_refs "$RULES_FILE" > "$TMP/rule-refs.txt" + if [ -s "$TMP/rule-refs.txt" ]; then + comm -23 "$TMP/rule-refs.txt" "$TMP/declared.txt" > "$TMP/rule-missing.txt" + if [ -s "$TMP/rule-missing.txt" ]; then + echo "ERROR: gpu-recording.rules.yaml references DCGM metrics not declared in any CSV:" >&2 + sed 's/^/ - /' "$TMP/rule-missing.txt" >&2 + failed=1 + fi + fi + + [ "$failed" -eq 0 ] +} + @test "every tracked GPU dashboard listed in dashboards-infra.list exists on disk" { TMP=$(mktemp -d) trap 'rm -rf "$TMP"' EXIT diff --git a/hack/dcgm-default-counters.csv b/hack/dcgm-default-counters.csv new file mode 100644 index 00000000..b5e94540 --- /dev/null +++ b/hack/dcgm-default-counters.csv @@ -0,0 +1,104 @@ +# Snapshot of the upstream DCGM Exporter default-counters.csv used as a +# fixture by hack/check-gpu-recording-rules.bats. The test verifies that +# every DCGM_FI_* metric referenced by a tracked GPU dashboard is either +# declared here (upstream defaults) or in +# packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml +# (the project's custom CSV). +# +# Source: https://github.com/NVIDIA/dcgm-exporter/blob/4.1.1-4.0.4/etc/default-counters.csv +# Pinned to the DCGM Exporter image tag shipped by the gpu-operator +# chart under packages/system/gpu-operator/charts/gpu-operator/values.yaml +# (dcgmExporter.version = 4.1.1-4.0.4-ubuntu22.04). When that image is +# bumped, refresh this file from the matching tag in the NVIDIA/dcgm-exporter +# repo and re-run `./hack/cozytest.sh hack/check-gpu-recording-rules.bats`. + +# Format +# If line starts with a '#' it is considered a comment +# DCGM FIELD, Prometheus metric type, help message + +# Clocks +DCGM_FI_DEV_SM_CLOCK, gauge, SM clock frequency (in MHz). +DCGM_FI_DEV_MEM_CLOCK, gauge, Memory clock frequency (in MHz). + +# Temperature +DCGM_FI_DEV_MEMORY_TEMP, gauge, Memory temperature (in C). +DCGM_FI_DEV_GPU_TEMP, gauge, GPU temperature (in C). + +# Power +DCGM_FI_DEV_POWER_USAGE, gauge, Power draw (in W). +DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION, counter, Total energy consumption since boot (in mJ). + +# PCIE +# DCGM_FI_PROF_PCIE_TX_BYTES, counter, Total number of bytes transmitted through PCIe TX via NVML. +# DCGM_FI_PROF_PCIE_RX_BYTES, counter, Total number of bytes received through PCIe RX via NVML. +DCGM_FI_DEV_PCIE_REPLAY_COUNTER, counter, Total number of PCIe retries. + +# Utilization (the sample period varies depending on the product) +DCGM_FI_DEV_GPU_UTIL, gauge, GPU utilization (in %). +DCGM_FI_DEV_MEM_COPY_UTIL, gauge, Memory utilization (in %). +DCGM_FI_DEV_ENC_UTIL, gauge, Encoder utilization (in %). +DCGM_FI_DEV_DEC_UTIL , gauge, Decoder utilization (in %). + +# Errors and violations +DCGM_FI_DEV_XID_ERRORS, gauge, Value of the last XID error encountered. +# DCGM_FI_DEV_POWER_VIOLATION, counter, Throttling duration due to power constraints (in us). +# DCGM_FI_DEV_THERMAL_VIOLATION, counter, Throttling duration due to thermal constraints (in us). +# DCGM_FI_DEV_SYNC_BOOST_VIOLATION, counter, Throttling duration due to sync-boost constraints (in us). +# DCGM_FI_DEV_BOARD_LIMIT_VIOLATION, counter, Throttling duration due to board limit constraints (in us). +# DCGM_FI_DEV_LOW_UTIL_VIOLATION, counter, Throttling duration due to low utilization (in us). +# DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (in us). + +# Memory usage +DCGM_FI_DEV_FB_FREE, gauge, Framebuffer memory free (in MiB). +DCGM_FI_DEV_FB_USED, gauge, Framebuffer memory used (in MiB). + +# ECC +# DCGM_FI_DEV_ECC_SBE_VOL_TOTAL, counter, Total number of single-bit volatile ECC errors. +# DCGM_FI_DEV_ECC_DBE_VOL_TOTAL, counter, Total number of double-bit volatile ECC errors. +# DCGM_FI_DEV_ECC_SBE_AGG_TOTAL, counter, Total number of single-bit persistent ECC errors. +# DCGM_FI_DEV_ECC_DBE_AGG_TOTAL, counter, Total number of double-bit persistent ECC errors. + +# Retired pages +# DCGM_FI_DEV_RETIRED_SBE, counter, Total number of retired pages due to single-bit errors. +# DCGM_FI_DEV_RETIRED_DBE, counter, Total number of retired pages due to double-bit errors. +# DCGM_FI_DEV_RETIRED_PENDING, counter, Total number of pages pending retirement. + +# NVLink +# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_TOTAL, counter, Total number of NVLink flow-control CRC errors. +# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_TOTAL, counter, Total number of NVLink data CRC errors. +# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_TOTAL, counter, Total number of NVLink retries. +# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_TOTAL, counter, Total number of NVLink recovery errors. +DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL, counter, Total number of NVLink bandwidth counters for all lanes. +# DCGM_FI_DEV_NVLINK_BANDWIDTH_L0, counter, The number of bytes of active NVLink rx or tx data including both header and payload. + +# VGPU License status +DCGM_FI_DEV_VGPU_LICENSE_STATUS, gauge, vGPU License status + +# Remapped rows +DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for uncorrectable errors +DCGM_FI_DEV_CORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for correctable errors +DCGM_FI_DEV_ROW_REMAP_FAILURE, gauge, Whether remapping of rows has failed + +# Static configuration information. These appear as labels on the other metrics +DCGM_FI_DRIVER_VERSION, label, Driver Version +# DCGM_FI_NVML_VERSION, label, NVML Version +# DCGM_FI_DEV_BRAND, label, Device Brand +# DCGM_FI_DEV_SERIAL, label, Device Serial Number +# DCGM_FI_DEV_OEM_INFOROM_VER, label, OEM inforom version +# DCGM_FI_DEV_ECC_INFOROM_VER, label, ECC inforom version +# DCGM_FI_DEV_POWER_INFOROM_VER, label, Power management object inforom version +# DCGM_FI_DEV_INFOROM_IMAGE_VER, label, Inforom image version +# DCGM_FI_DEV_VBIOS_VERSION, label, VBIOS version of the device + +# Datacenter Profiling (DCP) metrics +# NOTE: supported on Nvidia datacenter Volta GPUs and newer +DCGM_FI_PROF_GR_ENGINE_ACTIVE, gauge, Ratio of time the graphics engine is active. +# DCGM_FI_PROF_SM_ACTIVE, gauge, The ratio of cycles an SM has at least 1 warp assigned. +# DCGM_FI_PROF_SM_OCCUPANCY, gauge, The ratio of number of warps resident on an SM. +DCGM_FI_PROF_PIPE_TENSOR_ACTIVE, gauge, Ratio of cycles the tensor (HMMA) pipe is active. +DCGM_FI_PROF_DRAM_ACTIVE, gauge, Ratio of cycles the device memory interface is active sending or receiving data. +# DCGM_FI_PROF_PIPE_FP64_ACTIVE, gauge, Ratio of cycles the fp64 pipes are active. +# DCGM_FI_PROF_PIPE_FP32_ACTIVE, gauge, Ratio of cycles the fp32 pipes are active. +# DCGM_FI_PROF_PIPE_FP16_ACTIVE, gauge, Ratio of cycles the fp16 pipes are active. +DCGM_FI_PROF_PCIE_TX_BYTES, gauge, The rate of data transmitted over the PCIe bus - including both protocol headers and data payloads - in bytes per second. +DCGM_FI_PROF_PCIE_RX_BYTES, gauge, The rate of data received over the PCIe bus - including both protocol headers and data payloads - in bytes per second. diff --git a/packages/system/gpu-operator/examples/README.md b/packages/system/gpu-operator/examples/README.md index 5374bffc..9599acb7 100644 --- a/packages/system/gpu-operator/examples/README.md +++ b/packages/system/gpu-operator/examples/README.md @@ -68,34 +68,34 @@ scraped, or when optional counters are missing. What each dashboard needs on top of the upstream DCGM Exporter [`default-counters.csv`][default-csv]: -| Dashboard | Scope | Needs beyond defaults | -| ----------------- | ---------------------------------- | -------------------------------------------------------------------------------------------- | -| `gpu-performance` | Per-node, per-GPU deep dive | `DCGM_FI_PROF_PIPE_TENSOR_ACTIVE`, `DCGM_FI_PROF_GR_ENGINE_ACTIVE`, `DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION` | -| `gpu-efficiency` | Per-workload util vs tensor active | `DCGM_FI_PROF_PIPE_TENSOR_ACTIVE` | -| `gpu-fleet` | Cluster-wide admin inventory | nothing (works on default counters) | -| `gpu-quotas` | Kube-quota vs live usage | nothing (kube-state-metrics + default counters) | -| `gpu-tenants` | Per-namespace tenant view | `DCGM_FI_PROF_PIPE_TENSOR_ACTIVE` for the tensor-saturation panel; other panels work on defaults | +| Dashboard | Scope | Needs beyond defaults | +| ----------------- | ---------------------------------- | ----------------------------------------------------------------------- | +| `gpu-performance` | Per-node, per-GPU deep dive | `DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION` | +| `gpu-efficiency` | Per-workload util vs tensor active | `DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION` (via `gpu:*_throttle_fraction:rate5m` recording rules) | +| `gpu-fleet` | Cluster-wide admin inventory | `DCGM_FI_DEV_POWER_MGMT_LIMIT` (for the TDP vs draw panel) | +| `gpu-quotas` | Kube-quota vs live usage | nothing (kube-state-metrics + default counters) | +| `gpu-tenants` | Per-namespace tenant view | nothing (works on default counters) | -The throttling counters (`DCGM_FI_DEV_POWER_VIOLATION`, -`DCGM_FI_DEV_THERMAL_VIOLATION`) are only required by `gpu-performance`. -The profiling counters (`DCGM_FI_PROF_*`) are required by -`gpu-performance` and `gpu-efficiency`, and unlock the tensor panel in -`gpu-tenants`. Everything else the recording rules consume — -utilization, FB used/free, power, temperature, energy — is already in -the default counter set. +`DCGM_FI_PROF_PIPE_TENSOR_ACTIVE` and `DCGM_FI_PROF_GR_ENGINE_ACTIVE` +are already in the upstream default set for the pinned DCGM Exporter +version, so the tensor-saturation and engine-active panels work without +any CSV override. The three counters listed in the table — throttling +violations and the power management limit — are the only extras the +tracked dashboards need. The recording rules in +`gpu-recording.rules.yaml` consume utilization, FB used, power, +temperature and the tensor-active profiling counter, all of which are +in the default set. ## Verification status -> **Pending verification on an updated GPU Operator release.** -> -> The minimum-CSV claims above are derived by cross-referencing -> `gpu-recording.rules.yaml` and each dashboard against the DCGM -> Exporter [`default-counters.csv`][default-csv] for the version pinned -> in the currently shipped `gpu-operator` package. The package in this -> branch is **not** the latest GPU Operator release; once we move to a -> newer version, the claims must be re-checked because the upstream -> default set occasionally adds or removes counters between releases. -> Until then, treat the CSV in `dcgm-custom-metrics.yaml` as a -> known-good superset rather than a minimal config. +The minimum-CSV claims above are verified by +`hack/check-gpu-recording-rules.bats`, which cross-checks every +`DCGM_FI_*` reference in the tracked GPU dashboards and recording rules +against the union of the upstream default set (snapshotted at +`hack/dcgm-default-counters.csv` for the pinned DCGM Exporter version) +and the custom CSV in `dcgm-custom-metrics.yaml`. When the DCGM +Exporter image in `packages/system/gpu-operator/charts/gpu-operator/values.yaml` +is bumped, refresh the snapshot from the matching tag of the +[`NVIDIA/dcgm-exporter`][default-csv] repository and rerun the test. [default-csv]: https://github.com/NVIDIA/dcgm-exporter/blob/main/etc/default-counters.csv diff --git a/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml b/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml index 14b89b5c..92064701 100644 --- a/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml +++ b/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml @@ -26,6 +26,7 @@ data: # Power DCGM_FI_DEV_POWER_USAGE, gauge, Power draw (in W). + DCGM_FI_DEV_POWER_MGMT_LIMIT, gauge, Current power management limit (in W). DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION, counter, Total energy consumption since boot (in mJ). # PCIE From 2518e09d6775ae9916c52dba16b52d88a0e7beaa Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 07:03:42 +0300 Subject: [PATCH 017/130] refactor(monitoring): store pod:tensor_saturation as unitless ratio Align pod:tensor_saturation:avg5m with namespace:tensor_active:avg and DCGM's native 0..1 range by dropping the * 100 from the recording rule and multiplying at display time in gpu-efficiency.json. Also scope pod:util_per_watt:avg5m with avg by (Hostname, gpu, UUID, namespace, pod) so the series mirrors pod:tensor_saturation's grouping and stays usable in topk queries. Signed-off-by: Arsolitt --- dashboards/gpu/gpu-efficiency.json | 4 ++-- .../alerts/gpu-recording.rules.yaml | 16 ++++++++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/dashboards/gpu/gpu-efficiency.json b/dashboards/gpu/gpu-efficiency.json index 07000f71..dddb2001 100644 --- a/dashboards/gpu/gpu-efficiency.json +++ b/dashboards/gpu/gpu-efficiency.json @@ -35,7 +35,7 @@ "id": 2, "targets": [ { - "expr": "avg(pod:tensor_saturation:avg5m{namespace=~\"$namespace\"})", + "expr": "avg(pod:tensor_saturation:avg5m{namespace=~\"$namespace\"}) * 100", "refId": "A" } ], @@ -363,7 +363,7 @@ "id": 21, "targets": [ { - "expr": "topk(20, pod:tensor_saturation:avg5m)", + "expr": "topk(20, pod:tensor_saturation:avg5m * 100)", "instant": true, "range": false, "format": "table", diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml index a24951f5..a6ecb20a 100644 --- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml +++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml @@ -64,20 +64,24 @@ spec: rules: # Tensor hardware saturation — the honest "am I using the GPU" metric # for AI/LLM workloads. Unlike NVML, idle tensor cores are visible. + # Stored as a 0..1 ratio to stay consistent with namespace:tensor_active:avg + # and DCGM's native units. Consumers multiply by 100 at display time. - record: pod:tensor_saturation:avg5m expr: | avg by (Hostname, gpu, UUID, namespace, pod) ( avg_over_time(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]) - ) * 100 + ) # Power efficiency — utilization per watt, reveals unoptimized clients. - record: pod:util_per_watt:avg5m expr: | - avg_over_time(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]) - / - clamp_min( - avg_over_time(DCGM_FI_DEV_POWER_USAGE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]), - 1 + avg by (Hostname, gpu, UUID, namespace, pod) ( + avg_over_time(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]) + / + clamp_min( + avg_over_time(DCGM_FI_DEV_POWER_USAGE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]), + 1 + ) ) # Fraction of time power-throttled (TDP cap) — 1.0 = fully throttled. From 5b210ac7fd29e15804fe787f6de8f29455eeeab4 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 07:11:28 +0300 Subject: [PATCH 018/130] docs(monitoring): mark gpu-fleet average utilization as legacy NVML Clarify that the "Average utilization" panel on gpu-fleet reflects the legacy NVML view (DCGM_FI_DEV_GPU_UTIL) rather than engine-active profiling. For AI/LLM workloads the NVML number is optimistic; the gpu-efficiency dashboard carries the profiling-based view. Signed-off-by: Arsolitt --- dashboards/gpu/gpu-fleet.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboards/gpu/gpu-fleet.json b/dashboards/gpu/gpu-fleet.json index 4cd5b4b5..ea169433 100644 --- a/dashboards/gpu/gpu-fleet.json +++ b/dashboards/gpu/gpu-fleet.json @@ -268,7 +268,7 @@ } ], "title": "Average utilization", - "description": "NVML utilization across all tenant GPUs (engine-active %).", + "description": "Legacy NVML utilization across all tenant GPUs.", "transparent": false, "datasource": { "type": "prometheus", From 4e37f64553728bab5b59ae7d7e8a50ee91c23232 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 07:19:05 +0300 Subject: [PATCH 019/130] docs(gpu-operator): document tolerate-all on compat DaemonSet Explain why tolerations: [{operator: Exists}] is safe on the driver compat DaemonSet: the nodeSelector already confines scheduling to GPU nodes, so the blanket toleration only kicks in when those nodes carry the dedicated=gpu / nvidia.com/gpu taints that the GPU Operator's default policy and many deployments apply. Signed-off-by: Arsolitt --- .../system/gpu-operator/examples/nvidia-driver-compat.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml b/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml index 63e07c4f..bad8609e 100644 --- a/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml +++ b/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml @@ -35,6 +35,13 @@ spec: # cluster uses to mark GPU hosts. nodeSelector: nvidia.com/gpu.present: "true" + # Tolerate-all is intentionally broad: the nodeSelector above already + # confines scheduling to GPU nodes, so the "Exists" toleration only + # matters when those nodes carry extra taints (dedicated=gpu:NoSchedule, + # nvidia.com/gpu:NoSchedule from the GPU Operator's default policy, + # etc.). This mirrors the stance of the upstream nvidia-driver-daemonset + # and nvidia-device-plugin DaemonSets — blanket tolerations plus a + # narrow nodeSelector. tolerations: - operator: Exists initContainers: From 5db6ec3e1fc8b2d30d3aa017f0ff7bc4dd943000 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 07:27:33 +0300 Subject: [PATCH 020/130] fix(dashboard): scope GPU panels to selected namespace Pod-level panels on the efficiency dashboard and DCGM-level panels on the performance dashboard ignored the $namespace template variable, so changing it left the visualizations unchanged. Add the filter to each query. Performance-side queries use the `$namespace|` empty-tolerant form so host-level DCGM series without a namespace label remain visible when a specific namespace is selected. Signed-off-by: Arsolitt --- dashboards/gpu/gpu-efficiency.json | 4 ++-- dashboards/gpu/gpu-performance.json | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dashboards/gpu/gpu-efficiency.json b/dashboards/gpu/gpu-efficiency.json index dddb2001..e2e8c105 100644 --- a/dashboards/gpu/gpu-efficiency.json +++ b/dashboards/gpu/gpu-efficiency.json @@ -363,7 +363,7 @@ "id": 21, "targets": [ { - "expr": "topk(20, pod:tensor_saturation:avg5m * 100)", + "expr": "topk(20, pod:tensor_saturation:avg5m{namespace=~\"$namespace\"} * 100)", "instant": true, "range": false, "format": "table", @@ -501,7 +501,7 @@ "id": 22, "targets": [ { - "expr": "topk(20, pod:util_per_watt:avg5m)", + "expr": "topk(20, pod:util_per_watt:avg5m{namespace=~\"$namespace\"})", "instant": true, "range": false, "format": "table", diff --git a/dashboards/gpu/gpu-performance.json b/dashboards/gpu/gpu-performance.json index b0f26958..e16f8c47 100644 --- a/dashboards/gpu/gpu-performance.json +++ b/dashboards/gpu/gpu-performance.json @@ -553,7 +553,7 @@ "id": 22, "targets": [ { - "expr": "DCGM_FI_DEV_FB_FREE{Hostname=~\"$Hostname\"} * 1048576", + "expr": "DCGM_FI_DEV_FB_FREE{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"} * 1048576", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } @@ -617,7 +617,7 @@ "id": 31, "targets": [ { - "expr": "DCGM_FI_DEV_POWER_USAGE{Hostname=~\"$Hostname\"}", + "expr": "DCGM_FI_DEV_POWER_USAGE{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"}", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } @@ -668,7 +668,7 @@ "id": 32, "targets": [ { - "expr": "DCGM_FI_DEV_GPU_TEMP{Hostname=~\"$Hostname\"}", + "expr": "DCGM_FI_DEV_GPU_TEMP{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"}", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } @@ -751,7 +751,7 @@ "id": 41, "targets": [ { - "expr": "max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS{Hostname=~\"$Hostname\"})", + "expr": "max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"})", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } @@ -809,7 +809,7 @@ "id": 42, "targets": [ { - "expr": "rate(DCGM_FI_DEV_POWER_VIOLATION{Hostname=~\"$Hostname\"}[5m])", + "expr": "rate(DCGM_FI_DEV_POWER_VIOLATION{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"}[5m])", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } @@ -857,7 +857,7 @@ "id": 43, "targets": [ { - "expr": "rate(DCGM_FI_DEV_THERMAL_VIOLATION{Hostname=~\"$Hostname\"}[5m])", + "expr": "rate(DCGM_FI_DEV_THERMAL_VIOLATION{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"}[5m])", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } From b64bfcc414ec4faddecf49ff9ece8ef169896bc0 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 07:36:14 +0300 Subject: [PATCH 021/130] fix(dashboard): deduplicate pending GPU pods by (namespace, pod) The Pending GPU pods counter on gpu-quotas joined raw kube_pod_container_resource_requests (per-container series) against kube_pod_status_phase (per-pod series). Multi-container pods were counted once per requesting container instead of once per pod, so the widget over-reported whenever a Pending pod had more than one GPU container. Collapse the requests to pod level before the join. Signed-off-by: Arsolitt --- dashboards/gpu/gpu-quotas.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json index 3bc01a83..624f072b 100644 --- a/dashboards/gpu/gpu-quotas.json +++ b/dashboards/gpu/gpu-quotas.json @@ -208,7 +208,7 @@ "id": 5, "targets": [ { - "expr": "count((kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"} \u003e 0) * on(namespace, pod) group_left() (kube_pod_status_phase{phase=\"Pending\"} == 1)) or vector(0)", + "expr": "count((sum by (namespace, pod) (kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}) \u003e 0) * on(namespace, pod) group_left() (kube_pod_status_phase{phase=\"Pending\"} == 1)) or vector(0)", "refId": "A" } ], From 51b0dedd0820c2a23978880baac0b8a6a3cbca94 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 07:45:02 +0300 Subject: [PATCH 022/130] chore(monitoring): tidy GPU VMRule top-level structure Drop the hardcoded metadata.namespace so the rule inherits the chart's release namespace, and add an explicit empty params field on every group for schema consistency. No behavior change. Signed-off-by: Arsolitt --- .../system/monitoring-agents/alerts/gpu-recording.rules.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml index a6ecb20a..22b3edf0 100644 --- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml +++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml @@ -2,11 +2,11 @@ apiVersion: operator.victoriametrics.com/v1beta1 kind: VMRule metadata: name: alerts-gpu-recording.rules - namespace: cozy-monitoring spec: groups: - name: gpu.recording.cluster.1m interval: 1m + params: {} rules: - record: cluster:gpu_count:total expr: count(group by (UUID) (DCGM_FI_DEV_GPU_UTIL)) @@ -25,6 +25,7 @@ spec: - name: gpu.recording.namespace.1m interval: 1m + params: {} rules: # Kube-requested GPU count per namespace — billable view, includes # Pending pods. Use this for GPU-hour reporting via @@ -61,6 +62,7 @@ spec: - name: gpu.recording.efficiency.1m interval: 1m + params: {} rules: # Tensor hardware saturation — the honest "am I using the GPU" metric # for AI/LLM workloads. Unlike NVML, idle tensor cores are visible. From 0634654d63d0a9b6ef17074161fce84443e3ba95 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 07:53:48 +0300 Subject: [PATCH 023/130] fix(monitoring): exclude terminated pods from GPU allocation count MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit kube-state-metrics keeps kube_pod_container_resource_requests series for Failed/Succeeded pods until the apiserver garbage-collects them, which could inflate :allocated beyond what tenants actually hold and drive cluster:gpu_count:free negative. Join the request metric against kube_pod_status_phase filtered to Pending|Running — the canonical pattern from Kubernetes' own container_resource recording rules — on both the cluster and namespace aggregates. Add clamp_min(..., 0) on cluster:gpu_count:free as a second line of defence against transient label drift between kube-state-metrics and DCGM. Signed-off-by: Arsolitt --- .../alerts/gpu-recording.rules.yaml | 35 +++++++++++++++---- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml index 22b3edf0..de81b8cc 100644 --- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml +++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml @@ -10,14 +10,29 @@ spec: rules: - record: cluster:gpu_count:total expr: count(group by (UUID) (DCGM_FI_DEV_GPU_UTIL)) - # Kube-allocated GPU count: GPUs requested by *all* pods regardless of - # phase (Pending+Running). Source of truth for "what tenants asked for" - # — used for capacity planning and billing. Includes system pods so it - # stays consistent with cluster:gpu_count:total when computing :free. + # Kube-allocated GPU count: GPUs requested by *active* pods — Pending + # and Running only. Source of truth for "what tenants asked for" — used + # for capacity planning and billing. Includes system pods so it stays + # consistent with cluster:gpu_count:total when computing :free. + # + # Phase filter via group_left() on kube_pod_status_phase matches the + # canonical pattern from k8s.rules.container_resource.yaml. Without it + # kube-state-metrics keeps series for Failed/Succeeded pods until the + # apiserver garbage-collects them, which inflates billing hours and + # can push :free negative. - record: cluster:gpu_count:allocated - expr: sum(kube_pod_container_resource_requests{resource="nvidia_com_gpu"}) + expr: | + sum( + kube_pod_container_resource_requests{resource="nvidia_com_gpu"} + * on (namespace, pod) group_left() max by (namespace, pod) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + # clamp_min guards against transients at DCGM/kube-state-metrics + # restart where cluster:gpu_count:total dips before :allocated catches + # up, or rare label drift between the two sources. - record: cluster:gpu_count:free - expr: cluster:gpu_count:total - (cluster:gpu_count:allocated or vector(0)) + expr: clamp_min(cluster:gpu_count:total - (cluster:gpu_count:allocated or vector(0)), 0) - record: cluster:gpu_util:avg expr: avg(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}) - record: cluster:gpu_power_watts:sum @@ -50,7 +65,13 @@ spec: # keeps system pods to stay aligned with cluster:gpu_count:total # when computing :free. - record: namespace:gpu_count:allocated - expr: sum by (namespace) (kube_pod_container_resource_requests{resource="nvidia_com_gpu", namespace!="", namespace!~"cozy-.*|kube-.*"}) + expr: | + sum by (namespace) ( + kube_pod_container_resource_requests{resource="nvidia_com_gpu", namespace!="", namespace!~"cozy-.*|kube-.*"} + * on (namespace, pod) group_left() max by (namespace, pod) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) - record: namespace:gpu_util:avg expr: avg by (namespace) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}) - record: namespace:tensor_active:avg From 605bcd338c744905097dc5c34a9b26f20b999661 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 08:03:21 +0300 Subject: [PATCH 024/130] fix(monitoring): pin label matching on GPU efficiency and throttle rules pod:util_per_watt:avg5m divided two DCGM metrics without an explicit on(...) clause, so the match used the intersection of their label sets. If dcgm-exporter relabeling ever diverges between DCGM_FI_DEV_GPU_UTIL and DCGM_FI_DEV_POWER_USAGE (e.g. a pod-mapping label appears on one but not the other after a config change), the entire result drops to empty silently. Pin the match to the labels we group by so divergence becomes a missing side, not a missing rule. Throttle fractions had a related shape problem: dcgm-exporter emits one series per GPU for each pod-mapping combination. On a shared GPU (restart races, MIG/MPS) the same physical counter appears under multiple pod labels and downstream avg(...) panels get diluted by the pod count. Fold duplicates with max by (Hostname, gpu, UUID) before clamp_max so the fraction is tied to the physical GPU. Signed-off-by: Arsolitt --- .../alerts/gpu-recording.rules.yaml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml index de81b8cc..c56ea851 100644 --- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml +++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml @@ -96,11 +96,14 @@ spec: ) # Power efficiency — utilization per watt, reveals unoptimized clients. + # Explicit on(...) pins the bigop matching set to the labels we + # group by — protects against empty results if dcgm-exporter + # relabeling ever diverges between the two metrics. - record: pod:util_per_watt:avg5m expr: | avg by (Hostname, gpu, UUID, namespace, pod) ( avg_over_time(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]) - / + / on (Hostname, gpu, UUID, namespace, pod) clamp_min( avg_over_time(DCGM_FI_DEV_POWER_USAGE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]), 1 @@ -112,9 +115,16 @@ spec: # counter grows in nanoseconds in practice — divide by 1e9 to get a # 0..1 fraction (verified empirically when /1e6 yielded >100× reality). # clamp_max protects against rate() artefacts at counter resets. + # + # max by (Hostname, gpu, UUID) collapses duplicate series from + # dcgm-exporter's pod-mapping labels (one physical GPU may appear + # under multiple pod/container labels during restarts or under + # MIG/MPS). Throttling is a physical-GPU property; taking max + # keeps consumer avg(...) honest — a naive avg() would dilute the + # signal by the number of pods sharing the GPU. - record: gpu:power_throttle_fraction:rate5m - expr: clamp_max(rate(DCGM_FI_DEV_POWER_VIOLATION[5m]) / 1e9, 1) + expr: clamp_max(max by (Hostname, gpu, UUID) (rate(DCGM_FI_DEV_POWER_VIOLATION[5m]) / 1e9), 1) # Fraction of time thermal-throttled. - record: gpu:thermal_throttle_fraction:rate5m - expr: clamp_max(rate(DCGM_FI_DEV_THERMAL_VIOLATION[5m]) / 1e9, 1) + expr: clamp_max(max by (Hostname, gpu, UUID) (rate(DCGM_FI_DEV_THERMAL_VIOLATION[5m]) / 1e9), 1) From 165e175d70566b6890abea3efd01c18784f9b9ff Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 08:11:57 +0300 Subject: [PATCH 025/130] feat(monitoring): alert on DCGM throttle divisor drift MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The /1e9 divisor in gpu:{power,thermal}_throttle_fraction:rate5m was derived empirically against DCGM 3.x on A10 — the counter documents itself as microseconds but ticks in nanoseconds in practice. If a future exporter release honors the documented units, pre-clamp values would exceed 1.0 while clamp_max(..., 1) silently masks the drift, plateauing every throttle fraction at 100% and making the panels lie in unison. Add a validation group that fires when the raw max/1e9 value exceeds 1.0 for 15m, so we notice and rescale to /1e6 before dashboards silently mislead operators. Signed-off-by: Arsolitt --- .../alerts/gpu-recording.rules.yaml | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml index c56ea851..7589f894 100644 --- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml +++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml @@ -128,3 +128,33 @@ spec: # Fraction of time thermal-throttled. - record: gpu:thermal_throttle_fraction:rate5m expr: clamp_max(max by (Hostname, gpu, UUID) (rate(DCGM_FI_DEV_THERMAL_VIOLATION[5m]) / 1e9), 1) + + # Regression watch — the /1e9 divisor on *_VIOLATION was derived + # empirically against DCGM 3.x on A10. If a future DCGM version + # restores the documented µs units, pre-clamp values would exceed 1.0 + # while clamp_max(..., 1) silently masks the drift — recorded + # throttle fractions would plateau at 1.0 and consumers would think + # every GPU is always throttled. This alert fires on that condition + # so we can rescale to /1e6 before dashboards start lying. + - name: gpu.recording.throttle.validation.5m + interval: 5m + params: {} + rules: + - alert: GPUThrottleFractionOverOne + expr: | + max by (Hostname, gpu, UUID) (rate(DCGM_FI_DEV_POWER_VIOLATION[5m]) / 1e9) > 1 + or + max by (Hostname, gpu, UUID) (rate(DCGM_FI_DEV_THERMAL_VIOLATION[5m]) / 1e9) > 1 + for: 15m + labels: + severity: warning + component: gpu-monitoring + annotations: + summary: "DCGM throttle fraction exceeds 1.0 pre-clamp on {{ $labels.Hostname }}/{{ $labels.gpu }}" + description: | + gpu:{power,thermal}_throttle_fraction:rate5m clamps to 1.0, but the raw + rate(DCGM_FI_DEV_*_VIOLATION)/1e9 value is already >1 on this GPU — the + empirical nanosecond assumption is broken. Likely the DCGM exporter + version changed and the counter now grows in its documented µs units. + Re-verify on the exporter Pod and adjust the divisor (/1e9 → /1e6) in + gpu.recording.efficiency.1m before dashboards plateau at 100% throttle. From 549b341675ea09ce2d61de9c931267adb55801ea Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sun, 19 Apr 2026 10:59:42 +0300 Subject: [PATCH 026/130] fix(efficiency): drop namespace filter on cluster-level throttle metrics Signed-off-by: Arsolitt --- dashboards/gpu/gpu-efficiency.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dashboards/gpu/gpu-efficiency.json b/dashboards/gpu/gpu-efficiency.json index e2e8c105..3bd59bc0 100644 --- a/dashboards/gpu/gpu-efficiency.json +++ b/dashboards/gpu/gpu-efficiency.json @@ -166,12 +166,12 @@ "id": 4, "targets": [ { - "expr": "avg(gpu:power_throttle_fraction:rate5m{namespace=~\"$namespace\"})", + "expr": "avg(gpu:power_throttle_fraction:rate5m)", "refId": "A" } ], "title": "Avg Power Throttling", - "description": "Fraction of time GPUs hit the TDP cap and lose performance. \u003e5% means tenants underutilize billed FLOPS.", + "description": "Fraction of time GPUs hit the TDP cap and lose performance. \u003e5% means tenants underutilize billed FLOPS. Cluster-wide — rule aggregates by (Hostname, gpu, UUID) and drops namespace label.", "transparent": false, "datasource": { "type": "prometheus", @@ -658,7 +658,7 @@ } ], "title": "Power throttle fraction per GPU", - "description": "Fraction of time the GPU was power-throttled. 1.0 = always throttled.", + "description": "Fraction of time the GPU was power-throttled. 1.0 = always throttled. Cluster-wide — rule aggregates by (Hostname, gpu, UUID) and drops namespace label.", "transparent": false, "datasource": { "type": "prometheus", @@ -729,7 +729,7 @@ } ], "title": "Thermal throttle fraction per GPU", - "description": "Fraction of time the GPU was thermal-throttled.", + "description": "Fraction of time the GPU was thermal-throttled. Cluster-wide — rule aggregates by (Hostname, gpu, UUID) and drops namespace label.", "transparent": false, "datasource": { "type": "prometheus", From 5e070840d66aa9a5da934db330a8ec16d5448e26 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sun, 19 Apr 2026 11:00:02 +0300 Subject: [PATCH 027/130] fix(fleet): count GPU nodes via DCGM instead of kube_node_labels Signed-off-by: Arsolitt --- dashboards/gpu/gpu-fleet.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboards/gpu/gpu-fleet.json b/dashboards/gpu/gpu-fleet.json index ea169433..8139549e 100644 --- a/dashboards/gpu/gpu-fleet.json +++ b/dashboards/gpu/gpu-fleet.json @@ -209,12 +209,12 @@ "id": 5, "targets": [ { - "expr": "count(kube_node_labels{label_nvidia_com_gpu_present=\"true\"})", + "expr": "count(count by (Hostname) (DCGM_FI_DEV_GPU_UTIL))", "refId": "A" } ], "title": "Nodes with GPU", - "description": "Kubernetes nodes advertising nvidia.com/gpu.present=true.", + "description": "Nodes reporting at least one GPU to DCGM. Counted via DCGM_FI_DEV_GPU_UTIL because kube-state-metrics does not expose node labels by default.", "transparent": false, "datasource": { "type": "prometheus", From eefb3651a0b8c16ce4b6aa812fefed3b58019e15 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sun, 19 Apr 2026 11:00:53 +0300 Subject: [PATCH 028/130] fix(performance): drop namespace filter on per-GPU physical metrics Signed-off-by: Arsolitt --- dashboards/gpu/gpu-performance.json | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/dashboards/gpu/gpu-performance.json b/dashboards/gpu/gpu-performance.json index e16f8c47..e581a76b 100644 --- a/dashboards/gpu/gpu-performance.json +++ b/dashboards/gpu/gpu-performance.json @@ -553,7 +553,7 @@ "id": 22, "targets": [ { - "expr": "DCGM_FI_DEV_FB_FREE{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"} * 1048576", + "expr": "DCGM_FI_DEV_FB_FREE{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 1048576", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } @@ -617,12 +617,13 @@ "id": 31, "targets": [ { - "expr": "DCGM_FI_DEV_POWER_USAGE{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"}", + "expr": "DCGM_FI_DEV_POWER_USAGE{Hostname=~\"$Hostname\"}", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } ], "title": "Power Usage per GPU", + "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.", "transparent": false, "datasource": { "type": "prometheus", @@ -668,12 +669,13 @@ "id": 32, "targets": [ { - "expr": "DCGM_FI_DEV_GPU_TEMP{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"}", + "expr": "DCGM_FI_DEV_GPU_TEMP{Hostname=~\"$Hostname\"}", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } ], "title": "GPU Temperature", + "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.", "transparent": false, "datasource": { "type": "prometheus", @@ -751,12 +753,13 @@ "id": 41, "targets": [ { - "expr": "max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"})", + "expr": "max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS{Hostname=~\"$Hostname\"})", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } ], "title": "XID errors (latest)", + "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.", "transparent": false, "datasource": { "type": "prometheus", @@ -809,12 +812,13 @@ "id": 42, "targets": [ { - "expr": "rate(DCGM_FI_DEV_POWER_VIOLATION{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"}[5m])", + "expr": "rate(DCGM_FI_DEV_POWER_VIOLATION{Hostname=~\"$Hostname\"}[5m])", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } ], "title": "Power Violation (µs/s throttled due to power)", + "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.", "transparent": false, "datasource": { "type": "prometheus", @@ -857,12 +861,13 @@ "id": 43, "targets": [ { - "expr": "rate(DCGM_FI_DEV_THERMAL_VIOLATION{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"}[5m])", + "expr": "rate(DCGM_FI_DEV_THERMAL_VIOLATION{Hostname=~\"$Hostname\"}[5m])", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } ], "title": "Thermal Violation (µs/s throttled due to thermals)", + "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.", "transparent": false, "datasource": { "type": "prometheus", From 14d9188fcda1a9780912f43d3b3ad3fd11f4c6f4 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sun, 19 Apr 2026 11:01:22 +0300 Subject: [PATCH 029/130] fix(quotas): exclude terminated pods from GPU request panel Signed-off-by: Arsolitt --- dashboards/gpu/gpu-quotas.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json index 624f072b..29e727ce 100644 --- a/dashboards/gpu/gpu-quotas.json +++ b/dashboards/gpu/gpu-quotas.json @@ -423,7 +423,7 @@ "id": 21, "targets": [ { - "expr": "kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left(phase) (kube_pod_status_phase{phase=~\"Running|Pending|Failed|Succeeded\"} == 1)", + "expr": "kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left(phase) (kube_pod_status_phase{phase=~\"Running|Pending\"} == 1)", "instant": true, "range": false, "format": "table", From 950c5dd6695872850e0ec87f2c3dbb28f8dbef52 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sun, 19 Apr 2026 11:01:40 +0300 Subject: [PATCH 030/130] fix(fleet): guard TDP division and document DCGM dependency Signed-off-by: Arsolitt --- dashboards/gpu/gpu-fleet.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboards/gpu/gpu-fleet.json b/dashboards/gpu/gpu-fleet.json index 8139549e..a7d3f085 100644 --- a/dashboards/gpu/gpu-fleet.json +++ b/dashboards/gpu/gpu-fleet.json @@ -467,13 +467,13 @@ "id": 12, "targets": [ { - "expr": "sum by (Hostname) (DCGM_FI_DEV_POWER_USAGE) / sum by (Hostname) (DCGM_FI_DEV_POWER_MGMT_LIMIT) * 100", + "expr": "sum by (Hostname) (DCGM_FI_DEV_POWER_USAGE) / clamp_min(sum by (Hostname) (DCGM_FI_DEV_POWER_MGMT_LIMIT), 1) * 100", "legendFormat": "{{Hostname}}", "refId": "A" } ], "title": "Power draw per node (% of TDP)", - "description": "GPU power usage as a fraction of the node's combined TDP cap. Works across GPU SKUs without per-model thresholds.", + "description": "GPU power usage as a fraction of the node's combined TDP cap. Works across GPU SKUs without per-model thresholds. Requires DCGM_FI_DEV_POWER_MGMT_LIMIT from custom DCGM CSV (see packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml).", "transparent": false, "datasource": { "type": "prometheus", From 43fe172d2f08c9bdf669473913cb485b4e6a6b86 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sun, 19 Apr 2026 11:02:05 +0300 Subject: [PATCH 031/130] docs(gpu-operator): document POWER/THERMAL_VIOLATION and PSS requirements Signed-off-by: Arsolitt --- .../system/gpu-operator/examples/README.md | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/packages/system/gpu-operator/examples/README.md b/packages/system/gpu-operator/examples/README.md index 9599acb7..7922d85a 100644 --- a/packages/system/gpu-operator/examples/README.md +++ b/packages/system/gpu-operator/examples/README.md @@ -56,6 +56,14 @@ files into a directory the validator does inspect and creates the [1]: https://github.com/NVIDIA/gpu-operator/issues/1687 +The compat DaemonSet runs privileged and bind-mounts host paths, so +the target namespace must allow privileged pods. On clusters that +enforce the Kubernetes Pod Security Standards at `baseline` or +`restricted`, label the namespace with +`pod-security.kubernetes.io/enforce: privileged` (and the matching +`audit`/`warn` labels if the admission webhook is configured to +surface violations) before applying the manifest. + ## Dashboards and what DCGM metrics they need Five GPU dashboards live under `gpu/*` in @@ -83,8 +91,20 @@ any CSV override. The three counters listed in the table — throttling violations and the power management limit — are the only extras the tracked dashboards need. The recording rules in `gpu-recording.rules.yaml` consume utilization, FB used, power, -temperature and the tensor-active profiling counter, all of which are -in the default set. +temperature and the tensor-active profiling counter from the default +set, plus `DCGM_FI_DEV_POWER_VIOLATION` and +`DCGM_FI_DEV_THERMAL_VIOLATION` — used by the +`gpu.recording.efficiency.1m` group to derive the +`gpu:power_throttle_fraction:rate5m` and +`gpu:thermal_throttle_fraction:rate5m` series consumed by the +throttling panels on the efficiency and fleet dashboards. + +The `gpu.recording.throttle.validation.5m` group additionally ships the +`GPUThrottleFractionOverOne` alert (severity `warning`) as a regression +detector: it fires when either throttle-fraction series exceeds 1.0, +which would indicate that DCGM changed the scale/divisor of the +underlying violation counters and the recording rules need to be +re-derived. ## Verification status From f8b99008737b7f1067d89220a517edbed77ef777 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sun, 19 Apr 2026 11:08:25 +0300 Subject: [PATCH 032/130] fix(quotas): use allocated recording rules to exclude terminated pods Signed-off-by: Arsolitt --- dashboards/gpu/gpu-quotas.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json index 29e727ce..f8ce4ba8 100644 --- a/dashboards/gpu/gpu-quotas.json +++ b/dashboards/gpu/gpu-quotas.json @@ -278,7 +278,7 @@ "id": 11, "targets": [ { - "expr": "sum by (namespace) (kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"})", + "expr": "sum by (namespace) (namespace:gpu_count:allocated{namespace=~\"$namespace\"})", "legendFormat": "{{namespace}}", "refId": "A" } @@ -346,7 +346,7 @@ "refId": "A" }, { - "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"})", + "expr": "sum(namespace:gpu_count:allocated{namespace=~\"$namespace\"})", "legendFormat": "Requested", "refId": "B" } From a3241bf51bf255e24b77ebc23de89e33064370eb Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sun, 19 Apr 2026 11:08:34 +0300 Subject: [PATCH 033/130] fix(quotas): apply phase join to GPU limits column Signed-off-by: Arsolitt --- dashboards/gpu/gpu-quotas.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json index f8ce4ba8..ce1e387e 100644 --- a/dashboards/gpu/gpu-quotas.json +++ b/dashboards/gpu/gpu-quotas.json @@ -430,7 +430,7 @@ "refId": "requested" }, { - "expr": "kube_pod_container_resource_limits{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"}", + "expr": "kube_pod_container_resource_limits{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left() (kube_pod_status_phase{phase=~\"Running|Pending\"} == 1)", "instant": true, "range": false, "format": "table", From 2cc60f170c3c370a6939e2dfffc20185fb29d0c1 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sun, 19 Apr 2026 11:08:47 +0300 Subject: [PATCH 034/130] docs(gpu-operator): reflect recording-rule dependency for gpu-quotas Signed-off-by: Arsolitt --- packages/system/gpu-operator/examples/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/system/gpu-operator/examples/README.md b/packages/system/gpu-operator/examples/README.md index 7922d85a..f9094549 100644 --- a/packages/system/gpu-operator/examples/README.md +++ b/packages/system/gpu-operator/examples/README.md @@ -81,7 +81,7 @@ What each dashboard needs on top of the upstream DCGM Exporter | `gpu-performance` | Per-node, per-GPU deep dive | `DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION` | | `gpu-efficiency` | Per-workload util vs tensor active | `DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION` (via `gpu:*_throttle_fraction:rate5m` recording rules) | | `gpu-fleet` | Cluster-wide admin inventory | `DCGM_FI_DEV_POWER_MGMT_LIMIT` (for the TDP vs draw panel) | -| `gpu-quotas` | Kube-quota vs live usage | nothing (kube-state-metrics + default counters) | +| `gpu-quotas` | Kube-quota vs live usage | `kube_pod_container_resource_requests`, `kube_pod_status_phase`, `DCGM_FI_DEV_GPU_UTIL` (via `namespace:gpu_count:allocated` / `cluster:gpu_count:*` recording rules) | | `gpu-tenants` | Per-namespace tenant view | nothing (works on default counters) | `DCGM_FI_PROF_PIPE_TENSOR_ACTIVE` and `DCGM_FI_PROF_GR_ENGINE_ACTIVE` From 95ea20119e7fddc601addc9f79777d0e24aadd30 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sun, 19 Apr 2026 11:09:01 +0300 Subject: [PATCH 035/130] fix(gpu-operator): drop unused hostPID on driver-compat DaemonSet Signed-off-by: Arsolitt --- .../system/gpu-operator/examples/nvidia-driver-compat.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml b/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml index bad8609e..7706b221 100644 --- a/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml +++ b/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml @@ -24,8 +24,11 @@ spec: metadata: labels: app: nvidia-driver-compat + # DaemonSet rather than a one-shot Job: staging is idempotent per-node, + # must re-run after every Talos reboot (hostPath contents are wiped on + # reboot when the system extension re-installs), and the pause container + # keeps the pod visible for operator observability (kubectl get pods). spec: - hostPID: true priorityClassName: system-node-critical # Restrict to GPU nodes only. Without this the DaemonSet schedules onto # every node (including control-plane and CPU-only workers) and burns From 5e8194c850a889d03ba60e900c7e91d3f25b8e53 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sun, 19 Apr 2026 11:09:50 +0300 Subject: [PATCH 036/130] docs(monitoring): explain cluster-layer filter asymmetry in GPU rules Signed-off-by: Arsolitt --- .../alerts/gpu-recording.rules.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml index 7589f894..681ac3e8 100644 --- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml +++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml @@ -33,6 +33,24 @@ spec: # up, or rare label drift between the two sources. - record: cluster:gpu_count:free expr: clamp_min(cluster:gpu_count:total - (cluster:gpu_count:allocated or vector(0)), 0) + # Cluster-layer filter asymmetry — intentional, not a bug: + # + # * cluster:gpu_count:total and cluster:gpu_power_watts:sum do NOT + # filter cozy-*/kube-* namespaces because they describe the + # physical hardware fleet. A GPU draws power and occupies a + # slot regardless of which namespace's pod happens to hold it; + # excluding infra pods would under-report raw capacity and + # break capacity-planning math. + # + # * cluster:gpu_util:avg DOES filter infra namespaces because it + # is a tenant-oriented KPI. Admins want to see the mean + # utilization of tenant workloads, not have it diluted by + # GPU Operator DaemonSet pods whose containers hold a GPU + # handle but do no useful compute. + # + # Do not "normalize" by adding the filter to the hardware rules — + # downstream consumers (capacity planning, billing) rely on the + # raw totals. - record: cluster:gpu_util:avg expr: avg(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}) - record: cluster:gpu_power_watts:sum From f5f083e84102d8c7e743766774cd54b226fc6ef1 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sun, 19 Apr 2026 11:09:51 +0300 Subject: [PATCH 037/130] feat(monitoring): annotate GPUThrottleFractionOverOne with verified hardware Signed-off-by: Arsolitt --- .../alerts/gpu-recording.rules.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml index 681ac3e8..093a2939 100644 --- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml +++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml @@ -99,6 +99,16 @@ spec: - record: namespace:power_watts:sum expr: sum by (namespace) (DCGM_FI_DEV_POWER_USAGE{namespace!="", namespace!~"cozy-.*|kube-.*"}) + # Known gap: there is no lower-bound sanity alert for under-reporting. + # If DCGM ever switches POWER/THERMAL_VIOLATION counter units the other + # direction — e.g. from the current nanoseconds back to microseconds, + # making rate()/1e9 produce values around 0.001 instead of real + # fractions — the throttle signal would silently collapse to near-zero + # instead of plateauing at 1.0. GPUThrottleFractionOverOne below catches + # the >1 drift; the <<1 drift would require correlation with independent + # signals (power draw vs TDP, clock dips) that we do not yet wire up. + # Documented here so future maintainers know this is a known gap, not + # an oversight. - name: gpu.recording.efficiency.1m interval: 1m params: {} @@ -176,3 +186,5 @@ spec: version changed and the counter now grows in its documented µs units. Re-verify on the exporter Pod and adjust the divisor (/1e9 → /1e6) in gpu.recording.efficiency.1m before dashboards plateau at 100% throttle. + verified_on: "NVIDIA A10, DCGM 3.x — other GPU families (H100, L40S) may require a different divisor" + runbook: "If this alert fires, DCGM may have changed POWER_VIOLATION/THERMAL_VIOLATION counter units. Recalibrate by comparing raw counter rate against known throttle events; see gpu-recording.rules.yaml comments." From 16b2fd008ba7bc4db6dae122202884791a22887c Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sun, 19 Apr 2026 11:10:43 +0300 Subject: [PATCH 038/130] docs(monitoring): comment bats regex rule-name convention Signed-off-by: Arsolitt --- hack/check-gpu-recording-rules.bats | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hack/check-gpu-recording-rules.bats b/hack/check-gpu-recording-rules.bats index 4981959a..bae6785e 100644 --- a/hack/check-gpu-recording-rules.bats +++ b/hack/check-gpu-recording-rules.bats @@ -62,6 +62,10 @@ extract_rules() { # whole expression must contain at least two ':' characters. extract_refs() { json_file=$1 + # Prometheus convention allows 2-segment rule names (level:metric); this + # regex is tuned to the 3+ segment convention used in this repo + # (level:metric:op — e.g. cluster:gpu_count:total). Update if future + # rules use 2 segments, otherwise they will be silently skipped. grep -hoE '[a-z][a-z0-9_]*:[a-z0-9_]+:[a-z0-9_]+' "$json_file" | sort -u } From b5232bd15c55fb4a1faf25fa3f51e201a3002a60 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sun, 19 Apr 2026 11:10:46 +0300 Subject: [PATCH 039/130] feat(gpu-operator): enable NVLINK bandwidth in default DCGM CSV Signed-off-by: Arsolitt --- .../system/gpu-operator/examples/dcgm-custom-metrics.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml b/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml index 92064701..8e57788b 100644 --- a/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml +++ b/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml @@ -70,8 +70,10 @@ data: DCGM_FI_DEV_LOW_UTIL_VIOLATION, counter, Throttling duration due to low utilization (in us). DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (in us). - # NVLink — enable only for GPUs that actually have NVLink (A10 has none). - # DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL, counter, Total number of NVLink bandwidth counters for all lanes. + # NVLink — DCGM silently drops this metric on GPUs without NVLink, so + # enabling it unconditionally is safe and keeps this file reusable + # across GPU families. + DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL, counter, Total number of NVLink bandwidth counters for all lanes. # DCP (profiling) metrics — supported from Ampere onwards. DCGM_FI_PROF_GR_ENGINE_ACTIVE, gauge, Ratio of time the graphics engine is active. From 4d9a61a0ec49538c70d7b0e3c7d3cd37949781fa Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sun, 19 Apr 2026 11:10:47 +0300 Subject: [PATCH 040/130] docs(gpu-operator): document native-talos service-monitor interval Signed-off-by: Arsolitt --- packages/system/gpu-operator/examples/values-native-talos.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/system/gpu-operator/examples/values-native-talos.yaml b/packages/system/gpu-operator/examples/values-native-talos.yaml index 86e436c4..5a3d7786 100644 --- a/packages/system/gpu-operator/examples/values-native-talos.yaml +++ b/packages/system/gpu-operator/examples/values-native-talos.yaml @@ -39,6 +39,9 @@ spec: dcgmExporter: serviceMonitor: enabled: true + # Matches the 1m evaluation cadence of GPU recording rules with + # 4x oversampling headroom. DCGM_FI_PROF_* counters have + # documented overhead concerns below 1s; 15s is safe. interval: "15s" honorLabels: true config: From 531bc00524be0789ba99e2082234a1104e010b51 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Wed, 8 Apr 2026 22:17:34 +0300 Subject: [PATCH 041/130] feat(postgres): add serverName parameter for backup recovery Add serverName field to bootstrap configuration to explicitly specify Barman server name from backup.info. This fixes "no target backup found" errors when server_name in backup.info differs from Kubernetes cluster name. Signed-off-by: IvanHunters --- api/apps/v1alpha1/postgresql/types.go | 5 ++++- packages/apps/postgres/README.md | 13 +++++++------ packages/apps/postgres/templates/db.yaml | 3 +++ packages/apps/postgres/values.schema.json | 7 ++++++- packages/apps/postgres/values.yaml | 4 +++- packages/system/postgres-rd/cozyrds/postgres.yaml | 4 ++-- 6 files changed, 25 insertions(+), 11 deletions(-) diff --git a/api/apps/v1alpha1/postgresql/types.go b/api/apps/v1alpha1/postgresql/types.go index fa85d51f..56580018 100644 --- a/api/apps/v1alpha1/postgresql/types.go +++ b/api/apps/v1alpha1/postgresql/types.go @@ -86,12 +86,15 @@ type Bootstrap struct { // Whether to restore from a backup. // +kubebuilder:default:=false Enabled bool `json:"enabled"` - // Previous cluster name before deletion. + // Previous cluster name before deletion (matches serverName in backup.info). // +kubebuilder:default:="" OldName string `json:"oldName"` // Timestamp (RFC3339) for point-in-time recovery; empty means latest. // +kubebuilder:default:="" RecoveryTime string `json:"recoveryTime,omitempty"` + // Barman server name from the old cluster's backup.info. Use when the original cluster used a different serverName than its Kubernetes cluster name. + // +kubebuilder:default:="" + ServerName string `json:"serverName,omitempty"` } type Database struct { diff --git a/packages/apps/postgres/README.md b/packages/apps/postgres/README.md index 5a550f7a..648b3d71 100644 --- a/packages/apps/postgres/README.md +++ b/packages/apps/postgres/README.md @@ -133,12 +133,13 @@ See: ### Bootstrap (recovery) parameters -| Name | Description | Type | Value | -| ------------------------ | ------------------------------------------------------------------- | -------- | ------- | -| `bootstrap` | Bootstrap configuration. | `object` | `{}` | -| `bootstrap.enabled` | Whether to restore from a backup. | `bool` | `false` | -| `bootstrap.recoveryTime` | Timestamp (RFC3339) for point-in-time recovery; empty means latest. | `string` | `""` | -| `bootstrap.oldName` | Previous cluster name before deletion. | `string` | `""` | +| Name | Description | Type | Value | +| ------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------- | +| `bootstrap` | Bootstrap configuration. | `object` | `{}` | +| `bootstrap.enabled` | Whether to restore from a backup. | `bool` | `false` | +| `bootstrap.recoveryTime` | Timestamp (RFC3339) for point-in-time recovery; empty means latest. | `string` | `""` | +| `bootstrap.oldName` | Previous cluster name before deletion (matches serverName in backup.info). | `string` | `""` | +| `bootstrap.serverName` | Barman server name from the old cluster's backup.info. Use when the original cluster used a different serverName than its Kubernetes cluster name. | `string` | `""` | ## Parameter examples and reference diff --git a/packages/apps/postgres/templates/db.yaml b/packages/apps/postgres/templates/db.yaml index 5c40b2c7..2cdaec1d 100644 --- a/packages/apps/postgres/templates/db.yaml +++ b/packages/apps/postgres/templates/db.yaml @@ -32,6 +32,9 @@ spec: - name: {{ .Values.bootstrap.oldName }} barmanObjectStore: destinationPath: {{ .Values.backup.destinationPath }} + {{- if .Values.bootstrap.serverName }} + serverName: {{ .Values.bootstrap.serverName }} + {{- end }} endpointURL: {{ .Values.backup.endpointURL }} s3Credentials: accessKeyId: diff --git a/packages/apps/postgres/values.schema.json b/packages/apps/postgres/values.schema.json index b2a4aeba..28acc5a3 100644 --- a/packages/apps/postgres/values.schema.json +++ b/packages/apps/postgres/values.schema.json @@ -246,7 +246,7 @@ "default": false }, "oldName": { - "description": "Previous cluster name before deletion.", + "description": "Previous cluster name before deletion (matches serverName in backup.info).", "type": "string", "default": "" }, @@ -254,6 +254,11 @@ "description": "Timestamp (RFC3339) for point-in-time recovery; empty means latest.", "type": "string", "default": "" + }, + "serverName": { + "description": "Barman server name from the old cluster's backup.info. Use when the original cluster used a different serverName than its Kubernetes cluster name.", + "type": "string", + "default": "" } } } diff --git a/packages/apps/postgres/values.yaml b/packages/apps/postgres/values.yaml index b8f07f63..a4a4bf86 100644 --- a/packages/apps/postgres/values.yaml +++ b/packages/apps/postgres/values.yaml @@ -153,7 +153,8 @@ backup: ## @typedef {struct} Bootstrap - Bootstrap configuration for restoring a database cluster from a backup. ## @field {bool} enabled - Whether to restore from a backup. ## @field {string} [recoveryTime] - Timestamp (RFC3339) for point-in-time recovery; empty means latest. -## @field {string} oldName - Previous cluster name before deletion. +## @field {string} oldName - Previous cluster name before deletion (matches serverName in backup.info). +## @field {string} [serverName] - Barman server name from the old cluster's backup.info. Use when the original cluster used a different serverName than its Kubernetes cluster name. ## @param {Bootstrap} bootstrap - Bootstrap configuration. bootstrap: @@ -161,3 +162,4 @@ bootstrap: # example: 2020-11-26 15:22:00.00000+00 recoveryTime: "" oldName: "" + serverName: "" diff --git a/packages/system/postgres-rd/cozyrds/postgres.yaml b/packages/system/postgres-rd/cozyrds/postgres.yaml index c74c5783..41457db4 100644 --- a/packages/system/postgres-rd/cozyrds/postgres.yaml +++ b/packages/system/postgres-rd/cozyrds/postgres.yaml @@ -8,7 +8,7 @@ spec: singular: postgres plural: postgreses openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Postgres replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each PostgreSQL replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"version":{"description":"PostgreSQL major version to deploy","type":"string","default":"v18","enum":["v18","v17","v16","v15","v14","v13"]},"postgresql":{"description":"PostgreSQL server configuration.","type":"object","default":{},"properties":{"parameters":{"description":"PostgreSQL server parameters.","type":"object","default":{},"properties":{"max_connections":{"description":"Maximum number of concurrent connections to the database server.","type":"integer","default":100}}}}},"quorum":{"description":"Quorum configuration for synchronous replication.","type":"object","default":{},"required":["maxSyncReplicas","minSyncReplicas"],"properties":{"maxSyncReplicas":{"description":"Maximum number of synchronous replicas allowed (must be less than total replicas).","type":"integer","default":0},"minSyncReplicas":{"description":"Minimum number of synchronous replicas required for commit.","type":"integer","default":0}}},"users":{"description":"Users configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"password":{"description":"Password for the user.","type":"string"},"replication":{"description":"Whether the user has replication privileges.","type":"boolean"}}}},"databases":{"description":"Databases configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"extensions":{"description":"List of enabled PostgreSQL extensions.","type":"array","items":{"type":"string"}},"roles":{"description":"Roles assigned to users.","type":"object","properties":{"admin":{"description":"List of users with admin privileges.","type":"array","items":{"type":"string"}},"readonly":{"description":"List of users with read-only privileges.","type":"array","items":{"type":"string"}}}}}}},"backup":{"description":"Backup configuration.","type":"object","default":{},"required":["enabled"],"properties":{"destinationPath":{"description":"Destination path for backups (e.g. s3://bucket/path/).","type":"string","default":"s3://bucket/path/to/folder/"},"enabled":{"description":"Enable regular backups.","type":"boolean","default":false},"endpointURL":{"description":"S3 endpoint URL for uploads.","type":"string","default":"http://minio-gateway-service:9000"},"retentionPolicy":{"description":"Retention policy (e.g. \"30d\").","type":"string","default":"30d"},"s3AccessKey":{"description":"Access key for S3 authentication.","type":"string","default":""},"s3SecretKey":{"description":"Secret key for S3 authentication.","type":"string","default":""},"schedule":{"description":"Cron schedule for automated backups.","type":"string","default":"0 2 * * * *"}}},"bootstrap":{"description":"Bootstrap configuration.","type":"object","default":{},"required":["enabled","oldName"],"properties":{"enabled":{"description":"Whether to restore from a backup.","type":"boolean","default":false},"oldName":{"description":"Previous cluster name before deletion.","type":"string","default":""},"recoveryTime":{"description":"Timestamp (RFC3339) for point-in-time recovery; empty means latest.","type":"string","default":""}}}}} + {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Postgres replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each PostgreSQL replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"version":{"description":"PostgreSQL major version to deploy","type":"string","default":"v18","enum":["v18","v17","v16","v15","v14","v13"]},"postgresql":{"description":"PostgreSQL server configuration.","type":"object","default":{},"properties":{"parameters":{"description":"PostgreSQL server parameters.","type":"object","default":{},"properties":{"max_connections":{"description":"Maximum number of concurrent connections to the database server.","type":"integer","default":100}}}}},"quorum":{"description":"Quorum configuration for synchronous replication.","type":"object","default":{},"required":["maxSyncReplicas","minSyncReplicas"],"properties":{"maxSyncReplicas":{"description":"Maximum number of synchronous replicas allowed (must be less than total replicas).","type":"integer","default":0},"minSyncReplicas":{"description":"Minimum number of synchronous replicas required for commit.","type":"integer","default":0}}},"users":{"description":"Users configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"password":{"description":"Password for the user.","type":"string"},"replication":{"description":"Whether the user has replication privileges.","type":"boolean"}}}},"databases":{"description":"Databases configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"extensions":{"description":"List of enabled PostgreSQL extensions.","type":"array","items":{"type":"string"}},"roles":{"description":"Roles assigned to users.","type":"object","properties":{"admin":{"description":"List of users with admin privileges.","type":"array","items":{"type":"string"}},"readonly":{"description":"List of users with read-only privileges.","type":"array","items":{"type":"string"}}}}}}},"backup":{"description":"Backup configuration.","type":"object","default":{},"required":["enabled"],"properties":{"destinationPath":{"description":"Destination path for backups (e.g. s3://bucket/path/).","type":"string","default":"s3://bucket/path/to/folder/"},"enabled":{"description":"Enable regular backups.","type":"boolean","default":false},"endpointURL":{"description":"S3 endpoint URL for uploads.","type":"string","default":"http://minio-gateway-service:9000"},"retentionPolicy":{"description":"Retention policy (e.g. \"30d\").","type":"string","default":"30d"},"s3AccessKey":{"description":"Access key for S3 authentication.","type":"string","default":""},"s3SecretKey":{"description":"Secret key for S3 authentication.","type":"string","default":""},"schedule":{"description":"Cron schedule for automated backups.","type":"string","default":"0 2 * * * *"}}},"bootstrap":{"description":"Bootstrap configuration.","type":"object","default":{},"required":["enabled","oldName"],"properties":{"enabled":{"description":"Whether to restore from a backup.","type":"boolean","default":false},"oldName":{"description":"Previous cluster name before deletion (matches serverName in backup.info).","type":"string","default":""},"recoveryTime":{"description":"Timestamp (RFC3339) for point-in-time recovery; empty means latest.","type":"string","default":""},"serverName":{"description":"Barman server name from the old cluster's backup.info. Use when the original cluster used a different serverName than its Kubernetes cluster name.","type":"string","default":""}}}}} release: prefix: postgres- labels: @@ -33,7 +33,7 @@ spec: # labelSelector: # helm.toolkit.fluxcd.io/name: "{reqs[0]['metadata','name']}" - keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "replicas"], ["spec", "resources"], ["spec", "resourcesPreset"], ["spec", "size"], ["spec", "storageClass"], ["spec", "external"], ["spec", "version"], ["spec", "postgresql"], ["spec", "postgresql", "parameters"], ["spec", "postgresql", "parameters", "max_connections"], ["spec", "quorum"], ["spec", "quorum", "minSyncReplicas"], ["spec", "quorum", "maxSyncReplicas"], ["spec", "users"], ["spec", "databases"], ["spec", "backup"], ["spec", "backup", "enabled"], ["spec", "backup", "retentionPolicy"], ["spec", "backup", "destinationPath"], ["spec", "backup", "endpointURL"], ["spec", "backup", "schedule"], ["spec", "backup", "s3AccessKey"], ["spec", "backup", "s3SecretKey"], ["spec", "bootstrap"], ["spec", "bootstrap", "enabled"], ["spec", "bootstrap", "recoveryTime"], ["spec", "bootstrap", "oldName"]] + keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "replicas"], ["spec", "resources"], ["spec", "resourcesPreset"], ["spec", "size"], ["spec", "storageClass"], ["spec", "external"], ["spec", "version"], ["spec", "postgresql"], ["spec", "postgresql", "parameters"], ["spec", "postgresql", "parameters", "max_connections"], ["spec", "quorum"], ["spec", "quorum", "minSyncReplicas"], ["spec", "quorum", "maxSyncReplicas"], ["spec", "users"], ["spec", "databases"], ["spec", "backup"], ["spec", "backup", "enabled"], ["spec", "backup", "retentionPolicy"], ["spec", "backup", "destinationPath"], ["spec", "backup", "endpointURL"], ["spec", "backup", "schedule"], ["spec", "backup", "s3AccessKey"], ["spec", "backup", "s3SecretKey"], ["spec", "bootstrap"], ["spec", "bootstrap", "enabled"], ["spec", "bootstrap", "recoveryTime"], ["spec", "bootstrap", "oldName"], ["spec", "bootstrap", "serverName"]] secrets: exclude: [] include: From 9f41dc3228b18308f9cd6819a8b2470778f27f0c Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Tue, 21 Apr 2026 13:17:32 +0300 Subject: [PATCH 042/130] docs(postgres): clarify bootstrap field descriptions Update oldName and serverName field descriptions based on code review feedback to avoid confusion about their actual roles: - oldName: Remove misleading "(matches serverName in backup.info)" text. This field represents the Kubernetes cluster resource name, not the Barman server name. - serverName: Provide clearer explanation that it's the S3 path prefix (barmanObjectStore.serverName) used by the original cluster, and should only be set when it differs from the Kubernetes resource name. Updated in: - values.yaml (source of truth for field documentation) - types.go (Go API type comments) - values.schema.json (JSON schema for validation) - postgres.yaml (CRD with embedded OpenAPI schema) Signed-off-by: IvanHunters --- api/apps/v1alpha1/postgresql/types.go | 4 ++-- api/backups/v1alpha1/zz_generated.deepcopy.go | 5 +++++ packages/apps/postgres/README.md | 14 +++++++------- packages/apps/postgres/values.schema.json | 4 ++-- packages/apps/postgres/values.yaml | 4 ++-- packages/system/postgres-rd/cozyrds/postgres.yaml | 2 +- 6 files changed, 19 insertions(+), 14 deletions(-) diff --git a/api/apps/v1alpha1/postgresql/types.go b/api/apps/v1alpha1/postgresql/types.go index 56580018..a2ff77a8 100644 --- a/api/apps/v1alpha1/postgresql/types.go +++ b/api/apps/v1alpha1/postgresql/types.go @@ -86,13 +86,13 @@ type Bootstrap struct { // Whether to restore from a backup. // +kubebuilder:default:=false Enabled bool `json:"enabled"` - // Previous cluster name before deletion (matches serverName in backup.info). + // Previous cluster name before deletion. // +kubebuilder:default:="" OldName string `json:"oldName"` // Timestamp (RFC3339) for point-in-time recovery; empty means latest. // +kubebuilder:default:="" RecoveryTime string `json:"recoveryTime,omitempty"` - // Barman server name from the old cluster's backup.info. Use when the original cluster used a different serverName than its Kubernetes cluster name. + // Barman server name (S3 path prefix) used by the original cluster when writing backups. Set this only when the original cluster had an explicit barmanObjectStore.serverName that differed from its Kubernetes resource name. // +kubebuilder:default:="" ServerName string `json:"serverName,omitempty"` } diff --git a/api/backups/v1alpha1/zz_generated.deepcopy.go b/api/backups/v1alpha1/zz_generated.deepcopy.go index 89f6171f..61b8f839 100644 --- a/api/backups/v1alpha1/zz_generated.deepcopy.go +++ b/api/backups/v1alpha1/zz_generated.deepcopy.go @@ -620,6 +620,11 @@ func (in *RestoreJobSpec) DeepCopyInto(out *RestoreJobSpec) { *out = new(v1.TypedLocalObjectReference) (*in).DeepCopyInto(*out) } + if in.Options != nil { + in, out := &in.Options, &out.Options + *out = new(runtime.RawExtension) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestoreJobSpec. diff --git a/packages/apps/postgres/README.md b/packages/apps/postgres/README.md index 648b3d71..4cda7284 100644 --- a/packages/apps/postgres/README.md +++ b/packages/apps/postgres/README.md @@ -133,13 +133,13 @@ See: ### Bootstrap (recovery) parameters -| Name | Description | Type | Value | -| ------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------- | -| `bootstrap` | Bootstrap configuration. | `object` | `{}` | -| `bootstrap.enabled` | Whether to restore from a backup. | `bool` | `false` | -| `bootstrap.recoveryTime` | Timestamp (RFC3339) for point-in-time recovery; empty means latest. | `string` | `""` | -| `bootstrap.oldName` | Previous cluster name before deletion (matches serverName in backup.info). | `string` | `""` | -| `bootstrap.serverName` | Barman server name from the old cluster's backup.info. Use when the original cluster used a different serverName than its Kubernetes cluster name. | `string` | `""` | +| Name | Description | Type | Value | +| ------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------- | +| `bootstrap` | Bootstrap configuration. | `object` | `{}` | +| `bootstrap.enabled` | Whether to restore from a backup. | `bool` | `false` | +| `bootstrap.recoveryTime` | Timestamp (RFC3339) for point-in-time recovery; empty means latest. | `string` | `""` | +| `bootstrap.oldName` | Previous cluster name before deletion. | `string` | `""` | +| `bootstrap.serverName` | Barman server name (S3 path prefix) used by the original cluster when writing backups. Set this only when the original cluster had an explicit barmanObjectStore.serverName that differed from its Kubernetes resource name. | `string` | `""` | ## Parameter examples and reference diff --git a/packages/apps/postgres/values.schema.json b/packages/apps/postgres/values.schema.json index 28acc5a3..98e29822 100644 --- a/packages/apps/postgres/values.schema.json +++ b/packages/apps/postgres/values.schema.json @@ -246,7 +246,7 @@ "default": false }, "oldName": { - "description": "Previous cluster name before deletion (matches serverName in backup.info).", + "description": "Previous cluster name before deletion.", "type": "string", "default": "" }, @@ -256,7 +256,7 @@ "default": "" }, "serverName": { - "description": "Barman server name from the old cluster's backup.info. Use when the original cluster used a different serverName than its Kubernetes cluster name.", + "description": "Barman server name (S3 path prefix) used by the original cluster when writing backups. Set this only when the original cluster had an explicit barmanObjectStore.serverName that differed from its Kubernetes resource name.", "type": "string", "default": "" } diff --git a/packages/apps/postgres/values.yaml b/packages/apps/postgres/values.yaml index a4a4bf86..2ceaa9cf 100644 --- a/packages/apps/postgres/values.yaml +++ b/packages/apps/postgres/values.yaml @@ -153,8 +153,8 @@ backup: ## @typedef {struct} Bootstrap - Bootstrap configuration for restoring a database cluster from a backup. ## @field {bool} enabled - Whether to restore from a backup. ## @field {string} [recoveryTime] - Timestamp (RFC3339) for point-in-time recovery; empty means latest. -## @field {string} oldName - Previous cluster name before deletion (matches serverName in backup.info). -## @field {string} [serverName] - Barman server name from the old cluster's backup.info. Use when the original cluster used a different serverName than its Kubernetes cluster name. +## @field {string} oldName - Previous cluster name before deletion. +## @field {string} [serverName] - Barman server name (S3 path prefix) used by the original cluster when writing backups. Set this only when the original cluster had an explicit barmanObjectStore.serverName that differed from its Kubernetes resource name. ## @param {Bootstrap} bootstrap - Bootstrap configuration. bootstrap: diff --git a/packages/system/postgres-rd/cozyrds/postgres.yaml b/packages/system/postgres-rd/cozyrds/postgres.yaml index 41457db4..72d013f1 100644 --- a/packages/system/postgres-rd/cozyrds/postgres.yaml +++ b/packages/system/postgres-rd/cozyrds/postgres.yaml @@ -8,7 +8,7 @@ spec: singular: postgres plural: postgreses openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Postgres replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each PostgreSQL replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"version":{"description":"PostgreSQL major version to deploy","type":"string","default":"v18","enum":["v18","v17","v16","v15","v14","v13"]},"postgresql":{"description":"PostgreSQL server configuration.","type":"object","default":{},"properties":{"parameters":{"description":"PostgreSQL server parameters.","type":"object","default":{},"properties":{"max_connections":{"description":"Maximum number of concurrent connections to the database server.","type":"integer","default":100}}}}},"quorum":{"description":"Quorum configuration for synchronous replication.","type":"object","default":{},"required":["maxSyncReplicas","minSyncReplicas"],"properties":{"maxSyncReplicas":{"description":"Maximum number of synchronous replicas allowed (must be less than total replicas).","type":"integer","default":0},"minSyncReplicas":{"description":"Minimum number of synchronous replicas required for commit.","type":"integer","default":0}}},"users":{"description":"Users configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"password":{"description":"Password for the user.","type":"string"},"replication":{"description":"Whether the user has replication privileges.","type":"boolean"}}}},"databases":{"description":"Databases configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"extensions":{"description":"List of enabled PostgreSQL extensions.","type":"array","items":{"type":"string"}},"roles":{"description":"Roles assigned to users.","type":"object","properties":{"admin":{"description":"List of users with admin privileges.","type":"array","items":{"type":"string"}},"readonly":{"description":"List of users with read-only privileges.","type":"array","items":{"type":"string"}}}}}}},"backup":{"description":"Backup configuration.","type":"object","default":{},"required":["enabled"],"properties":{"destinationPath":{"description":"Destination path for backups (e.g. s3://bucket/path/).","type":"string","default":"s3://bucket/path/to/folder/"},"enabled":{"description":"Enable regular backups.","type":"boolean","default":false},"endpointURL":{"description":"S3 endpoint URL for uploads.","type":"string","default":"http://minio-gateway-service:9000"},"retentionPolicy":{"description":"Retention policy (e.g. \"30d\").","type":"string","default":"30d"},"s3AccessKey":{"description":"Access key for S3 authentication.","type":"string","default":""},"s3SecretKey":{"description":"Secret key for S3 authentication.","type":"string","default":""},"schedule":{"description":"Cron schedule for automated backups.","type":"string","default":"0 2 * * * *"}}},"bootstrap":{"description":"Bootstrap configuration.","type":"object","default":{},"required":["enabled","oldName"],"properties":{"enabled":{"description":"Whether to restore from a backup.","type":"boolean","default":false},"oldName":{"description":"Previous cluster name before deletion (matches serverName in backup.info).","type":"string","default":""},"recoveryTime":{"description":"Timestamp (RFC3339) for point-in-time recovery; empty means latest.","type":"string","default":""},"serverName":{"description":"Barman server name from the old cluster's backup.info. Use when the original cluster used a different serverName than its Kubernetes cluster name.","type":"string","default":""}}}}} + {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Postgres replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each PostgreSQL replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"version":{"description":"PostgreSQL major version to deploy","type":"string","default":"v18","enum":["v18","v17","v16","v15","v14","v13"]},"postgresql":{"description":"PostgreSQL server configuration.","type":"object","default":{},"properties":{"parameters":{"description":"PostgreSQL server parameters.","type":"object","default":{},"properties":{"max_connections":{"description":"Maximum number of concurrent connections to the database server.","type":"integer","default":100}}}}},"quorum":{"description":"Quorum configuration for synchronous replication.","type":"object","default":{},"required":["maxSyncReplicas","minSyncReplicas"],"properties":{"maxSyncReplicas":{"description":"Maximum number of synchronous replicas allowed (must be less than total replicas).","type":"integer","default":0},"minSyncReplicas":{"description":"Minimum number of synchronous replicas required for commit.","type":"integer","default":0}}},"users":{"description":"Users configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"password":{"description":"Password for the user.","type":"string"},"replication":{"description":"Whether the user has replication privileges.","type":"boolean"}}}},"databases":{"description":"Databases configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"extensions":{"description":"List of enabled PostgreSQL extensions.","type":"array","items":{"type":"string"}},"roles":{"description":"Roles assigned to users.","type":"object","properties":{"admin":{"description":"List of users with admin privileges.","type":"array","items":{"type":"string"}},"readonly":{"description":"List of users with read-only privileges.","type":"array","items":{"type":"string"}}}}}}},"backup":{"description":"Backup configuration.","type":"object","default":{},"required":["enabled"],"properties":{"destinationPath":{"description":"Destination path for backups (e.g. s3://bucket/path/).","type":"string","default":"s3://bucket/path/to/folder/"},"enabled":{"description":"Enable regular backups.","type":"boolean","default":false},"endpointURL":{"description":"S3 endpoint URL for uploads.","type":"string","default":"http://minio-gateway-service:9000"},"retentionPolicy":{"description":"Retention policy (e.g. \"30d\").","type":"string","default":"30d"},"s3AccessKey":{"description":"Access key for S3 authentication.","type":"string","default":""},"s3SecretKey":{"description":"Secret key for S3 authentication.","type":"string","default":""},"schedule":{"description":"Cron schedule for automated backups.","type":"string","default":"0 2 * * * *"}}},"bootstrap":{"description":"Bootstrap configuration.","type":"object","default":{},"required":["enabled","oldName"],"properties":{"enabled":{"description":"Whether to restore from a backup.","type":"boolean","default":false},"oldName":{"description":"Previous cluster name before deletion.","type":"string","default":""},"recoveryTime":{"description":"Timestamp (RFC3339) for point-in-time recovery; empty means latest.","type":"string","default":""},"serverName":{"description":"Barman server name (S3 path prefix) used by the original cluster when writing backups. Set this only when the original cluster had an explicit barmanObjectStore.serverName that differed from its Kubernetes resource name.","type":"string","default":""}}}}} release: prefix: postgres- labels: From 0fdb25df724ba5d44c794271cc0661d97b1d9b13 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 22 Apr 2026 11:59:50 +0500 Subject: [PATCH 043/130] ci(api): add codegen drift check Run root 'make generate' as a pre-commit hook and as a dedicated CI workflow so missed codegen updates (CRDs, deepcopy, clients, RBAC) are caught instead of merging stale generated files. Pre-commit hook is scoped to files that actually affect codegen (api/, pkg/apis/, hack/update-codegen.sh, hack/boilerplate.go.txt) so unrelated commits are not slowed down. CI job sets up Go from go.mod, runs make generate, and fails on drift with a pointer to the local fix. Assisted-By: Claude Signed-off-by: Myasnikov Daniil --- .github/workflows/codegen-drift.yml | 42 +++++++++++++++++++++++++++++ .pre-commit-config.yaml | 11 ++++++++ 2 files changed, 53 insertions(+) create mode 100644 .github/workflows/codegen-drift.yml diff --git a/.github/workflows/codegen-drift.yml b/.github/workflows/codegen-drift.yml new file mode 100644 index 00000000..bd7927f7 --- /dev/null +++ b/.github/workflows/codegen-drift.yml @@ -0,0 +1,42 @@ +name: Codegen Drift Check + +on: + pull_request: + types: [opened, synchronize, reopened] + paths: + - 'api/**' + - 'pkg/apis/**' + - 'hack/update-codegen.sh' + - 'hack/boilerplate.go.txt' + - 'go.mod' + - 'go.sum' + - '.github/workflows/codegen-drift.yml' + +concurrency: + group: codegen-drift-${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + codegen-drift: + name: Verify generated code is up to date + runs-on: ubuntu-22.04 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Run make generate + run: make generate + + - name: Fail on drift + run: | + if ! git diff --exit-code; then + echo "::error::'make generate' produced changes. Run 'make generate' locally and commit the result." + git status --short + exit 1 + fi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ac0f7e30..836f79c5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,17 @@ repos: - repo: local hooks: + - id: run-make-generate-root + name: Run 'make generate' at repo root + entry: | + flock -x .git/pre-commit.lock sh -c ' + echo "Running make generate at repo root" + make generate || exit $? + git diff --color=always | cat + ' + language: system + files: ^(api/|pkg/apis/|hack/update-codegen\.sh$|hack/boilerplate\.go\.txt$) + pass_filenames: false - id: run-make-generate name: Run 'make generate' in all app directories entry: | From 860f431187c3f577bfd2b7ab60201583bdef9384 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 22 Apr 2026 12:00:22 +0500 Subject: [PATCH 044/130] chore(api): regenerate deepcopy for RestoreJobSpec.Options The Options field was added to RestoreJobSpec without re-running 'make generate', leaving zz_generated.deepcopy.go out of sync. Regenerated to include the missing DeepCopyInto handling for the runtime.RawExtension pointer. Assisted-By: Claude Signed-off-by: Myasnikov Daniil --- api/backups/v1alpha1/zz_generated.deepcopy.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/api/backups/v1alpha1/zz_generated.deepcopy.go b/api/backups/v1alpha1/zz_generated.deepcopy.go index 89f6171f..61b8f839 100644 --- a/api/backups/v1alpha1/zz_generated.deepcopy.go +++ b/api/backups/v1alpha1/zz_generated.deepcopy.go @@ -620,6 +620,11 @@ func (in *RestoreJobSpec) DeepCopyInto(out *RestoreJobSpec) { *out = new(v1.TypedLocalObjectReference) (*in).DeepCopyInto(*out) } + if in.Options != nil { + in, out := &in.Options, &out.Options + *out = new(runtime.RawExtension) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestoreJobSpec. From 51c2ec0ad41722f34d6cb8a5d111ceac021c4464 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Wed, 22 Apr 2026 13:25:16 +0300 Subject: [PATCH 045/130] feat(hami): add HAMi GPU virtualization system chart Add HAMi v2.8.1 (CNCF Sandbox) as a new system package for fractional GPU sharing in tenant Kubernetes clusters. The chart enables workloads to request specific amounts of GPU memory and compute cores instead of claiming entire GPUs. Vendored upstream chart includes device plugin, scheduler extender, mutating webhook, and DRA subchart. Wrapper values configure nvidia RuntimeClass and clear hardcoded node config from upstream defaults. Assisted-By: Claude Signed-off-by: Arsolitt --- packages/system/hami/Chart.yaml | 3 + packages/system/hami/Makefile | 11 + packages/system/hami/charts/hami/Chart.lock | 6 + packages/system/hami/charts/hami/Chart.yaml | 22 + packages/system/hami/charts/hami/README.md | 237 +++++++++ .../charts/hami/charts/hami-dra/.helmignore | 24 + .../charts/hami/charts/hami-dra/Chart.yaml | 18 + .../charts/hami-dra/templates/_commons.tpl | 84 ++++ .../charts/hami-dra/templates/_helpers.tpl | 73 +++ .../hami-dra-driver/deviceclass.yaml | 11 + .../nvidia-dra-driver-daemonset.yaml | 145 ++++++ .../templates/hami-dra-driver/rbac.yaml | 110 ++++ .../monitor/monitor-clusterrole.yaml | 11 + .../monitor/monitor-clusterrolebinding.yaml | 15 + .../templates/monitor/monitor-deployment.yaml | 62 +++ .../templates/monitor/monitor-service.yaml | 26 + .../monitor/monitor-serviceaccount.yaml | 8 + .../templates/webhook/cert-manager.yaml | 29 ++ .../templates/webhook/cert-secret.yaml | 13 + .../templates/webhook/device-config.yaml | 394 +++++++++++++++ .../webhook/mutatingwebhookconfiguration.yaml | 28 ++ .../validatingwebhookconfiguration.yaml | 29 ++ .../webhook/webhook-clusterrole.yaml | 14 + .../webhook/webhook-clusterrolebinding.yaml | 12 + .../templates/webhook/webhook-deployment.yaml | 59 +++ .../templates/webhook/webhook-service.yaml | 15 + .../webhook/webhook-serviceaccount.yaml | 5 + .../charts/hami/charts/hami-dra/values.yaml | 166 ++++++ .../hami/charts/hami/templates/NOTES.txt | 3 + .../hami/charts/hami/templates/_commons.tpl | 49 ++ .../hami/charts/hami/templates/_helpers.tpl | 163 ++++++ .../templates/device-plugin/configmap.yaml | 13 + .../device-plugin/daemonsetmock.yaml | 55 ++ .../device-plugin/daemonsetnvidia.yaml | 262 ++++++++++ .../templates/device-plugin/monitorrole.yaml | 29 ++ .../device-plugin/monitorrolebinding.yaml | 17 + .../device-plugin/monitorservice.yaml | 29 ++ .../device-plugin/monitorserviceaccount.yaml | 10 + .../device-plugin/runtime-class.yaml | 11 + .../hami/templates/scheduler/certmanager.yaml | 31 ++ .../hami/templates/scheduler/clusterrole.yaml | 36 ++ .../scheduler/clusterrolebinding.yaml | 49 ++ .../hami/templates/scheduler/configmap.yaml | 142 ++++++ .../templates/scheduler/configmapnew.yaml | 102 ++++ .../hami/templates/scheduler/deployment.yaml | 227 +++++++++ .../templates/scheduler/device-configmap.yaml | 410 +++++++++++++++ .../scheduler/job-patch/clusterrole.yaml | 30 ++ .../job-patch/clusterrolebinding.yaml | 22 + .../scheduler/job-patch/job-createSecret.yaml | 70 +++ .../scheduler/job-patch/job-patchWebhook.yaml | 65 +++ .../templates/scheduler/job-patch/psp.yaml | 40 ++ .../templates/scheduler/job-patch/role.yaml | 23 + .../scheduler/job-patch/rolebinding.yaml | 23 + .../scheduler/job-patch/serviceaccount.yaml | 15 + .../charts/hami/templates/scheduler/role.yaml | 14 + .../hami/templates/scheduler/rolebinding.yaml | 33 ++ .../hami/templates/scheduler/service.yaml | 36 ++ .../templates/scheduler/serviceaccount.yaml | 18 + .../hami/templates/scheduler/webhook.yaml | 57 +++ packages/system/hami/charts/hami/values.yaml | 476 ++++++++++++++++++ packages/system/hami/values.yaml | 8 + 61 files changed, 4198 insertions(+) create mode 100644 packages/system/hami/Chart.yaml create mode 100644 packages/system/hami/Makefile create mode 100644 packages/system/hami/charts/hami/Chart.lock create mode 100644 packages/system/hami/charts/hami/Chart.yaml create mode 100644 packages/system/hami/charts/hami/README.md create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/.helmignore create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/_commons.tpl create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/_helpers.tpl create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/deviceclass.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/nvidia-dra-driver-daemonset.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/rbac.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrole.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrolebinding.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-deployment.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-service.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-serviceaccount.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-manager.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-secret.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/device-config.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/mutatingwebhookconfiguration.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/validatingwebhookconfiguration.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrole.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrolebinding.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-deployment.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-service.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-serviceaccount.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/values.yaml create mode 100644 packages/system/hami/charts/hami/templates/NOTES.txt create mode 100644 packages/system/hami/charts/hami/templates/_commons.tpl create mode 100644 packages/system/hami/charts/hami/templates/_helpers.tpl create mode 100644 packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml create mode 100644 packages/system/hami/charts/hami/templates/device-plugin/daemonsetmock.yaml create mode 100644 packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml create mode 100644 packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml create mode 100644 packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml create mode 100644 packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml create mode 100644 packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml create mode 100644 packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/configmap.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/deployment.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/role.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/service.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/webhook.yaml create mode 100644 packages/system/hami/charts/hami/values.yaml create mode 100644 packages/system/hami/values.yaml diff --git a/packages/system/hami/Chart.yaml b/packages/system/hami/Chart.yaml new file mode 100644 index 00000000..3fcf4c5d --- /dev/null +++ b/packages/system/hami/Chart.yaml @@ -0,0 +1,3 @@ +apiVersion: v2 +name: cozy-hami +version: 0.0.0 # Placeholder, the actual version will be automatically set during the build process diff --git a/packages/system/hami/Makefile b/packages/system/hami/Makefile new file mode 100644 index 00000000..b7a169c2 --- /dev/null +++ b/packages/system/hami/Makefile @@ -0,0 +1,11 @@ +export NAME=hami +export NAMESPACE=cozy-$(NAME) + +include ../../../hack/common-envs.mk +include ../../../hack/package.mk + +update: + rm -rf charts + helm repo add hami-charts https://project-hami.github.io/HAMi/ + helm repo update hami-charts + helm pull hami-charts/hami --untar --untardir charts diff --git a/packages/system/hami/charts/hami/Chart.lock b/packages/system/hami/charts/hami/Chart.lock new file mode 100644 index 00000000..25856c07 --- /dev/null +++ b/packages/system/hami/charts/hami/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: hami-dra + repository: https://project-hami.github.io/HAMi-DRA/ + version: 0.1.0 +digest: sha256:374551539570bcee82c1c4dea93eac59e6f410b6d5967d188efd630e203d43bb +generated: "2026-04-17T04:06:47.689117678Z" diff --git a/packages/system/hami/charts/hami/Chart.yaml b/packages/system/hami/charts/hami/Chart.yaml new file mode 100644 index 00000000..aba7e95c --- /dev/null +++ b/packages/system/hami/charts/hami/Chart.yaml @@ -0,0 +1,22 @@ +apiVersion: v2 +appVersion: 2.8.1 +dependencies: +- condition: dra.enabled + name: hami-dra + repository: https://project-hami.github.io/HAMi-DRA/ + version: 0.1.0 +description: Heterogeneous AI Computing Virtualization Middleware +keywords: +- vgpu +- gpu +kubeVersion: '>= 1.18.0-0' +maintainers: +- email: archlitchi@gmail.com + name: limengxuan +- email: xiaozhang0210@hotmail.com + name: zhangxiao +name: hami +sources: +- https://github.com/Project-HAMi/HAMi +type: application +version: 2.8.1 diff --git a/packages/system/hami/charts/hami/README.md b/packages/system/hami/charts/hami/README.md new file mode 100644 index 00000000..e8217290 --- /dev/null +++ b/packages/system/hami/charts/hami/README.md @@ -0,0 +1,237 @@ +# HAMi Helm Chart Values Documentation + +This document provides detailed descriptions of all configurable values parameters for the HAMi Helm Chart. + +## Global Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `global.imageRegistry` | Global Docker image registry | `""` | +| `global.imagePullSecrets` | Global Docker image pull secrets | `[]` | +| `global.imageTag` | Image tag | `"v2.8.1"` | +| `global.gpuHookPath` | GPU Hook path | `/usr/local` | +| `global.labels` | Global labels | `{}` | +| `global.annotations` | Global annotations | `{}` | +| `global.managedNodeSelectorEnable` | Whether to enable managed node selector | `false` | +| `global.managedNodeSelector.usage` | Managed node selector usage | `"gpu"` | +| `nameOverride` | Name override | `""` | +| `fullnameOverride` | Full name override | `""` | +| `namespaceOverride` | Namespace override | `""` | + +## Resource Name Configuration + +### NVIDIA GPU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `resourceName` | GPU resource name | `"nvidia.com/gpu"` | +| `resourceMem` | GPU memory resource name | `"nvidia.com/gpumem"` | +| `resourceMemPercentage` | GPU memory percentage resource name | `"nvidia.com/gpumem-percentage"` | +| `resourceCores` | GPU core resource name | `"nvidia.com/gpucores"` | +| `resourcePriority` | GPU priority resource name | `"nvidia.com/priority"` | + +### Cambricon MLU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `mluResourceName` | MLU resource name | `"cambricon.com/vmlu"` | +| `mluResourceMem` | MLU memory resource name | `"cambricon.com/mlu.smlu.vmemory"` | +| `mluResourceCores` | MLU core resource name | `"cambricon.com/mlu.smlu.vcore"` | + +### Hygon DCU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `dcuResourceName` | DCU resource name | `"hygon.com/dcunum"` | +| `dcuResourceMem` | DCU memory resource name | `"hygon.com/dcumem"` | +| `dcuResourceCores` | DCU core resource name | `"hygon.com/dcucores"` | + +### Metax GPU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `metaxResourceName` | GPU resource name | `"metax-tech.com/sgpu"` | +| `metaxResourceCore` | GPU core resource name | `"metax-tech.com/vcore"` | +| `metaxResourceMem` | GPU memory resource name | `"metax-tech.com/vmemory"` | +| `metaxsGPUTopologyAware` | GPU topology awareness | `"false"` | + +### Enflame GCU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `enflameResourceNameVGCU` | vGCU resource name | `"enflame.com/vgcu"` | +| `enflameResourceNameVGCUPercentage` | vGCU percentage resource name | `"enflame.com/vgcu-percentage"` | + +### Kunlunxin XPU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `kunlunResourceName` | XPU resource name | `"kunlunxin.com/xpu"` | + +## Scheduler Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `schedulerName` | Scheduler name | `"hami-scheduler"` | +| `scheduler.nodeName` | Define node name, scheduler will schedule to this node | `""` | +| `scheduler.overwriteEnv` | Whether to overwrite environment variables | `"false"` | +| `scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy` | Node scheduler policy | `binpack` | +| `scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy` | GPU scheduler policy | `spread` | +| `scheduler.metricsBindAddress` | Metrics bind address | `":9395"` | +| `scheduler.forceOverwriteDefaultScheduler` | Whether to force overwrite default scheduler | `true` | +| `scheduler.livenessProbe` | Whether to enable liveness probe | `false` | +| `scheduler.leaderElect` | Whether to enable leader election | `true` | +| `scheduler.replicas` | Number of replicas | `1` | + +### Kube Scheduler Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `scheduler.kubeScheduler.enabled` | Whether to run kube-scheduler container in scheduler pod | `true` | +| `scheduler.kubeScheduler.image.registry` | Kube scheduler image registry | `"registry.cn-hangzhou.aliyuncs.com"` | +| `scheduler.kubeScheduler.image.repository` | Kube scheduler image repository | `"google_containers/kube-scheduler"` | +| `scheduler.kubeScheduler.image.tag` | Kube scheduler image tag | `""` | +| `scheduler.kubeScheduler.image.pullPolicy` | Kube scheduler image pull policy | `IfNotPresent` | +| `scheduler.kubeScheduler.image.pullSecrets` | Kube scheduler image pull secrets | `[]` | +| `scheduler.kubeScheduler.extraNewArgs` | Extra new arguments | `["--config=/config/config.yaml", "-v=4"]` | +| `scheduler.kubeScheduler.extraArgs` | Extra arguments | `["--policy-config-file=/config/config.json", "-v=4"]` | + +### Extender Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `scheduler.extender.image.registry` | Scheduler extender image registry | `"docker.io"` | +| `scheduler.extender.image.repository` | Scheduler extender image repository | `"projecthami/hami"` | +| `scheduler.extender.image.tag` | Scheduler extender image tag | `""` | +| `scheduler.extender.image.pullPolicy` | Scheduler extender image pull policy | `IfNotPresent` | +| `scheduler.extender.image.pullSecrets` | Scheduler extender image pull secrets | `[]` | +| `scheduler.extender.extraArgs` | Scheduler extender extra arguments | `["--debug", "-v=4"]` | + +### Admission Webhook Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `scheduler.admissionWebhook.enabled` | Whether to enable admission webhook | `true` | +| `scheduler.admissionWebhook.customURL.enabled` | Whether to enable custom URL | `false` | +| `scheduler.admissionWebhook.customURL.host` | Custom URL host | `127.0.0.1` | +| `scheduler.admissionWebhook.customURL.port` | Custom URL port | `31998` | +| `scheduler.admissionWebhook.customURL.path` | Custom URL path | `/webhook` | +| `scheduler.admissionWebhook.reinvocationPolicy` | Reinvocation policy | `Never` | +| `scheduler.admissionWebhook.failurePolicy` | Failure policy | `Ignore` | + +### TLS Certificate Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `scheduler.certManager.enabled` | Whether to use cert-manager to generate self-signed certificates | `false` | +| `scheduler.patch.enabled` | Whether to use kube-webhook-certgen to generate self-signed certificates | `true` | +| `scheduler.patch.image.registry` | Certgen image registry | `"docker.io"` | +| `scheduler.patch.image.repository` | Certgen image repository | `"jettech/kube-webhook-certgen"` | +| `scheduler.patch.image.tag` | Certgen image tag | `"v1.5.2"` | +| `scheduler.patch.image.pullPolicy` | Certgen image pull policy | `IfNotPresent` | +| `scheduler.patch.imageNew.registry` | New certgen image registry | `"docker.io"` | +| `scheduler.patch.imageNew.repository` | New certgen image repository | `"liangjw/kube-webhook-certgen"` | +| `scheduler.patch.imageNew.tag` | New certgen image tag | `"v1.1.1"` | + +### Scheduler Service Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `scheduler.service.type` | Service type | `NodePort` | +| `scheduler.service.httpPort` | HTTP port | `443` | +| `scheduler.service.schedulerPort` | Scheduler NodePort | `31998` | +| `scheduler.service.monitorPort` | Monitor port | `31993` | +| `scheduler.service.monitorTargetPort` | Monitor target port | `9395` | + +## Device Plugin Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devicePlugin.image.registry` | Device plugin image registry | `"docker.io"` | +| `devicePlugin.image.repository` | Device plugin image repository | `"projecthami/hami"` | +| `devicePlugin.image.tag` | Device plugin image tag | `""` | +| `devicePlugin.image.pullPolicy` | Device plugin image pull policy | `IfNotPresent` | +| `devicePlugin.image.pullSecrets` | Device plugin image pull secrets | `[]` | + +### Monitor Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devicePlugin.monitor.image.registry` | Monitor image registry | `"docker.io"` | +| `devicePlugin.monitor.image.repository` | Monitor image repository | `"projecthami/hami"` | +| `devicePlugin.monitor.image.tag` | Monitor image tag | `""` | +| `devicePlugin.monitor.image.pullPolicy` | Monitor image pull policy | `IfNotPresent` | +| `devicePlugin.monitor.image.pullSecrets` | Monitor image pull secrets | `[]` | +| `devicePlugin.monitor.ctrPath` | Container path | `/usr/local/vgpu/containers` | +| `devicePlugin.monitor.extraArgs` | Monitor extra arguments | `["-v=4"]` | +| `devicePlugin.monitor.extraEnvs` | Monitor extra environments | `{}` | + +### Device Plugin Other Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devicePlugin.deviceSplitCount` | Integer type, default value: 10. Maximum number of tasks assigned to a single GPU device | `10` | +| `devicePlugin.deviceMemoryScaling` | Device memory scaling ratio | `1` | +| `devicePlugin.deviceCoreScaling` | Device core scaling ratio | `1` | +| `devicePlugin.runtimeClassName` | Runtime class name | `""` | +| `devicePlugin.createRuntimeClass` | Whether to create runtime class | `false` | +| `devicePlugin.migStrategy` | String type, "none" means ignore MIG functionality, "mixed" means allocate MIG devices through independent resources | `"none"` | +| `devicePlugin.disablecorelimit` | String type, "true" means disable core limit, "false" means enable core limit | `"false"` | +| `devicePlugin.passDeviceSpecsEnabled` | Whether to enable passing device specs | `false` | +| `devicePlugin.extraArgs` | Device plugin extra arguments | `["-v=4"]` | +| `devicePlugin.nodeConfiguration.config` | Node configuration for device plugin by json | An example of default configuration. | +| `devicePlugin.nodeConfiguration.externalConfigName` | Node configuration for device plugin by external congimap | `""` | +| `devicePlugin.extraEnvs` | Device plugin extra environments | `{}` | + +### Device Plugin Service Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devicePlugin.service.type` | Service type | `NodePort` | +| `devicePlugin.service.httpPort` | HTTP port | `31992` | + +### Device Plugin Deployment Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devicePlugin.pluginPath` | Plugin path | `/var/lib/kubelet/device-plugins` | +| `devicePlugin.libPath` | Library path | `/usr/local/vgpu` | +| `devicePlugin.nvidiaNodeSelector` | NVIDIA node selector | `{"gpu": "on"}` | +| `devicePlugin.updateStrategy.type` | Update strategy type | `RollingUpdate` | +| `devicePlugin.updateStrategy.rollingUpdate.maxUnavailable` | Maximum unavailable count | `1` | + +## Device Configuration + +### AWS Neuron +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.awsneuron.customresources` | Custom resources | `["aws.amazon.com/neuron", "aws.amazon.com/neuroncore"]` | + +### Kunlunxin +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.kunlun.enabled` | Whether to enable | `true` | +| `devices.kunlun.customresources` | Custom resources | `["kunlunxin.com/xpu"]` | + +### Mthreads +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.mthreads.enabled` | Whether to enable | `true` | +| `devices.mthreads.customresources` | Custom resources | `["mthreads.com/vgpu"]` | + +### NVIDIA +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.nvidia.gpuCorePolicy` | GPU core policy | `default` | +| `devices.nvidia.libCudaLogLevel` | CUDA library log level | `1` | + +### Huawei Ascend +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.ascend.enabled` | Whether to enable | `false` | +| `devices.ascend.image` | Image | `""` | +| `devices.ascend.imagePullPolicy` | Image pull policy | `IfNotPresent` | +| `devices.ascend.extraArgs` | Extra arguments | `[]` | +| `devices.ascend.nodeSelector` | Node selector | `{"ascend": "on"}` | +| `devices.ascend.tolerations` | Tolerations | `[]` | +| `devices.ascend.customresources` | Custom resources | `["huawei.com/Ascend910A", "huawei.com/Ascend910A-memory", ...]` | + +### Iluvatar +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.iluvatar.enabled` | Whether to enable | `false` | +| `devices.iluvatar.customresources` | Custom resources | `["iluvatar.ai/BI-V150-vgpu", "iluvatar.ai/BI-V150.vMem","iluvatar.ai/BI-V150.vCore", ...]` | diff --git a/packages/system/hami/charts/hami/charts/hami-dra/.helmignore b/packages/system/hami/charts/hami/charts/hami-dra/.helmignore new file mode 100644 index 00000000..898df488 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/.helmignore @@ -0,0 +1,24 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ + diff --git a/packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml b/packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml new file mode 100644 index 00000000..d01f678f --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml @@ -0,0 +1,18 @@ +apiVersion: v2 +appVersion: 0.1.0 +description: A Helm chart for HAMi DRA +home: https://github.com/Project-HAMi/HAMi-DRA +keywords: +- webhook +- kubernetes +- admission-controller +- hami +- dra +maintainers: +- name: hami-dra + url: https://github.com/Project-HAMi/HAMi-DRA +name: hami-dra +sources: +- https://github.com/Project-HAMi/HAMi-DRA +type: application +version: 0.1.0 diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/_commons.tpl b/packages/system/hami/charts/hami/charts/hami-dra/templates/_commons.tpl new file mode 100644 index 00000000..02fbed20 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/_commons.tpl @@ -0,0 +1,84 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* +Return the proper image name +{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" $) }} +*/}} +{{- define "common.images.image" -}} +{{- $registryName := .imageRoot.registry -}} +{{- $repositoryName := .imageRoot.repository -}} +{{- $tag := .imageRoot.tag | toString -}} +{{- if .global }} + {{- if .global.imageRegistry }} + {{- $registryName = .global.imageRegistry -}} + {{- end -}} +{{- end -}} +{{- if .tag }} + {{- if .tag.imageTag }} + {{- $tag = .tag.imageTag -}} + {{- end -}} +{{- end -}} +{{- if $registryName }} +{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- else -}} +{{- printf "%s:%s" $repositoryName $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead) +{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }} +*/}} +{{- define "common.images.pullSecrets" -}} + {{- $pullSecrets := list }} + + {{- if .global }} + {{- range .global.imagePullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- range .images -}} + {{- range .pullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- if (not (empty $pullSecrets)) }} +imagePullSecrets: + {{- range $pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end -}} + +{{/* +Renders a value that contains template. +Usage: +{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $) }} +*/}} +{{- define "common.tplvalues.render" -}} + {{- if typeIs "string" .value }} + {{- tpl .value .context }} + {{- else }} + {{- tpl (.value | toYaml) .context }} + {{- end }} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "common.names.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/_helpers.tpl b/packages/system/hami/charts/hami/charts/hami-dra/templates/_helpers.tpl new file mode 100644 index 00000000..99beec77 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/_helpers.tpl @@ -0,0 +1,73 @@ +{{- define "hami.dra.webhook.fullname" -}} +{{- printf "%s-%s" (include "common.names.fullname" .) "webhook" | trunc 63 | trimSuffix "-" -}} +{{- end }} + +{{- define "hami.dra.webhook.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.webhook.image "global" .Values.global) }} +{{- end -}} + +{{- define "hami.dra.webhook.imagePullSecrets" -}} +{{ include "common.images.pullSecrets" (dict "images" (list .Values.webhook.image) "global" .Values.global) }} +{{- end -}} + +{{- define "hami.dra.driver.nvidia.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.drivers.nvidia.image "global" .Values.global) }} +{{- end -}} + +{{- define "hami.dra.driver.nvidia.imagePullSecrets" -}} +{{ include "common.images.pullSecrets" (dict "images" (list .Values.drivers.nvidia.image) "global" .Values.global) }} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "hami-dra-webhook.labels" -}} +helm.sh/chart: {{ .Chart.Name }} +{{ include "hami-dra-webhook.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "hami-dra-webhook.selectorLabels" -}} +app.kubernetes.io/name: {{ .Release.Name }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: webhook +{{- end }} + +{{- define "hami.dra.monitor.fullname" -}} +{{- printf "%s-%s" (include "common.names.fullname" .) "monitor" | trunc 63 | trimSuffix "-" -}} +{{- end }} + +{{- define "hami.dra.monitor.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.monitor.image "global" .Values.global) }} +{{- end -}} + +{{- define "hami.dra.monitor.imagePullSecrets" -}} +{{ include "common.images.pullSecrets" (dict "images" (list .Values.monitor.image) "global" .Values.global) }} +{{- end -}} + +{{/* +Common labels for monitor +*/}} +{{- define "hami-dra-monitor.labels" -}} +helm.sh/chart: {{ .Chart.Name }} +{{ include "hami-dra-monitor.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels for monitor +*/}} +{{- define "hami-dra-monitor.selectorLabels" -}} +app.kubernetes.io/name: {{ .Release.Name }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: monitor +{{- end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/deviceclass.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/deviceclass.yaml new file mode 100644 index 00000000..bcb6d2c5 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/deviceclass.yaml @@ -0,0 +1,11 @@ +{{- if .Values.drivers.nvidia.enabled }} +apiVersion: resource.k8s.io/v1 +kind: DeviceClass +metadata: + name: hami-core-gpu.project-hami.io +spec: + selectors: + - cel: + expression: |- + device.driver == "hami-core-gpu.project-hami.io" && device.attributes["hami-core-gpu.project-hami.io"].type == "hami-gpu" +{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/nvidia-dra-driver-daemonset.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/nvidia-dra-driver-daemonset.yaml new file mode 100644 index 00000000..a3fb0d01 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/nvidia-dra-driver-daemonset.yaml @@ -0,0 +1,145 @@ +{{- if .Values.drivers.nvidia.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: hami-dra-driver-kubelet-plugin + namespace: {{ .Release.Namespace }} +spec: + revisionHistoryLimit: 10 + selector: + matchLabels: + hami-dra-driver-component: kubelet-plugin + template: + metadata: + labels: + app.kubernetes.io/instance: hami-dra-driver + app.kubernetes.io/name: hami-dra-driver + hami-dra-driver-component: kubelet-plugin + spec: + {{- include "hami.dra.driver.nvidia.imagePullSecrets" . | nindent 6 }} + initContainers: + - name: init-container + image: {{ include "hami.dra.driver.nvidia.image" . }} + securityContext: + privileged: true + command: [bash, /usr/bin/kubelet-plugin-prestart.sh] + env: + - name: NVIDIA_DRIVER_ROOT + value: {{ if .Values.drivers.nvidia.containerDriver }}/run/nvidia/driver{{ else }}/{{ end }} + - name: NVIDIA_VISIBLE_DEVICES + value: void + - name: KUBELET_REGISTRAR_DIRECTORY_PATH + value: /var/lib/kubelet/plugins_registry + - name: KUBELET_PLUGINS_DIRECTORY_PATH + value: /var/lib/kubelet/plugins + volumeMounts: + - name: driver-root-parent + mountPath: /driver-root-parent + readOnly: true + containers: + - args: + - |- + # Conditionally mask the params file to prevent this container from + # recreating any missing GPU device nodes. This is necessary, for + # example, when running under nvkind to limit the set GPUs governed + # by the plugin even though it has cgroup access to all of them. + if [ "${MASK_NVIDIA_DRIVER_PARAMS}" = "true" ]; then + cp /proc/driver/nvidia/params root/gpu-params + sed -i 's/^ModifyDeviceFiles: 1$/ModifyDeviceFiles: 0/' root/gpu-params + mount --bind root/gpu-params /proc/driver/nvidia/params + fi + hami-kubelet-plugin -v 6 + command: + - bash + - -c + env: + - name: MASK_NVIDIA_DRIVER_PARAMS + value: "" + - name: NVIDIA_DRIVER_ROOT + value: {{ if .Values.drivers.nvidia.containerDriver }}/run/nvidia/driver{{ else }}/{{ end }} + - name: NVIDIA_VISIBLE_DEVICES + value: void + - name: CDI_ROOT + value: /var/run/cdi + - name: NVIDIA_MIG_CONFIG_DEVICES + value: all + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: KUBELET_REGISTRAR_DIRECTORY_PATH + value: /var/lib/kubelet/plugins_registry + - name: KUBELET_PLUGINS_DIRECTORY_PATH + value: /var/lib/kubelet/plugins + - name: IMAGE_NAME + value: {{ include "hami.dra.driver.nvidia.image" . }} + lifecycle: + postStart: + exec: + command: ["/bin/bash", "-c", "/usr/bin/vgpu-init.sh /usr/local/vgpu/"] + image: {{ include "hami.dra.driver.nvidia.image" . }} + imagePullPolicy: {{ .Values.drivers.nvidia.image.pullPolicy | default "IfNotPresent" }} + name: gpus + securityContext: + privileged: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /var/lib/kubelet/plugins_registry + name: plugins-registry + - mountPath: /var/lib/kubelet/plugins + mountPropagation: Bidirectional + name: plugins + - mountPath: /var/run/cdi + name: cdi + - mountPath: /driver-root + mountPropagation: HostToContainer + name: driver-root + readOnly: true + - mountPath: /usr/local/vgpu + name: host-vgpu + - mountPath: /tmp + name: host-tmp + dnsPolicy: ClusterFirst + serviceAccount: hami-dra-driver-service-account + serviceAccountName: hami-dra-driver-service-account + volumes: + - hostPath: + path: /var/lib/kubelet/plugins_registry + type: "" + name: plugins-registry + - hostPath: + path: /var/lib/kubelet/plugins + type: "" + name: plugins + - hostPath: + path: /var/run/cdi + type: "" + name: cdi + - hostPath: + path: {{if .Values.drivers.nvidia.containerDriver }}/run/nvidia{{ else }}/{{ end }} + type: DirectoryOrCreate + name: driver-root-parent + - hostPath: + path: {{if .Values.drivers.nvidia.containerDriver }}/run/nvidia/driver{{ else }}/{{ end }} + type: DirectoryOrCreate + name: driver-root + - hostPath: + path: /dev + type: "" + name: host-dev + - hostPath: + path: /usr/local/vgpu + type: DirectoryOrCreate + name: host-vgpu + - hostPath: + path: /tmp + type: "" + name: host-tmp +{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/rbac.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/rbac.yaml new file mode 100644 index 00000000..435c37cc --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/rbac.yaml @@ -0,0 +1,110 @@ +{{- if .Values.drivers.nvidia.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hami-dra-driver-service-account + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +items: +- apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: hami-dra-driver-role + namespace: {{ .Release.Namespace }} + rules: + - apiGroups: + - apps + resources: + - daemonsets + - deployments + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +kind: List +metadata: + resourceVersion: "" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: hami-dra-driver-role-binding + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: hami-dra-driver-role +subjects: +- kind: ServiceAccount + name: hami-dra-driver-service-account + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: hami-dra-driver-clusterrole +rules: +- apiGroups: + - resource.k8s.io + resources: + - resourceclaims + verbs: + - get + - list + - watch +- apiGroups: + - resource.k8s.io + resources: + - resourceclaimtemplates + verbs: + - get + - list + - watch + - create + - update + - delete +- apiGroups: + - resource.k8s.io + resources: + - resourceslices + verbs: + - get + - list + - watch + - create + - update + - delete +- apiGroups: + - resource.k8s.io + resources: + - resourceclaims/status + verbs: + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: hami-dra-driver-clusterrole-binding-hami-dra-driver +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hami-dra-driver-clusterrole +subjects: +- kind: ServiceAccount + name: hami-dra-driver-service-account + namespace: {{ .Release.Namespace }} +{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrole.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrole.yaml new file mode 100644 index 00000000..45c24863 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrole.yaml @@ -0,0 +1,11 @@ +{{- if .Values.monitor.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "hami.dra.monitor.fullname" . }} +rules: +- apiGroups: ["resource.k8s.io"] + resources: ["resourceslices", "resourceclaims"] + verbs: ["get", "list", "watch"] +{{- end }} + diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrolebinding.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrolebinding.yaml new file mode 100644 index 00000000..766d3098 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrolebinding.yaml @@ -0,0 +1,15 @@ +{{- if .Values.monitor.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami.dra.monitor.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "hami.dra.monitor.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "hami.dra.monitor.fullname" . }} + namespace: {{ .Release.Namespace }} +{{- end }} + diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-deployment.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-deployment.yaml new file mode 100644 index 00000000..b544b376 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-deployment.yaml @@ -0,0 +1,62 @@ +{{- if .Values.monitor.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "hami.dra.monitor.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "hami.dra.monitor.fullname" . }} + {{- include "hami-dra-monitor.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.monitor.replicas | default 1 }} + selector: + matchLabels: + {{- include "hami-dra-monitor.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "hami-dra-monitor.selectorLabels" . | nindent 8 }} + spec: + {{- include "hami.dra.monitor.imagePullSecrets" . | nindent 6 }} + serviceAccountName: {{ include "hami.dra.monitor.fullname" . }} + containers: + - name: monitor + image: {{ include "hami.dra.monitor.image" . }} + imagePullPolicy: {{ .Values.monitor.image.pullPolicy | default "IfNotPresent" }} + command: + - /bin/monitor + - --v={{ .Values.monitor.logLevel | default 2 }} + - --metrics-bind-address={{ .Values.monitor.metricsBindAddress | default ":8080" }} + - --health-probe-bind-address={{ .Values.monitor.healthProbeBindAddress | default ":8000" }} + - --kube-api-qps={{ .Values.monitor.kubeAPIQPS | default 40.0 }} + - --kube-api-burst={{ .Values.monitor.kubeAPIBurst | default 60 }} + - --collect-interval={{ .Values.monitor.collectInterval | default "30s" }} + ports: + - containerPort: 8080 + name: metrics + protocol: TCP + - containerPort: 8000 + name: health + protocol: TCP + {{- if .Values.monitor.resources }} + resources: + {{- toYaml .Values.monitor.resources | nindent 12 }} + {{- end }} + livenessProbe: + httpGet: + path: /healthz + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8000 + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 +{{- end }} + diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-service.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-service.yaml new file mode 100644 index 00000000..d740cc38 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-service.yaml @@ -0,0 +1,26 @@ +{{- if .Values.monitor.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "hami.dra.monitor.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "hami-dra-monitor.selectorLabels" . | nindent 4 }} +spec: + type: {{ .Values.monitor.service.type | default "ClusterIP" }} + selector: + {{- include "hami-dra-monitor.selectorLabels" . | nindent 4 }} + ports: + - port: 8080 + targetPort: 8080 + name: metrics + protocol: TCP + {{- if and (eq (.Values.monitor.service.type | default "ClusterIP") "NodePort") .Values.monitor.service.nodePort.metrics }} + nodePort: {{ .Values.monitor.service.nodePort.metrics }} + {{- end }} + - port: 8000 + targetPort: 8000 + name: health + protocol: TCP +{{- end }} + diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-serviceaccount.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-serviceaccount.yaml new file mode 100644 index 00000000..c61ad454 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-serviceaccount.yaml @@ -0,0 +1,8 @@ +{{- if .Values.monitor.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "hami.dra.monitor.fullname" . }} + namespace: {{ .Release.Namespace }} +{{- end }} + diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-manager.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-manager.yaml new file mode 100644 index 00000000..dd31c71d --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-manager.yaml @@ -0,0 +1,29 @@ +{{- if .Values.certs.certManager.enabled }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ .Release.Name }}-dra-webhook-serving-cert + namespace: {{ .Release.Namespace }} + labels: + {{- include "hami-dra-webhook.labels" . | nindent 4 }} +spec: + dnsNames: + - {{ .Release.Name }}-dra-webhook.{{ .Release.Namespace }}.svc + - {{ .Release.Name }}-dra-webhook.{{ .Release.Namespace }}.svc.cluster.local + issuerRef: + kind: Issuer + name: {{ .Release.Name }}-dra-webhook-selfsigned-issuer + secretName: {{ .Release.Name }}-dra-webhook-tls + privateKey: + rotationPolicy: {{ .Values.certs.certManager.privateKeyRotationPolicy | default "Never" }} +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ .Release.Name }}-dra-webhook-selfsigned-issuer + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/component: hami-dra-webhook +spec: + selfSigned: {} +{{- end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-secret.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-secret.yaml new file mode 100644 index 00000000..eadbb517 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-secret.yaml @@ -0,0 +1,13 @@ +{{- if eq .Values.certs.mode "custom" }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "hami-dra-webhook.tlsSecretName" . }} + namespace: {{ .Release.Namespace }} +type: kubernetes.io/tls +data: + tls.crt: | + {{- .Values.certs.custom.crt | b64enc | indent 4 }} + tls.key: | + {{- .Values.certs.custom.key | b64enc | indent 4 }} +{{- end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/device-config.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/device-config.yaml new file mode 100644 index 00000000..97283f3e --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/device-config.yaml @@ -0,0 +1,394 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "hami.dra.webhook.fullname" . }}-device-config + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-dra-webhook.selectorLabels" . | nindent 4 }} +data: + device-config.yaml: |- + {{- if .Files.Glob "files/device-config.yaml" }} + {{- .Files.Get "files/device-config.yaml" | nindent 4}} + {{- else }} + nvidia: + resourceCountName: {{ .Values.resourceName }} + resourceMemoryName: {{ .Values.resourceMem }} + resourceMemoryPercentageName: {{ .Values.resourceMemPercentage }} + resourceCoreName: {{ .Values.resourceCores }} + resourcePriorityName: {{ .Values.resourcePriority }} + overwriteEnv: false + defaultMemory: 0 + defaultCores: 0 + defaultGPUNum: 1 + knownMigGeometries: + - models: [ "A30" ] + allowedGeometries: + - + - name: 1g.6gb + core: 25 + memory: 6144 + count: 4 + - + - name: 2g.12gb + core: 50 + memory: 12288 + count: 2 + - + - name: 4g.24gb + core: 100 + memory: 24576 + count: 1 + - models: [ "A100-SXM4-40GB", "A100-40GB-PCIe", "A100-PCIE-40GB"] + allowedGeometries: + - + - name: 1g.5gb + core: 14 + memory: 5120 + count: 7 + - + - name: 1g.5gb + core: 14 + memory: 5120 + count: 1 + - name: 2g.10gb + core: 28 + memory: 10240 + count: 3 + - + - name: 3g.20gb + core: 42 + memory: 20480 + count: 2 + - + - name: 7g.40gb + core: 100 + memory: 40960 + count: 1 + - models: [ "A100-SXM4-80GB", "A100-80GB-PCIe", "A100-PCIE-80GB"] + allowedGeometries: + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 7 + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 1 + - name: 2g.20gb + core: 28 + memory: 20480 + count: 3 + - + - name: 3g.40gb + core: 42 + memory: 40960 + count: 2 + - + - name: 7g.79gb + core: 100 + memory: 80896 + count: 1 + - models: [ "H100-PCIE-80GB", "H100-SXM5-80GB"] + allowedGeometries: + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 7 + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 1 + - name: 2g.20gb + core: 28 + memory: 20480 + count: 3 + - + - name: 3g.40gb + core: 42 + memory: 40960 + count: 2 + - + - name: 7g.80gb + core: 100 + memory: 81920 + count: 1 + - models: [ "H100-PCIE-94GB", "H100-SXM5-94GB"] + allowedGeometries: + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 7 + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 1 + - name: 2g.24gb + core: 28 + memory: 24576 + count: 3 + - + - name: 3g.47gb + core: 42 + memory: 48128 + count: 2 + - + - name: 7g.94gb + core: 100 + memory: 96256 + count: 1 + - models: [ "H20", "H100 on GH200"] + allowedGeometries: + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 7 + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 1 + - name: 2g.24gb + core: 28 + memory: 24576 + count: 3 + - + - name: 3g.48gb + core: 42 + memory: 49152 + count: 2 + - + - name: 7g.96gb + core: 100 + memory: 98304 + count: 1 + - models: [ "H200 NVL", "H200-SXM5"] + allowedGeometries: + - + - name: 1g.18gb + core: 14 + memory: 18432 + count: 7 + - + - name: 1g.18gb + core: 14 + memory: 18432 + count: 1 + - name: 2g.35gb + core: 28 + memory: 35840 + count: 3 + - + - name: 3g.71gb + core: 42 + memory: 72704 + count: 2 + - + - name: 7g.141gb + core: 100 + memory: 144384 + count: 1 + - models: [ "B200" ] + allowedGeometries: + - + - name: 1g.23gb + core: 14 + memory: 23552 + count: 7 + - + - name: 1g.23gb + core: 14 + memory: 23552 + count: 1 + - name: 2g.45gb + core: 28 + memory: 46080 + count: 3 + - + - name: 3g.90gb + core: 42 + memory: 92160 + count: 2 + - + - name: 7g.180gb + core: 100 + memory: 184320 + count: 1 + cambricon: + resourceCountName: {{ .Values.mluResourceName }} + resourceMemoryName: {{ .Values.mluResourceMem }} + resourceCoreName: {{ .Values.mluResourceCores }} + hygon: + resourceCountName: {{ .Values.dcuResourceName }} + resourceMemoryName: {{ .Values.dcuResourceMem }} + resourceCoreName: {{ .Values.dcuResourceCores }} + metax: + resourceCountName: "metax-tech.com/gpu" + resourceVCountName: {{ .Values.metaxResourceName }} + resourceVMemoryName: {{ .Values.metaxResourceMem }} + resourceVCoreName: {{ .Values.metaxResourceCore }} + sgpuTopologyAware: {{ .Values.metaxsGPUTopologyAware }} + enflame: + resourceNameGCU: "enflame.com/gcu" + resourceNameVGCU: {{ .Values.enflameResourceNameVGCU }} + resourceNameVGCUPercentage: {{ .Values.enflameResourceNameVGCUPercentage }} + mthreads: + resourceCountName: "mthreads.com/vgpu" + resourceMemoryName: "mthreads.com/sgpu-memory" + resourceCoreName: "mthreads.com/sgpu-core" + iluvatars: + - chipName: MR-V100 + commonWord: MR-V100 + resourceCountName: iluvatar.ai/MR-V100-vgpu + resourceMemoryName: iluvatar.ai/MR-V100.vMem + resourceCoreName: iluvatar.ai/MR-V100.vCore + - chipName: MR-V50 + commonWord: MR-V50 + resourceCountName: iluvatar.ai/MR-V50-vgpu + resourceMemoryName: iluvatar.ai/MR-V50.vMem + resourceCoreName: iluvatar.ai/MR-V50.vCore + - chipName: BI-V150 + commonWord: BI-V150 + resourceCountName: iluvatar.ai/BI-V150-vgpu + resourceMemoryName: iluvatar.ai/BI-V150.vMem + resourceCoreName: iluvatar.ai/BI-V150.vCore + - chipName: BI-V100 + commonWord: BI-V100 + resourceCountName: iluvatar.ai/BI-V100-vgpu + resourceMemoryName: iluvatar.ai/BI-V100.vMem + resourceCoreName: iluvatar.ai/BI-V100.vCore + kunlun: + resourceCountName: {{ .Values.kunlunResourceName }} + resourceVCountName: {{ .Values.kunlunResourceVCountName }} + resourceVMemoryName: {{ .Values.kunlunResourceVMemoryName }} + awsneuron: + resourceCountName: "aws.amazon.com/neuron" + resourceCoreName: "aws.amazon.com/neuroncore" + amd: + resourceCountName: "amd.com/gpu" + vnpus: + - chipName: 910A + commonWord: Ascend910A + resourceName: huawei.com/Ascend910A + resourceMemoryName: huawei.com/Ascend910A-memory + memoryAllocatable: 32768 + memoryCapacity: 32768 + aiCore: 30 + templates: + - name: vir02 + memory: 2184 + aiCore: 2 + - name: vir04 + memory: 4369 + aiCore: 4 + - name: vir08 + memory: 8738 + aiCore: 8 + - name: vir16 + memory: 17476 + aiCore: 16 + - chipName: 910B2 + commonWord: Ascend910B2 + resourceName: huawei.com/Ascend910B2 + resourceMemoryName: huawei.com/Ascend910B2-memory + memoryAllocatable: 65536 + memoryCapacity: 65536 + aiCore: 24 + aiCPU: 6 + templates: + - name: vir03_1c_8g + memory: 8192 + aiCore: 3 + aiCPU: 1 + - name: vir06_1c_16g + memory: 16384 + aiCore: 6 + aiCPU: 1 + - name: vir12_3c_32g + memory: 32768 + aiCore: 12 + aiCPU: 3 + - chipName: 910B3 + commonWord: Ascend910B3 + resourceName: huawei.com/Ascend910B3 + resourceMemoryName: huawei.com/Ascend910B3-memory + memoryAllocatable: 65536 + memoryCapacity: 65536 + aiCore: 20 + aiCPU: 7 + templates: + - name: vir05_1c_16g + memory: 16384 + aiCore: 5 + aiCPU: 1 + - name: vir10_3c_32g + memory: 32768 + aiCore: 10 + aiCPU: 3 + - chipName: 910B4-1 + commonWord: Ascend910B4-1 + resourceName: huawei.com/Ascend910B4-1 + resourceMemoryName: huawei.com/Ascend910B4-1-memory + memoryAllocatable: 65536 + memoryCapacity: 65536 + aiCore: 20 + aiCPU: 7 + templates: + # NOTE: Names of vnpu template for 910B4-1 are fixed by Ascend runtime and must not be changed. + # The memory is used for scheduling so the correct values must be set. + # Template vir05_1c_8g actually provides 16GB memory, + - name: vir05_1c_8g + memory: 16384 + aiCore: 5 + aiCPU: 1 + # Template vir10_3c_16g actually provides 32GB memory + - name: vir10_3c_16g + memory: 32768 + aiCore: 10 + aiCPU: 3 + - chipName: 910B4 + commonWord: Ascend910B4 + resourceName: huawei.com/Ascend910B4 + resourceMemoryName: huawei.com/Ascend910B4-memory + memoryAllocatable: 32768 + memoryCapacity: 32768 + aiCore: 20 + aiCPU: 7 + templates: + - name: vir05_1c_8g + memory: 8192 + aiCore: 5 + aiCPU: 1 + - name: vir10_3c_16g + memory: 16384 + aiCore: 10 + aiCPU: 3 + - chipName: 310P3 + commonWord: Ascend310P + resourceName: huawei.com/Ascend310P + resourceMemoryName: huawei.com/Ascend310P-memory + memoryAllocatable: 21527 + memoryCapacity: 24576 + aiCore: 8 + aiCPU: 7 + templates: + - name: vir01 + memory: 3072 + aiCore: 1 + aiCPU: 1 + - name: vir02 + memory: 6144 + aiCore: 2 + aiCPU: 2 + - name: vir04 + memory: 12288 + aiCore: 4 + aiCPU: 4 + {{ end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/mutatingwebhookconfiguration.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/mutatingwebhookconfiguration.yaml new file mode 100644 index 00000000..c763b6aa --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/mutatingwebhookconfiguration.yaml @@ -0,0 +1,28 @@ +{{- if .Values.webhook.config.mutating.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: {{ .Release.Name }}-mutatingwebhookconfiguration + annotations: + cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ .Release.Name }}-dra-webhook-serving-cert +webhooks: + - name: mutate.hami.io + admissionReviewVersions: ["v1"] + clientConfig: + service: + name: {{ .Release.Name }}-dra-webhook + namespace: {{ .Release.Namespace }} + path: /mutate + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - pods + scope: '*' + sideEffects: None + timeoutSeconds: 10 +{{- end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/validatingwebhookconfiguration.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/validatingwebhookconfiguration.yaml new file mode 100644 index 00000000..acdcbb96 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/validatingwebhookconfiguration.yaml @@ -0,0 +1,29 @@ +{{- if .Values.webhook.config.validating.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: {{ .Release.Name }}-validatingwebhookconfiguration + annotations: + cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ .Release.Name }}-dra-webhook-serving-cert +webhooks: + - name: validate.hami.io + admissionReviewVersions: ["v1"] + clientConfig: + service: + name: {{ .Release.Name }}-dra-webhook + namespace: {{ .Release.Namespace }} + path: /validate + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - DELETE + resources: + - pods + scope: '*' + sideEffects: None + timeoutSeconds: 10 + failurePolicy: Ignore +{{- end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrole.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrole.yaml new file mode 100644 index 00000000..28916a36 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrole.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "hami.dra.webhook.fullname" . }} +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] +- apiGroups: ["admissionregistration.k8s.io"] + resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"] + verbs: ["get", "list", "watch"] +- apiGroups: ["resource.k8s.io"] + resources: ["resourceclaims"] + verbs: ["*"] diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrolebinding.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrolebinding.yaml new file mode 100644 index 00000000..daad52f2 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrolebinding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami.dra.webhook.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "hami.dra.webhook.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "hami.dra.webhook.fullname" . }} + namespace: {{ .Release.Namespace }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-deployment.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-deployment.yaml new file mode 100644 index 00000000..8e3cf90c --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-deployment.yaml @@ -0,0 +1,59 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "hami.dra.webhook.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "hami.dra.webhook.fullname" . }} + {{- include "hami-dra-webhook.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "hami-dra-webhook.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "hami-dra-webhook.selectorLabels" . | nindent 8 }} + spec: + {{- include "hami.dra.webhook.imagePullSecrets" . | nindent 6 }} + serviceAccountName: {{ include "hami.dra.webhook.fullname" . }} + containers: + - name: webhook + image: {{ include "hami.dra.webhook.image" . }} + imagePullPolicy: {{ .Values.webhook.image.pullPolicy | default "IfNotPresent" }} + command: + - /bin/webhook + - --v=5 + - --bind-address=0.0.0.0 + - --secure-port=8443 + - --cert-dir=/tls + - --tls-cert-file-name=tls.crt + - --tls-private-key-file-name=tls.key + - --metrics-bind-address=:8080 + - --health-probe-bind-address=:8000 + - --device-config-file=/device-config.yaml + ports: + - containerPort: 8443 + name: webhook + protocol: TCP + - containerPort: 8080 + name: metrics + protocol: TCP + - containerPort: 8000 + name: health + protocol: TCP + volumeMounts: + - name: device-config + mountPath: /device-config.yaml + subPath: device-config.yaml + - name: tls-config + mountPath: /tls + readOnly: true + volumes: + - name: device-config + configMap: + name: {{ include "hami.dra.webhook.fullname" . }}-device-config + - name: tls-config + secret: + secretName: {{ if .Values.certs.certManager.enabled }}{{ .Release.Name }}-dra-webhook-tls{{ else }}{{ .Release.Name }}-tls{{ end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-service.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-service.yaml new file mode 100644 index 00000000..290174ec --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-dra-webhook + namespace: {{ .Release.Namespace }} + labels: + {{- include "hami-dra-webhook.selectorLabels" . | nindent 4 }} +spec: + type: "ClusterIP" + selector: + {{- include "hami-dra-webhook.selectorLabels" . | nindent 4 }} + ports: + - port: 443 + targetPort: 8443 + name: webhook diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-serviceaccount.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-serviceaccount.yaml new file mode 100644 index 00000000..b0dc2fb2 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-serviceaccount.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "hami.dra.webhook.fullname" . }} + namespace: {{ .Release.Namespace }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/values.yaml b/packages/system/hami/charts/hami/charts/hami-dra/values.yaml new file mode 100644 index 00000000..ebbbfed7 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/values.yaml @@ -0,0 +1,166 @@ +## @section Global configuration +## @param global.imageRegistry Global Docker image registry +## @param global.imagePullSecrets Global Docker image pull secrets +global: + ## @param global.imageRegistry Global Docker image registry + imageRegistry: "" + ## E.g. + ## imagePullSecrets: + ## - myRegistryKeySecretName + ## @param global.imagePullSecrets Global Docker image pull secrets + imagePullSecrets: [] + +#Nvidia GPU Parameters +resourceName: "nvidia.com/gpu" +resourceMem: "nvidia.com/gpumem" +resourceMemPercentage: "nvidia.com/gpumem-percentage" +resourceCores: "nvidia.com/gpucores" +resourcePriority: "nvidia.com/priority" + +#MLU Parameters +mluResourceName: "cambricon.com/vmlu" +mluResourceMem: "cambricon.com/mlu.smlu.vmemory" +mluResourceCores: "cambricon.com/mlu.smlu.vcore" + +#Hygon DCU Parameters +dcuResourceName: "hygon.com/dcunum" +dcuResourceMem: "hygon.com/dcumem" +dcuResourceCores: "hygon.com/dcucores" + +#Metax sGPU Parameters +metaxResourceName: "metax-tech.com/sgpu" +metaxResourceCore: "metax-tech.com/vcore" +metaxResourceMem: "metax-tech.com/vmemory" +metaxsGPUTopologyAware: "false" + +#Enflame VGCU Parameters +enflameResourceNameVGCU: "enflame.com/vgcu" +enflameResourceNameVGCUPercentage: "enflame.com/vgcu-percentage" + +#Kunlun XPU Parameters +kunlunResourceName: "kunlunxin.com/xpu" +kunlunResourceVCountName: "kunlunxin.com/vxpu" +kunlunResourceVMemoryName: "kunlunxin.com/vxpu-memory" + +# Webhook deployment configuration +webhook: + image: + registry: ghcr.io + repository: project-hami/hami-dra-webhook + tag: "v0.1.0" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## @param resources controllerManager resource requests and limits + resources: + # If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 128Mi + ## @param nodeSelector controllerManager node labels for pod assignment + args: + webhookName: "" # Default: {{ .Release.Name }}-hami-dra-webhook + secretName: "" # Default: {{ .Release.Name }}-hami-dra-webhook-tls + # Webhook configuration + config: + mutating: + enabled: true + validating: + enabled: true + +# Monitor deployment configuration +monitor: + enabled: true + replicas: 1 + image: + registry: ghcr.io + repository: project-hami/hami-dra-monitor + tag: "v0.1.0" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## @param resources monitor resource requests and limits + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 128Mi + ## Monitor configuration + logLevel: 2 + metricsBindAddress: ":8080" + healthProbeBindAddress: ":8000" + kubeAPIQPS: 40.0 + kubeAPIBurst: 60 + collectInterval: "30s" + ## Monitor service configuration + service: + ## Service type: ClusterIP, NodePort, or LoadBalancer + ## Defaults to ClusterIP + type: ClusterIP + ## NodePort configuration (only used when type is NodePort) + nodePort: + ## Metrics port (NodePort). If not specified, Kubernetes will assign a random port + metrics: "" + +# Certificate configuration +certs: + # Custom certificate (used when mode is "custom") + custom: + crt: "" + key: "" + # Cert-manager configuration (used when mode is "cert-manager") + certManager: + enabled: true + # Private key rotation policy: "Never" or "Always" + # In cert-manager >= v1.18.0, the default changed from "Never" to "Always" + # Setting to "Never" avoids frequent certificate rotation for webhooks + privateKeyRotationPolicy: "Never" + issuer: + clusterIssuerName: "" # Required when type is "clusterIssuer" + +# DRA Drivers configuration +drivers: + nvidia: + enabled: true + # If you are using gpu driver on host, you need to set this to false + containerDriver: true + image: + registry: ghcr.io + repository: projecthami/k8s-dra-driver + tag: "v0.0.1-dev" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] diff --git a/packages/system/hami/charts/hami/templates/NOTES.txt b/packages/system/hami/charts/hami/templates/NOTES.txt new file mode 100644 index 00000000..15bb1218 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/NOTES.txt @@ -0,0 +1,3 @@ +** Please be patient while the chart is being deployed ** +Resource name: {{ .Values.resourceName }} + diff --git a/packages/system/hami/charts/hami/templates/_commons.tpl b/packages/system/hami/charts/hami/templates/_commons.tpl new file mode 100644 index 00000000..78a262c9 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/_commons.tpl @@ -0,0 +1,49 @@ +{{/* +Return the proper image name +{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" $) }} +*/}} +{{- define "common.images.image" -}} +{{- $registryName := .imageRoot.registry -}} +{{- $repositoryName := .imageRoot.repository -}} +{{- $tag := .imageRoot.tag | toString -}} +{{- if .global }} + {{- if .global.imageRegistry }} + {{- $registryName = .global.imageRegistry -}} + {{- end -}} +{{- end -}} +{{- if .tag }} + {{- $tag = .tag | toString -}} +{{- end -}} +{{- if $registryName }} +{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- else -}} +{{- printf "%s:%s" $repositoryName $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead) +{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }} +*/}} +{{- define "common.images.pullSecrets" -}} + {{- $pullSecrets := list }} + + {{- if .global }} + {{- range .global.imagePullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- range .images -}} + {{- range .pullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- if (not (empty $pullSecrets)) }} +imagePullSecrets: + {{- range $pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end -}} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/_helpers.tpl b/packages/system/hami/charts/hami/templates/_helpers.tpl new file mode 100644 index 00000000..ffcc61da --- /dev/null +++ b/packages/system/hami/charts/hami/templates/_helpers.tpl @@ -0,0 +1,163 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "hami-vgpu.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "hami-vgpu.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "hami-vgpu.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +The app name for Scheduler +*/}} +{{- define "hami-vgpu.scheduler" -}} +{{- printf "%s-scheduler" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +The app name for DevicePlugin +*/}} +{{- define "hami-vgpu.device-plugin" -}} +{{- printf "%s-device-plugin" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* + The app name for MockDevicePlugin + */}} +{{- define "hami-vgpu.mock-device-plugin" -}} +{{- printf "%s-mock-device-plugin" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +The tls secret name for Scheduler +*/}} +{{- define "hami-vgpu.scheduler.tls" -}} +{{- printf "%s-scheduler-tls" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +The webhook name +*/}} +{{- define "hami-vgpu.scheduler.webhook" -}} +{{- printf "%s-webhook" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "hami-vgpu.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "hami-vgpu.labels" -}} +helm.sh/chart: {{ include "hami-vgpu.chart" . }} +{{ include "hami-vgpu.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "hami-vgpu.selectorLabels" -}} +app.kubernetes.io/name: {{ include "hami-vgpu.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + + +{{/* + Resolve the tag for kubeScheduler. +*/}} +{{- define "resolvedKubeSchedulerTag" -}} +{{- if .Values.scheduler.kubeScheduler.image.tag }} +{{- .Values.scheduler.kubeScheduler.image.tag | trim -}} +{{- else }} +{{- include "strippedKubeVersion" . | trim -}} +{{- end }} +{{- end }} + +{{/* + Return the stripped Kubernetes version string by removing extra parts after semantic version number. + v1.31.1+k3s1 -> v1.31.1 + v1.30.8-eks-2d5f260 -> v1.30.8 + v1.31.1 -> v1.31.1 +*/}} +{{- define "strippedKubeVersion" -}} +{{ regexReplaceAll "^(v[0-9]+\\.[0-9]+\\.[0-9]+)(.*)$" .Capabilities.KubeVersion.Version "$1" }} +{{- end -}} + +{{- define "hami.scheduler.kubeScheduler.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.kubeScheduler.image "global" .Values.global "tag" (include "resolvedKubeSchedulerTag" .)) }} +{{- end -}} + +{{- define "hami.scheduler.extender.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.extender.image "global" .Values.global "tag" .Values.global.imageTag) }} +{{- end -}} + +{{- define "hami.devicePlugin.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.devicePlugin.image "global" .Values.global "tag" .Values.global.imageTag) }} +{{- end -}} + +{{- define "hami.mockDevicePlugin.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.mockDevicePlugin.image "global" .Values.global "tag" .Values.mockDevicePlugin.tag) }} +{{- end -}} + +{{- define "hami.devicePlugin.monitor.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.devicePlugin.monitor.image "global" .Values.global "tag" .Values.global.imageTag) }} +{{- end -}} + +{{- define "hami.scheduler.patch.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.patch.image "global" .Values.global) }} +{{- end -}} + +{{- define "hami.scheduler.patch.new.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.patch.imageNew "global" .Values.global) }} +{{- end -}} + +{{- define "hami.scheduler.extender.imagePullSecrets" -}} +{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.extender.image) "global" .Values.global) }} +{{- end -}} + +{{- define "hami.devicePlugin.imagePullSecrets" -}} +{{ include "common.images.pullSecrets" (dict "images" (list .Values.devicePlugin.image) "global" .Values.global) }} +{{- end -}} + +{{- define "hami.scheduler.patch.imagePullSecrets" -}} +{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.patch.image) "global" .Values.global) }} +{{- end -}} + +{{- define "hami.scheduler.patch.new.imagePullSecrets" -}} +{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.patch.imageNew) "global" .Values.global) }} +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml b/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml new file mode 100644 index 00000000..b36347f2 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml @@ -0,0 +1,13 @@ +{{- if and .Values.devicePlugin.enabled (not .Values.devicePlugin.nodeConfiguration.externalConfigName) (not .Values.dra.enabled) -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "hami-vgpu.device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-device-plugin + {{- include "hami-vgpu.labels" . | nindent 4 }} +data: + config.json: | +{{- .Values.devicePlugin.nodeConfiguration.config | nindent 4 }} +{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/device-plugin/daemonsetmock.yaml b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetmock.yaml new file mode 100644 index 00000000..a0dc4ff3 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetmock.yaml @@ -0,0 +1,55 @@ +{{- if .Values.mockDevicePlugin.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "hami-vgpu.mock-device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +spec: + selector: + matchLabels: + app.kubernetes.io/component: hami-mock-device-plugin + {{- include "hami-vgpu.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" + labels: + app.kubernetes.io/component: hami-mock-device-plugin + {{- include "hami-vgpu.selectorLabels" . | nindent 8 }} + spec: + serviceAccountName: {{ include "hami-vgpu.mock-device-plugin" . }} + tolerations: + - key: CriticalAddonsOnly + operator: Exists + containers: + - image: {{ include "hami.mockDevicePlugin.image" . }} + imagePullPolicy: {{ .Values.mockDevicePlugin.image.pullPolicy }} + name: hami-mock-dp-cntr + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + command: + - ./k8s-device-plugin + - -v=5 + - --device-config-file=/device-config.yaml + volumeMounts: + - name: dp + mountPath: /var/lib/kubelet/device-plugins + - name: sys + mountPath: /sys + - name: device-config + mountPath: /device-config.yaml + subPath: device-config.yaml + volumes: + - name: dp + hostPath: + path: /var/lib/kubelet/device-plugins + - name: sys + hostPath: + path: /sys + - name: device-config + configMap: + name: {{ include "hami-vgpu.scheduler" . }}-device +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml new file mode 100644 index 00000000..577dc0f9 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml @@ -0,0 +1,262 @@ +{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "hami-vgpu.device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-device-plugin + {{- include "hami-vgpu.labels" . | nindent 4 }} + {{- with .Values.global.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.global.annotations }} + annotations: {{ toYaml .Values.global.annotations | nindent 4}} + {{- end }} +spec: + updateStrategy: + {{- with .Values.devicePlugin.updateStrategy }} + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/component: hami-device-plugin + {{- include "hami-vgpu.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app.kubernetes.io/component: hami-device-plugin + hami.io/webhook: ignore + {{- include "hami-vgpu.selectorLabels" . | nindent 8 }} + annotations: + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + checksum/hami-scheduler-newversion-config: {{ include (print $.Template.BasePath "/scheduler/configmapnew.yaml") . | sha256sum }} + {{- else }} + checksum/hami-scheduler-config: {{ include (print $.Template.BasePath "/scheduler/configmap.yaml") . | sha256sum }} + {{- end }} + checksum/hami-scheduler-device-config: {{ include (print $.Template.BasePath "/scheduler/device-configmap.yaml") . | sha256sum }} + {{- if .Values.devicePlugin.podAnnotations }} + {{- toYaml .Values.devicePlugin.podAnnotations | nindent 8 }} + {{- end }} + spec: + {{- if .Values.devicePlugin.runtimeClassName }} + runtimeClassName: {{ .Values.devicePlugin.runtimeClassName }} + {{- end }} + serviceAccountName: {{ include "hami-vgpu.device-plugin" . }} + priorityClassName: system-node-critical + hostPID: true + hostNetwork: true + {{- include "hami.devicePlugin.imagePullSecrets" . | nindent 6 }} + {{- if .Values.devicePlugin.gpuOperatorToolkitReady.enabled }} + initContainers: + - name: toolkit-validation + image: {{ include "hami.devicePlugin.image" . }} + imagePullPolicy: {{ .Values.devicePlugin.image.pullPolicy }} + securityContext: + privileged: true + runAsUser: 0 + command: ["sh", "-c"] + args: + - | + echo "Waiting for NVIDIA Toolkit to be ready..." + until [ -f {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }}/toolkit-ready ]; do + echo "Waiting for {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }}/toolkit-ready..." + sleep 5 + done + echo "NVIDIA Toolkit is ready!" + volumeMounts: + - name: nvidia-validations + mountPath: {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath | quote }} + mountPropagation: HostToContainer + readOnly: true + {{- end }} + containers: + - name: device-plugin + image: {{ include "hami.devicePlugin.image" . }} + imagePullPolicy: {{ .Values.devicePlugin.image.pullPolicy }} + lifecycle: + postStart: + exec: + command: ["/bin/sh","-c", {{ printf "/k8s-vgpu/bin/vgpu-init.sh %s/vgpu/" .Values.global.gpuHookPath | quote }}] + command: + - nvidia-device-plugin + - --config-file=/device-config.yaml + - --mig-strategy={{ .Values.devicePlugin.migStrategy }} + - --disable-core-limit={{ .Values.devicePlugin.disablecorelimit }} + {{- range .Values.devicePlugin.extraArgs }} + - {{ . }} + {{- end }} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: NVIDIA_MIG_MONITOR_DEVICES + value: all + - name: DEVICE_LIST_STRATEGY + value: {{ .Values.devicePlugin.deviceListStrategy }} + - name: HOOK_PATH + value: {{ .Values.global.gpuHookPath }} + {{- if typeIs "bool" .Values.devicePlugin.passDeviceSpecsEnabled }} + - name: PASS_DEVICE_SPECS + value: {{ .Values.devicePlugin.passDeviceSpecsEnabled | quote }} + {{- end }} + {{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }} + - name: NVIDIA_DRIVER_ROOT + value: {{ .Values.devicePlugin.nvidiaDriverRoot }} + {{- end }} + {{- if typeIs "string" .Values.devicePlugin.nvidiaHookPath }} + - name: NVIDIA_CDI_HOOK_PATH + value: {{ .Values.devicePlugin.nvidiaHookPath }} + {{- end }} + {{- if typeIs "bool" .Values.devicePlugin.gdrcopyEnabled }} + - name: GDRCOPY_ENABLED + value: {{ .Values.devicePlugin.gdrcopyEnabled | quote }} + {{- end }} + {{- if typeIs "bool" .Values.devicePlugin.gdsEnabled }} + - name: GDS_ENABLED + value: {{ .Values.devicePlugin.gdsEnabled | quote }} + {{- end }} + {{- if typeIs "bool" .Values.devicePlugin.mofedEnabled }} + - name: MOFED_ENABLED + value: {{ .Values.devicePlugin.mofedEnabled | quote }} + {{- end }} + {{- if eq (.Values.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy | default "spread") "topology-aware" }} + - name: ENABLE_TOPOLOGY_SCORE + value: "true" + {{- end }} + {{- with .Values.devicePlugin.extraEnvs }} + {{- . | toYaml | nindent 12 }} + {{- end }} + securityContext: + privileged: true + allowPrivilegeEscalation: true + capabilities: + drop: ["ALL"] + add: ["SYS_ADMIN"] + resources: + {{- toYaml .Values.devicePlugin.resources | nindent 12 }} + volumeMounts: + - name: device-plugin + mountPath: /var/lib/kubelet/device-plugins + - name: lib + mountPath: {{ printf "%s%s" .Values.global.gpuHookPath "/vgpu" }} + - name: usrbin + mountPath: /usrbin + - name: deviceconfig + mountPath: /config + - name: hosttmp + mountPath: /tmp + - name: device-config + mountPath: /device-config.yaml + subPath: device-config.yaml + - name: cdi-root + mountPath: /var/run/cdi + {{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }} + # We always mount the driver root at /driver-root in the container. + # This is required for CDI detection to work correctly. + - name: driver-root + mountPath: /driver-root + readOnly: true + {{- end }} + - name: vgpu-monitor + image: {{ include "hami.devicePlugin.monitor.image" . }} + imagePullPolicy: {{ .Values.devicePlugin.monitor.image.pullPolicy }} + command: + - "vGPUmonitor" + {{- range .Values.devicePlugin.monitor.extraArgs }} + - {{ . }} + {{- end }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + add: ["SYS_ADMIN"] + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: NVIDIA_VISIBLE_DEVICES + value: "all" + - name: NVIDIA_MIG_MONITOR_DEVICES + value: "all" + - name: HOOK_PATH + value: "{{ .Values.global.gpuHookPath }}/vgpu" + - name: HAMI_RESYNC_INTERVAL + value: {{ .Values.devicePlugin.monitor.resyncInterval | default "5m" | quote }} + {{- with .Values.devicePlugin.monitor.extraEnvs }} + {{- . | toYaml | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.devicePlugin.monitor.resources | nindent 12 }} + volumeMounts: + - name: ctrs + mountPath: {{ .Values.devicePlugin.monitor.ctrPath }} + - name: dockers + mountPath: /run/docker + - name: containerds + mountPath: /run/containerd + - name: sysinfo + mountPath: /sysinfo + - name: hostvar + mountPath: /hostvar + - name: hosttmp + mountPath: /tmp + volumes: + - name: ctrs + hostPath: + path: {{ .Values.devicePlugin.monitor.ctrPath }} + - name: hosttmp + hostPath: + path: /tmp + - name: dockers + hostPath: + path: /run/docker + - name: containerds + hostPath: + path: /run/containerd + - name: device-plugin + hostPath: + path: {{ .Values.devicePlugin.pluginPath }} + - name: lib + hostPath: + path: {{ .Values.devicePlugin.libPath }} + {{- if .Values.devicePlugin.gpuOperatorToolkitReady.enabled }} + - name: nvidia-validations + hostPath: + path: {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }} + type: DirectoryOrCreate + {{- end }} + {{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }} + - name: driver-root + hostPath: + path: {{ .Values.devicePlugin.nvidiaDriverRoot }} + type: Directory + {{- end }} + - name: cdi-root + hostPath: + path: /var/run/cdi + type: DirectoryOrCreate + - name: usrbin + hostPath: + path: /usr/bin + - name: sysinfo + hostPath: + path: /sys + - name: hostvar + hostPath: + path: /var + - name: deviceconfig + configMap: + name: {{ .Values.devicePlugin.nodeConfiguration.externalConfigName | default (include "hami-vgpu.device-plugin" .) }} + - name: device-config + configMap: + name: {{ include "hami-vgpu.scheduler" . }}-device + {{- if .Values.devicePlugin.nvidiaNodeSelector }} + nodeSelector: {{ toYaml .Values.devicePlugin.nvidiaNodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.devicePlugin.tolerations }} + tolerations: {{ toYaml .Values.devicePlugin.tolerations | nindent 8 }} + {{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml new file mode 100644 index 00000000..c51ff6c6 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "hami-vgpu.device-plugin" . }}-monitor +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - create + - watch + - list + - update + - patch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - update + - list + - patch +{{- end -}} + + diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml new file mode 100644 index 00000000..93a118dd --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.device-plugin" . }} + labels: + app.kubernetes.io/component: "hami-device-plugin" + {{- include "hami-vgpu.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "hami-vgpu.device-plugin" . }}-monitor +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml new file mode 100644 index 00000000..24daca9b --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "hami-vgpu.device-plugin" . }}-monitor + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-device-plugin + {{- include "hami-vgpu.labels" . | nindent 4 }} + {{- if .Values.devicePlugin.service.labels }} # Use devicePlugin instead of scheduler + {{ toYaml .Values.devicePlugin.service.labels | indent 4 }} + {{- end }} + {{- if .Values.devicePlugin.service.annotations }} # Use devicePlugin instead of scheduler + annotations: {{ toYaml .Values.devicePlugin.service.annotations | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.devicePlugin.service.type | default "NodePort" }} # Default type is NodePort + ports: + - name: monitorport + port: {{ .Values.devicePlugin.service.httpPort | default 31992 }} # Default HTTP port is 31992 + targetPort: 9394 + {{- if eq (.Values.devicePlugin.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort + nodePort: {{ .Values.devicePlugin.service.httpPort | default 31992 }} + {{- end }} + protocol: TCP + selector: + app.kubernetes.io/component: hami-device-plugin + {{- include "hami-vgpu.selectorLabels" . | nindent 4 }} +{{- end -}} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml new file mode 100644 index 00000000..b10d2cb9 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml @@ -0,0 +1,10 @@ +{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "hami-vgpu.device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: "hami-device-plugin" + {{- include "hami-vgpu.labels" . | nindent 4 }} +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml b/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml new file mode 100644 index 00000000..cd3d14cc --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml @@ -0,0 +1,11 @@ +{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}} +{{- if and .Values.devicePlugin.createRuntimeClass .Values.devicePlugin.runtimeClassName }} +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: {{ .Values.devicePlugin.runtimeClassName }} + annotations: + helm.sh/hook: pre-install,pre-upgrade +handler: nvidia +{{- end }} +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml b/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml new file mode 100644 index 00000000..b6edb605 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml @@ -0,0 +1,31 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} +{{- if .Values.scheduler.certManager.enabled }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-serving-cert + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} +spec: + dnsNames: + - {{ include "hami-vgpu.scheduler" . }}.{{ include "hami-vgpu.namespace" . }}.svc + - {{ include "hami-vgpu.scheduler" . }}.{{ include "hami-vgpu.namespace" . }}.svc.cluster.local + issuerRef: + kind: Issuer + name: {{ include "hami-vgpu.scheduler" . }}-selfsigned-issuer + secretName: {{ include "hami-vgpu.scheduler.tls" . }} +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-selfsigned-issuer + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} +spec: + selfSigned: {} +{{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml b/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml new file mode 100644 index 00000000..9c510c00 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml @@ -0,0 +1,36 @@ +{{- if not .Values.dra.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["pods", "configmaps"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: [""] + resources: ["pods/binding"] + verbs: ["create"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "patch", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "get", "list"] + - apiGroups: [""] + resources: ["resourcequotas"] + verbs: ["get", "list", "watch"] +--- +{{- if .Values.mockDevicePlugin.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "hami-vgpu.mock-device-plugin" . }} +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "update", "list", "patch"] +{{- end -}} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml new file mode 100644 index 00000000..69d75986 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml @@ -0,0 +1,49 @@ +{{- if not .Values.dra.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-kube + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-scheduler +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-volume + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:volume-scheduler +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "hami-vgpu.scheduler" . }} +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml b/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml new file mode 100644 index 00000000..75889146 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml @@ -0,0 +1,142 @@ +{{- if and .Values.scheduler.kubeScheduler.enabled (not .Values.dra.enabled) -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} +data: + config.json: | + { + "kind": "Policy", + "apiVersion": "v1", + "extenders": [ + { + {{- if .Values.scheduler.admissionWebhook.enabled }} + "urlPrefix": "https://127.0.0.1:443", + "enableHttps": true, + "tlsConfig": { + "insecure": true + }, + {{- else }} + "urlPrefix": "http://127.0.0.1:80", + "enableHttps": false, + {{- end }} + "filterVerb": "filter", + "bindVerb": "bind", + "weight": 1, + "nodeCacheCapable": true, + "httpTimeout": 30000000000, + "managedResources": [ + {{- range .Values.devices.amd.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- if .Values.devices.ascend.enabled }} + {{- range .Values.devices.ascend.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- end }} + {{- if .Values.devices.mthreads.enabled }} + {{- range .Values.devices.mthreads.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- end }} + {{- if .Values.devices.enflame.enabled }} + {{- range .Values.devices.enflame.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- end }} + {{- if .Values.devices.kunlun.enabled }} + {{- range .Values.devices.kunlun.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- end }} + {{- range .Values.devices.awsneuron.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- if .Values.devices.iluvatar.enabled }} + {{- range .Values.devices.iluvatar.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- end }} + { + "name": "{{ .Values.resourceName }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.resourceMem }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.resourceCores }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.resourceMemPercentage }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.resourcePriority }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.mluResourceName }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.dcuResourceName }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.dcuResourceMem }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.dcuResourceCores }}", + "ignoredByScheduler": true + }, + { + "name": "metax-tech.com/gpu", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.metaxResourceName }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.metaxResourceCore }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.metaxResourceMem }}", + "ignoredByScheduler": true + } + ], + "ignoreable": false + } + ] + } +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml b/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml new file mode 100644 index 00000000..e2a91d8b --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml @@ -0,0 +1,102 @@ +{{- if and .Values.scheduler.kubeScheduler.enabled (not .Values.dra.enabled) -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-newversion + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} +data: + config.yaml: | + {{- if gt (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 25}} + apiVersion: kubescheduler.config.k8s.io/v1 + {{- else }} + apiVersion: kubescheduler.config.k8s.io/v1beta2 + {{- end }} + kind: KubeSchedulerConfiguration + leaderElection: + leaderElect: false + profiles: + - schedulerName: {{ .Values.schedulerName }} + extenders: + {{- if .Values.scheduler.admissionWebhook.enabled }} + - urlPrefix: "https://127.0.0.1:443" + enableHTTPS: true + tlsConfig: + insecure: true + {{- else }} + - urlPrefix: "http://127.0.0.1:80" + enableHTTPS: false + {{- end }} + filterVerb: filter + bindVerb: bind + nodeCacheCapable: true + weight: 1 + httpTimeout: 30s + managedResources: + - name: {{ .Values.resourceName }} + ignoredByScheduler: true + - name: {{ .Values.resourceMem }} + ignoredByScheduler: true + - name: {{ .Values.resourceCores }} + ignoredByScheduler: true + - name: {{ .Values.resourceMemPercentage }} + ignoredByScheduler: true + - name: {{ .Values.resourcePriority }} + ignoredByScheduler: true + - name: {{ .Values.mluResourceName }} + ignoredByScheduler: true + - name: {{ .Values.dcuResourceName }} + ignoredByScheduler: true + - name: {{ .Values.dcuResourceMem }} + ignoredByScheduler: true + - name: {{ .Values.dcuResourceCores }} + ignoredByScheduler: true + - name: "metax-tech.com/gpu" + ignoredByScheduler: true + - name: {{ .Values.metaxResourceName }} + ignoredByScheduler: true + - name: {{ .Values.metaxResourceCore }} + ignoredByScheduler: true + - name: {{ .Values.metaxResourceMem }} + ignoredByScheduler: true + {{- if .Values.devices.ascend.enabled }} + {{- range .Values.devices.ascend.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- end }} + {{- if .Values.devices.mthreads.enabled }} + {{- range .Values.devices.mthreads.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- end }} + {{- if .Values.devices.enflame.enabled }} + {{- range .Values.devices.enflame.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- end }} + {{- if .Values.devices.kunlun.enabled }} + {{- range .Values.devices.kunlun.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- end }} + {{- range .Values.devices.awsneuron.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- if .Values.devices.iluvatar.enabled }} + {{- range .Values.devices.iluvatar.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- end }} + {{- range .Values.devices.amd.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml b/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml new file mode 100644 index 00000000..6364cae3 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml @@ -0,0 +1,227 @@ +{{- if not .Values.dra.enabled -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} + {{- with .Values.global.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.global.annotations }} + annotations: {{ toYaml .Values.global.annotations | nindent 4}} + {{- end }} +spec: + {{- if .Values.scheduler.leaderElect }} + replicas: {{ .Values.scheduler.replicas }} + {{- else }} + replicas: 1 + {{- end }} + selector: + matchLabels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.selectorLabels" . | nindent 8 }} + hami.io/webhook: ignore + annotations: + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + checksum/hami-scheduler-newversion-config: {{ include (print $.Template.BasePath "/scheduler/configmapnew.yaml") . | sha256sum }} + {{- else }} + checksum/hami-scheduler-config: {{ include (print $.Template.BasePath "/scheduler/configmap.yaml") . | sha256sum }} + {{- end }} + checksum/hami-scheduler-device-config: {{ include (print $.Template.BasePath "/scheduler/device-configmap.yaml") . | sha256sum }} + {{- if .Values.scheduler.podAnnotations }} + {{- toYaml .Values.scheduler.podAnnotations | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "hami-vgpu.scheduler" . }} + priorityClassName: system-node-critical + {{- include "hami.scheduler.extender.imagePullSecrets" . | nindent 6 }} + containers: + {{- if .Values.scheduler.kubeScheduler.enabled }} + - name: kube-scheduler + image: {{ include "hami.scheduler.kubeScheduler.image" . }} + imagePullPolicy: {{ .Values.scheduler.kubeScheduler.image.pullPolicy }} + command: + - kube-scheduler + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + {{- range .Values.scheduler.kubeScheduler.extraNewArgs }} + - {{ . }} + {{- end }} + {{- else }} + - --scheduler-name={{ .Values.schedulerName }} + {{- range .Values.scheduler.kubeScheduler.extraArgs }} + - {{ . }} + {{- end }} + {{- end }} + - --leader-elect={{ .Values.scheduler.leaderElect }} + - --leader-elect-resource-name={{ .Values.schedulerName }} + - --leader-elect-resource-namespace={{ include "hami-vgpu.namespace" . }} + resources: + {{- toYaml .Values.scheduler.kubeScheduler.resources | nindent 12 }} + volumeMounts: + - name: scheduler-config + mountPath: /config + {{- end }} + {{- if .Values.scheduler.livenessProbe }} + livenessProbe: + failureThreshold: 8 + httpGet: + path: /healthz + port: 10259 + scheme: HTTPS + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 15 + {{- end }} + - name: vgpu-scheduler-extender + image: {{ include "hami.scheduler.extender.image" . }} + imagePullPolicy: {{ .Values.scheduler.extender.image.pullPolicy }} + env: + {{- if .Values.scheduler.nodeLockExpire }} + - name: HAMI_NODELOCK_EXPIRE + value: "{{ .Values.scheduler.nodeLockExpire }}" + {{- end }} + {{- if .Values.global.managedNodeSelectorEnable }} + {{- range $key, $value := .Values.global.managedNodeSelector }} + - name: NODE_SELECTOR_{{ $key | upper | replace "-" "_" }} + value: "{{ $value }}" + {{- end }} + {{- end }} + command: + - scheduler + {{- if .Values.scheduler.admissionWebhook.enabled }} + - --http_bind=0.0.0.0:443 + - --cert_file=/tls/tls.crt + - --key_file=/tls/tls.key + {{- else }} + - --http_bind=0.0.0.0:80 + {{- end }} + - --scheduler-name={{ .Values.schedulerName }} + - --metrics-bind-address={{ .Values.scheduler.metricsBindAddress }} + - --node-scheduler-policy={{ .Values.scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy }} + - --gpu-scheduler-policy={{ .Values.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy }} + - --force-overwrite-default-scheduler={{ .Values.scheduler.forceOverwriteDefaultScheduler}} + - --device-config-file=/device-config.yaml + - --leader-elect={{ .Values.scheduler.leaderElect }} + - --leader-elect-resource-name={{ .Values.schedulerName }} + - --leader-elect-resource-namespace={{ include "hami-vgpu.namespace" . }} + {{- if .Values.devices.ascend.enabled }} + - --enable-ascend=true + {{- end }} + {{- if .Values.devices.iluvatar.enabled }} + - --enable-iluvatar=true + {{- end }} + {{- if .Values.scheduler.nodeLabelSelector }} + - --node-label-selector={{- $first := true -}} + {{- range $key, $value := .Values.scheduler.nodeLabelSelector -}} + {{- if not $first }},{{ end -}} + {{- $key }}={{ $value -}} + {{- $first = false -}} + {{- end -}} + {{- end }} + {{- range .Values.scheduler.extender.extraArgs }} + - {{ . }} + {{- end }} + ports: + {{- if .Values.scheduler.admissionWebhook.enabled }} + - name: https + containerPort: 443 + protocol: TCP + {{- else }} + - name: http + containerPort: 80 + protocol: TCP + {{- end }} + - name: metrics + containerPort: {{ last (splitList ":" .Values.scheduler.metricsBindAddress) | int }} + protocol: TCP + resources: + {{- toYaml .Values.scheduler.extender.resources | nindent 12 }} + volumeMounts: + - name: device-config + mountPath: /device-config.yaml + subPath: device-config.yaml + {{- if .Values.scheduler.admissionWebhook.enabled }} + - name: tls-config + mountPath: /tls + {{- end }} + {{- if .Values.scheduler.livenessProbe }} + livenessProbe: + httpGet: + path: /healthz + {{- if .Values.scheduler.admissionWebhook.enabled }} + port: https + scheme: HTTPS + {{- else }} + port: http + scheme: HTTP + {{- end }} + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 3 + timeoutSeconds: 5 + {{- end }} + {{- if .Values.scheduler.leaderElect }} + readinessProbe: + httpGet: + path: /readyz + {{- if .Values.scheduler.admissionWebhook.enabled }} + port: https + scheme: HTTPS + {{- else }} + port: http + scheme: HTTP + {{- end }} + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 3 + timeoutSeconds: 5 + {{- end }} + {{- if .Values.scheduler.leaderElect }} + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app.kubernetes.io/component + operator: In + values: + - hami-scheduler + topologyKey: "kubernetes.io/hostname" + {{- end }} + volumes: + {{- if .Values.scheduler.admissionWebhook.enabled }} + - name: tls-config + secret: + secretName: {{ template "hami-vgpu.scheduler.tls" . }} + {{- end }} + {{- if .Values.scheduler.kubeScheduler.enabled }} + - name: scheduler-config + configMap: + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + name: {{ template "hami-vgpu.scheduler" . }}-newversion + {{- else }} + name: {{ template "hami-vgpu.scheduler" . }} + {{- end }} + {{- end }} + - name: device-config + configMap: + name: {{ include "hami-vgpu.scheduler" . }}-device + {{- if .Values.scheduler.nodeSelector }} + nodeSelector: {{ toYaml .Values.scheduler.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.scheduler.tolerations }} + tolerations: {{ toYaml .Values.scheduler.tolerations | nindent 8 }} + {{- end }} + {{- if .Values.scheduler.nodeName }} + nodeName: {{ .Values.scheduler.nodeName }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml b/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml new file mode 100644 index 00000000..ccb945ac --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml @@ -0,0 +1,410 @@ +{{- if not .Values.dra.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-device + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} +data: + device-config.yaml: |- + {{- if .Files.Glob "files/device-config.yaml" }} + {{- .Files.Get "files/device-config.yaml" | nindent 4}} + {{- else }} + nvidia: + resourceCountName: {{ .Values.resourceName }} + resourceMemoryName: {{ .Values.resourceMem }} + resourceMemoryPercentageName: {{ .Values.resourceMemPercentage }} + resourceCoreName: {{ .Values.resourceCores }} + resourcePriorityName: {{ .Values.resourcePriority }} + overwriteEnv: false + defaultMemory: 0 + defaultCores: 0 + defaultGPUNum: 1 + memoryFactor: 1 + deviceSplitCount: {{ .Values.devicePlugin.deviceSplitCount }} + deviceMemoryScaling: {{ .Values.devicePlugin.deviceMemoryScaling }} + deviceCoreScaling: {{ .Values.devicePlugin.deviceCoreScaling }} + gpuCorePolicy: {{ .Values.devices.nvidia.gpuCorePolicy }} + libCudaLogLevel: {{ .Values.devices.nvidia.libCudaLogLevel }} + runtimeClassName: "{{ .Values.devicePlugin.runtimeClassName }}" + knownMigGeometries: + - models: [ "A30" ] + allowedGeometries: + - + - name: 1g.6gb + core: 25 + memory: 6144 + count: 4 + - + - name: 2g.12gb + core: 50 + memory: 12288 + count: 2 + - + - name: 4g.24gb + core: 100 + memory: 24576 + count: 1 + - models: [ "A100-SXM4-40GB", "A100-40GB-PCIe", "A100-PCIE-40GB"] + allowedGeometries: + - + - name: 1g.5gb + core: 14 + memory: 5120 + count: 7 + - + - name: 1g.5gb + core: 14 + memory: 5120 + count: 1 + - name: 2g.10gb + core: 28 + memory: 10240 + count: 3 + - + - name: 3g.20gb + core: 42 + memory: 20480 + count: 2 + - + - name: 7g.40gb + core: 100 + memory: 40960 + count: 1 + - models: [ "A100-SXM4-80GB", "A100-80GB-PCIe", "A100-PCIE-80GB"] + allowedGeometries: + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 7 + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 1 + - name: 2g.20gb + core: 28 + memory: 20480 + count: 3 + - + - name: 3g.40gb + core: 42 + memory: 40960 + count: 2 + - + - name: 7g.79gb + core: 100 + memory: 80896 + count: 1 + - models: [ "H100-PCIE-80GB", "H100-SXM5-80GB"] + allowedGeometries: + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 7 + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 1 + - name: 2g.20gb + core: 28 + memory: 20480 + count: 3 + - + - name: 3g.40gb + core: 42 + memory: 40960 + count: 2 + - + - name: 7g.80gb + core: 100 + memory: 81920 + count: 1 + - models: [ "H100-PCIE-94GB", "H100-SXM5-94GB"] + allowedGeometries: + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 7 + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 1 + - name: 2g.24gb + core: 28 + memory: 24576 + count: 3 + - + - name: 3g.47gb + core: 42 + memory: 48128 + count: 2 + - + - name: 7g.94gb + core: 100 + memory: 96256 + count: 1 + - models: [ "H20", "H100 on GH200"] + allowedGeometries: + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 7 + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 1 + - name: 2g.24gb + core: 28 + memory: 24576 + count: 3 + - + - name: 3g.48gb + core: 42 + memory: 49152 + count: 2 + - + - name: 7g.96gb + core: 100 + memory: 98304 + count: 1 + - models: [ "H200 NVL", "H200-SXM5"] + allowedGeometries: + - + - name: 1g.18gb + core: 14 + memory: 18432 + count: 7 + - + - name: 1g.18gb + core: 14 + memory: 18432 + count: 1 + - name: 2g.35gb + core: 28 + memory: 35840 + count: 3 + - + - name: 3g.71gb + core: 42 + memory: 72704 + count: 2 + - + - name: 7g.141gb + core: 100 + memory: 144384 + count: 1 + - models: [ "B200" ] + allowedGeometries: + - + - name: 1g.23gb + core: 14 + memory: 23552 + count: 7 + - + - name: 1g.23gb + core: 14 + memory: 23552 + count: 1 + - name: 2g.45gb + core: 28 + memory: 46080 + count: 3 + - + - name: 3g.90gb + core: 42 + memory: 92160 + count: 2 + - + - name: 7g.180gb + core: 100 + memory: 184320 + count: 1 + cambricon: + resourceCountName: {{ .Values.mluResourceName }} + resourceMemoryName: {{ .Values.mluResourceMem }} + resourceCoreName: {{ .Values.mluResourceCores }} + hygon: + resourceCountName: {{ .Values.dcuResourceName }} + resourceMemoryName: {{ .Values.dcuResourceMem }} + resourceCoreName: {{ .Values.dcuResourceCores }} + memoryFactor: 1 + metax: + resourceCountName: "metax-tech.com/gpu" + resourceVCountName: {{ .Values.metaxResourceName }} + resourceVMemoryName: {{ .Values.metaxResourceMem }} + resourceVCoreName: {{ .Values.metaxResourceCore }} + sgpuTopologyAware: {{ .Values.metaxsGPUTopologyAware }} + enflame: + resourceNameGCU: "enflame.com/gcu" + resourceNameVGCU: {{ .Values.enflameResourceNameVGCU }} + resourceNameVGCUPercentage: {{ .Values.enflameResourceNameVGCUPercentage }} + mthreads: + resourceCountName: "mthreads.com/vgpu" + resourceMemoryName: "mthreads.com/sgpu-memory" + resourceCoreName: "mthreads.com/sgpu-core" + iluvatars: + - chipName: MR-V100 + commonWord: MR-V100 + resourceCountName: iluvatar.ai/MR-V100-vgpu + resourceMemoryName: iluvatar.ai/MR-V100.vMem + resourceCoreName: iluvatar.ai/MR-V100.vCore + - chipName: MR-V50 + commonWord: MR-V50 + resourceCountName: iluvatar.ai/MR-V50-vgpu + resourceMemoryName: iluvatar.ai/MR-V50.vMem + resourceCoreName: iluvatar.ai/MR-V50.vCore + - chipName: BI-V150 + commonWord: BI-V150 + resourceCountName: iluvatar.ai/BI-V150-vgpu + resourceMemoryName: iluvatar.ai/BI-V150.vMem + resourceCoreName: iluvatar.ai/BI-V150.vCore + - chipName: BI-V100 + commonWord: BI-V100 + resourceCountName: iluvatar.ai/BI-V100-vgpu + resourceMemoryName: iluvatar.ai/BI-V100.vMem + resourceCoreName: iluvatar.ai/BI-V100.vCore + kunlun: + resourceCountName: {{ .Values.kunlunResourceName }} + resourceVCountName: {{ .Values.kunlunResourceVCountName }} + resourceVMemoryName: {{ .Values.kunlunResourceVMemoryName }} + awsneuron: + resourceCountName: "aws.amazon.com/neuron" + resourceCoreName: "aws.amazon.com/neuroncore" + amd: + resourceCountName: "amd.com/gpu" + vnpus: + - chipName: 910A + commonWord: Ascend910A + resourceName: huawei.com/Ascend910A + resourceMemoryName: huawei.com/Ascend910A-memory + memoryAllocatable: 32768 + memoryCapacity: 32768 + memoryFactor: 1 + aiCore: 30 + templates: + - name: vir02 + memory: 2184 + aiCore: 2 + - name: vir04 + memory: 4369 + aiCore: 4 + - name: vir08 + memory: 8738 + aiCore: 8 + - name: vir16 + memory: 17476 + aiCore: 16 + - chipName: 910B2 + commonWord: Ascend910B2 + resourceName: huawei.com/Ascend910B2 + resourceMemoryName: huawei.com/Ascend910B2-memory + memoryAllocatable: 65536 + memoryCapacity: 65536 + memoryFactor: 1 + aiCore: 24 + aiCPU: 6 + templates: + - name: vir03_1c_8g + memory: 8192 + aiCore: 3 + aiCPU: 1 + - name: vir06_1c_16g + memory: 16384 + aiCore: 6 + aiCPU: 1 + - name: vir12_3c_32g + memory: 32768 + aiCore: 12 + aiCPU: 3 + - chipName: 910B3 + commonWord: Ascend910B3 + resourceName: huawei.com/Ascend910B3 + resourceMemoryName: huawei.com/Ascend910B3-memory + memoryAllocatable: 65536 + memoryCapacity: 65536 + memoryFactor: 1 + aiCore: 20 + aiCPU: 7 + templates: + - name: vir05_1c_16g + memory: 16384 + aiCore: 5 + aiCPU: 1 + - name: vir10_3c_32g + memory: 32768 + aiCore: 10 + aiCPU: 3 + - chipName: 910B4-1 + commonWord: Ascend910B4-1 + resourceName: huawei.com/Ascend910B4-1 + resourceMemoryName: huawei.com/Ascend910B4-1-memory + memoryAllocatable: 65536 + memoryCapacity: 65536 + memoryFactor: 1 + aiCore: 20 + aiCPU: 7 + templates: + # NOTE: Names of vnpu template for 910B4-1 are fixed by Ascend runtime and must not be changed. + # The memory is used for scheduling so the correct values must be set. + # Template vir05_1c_8g actually provides 16GB memory, + - name: vir05_1c_8g + memory: 16384 + aiCore: 5 + aiCPU: 1 + # Template vir10_3c_16g actually provides 32GB memory + - name: vir10_3c_16g + memory: 32768 + aiCore: 10 + aiCPU: 3 + - chipName: 910B4 + commonWord: Ascend910B4 + resourceName: huawei.com/Ascend910B4 + resourceMemoryName: huawei.com/Ascend910B4-memory + memoryAllocatable: 32768 + memoryCapacity: 32768 + memoryFactor: 1 + aiCore: 20 + aiCPU: 7 + templates: + - name: vir05_1c_8g + memory: 8192 + aiCore: 5 + aiCPU: 1 + - name: vir10_3c_16g + memory: 16384 + aiCore: 10 + aiCPU: 3 + - chipName: 310P3 + commonWord: Ascend310P + resourceName: huawei.com/Ascend310P + resourceMemoryName: huawei.com/Ascend310P-memory + memoryAllocatable: 21527 + memoryCapacity: 24576 + memoryFactor: 1 + aiCore: 8 + aiCPU: 7 + templates: + - name: vir01 + memory: 3072 + aiCore: 1 + aiCPU: 1 + - name: vir02 + memory: 6144 + aiCore: 2 + aiCPU: 2 + - name: vir04 + memory: 12288 + aiCore: 4 + aiCPU: 4 + {{ end }} +{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml new file mode 100644 index 00000000..b089ae82 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml @@ -0,0 +1,30 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} +{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +rules: + - apiGroups: + - admissionregistration.k8s.io + resources: + #- validatingwebhookconfigurations + - mutatingwebhookconfigurations + verbs: + - get + - update +{{- if .Values.podSecurityPolicy.enabled }} + - apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ include "hami-vgpu.fullname" . }}-admission +{{- end }} +{{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml new file mode 100644 index 00000000..64f01ecf --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} +{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "hami-vgpu.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.fullname" . }}-admission + namespace: {{ include "hami-vgpu.namespace" . }} +{{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml new file mode 100644 index 00000000..400168a7 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml @@ -0,0 +1,70 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} +{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission-create + namespace: {{ include "hami-vgpu.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission-create + {{- if .Values.scheduler.patch.podAnnotations }} + annotations: {{ toYaml .Values.scheduler.patch.podAnnotations | nindent 8 }} + {{- end }} + labels: + {{- include "hami-vgpu.labels" . | nindent 8 }} + app.kubernetes.io/component: admission-webhook + hami.io/webhook: ignore + spec: + {{- if .Values.scheduler.patch.priorityClassName }} + priorityClassName: {{ .Values.scheduler.patch.priorityClassName }} + {{- end }} + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + {{- include "hami.scheduler.patch.new.imagePullSecrets" . | nindent 6 }} + {{- else }} + {{- include "hami.scheduler.patch.imagePullSecrets" . | nindent 6 }} + {{- end }} + containers: + - name: create + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + image: {{ include "hami.scheduler.patch.new.image" . }} + imagePullPolicy: {{ .Values.scheduler.patch.imageNew.pullPolicy }} + {{- else }} + image: {{ include "hami.scheduler.patch.image" . }} + imagePullPolicy: {{ .Values.scheduler.patch.image.pullPolicy }} + {{- end }} + args: + - create + - --cert-name=tls.crt + - --key-name=tls.key + {{- if .Values.scheduler.admissionWebhook.customURL.enabled }} + - --host={{ printf "%s.%s.svc,127.0.0.1,%s" (include "hami-vgpu.scheduler" .) (include "hami-vgpu.namespace" .) .Values.scheduler.admissionWebhook.customURL.host}} + {{- else }} + - --host={{ printf "%s.%s.svc,127.0.0.1" (include "hami-vgpu.scheduler" .) (include "hami-vgpu.namespace" .) }} + {{- end }} + - --namespace={{ include "hami-vgpu.namespace" . }} + - --secret-name={{ include "hami-vgpu.scheduler.tls" . }} + restartPolicy: OnFailure + serviceAccountName: {{ include "hami-vgpu.fullname" . }}-admission + {{- if .Values.scheduler.patch.nodeSelector }} + nodeSelector: {{ toYaml .Values.scheduler.patch.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.scheduler.patch.tolerations }} + tolerations: {{ toYaml .Values.scheduler.patch.tolerations | nindent 8 }} + {{- end }} + securityContext: + runAsNonRoot: true + runAsUser: {{ .Values.scheduler.patch.runAsUser }} +{{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml new file mode 100644 index 00000000..71e97f6b --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml @@ -0,0 +1,65 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} +{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission-patch + namespace: {{ include "hami-vgpu.namespace" . }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission-patch + {{- if .Values.scheduler.patch.podAnnotations }} + annotations: {{ toYaml .Values.scheduler.patch.podAnnotations | nindent 8 }} + {{- end }} + labels: + {{- include "hami-vgpu.labels" . | nindent 8 }} + app.kubernetes.io/component: admission-webhook + hami.io/webhook: ignore + spec: + {{- if .Values.scheduler.patch.priorityClassName }} + priorityClassName: {{ .Values.scheduler.patch.priorityClassName }} + {{- end }} + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + {{- include "hami.scheduler.patch.new.imagePullSecrets" . | nindent 6 }} + {{- else }} + {{- include "hami.scheduler.patch.imagePullSecrets" . | nindent 6 }} + {{- end }} + containers: + - name: patch + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + image: {{ include "hami.scheduler.patch.new.image" . }} + imagePullPolicy: {{ .Values.scheduler.patch.imageNew.pullPolicy }} + {{- else }} + image: {{ include "hami.scheduler.patch.image" . }} + imagePullPolicy: {{ .Values.scheduler.patch.image.pullPolicy }} + {{- end }} + args: + - patch + - --webhook-name={{ include "hami-vgpu.scheduler.webhook" . }} + - --namespace={{ include "hami-vgpu.namespace" . }} + - --patch-validating=false + - --secret-name={{ include "hami-vgpu.scheduler.tls" . }} + restartPolicy: OnFailure + serviceAccountName: {{ include "hami-vgpu.fullname" . }}-admission + {{- if .Values.scheduler.patch.nodeSelector }} + nodeSelector: {{ toYaml .Values.scheduler.patch.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.scheduler.patch.tolerations }} + tolerations: {{ toYaml .Values.scheduler.patch.tolerations | nindent 8 }} + {{- end }} + securityContext: + runAsNonRoot: true + runAsUser: {{ .Values.scheduler.patch.runAsUser }} +{{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml new file mode 100644 index 00000000..a48270e7 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml @@ -0,0 +1,40 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} +{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +{{- if .Values.podSecurityPolicy.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +spec: + allowPrivilegeEscalation: false + fsGroup: + ranges: + - max: 65535 + min: 1 + rule: MustRunAs + requiredDropCapabilities: + - ALL + runAsUser: + rule: MustRunAsNonRoot + seLinux: + rule: RunAsAny + supplementalGroups: + ranges: + - max: 65535 + min: 1 + rule: MustRunAs + volumes: + - configMap + - emptyDir + - projected + - secret + - downwardAPI +{{- end }} +{{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml new file mode 100644 index 00000000..74e45681 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml @@ -0,0 +1,23 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} +{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission + namespace: {{ include "hami-vgpu.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create +{{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml new file mode 100644 index 00000000..32da5ba4 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml @@ -0,0 +1,23 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} +{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission + namespace: {{ include "hami-vgpu.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "hami-vgpu.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.fullname" . }}-admission + namespace: {{ include "hami-vgpu.namespace" . }} +{{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml new file mode 100644 index 00000000..d2eb41de --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) }} +{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission + namespace: {{ include "hami-vgpu.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +{{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/role.yaml b/packages/system/hami/charts/hami/templates/scheduler/role.yaml new file mode 100644 index 00000000..6e681967 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/role.yaml @@ -0,0 +1,14 @@ +{{- if not .Values.dra.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create", "list", "watch", "get", "update", "patch"] +{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml new file mode 100644 index 00000000..934f8223 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml @@ -0,0 +1,33 @@ +{{- if not .Values.dra.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "hami-vgpu.scheduler" . }} +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +--- +{{- if .Values.mockDevicePlugin.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.mock-device-plugin" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "hami-vgpu.mock-device-plugin" . }} +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.mock-device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +{{- end -}} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/service.yaml b/packages/system/hami/charts/hami/templates/scheduler/service.yaml new file mode 100644 index 00000000..3e18bc98 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/service.yaml @@ -0,0 +1,36 @@ +{{- if not .Values.dra.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} + {{- if .Values.scheduler.service.labels }} + {{ toYaml .Values.scheduler.service.labels | indent 4 }} + {{- end }} + {{- if .Values.scheduler.service.annotations }} + annotations: {{ toYaml .Values.scheduler.service.annotations | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.scheduler.service.type | default "NodePort" }} # Default type is NodePort + ports: + - name: http + port: {{ .Values.scheduler.service.httpPort | default 443 }} # Default HTTP port is 443 + targetPort: {{ .Values.scheduler.service.httpTargetPort | default 443 }} + {{- if eq (.Values.scheduler.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort + nodePort: {{ .Values.scheduler.service.schedulerPort | default 31998 }} + {{- end }} + protocol: TCP + - name: monitor + port: {{ .Values.scheduler.service.monitorPort | default 31993 }} # Default monitoring port is 31993 + targetPort: {{ .Values.scheduler.service.monitorTargetPort | default 9395 }} + {{- if eq (.Values.scheduler.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort + nodePort: {{ .Values.scheduler.service.monitorPort | default 31993 }} + {{- end }} + protocol: TCP + selector: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.selectorLabels" . | nindent 4 }} +{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml b/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml new file mode 100644 index 00000000..929bf59a --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml @@ -0,0 +1,18 @@ +{{- if not .Values.dra.enabled -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +--- +{{- if .Values.mockDevicePlugin.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "hami-vgpu.mock-device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +{{- end -}} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml b/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml new file mode 100644 index 00000000..148feb5c --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml @@ -0,0 +1,57 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + {{- if .Values.scheduler.certManager.enabled }} + annotations: + cert-manager.io/inject-ca-from: {{ include "hami-vgpu.namespace" . }}/{{ include "hami-vgpu.scheduler" . }}-serving-cert + {{- end }} + name: {{ include "hami-vgpu.scheduler.webhook" . }} +webhooks: + - admissionReviewVersions: + - v1beta1 + clientConfig: + {{- if .Values.scheduler.admissionWebhook.customURL.enabled }} + url: https://{{ .Values.scheduler.admissionWebhook.customURL.host}}:{{.Values.scheduler.admissionWebhook.customURL.port}}{{.Values.scheduler.admissionWebhook.customURL.path}} + {{- else }} + service: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + path: /webhook + port: {{ .Values.scheduler.service.httpPort }} + {{- end }} + failurePolicy: {{ .Values.scheduler.admissionWebhook.failurePolicy }} + matchPolicy: Equivalent + name: vgpu.hami.io + namespaceSelector: + matchExpressions: + - key: hami.io/webhook + operator: NotIn + values: + - ignore + {{- if .Values.scheduler.admissionWebhook.whitelistNamespaces }} + - key: kubernetes.io/metadata.name + operator: NotIn + values: + {{- toYaml .Values.scheduler.admissionWebhook.whitelistNamespaces | nindent 10 }} + {{- end }} + objectSelector: + matchExpressions: + - key: hami.io/webhook + operator: NotIn + values: + - ignore + reinvocationPolicy: {{ .Values.scheduler.admissionWebhook.reinvocationPolicy }} + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - pods + scope: '*' + sideEffects: None + timeoutSeconds: 10 +{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/values.yaml b/packages/system/hami/charts/hami/values.yaml new file mode 100644 index 00000000..9fbeca70 --- /dev/null +++ b/packages/system/hami/charts/hami/values.yaml @@ -0,0 +1,476 @@ +## @section Global configuration +## @param global.imageRegistry Global Docker image registry +## @param global.imagePullSecrets Global Docker image pull secrets +global: + ## @param global.imageRegistry Global Docker image registry + imageRegistry: "" + ## E.g. + ## imagePullSecrets: + ## - myRegistryKeySecretName + ## @param global.imagePullSecrets Global Docker image pull secrets + imagePullSecrets: [] + imageTag: "v2.8.1" + gpuHookPath: /usr/local + labels: {} + annotations: {} + managedNodeSelectorEnable: false + managedNodeSelector: + usage: "gpu" + +nameOverride: "" +fullnameOverride: "" +namespaceOverride: "" + +#Nvidia GPU Parameters +resourceName: "nvidia.com/gpu" +resourceMem: "nvidia.com/gpumem" +resourceMemPercentage: "nvidia.com/gpumem-percentage" +resourceCores: "nvidia.com/gpucores" +resourcePriority: "nvidia.com/priority" + +#MLU Parameters +mluResourceName: "cambricon.com/vmlu" +mluResourceMem: "cambricon.com/mlu.smlu.vmemory" +mluResourceCores: "cambricon.com/mlu.smlu.vcore" + +#Hygon DCU Parameters +dcuResourceName: "hygon.com/dcunum" +dcuResourceMem: "hygon.com/dcumem" +dcuResourceCores: "hygon.com/dcucores" + +#Metax sGPU Parameters +metaxResourceName: "metax-tech.com/sgpu" +metaxResourceCore: "metax-tech.com/vcore" +metaxResourceMem: "metax-tech.com/vmemory" +metaxsGPUTopologyAware: "false" + +#Enflame VGCU Parameters +enflameResourceNameVGCU: "enflame.com/vgcu" +enflameResourceNameVGCUPercentage: "enflame.com/vgcu-percentage" + +#Kunlun XPU Parameters +kunlunResourceName: "kunlunxin.com/xpu" +kunlunResourceVCountName: "kunlunxin.com/vxpu" +kunlunResourceVMemoryName: "kunlunxin.com/vxpu-memory" + +schedulerName: "hami-scheduler" + +podSecurityPolicy: + enabled: false + +scheduler: + # @param nodeName defines the node name and the nvidia-vgpu-scheduler-scheduler will schedule to the node. + # if we install the nvidia-vgpu-scheduler-scheduler as default scheduler, we need to remove the k8s default + # scheduler pod from the cluster first, we must specify node name to skip the schedule workflow. + nodeName: "" + #nodeLabelSelector: + # "gpu": "on" + overwriteEnv: "false" + defaultSchedulerPolicy: + nodeSchedulerPolicy: binpack + gpuSchedulerPolicy: spread + metricsBindAddress: ":9395" + # If set to false, When Pod.Spec.SchedulerName equals to the const DefaultSchedulerName in k8s.io/api/core/v1 package, webhook will not overwrite it + forceOverwriteDefaultScheduler: true + livenessProbe: false + leaderElect: true + # when leaderElect is true, replicas is available, otherwise replicas is 1. + replicas: 1 + kubeScheduler: + # @param enabled indicate whether to run kube-scheduler container in the scheduler pod, it's true by default. + enabled: true + ## @param image.registry kube scheduler image registry + ## @param image.repository kube scheduler image repository + ## @param image.tag kube scheduler image tag (immutable tags are recommended) + ## @param image.pullPolicy kube scheduler image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + image: + registry: "registry.cn-hangzhou.aliyuncs.com" + repository: "google_containers/kube-scheduler" + tag: "" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + resources: {} + # If you do want to specify resources, uncomment the following lines, adjust them as necessary. + # and remove the curly braces after 'resources:'. +# limits: +# cpu: 1000m +# memory: 1000Mi +# requests: +# cpu: 100m +# memory: 100Mi + extraNewArgs: + - --config=/config/config.yaml + - -v=4 + extraArgs: + - --policy-config-file=/config/config.json + - -v=4 + extender: + ## @param image.registry scheduler extender image registry + ## @param image.repository scheduler extender image repository + ## @param image.tag scheduler extender image tag (immutable tags are recommended) + ## @param image.pullPolicy scheduler extender image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + image: + registry: "docker.io" + repository: "projecthami/hami" + tag: "" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + resources: {} + # If you do want to specify resources, uncomment the following lines, adjust them as necessary, + # and remove the curly braces after 'resources:'. +# limits: +# cpu: 1000m +# memory: 1000Mi +# requests: +# cpu: 100m +# memory: 100Mi + extraArgs: + - --debug + - -v=4 + nodeLockExpire: "5m" + podAnnotations: {} + tolerations: [] + #serviceAccountName: "hami-vgpu-scheduler-sa" + admissionWebhook: + # If set to false, the admission webhook is not installed and any pods that should use HAMi must be + # configured to use the right 'schedulerName' and other device-specific configurations. + enabled: true + customURL: + enabled: false + # must be an endpoint using https. + # should generate host certs here + host: 127.0.0.1 # hostname or ip, can be your node'IP if you want to use https://:/ + port: 31998 + path: /webhook + whitelistNamespaces: + # Specify the namespaces that the webhook will not be applied to. + # - default + # - kube-system + # - istio-system + reinvocationPolicy: Never + failurePolicy: Ignore + ## TLS Certificate Option 1: Use cert-manager to generate self-signed certificate. + ## If enabled, always takes precedence over options 2. + certManager: + enabled: false + ## TLS Certificate Option 2: Use kube-webhook-certgen to generate self-signed certificate. + ## If true and certManager.enabled is false, Helm will automatically create a self-signed cert and secret for you. + patch: + enabled: true + ## @param image.registry scheduler extender image registry + ## @param image.repository scheduler extender image repository + ## @param image.tag scheduler extender image tag (immutable tags are recommended) + ## @param image.pullPolicy scheduler extender image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + image: + registry: "docker.io" + repository: "jettech/kube-webhook-certgen" + tag: "v1.5.2" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## @param image.registry scheduler extender image registry + ## @param image.repository scheduler extender image repository + ## @param image.tag scheduler extender image tag (immutable tags are recommended) + ## @param image.pullPolicy scheduler extender image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + imageNew: + registry: "docker.io" + repository: "liangjw/kube-webhook-certgen" + tag: "v1.1.1" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + priorityClassName: "" + podAnnotations: {} + nodeSelector: {} + tolerations: [] + runAsUser: 2000 + service: + type: NodePort # Default type is NodePort, can be changed to ClusterIP + httpPort: 443 # HTTP port + schedulerPort: 31998 # NodePort for HTTP + monitorPort: 31993 # Monitoring port + monitorTargetPort: 9395 + httpTargetPort: 443 + labels: {} + annotations: {} + +devicePlugin: + enabled: true + gpuOperatorToolkitReady: + enabled: false + hostPath: "/run/nvidia/validations" + ## @param image.registry devicePlugin image registry + ## @param image.repository devicePlugin image repository + ## @param image.tag devicePlugin image tag (immutable tags are recommended) + ## @param image.pullPolicy devicePlugin image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + image: + registry: "docker.io" + repository: "projecthami/hami" + tag: "" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + monitor: + ## @param image.registry monitor image registry + ## @param image.repository monitor image repository + ## @param image.tag monitor image tag (immutable tags are recommended) + ## @param image.pullPolicy monitor image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + image: + registry: "docker.io" + repository: "projecthami/hami" + tag: "" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ctrPath: /usr/local/vgpu/containers + resyncInterval: "5m" + extraArgs: + - -v=4 + extraEnvs: {} + resources: {} + # If you do want to specify resources, uncomment the following lines, adjust them as necessary. + # and remove the curly braces after 'resources:'. +# limits: +# cpu: 1000m +# memory: 1000Mi +# requests: +# cpu: 100m +# memory: 100Mi + + deviceSplitCount: 10 + deviceMemoryScaling: 1 + deviceCoreScaling: 1 + # Node configuration for device plugin, Priority: externalConfigName > config > default config + nodeConfiguration: + # If you want to use a custom config.json, you can set the content here. + # If this is set, it will override the default config.json(An example is as follows). + config: | + { + "nodeconfig": [ + { + "name": "your-node-name", + "operatingmode": "hami-core", + "devicememoryscaling": 1, + "devicesplitcount": 10, + "migstrategy": "none", + "filterdevices": { + "uuid": [], + "index": [] + } + } + ] + } + # If you want to use an existing ConfigMap, you can set the name here. + # If this is set, the chart will not create the ConfigMap and will use the existing one. + externalConfigName: "" + # The runtime class name to be used by the device plugin, and added to the pod.spec.runtimeClassName of applications utilizing NVIDIA GPUs + runtimeClassName: "" + # Whether to create runtime class, name comes from runtimeClassName when it is set + createRuntimeClass: false + migStrategy: "none" + disablecorelimit: "false" + passDeviceSpecsEnabled: false + deviceListStrategy: "envvar" + nvidiaHookPath: null + nvidiaDriverRoot: null + gdrcopyEnabled: null + gdsEnabled: null + mofedEnabled: null + + extraArgs: + - -v=4 + extraEnvs: {} + service: + type: NodePort # Default type is NodePort, can be changed to ClusterIP + httpPort: 31992 + labels: {} + annotations: {} + + pluginPath: /var/lib/kubelet/device-plugins + libPath: /usr/local/vgpu + + podAnnotations: {} + nvidiaNodeSelector: + gpu: "on" + tolerations: [] + # The updateStrategy for DevicePlugin DaemonSet. + # If you want to update the DaemonSet by manual, set type as "OnDelete". + # We recommend use OnDelete update strategy because DevicePlugin pod restart will cause business pod restart, this behavior is destructive. + # Otherwise, you can use RollingUpdate update strategy to rolling update DevicePlugin pod. + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + + resources: {} + # If you do want to specify resources, uncomment the following lines, adjust them as necessary. + # and remove the curly braces after 'resources:'. +# limits: +# cpu: 1000m +# memory: 1000Mi +# requests: +# cpu: 100m +# memory: 100Mi + +mockDevicePlugin: + enabled: false + image: + registry: "docker.io" + repository: "projecthami/mock-device-plugin" + tag: "1.0.1" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] +# If this option is enabled, the DRA will be installed and the scheduler extender will not be installed. +dra: + enabled: false + +# Configuration for the hami-dra subchart, which includes DRA drivers. +# This is part of config for DRA, please refer to the DRA documentation for more details. +hami-dra: + monitor: + enabled: true + drivers: + nvidia: + enabled: true + # If you are using gpu driver on host, you need to set this to false + containerDriver: true + image: + repository: projecthami/k8s-dra-driver + tag: "v0.0.1-dev" + +devices: + amd: + customresources: + - amd.com/gpu + - amd.com/gpu-memory + awsneuron: + customresources: + - aws.amazon.com/neuron + - aws.amazon.com/neuroncore + kunlun: + enabled: true + customresources: + - kunlunxin.com/xpu + - kunlunxin.com/vxpu + - kunlunxin.com/vxpu-memory + enflame: + enabled: true + customresources: + - enflame.com/vgcu + - enflame.com/vgcu-percentage + - enflame.com/gcu + mthreads: + enabled: true + customresources: + - mthreads.com/vgpu + nvidia: + gpuCorePolicy: default + libCudaLogLevel: 1 + ascend: + enabled: false + image: "" + imagePullPolicy: IfNotPresent + extraArgs: [] + nodeSelector: + ascend: "on" + tolerations: [] + customresources: + - huawei.com/Ascend910A + - huawei.com/Ascend910A-memory + - huawei.com/Ascend910B2 + - huawei.com/Ascend910B2-memory + - huawei.com/Ascend910B3 + - huawei.com/Ascend910B3-memory + - huawei.com/Ascend910B4 + - huawei.com/Ascend910B4-memory + - huawei.com/Ascend910B4-1 + - huawei.com/Ascend910B4-1-memory + - huawei.com/Ascend310P + - huawei.com/Ascend310P-memory + iluvatar: + enabled: false + customresources: + - iluvatar.ai/BI-V100-vgpu + - iluvatar.ai/BI-V100.vCore + - iluvatar.ai/BI-V100.vMem + - iluvatar.ai/BI-V150-vgpu + - iluvatar.ai/BI-V150.vCore + - iluvatar.ai/BI-V150.vMem + - iluvatar.ai/MR-V100-vgpu + - iluvatar.ai/MR-V100.vCore + - iluvatar.ai/MR-V100.vMem + - iluvatar.ai/MR-V50-vgpu + - iluvatar.ai/MR-V50.vCore + - iluvatar.ai/MR-V50.vMem diff --git a/packages/system/hami/values.yaml b/packages/system/hami/values.yaml new file mode 100644 index 00000000..1155e2cb --- /dev/null +++ b/packages/system/hami/values.yaml @@ -0,0 +1,8 @@ +hami: + devicePlugin: + runtimeClassName: nvidia + nodeConfiguration: + config: | + { + "nodeconfig": [] + } From a3e2fbd742d0c9f8bf6a255836913f080f42b826 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Wed, 22 Apr 2026 13:25:23 +0300 Subject: [PATCH 046/130] feat(kubernetes): integrate HAMi as optional addon Add HAMi HelmRelease to the kubernetes app as a toggleable addon. When enabled, GPU Operator's built-in device plugin is automatically disabled via a values merge to avoid conflicts with HAMi's own device plugin. The HelmRelease fails fast if GPU Operator is not enabled, since HAMi depends on it for driver management and container toolkit. Assisted-By: Claude Signed-off-by: Arsolitt --- .../templates/helmreleases/gpu-operator.yaml | 13 ++++- .../templates/helmreleases/hami.yaml | 49 +++++++++++++++++++ packages/apps/kubernetes/values.schema.json | 23 +++++++++ packages/apps/kubernetes/values.yaml | 8 +++ 4 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 packages/apps/kubernetes/templates/helmreleases/hami.yaml diff --git a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml index 5ef48912..995aa430 100644 --- a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml @@ -1,3 +1,11 @@ +{{- define "cozystack.defaultGpuOperatorValues" -}} +{{- if and (hasKey .Values.addons "hami") .Values.addons.hami.enabled }} +gpu-operator: + devicePlugin: + enabled: false +{{- end }} +{{- end }} + {{- if .Values.addons.gpuOperator.enabled }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease @@ -29,7 +37,10 @@ spec: force: true remediation: retries: -1 - {{- with .Values.addons.gpuOperator.valuesOverride }} + {{- $defaults := fromYaml (include "cozystack.defaultGpuOperatorValues" .) }} + {{- $overrides := deepCopy (default (dict) .Values.addons.gpuOperator.valuesOverride) }} + {{- $merged := mergeOverwrite (default (dict) $defaults) $overrides }} + {{- with $merged }} values: {{- toYaml . | nindent 4 }} {{- end }} diff --git a/packages/apps/kubernetes/templates/helmreleases/hami.yaml b/packages/apps/kubernetes/templates/helmreleases/hami.yaml new file mode 100644 index 00000000..1f7a5358 --- /dev/null +++ b/packages/apps/kubernetes/templates/helmreleases/hami.yaml @@ -0,0 +1,49 @@ +{{- if .Values.addons.hami.enabled }} +{{- if not .Values.addons.gpuOperator.enabled }} +{{- fail "addons.hami requires addons.gpuOperator to be enabled" }} +{{- end }} +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: {{ .Release.Name }}-hami + labels: + cozystack.io/repository: system + cozystack.io/target-cluster-name: {{ .Release.Name }} + sharding.fluxcd.io/key: tenants +spec: + releaseName: hami + chartRef: + kind: ExternalArtifact + name: cozystack-kubernetes-application-kubevirt-kubernetes-hami + namespace: cozy-system + kubeConfig: + secretRef: + name: {{ .Release.Name }}-admin-kubeconfig + key: super-admin.svc + targetNamespace: cozy-hami + storageNamespace: cozy-hami + interval: 5m + timeout: 10m + install: + createNamespace: true + remediation: + retries: -1 + upgrade: + force: true + remediation: + retries: -1 + {{- with .Values.addons.hami.valuesOverride }} + values: + {{- toYaml . | nindent 4 }} + {{- end }} + + dependsOn: + {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }} + - name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} + {{- end }} + - name: {{ .Release.Name }}-cilium + namespace: {{ .Release.Namespace }} + - name: {{ .Release.Name }}-gpu-operator + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/packages/apps/kubernetes/values.schema.json b/packages/apps/kubernetes/values.schema.json index 1beeff9c..3d7a17dd 100644 --- a/packages/apps/kubernetes/values.schema.json +++ b/packages/apps/kubernetes/values.schema.json @@ -149,6 +149,7 @@ "fluxcd", "gatewayAPI", "gpuOperator", + "hami", "ingressNginx", "monitoringAgents", "velero", @@ -268,6 +269,28 @@ } } }, + "hami": { + "description": "HAMi GPU virtualization middleware. Enables fractional GPU sharing (memory and compute isolation) for workloads. Requires GPU Operator. Note: workload containers must use glibc < 2.34 (not musl/Alpine) for full compute isolation.", + "type": "object", + "default": {}, + "required": [ + "enabled", + "valuesOverride" + ], + "properties": { + "enabled": { + "description": "Enable HAMi.", + "type": "boolean", + "default": false + }, + "valuesOverride": { + "description": "Custom Helm values overrides.", + "type": "object", + "default": {}, + "x-kubernetes-preserve-unknown-fields": true + } + } + }, "ingressNginx": { "description": "Ingress-NGINX controller.", "type": "object", diff --git a/packages/apps/kubernetes/values.yaml b/packages/apps/kubernetes/values.yaml index a67b5d69..f601b979 100644 --- a/packages/apps/kubernetes/values.yaml +++ b/packages/apps/kubernetes/values.yaml @@ -94,6 +94,10 @@ host: "" ## @field {bool} enabled - Enable FluxCD. ## @field {object} valuesOverride - Custom Helm values overrides. +## @typedef {struct} HAMiAddon - HAMi GPU virtualization middleware. +## @field {bool} enabled - Enable HAMi (requires GPU Operator). +## @field {object} valuesOverride - Custom Helm values overrides. + ## @typedef {struct} MonitoringAgentsAddon - Monitoring agents (Fluent Bit, VMAgents). ## @field {bool} enabled - Enable monitoring agents. ## @field {object} valuesOverride - Custom Helm values overrides. @@ -114,6 +118,7 @@ host: "" ## @field {GatewayAPIAddon} gatewayAPI - Gateway API addon. ## @field {IngressNginxAddon} ingressNginx - Ingress-NGINX controller. ## @field {GPUOperatorAddon} gpuOperator - NVIDIA GPU Operator. +## @field {HAMiAddon} hami - HAMi GPU virtualization middleware. ## @field {FluxCDAddon} fluxcd - FluxCD GitOps operator. ## @field {MonitoringAgentsAddon} monitoringAgents - Monitoring agents. ## @field {VerticalPodAutoscalerAddon} verticalPodAutoscaler - Vertical Pod Autoscaler. @@ -137,6 +142,9 @@ addons: gpuOperator: enabled: false valuesOverride: {} + hami: + enabled: false + valuesOverride: {} fluxcd: enabled: false valuesOverride: {} From 273eb7811a36fb69141d80360705f081a3276194 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Wed, 22 Apr 2026 13:25:29 +0300 Subject: [PATCH 047/130] docs(hami): document glibc < 2.34 limitation and upstream issues HAMi-core relies on _dl_sym (private glibc symbol removed in 2.34) for CUDA interception. This breaks compute isolation on modern container images using Ubuntu 22.04+ and makes Alpine/musl completely incompatible. Include upstream issue references and a compatibility matrix so users can make informed decisions about base image selection. Assisted-By: Claude Signed-off-by: Arsolitt --- packages/system/hami/README.md | 81 ++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 packages/system/hami/README.md diff --git a/packages/system/hami/README.md b/packages/system/hami/README.md new file mode 100644 index 00000000..283f9e80 --- /dev/null +++ b/packages/system/hami/README.md @@ -0,0 +1,81 @@ +# HAMi — GPU Virtualization Middleware + +[HAMi](https://github.com/Project-HAMi/HAMi) (Heterogeneous AI Computing Virtualization Middleware) is a CNCF Sandbox project that enables fractional GPU sharing in Kubernetes. It allows workloads to request specific amounts of GPU memory and compute cores instead of claiming entire GPUs. + +## Architecture + +HAMi consists of four components: + +- **MutatingWebhook** — intercepts pod creation, injects `schedulerName: hami-scheduler` +- **Scheduler Extender** — extends kube-scheduler with GPU-aware Filter and Bind logic +- **Device Plugin** (DaemonSet) — registers vGPU resources via the Kubernetes Device Plugin API +- **HAMi-core** (`libvgpu.so`) — `LD_PRELOAD` library injected into workload containers, intercepts CUDA API calls to enforce memory and compute isolation + +## Prerequisites + +- GPU Operator must be enabled (`addons.gpuOperator.enabled: true`) +- NVIDIA driver >= 440 on host nodes +- nvidia-container-toolkit configured as the default container runtime +- GPU nodes labeled with `gpu=on` + +## Known Limitations + +### glibc < 2.34 requirement for workload containers + +HAMi-core uses `LD_PRELOAD` to intercept `dlsym()` for CUDA symbol resolution. The fallback code path relies on `_dl_sym`, a private glibc internal symbol that was removed in glibc 2.34 when libdl and libpthread were merged into libc.so. + +**This limitation affects workload containers only**, not the host OS or HAMi's own components. + +| Distribution | glibc | Result | +| --------------- | ----- | -------------------------------------------- | +| Ubuntu 18.04 | 2.27 | Full isolation (memory + compute) | +| Ubuntu 20.04 | 2.31 | Full isolation (memory + compute) | +| Ubuntu 22.04 | 2.35 | Memory isolation works, compute breaks | +| Ubuntu 24.04 | 2.39 | Both memory and compute isolation break | +| Alpine (musl) | N/A | Completely incompatible (`dlvsym` absent) | + +Most modern ML/AI base images (CUDA 12.x, PyTorch 2.x, TensorFlow 2.x) use Ubuntu 22.04+ with glibc >= 2.35, which means compute isolation will not work with these images until the upstream fix is merged. + +**Upstream tracking issues:** + +- [HAMi-core#174](https://github.com/Project-HAMi/HAMi-core/issues/174) — `_dl_sym` removal breaks HAMi-core on glibc >= 2.34 +- [HAMi#1190](https://github.com/Project-HAMi/HAMi/issues/1190) — degraded isolation across glibc versions +- [HAMi#173](https://github.com/Project-HAMi/HAMi/issues/173) — documentation incorrectly states glibc < 2.30 (actual boundary is 2.34) + +### musl libc (Alpine) incompatibility + +HAMi-core is completely incompatible with musl libc. The `dlvsym()` function used by HAMi-core is a glibc extension not available in musl. Only glibc-based container images (Debian, Ubuntu, RHEL, etc.) can use HAMi GPU isolation. + +## Usage + +Enable HAMi in your tenant Kubernetes cluster values: + +```yaml +addons: + gpuOperator: + enabled: true + hami: + enabled: true +``` + +When HAMi is enabled, GPU Operator's built-in device plugin is automatically disabled to avoid conflicts. + +### Requesting fractional GPU resources + +```yaml +resources: + limits: + nvidia.com/gpu: 1 + nvidia.com/gpumem: 3000 # 3000 MB of GPU memory + nvidia.com/gpucores: 30 # 30% of GPU compute cores +``` + +## Parameters + +| Name | Description | Default | +| --- | --- | --- | +| `hami.devicePlugin.runtimeClassName` | RuntimeClass for device plugin pods | `nvidia` | +| `hami.devicePlugin.deviceSplitCount` | Max virtual GPUs per physical GPU | `10` | +| `hami.devicePlugin.deviceMemoryScaling` | Memory overcommit factor (> 1.0 enables overcommit) | `1` | +| `hami.scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy` | Node packing strategy (`binpack` or `spread`) | `binpack` | +| `hami.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy` | GPU packing strategy (`binpack` or `spread`) | `spread` | From 43035562efff34f18c3c439026fe2a233dc8d17b Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Wed, 22 Apr 2026 13:25:35 +0300 Subject: [PATCH 048/130] test(kubernetes): add helm-unittest tests for HAMi integration Cover HelmRelease rendering, gpuOperator dependency validation, ExternalArtifact chartRef, namespace targeting, dependency chain, valuesOverride passthrough, and automatic devicePlugin disable in GPU Operator when HAMi is active. Assisted-By: Claude Signed-off-by: Arsolitt --- .../tests/gpu_operator_hami_test.yaml | 80 +++++++++++ packages/apps/kubernetes/tests/hami_test.yaml | 136 ++++++++++++++++++ 2 files changed, 216 insertions(+) create mode 100644 packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml create mode 100644 packages/apps/kubernetes/tests/hami_test.yaml diff --git a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml new file mode 100644 index 00000000..33256ae6 --- /dev/null +++ b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml @@ -0,0 +1,80 @@ +suite: GPU Operator HelmRelease HAMi integration tests +templates: + - templates/helmreleases/gpu-operator.yaml +tests: + - it: should disable devicePlugin when hami is enabled + set: + addons: + gpuOperator: + enabled: true + valuesOverride: {} + hami: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.values.gpu-operator.devicePlugin.enabled + value: false + + - it: should not have values when hami is disabled and no overrides + set: + addons: + gpuOperator: + enabled: true + valuesOverride: {} + hami: + enabled: false + valuesOverride: {} + asserts: + - notExists: + path: spec.values + + - it: should allow user overrides to merge with hami defaults + set: + addons: + gpuOperator: + enabled: true + valuesOverride: + gpu-operator: + driver: + enabled: false + hami: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.values.gpu-operator.devicePlugin.enabled + value: false + - equal: + path: spec.values.gpu-operator.driver.enabled + value: false + + - it: should let user override devicePlugin back to true if needed + set: + addons: + gpuOperator: + enabled: true + valuesOverride: + gpu-operator: + devicePlugin: + enabled: true + hami: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.values.gpu-operator.devicePlugin.enabled + value: true + + - it: should not render when gpuOperator is disabled + set: + addons: + gpuOperator: + enabled: false + valuesOverride: {} + hami: + enabled: false + valuesOverride: {} + asserts: + - hasDocuments: + count: 0 diff --git a/packages/apps/kubernetes/tests/hami_test.yaml b/packages/apps/kubernetes/tests/hami_test.yaml new file mode 100644 index 00000000..e9d32236 --- /dev/null +++ b/packages/apps/kubernetes/tests/hami_test.yaml @@ -0,0 +1,136 @@ +suite: HAMi HelmRelease tests +templates: + - templates/helmreleases/hami.yaml +tests: + - it: should not render when hami is disabled + set: + addons: + hami: + enabled: false + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - hasDocuments: + count: 0 + + - it: should render HelmRelease when hami is enabled + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - hasDocuments: + count: 1 + - isKind: + of: HelmRelease + + - it: should fail when gpuOperator is not enabled + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: false + valuesOverride: {} + asserts: + - failedTemplate: + errorMessage: "addons.hami requires addons.gpuOperator to be enabled" + + - it: should have correct metadata labels + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: metadata.labels["cozystack.io/repository"] + value: system + - equal: + path: metadata.labels["sharding.fluxcd.io/key"] + value: tenants + + - it: should use ExternalArtifact chartRef + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.chartRef.kind + value: ExternalArtifact + - equal: + path: spec.chartRef.namespace + value: cozy-system + + - it: should target cozy-hami namespace + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.targetNamespace + value: cozy-hami + - equal: + path: spec.storageNamespace + value: cozy-hami + + - it: should depend on gpu-operator and cilium + release: + name: test + namespace: test-ns + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - contains: + path: spec.dependsOn + content: + name: test-cilium + namespace: test-ns + - contains: + path: spec.dependsOn + content: + name: test-gpu-operator + namespace: test-ns + + - it: should pass through valuesOverride + set: + addons: + hami: + enabled: true + valuesOverride: + hami: + devicePlugin: + deviceSplitCount: 5 + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.values.hami.devicePlugin.deviceSplitCount + value: 5 From c1508940bda27b2979639981fd25461c9d4824b3 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Thu, 23 Apr 2026 13:34:21 +0500 Subject: [PATCH 049/130] fix(etcd): remove destructive post-upgrade cert-regeneration hook The etcd chart shipped a `post-upgrade` Helm hook that `kubectl delete`d the etcd TLS chain (`etcd-{ca,peer-ca,client,peer,server}-tls`) and then deleted etcd pods on every chart upgrade, gated by a semver compare of an `etcd-deployed-version` ConfigMap against `2.6.1`. The hook was added as a one-shot migration for the chart `2.6.0 -> 2.6.1` transition. Since commit f871fbdb ("Remove versions_map logic") all chart versions are stamped as `0.0.0+`, which per semver is always `< 2.6.1`. The gate therefore always resolves to "update certs", firing the destructive hook on every etcd upgrade. On clusters running Kamaji-managed tenant control planes this wipes the etcd CA, cert-manager re-issues it, and tenant kube-apiservers hit `x509: certificate signed by unknown authority` against `etcd..svc:2379` until each tenant DataStore is manually re-reconciled. Commit 47d81f70 ("Disabled private key rotation in CA certs") already fixed the underlying `rotationPolicy: Always` issue the migration was papering over, so the hook has no remaining use. Remove the hook Job, its RBAC, the version ConfigMap it read, and add a helm-unittest suite under `packages/extra/etcd/tests/` that guards against re-introducing the hook or the version ConfigMap. Assisted-By: Claude Signed-off-by: Myasnikov Daniil --- packages/extra/etcd/Makefile | 3 ++ packages/extra/etcd/templates/hook/job.yaml | 39 ------------------- packages/extra/etcd/templates/hook/role.yaml | 26 ------------- .../etcd/templates/hook/rolebinding.yaml | 15 ------- .../etcd/templates/hook/serviceaccount.yaml | 7 ---- packages/extra/etcd/templates/version.yaml | 6 --- .../etcd/tests/no-post-upgrade-hook_test.yaml | 34 ++++++++++++++++ 7 files changed, 37 insertions(+), 93 deletions(-) delete mode 100644 packages/extra/etcd/templates/hook/job.yaml delete mode 100644 packages/extra/etcd/templates/hook/role.yaml delete mode 100644 packages/extra/etcd/templates/hook/rolebinding.yaml delete mode 100644 packages/extra/etcd/templates/hook/serviceaccount.yaml delete mode 100644 packages/extra/etcd/templates/version.yaml create mode 100644 packages/extra/etcd/tests/no-post-upgrade-hook_test.yaml diff --git a/packages/extra/etcd/Makefile b/packages/extra/etcd/Makefile index f37d6e1e..2b3ed61e 100644 --- a/packages/extra/etcd/Makefile +++ b/packages/extra/etcd/Makefile @@ -5,3 +5,6 @@ include ../../../hack/package.mk generate: cozyvalues-gen -v values.yaml -s values.schema.json -r README.md ../../../hack/update-crd.sh + +test: + helm unittest . diff --git a/packages/extra/etcd/templates/hook/job.yaml b/packages/extra/etcd/templates/hook/job.yaml deleted file mode 100644 index 3bf2f84b..00000000 --- a/packages/extra/etcd/templates/hook/job.yaml +++ /dev/null @@ -1,39 +0,0 @@ -{{- $shouldUpdateCerts := true }} -{{- $configMap := lookup "v1" "ConfigMap" .Release.Namespace "etcd-deployed-version" }} -{{- if $configMap }} - {{- $deployedVersion := index $configMap "data" "version" }} - {{- if $deployedVersion | semverCompare ">= 2.6.1" }} - {{- $shouldUpdateCerts = false }} - {{- end }} -{{- end }} - -{{- if $shouldUpdateCerts }} ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: etcd-hook - annotations: - helm.sh/hook: post-upgrade - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded -spec: - template: - metadata: - labels: - policy.cozystack.io/allow-to-apiserver: "true" - spec: - serviceAccountName: etcd-hook - containers: - - name: kubectl - image: docker.io/alpine/k8s:1.33.4 - command: - - sh - args: - - -exc - - |- - kubectl --namespace={{ .Release.Namespace }} delete secrets etcd-ca-tls etcd-peer-ca-tls - sleep 10 - kubectl --namespace={{ .Release.Namespace }} delete secrets etcd-client-tls etcd-peer-tls etcd-server-tls - kubectl --namespace={{ .Release.Namespace }} delete pods --selector=app.kubernetes.io/instance=etcd,app.kubernetes.io/managed-by=etcd-operator,app.kubernetes.io/name=etcd,cozystack.io/service=etcd - restartPolicy: Never -{{- end }} diff --git a/packages/extra/etcd/templates/hook/role.yaml b/packages/extra/etcd/templates/hook/role.yaml deleted file mode 100644 index 327eeadb..00000000 --- a/packages/extra/etcd/templates/hook/role.yaml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - annotations: - helm.sh/hook: post-upgrade - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded - name: etcd-hook -rules: -- apiGroups: - - "" - resources: - - secrets - - pods - verbs: - - get - - list - - watch - - delete -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch diff --git a/packages/extra/etcd/templates/hook/rolebinding.yaml b/packages/extra/etcd/templates/hook/rolebinding.yaml deleted file mode 100644 index 0ee0ffd1..00000000 --- a/packages/extra/etcd/templates/hook/rolebinding.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: etcd-hook - annotations: - helm.sh/hook: post-upgrade - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: etcd-hook -subjects: - - kind: ServiceAccount - name: etcd-hook - namespace: {{ .Release.Namespace | quote }} diff --git a/packages/extra/etcd/templates/hook/serviceaccount.yaml b/packages/extra/etcd/templates/hook/serviceaccount.yaml deleted file mode 100644 index 552fb5fc..00000000 --- a/packages/extra/etcd/templates/hook/serviceaccount.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: etcd-hook - annotations: - helm.sh/hook: post-upgrade - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded diff --git a/packages/extra/etcd/templates/version.yaml b/packages/extra/etcd/templates/version.yaml deleted file mode 100644 index cc9375bb..00000000 --- a/packages/extra/etcd/templates/version.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: etcd-deployed-version -data: - version: {{ .Chart.Version }} diff --git a/packages/extra/etcd/tests/no-post-upgrade-hook_test.yaml b/packages/extra/etcd/tests/no-post-upgrade-hook_test.yaml new file mode 100644 index 00000000..0e4f5aa3 --- /dev/null +++ b/packages/extra/etcd/tests/no-post-upgrade-hook_test.yaml @@ -0,0 +1,34 @@ +suite: etcd chart does not ship a destructive post-upgrade cert-regeneration hook + +release: + name: etcd + namespace: tenant-root + +templates: + - templates/check-release-name.yaml + - templates/dashboard-resourcemap.yaml + - templates/datastore.yaml + - templates/etcd-defrag.yaml + - templates/hook/job.yaml + - templates/podscrape.yaml + - templates/prometheus-rules.yaml + - templates/version.yaml + +tests: + - it: renders no Job named etcd-hook + documentSelector: + path: metadata.name + value: etcd-hook + skipEmptyTemplates: true + asserts: + - hasDocuments: + count: 0 + + - it: renders no ConfigMap named etcd-deployed-version + documentSelector: + path: metadata.name + value: etcd-deployed-version + skipEmptyTemplates: true + asserts: + - hasDocuments: + count: 0 From 9222b6feda4d97eda7481569d5d9a466b69fec94 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Thu, 23 Apr 2026 14:32:20 +0500 Subject: [PATCH 050/130] ci(api): pre-fetch k8s.io/code-generator in codegen drift job hack/update-codegen.sh sources kube_codegen.sh from the Go module cache at ~/go/pkg/mod/k8s.io/code-generator@vX.Y.Z/, but the module is not declared in go.mod so a fresh runner has nothing to source from. Add a workflow step that parses the pinned version out of the script and pulls the module into the cache before running make generate. Assisted-By: Claude Signed-off-by: Myasnikov Daniil --- .github/workflows/codegen-drift.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/codegen-drift.yml b/.github/workflows/codegen-drift.yml index bd7927f7..b08f5d7e 100644 --- a/.github/workflows/codegen-drift.yml +++ b/.github/workflows/codegen-drift.yml @@ -30,6 +30,17 @@ jobs: go-version-file: go.mod cache: true + - name: Pre-fetch k8s.io/code-generator module + # hack/update-codegen.sh sources kube_codegen.sh from the Go module cache. + # The module is not declared in go.mod, so fetch it explicitly at the + # version pinned in the script. + run: | + version=$(grep -oP 'code-generator@\Kv[0-9.]+' hack/update-codegen.sh) + tmpdir=$(mktemp -d) + cd "$tmpdir" + go mod init codegen-fetch + go get "k8s.io/code-generator@${version}" + - name: Run make generate run: make generate From adc7abe5c16fc81ce1c063d153d9a920c0aac097 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 23 Apr 2026 14:55:21 +0300 Subject: [PATCH 051/130] feat(ingress): add loadBalancer exposure mode via CiliumLoadBalancerIPPool Service.spec.externalIPs is deprecated upstream in Kubernetes v1.36 (KEP-5707, kubernetes#137293). The AllowServiceExternalIPs feature gate is expected to default to off around v1.40 and the implementation to be removed around v1.43. For bare-metal installs that rely on externalIPs today, cozystack needs a migration path. This change adds an opt-in 'loadBalancer' exposure mode for the ingress-nginx Service: - New platform value 'publishing.exposure' (enum: externalIPs | loadBalancer, default externalIPs). Plumbed through cozystack-values into each tenant's ingress HelmRelease via the new 'expose-mode' key. - Unknown values and loadBalancer with an empty externalIPs list fail the chart render with explicit error messages, rather than silently producing a broken Service. - When exposure=loadBalancer and the current namespace matches publishing.ingressName, the Service becomes type: LoadBalancer with externalTrafficPolicy: Local. - A new template renders a CiliumLoadBalancerIPPool whose blocks come from publishing.externalIPs (IPv4 addresses get /32, IPv6 addresses get /128) and whose serviceSelector uses Cilium's synthetic io.kubernetes.service.namespace key combined with the standard app.kubernetes.io/name: ingress-nginx label. No custom label is written to the Service itself, avoiding cross-tenant collisions from user-defined labels. Default behaviour is unchanged: without opting in, the Service is still ClusterIP + spec.externalIPs as today. Scope: only ingress-nginx is migrated by this setting. Other cozystack components that still write Service.spec.externalIPs directly (notably the vpn app) must be migrated separately before the v1.40 feature gate flip. Tests: packages/extra/ingress/tests/exposure_test.yaml adds 13 helm-unittest cases covering both modes, IPv4/IPv6, empty-token filtering, unknown-mode rejection, and the non-matching-namespace fallback. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- packages/core/platform/templates/apps.yaml | 1 + packages/core/platform/values.yaml | 35 +++ packages/extra/ingress/Makefile | 3 + packages/extra/ingress/README.md | 14 + .../ingress/templates/cilium-lb-pool.yaml | 25 ++ .../ingress/templates/nginx-ingress.yaml | 21 +- .../extra/ingress/tests/exposure_test.yaml | 251 ++++++++++++++++++ 7 files changed, 348 insertions(+), 2 deletions(-) create mode 100644 packages/extra/ingress/templates/cilium-lb-pool.yaml create mode 100644 packages/extra/ingress/tests/exposure_test.yaml diff --git a/packages/core/platform/templates/apps.yaml b/packages/core/platform/templates/apps.yaml index 7ee0de52..6e1ae25a 100644 --- a/packages/core/platform/templates/apps.yaml +++ b/packages/core/platform/templates/apps.yaml @@ -30,6 +30,7 @@ stringData: expose-services: {{ .Values.publishing.exposedServices | join "," | quote }} expose-ingress: {{ .Values.publishing.ingressName | quote }} expose-external-ips: {{ .Values.publishing.externalIPs | join "," | quote }} + expose-mode: {{ .Values.publishing.exposure | default "externalIPs" | quote }} cluster-domain: {{ .Values.networking.clusterDomain | quote }} api-server-endpoint: {{ .Values.publishing.apiServerEndpoint | quote }} {{- with .Values.branding }} diff --git a/packages/core/platform/values.yaml b/packages/core/platform/values.yaml index f33926db..605e23bc 100644 --- a/packages/core/platform/values.yaml +++ b/packages/core/platform/values.yaml @@ -45,6 +45,41 @@ publishing: - cdi-uploadproxy apiServerEndpoint: "" # example: "https://api.example.org" externalIPs: [] + # Exposure mode for the ingress-nginx Service. When "externalIPs" (current + # default) is selected, the Service is created as ClusterIP with + # Service.spec.externalIPs set from publishing.externalIPs. When + # "loadBalancer" is selected, the Service is type: LoadBalancer and a + # CiliumLoadBalancerIPPool makes those same addresses allocatable via LB IPAM. + # + # Service.spec.externalIPs is deprecated upstream in Kubernetes v1.36 + # (KEP-5707). The AllowServiceExternalIPs feature gate is expected to default + # to false around v1.40 and the implementation removed around v1.43 — switch + # to "loadBalancer" before upgrading past v1.40. + # + # Caveats for the "loadBalancer" mode: + # - publishing.externalIPs must contain at least one non-empty address, + # otherwise the chart render fails with an explicit error (a LoadBalancer + # Service without a pool would sit in forever). + # - The ingress-nginx Service is created with externalTrafficPolicy: Local + # to preserve the client source IP. Traffic arriving on a node that does + # not host an ingress-nginx pod is dropped, so the external IP must be + # routed to a node that runs the ingress pod (floating IP / keepalived / + # upstream router / podAntiAffinity). + # - Cilium does NOT announce the IP on its own unless L2 announcements or + # BGP are enabled in the Cilium values (disabled by default in Cozystack). + # This mode assumes the operator already routes the externalIPs to a + # cluster node; enabling announcements is out of scope for this setting. + # - Switching this value on a running cluster causes the ingress-nginx + # Service to be recreated (the HelmRelease has upgrade.force: true and + # the Service kind changes between ClusterIP and LoadBalancer). Expect a + # brief interruption of ingress traffic during the flip. + # + # Scope: this setting only controls the ingress-nginx Service. Other + # cozystack components that currently write Service.spec.externalIPs directly + # (e.g. the vpn app at packages/apps/vpn/templates/service.yaml) are NOT + # migrated by flipping this value and must be addressed separately before + # the AllowServiceExternalIPs feature gate flips to off in ~v1.40. + exposure: externalIPs # "externalIPs" or "loadBalancer" certificates: solver: http01 # "http01" or "dns01" issuerName: letsencrypt-prod diff --git a/packages/extra/ingress/Makefile b/packages/extra/ingress/Makefile index 958ce484..65b53c03 100644 --- a/packages/extra/ingress/Makefile +++ b/packages/extra/ingress/Makefile @@ -10,3 +10,6 @@ get-cloudflare-ips: generate: cozyvalues-gen -v values.yaml -s values.schema.json -r README.md ../../../hack/update-crd.sh + +test: + helm unittest . diff --git a/packages/extra/ingress/README.md b/packages/extra/ingress/README.md index 0e786dfb..c541d8e9 100644 --- a/packages/extra/ingress/README.md +++ b/packages/extra/ingress/README.md @@ -14,3 +14,17 @@ | `resources.memory` | Memory (RAM) available to each replica. | `quantity` | `""` | | `resourcesPreset` | Default sizing preset used when `resources` is omitted. | `string` | `micro` | + +## Exposure mode + +The ingress Service type is driven by the cluster-wide `publishing.exposure` value in the platform chart, not by any key in this package. Two modes exist: + +- `externalIPs` (default) has three rendered shapes: + - Release namespace matches `publishing.ingressName` AND `publishing.externalIPs` is non-empty → Service is `ClusterIP` with `Service.spec.externalIPs` set from that list and `externalTrafficPolicy: Cluster`. + - Release namespace matches `publishing.ingressName` but `publishing.externalIPs` is empty → Service falls back to `type: LoadBalancer` with `externalTrafficPolicy: Local`. + - Release namespace does not match `publishing.ingressName` (non-root tenants) → Service is `type: LoadBalancer` with `externalTrafficPolicy: Local`. + `Service.spec.externalIPs` is deprecated upstream in Kubernetes v1.36 (KEP-5707); plan migration before v1.40. +- `loadBalancer` — Service is `type: LoadBalancer` with `externalTrafficPolicy: Local`, and a `CiliumLoadBalancerIPPool` makes the addresses in `publishing.externalIPs` allocatable via Cilium LB IPAM. Requires `publishing.externalIPs` to contain at least one non-empty address (render fails otherwise) and assumes the addresses are already routed to a cluster node (floating IP / upstream router). See the inline comment on `publishing.exposure` in the platform chart for full caveats, including the note that switching the value on a running cluster causes the ingress Service to be recreated. + +This setting only migrates ingress-nginx away from `Service.spec.externalIPs`. Other cozystack components that use the same deprecated field (e.g. the `vpn` app) must be migrated separately before Kubernetes v1.40 flips the `AllowServiceExternalIPs` feature gate off. + diff --git a/packages/extra/ingress/templates/cilium-lb-pool.yaml b/packages/extra/ingress/templates/cilium-lb-pool.yaml new file mode 100644 index 00000000..a383922b --- /dev/null +++ b/packages/extra/ingress/templates/cilium-lb-pool.yaml @@ -0,0 +1,25 @@ +{{- $exposeIngress := (index .Values._cluster "expose-ingress") | default "tenant-root" }} +{{- $exposeMode := (index .Values._cluster "expose-mode") | default "externalIPs" }} +{{- $exposeExternalIPs := (index .Values._cluster "expose-external-ips") | default "" | nospace }} +{{- $exposeIPsList := list }} +{{- range splitList "," $exposeExternalIPs }} + {{- $ip := . | trim }} + {{- if $ip }} + {{- $exposeIPsList = append $exposeIPsList $ip }} + {{- end }} +{{- end }} +{{- if and (eq $exposeMode "loadBalancer") (eq $exposeIngress .Release.Namespace) $exposeIPsList }} +apiVersion: cilium.io/v2 +kind: CiliumLoadBalancerIPPool +metadata: + name: {{ trimPrefix "tenant-" .Release.Namespace }}-ingress +spec: + blocks: + {{- range $exposeIPsList }} + - cidr: {{ . }}/{{ if contains ":" . }}128{{ else }}32{{ end }} + {{- end }} + serviceSelector: + matchLabels: + "io.kubernetes.service.namespace": {{ .Release.Namespace | quote }} + "app.kubernetes.io/name": ingress-nginx +{{- end }} diff --git a/packages/extra/ingress/templates/nginx-ingress.yaml b/packages/extra/ingress/templates/nginx-ingress.yaml index ca50d276..8e1f00fb 100644 --- a/packages/extra/ingress/templates/nginx-ingress.yaml +++ b/packages/extra/ingress/templates/nginx-ingress.yaml @@ -1,5 +1,19 @@ {{- $exposeIngress := (index .Values._cluster "expose-ingress") | default "tenant-root" }} {{- $exposeExternalIPs := (index .Values._cluster "expose-external-ips") | default "" | nospace }} +{{- $exposeMode := (index .Values._cluster "expose-mode") | default "externalIPs" }} +{{- $exposeIPsList := list }} +{{- range splitList "," $exposeExternalIPs }} + {{- $ip := . | trim }} + {{- if $ip }} + {{- $exposeIPsList = append $exposeIPsList $ip }} + {{- end }} +{{- end }} +{{- if not (has $exposeMode (list "externalIPs" "loadBalancer")) }} +{{- fail (printf "unknown publishing.exposure mode %q: must be \"externalIPs\" or \"loadBalancer\"" $exposeMode) }} +{{- end }} +{{- if and (eq $exposeMode "loadBalancer") (eq $exposeIngress .Release.Namespace) (not $exposeIPsList) }} +{{- fail "publishing.exposure=loadBalancer requires publishing.externalIPs to contain at least one non-empty address: the CiliumLoadBalancerIPPool has nothing to advertise and the Service would stay in ." }} +{{- end }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -41,9 +55,12 @@ spec: enabled: false {{- end }} service: - {{- if and (eq $exposeIngress .Release.Namespace) $exposeExternalIPs }} + {{- if and (eq $exposeIngress .Release.Namespace) (eq $exposeMode "loadBalancer") }} + type: LoadBalancer + externalTrafficPolicy: Local + {{- else if and (eq $exposeIngress .Release.Namespace) $exposeIPsList }} externalIPs: - {{- toYaml (splitList "," $exposeExternalIPs) | nindent 12 }} + {{- toYaml $exposeIPsList | nindent 12 }} type: ClusterIP externalTrafficPolicy: Cluster {{- else }} diff --git a/packages/extra/ingress/tests/exposure_test.yaml b/packages/extra/ingress/tests/exposure_test.yaml new file mode 100644 index 00000000..780e0715 --- /dev/null +++ b/packages/extra/ingress/tests/exposure_test.yaml @@ -0,0 +1,251 @@ +suite: ingress exposure modes +templates: + - templates/nginx-ingress.yaml + - templates/cilium-lb-pool.yaml + +release: + name: ingress + namespace: tenant-root + +tests: + - it: default exposure (externalIPs) renders ClusterIP Service with spec.externalIPs and no CiliumLoadBalancerIPPool + set: + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10,192.0.2.11" + asserts: + - template: templates/nginx-ingress.yaml + equal: + path: spec.values.ingress-nginx.controller.service.type + value: ClusterIP + - template: templates/nginx-ingress.yaml + equal: + path: spec.values.ingress-nginx.controller.service.externalTrafficPolicy + value: Cluster + - template: templates/nginx-ingress.yaml + equal: + path: spec.values.ingress-nginx.controller.service.externalIPs + value: + - 192.0.2.10 + - 192.0.2.11 + - template: templates/nginx-ingress.yaml + notExists: + path: spec.values.ingress-nginx.controller.service.labels + - template: templates/cilium-lb-pool.yaml + hasDocuments: + count: 0 + + - it: legacy config without expose-mode falls back to externalIPs behavior + set: + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10" + asserts: + - template: templates/nginx-ingress.yaml + equal: + path: spec.values.ingress-nginx.controller.service.type + value: ClusterIP + - template: templates/cilium-lb-pool.yaml + hasDocuments: + count: 0 + + - it: externalIPs mode in a namespace other than publishing.ingressName renders LoadBalancer fallback without externalIPs + set: + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10" + release: + namespace: tenant-other + asserts: + - template: templates/nginx-ingress.yaml + equal: + path: spec.values.ingress-nginx.controller.service.type + value: LoadBalancer + - template: templates/nginx-ingress.yaml + equal: + path: spec.values.ingress-nginx.controller.service.externalTrafficPolicy + value: Local + - template: templates/nginx-ingress.yaml + notExists: + path: spec.values.ingress-nginx.controller.service.externalIPs + - template: templates/cilium-lb-pool.yaml + hasDocuments: + count: 0 + + - it: loadBalancer mode renders LoadBalancer Service with lb-pool label and a v2 CiliumLoadBalancerIPPool + set: + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10,192.0.2.11" + expose-mode: loadBalancer + asserts: + - template: templates/nginx-ingress.yaml + equal: + path: spec.values.ingress-nginx.controller.service.type + value: LoadBalancer + - template: templates/nginx-ingress.yaml + equal: + path: spec.values.ingress-nginx.controller.service.externalTrafficPolicy + value: Local + - template: templates/nginx-ingress.yaml + notExists: + path: spec.values.ingress-nginx.controller.service.labels + - template: templates/nginx-ingress.yaml + notExists: + path: spec.values.ingress-nginx.controller.service.externalIPs + - template: templates/cilium-lb-pool.yaml + hasDocuments: + count: 1 + - template: templates/cilium-lb-pool.yaml + equal: + path: apiVersion + value: cilium.io/v2 + - template: templates/cilium-lb-pool.yaml + equal: + path: kind + value: CiliumLoadBalancerIPPool + - template: templates/cilium-lb-pool.yaml + equal: + path: metadata.name + value: root-ingress + - template: templates/cilium-lb-pool.yaml + equal: + path: spec.blocks + value: + - cidr: 192.0.2.10/32 + - cidr: 192.0.2.11/32 + - template: templates/cilium-lb-pool.yaml + equal: + path: spec.serviceSelector.matchLabels["io.kubernetes.service.namespace"] + value: tenant-root + - template: templates/cilium-lb-pool.yaml + equal: + path: spec.serviceSelector.matchLabels["app.kubernetes.io/name"] + value: ingress-nginx + + - it: loadBalancer mode with IPv6 address emits /128 CIDR + set: + _cluster: + expose-ingress: tenant-root + expose-external-ips: "2001:db8::1" + expose-mode: loadBalancer + asserts: + - template: templates/cilium-lb-pool.yaml + equal: + path: spec.blocks + value: + - cidr: 2001:db8::1/128 + + - it: loadBalancer mode with mixed IPv4 and IPv6 emits correct CIDR per family + set: + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10,2001:db8::1" + expose-mode: loadBalancer + asserts: + - template: templates/cilium-lb-pool.yaml + equal: + path: spec.blocks + value: + - cidr: 192.0.2.10/32 + - cidr: 2001:db8::1/128 + + - it: loadBalancer mode without externalIPs fails chart render with explicit message + set: + _cluster: + expose-ingress: tenant-root + expose-external-ips: "" + expose-mode: loadBalancer + asserts: + - template: templates/nginx-ingress.yaml + failedTemplate: + errorMessage: "publishing.exposure=loadBalancer requires publishing.externalIPs to contain at least one non-empty address: the CiliumLoadBalancerIPPool has nothing to advertise and the Service would stay in ." + + - it: unknown exposure mode is rejected with a clear error (case-sensitive enum) + set: + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10" + expose-mode: LoadBalancer + asserts: + - template: templates/nginx-ingress.yaml + failedTemplate: + errorMessage: "unknown publishing.exposure mode \"LoadBalancer\": must be \"externalIPs\" or \"loadBalancer\"" + + - it: another typo in exposure mode also fails + set: + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10" + expose-mode: loadbalancer + asserts: + - template: templates/nginx-ingress.yaml + failedTemplate: + errorMessage: "unknown publishing.exposure mode \"loadbalancer\": must be \"externalIPs\" or \"loadBalancer\"" + + - it: loadBalancer mode in a namespace other than publishing.ingressName falls back to LoadBalancer Service without pool + set: + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10" + expose-mode: loadBalancer + release: + namespace: tenant-other + asserts: + - template: templates/nginx-ingress.yaml + equal: + path: spec.values.ingress-nginx.controller.service.type + value: LoadBalancer + - template: templates/nginx-ingress.yaml + equal: + path: spec.values.ingress-nginx.controller.service.externalTrafficPolicy + value: Local + - template: templates/nginx-ingress.yaml + notExists: + path: spec.values.ingress-nginx.controller.service.labels + - template: templates/nginx-ingress.yaml + notExists: + path: spec.values.ingress-nginx.controller.service.externalIPs + - template: templates/cilium-lb-pool.yaml + hasDocuments: + count: 0 + + - it: loadBalancer mode filters out empty entries from externalIPs (trailing comma, leading comma) + set: + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10,,192.0.2.11," + expose-mode: loadBalancer + asserts: + - template: templates/cilium-lb-pool.yaml + hasDocuments: + count: 1 + - template: templates/cilium-lb-pool.yaml + equal: + path: spec.blocks + value: + - cidr: 192.0.2.10/32 + - cidr: 192.0.2.11/32 + + - it: loadBalancer mode with only-empty externalIPs fails chart render (comma-only input) + set: + _cluster: + expose-ingress: tenant-root + expose-external-ips: ",," + expose-mode: loadBalancer + asserts: + - template: templates/nginx-ingress.yaml + failedTemplate: + errorMessage: "publishing.exposure=loadBalancer requires publishing.externalIPs to contain at least one non-empty address: the CiliumLoadBalancerIPPool has nothing to advertise and the Service would stay in ." + + - it: externalIPs mode also filters out empty entries (trailing comma) + set: + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10," + asserts: + - template: templates/nginx-ingress.yaml + equal: + path: spec.values.ingress-nginx.controller.service.externalIPs + value: + - 192.0.2.10 From 64a3edff01533a0483db31686a6385dffa81ee55 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Thu, 23 Apr 2026 18:13:10 +0300 Subject: [PATCH 052/130] fix(api): prevent IDOR in TenantNamespace Get and Watch handlers Fixed two IDOR vulnerabilities allowing authenticated users to access metadata of any tenant namespace without proper authorization. Changes: - Added hasAccessToNamespace() for efficient single-namespace access checks - Get() now verifies access before returning namespace metadata - Watch() filters events per-namespace with proper authorization - Returns NotFound (not Forbidden) to prevent tenant enumeration Performance optimization: - hasAccessToNamespace() lists RoleBindings only in target namespace instead of listing all cluster RoleBindings (order of magnitude faster) - Watch handler logs authorization errors for security audit Additional fixes: - Handle ServiceAccount subjects with empty namespace correctly - Add klog error logging for failed authorization checks Signed-off-by: IvanHunters --- pkg/registry/core/tenantnamespace/rest.go | 89 ++++++++++++++++++++++- 1 file changed, 87 insertions(+), 2 deletions(-) diff --git a/pkg/registry/core/tenantnamespace/rest.go b/pkg/registry/core/tenantnamespace/rest.go index f1ed3fab..7724c0e1 100644 --- a/pkg/registry/core/tenantnamespace/rest.go +++ b/pkg/registry/core/tenantnamespace/rest.go @@ -24,6 +24,7 @@ import ( "k8s.io/apimachinery/pkg/watch" "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/rest" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" corev1alpha1 "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1" @@ -123,8 +124,18 @@ func (r *REST) Get( return nil, apierrors.NewNotFound(r.gvr.GroupResource(), name) } + // Check if user has access to this namespace + hasAccess, err := r.hasAccessToNamespace(ctx, name) + if err != nil { + return nil, err + } + if !hasAccess { + // Return NotFound instead of Forbidden to prevent enumeration + return nil, apierrors.NewNotFound(r.gvr.GroupResource(), name) + } + ns := &corev1.Namespace{} - err := r.c.Get(ctx, types.NamespacedName{Namespace: "", Name: name}, ns, &client.GetOptions{Raw: opts}) + err = r.c.Get(ctx, types.NamespacedName{Namespace: "", Name: name}, ns, &client.GetOptions{Raw: opts}) if err != nil { return nil, err } @@ -189,6 +200,17 @@ func (r *REST) Watch(ctx context.Context, opts *metainternal.ListOptions) (watch continue } + // Check if user has access to this namespace + hasAccess, err := r.hasAccessToNamespace(ctx, ns.Name) + if err != nil { + klog.Errorf("Failed to check access for namespace %s in watch: %v", ns.Name, err) + continue + } + if !hasAccess { + // User doesn't have access, skip this event + continue + } + out := &corev1alpha1.TenantNamespace{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1alpha1.SchemeGroupVersion.String(), @@ -359,7 +381,11 @@ func (r *REST) filterAccessible( break subjectLoop } case "ServiceAccount": - if u.GetName() == fmt.Sprintf("system:serviceaccount:%s:%s", subj.Namespace, subj.Name) { + saNamespace := subj.Namespace + if saNamespace == "" { + saNamespace = rbs.Items[i].Namespace + } + if u.GetName() == fmt.Sprintf("system:serviceaccount:%s:%s", saNamespace, subj.Name) { allowedNameSet[rbs.Items[i].Namespace] = struct{}{} break subjectLoop } @@ -373,6 +399,65 @@ func (r *REST) filterAccessible( return allowed, nil } +// hasAccessToNamespace checks if the user has access to a single namespace. +// This is optimized for Get/Watch operations where we check one namespace at a time. +// It lists RoleBindings only in the target namespace instead of all cluster RoleBindings. +func (r *REST) hasAccessToNamespace( + ctx context.Context, + namespace string, +) (bool, error) { + u, ok := request.UserFrom(ctx) + if !ok { + return false, fmt.Errorf("user missing in context") + } + + // Check privileged groups + groups := make(map[string]struct{}) + for _, group := range u.GetGroups() { + groups[group] = struct{}{} + } + if _, ok := groups["system:masters"]; ok { + return true, nil + } + if _, ok := groups["cozystack-cluster-admin"]; ok { + return true, nil + } + + // List RoleBindings only in the target namespace + rbs := &rbacv1.RoleBindingList{} + err := r.c.List(ctx, rbs, client.InNamespace(namespace)) + if err != nil { + return false, fmt.Errorf("failed to list rolebindings in %s: %w", namespace, err) + } + + // Check if user is in any RoleBinding subjects + for i := range rbs.Items { + for j := range rbs.Items[i].Subjects { + subj := rbs.Items[i].Subjects[j] + switch subj.Kind { + case "Group": + if _, ok := groups[subj.Name]; ok { + return true, nil + } + case "User": + if subj.Name == u.GetName() { + return true, nil + } + case "ServiceAccount": + saNamespace := subj.Namespace + if saNamespace == "" { + saNamespace = rbs.Items[i].Namespace + } + if u.GetName() == fmt.Sprintf("system:serviceaccount:%s:%s", saNamespace, subj.Name) { + return true, nil + } + } + } + } + + return false, nil +} + // ----------------------------------------------------------------------------- // Boiler-plate // ----------------------------------------------------------------------------- From 5b2501db91c82d8f9ce1d7187325927f7504847c Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Thu, 23 Apr 2026 18:17:47 +0300 Subject: [PATCH 053/130] test(api): add security tests for TenantNamespace IDOR fix Added comprehensive unit tests for authorization logic: - hasAccessToNamespace() tests: - User subject access (positive and negative cases) - Group subject access - ServiceAccount subject access - ServiceAccount with empty namespace (defaults to RoleBinding ns) - Privileged groups (system:masters, cozystack-cluster-admin) - Get() handler tests: - Returns namespace when user has access - Returns NotFound when user lacks access (not Forbidden) - Returns NotFound for non-tenant namespaces All tests verify that authorization correctly enforces RoleBinding-based access control and prevents IDOR vulnerabilities. Signed-off-by: IvanHunters --- .../core/tenantnamespace/rest_test.go | 461 ++++++++++++++++++ 1 file changed, 461 insertions(+) diff --git a/pkg/registry/core/tenantnamespace/rest_test.go b/pkg/registry/core/tenantnamespace/rest_test.go index 7f2979bc..eb678949 100644 --- a/pkg/registry/core/tenantnamespace/rest_test.go +++ b/pkg/registry/core/tenantnamespace/rest_test.go @@ -3,10 +3,18 @@ package tenantnamespace import ( + "context" "testing" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/apiserver/pkg/endpoints/request" + "sigs.k8s.io/controller-runtime/pkg/client/fake" ) func TestMakeListSortsAlphabetically(t *testing.T) { @@ -38,3 +46,456 @@ func TestMakeListSortsAlphabetically(t *testing.T) { } } } + +// Security tests for IDOR fix + +func TestHasAccessToNamespace_WithUserAccess(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = rbacv1.AddToScheme(scheme) + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, + } + + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-binding", + Namespace: "tenant-test", + }, + Subjects: []rbacv1.Subject{ + {Kind: "User", Name: "test-user", APIGroup: "rbac.authorization.k8s.io"}, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: "test-role", + }, + } + + client := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(ns, rb). + Build() + + r := &REST{ + c: client, + gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, + } + + u := &user.DefaultInfo{ + Name: "test-user", + Groups: []string{"system:authenticated"}, + } + ctx := request.WithUser(context.Background(), u) + + hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !hasAccess { + t.Error("expected user to have access, but got false") + } +} + +func TestHasAccessToNamespace_WithoutAccess(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = rbacv1.AddToScheme(scheme) + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, + } + + // RoleBinding for different user + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-binding", + Namespace: "tenant-test", + }, + Subjects: []rbacv1.Subject{ + {Kind: "User", Name: "other-user", APIGroup: "rbac.authorization.k8s.io"}, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: "test-role", + }, + } + + client := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(ns, rb). + Build() + + r := &REST{ + c: client, + gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, + } + + u := &user.DefaultInfo{ + Name: "test-user", + Groups: []string{"system:authenticated"}, + } + ctx := request.WithUser(context.Background(), u) + + hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if hasAccess { + t.Error("expected user to NOT have access, but got true") + } +} + +func TestHasAccessToNamespace_WithGroupAccess(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = rbacv1.AddToScheme(scheme) + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, + } + + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-binding", + Namespace: "tenant-test", + }, + Subjects: []rbacv1.Subject{ + {Kind: "Group", Name: "test-group", APIGroup: "rbac.authorization.k8s.io"}, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: "test-role", + }, + } + + client := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(ns, rb). + Build() + + r := &REST{ + c: client, + gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, + } + + u := &user.DefaultInfo{ + Name: "test-user", + Groups: []string{"system:authenticated", "test-group"}, + } + ctx := request.WithUser(context.Background(), u) + + hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !hasAccess { + t.Error("expected user to have access via group, but got false") + } +} + +func TestHasAccessToNamespace_SystemMasters(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = rbacv1.AddToScheme(scheme) + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, + } + + client := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(ns). + Build() + + r := &REST{ + c: client, + gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, + } + + u := &user.DefaultInfo{ + Name: "admin", + Groups: []string{"system:masters"}, + } + ctx := request.WithUser(context.Background(), u) + + hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !hasAccess { + t.Error("expected system:masters to have access, but got false") + } +} + +func TestHasAccessToNamespace_CozyAdminGroup(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = rbacv1.AddToScheme(scheme) + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, + } + + client := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(ns). + Build() + + r := &REST{ + c: client, + gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, + } + + u := &user.DefaultInfo{ + Name: "cozy-admin", + Groups: []string{"cozystack-cluster-admin"}, + } + ctx := request.WithUser(context.Background(), u) + + hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !hasAccess { + t.Error("expected cozystack-cluster-admin to have access, but got false") + } +} + +func TestHasAccessToNamespace_ServiceAccount(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = rbacv1.AddToScheme(scheme) + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, + } + + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-binding", + Namespace: "tenant-test", + }, + Subjects: []rbacv1.Subject{ + {Kind: "ServiceAccount", Name: "test-sa", Namespace: "tenant-test"}, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: "test-role", + }, + } + + client := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(ns, rb). + Build() + + r := &REST{ + c: client, + gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, + } + + u := &user.DefaultInfo{ + Name: "system:serviceaccount:tenant-test:test-sa", + Groups: []string{"system:authenticated"}, + } + ctx := request.WithUser(context.Background(), u) + + hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !hasAccess { + t.Error("expected service account to have access, but got false") + } +} + +func TestHasAccessToNamespace_ServiceAccountEmptyNamespace(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = rbacv1.AddToScheme(scheme) + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, + } + + // ServiceAccount subject with empty namespace should default to RoleBinding namespace + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-binding", + Namespace: "tenant-test", + }, + Subjects: []rbacv1.Subject{ + {Kind: "ServiceAccount", Name: "test-sa", Namespace: ""}, // Empty namespace + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: "test-role", + }, + } + + client := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(ns, rb). + Build() + + r := &REST{ + c: client, + gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, + } + + u := &user.DefaultInfo{ + Name: "system:serviceaccount:tenant-test:test-sa", + Groups: []string{"system:authenticated"}, + } + ctx := request.WithUser(context.Background(), u) + + hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !hasAccess { + t.Error("expected service account with empty namespace to have access, but got false") + } +} + +func TestGet_WithAccess(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = rbacv1.AddToScheme(scheme) + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, + } + + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-binding", + Namespace: "tenant-test", + }, + Subjects: []rbacv1.Subject{ + {Kind: "User", Name: "test-user", APIGroup: "rbac.authorization.k8s.io"}, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: "test-role", + }, + } + + client := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(ns, rb). + Build() + + r := &REST{ + c: client, + gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, + } + + u := &user.DefaultInfo{ + Name: "test-user", + Groups: []string{"system:authenticated"}, + } + ctx := request.WithUser(context.Background(), u) + + obj, err := r.Get(ctx, "tenant-test", &metav1.GetOptions{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if obj == nil { + t.Fatal("expected object, got nil") + } +} + +func TestGet_WithoutAccess(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = rbacv1.AddToScheme(scheme) + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, + } + + // RoleBinding for different user + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-binding", + Namespace: "tenant-test", + }, + Subjects: []rbacv1.Subject{ + {Kind: "User", Name: "other-user", APIGroup: "rbac.authorization.k8s.io"}, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: "test-role", + }, + } + + client := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(ns, rb). + Build() + + r := &REST{ + c: client, + gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, + } + + u := &user.DefaultInfo{ + Name: "test-user", + Groups: []string{"system:authenticated"}, + } + ctx := request.WithUser(context.Background(), u) + + obj, err := r.Get(ctx, "tenant-test", &metav1.GetOptions{}) + if err == nil { + t.Fatal("expected error, got nil") + } + if obj != nil { + t.Errorf("expected nil object, got %v", obj) + } + + // Verify it returns NotFound (not Forbidden) to prevent enumeration + if !apierrors.IsNotFound(err) { + t.Errorf("expected NotFound error, got %v", err) + } +} + +func TestGet_NonTenantNamespace(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = rbacv1.AddToScheme(scheme) + + client := fake.NewClientBuilder(). + WithScheme(scheme). + Build() + + r := &REST{ + c: client, + gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, + } + + u := &user.DefaultInfo{ + Name: "test-user", + Groups: []string{"system:masters"}, + } + ctx := request.WithUser(context.Background(), u) + + obj, err := r.Get(ctx, "default", &metav1.GetOptions{}) + if err == nil { + t.Fatal("expected error for non-tenant namespace, got nil") + } + if obj != nil { + t.Errorf("expected nil object, got %v", obj) + } + if !apierrors.IsNotFound(err) { + t.Errorf("expected NotFound error, got %v", err) + } +} From 9d552d4086726d2f5c6de5adaa2783082893a8c5 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Thu, 23 Apr 2026 22:19:35 +0500 Subject: [PATCH 054/130] ci(api): broaden codegen drift trigger paths and detect untracked files - Add generated-output dirs (pkg/generated, internal/crdinstall/manifests, packages/system/*/definitions) and Makefile to the workflow paths: filter so PRs that modify only generated artifacts still trigger the drift check. - Mirror the same paths in the root pre-commit hook's files: regex so manual edits to generated files or changes to the root generate target re-run make generate through pre-commit. - Switch drift detection in the workflow from `git diff --exit-code` to `git status --porcelain` so new untracked files produced by make generate (e.g. generated YAML/Go for a new API type) also fail the job; dump `git diff --color=always` on failure for easier debugging. Signed-off-by: Myasnikov Daniil --- .github/workflows/codegen-drift.yml | 10 +++++++++- .pre-commit-config.yaml | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codegen-drift.yml b/.github/workflows/codegen-drift.yml index b08f5d7e..a26caf5e 100644 --- a/.github/workflows/codegen-drift.yml +++ b/.github/workflows/codegen-drift.yml @@ -6,8 +6,15 @@ on: paths: - 'api/**' - 'pkg/apis/**' + - 'pkg/generated/**' + - 'internal/crdinstall/manifests/**' + - 'packages/system/cozystack-controller/definitions/**' + - 'packages/system/application-definition-crd/definition/**' + - 'packages/system/backup-controller/definitions/**' + - 'packages/system/backupstrategy-controller/definitions/**' - 'hack/update-codegen.sh' - 'hack/boilerplate.go.txt' + - 'Makefile' - 'go.mod' - 'go.sum' - '.github/workflows/codegen-drift.yml' @@ -46,8 +53,9 @@ jobs: - name: Fail on drift run: | - if ! git diff --exit-code; then + if [ -n "$(git status --porcelain)" ]; then echo "::error::'make generate' produced changes. Run 'make generate' locally and commit the result." git status --short + git diff --color=always exit 1 fi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 836f79c5..6492b92d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ repos: git diff --color=always | cat ' language: system - files: ^(api/|pkg/apis/|hack/update-codegen\.sh$|hack/boilerplate\.go\.txt$) + files: ^(api/|pkg/apis/|pkg/generated/|internal/crdinstall/manifests/|packages/system/cozystack-controller/definitions/|packages/system/application-definition-crd/definition/|packages/system/backup-controller/definitions/|packages/system/backupstrategy-controller/definitions/|hack/update-codegen\.sh$|hack/boilerplate\.go\.txt$|Makefile$) pass_filenames: false - id: run-make-generate name: Run 'make generate' in all app directories From 3c5521ee992b2dce74ff8956a6d8c581763bf640 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Thu, 23 Apr 2026 22:16:30 +0300 Subject: [PATCH 055/130] fix(hami): remove broken hami-dra subchart The hami-dra subchart renders resources even when dra.enabled=false because Helm dependency conditions don't work for vendored subcharts. The subchart also contains duplicate YAML label keys (app.kubernetes.io/component, app.kubernetes.io/name) and references unpublished container images (v0.0.1-dev). DRA support can be re-added when the upstream project stabilizes it. Signed-off-by: Arsolitt --- .../charts/hami/charts/hami-dra/.helmignore | 24 -- .../charts/hami/charts/hami-dra/Chart.yaml | 18 - .../charts/hami-dra/templates/_commons.tpl | 84 ---- .../charts/hami-dra/templates/_helpers.tpl | 73 ---- .../hami-dra-driver/deviceclass.yaml | 11 - .../nvidia-dra-driver-daemonset.yaml | 145 ------- .../templates/hami-dra-driver/rbac.yaml | 110 ----- .../monitor/monitor-clusterrole.yaml | 11 - .../monitor/monitor-clusterrolebinding.yaml | 15 - .../templates/monitor/monitor-deployment.yaml | 62 --- .../templates/monitor/monitor-service.yaml | 26 -- .../monitor/monitor-serviceaccount.yaml | 8 - .../templates/webhook/cert-manager.yaml | 29 -- .../templates/webhook/cert-secret.yaml | 13 - .../templates/webhook/device-config.yaml | 394 ------------------ .../webhook/mutatingwebhookconfiguration.yaml | 28 -- .../validatingwebhookconfiguration.yaml | 29 -- .../webhook/webhook-clusterrole.yaml | 14 - .../webhook/webhook-clusterrolebinding.yaml | 12 - .../templates/webhook/webhook-deployment.yaml | 59 --- .../templates/webhook/webhook-service.yaml | 15 - .../webhook/webhook-serviceaccount.yaml | 5 - .../charts/hami/charts/hami-dra/values.yaml | 166 -------- 23 files changed, 1351 deletions(-) delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/.helmignore delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/_commons.tpl delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/_helpers.tpl delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/deviceclass.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/nvidia-dra-driver-daemonset.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/rbac.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrole.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrolebinding.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-deployment.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-service.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-serviceaccount.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-manager.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-secret.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/device-config.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/mutatingwebhookconfiguration.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/validatingwebhookconfiguration.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrole.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrolebinding.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-deployment.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-service.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-serviceaccount.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/values.yaml diff --git a/packages/system/hami/charts/hami/charts/hami-dra/.helmignore b/packages/system/hami/charts/hami/charts/hami-dra/.helmignore deleted file mode 100644 index 898df488..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/.helmignore +++ /dev/null @@ -1,24 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ - diff --git a/packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml b/packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml deleted file mode 100644 index d01f678f..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: v2 -appVersion: 0.1.0 -description: A Helm chart for HAMi DRA -home: https://github.com/Project-HAMi/HAMi-DRA -keywords: -- webhook -- kubernetes -- admission-controller -- hami -- dra -maintainers: -- name: hami-dra - url: https://github.com/Project-HAMi/HAMi-DRA -name: hami-dra -sources: -- https://github.com/Project-HAMi/HAMi-DRA -type: application -version: 0.1.0 diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/_commons.tpl b/packages/system/hami/charts/hami/charts/hami-dra/templates/_commons.tpl deleted file mode 100644 index 02fbed20..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/_commons.tpl +++ /dev/null @@ -1,84 +0,0 @@ -{{/* vim: set filetype=mustache: */}} - -{{/* -Return the proper image name -{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" $) }} -*/}} -{{- define "common.images.image" -}} -{{- $registryName := .imageRoot.registry -}} -{{- $repositoryName := .imageRoot.repository -}} -{{- $tag := .imageRoot.tag | toString -}} -{{- if .global }} - {{- if .global.imageRegistry }} - {{- $registryName = .global.imageRegistry -}} - {{- end -}} -{{- end -}} -{{- if .tag }} - {{- if .tag.imageTag }} - {{- $tag = .tag.imageTag -}} - {{- end -}} -{{- end -}} -{{- if $registryName }} -{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} -{{- else -}} -{{- printf "%s:%s" $repositoryName $tag -}} -{{- end -}} -{{- end -}} - -{{/* -Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead) -{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }} -*/}} -{{- define "common.images.pullSecrets" -}} - {{- $pullSecrets := list }} - - {{- if .global }} - {{- range .global.imagePullSecrets -}} - {{- $pullSecrets = append $pullSecrets . -}} - {{- end -}} - {{- end -}} - - {{- range .images -}} - {{- range .pullSecrets -}} - {{- $pullSecrets = append $pullSecrets . -}} - {{- end -}} - {{- end -}} - - {{- if (not (empty $pullSecrets)) }} -imagePullSecrets: - {{- range $pullSecrets }} - - name: {{ . }} - {{- end }} - {{- end }} -{{- end -}} - -{{/* -Renders a value that contains template. -Usage: -{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $) }} -*/}} -{{- define "common.tplvalues.render" -}} - {{- if typeIs "string" .value }} - {{- tpl .value .context }} - {{- else }} - {{- tpl (.value | toYaml) .context }} - {{- end }} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "common.names.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/_helpers.tpl b/packages/system/hami/charts/hami/charts/hami-dra/templates/_helpers.tpl deleted file mode 100644 index 99beec77..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/_helpers.tpl +++ /dev/null @@ -1,73 +0,0 @@ -{{- define "hami.dra.webhook.fullname" -}} -{{- printf "%s-%s" (include "common.names.fullname" .) "webhook" | trunc 63 | trimSuffix "-" -}} -{{- end }} - -{{- define "hami.dra.webhook.image" -}} -{{ include "common.images.image" (dict "imageRoot" .Values.webhook.image "global" .Values.global) }} -{{- end -}} - -{{- define "hami.dra.webhook.imagePullSecrets" -}} -{{ include "common.images.pullSecrets" (dict "images" (list .Values.webhook.image) "global" .Values.global) }} -{{- end -}} - -{{- define "hami.dra.driver.nvidia.image" -}} -{{ include "common.images.image" (dict "imageRoot" .Values.drivers.nvidia.image "global" .Values.global) }} -{{- end -}} - -{{- define "hami.dra.driver.nvidia.imagePullSecrets" -}} -{{ include "common.images.pullSecrets" (dict "images" (list .Values.drivers.nvidia.image) "global" .Values.global) }} -{{- end -}} - -{{/* -Common labels -*/}} -{{- define "hami-dra-webhook.labels" -}} -helm.sh/chart: {{ .Chart.Name }} -{{ include "hami-dra-webhook.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "hami-dra-webhook.selectorLabels" -}} -app.kubernetes.io/name: {{ .Release.Name }} -app.kubernetes.io/instance: {{ .Release.Name }} -app.kubernetes.io/component: webhook -{{- end }} - -{{- define "hami.dra.monitor.fullname" -}} -{{- printf "%s-%s" (include "common.names.fullname" .) "monitor" | trunc 63 | trimSuffix "-" -}} -{{- end }} - -{{- define "hami.dra.monitor.image" -}} -{{ include "common.images.image" (dict "imageRoot" .Values.monitor.image "global" .Values.global) }} -{{- end -}} - -{{- define "hami.dra.monitor.imagePullSecrets" -}} -{{ include "common.images.pullSecrets" (dict "images" (list .Values.monitor.image) "global" .Values.global) }} -{{- end -}} - -{{/* -Common labels for monitor -*/}} -{{- define "hami-dra-monitor.labels" -}} -helm.sh/chart: {{ .Chart.Name }} -{{ include "hami-dra-monitor.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels for monitor -*/}} -{{- define "hami-dra-monitor.selectorLabels" -}} -app.kubernetes.io/name: {{ .Release.Name }} -app.kubernetes.io/instance: {{ .Release.Name }} -app.kubernetes.io/component: monitor -{{- end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/deviceclass.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/deviceclass.yaml deleted file mode 100644 index bcb6d2c5..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/deviceclass.yaml +++ /dev/null @@ -1,11 +0,0 @@ -{{- if .Values.drivers.nvidia.enabled }} -apiVersion: resource.k8s.io/v1 -kind: DeviceClass -metadata: - name: hami-core-gpu.project-hami.io -spec: - selectors: - - cel: - expression: |- - device.driver == "hami-core-gpu.project-hami.io" && device.attributes["hami-core-gpu.project-hami.io"].type == "hami-gpu" -{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/nvidia-dra-driver-daemonset.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/nvidia-dra-driver-daemonset.yaml deleted file mode 100644 index a3fb0d01..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/nvidia-dra-driver-daemonset.yaml +++ /dev/null @@ -1,145 +0,0 @@ -{{- if .Values.drivers.nvidia.enabled }} -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: hami-dra-driver-kubelet-plugin - namespace: {{ .Release.Namespace }} -spec: - revisionHistoryLimit: 10 - selector: - matchLabels: - hami-dra-driver-component: kubelet-plugin - template: - metadata: - labels: - app.kubernetes.io/instance: hami-dra-driver - app.kubernetes.io/name: hami-dra-driver - hami-dra-driver-component: kubelet-plugin - spec: - {{- include "hami.dra.driver.nvidia.imagePullSecrets" . | nindent 6 }} - initContainers: - - name: init-container - image: {{ include "hami.dra.driver.nvidia.image" . }} - securityContext: - privileged: true - command: [bash, /usr/bin/kubelet-plugin-prestart.sh] - env: - - name: NVIDIA_DRIVER_ROOT - value: {{ if .Values.drivers.nvidia.containerDriver }}/run/nvidia/driver{{ else }}/{{ end }} - - name: NVIDIA_VISIBLE_DEVICES - value: void - - name: KUBELET_REGISTRAR_DIRECTORY_PATH - value: /var/lib/kubelet/plugins_registry - - name: KUBELET_PLUGINS_DIRECTORY_PATH - value: /var/lib/kubelet/plugins - volumeMounts: - - name: driver-root-parent - mountPath: /driver-root-parent - readOnly: true - containers: - - args: - - |- - # Conditionally mask the params file to prevent this container from - # recreating any missing GPU device nodes. This is necessary, for - # example, when running under nvkind to limit the set GPUs governed - # by the plugin even though it has cgroup access to all of them. - if [ "${MASK_NVIDIA_DRIVER_PARAMS}" = "true" ]; then - cp /proc/driver/nvidia/params root/gpu-params - sed -i 's/^ModifyDeviceFiles: 1$/ModifyDeviceFiles: 0/' root/gpu-params - mount --bind root/gpu-params /proc/driver/nvidia/params - fi - hami-kubelet-plugin -v 6 - command: - - bash - - -c - env: - - name: MASK_NVIDIA_DRIVER_PARAMS - value: "" - - name: NVIDIA_DRIVER_ROOT - value: {{ if .Values.drivers.nvidia.containerDriver }}/run/nvidia/driver{{ else }}/{{ end }} - - name: NVIDIA_VISIBLE_DEVICES - value: void - - name: CDI_ROOT - value: /var/run/cdi - - name: NVIDIA_MIG_CONFIG_DEVICES - value: all - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: KUBELET_REGISTRAR_DIRECTORY_PATH - value: /var/lib/kubelet/plugins_registry - - name: KUBELET_PLUGINS_DIRECTORY_PATH - value: /var/lib/kubelet/plugins - - name: IMAGE_NAME - value: {{ include "hami.dra.driver.nvidia.image" . }} - lifecycle: - postStart: - exec: - command: ["/bin/bash", "-c", "/usr/bin/vgpu-init.sh /usr/local/vgpu/"] - image: {{ include "hami.dra.driver.nvidia.image" . }} - imagePullPolicy: {{ .Values.drivers.nvidia.image.pullPolicy | default "IfNotPresent" }} - name: gpus - securityContext: - privileged: true - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - volumeMounts: - - mountPath: /var/lib/kubelet/plugins_registry - name: plugins-registry - - mountPath: /var/lib/kubelet/plugins - mountPropagation: Bidirectional - name: plugins - - mountPath: /var/run/cdi - name: cdi - - mountPath: /driver-root - mountPropagation: HostToContainer - name: driver-root - readOnly: true - - mountPath: /usr/local/vgpu - name: host-vgpu - - mountPath: /tmp - name: host-tmp - dnsPolicy: ClusterFirst - serviceAccount: hami-dra-driver-service-account - serviceAccountName: hami-dra-driver-service-account - volumes: - - hostPath: - path: /var/lib/kubelet/plugins_registry - type: "" - name: plugins-registry - - hostPath: - path: /var/lib/kubelet/plugins - type: "" - name: plugins - - hostPath: - path: /var/run/cdi - type: "" - name: cdi - - hostPath: - path: {{if .Values.drivers.nvidia.containerDriver }}/run/nvidia{{ else }}/{{ end }} - type: DirectoryOrCreate - name: driver-root-parent - - hostPath: - path: {{if .Values.drivers.nvidia.containerDriver }}/run/nvidia/driver{{ else }}/{{ end }} - type: DirectoryOrCreate - name: driver-root - - hostPath: - path: /dev - type: "" - name: host-dev - - hostPath: - path: /usr/local/vgpu - type: DirectoryOrCreate - name: host-vgpu - - hostPath: - path: /tmp - type: "" - name: host-tmp -{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/rbac.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/rbac.yaml deleted file mode 100644 index 435c37cc..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/rbac.yaml +++ /dev/null @@ -1,110 +0,0 @@ -{{- if .Values.drivers.nvidia.enabled }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: hami-dra-driver-service-account - namespace: {{ .Release.Namespace }} ---- -apiVersion: v1 -items: -- apiVersion: rbac.authorization.k8s.io/v1 - kind: Role - metadata: - name: hami-dra-driver-role - namespace: {{ .Release.Namespace }} - rules: - - apiGroups: - - apps - resources: - - daemonsets - - deployments - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -kind: List -metadata: - resourceVersion: "" ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: hami-dra-driver-role-binding - namespace: {{ .Release.Namespace }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: hami-dra-driver-role -subjects: -- kind: ServiceAccount - name: hami-dra-driver-service-account - namespace: {{ .Release.Namespace }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: hami-dra-driver-clusterrole -rules: -- apiGroups: - - resource.k8s.io - resources: - - resourceclaims - verbs: - - get - - list - - watch -- apiGroups: - - resource.k8s.io - resources: - - resourceclaimtemplates - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - resource.k8s.io - resources: - - resourceslices - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - resource.k8s.io - resources: - - resourceclaims/status - verbs: - - update -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - watch - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: hami-dra-driver-clusterrole-binding-hami-dra-driver -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: hami-dra-driver-clusterrole -subjects: -- kind: ServiceAccount - name: hami-dra-driver-service-account - namespace: {{ .Release.Namespace }} -{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrole.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrole.yaml deleted file mode 100644 index 45c24863..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrole.yaml +++ /dev/null @@ -1,11 +0,0 @@ -{{- if .Values.monitor.enabled }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "hami.dra.monitor.fullname" . }} -rules: -- apiGroups: ["resource.k8s.io"] - resources: ["resourceslices", "resourceclaims"] - verbs: ["get", "list", "watch"] -{{- end }} - diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrolebinding.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrolebinding.yaml deleted file mode 100644 index 766d3098..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrolebinding.yaml +++ /dev/null @@ -1,15 +0,0 @@ -{{- if .Values.monitor.enabled }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "hami.dra.monitor.fullname" . }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "hami.dra.monitor.fullname" . }} -subjects: - - kind: ServiceAccount - name: {{ include "hami.dra.monitor.fullname" . }} - namespace: {{ .Release.Namespace }} -{{- end }} - diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-deployment.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-deployment.yaml deleted file mode 100644 index b544b376..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-deployment.yaml +++ /dev/null @@ -1,62 +0,0 @@ -{{- if .Values.monitor.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "hami.dra.monitor.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: - app.kubernetes.io/name: {{ include "hami.dra.monitor.fullname" . }} - {{- include "hami-dra-monitor.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.monitor.replicas | default 1 }} - selector: - matchLabels: - {{- include "hami-dra-monitor.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - {{- include "hami-dra-monitor.selectorLabels" . | nindent 8 }} - spec: - {{- include "hami.dra.monitor.imagePullSecrets" . | nindent 6 }} - serviceAccountName: {{ include "hami.dra.monitor.fullname" . }} - containers: - - name: monitor - image: {{ include "hami.dra.monitor.image" . }} - imagePullPolicy: {{ .Values.monitor.image.pullPolicy | default "IfNotPresent" }} - command: - - /bin/monitor - - --v={{ .Values.monitor.logLevel | default 2 }} - - --metrics-bind-address={{ .Values.monitor.metricsBindAddress | default ":8080" }} - - --health-probe-bind-address={{ .Values.monitor.healthProbeBindAddress | default ":8000" }} - - --kube-api-qps={{ .Values.monitor.kubeAPIQPS | default 40.0 }} - - --kube-api-burst={{ .Values.monitor.kubeAPIBurst | default 60 }} - - --collect-interval={{ .Values.monitor.collectInterval | default "30s" }} - ports: - - containerPort: 8080 - name: metrics - protocol: TCP - - containerPort: 8000 - name: health - protocol: TCP - {{- if .Values.monitor.resources }} - resources: - {{- toYaml .Values.monitor.resources | nindent 12 }} - {{- end }} - livenessProbe: - httpGet: - path: /healthz - port: 8000 - initialDelaySeconds: 10 - periodSeconds: 10 - timeoutSeconds: 5 - failureThreshold: 3 - readinessProbe: - httpGet: - path: /readyz - port: 8000 - initialDelaySeconds: 5 - periodSeconds: 5 - timeoutSeconds: 3 - failureThreshold: 3 -{{- end }} - diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-service.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-service.yaml deleted file mode 100644 index d740cc38..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-service.yaml +++ /dev/null @@ -1,26 +0,0 @@ -{{- if .Values.monitor.enabled }} -apiVersion: v1 -kind: Service -metadata: - name: {{ include "hami.dra.monitor.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: - {{- include "hami-dra-monitor.selectorLabels" . | nindent 4 }} -spec: - type: {{ .Values.monitor.service.type | default "ClusterIP" }} - selector: - {{- include "hami-dra-monitor.selectorLabels" . | nindent 4 }} - ports: - - port: 8080 - targetPort: 8080 - name: metrics - protocol: TCP - {{- if and (eq (.Values.monitor.service.type | default "ClusterIP") "NodePort") .Values.monitor.service.nodePort.metrics }} - nodePort: {{ .Values.monitor.service.nodePort.metrics }} - {{- end }} - - port: 8000 - targetPort: 8000 - name: health - protocol: TCP -{{- end }} - diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-serviceaccount.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-serviceaccount.yaml deleted file mode 100644 index c61ad454..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-serviceaccount.yaml +++ /dev/null @@ -1,8 +0,0 @@ -{{- if .Values.monitor.enabled }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "hami.dra.monitor.fullname" . }} - namespace: {{ .Release.Namespace }} -{{- end }} - diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-manager.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-manager.yaml deleted file mode 100644 index dd31c71d..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-manager.yaml +++ /dev/null @@ -1,29 +0,0 @@ -{{- if .Values.certs.certManager.enabled }} -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: {{ .Release.Name }}-dra-webhook-serving-cert - namespace: {{ .Release.Namespace }} - labels: - {{- include "hami-dra-webhook.labels" . | nindent 4 }} -spec: - dnsNames: - - {{ .Release.Name }}-dra-webhook.{{ .Release.Namespace }}.svc - - {{ .Release.Name }}-dra-webhook.{{ .Release.Namespace }}.svc.cluster.local - issuerRef: - kind: Issuer - name: {{ .Release.Name }}-dra-webhook-selfsigned-issuer - secretName: {{ .Release.Name }}-dra-webhook-tls - privateKey: - rotationPolicy: {{ .Values.certs.certManager.privateKeyRotationPolicy | default "Never" }} ---- -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - name: {{ .Release.Name }}-dra-webhook-selfsigned-issuer - namespace: {{ .Release.Namespace }} - labels: - app.kubernetes.io/component: hami-dra-webhook -spec: - selfSigned: {} -{{- end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-secret.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-secret.yaml deleted file mode 100644 index eadbb517..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-secret.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- if eq .Values.certs.mode "custom" }} -apiVersion: v1 -kind: Secret -metadata: - name: {{ include "hami-dra-webhook.tlsSecretName" . }} - namespace: {{ .Release.Namespace }} -type: kubernetes.io/tls -data: - tls.crt: | - {{- .Values.certs.custom.crt | b64enc | indent 4 }} - tls.key: | - {{- .Values.certs.custom.key | b64enc | indent 4 }} -{{- end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/device-config.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/device-config.yaml deleted file mode 100644 index 97283f3e..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/device-config.yaml +++ /dev/null @@ -1,394 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "hami.dra.webhook.fullname" . }}-device-config - namespace: {{ .Release.Namespace }} - labels: - app.kubernetes.io/component: hami-scheduler - {{- include "hami-dra-webhook.selectorLabels" . | nindent 4 }} -data: - device-config.yaml: |- - {{- if .Files.Glob "files/device-config.yaml" }} - {{- .Files.Get "files/device-config.yaml" | nindent 4}} - {{- else }} - nvidia: - resourceCountName: {{ .Values.resourceName }} - resourceMemoryName: {{ .Values.resourceMem }} - resourceMemoryPercentageName: {{ .Values.resourceMemPercentage }} - resourceCoreName: {{ .Values.resourceCores }} - resourcePriorityName: {{ .Values.resourcePriority }} - overwriteEnv: false - defaultMemory: 0 - defaultCores: 0 - defaultGPUNum: 1 - knownMigGeometries: - - models: [ "A30" ] - allowedGeometries: - - - - name: 1g.6gb - core: 25 - memory: 6144 - count: 4 - - - - name: 2g.12gb - core: 50 - memory: 12288 - count: 2 - - - - name: 4g.24gb - core: 100 - memory: 24576 - count: 1 - - models: [ "A100-SXM4-40GB", "A100-40GB-PCIe", "A100-PCIE-40GB"] - allowedGeometries: - - - - name: 1g.5gb - core: 14 - memory: 5120 - count: 7 - - - - name: 1g.5gb - core: 14 - memory: 5120 - count: 1 - - name: 2g.10gb - core: 28 - memory: 10240 - count: 3 - - - - name: 3g.20gb - core: 42 - memory: 20480 - count: 2 - - - - name: 7g.40gb - core: 100 - memory: 40960 - count: 1 - - models: [ "A100-SXM4-80GB", "A100-80GB-PCIe", "A100-PCIE-80GB"] - allowedGeometries: - - - - name: 1g.10gb - core: 14 - memory: 10240 - count: 7 - - - - name: 1g.10gb - core: 14 - memory: 10240 - count: 1 - - name: 2g.20gb - core: 28 - memory: 20480 - count: 3 - - - - name: 3g.40gb - core: 42 - memory: 40960 - count: 2 - - - - name: 7g.79gb - core: 100 - memory: 80896 - count: 1 - - models: [ "H100-PCIE-80GB", "H100-SXM5-80GB"] - allowedGeometries: - - - - name: 1g.10gb - core: 14 - memory: 10240 - count: 7 - - - - name: 1g.10gb - core: 14 - memory: 10240 - count: 1 - - name: 2g.20gb - core: 28 - memory: 20480 - count: 3 - - - - name: 3g.40gb - core: 42 - memory: 40960 - count: 2 - - - - name: 7g.80gb - core: 100 - memory: 81920 - count: 1 - - models: [ "H100-PCIE-94GB", "H100-SXM5-94GB"] - allowedGeometries: - - - - name: 1g.12gb - core: 14 - memory: 12288 - count: 7 - - - - name: 1g.12gb - core: 14 - memory: 12288 - count: 1 - - name: 2g.24gb - core: 28 - memory: 24576 - count: 3 - - - - name: 3g.47gb - core: 42 - memory: 48128 - count: 2 - - - - name: 7g.94gb - core: 100 - memory: 96256 - count: 1 - - models: [ "H20", "H100 on GH200"] - allowedGeometries: - - - - name: 1g.12gb - core: 14 - memory: 12288 - count: 7 - - - - name: 1g.12gb - core: 14 - memory: 12288 - count: 1 - - name: 2g.24gb - core: 28 - memory: 24576 - count: 3 - - - - name: 3g.48gb - core: 42 - memory: 49152 - count: 2 - - - - name: 7g.96gb - core: 100 - memory: 98304 - count: 1 - - models: [ "H200 NVL", "H200-SXM5"] - allowedGeometries: - - - - name: 1g.18gb - core: 14 - memory: 18432 - count: 7 - - - - name: 1g.18gb - core: 14 - memory: 18432 - count: 1 - - name: 2g.35gb - core: 28 - memory: 35840 - count: 3 - - - - name: 3g.71gb - core: 42 - memory: 72704 - count: 2 - - - - name: 7g.141gb - core: 100 - memory: 144384 - count: 1 - - models: [ "B200" ] - allowedGeometries: - - - - name: 1g.23gb - core: 14 - memory: 23552 - count: 7 - - - - name: 1g.23gb - core: 14 - memory: 23552 - count: 1 - - name: 2g.45gb - core: 28 - memory: 46080 - count: 3 - - - - name: 3g.90gb - core: 42 - memory: 92160 - count: 2 - - - - name: 7g.180gb - core: 100 - memory: 184320 - count: 1 - cambricon: - resourceCountName: {{ .Values.mluResourceName }} - resourceMemoryName: {{ .Values.mluResourceMem }} - resourceCoreName: {{ .Values.mluResourceCores }} - hygon: - resourceCountName: {{ .Values.dcuResourceName }} - resourceMemoryName: {{ .Values.dcuResourceMem }} - resourceCoreName: {{ .Values.dcuResourceCores }} - metax: - resourceCountName: "metax-tech.com/gpu" - resourceVCountName: {{ .Values.metaxResourceName }} - resourceVMemoryName: {{ .Values.metaxResourceMem }} - resourceVCoreName: {{ .Values.metaxResourceCore }} - sgpuTopologyAware: {{ .Values.metaxsGPUTopologyAware }} - enflame: - resourceNameGCU: "enflame.com/gcu" - resourceNameVGCU: {{ .Values.enflameResourceNameVGCU }} - resourceNameVGCUPercentage: {{ .Values.enflameResourceNameVGCUPercentage }} - mthreads: - resourceCountName: "mthreads.com/vgpu" - resourceMemoryName: "mthreads.com/sgpu-memory" - resourceCoreName: "mthreads.com/sgpu-core" - iluvatars: - - chipName: MR-V100 - commonWord: MR-V100 - resourceCountName: iluvatar.ai/MR-V100-vgpu - resourceMemoryName: iluvatar.ai/MR-V100.vMem - resourceCoreName: iluvatar.ai/MR-V100.vCore - - chipName: MR-V50 - commonWord: MR-V50 - resourceCountName: iluvatar.ai/MR-V50-vgpu - resourceMemoryName: iluvatar.ai/MR-V50.vMem - resourceCoreName: iluvatar.ai/MR-V50.vCore - - chipName: BI-V150 - commonWord: BI-V150 - resourceCountName: iluvatar.ai/BI-V150-vgpu - resourceMemoryName: iluvatar.ai/BI-V150.vMem - resourceCoreName: iluvatar.ai/BI-V150.vCore - - chipName: BI-V100 - commonWord: BI-V100 - resourceCountName: iluvatar.ai/BI-V100-vgpu - resourceMemoryName: iluvatar.ai/BI-V100.vMem - resourceCoreName: iluvatar.ai/BI-V100.vCore - kunlun: - resourceCountName: {{ .Values.kunlunResourceName }} - resourceVCountName: {{ .Values.kunlunResourceVCountName }} - resourceVMemoryName: {{ .Values.kunlunResourceVMemoryName }} - awsneuron: - resourceCountName: "aws.amazon.com/neuron" - resourceCoreName: "aws.amazon.com/neuroncore" - amd: - resourceCountName: "amd.com/gpu" - vnpus: - - chipName: 910A - commonWord: Ascend910A - resourceName: huawei.com/Ascend910A - resourceMemoryName: huawei.com/Ascend910A-memory - memoryAllocatable: 32768 - memoryCapacity: 32768 - aiCore: 30 - templates: - - name: vir02 - memory: 2184 - aiCore: 2 - - name: vir04 - memory: 4369 - aiCore: 4 - - name: vir08 - memory: 8738 - aiCore: 8 - - name: vir16 - memory: 17476 - aiCore: 16 - - chipName: 910B2 - commonWord: Ascend910B2 - resourceName: huawei.com/Ascend910B2 - resourceMemoryName: huawei.com/Ascend910B2-memory - memoryAllocatable: 65536 - memoryCapacity: 65536 - aiCore: 24 - aiCPU: 6 - templates: - - name: vir03_1c_8g - memory: 8192 - aiCore: 3 - aiCPU: 1 - - name: vir06_1c_16g - memory: 16384 - aiCore: 6 - aiCPU: 1 - - name: vir12_3c_32g - memory: 32768 - aiCore: 12 - aiCPU: 3 - - chipName: 910B3 - commonWord: Ascend910B3 - resourceName: huawei.com/Ascend910B3 - resourceMemoryName: huawei.com/Ascend910B3-memory - memoryAllocatable: 65536 - memoryCapacity: 65536 - aiCore: 20 - aiCPU: 7 - templates: - - name: vir05_1c_16g - memory: 16384 - aiCore: 5 - aiCPU: 1 - - name: vir10_3c_32g - memory: 32768 - aiCore: 10 - aiCPU: 3 - - chipName: 910B4-1 - commonWord: Ascend910B4-1 - resourceName: huawei.com/Ascend910B4-1 - resourceMemoryName: huawei.com/Ascend910B4-1-memory - memoryAllocatable: 65536 - memoryCapacity: 65536 - aiCore: 20 - aiCPU: 7 - templates: - # NOTE: Names of vnpu template for 910B4-1 are fixed by Ascend runtime and must not be changed. - # The memory is used for scheduling so the correct values must be set. - # Template vir05_1c_8g actually provides 16GB memory, - - name: vir05_1c_8g - memory: 16384 - aiCore: 5 - aiCPU: 1 - # Template vir10_3c_16g actually provides 32GB memory - - name: vir10_3c_16g - memory: 32768 - aiCore: 10 - aiCPU: 3 - - chipName: 910B4 - commonWord: Ascend910B4 - resourceName: huawei.com/Ascend910B4 - resourceMemoryName: huawei.com/Ascend910B4-memory - memoryAllocatable: 32768 - memoryCapacity: 32768 - aiCore: 20 - aiCPU: 7 - templates: - - name: vir05_1c_8g - memory: 8192 - aiCore: 5 - aiCPU: 1 - - name: vir10_3c_16g - memory: 16384 - aiCore: 10 - aiCPU: 3 - - chipName: 310P3 - commonWord: Ascend310P - resourceName: huawei.com/Ascend310P - resourceMemoryName: huawei.com/Ascend310P-memory - memoryAllocatable: 21527 - memoryCapacity: 24576 - aiCore: 8 - aiCPU: 7 - templates: - - name: vir01 - memory: 3072 - aiCore: 1 - aiCPU: 1 - - name: vir02 - memory: 6144 - aiCore: 2 - aiCPU: 2 - - name: vir04 - memory: 12288 - aiCore: 4 - aiCPU: 4 - {{ end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/mutatingwebhookconfiguration.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/mutatingwebhookconfiguration.yaml deleted file mode 100644 index c763b6aa..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/mutatingwebhookconfiguration.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{- if .Values.webhook.config.mutating.enabled }} -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - name: {{ .Release.Name }}-mutatingwebhookconfiguration - annotations: - cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ .Release.Name }}-dra-webhook-serving-cert -webhooks: - - name: mutate.hami.io - admissionReviewVersions: ["v1"] - clientConfig: - service: - name: {{ .Release.Name }}-dra-webhook - namespace: {{ .Release.Namespace }} - path: /mutate - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - CREATE - resources: - - pods - scope: '*' - sideEffects: None - timeoutSeconds: 10 -{{- end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/validatingwebhookconfiguration.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/validatingwebhookconfiguration.yaml deleted file mode 100644 index acdcbb96..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/validatingwebhookconfiguration.yaml +++ /dev/null @@ -1,29 +0,0 @@ -{{- if .Values.webhook.config.validating.enabled }} -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: {{ .Release.Name }}-validatingwebhookconfiguration - annotations: - cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ .Release.Name }}-dra-webhook-serving-cert -webhooks: - - name: validate.hami.io - admissionReviewVersions: ["v1"] - clientConfig: - service: - name: {{ .Release.Name }}-dra-webhook - namespace: {{ .Release.Namespace }} - path: /validate - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - DELETE - resources: - - pods - scope: '*' - sideEffects: None - timeoutSeconds: 10 - failurePolicy: Ignore -{{- end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrole.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrole.yaml deleted file mode 100644 index 28916a36..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrole.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "hami.dra.webhook.fullname" . }} -rules: -- apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list", "watch"] -- apiGroups: ["admissionregistration.k8s.io"] - resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"] - verbs: ["get", "list", "watch"] -- apiGroups: ["resource.k8s.io"] - resources: ["resourceclaims"] - verbs: ["*"] diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrolebinding.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrolebinding.yaml deleted file mode 100644 index daad52f2..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrolebinding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "hami.dra.webhook.fullname" . }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "hami.dra.webhook.fullname" . }} -subjects: - - kind: ServiceAccount - name: {{ include "hami.dra.webhook.fullname" . }} - namespace: {{ .Release.Namespace }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-deployment.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-deployment.yaml deleted file mode 100644 index 8e3cf90c..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-deployment.yaml +++ /dev/null @@ -1,59 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "hami.dra.webhook.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: - app.kubernetes.io/name: {{ include "hami.dra.webhook.fullname" . }} - {{- include "hami-dra-webhook.labels" . | nindent 4 }} -spec: - replicas: 1 - selector: - matchLabels: - {{- include "hami-dra-webhook.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - {{- include "hami-dra-webhook.selectorLabels" . | nindent 8 }} - spec: - {{- include "hami.dra.webhook.imagePullSecrets" . | nindent 6 }} - serviceAccountName: {{ include "hami.dra.webhook.fullname" . }} - containers: - - name: webhook - image: {{ include "hami.dra.webhook.image" . }} - imagePullPolicy: {{ .Values.webhook.image.pullPolicy | default "IfNotPresent" }} - command: - - /bin/webhook - - --v=5 - - --bind-address=0.0.0.0 - - --secure-port=8443 - - --cert-dir=/tls - - --tls-cert-file-name=tls.crt - - --tls-private-key-file-name=tls.key - - --metrics-bind-address=:8080 - - --health-probe-bind-address=:8000 - - --device-config-file=/device-config.yaml - ports: - - containerPort: 8443 - name: webhook - protocol: TCP - - containerPort: 8080 - name: metrics - protocol: TCP - - containerPort: 8000 - name: health - protocol: TCP - volumeMounts: - - name: device-config - mountPath: /device-config.yaml - subPath: device-config.yaml - - name: tls-config - mountPath: /tls - readOnly: true - volumes: - - name: device-config - configMap: - name: {{ include "hami.dra.webhook.fullname" . }}-device-config - - name: tls-config - secret: - secretName: {{ if .Values.certs.certManager.enabled }}{{ .Release.Name }}-dra-webhook-tls{{ else }}{{ .Release.Name }}-tls{{ end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-service.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-service.yaml deleted file mode 100644 index 290174ec..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ .Release.Name }}-dra-webhook - namespace: {{ .Release.Namespace }} - labels: - {{- include "hami-dra-webhook.selectorLabels" . | nindent 4 }} -spec: - type: "ClusterIP" - selector: - {{- include "hami-dra-webhook.selectorLabels" . | nindent 4 }} - ports: - - port: 443 - targetPort: 8443 - name: webhook diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-serviceaccount.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-serviceaccount.yaml deleted file mode 100644 index b0dc2fb2..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-serviceaccount.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "hami.dra.webhook.fullname" . }} - namespace: {{ .Release.Namespace }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/values.yaml b/packages/system/hami/charts/hami/charts/hami-dra/values.yaml deleted file mode 100644 index ebbbfed7..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/values.yaml +++ /dev/null @@ -1,166 +0,0 @@ -## @section Global configuration -## @param global.imageRegistry Global Docker image registry -## @param global.imagePullSecrets Global Docker image pull secrets -global: - ## @param global.imageRegistry Global Docker image registry - imageRegistry: "" - ## E.g. - ## imagePullSecrets: - ## - myRegistryKeySecretName - ## @param global.imagePullSecrets Global Docker image pull secrets - imagePullSecrets: [] - -#Nvidia GPU Parameters -resourceName: "nvidia.com/gpu" -resourceMem: "nvidia.com/gpumem" -resourceMemPercentage: "nvidia.com/gpumem-percentage" -resourceCores: "nvidia.com/gpucores" -resourcePriority: "nvidia.com/priority" - -#MLU Parameters -mluResourceName: "cambricon.com/vmlu" -mluResourceMem: "cambricon.com/mlu.smlu.vmemory" -mluResourceCores: "cambricon.com/mlu.smlu.vcore" - -#Hygon DCU Parameters -dcuResourceName: "hygon.com/dcunum" -dcuResourceMem: "hygon.com/dcumem" -dcuResourceCores: "hygon.com/dcucores" - -#Metax sGPU Parameters -metaxResourceName: "metax-tech.com/sgpu" -metaxResourceCore: "metax-tech.com/vcore" -metaxResourceMem: "metax-tech.com/vmemory" -metaxsGPUTopologyAware: "false" - -#Enflame VGCU Parameters -enflameResourceNameVGCU: "enflame.com/vgcu" -enflameResourceNameVGCUPercentage: "enflame.com/vgcu-percentage" - -#Kunlun XPU Parameters -kunlunResourceName: "kunlunxin.com/xpu" -kunlunResourceVCountName: "kunlunxin.com/vxpu" -kunlunResourceVMemoryName: "kunlunxin.com/vxpu-memory" - -# Webhook deployment configuration -webhook: - image: - registry: ghcr.io - repository: project-hami/hami-dra-webhook - tag: "v0.1.0" - ## Specify a imagePullPolicy - ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' - ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images - ## - pullPolicy: Always - ## Optionally specify an array of imagePullSecrets. - ## Secrets must be manually created in the namespace. - ## Example: - ## pullSecrets: - ## - myRegistryKeySecretName - ## - pullSecrets: [] - ## @param resources controllerManager resource requests and limits - resources: - # If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 128Mi - ## @param nodeSelector controllerManager node labels for pod assignment - args: - webhookName: "" # Default: {{ .Release.Name }}-hami-dra-webhook - secretName: "" # Default: {{ .Release.Name }}-hami-dra-webhook-tls - # Webhook configuration - config: - mutating: - enabled: true - validating: - enabled: true - -# Monitor deployment configuration -monitor: - enabled: true - replicas: 1 - image: - registry: ghcr.io - repository: project-hami/hami-dra-monitor - tag: "v0.1.0" - ## Specify a imagePullPolicy - ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' - ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images - ## - pullPolicy: Always - ## Optionally specify an array of imagePullSecrets. - ## Secrets must be manually created in the namespace. - ## Example: - ## pullSecrets: - ## - myRegistryKeySecretName - ## - pullSecrets: [] - ## @param resources monitor resource requests and limits - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 128Mi - ## Monitor configuration - logLevel: 2 - metricsBindAddress: ":8080" - healthProbeBindAddress: ":8000" - kubeAPIQPS: 40.0 - kubeAPIBurst: 60 - collectInterval: "30s" - ## Monitor service configuration - service: - ## Service type: ClusterIP, NodePort, or LoadBalancer - ## Defaults to ClusterIP - type: ClusterIP - ## NodePort configuration (only used when type is NodePort) - nodePort: - ## Metrics port (NodePort). If not specified, Kubernetes will assign a random port - metrics: "" - -# Certificate configuration -certs: - # Custom certificate (used when mode is "custom") - custom: - crt: "" - key: "" - # Cert-manager configuration (used when mode is "cert-manager") - certManager: - enabled: true - # Private key rotation policy: "Never" or "Always" - # In cert-manager >= v1.18.0, the default changed from "Never" to "Always" - # Setting to "Never" avoids frequent certificate rotation for webhooks - privateKeyRotationPolicy: "Never" - issuer: - clusterIssuerName: "" # Required when type is "clusterIssuer" - -# DRA Drivers configuration -drivers: - nvidia: - enabled: true - # If you are using gpu driver on host, you need to set this to false - containerDriver: true - image: - registry: ghcr.io - repository: projecthami/k8s-dra-driver - tag: "v0.0.1-dev" - ## Specify a imagePullPolicy - ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' - ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images - ## - pullPolicy: IfNotPresent - ## Optionally specify an array of imagePullSecrets. - ## Secrets must be manually created in the namespace. - ## Example: - ## pullSecrets: - ## - myRegistryKeySecretName - ## - pullSecrets: [] From 385ea7a17f8cfc4526c50db036c105c63a5817fc Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Thu, 23 Apr 2026 22:16:34 +0300 Subject: [PATCH 056/130] feat(platform): register HAMi as optional system package Add PackageSource for HAMi with dependency on gpu-operator. Include HAMi in the iaas bundle as an optional package, allowing host-level deployment alongside the existing kubernetes addon path. Signed-off-by: Arsolitt --- packages/core/platform/sources/hami.yaml | 24 +++++++++++++++++++ .../core/platform/templates/bundles/iaas.yaml | 1 + 2 files changed, 25 insertions(+) create mode 100644 packages/core/platform/sources/hami.yaml diff --git a/packages/core/platform/sources/hami.yaml b/packages/core/platform/sources/hami.yaml new file mode 100644 index 00000000..0184e405 --- /dev/null +++ b/packages/core/platform/sources/hami.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: cozystack.io/v1alpha1 +kind: PackageSource +metadata: + name: cozystack.hami +spec: + sourceRef: + kind: OCIRepository + name: cozystack-packages + namespace: cozy-system + path: / + variants: + - name: default + dependsOn: + - cozystack.gpu-operator + components: + - name: hami + path: system/hami + valuesFiles: + - values.yaml + install: + privileged: true + namespace: cozy-hami + releaseName: hami diff --git a/packages/core/platform/templates/bundles/iaas.yaml b/packages/core/platform/templates/bundles/iaas.yaml index ced0a322..66ec98b2 100644 --- a/packages/core/platform/templates/bundles/iaas.yaml +++ b/packages/core/platform/templates/bundles/iaas.yaml @@ -10,6 +10,7 @@ {{include "cozystack.platform.package.default" (list "cozystack.kubevirt-cdi" $) }} {{include "cozystack.platform.package.optional.default" (list "cozystack.vm-default-images" $) }} {{include "cozystack.platform.package.optional.default" (list "cozystack.gpu-operator" $) }} +{{include "cozystack.platform.package.optional.default" (list "cozystack.hami" $) }} {{include "cozystack.platform.package.default" (list "cozystack.kamaji" $) }} {{include "cozystack.platform.package.default" (list "cozystack.capi-operator" $) }} {{include "cozystack.platform.package.default" (list "cozystack.capi-provider-bootstrap-kubeadm" $) }} From 6af74cec0ea436cbd5aefce5e5c8bedf86e5b81a Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 23 Apr 2026 23:24:38 +0300 Subject: [PATCH 057/130] fix(ingress): accept pre-CIDR externalIPs in CiliumLoadBalancerIPPool Address review feedback from gemini-code-assist on packages/extra/ingress/templates/cilium-lb-pool.yaml:19: if the operator passes an externalIP already in CIDR form (192.0.2.10/32 or 2001:db8::1/128), the template appended a second /32 or /128 suffix producing an invalid CiliumLoadBalancerIPPool block. Guard the suffix append on the absence of "/" in the input. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- .../extra/ingress/templates/cilium-lb-pool.yaml | 2 +- packages/extra/ingress/tests/exposure_test.yaml | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/extra/ingress/templates/cilium-lb-pool.yaml b/packages/extra/ingress/templates/cilium-lb-pool.yaml index a383922b..048eb58b 100644 --- a/packages/extra/ingress/templates/cilium-lb-pool.yaml +++ b/packages/extra/ingress/templates/cilium-lb-pool.yaml @@ -16,7 +16,7 @@ metadata: spec: blocks: {{- range $exposeIPsList }} - - cidr: {{ . }}/{{ if contains ":" . }}128{{ else }}32{{ end }} + - cidr: {{ . }}{{ if not (contains "/" .) }}/{{ if contains ":" . }}128{{ else }}32{{ end }}{{ end }} {{- end }} serviceSelector: matchLabels: diff --git a/packages/extra/ingress/tests/exposure_test.yaml b/packages/extra/ingress/tests/exposure_test.yaml index 780e0715..323c8e68 100644 --- a/packages/extra/ingress/tests/exposure_test.yaml +++ b/packages/extra/ingress/tests/exposure_test.yaml @@ -249,3 +249,17 @@ tests: path: spec.values.ingress-nginx.controller.service.externalIPs value: - 192.0.2.10 + + - it: loadBalancer mode accepts pre-CIDR input without double-suffixing + set: + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10/32,2001:db8::1/128" + expose-mode: loadBalancer + asserts: + - template: templates/cilium-lb-pool.yaml + equal: + path: spec.blocks + value: + - cidr: 192.0.2.10/32 + - cidr: 2001:db8::1/128 From 9b87bda06af20d7c1ca8d96b8f338f3bc937d0c5 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Fri, 24 Apr 2026 03:14:41 +0300 Subject: [PATCH 058/130] fix(postgres-operator): block HelmRelease Ready until the cnpg webhook actually serves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cnpg admission webhook's controller pod passes readinessProbe as soon as the local HTTPS server binds to :9443. The HelmRelease marks itself Ready right after that via helm install --wait. But the Service cnpg-webhook-service needs its EndpointSlice populated and the data plane (kube-proxy / cilium) programmed before kube-apiserver can reach the webhook through the Service ClusterIP. That gap is short but not zero, and any HelmRelease that depends on postgres-operator (cozy-keycloak, tenant Postgres apps) can fire its own install inside the window and hit Internal error occurred: failed calling webhook "mcluster.cnpg.io": failed to call webhook: Post "https://cnpg-webhook-service.cozy-postgres-operator.svc:443/...": dial tcp :443: connect: connection refused which fails the install of the downstream release. Seen on cozystack/cozystack#2470 E2E run 24862782568. Add a post-install,post-upgrade Helm hook Job that blocks the release from reporting Ready until the webhook answers /readyz through the apiserver service proxy. Apiserver proxy routes the call over the same Service IP → EndpointSlice → pod path the admission webhook uses, so once it responds, the webhook admission path is also working. RBAC is minimal: a dedicated ServiceAccount with a ClusterRole that only grants get on services/proxy scoped to https:cnpg-webhook-service:webhook-server. The Job times out after 120s with 60 attempts at 2s intervals — longer than any data-plane programming delay seen on E2E, but bounded. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- packages/system/postgres-operator/Makefile | 3 + .../templates/webhook-ready-hook.yaml | 139 ++++++++ .../tests/webhook-ready-hook_test.yaml | 308 ++++++++++++++++++ packages/system/postgres-operator/values.yaml | 25 ++ 4 files changed, 475 insertions(+) create mode 100644 packages/system/postgres-operator/templates/webhook-ready-hook.yaml create mode 100644 packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml diff --git a/packages/system/postgres-operator/Makefile b/packages/system/postgres-operator/Makefile index f6d7ddaa..5279f8a7 100644 --- a/packages/system/postgres-operator/Makefile +++ b/packages/system/postgres-operator/Makefile @@ -3,6 +3,9 @@ export NAMESPACE=cozy-$(NAME) include ../../../hack/package.mk +test: + helm unittest . + update: rm -rf charts helm repo add cnpg https://cloudnative-pg.github.io/charts diff --git a/packages/system/postgres-operator/templates/webhook-ready-hook.yaml b/packages/system/postgres-operator/templates/webhook-ready-hook.yaml new file mode 100644 index 00000000..2ad4e7c9 --- /dev/null +++ b/packages/system/postgres-operator/templates/webhook-ready-hook.yaml @@ -0,0 +1,139 @@ +{{- /* + Post-install gate: block the HelmRelease from reporting Ready until the + cnpg admission webhook actually serves through the cluster Service. Helm + --wait on the controller pod passes once its readinessProbe passes, but + EndpointSlice propagation and kube-proxy/cilium data-plane programming + can lag by a second or two — long enough for any HelmRelease that + depends on postgres-operator (e.g. cozy-keycloak, tenant Postgres apps) + to fire its own install and have kube-apiserver hit the mcluster.cnpg.io + mutating webhook with "dial tcp :443: connect: connection refused". + + The Job uses the apiserver service proxy, which exercises the same + endpoint-resolution and apiserver-initiated pod dial that the admission + webhook path uses. Once /readyz answers through the proxy the data-plane + race is resolved. It does not verify the webhook's TLS CA bundle, so + this gate is scoped to reachability regressions, not cert rotation. + + The service name and port name are hardcoded literals. Upstream cnpg + pins the service name in charts/cloudnative-pg/values.yaml with a + comment "DO NOT CHANGE THE SERVICE NAME as it is currently used to + generate the certificate and can not be configured". The port name is + fixed in charts/cloudnative-pg/templates/service.yaml (ports[0].name: + webhook-server). If a future `make update` ever changes either literal + upstream, the sync-check helm-unittest test + (tests/webhook-ready-hook_test.yaml) renders the subchart Service and + fails if the literal drifts — forcing this template to be updated in + the same change. +*/}} +{{- $_ := required "webhookReady.image.repository must be set to the container image providing kubectl for the post-install readiness Job" .Values.webhookReady.image.repository -}} +{{- $_ := required "webhookReady.image.tag must be set for the post-install readiness Job" .Values.webhookReady.image.tag -}} +{{- /* $svcName and $portName are hardcoded literals; see header comment. */ -}} +{{- $svcName := "cnpg-webhook-service" -}} +{{- /* $portName is the service port NAME, not number — matches ports[0].name in the vendored subchart's Service. */ -}} +{{- $portName := "webhook-server" -}} +{{- $resourceName := printf "https:%s:%s" $svcName $portName -}} +{{- $maxAttempts := .Values.webhookReady.maxAttempts | default 60 -}} +{{- $sleepSeconds := .Values.webhookReady.sleepSeconds | default 2 -}} +{{- /* Derive activeDeadlineSeconds from retries + 60s slack so a values override that raises maxAttempts doesn't get silently cut. */ -}} +{{- $deadline := add (mul (int $maxAttempts) (int $sleepSeconds)) 60 -}} +{{- $image := printf "%s:%s" .Values.webhookReady.image.repository .Values.webhookReady.image.tag -}} +{{- if .Values.webhookReady.image.digest }} +{{- $image = printf "%s@%s" $image .Values.webhookReady.image.digest -}} +{{- end }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }}-webhook-ready + namespace: {{ .Release.Namespace }} + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "0" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Release.Name }}-webhook-ready + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "0" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +rules: + - apiGroups: [""] + resources: ["services/proxy"] + resourceNames: [{{ $resourceName | quote }}] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Release.Name }}-webhook-ready + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "0" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Release.Name }}-webhook-ready +subjects: + - kind: ServiceAccount + name: {{ .Release.Name }}-webhook-ready + namespace: {{ .Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-webhook-ready + namespace: {{ .Release.Namespace }} + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "10" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +spec: + backoffLimit: {{ .Values.webhookReady.backoffLimit | default 2 }} + activeDeadlineSeconds: {{ $deadline }} + template: + spec: + restartPolicy: Never + serviceAccountName: {{ .Release.Name }}-webhook-ready + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: wait + image: {{ $image }} + imagePullPolicy: {{ if .Values.webhookReady.image.digest }}IfNotPresent{{ else }}Always{{ end }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + command: + - sh + - -c + - | + set -e + ns={{ .Release.Namespace }} + proxy="/api/v1/namespaces/${ns}/services/{{ $resourceName }}/proxy/readyz" + max_attempts={{ $maxAttempts }} + sleep_seconds={{ $sleepSeconds }} + i=0 + last_err="" + until last_err=$(kubectl get --raw "$proxy" 2>&1 >/dev/null); do + i=$((i + 1)) + if [ $i -gt $max_attempts ]; then + echo "timeout: cnpg webhook did not respond through the apiserver proxy after ${max_attempts} attempts (${sleep_seconds}s each)" + echo "last error: ${last_err}" + exit 1 + fi + if [ $((i % 10)) -eq 1 ]; then + echo "attempt $i/${max_attempts}: ${last_err}" + fi + sleep "$sleep_seconds" + done + echo "cnpg webhook is ready" diff --git a/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml b/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml new file mode 100644 index 00000000..69f5a6b5 --- /dev/null +++ b/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml @@ -0,0 +1,308 @@ +suite: cnpg webhook post-install readiness gate + +templates: + - templates/webhook-ready-hook.yaml + - charts/cloudnative-pg/templates/service.yaml + +release: + name: postgres-operator + namespace: cozy-postgres-operator + +tests: + - it: renders four hook objects (SA + ClusterRole + ClusterRoleBinding + Job) + template: templates/webhook-ready-hook.yaml + asserts: + - hasDocuments: + count: 4 + + - it: every rendered object carries post-install and post-upgrade hook annotations + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 0 + equal: + path: metadata.annotations["helm.sh/hook"] + value: post-install,post-upgrade + - documentIndex: 1 + equal: + path: metadata.annotations["helm.sh/hook"] + value: post-install,post-upgrade + - documentIndex: 2 + equal: + path: metadata.annotations["helm.sh/hook"] + value: post-install,post-upgrade + - documentIndex: 3 + equal: + path: metadata.annotations["helm.sh/hook"] + value: post-install,post-upgrade + + - it: RBAC is created before the Job (hook-weight ordering) + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 0 + equal: + path: kind + value: ServiceAccount + - documentIndex: 0 + equal: + path: metadata.annotations["helm.sh/hook-weight"] + value: "0" + - documentIndex: 1 + equal: + path: kind + value: ClusterRole + - documentIndex: 1 + equal: + path: metadata.annotations["helm.sh/hook-weight"] + value: "0" + - documentIndex: 2 + equal: + path: kind + value: ClusterRoleBinding + - documentIndex: 2 + equal: + path: metadata.annotations["helm.sh/hook-weight"] + value: "0" + - documentIndex: 3 + equal: + path: kind + value: Job + - documentIndex: 3 + equal: + path: metadata.annotations["helm.sh/hook-weight"] + value: "10" + + - it: RBAC resourceName matches the exact proxy URL segment the Job probes + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 1 + equal: + path: rules[0].resources[0] + value: services/proxy + - documentIndex: 1 + equal: + path: rules[0].resourceNames[0] + value: https:cnpg-webhook-service:webhook-server + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: "/api/v1/namespaces/\\$\\{ns\\}/services/https:cnpg-webhook-service:webhook-server/proxy/readyz" + + - it: hardcoded service name in the hook matches the vendored cnpg subchart Service (drift guard for make update) + template: charts/cloudnative-pg/templates/service.yaml + asserts: + - equal: + path: metadata.name + value: cnpg-webhook-service + - equal: + path: spec.ports[0].name + value: webhook-server + + - it: Job calls kubectl get --raw on the proxy path + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: "kubectl get --raw" + + - it: Job image is digest-pinned when webhookReady.image.digest is set + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: v1.32 + digest: sha256:b9ef7d8dbe65bcc81a46c09b8dc7543103055021c4f43287bf59e92a8f4fe05c + asserts: + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].image + value: docker.io/clastix/kubectl:v1.32@sha256:b9ef7d8dbe65bcc81a46c09b8dc7543103055021c4f43287bf59e92a8f4fe05c + + - it: Job image falls back to tag-only when digest is not configured + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: v1.32 + digest: "" + asserts: + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].image + value: docker.io/clastix/kubectl:v1.32 + + - it: backoffLimit defaults to 2 so transient pod-level failures retry instead of killing the HelmRelease + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 3 + equal: + path: spec.backoffLimit + value: 2 + + - it: backoffLimit is overridable + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + backoffLimit: 5 + asserts: + - documentIndex: 3 + equal: + path: spec.backoffLimit + value: 5 + + - it: chart render fails when webhookReady.image.repository is empty + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: "" + tag: v1.32 + asserts: + - failedTemplate: + errorMessage: "webhookReady.image.repository must be set to the container image providing kubectl for the post-install readiness Job" + + - it: chart render fails when webhookReady.image.tag is empty + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: "" + asserts: + - failedTemplate: + errorMessage: "webhookReady.image.tag must be set for the post-install readiness Job" + + - it: retry loop bounds default when blanked so a wiped override still produces a working Job + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: v1.32 + maxAttempts: null + sleepSeconds: null + asserts: + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: "max_attempts=60" + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: "sleep_seconds=2" + + - it: Job pod runs non-root with seccomp RuntimeDefault for restricted-PSA clusters + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 3 + equal: + path: spec.template.spec.securityContext.runAsNonRoot + value: true + - documentIndex: 3 + equal: + path: spec.template.spec.securityContext.seccompProfile.type + value: RuntimeDefault + + - it: Job container drops all capabilities and runs read-only rootfs + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation + value: false + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem + value: true + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].securityContext.capabilities.drop[0] + value: ALL + + - it: imagePullPolicy is IfNotPresent when digest-pinned, Always when tag-only + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: v1.32 + digest: sha256:b9ef7d8dbe65bcc81a46c09b8dc7543103055021c4f43287bf59e92a8f4fe05c + asserts: + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: IfNotPresent + + - it: imagePullPolicy is Always when no digest is configured + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: v1.32 + digest: "" + asserts: + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always + + - it: retry loop captures and surfaces the last kubectl error message on timeout + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'last_err=\$\(kubectl get --raw' + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'last error: \$\{last_err\}' + + - it: retry loop error message stays in sync when maxAttempts is bumped + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: v1.32 + maxAttempts: 90 + sleepSeconds: 3 + asserts: + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: "max_attempts=90" + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: "sleep_seconds=3" + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'after \$\{max_attempts\} attempts' + + - it: activeDeadlineSeconds scales with retry bounds so an override raise does not silently cut + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: v1.32 + maxAttempts: 180 + sleepSeconds: 3 + asserts: + - documentIndex: 3 + equal: + path: spec.activeDeadlineSeconds + value: 600 + + - it: activeDeadlineSeconds defaults include 60s slack over the default retry window + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 3 + equal: + path: spec.activeDeadlineSeconds + value: 180 diff --git a/packages/system/postgres-operator/values.yaml b/packages/system/postgres-operator/values.yaml index cae3dc53..854a3ac6 100644 --- a/packages/system/postgres-operator/values.yaml +++ b/packages/system/postgres-operator/values.yaml @@ -3,3 +3,28 @@ cloudnative-pg: create: true image: tag: "1.27.3" +# Image used by the post-install webhook-readiness Job (see templates/webhook-ready-hook.yaml). +# Any image with kubectl on PATH works; the Job calls the apiserver service proxy with the +# hook ServiceAccount's token to confirm the mcluster.cnpg.io webhook is reachable end-to-end +# before the HelmRelease reports Ready, closing the "connection refused" bootstrap race. +# +# The tag is digest-pinned so an upstream retag does not change what runs on every install +# and upgrade across the fleet. Refresh by resolving the current manifest-list digest +# (`docker manifest inspect docker.io/clastix/kubectl:v1.32`) and updating `digest` below. +# renovate: datasource=docker depName=docker.io/clastix/kubectl +webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: v1.32 + digest: sha256:b9ef7d8dbe65bcc81a46c09b8dc7543103055021c4f43287bf59e92a8f4fe05c + # Retry loop bounds for the readiness probe. Defaults total ~120s wall clock. + # Both are `default`-coerced in the template so an override that blanks them still + # produces a working Job. + maxAttempts: 60 + sleepSeconds: 2 + # Pod-level retry budget for transient node/registry failures (image pull rate limit, + # OOM, CNI hiccup). The shell loop inside the container only covers reachability retries. + # Pod retries and activeDeadlineSeconds (wall-clock bound on the whole Job across all + # pod retries) are ANDed: a 5-minute ImagePullBackOff on attempt 0 leaves only ~60s for + # subsequent retries before the Job is cut. + backoffLimit: 2 From 3b3540448975391ad7582db58d83ce5b18b15129 Mon Sep 17 00:00:00 2001 From: Denis Chernosov Date: Fri, 24 Apr 2026 11:32:20 +0400 Subject: [PATCH 059/130] fix(cozystack-engine): add managed Kubernetes without visible control-plane nodes support to lineage-controller-webhook (#2417) Signed-off-by: Denis Chernosov --- .../templates/daemonset.yaml | 2 + .../templates/deployment.yaml | 49 +++++++++++++++++++ .../templates/pdb.yaml | 11 +++++ .../lineage-controller-webhook/values.yaml | 5 ++ 4 files changed, 67 insertions(+) create mode 100644 packages/system/lineage-controller-webhook/templates/deployment.yaml create mode 100644 packages/system/lineage-controller-webhook/templates/pdb.yaml diff --git a/packages/system/lineage-controller-webhook/templates/daemonset.yaml b/packages/system/lineage-controller-webhook/templates/daemonset.yaml index 860aee6e..536ea24a 100644 --- a/packages/system/lineage-controller-webhook/templates/daemonset.yaml +++ b/packages/system/lineage-controller-webhook/templates/daemonset.yaml @@ -1,3 +1,4 @@ +{{- if not .Values.lineageControllerWebhook.deployment.enabled }} apiVersion: apps/v1 kind: DaemonSet metadata: @@ -51,3 +52,4 @@ spec: secret: secretName: lineage-controller-webhook-cert defaultMode: 0400 +{{- end }} \ No newline at end of file diff --git a/packages/system/lineage-controller-webhook/templates/deployment.yaml b/packages/system/lineage-controller-webhook/templates/deployment.yaml new file mode 100644 index 00000000..87717fd3 --- /dev/null +++ b/packages/system/lineage-controller-webhook/templates/deployment.yaml @@ -0,0 +1,49 @@ +{{- if .Values.lineageControllerWebhook.deployment.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: lineage-controller-webhook + labels: + app: lineage-controller-webhook +spec: + replicas: {{ .Values.lineageControllerWebhook.deployment.replicas }} + selector: + matchLabels: + app: lineage-controller-webhook + template: + metadata: + labels: + app: lineage-controller-webhook + spec: + serviceAccountName: lineage-controller-webhook + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app: lineage-controller-webhook + containers: + - name: lineage-controller-webhook + image: "{{ .Values.lineageControllerWebhook.image }}" + args: + {{- if .Values.lineageControllerWebhook.debug }} + - --zap-log-level=debug + {{- else }} + - --zap-log-level=info + {{- end }} + ports: + - name: webhook + containerPort: 9443 + volumeMounts: + - name: webhook-certs + mountPath: /tmp/k8s-webhook-server/serving-certs + readOnly: true + volumes: + - name: webhook-certs + secret: + secretName: lineage-controller-webhook-cert + defaultMode: 0400 +{{- end }} \ No newline at end of file diff --git a/packages/system/lineage-controller-webhook/templates/pdb.yaml b/packages/system/lineage-controller-webhook/templates/pdb.yaml new file mode 100644 index 00000000..b0a9d4e4 --- /dev/null +++ b/packages/system/lineage-controller-webhook/templates/pdb.yaml @@ -0,0 +1,11 @@ +{{- if .Values.lineageControllerWebhook.deployment.enabled }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: lineage-controller-webhook +spec: + minAvailable: {{ if gt (int .Values.lineageControllerWebhook.deployment.replicas) 1 }}1{{ else }}0{{ end }} + selector: + matchLabels: + app: lineage-controller-webhook +{{- end }} \ No newline at end of file diff --git a/packages/system/lineage-controller-webhook/values.yaml b/packages/system/lineage-controller-webhook/values.yaml index c5671548..2b84f4e1 100644 --- a/packages/system/lineage-controller-webhook/values.yaml +++ b/packages/system/lineage-controller-webhook/values.yaml @@ -2,9 +2,14 @@ lineageControllerWebhook: image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.3.0@sha256:e8984709686a5eaf19b89da378d7b8c688ea5607e0783a88d9c9e4ccfca96fb0 debug: false localK8sAPIEndpoint: + # incompatable with Deployment mode enabled: true # nodeSelector for the DaemonSet # Talos uses empty value: "node-role.kubernetes.io/control-plane": "" # Generic k8s (k3s, kubeadm) uses: "node-role.kubernetes.io/control-plane": "true" nodeSelector: node-role.kubernetes.io/control-plane: "" + # if enabled, replace DaemonSet by Deployment + deployment: + enabled: false + replicas: 1 From ab7deb2b055c2a677c4c84c1bb22271306ebf372 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Fri, 24 Apr 2026 13:03:49 +0300 Subject: [PATCH 060/130] chore(kubernetes): regenerate after HAMi addon integration Run make generate to update generated types, schema, README, and CRD definitions for the new HAMi addon. Signed-off-by: Arsolitt --- api/apps/v1alpha1/kubernetes/types.go | 12 ++++++++++++ packages/apps/kubernetes/README.md | 3 +++ packages/apps/kubernetes/values.schema.json | 4 ++-- .../system/kubernetes-rd/cozyrds/kubernetes.yaml | 4 ++-- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/api/apps/v1alpha1/kubernetes/types.go b/api/apps/v1alpha1/kubernetes/types.go index 774fcdc5..39a05567 100644 --- a/api/apps/v1alpha1/kubernetes/types.go +++ b/api/apps/v1alpha1/kubernetes/types.go @@ -66,6 +66,9 @@ type Addons struct { // NVIDIA GPU Operator. // +kubebuilder:default:={} GpuOperator GPUOperatorAddon `json:"gpuOperator"` + // HAMi GPU virtualization middleware. + // +kubebuilder:default:={} + Hami HAMiAddon `json:"hami"` // Ingress-NGINX controller. // +kubebuilder:default:={} IngressNginx IngressNginxAddon `json:"ingressNginx"` @@ -157,6 +160,15 @@ type GatewayAPIAddon struct { Enabled bool `json:"enabled"` } +type HAMiAddon struct { + // Enable HAMi (requires GPU Operator). + // +kubebuilder:default:=false + Enabled bool `json:"enabled"` + // Custom Helm values overrides. + // +kubebuilder:default:={} + ValuesOverride k8sRuntime.RawExtension `json:"valuesOverride"` +} + type IngressNginxAddon struct { // Enable the controller (requires nodes labeled `ingress-nginx`). // +kubebuilder:default:=false diff --git a/packages/apps/kubernetes/README.md b/packages/apps/kubernetes/README.md index d62d3a39..d705af64 100644 --- a/packages/apps/kubernetes/README.md +++ b/packages/apps/kubernetes/README.md @@ -128,6 +128,9 @@ See the reference for components utilized in this service: | `addons.gpuOperator` | NVIDIA GPU Operator. | `object` | `{}` | | `addons.gpuOperator.enabled` | Enable GPU Operator. | `bool` | `false` | | `addons.gpuOperator.valuesOverride` | Custom Helm values overrides. | `object` | `{}` | +| `addons.hami` | HAMi GPU virtualization middleware. | `object` | `{}` | +| `addons.hami.enabled` | Enable HAMi (requires GPU Operator). | `bool` | `false` | +| `addons.hami.valuesOverride` | Custom Helm values overrides. | `object` | `{}` | | `addons.fluxcd` | FluxCD GitOps operator. | `object` | `{}` | | `addons.fluxcd.enabled` | Enable FluxCD. | `bool` | `false` | | `addons.fluxcd.valuesOverride` | Custom Helm values overrides. | `object` | `{}` | diff --git a/packages/apps/kubernetes/values.schema.json b/packages/apps/kubernetes/values.schema.json index 3d7a17dd..1e457f1f 100644 --- a/packages/apps/kubernetes/values.schema.json +++ b/packages/apps/kubernetes/values.schema.json @@ -270,7 +270,7 @@ } }, "hami": { - "description": "HAMi GPU virtualization middleware. Enables fractional GPU sharing (memory and compute isolation) for workloads. Requires GPU Operator. Note: workload containers must use glibc < 2.34 (not musl/Alpine) for full compute isolation.", + "description": "HAMi GPU virtualization middleware.", "type": "object", "default": {}, "required": [ @@ -279,7 +279,7 @@ ], "properties": { "enabled": { - "description": "Enable HAMi.", + "description": "Enable HAMi (requires GPU Operator).", "type": "boolean", "default": false }, diff --git a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml index 5e9e8f94..cdf900e1 100644 --- a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml +++ b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml @@ -8,7 +8,7 @@ spec: singular: kubernetes plural: kuberneteses openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"ephemeralStorage":"20Gi","gpus":[],"instanceType":"u1.medium","maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"]}},"additionalProperties":{"type":"object","required":["ephemeralStorage","instanceType","maxReplicas","minReplicas","resources"],"properties":{"ephemeralStorage":{"description":"Ephemeral storage size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string"}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium"},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `.` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","ingressNginx","monitoringAgents","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"large","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}}}} + {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"ephemeralStorage":"20Gi","gpus":[],"instanceType":"u1.medium","maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"]}},"additionalProperties":{"type":"object","required":["ephemeralStorage","instanceType","maxReplicas","minReplicas","resources"],"properties":{"ephemeralStorage":{"description":"Ephemeral storage size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string"}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium"},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `.` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","hami","ingressNginx","monitoringAgents","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"hami":{"description":"HAMi GPU virtualization middleware.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable HAMi (requires GPU Operator).","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"large","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}}}} release: prefix: kubernetes- labels: @@ -26,7 +26,7 @@ spec: tags: - compute icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODFfMjg0NSkiLz4KPHBhdGggZD0iTTcxLjk5NjggMTlDNzAuMzAzOSAxOS4wMDAyIDY4LjkzMTIgMjAuNTMyMiA2OC45MzE0IDIyLjQyMjFDNjguOTMxNCAyMi40NTExIDY4LjkzNzMgMjIuNDc4OCA2OC45Mzc5IDIyLjUwNzZDNjguOTM1NCAyMi43NjQ0IDY4LjkyMzEgMjMuMDczNyA2OC45MzE0IDIzLjI5NzNDNjguOTcxNyAyNC4zODczIDY5LjIwODIgMjUuMjIxNiA2OS4zNTA2IDI2LjIyNThDNjkuNjA4NCAyOC4zNzUyIDY5LjgyNDUgMzAuMTU2OSA2OS42OTEyIDMxLjgxM0M2OS41NjE1IDMyLjQzNzUgNjkuMTAzNyAzMy4wMDg2IDY4LjY5NTYgMzMuNDA1Nkw2OC42MjM1IDM0LjcwODZDNjYuNzgzOSAzNC44NjE3IDY0LjkzMTkgMzUuMTQyMSA2My4wODIxIDM1LjU2NDFDNTUuMTIyNiAzNy4zNzk4IDQ4LjI2OTUgNDEuNDk5MSA0My4wNTIgNDcuMDYwOUM0Mi43MTM0IDQ2LjgyODggNDIuMTIxMSA0Ni40MDE5IDQxLjk0NSA0Ni4yNzEyQzQxLjM5NzcgNDYuMzQ1NCA0MC44NDQ1IDQ2LjUxNTEgNDAuMTI0MSA0Ni4wOTM1QzM4Ljc1MjIgNDUuMTY1NyAzNy41MDI4IDQzLjg4NTEgMzUuOTkxIDQyLjM0MjRDMzUuMjk4MiA0MS42MDQ0IDM0Ljc5NjYgNDAuOTAxOCAzMy45NzM1IDQwLjE5MDRDMzMuNzg2NiA0MC4wMjg5IDMzLjUwMTQgMzkuODEwNCAzMy4yOTIzIDM5LjY0NDJDMzIuNjQ4OSAzOS4xMjg4IDMxLjg5IDM4Ljg2IDMxLjE1NyAzOC44MzQ4QzMwLjIxNDcgMzguODAyNCAyOS4zMDc1IDM5LjE3MjUgMjguNzEzOCAzOS45MjA2QzI3LjY1ODQgNDEuMjUwNiAyNy45OTYzIDQzLjI4MzMgMjkuNDY3MSA0NC40NjE0QzI5LjQ4MiA0NC40NzM0IDI5LjQ5NzkgNDQuNDgyNyAyOS41MTI5IDQ0LjQ5NDNDMjkuNzE1IDQ0LjY1ODkgMjkuOTYyNSA0NC44Njk4IDMwLjE0ODMgNDUuMDA3NkMzMS4wMjE3IDQ1LjY1NTUgMzEuODE5NSA0NS45ODcyIDMyLjY4OTcgNDYuNTAxNUMzNC41MjMxIDQ3LjYzOTEgMzYuMDQzIDQ4LjU4MjMgMzcuMjQ4NiA0OS43MTk2QzM3LjcxOTQgNTAuMjIzNyAzNy44MDE2IDUxLjExMjIgMzcuODY0MyA1MS40OTY0TDM4Ljg0NjggNTIuMzc4MkMzMy41ODcyIDYwLjMzMDggMzEuMTUzIDcwLjE1MzkgMzIuNTkxNSA4MC4xNjI3TDMxLjMwNzcgODAuNTM3OEMzMC45NjkzIDgwLjk3NjggMzAuNDkxMiA4MS42Njc2IDI5Ljk5MTEgODEuODczOEMyOC40MTM4IDgyLjM3MjkgMjYuNjM4NyA4Mi41NTYyIDI0LjQ5NTYgODIuNzgxOUMyMy40ODk0IDgyLjg2NiAyMi42MjEzIDgyLjgxNTggMjEuNTU0NiA4My4wMTg4QzIxLjMxOTggODMuMDYzNSAyMC45OTI3IDgzLjE0OTEgMjAuNzM1OCA4My4yMDk3QzIwLjcyNjkgODMuMjExNiAyMC43MTg2IDgzLjIxNDIgMjAuNzA5NiA4My4yMTYyQzIwLjY5NTYgODMuMjE5NSAyMC42NzcyIDgzLjIyNjMgMjAuNjYzOCA4My4yMjk0QzE4Ljg1NyA4My42NjggMTcuNjk2MyA4NS4zMzY1IDE4LjA2OTkgODYuOTgwNUMxOC40NDM3IDg4LjYyNDggMjAuMjA4NiA4OS42MjQ4IDIyLjAyNjIgODkuMjMxMkMyMi4wMzkzIDg5LjIyODIgMjIuMDU4NCA4OS4yMjc3IDIyLjA3MiA4OS4yMjQ2QzIyLjA5MjYgODkuMjE5OSAyMi4xMTA2IDg5LjIwOTkgMjIuMTMxIDg5LjIwNDlDMjIuMzg0NCA4OS4xNDkgMjIuNzAxOSA4OS4wODY4IDIyLjkyMzYgODkuMDI3MkMyMy45NzIzIDg4Ljc0NTEgMjQuNzMxOCA4OC4zMzA2IDI1LjY3NDYgODcuOTY3N0MyNy43MDI5IDg3LjIzNjggMjkuMzgyOCA4Ni42MjYyIDMxLjAxOTUgODYuMzg4M0MzMS43MDMgODYuMzM0NSAzMi40MjMyIDg2LjgxMiAzMi43ODE0IDg3LjAxMzRMMzQuMTE3NyA4Ni43ODMxQzM3LjE5MjYgOTYuMzYxMyA0My42MzY2IDEwNC4xMDMgNTEuNzk2MyAxMDguOTYxTDUxLjIzOTYgMTEwLjMwM0M1MS40NDAzIDExMC44MjQgNTEuNjYxNiAxMTEuNTMgNTEuNTEyMSAxMTIuMDQ1QzUwLjkxNzEgMTEzLjU5NSA0OS44OTggMTE1LjIzMSA0OC43Mzc0IDExNy4wNTVDNDguMTc1NSAxMTcuODk4IDQ3LjYwMDQgMTE4LjU1MiA0Ny4wOTM0IDExOS41MTZDNDYuOTcyIDExOS43NDcgNDYuODE3NSAxMjAuMTAyIDQ2LjcwMDQgMTIwLjM0NkM0NS45MTI1IDEyMi4wMzkgNDYuNDkwNCAxMjMuOTkgNDguMDAzOCAxMjQuNzIyQzQ5LjUyNjggMTI1LjQ1OSA1MS40MTcxIDEyNC42ODIgNTIuMjM1MiAxMjIuOTg1QzUyLjIzNjQgMTIyLjk4MiA1Mi4yNDA2IDEyMi45OCA1Mi4yNDE3IDEyMi45NzhDNTIuMjQyNiAxMjIuOTc2IDUyLjI0MDkgMTIyLjk3MyA1Mi4yNDE3IDEyMi45NzFDNTIuMzU4MiAxMjIuNzMxIDUyLjUyMzMgMTIyLjQxNSA1Mi42MjE2IDEyMi4xODhDNTMuMDU2IDEyMS4xODkgNTMuMjAwNSAxMjAuMzMyIDUzLjUwNTkgMTE5LjM2NUM1NC4zMTcgMTE3LjMxOCA1NC43NjI2IDExNS4xNyA1NS44NzkxIDExMy44MzJDNTYuMTg0OSAxMTMuNDY2IDU2LjY4MzMgMTEzLjMyNSA1Ny4yMDAxIDExMy4xODZMNTcuODk0NCAxMTEuOTIyQzY1LjAwOCAxMTQuNjY1IDcyLjk3MDUgMTE1LjQwMiA4MC45MjQ1IDExMy41ODdDODIuNzM5MSAxMTMuMTczIDg0LjQ5MDggMTEyLjYzNyA4Ni4xODQzIDExMS45OTRDODYuMzc5NCAxMTIuMzQyIDg2Ljc0MiAxMTMuMDExIDg2LjgzOTMgMTEzLjE3OUM4Ny4zNjQ0IDExMy4zNTEgODcuOTM3NyAxMTMuNDM5IDg4LjQwNDcgMTE0LjEzM0M4OS4yNDAxIDExNS41NjcgODkuODExNCAxMTcuMjYzIDkwLjUwNzMgMTE5LjMxMkM5MC44MTI4IDEyMC4yNzkgOTAuOTYzOCAxMjEuMTM2IDkxLjM5ODEgMTIyLjEzNkM5MS40OTcxIDEyMi4zNjMgOTEuNjYxNCAxMjIuNjg0IDkxLjc3OCAxMjIuOTI1QzkyLjU5NDQgMTI0LjYyOCA5NC40OTA3IDEyNS40MDcgOTYuMDE1OSAxMjQuNjY5Qzk3LjUyOTIgMTIzLjkzNyA5OC4xMDc3IDEyMS45ODYgOTcuMzE5NCAxMjAuMjkzQzk3LjIwMjMgMTIwLjA0OSA5Ny4wNDEyIDExOS42OTUgOTYuOTE5OCAxMTkuNDY0Qzk2LjQxMjcgMTE4LjQ5OSA5NS44Mzc3IDExNy44NTIgOTUuMjc1OCAxMTcuMDA5Qzk0LjExNTIgMTE1LjE4NSA5My4xNTI2IDExMy42NyA5Mi41NTc1IDExMi4xMkM5Mi4zMDg3IDExMS4zMiA5Mi41OTk1IDExMC44MjMgOTIuNzkzMyAxMTAuMzAzQzkyLjY3NzIgMTEwLjE3IDkyLjQyODggMTA5LjQxNCA5Mi4yODI0IDEwOS4wNTlDMTAwLjc2MiAxMDQuMDI5IDEwNy4wMTcgOTUuOTk4NSAxMDkuOTU1IDg2LjcyMzlDMTEwLjM1MSA4Ni43ODY1IDExMS4wNDEgODYuOTA5MSAxMTEuMjY1IDg2Ljk1NDJDMTExLjcyNiA4Ni42NDg3IDExMi4xNDkgODYuMjUwMSAxMTIuOTgxIDg2LjMxNTlDMTE0LjYxNyA4Ni41NTM3IDExNi4yOTcgODcuMTY0NSAxMTguMzI2IDg3Ljg5NTNDMTE5LjI2OCA4OC4yNTgxIDEyMC4wMjggODguNjc5MyAxMjEuMDc3IDg4Ljk2MTRDMTIxLjI5OCA4OS4wMjEgMTIxLjYxNiA4OS4wNzY2IDEyMS44NjkgODkuMTMyNUMxMjEuODg5IDg5LjEzNzUgMTIxLjkwOCA4OS4xNDc1IDEyMS45MjggODkuMTUyMkMxMjEuOTQyIDg5LjE1NTMgMTIxLjk2MSA4OS4xNTU4IDEyMS45NzQgODkuMTU4OEMxMjMuNzkyIDg5LjU1MiAxMjUuNTU3IDg4LjU1MjYgMTI1LjkzIDg2LjkwODFDMTI2LjMwMyA4NS4yNjQxIDEyNS4xNDMgODMuNTk1MiAxMjMuMzM2IDgzLjE1N0MxMjMuMDc0IDgzLjA5NyAxMjIuNzAxIDgyLjk5NSAxMjIuNDQ2IDgyLjk0NjVDMTIxLjM3OSA4Mi43NDM1IDEyMC41MTEgODIuNzkzNSAxMTkuNTA1IDgyLjcwOTVDMTE3LjM2MSA4Mi40ODM5IDExNS41ODYgODIuMzAwNCAxMTQuMDA5IDgxLjgwMTRDMTEzLjM2NiA4MS41NTA3IDExMi45MDggODAuNzgxOSAxMTIuNjg2IDgwLjQ2NTVMMTExLjQ0OCA4MC4xMDM1QzExMi4wOSA3NS40MzggMTExLjkxNyA3MC41ODI1IDExMC44MDYgNjUuNzI0M0MxMDkuNjg1IDYwLjgyMDggMTA3LjcwNCA1Ni4zMzYxIDEwNS4wNjIgNTIuMzg0OEMxMDUuMzc5IDUyLjA5NDggMTA1Ljk3OSA1MS41NjEyIDEwNi4xNDkgNTEuNDA0M0MxMDYuMTk5IDUwLjg1MTcgMTA2LjE1NiA1MC4yNzIyIDEwNi43MjUgNDkuNjYwM0MxMDcuOTMxIDQ4LjUyMyAxMDkuNDUxIDQ3LjU3OTkgMTExLjI4NCA0Ni40NDIzQzExMi4xNTQgNDUuOTI3OSAxMTIuOTU5IDQ1LjU5NjQgMTEzLjgzMiA0NC45NDg0QzExNC4wMyA0NC44MDE5IDExNC4yOTkgNDQuNTY5OSAxMTQuNTA3IDQ0LjQwMjJDMTE1Ljk3NyA0My4yMjM3IDExNi4zMTYgNDEuMTkxMSAxMTUuMjYgMzkuODYxNEMxMTQuMjA0IDM4LjUzMTcgMTEyLjE1OSAzOC40MDY1IDExMC42ODggMzkuNTg1QzExMC40NzkgMzkuNzUxNiAxMTAuMTk1IDM5Ljk2ODggMTEwLjAwNyA0MC4xMzEyQzEwOS4xODQgNDAuODQyNiAxMDguNjc2IDQxLjU0NTIgMTA3Ljk4MyA0Mi4yODMyQzEwNi40NzEgNDMuODI1OSAxMDUuMjIyIDQ1LjExMyAxMDMuODUgNDYuMDQwOUMxMDMuMjU1IDQ2LjM4ODUgMTAyLjM4NSA0Ni4yNjgyIDEwMS45OSA0Ni4yNDQ5TDEwMC44MjQgNDcuMDgwNkM5NC4xNzUzIDQwLjA3NjMgODUuMTIzNSAzNS41OTgyIDc1LjM3NjYgMzQuNzI4M0M3NS4zNDk0IDM0LjMxNzkgNzUuMzEzNyAzMy41NzYxIDc1LjMwNDYgMzMuMzUyOUM3NC45MDU2IDMyLjk2OTMgNzQuNDIzNSAzMi42NDE4IDc0LjMwMjQgMzEuODEzQzc0LjE2OTEgMzAuMTU2OSA3NC4zOTE3IDI4LjM3NTIgNzQuNjQ5NiAyNi4yMjU4Qzc0Ljc5MTkgMjUuMjIxNiA3NS4wMjg0IDI0LjM4NzMgNzUuMDY4OCAyMy4yOTczQzc1LjA3OCAyMy4wNDk1IDc1LjA2MzIgMjIuNjkgNzUuMDYyMiAyMi40MjIxQzc1LjA2MiAyMC41MzIyIDczLjY4OTggMTguOTk5OCA3MS45OTY4IDE5Wk02OC4xNTg1IDQyLjg4ODZMNjcuMjQ4IDU5LjA0NDdMNjcuMTgyNSA1OS4wNzc2QzY3LjEyMTQgNjAuNTIyOSA2NS45Mzc1IDYxLjY3NyA2NC40ODM5IDYxLjY3N0M2My44ODg0IDYxLjY3NyA2My4zMzg4IDYxLjQ4NDkgNjIuODkyMiA2MS4xNTcxTDYyLjg2NiA2MS4xNzAzTDQ5LjY4MDcgNTEuNzc5NEM1My43MzMxIDQ3Ljc3NTkgNTguOTE2NCA0NC44MTcyIDY0Ljg5IDQzLjQ1NDZDNjUuOTgxMiA0My4yMDU2IDY3LjA3MTkgNDMuMDIwOSA2OC4xNTg1IDQyLjg4ODZaTTc1Ljg0MTcgNDIuODg4NkM4Mi44MTU5IDQzLjc1MDQgODkuMjY1NyA0Ni45MjMyIDk0LjIwODEgNTEuNzg2TDgxLjEwOCA2MS4xMTc2TDgxLjA2MjEgNjEuMDk3OUM3OS44OTk0IDYxLjk1MTIgNzguMjYxMSA2MS43Mzk0IDc3LjM1NDggNjAuNTk3OEM3Ni45ODM1IDYwLjEzMDEgNzYuNzg4NyA1OS41ODAxIDc2Ljc2NTMgNTkuMDI0OUw3Ni43NTIyIDU5LjAxODRMNzUuODQxNyA0Mi44ODg2Wk00NC44OTkxIDU3LjgxNEw1Ni45MzgyIDY4LjYzM0w1Ni45MjUxIDY4LjY5ODhDNTguMDExNyA2OS42NDc5IDU4LjE3MiA3MS4yOTQ5IDU3LjI2NTcgNzIuNDM2OEM1Ni44OTQ0IDcyLjkwNDUgNTYuMzk3NSA3My4yMTgyIDU1Ljg2MzkgNzMuMzY0N0w1NS44NTA4IDczLjQxNzNMNDAuNDE4OCA3Ny44OTIzQzM5LjYzMzQgNzAuNjc2NSA0MS4zMjYxIDYzLjY2MjEgNDQuODk5MSA1Ny44MTRaTTk5LjAwOTQgNTcuODIwNkMxMDAuNzk4IDYwLjczMzYgMTAyLjE1MyA2My45ODcxIDEwMi45NTkgNjcuNTE0M0MxMDMuNzU2IDcwLjk5OTEgMTAzLjk1NiA3NC40Nzc4IDEwMy42MjcgNzcuODM5N0w4OC4xMTY2IDczLjM1MTVMODguMTAzNSA3My4yODU3Qzg2LjcxNDUgNzIuOTA0MyA4NS44NjA5IDcxLjQ4NDggODYuMTg0MyA3MC4wNjExQzg2LjMxNjggNjkuNDc3OCA4Ni42MjQ5IDY4Ljk4NDQgODcuMDQyMyA2OC42MTk4TDg3LjAzNTggNjguNTg2OUw5OS4wMDk0IDU3LjgyMDZaTTY5LjUyNzQgNjkuNDY4OEg3NC40NTk2TDc3LjUyNTEgNzMuMzE4Nkw3Ni40MjQ3IDc4LjEyMjZMNzEuOTk2OCA4MC4yNjE0TDY3LjU1NTggNzguMTE2MUw2Ni40NTU0IDczLjMxMkw2OS41Mjc0IDY5LjQ2ODhaTTg1LjMzOTMgODIuNjQzN0M4NS41NDg5IDgyLjYzMzEgODUuNzU3NiA4Mi42NTIgODUuOTYxNiA4Mi42ODk4TDg1Ljk4NzggODIuNjU2OUwxMDEuOTUgODUuMzY4MkM5OS42MTQyIDkxLjk2MjQgOTUuMTQ0IDk3LjY3NSA4OS4xNzExIDEwMS40OThMODIuOTc0NyA4Ni40NjA2TDgyLjk5NDQgODYuNDM0M0M4Mi40MjUyIDg1LjEwNTUgODIuOTk0OCA4My41NDcyIDg0LjMwNDQgODIuOTEzNUM4NC42Mzk3IDgyLjc1MTMgODQuOTkgODIuNjYxNCA4NS4zMzkzIDgyLjY0MzdaTTU4LjUyOTggODIuNzA5NUM1OS43NDggODIuNzI2NyA2MC44NDA2IDgzLjU3NjEgNjEuMTIzNyA4NC44MjJDNjEuMjU2MiA4NS40MDUyIDYxLjE5MTcgODUuOTgzMSA2MC45NzMgODYuNDkzNUw2MS4wMTg5IDg2LjU1MjdMNTQuODg4IDEwMS40MzlDNDkuMTU1OSA5Ny43NDMyIDQ0LjU5MDQgOTIuMjA5OSA0Mi4xNDgxIDg1LjQyMDhMNTcuOTczMSA4Mi43MjI3TDU3Ljk5OTMgODIuNzU1NkM1OC4xNzYzIDgyLjcyMjkgNTguMzU1OCA4Mi43MDcxIDU4LjUyOTggODIuNzA5NVpNNzEuODk4NiA4OS4yMzEyQzcyLjMyMjkgODkuMjE1NSA3Mi43NTM0IDg5LjMwMyA3My4xNjI3IDg5LjUwMUM3My42OTkyIDg5Ljc2MDYgNzQuMTEzNiA5MC4xNjkyIDc0LjM3NDUgOTAuNjU5Mkg3NC40MzM0TDgyLjIzNDYgMTA0LjgyMUM4MS4yMjIxIDEwNS4xNjIgODAuMTgxMyAxMDUuNDU0IDc5LjExNjcgMTA1LjY5N0M3My4xNTA1IDEwNy4wNTggNjcuMjAzMiAxMDYuNjQ1IDYxLjgxOCAxMDQuODAyTDY5LjU5OTUgOTAuNjY1OEg2OS42MTI2QzcwLjA3OTUgODkuNzg4OCA3MC45NjUgODkuMjY1NiA3MS44OTg2IDg5LjIzMTJaIiBmaWxsPSJ3aGl0ZSIgc3Ryb2tlPSJ3aGl0ZSIgc3Ryb2tlLXdpZHRoPSIwLjI1Ii8+CjxkZWZzPgo8bGluZWFyR3JhZGllbnQgaWQ9InBhaW50MF9saW5lYXJfNjgxXzI4NDUiIHgxPSIxMCIgeTE9IjE1LjUiIHgyPSIxNDQiIHkyPSIxMzEuNSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiPgo8c3RvcCBzdG9wLWNvbG9yPSIjNEQ4N0ZGIi8+CjxzdG9wIG9mZnNldD0iMSIgc3RvcC1jb2xvcj0iIzA1NDdEMCIvPgo8L2xpbmVhckdyYWRpZW50Pgo8L2RlZnM+Cjwvc3ZnPgo= - keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "nodeGroups"], ["spec", "nodeGroups", "md0"], ["spec", "nodeGroups", "md0", "minReplicas"], ["spec", "nodeGroups", "md0", "maxReplicas"], ["spec", "nodeGroups", "md0", "instanceType"], ["spec", "nodeGroups", "md0", "ephemeralStorage"], ["spec", "nodeGroups", "md0", "roles"], ["spec", "nodeGroups", "md0", "resources"], ["spec", "nodeGroups", "md0", "gpus"], ["spec", "version"], ["spec", "host"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "fluxcd"], ["spec", "addons", "fluxcd", "enabled"], ["spec", "addons", "fluxcd", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"]] + keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "nodeGroups"], ["spec", "nodeGroups", "md0"], ["spec", "nodeGroups", "md0", "minReplicas"], ["spec", "nodeGroups", "md0", "maxReplicas"], ["spec", "nodeGroups", "md0", "instanceType"], ["spec", "nodeGroups", "md0", "ephemeralStorage"], ["spec", "nodeGroups", "md0", "roles"], ["spec", "nodeGroups", "md0", "resources"], ["spec", "nodeGroups", "md0", "gpus"], ["spec", "version"], ["spec", "host"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "hami"], ["spec", "addons", "hami", "enabled"], ["spec", "addons", "hami", "valuesOverride"], ["spec", "addons", "fluxcd"], ["spec", "addons", "fluxcd", "enabled"], ["spec", "addons", "fluxcd", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"]] secrets: exclude: [] include: From 648ad82dd3283076cf85f6b4b8953ceda8199476 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Fri, 24 Apr 2026 13:31:57 +0300 Subject: [PATCH 061/130] refactor(postgres-operator): scope webhook-ready RBAC to release namespace Address review feedback from gemini-code-assist on packages/system/postgres-operator/templates/webhook-ready-hook.yaml:83. The Job targets the cnpg-webhook-service/services/proxy subresource, which is namespaced and lives in the release namespace. A namespaced Role and RoleBinding grant the exact permission needed without creating global RBAC for a namespaced probe, which is the principle of least privilege. Also update the kind assertions in tests/webhook-ready-hook_test.yaml so the unittest suite tracks the new Role/RoleBinding objects. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- .../postgres-operator/templates/webhook-ready-hook.yaml | 8 +++++--- .../postgres-operator/tests/webhook-ready-hook_test.yaml | 6 +++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/system/postgres-operator/templates/webhook-ready-hook.yaml b/packages/system/postgres-operator/templates/webhook-ready-hook.yaml index 2ad4e7c9..70e51621 100644 --- a/packages/system/postgres-operator/templates/webhook-ready-hook.yaml +++ b/packages/system/postgres-operator/templates/webhook-ready-hook.yaml @@ -52,9 +52,10 @@ metadata: helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded --- apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole +kind: Role metadata: name: {{ .Release.Name }}-webhook-ready + namespace: {{ .Release.Namespace }} annotations: helm.sh/hook: post-install,post-upgrade helm.sh/hook-weight: "0" @@ -66,16 +67,17 @@ rules: verbs: ["get"] --- apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding +kind: RoleBinding metadata: name: {{ .Release.Name }}-webhook-ready + namespace: {{ .Release.Namespace }} annotations: helm.sh/hook: post-install,post-upgrade helm.sh/hook-weight: "0" helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded roleRef: apiGroup: rbac.authorization.k8s.io - kind: ClusterRole + kind: Role name: {{ .Release.Name }}-webhook-ready subjects: - kind: ServiceAccount diff --git a/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml b/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml index 69f5a6b5..eb7863da 100644 --- a/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml +++ b/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml @@ -9,7 +9,7 @@ release: namespace: cozy-postgres-operator tests: - - it: renders four hook objects (SA + ClusterRole + ClusterRoleBinding + Job) + - it: renders four hook objects (SA + Role + RoleBinding + Job) template: templates/webhook-ready-hook.yaml asserts: - hasDocuments: @@ -49,7 +49,7 @@ tests: - documentIndex: 1 equal: path: kind - value: ClusterRole + value: Role - documentIndex: 1 equal: path: metadata.annotations["helm.sh/hook-weight"] @@ -57,7 +57,7 @@ tests: - documentIndex: 2 equal: path: kind - value: ClusterRoleBinding + value: RoleBinding - documentIndex: 2 equal: path: metadata.annotations["helm.sh/hook-weight"] From fc057c039385e7b40cca3f2b6ca77a8f80bab49a Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Fri, 24 Apr 2026 13:32:33 +0300 Subject: [PATCH 062/130] fix(postgres-operator): set resources on webhook-ready wait container Address review feedback from gemini-code-assist on packages/system/postgres-operator/templates/webhook-ready-hook.yaml:115. The wait container had no resource requests or limits, so schedulers treated it as BestEffort and downstream quota enforcement had no signal. Set small requests (10m CPU, 32Mi memory) and conservative limits (100m CPU, 64Mi memory) matching the actual footprint of the kubectl polling loop. Add a matching unittest assertion so the values stay in sync if anyone touches the template. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- .../templates/webhook-ready-hook.yaml | 7 +++++++ .../tests/webhook-ready-hook_test.yaml | 20 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/packages/system/postgres-operator/templates/webhook-ready-hook.yaml b/packages/system/postgres-operator/templates/webhook-ready-hook.yaml index 70e51621..51a8ba81 100644 --- a/packages/system/postgres-operator/templates/webhook-ready-hook.yaml +++ b/packages/system/postgres-operator/templates/webhook-ready-hook.yaml @@ -110,6 +110,13 @@ spec: - name: wait image: {{ $image }} imagePullPolicy: {{ if .Values.webhookReady.image.digest }}IfNotPresent{{ else }}Always{{ end }} + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true diff --git a/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml b/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml index eb7863da..3253c663 100644 --- a/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml +++ b/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml @@ -205,6 +205,26 @@ tests: path: spec.template.spec.securityContext.seccompProfile.type value: RuntimeDefault + - it: wait container declares resource requests and limits for predictable scheduling + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].resources.requests.cpu + value: 10m + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].resources.requests.memory + value: 32Mi + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].resources.limits.cpu + value: 100m + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].resources.limits.memory + value: 64Mi + - it: Job container drops all capabilities and runs read-only rootfs template: templates/webhook-ready-hook.yaml asserts: From 7da29afe6e1f33b69d6040fd1051924d7937a13b Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Fri, 24 Apr 2026 13:33:07 +0300 Subject: [PATCH 063/130] docs(postgres-operator): fix backoffLimit/activeDeadlineSeconds comment math Address review feedback from coderabbitai on packages/system/postgres-operator/values.yaml:30. The old comment's 5-minute ImagePullBackOff scenario conflicted with the 180s activeDeadlineSeconds that the default maxAttempts/sleepSeconds resolve to, so the numbers could not both be taken at face value. Rewrite the comment to state the actual deadline math and frame the two gates as an AND with activeDeadlineSeconds being the shorter one under defaults, so readers understand why backoffLimit has little headroom without an accompanying maxAttempts/sleepSeconds bump. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- packages/system/postgres-operator/values.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/system/postgres-operator/values.yaml b/packages/system/postgres-operator/values.yaml index 854a3ac6..c408220e 100644 --- a/packages/system/postgres-operator/values.yaml +++ b/packages/system/postgres-operator/values.yaml @@ -25,6 +25,8 @@ webhookReady: # Pod-level retry budget for transient node/registry failures (image pull rate limit, # OOM, CNI hiccup). The shell loop inside the container only covers reachability retries. # Pod retries and activeDeadlineSeconds (wall-clock bound on the whole Job across all - # pod retries) are ANDed: a 5-minute ImagePullBackOff on attempt 0 leaves only ~60s for - # subsequent retries before the Job is cut. + # pod retries) are ANDed, so activeDeadlineSeconds is the shorter of the two gates with + # the defaults above: the 60*2+60 = 180s deadline cuts the Job before backoffLimit=2 + # ever matters if pod-level failures eat more than ~60s of the budget. Raise + # maxAttempts/sleepSeconds alongside backoffLimit when tuning for slow image pulls. backoffLimit: 2 From 2734dc0bcb96aaa087cbff6b2ee261e1bde8bffa Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Fri, 24 Apr 2026 14:00:48 +0300 Subject: [PATCH 064/130] fix(hami): clean up leftover hami-dra references and harden defaults Remove broken hami-dra subchart dependency from vendored chart (Chart.yaml, Chart.lock, values.yaml) and strip DRA condition guards from all templates since the subchart was already deleted. Override devicePlugin updateStrategy to OnDelete to prevent destructive rolling updates of GPU workloads. Align gpu-operator template with project pattern (unconditional values emission). Add nodeConfiguration format documentation and test for conflicting valuesOverride scenario. Signed-off-by: Arsolitt --- .../templates/helmreleases/gpu-operator.yaml | 6 ++--- .../tests/gpu_operator_hami_test.yaml | 25 ++++++++++++++++++- packages/system/hami/charts/hami/Chart.lock | 6 ----- packages/system/hami/charts/hami/Chart.yaml | 6 +---- .../templates/device-plugin/configmap.yaml | 2 +- .../device-plugin/daemonsetnvidia.yaml | 2 +- .../templates/device-plugin/monitorrole.yaml | 2 +- .../device-plugin/monitorrolebinding.yaml | 2 +- .../device-plugin/monitorservice.yaml | 2 +- .../device-plugin/monitorserviceaccount.yaml | 2 +- .../device-plugin/runtime-class.yaml | 4 +-- .../hami/templates/scheduler/certmanager.yaml | 2 +- .../hami/templates/scheduler/clusterrole.yaml | 2 -- .../scheduler/clusterrolebinding.yaml | 4 +-- .../hami/templates/scheduler/configmap.yaml | 2 +- .../templates/scheduler/configmapnew.yaml | 2 +- .../hami/templates/scheduler/deployment.yaml | 4 +-- .../templates/scheduler/device-configmap.yaml | 4 +-- .../scheduler/job-patch/clusterrole.yaml | 4 +-- .../job-patch/clusterrolebinding.yaml | 4 +-- .../scheduler/job-patch/job-createSecret.yaml | 4 +-- .../scheduler/job-patch/job-patchWebhook.yaml | 4 +-- .../templates/scheduler/job-patch/psp.yaml | 4 +-- .../templates/scheduler/job-patch/role.yaml | 4 +-- .../scheduler/job-patch/rolebinding.yaml | 4 +-- .../scheduler/job-patch/serviceaccount.yaml | 4 +-- .../charts/hami/templates/scheduler/role.yaml | 4 +-- .../hami/templates/scheduler/rolebinding.yaml | 2 -- .../hami/templates/scheduler/service.yaml | 4 +-- .../templates/scheduler/serviceaccount.yaml | 2 -- .../hami/templates/scheduler/webhook.yaml | 2 +- packages/system/hami/charts/hami/values.yaml | 18 ------------- packages/system/hami/values.yaml | 4 +++ 33 files changed, 55 insertions(+), 92 deletions(-) delete mode 100644 packages/system/hami/charts/hami/Chart.lock diff --git a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml index 995aa430..7a5ecaf1 100644 --- a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml @@ -1,5 +1,5 @@ {{- define "cozystack.defaultGpuOperatorValues" -}} -{{- if and (hasKey .Values.addons "hami") .Values.addons.hami.enabled }} +{{- if .Values.addons.hami.enabled }} gpu-operator: devicePlugin: enabled: false @@ -40,10 +40,8 @@ spec: {{- $defaults := fromYaml (include "cozystack.defaultGpuOperatorValues" .) }} {{- $overrides := deepCopy (default (dict) .Values.addons.gpuOperator.valuesOverride) }} {{- $merged := mergeOverwrite (default (dict) $defaults) $overrides }} - {{- with $merged }} values: - {{- toYaml . | nindent 4 }} - {{- end }} + {{- if $merged }}{{ toYaml $merged | nindent 4 }}{{ end }} dependsOn: {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }} diff --git a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml index 33256ae6..11de743a 100644 --- a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml +++ b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml @@ -26,8 +26,9 @@ tests: enabled: false valuesOverride: {} asserts: - - notExists: + - equal: path: spec.values + value: null - it: should allow user overrides to merge with hami defaults set: @@ -66,6 +67,28 @@ tests: path: spec.values.gpu-operator.devicePlugin.enabled value: true + - it: should let user explicitly override devicePlugin.enabled to true with hami enabled + set: + addons: + gpuOperator: + enabled: true + valuesOverride: + gpu-operator: + devicePlugin: + enabled: true + driver: + enabled: false + hami: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.values.gpu-operator.devicePlugin.enabled + value: true + - equal: + path: spec.values.gpu-operator.driver.enabled + value: false + - it: should not render when gpuOperator is disabled set: addons: diff --git a/packages/system/hami/charts/hami/Chart.lock b/packages/system/hami/charts/hami/Chart.lock deleted file mode 100644 index 25856c07..00000000 --- a/packages/system/hami/charts/hami/Chart.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: hami-dra - repository: https://project-hami.github.io/HAMi-DRA/ - version: 0.1.0 -digest: sha256:374551539570bcee82c1c4dea93eac59e6f410b6d5967d188efd630e203d43bb -generated: "2026-04-17T04:06:47.689117678Z" diff --git a/packages/system/hami/charts/hami/Chart.yaml b/packages/system/hami/charts/hami/Chart.yaml index aba7e95c..55f32ab6 100644 --- a/packages/system/hami/charts/hami/Chart.yaml +++ b/packages/system/hami/charts/hami/Chart.yaml @@ -1,10 +1,6 @@ apiVersion: v2 appVersion: 2.8.1 -dependencies: -- condition: dra.enabled - name: hami-dra - repository: https://project-hami.github.io/HAMi-DRA/ - version: 0.1.0 +dependencies: [] description: Heterogeneous AI Computing Virtualization Middleware keywords: - vgpu diff --git a/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml b/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml index b36347f2..db2ddacb 100644 --- a/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml +++ b/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.devicePlugin.enabled (not .Values.devicePlugin.nodeConfiguration.externalConfigName) (not .Values.dra.enabled) -}} +{{- if and .Values.devicePlugin.enabled (not .Values.devicePlugin.nodeConfiguration.externalConfigName) -}} apiVersion: v1 kind: ConfigMap metadata: diff --git a/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml index 577dc0f9..1f4c24f3 100644 --- a/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml +++ b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) }} +{{- if .Values.devicePlugin.enabled }} apiVersion: apps/v1 kind: DaemonSet metadata: diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml index c51ff6c6..377f6a86 100644 --- a/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}} +{{- if .Values.devicePlugin.enabled -}} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml index 93a118dd..2f0a14ba 100644 --- a/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}} +{{- if .Values.devicePlugin.enabled -}} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml index 24daca9b..0fb156cc 100644 --- a/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}} +{{- if .Values.devicePlugin.enabled -}} apiVersion: v1 kind: Service metadata: diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml index b10d2cb9..6e3c2a44 100644 --- a/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}} +{{- if .Values.devicePlugin.enabled -}} apiVersion: v1 kind: ServiceAccount metadata: diff --git a/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml b/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml index cd3d14cc..ebcc2c9c 100644 --- a/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml +++ b/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml @@ -1,5 +1,4 @@ -{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}} -{{- if and .Values.devicePlugin.createRuntimeClass .Values.devicePlugin.runtimeClassName }} +{{- if and .Values.devicePlugin.enabled .Values.devicePlugin.createRuntimeClass .Values.devicePlugin.runtimeClassName -}} apiVersion: node.k8s.io/v1 kind: RuntimeClass metadata: @@ -7,5 +6,4 @@ metadata: annotations: helm.sh/hook: pre-install,pre-upgrade handler: nvidia -{{- end }} {{- end -}} diff --git a/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml b/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml index b6edb605..e6d28721 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} +{{- if .Values.scheduler.admissionWebhook.enabled -}} {{- if .Values.scheduler.certManager.enabled }} apiVersion: cert-manager.io/v1 kind: Certificate diff --git a/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml b/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml index 9c510c00..81c4fddf 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml @@ -1,4 +1,3 @@ -{{- if not .Values.dra.enabled -}} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -33,4 +32,3 @@ rules: resources: ["nodes"] verbs: ["get", "update", "list", "patch"] {{- end -}} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml index 69d75986..aa0f3c66 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml @@ -1,4 +1,3 @@ -{{- if not .Values.dra.enabled -}} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -45,5 +44,4 @@ roleRef: subjects: - kind: ServiceAccount name: {{ include "hami-vgpu.scheduler" . }} - namespace: {{ include "hami-vgpu.namespace" . }} -{{- end }} \ No newline at end of file + namespace: {{ include "hami-vgpu.namespace" . }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml b/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml index 75889146..109ecbce 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.scheduler.kubeScheduler.enabled (not .Values.dra.enabled) -}} +{{- if .Values.scheduler.kubeScheduler.enabled -}} apiVersion: v1 kind: ConfigMap metadata: diff --git a/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml b/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml index e2a91d8b..6f6db097 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.scheduler.kubeScheduler.enabled (not .Values.dra.enabled) -}} +{{- if .Values.scheduler.kubeScheduler.enabled -}} apiVersion: v1 kind: ConfigMap metadata: diff --git a/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml b/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml index 6364cae3..12911a42 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml @@ -1,4 +1,3 @@ -{{- if not .Values.dra.enabled -}} apiVersion: apps/v1 kind: Deployment metadata: @@ -223,5 +222,4 @@ spec: {{- end }} {{- if .Values.scheduler.nodeName }} nodeName: {{ .Values.scheduler.nodeName }} - {{- end }} -{{- end }} \ No newline at end of file + {{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml b/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml index ccb945ac..8738a06f 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml @@ -1,4 +1,3 @@ -{{- if not .Values.dra.enabled -}} apiVersion: v1 kind: ConfigMap metadata: @@ -406,5 +405,4 @@ data: memory: 12288 aiCore: 4 aiCPU: 4 - {{ end }} -{{- end }} \ No newline at end of file + {{ end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml index b089ae82..412b1094 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml @@ -1,5 +1,4 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} -{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -27,4 +26,3 @@ rules: - {{ include "hami-vgpu.fullname" . }}-admission {{- end }} {{- end }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml index 64f01ecf..2b82f926 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml @@ -1,5 +1,4 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} -{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -19,4 +18,3 @@ subjects: name: {{ include "hami-vgpu.fullname" . }}-admission namespace: {{ include "hami-vgpu.namespace" . }} {{- end }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml index 400168a7..0e36d95f 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml @@ -1,5 +1,4 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} -{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} apiVersion: batch/v1 kind: Job metadata: @@ -67,4 +66,3 @@ spec: runAsNonRoot: true runAsUser: {{ .Values.scheduler.patch.runAsUser }} {{- end }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml index 71e97f6b..ce52042c 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml @@ -1,5 +1,4 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} -{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} apiVersion: batch/v1 kind: Job metadata: @@ -62,4 +61,3 @@ spec: runAsNonRoot: true runAsUser: {{ .Values.scheduler.patch.runAsUser }} {{- end }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml index a48270e7..1e9cd7da 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml @@ -1,5 +1,4 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} -{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} {{- if .Values.podSecurityPolicy.enabled }} apiVersion: policy/v1beta1 kind: PodSecurityPolicy @@ -37,4 +36,3 @@ spec: - downwardAPI {{- end }} {{- end }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml index 74e45681..56682f8e 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml @@ -1,5 +1,4 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} -{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: @@ -20,4 +19,3 @@ rules: - get - create {{- end }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml index 32da5ba4..7239b128 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml @@ -1,5 +1,4 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} -{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: @@ -20,4 +19,3 @@ subjects: name: {{ include "hami-vgpu.fullname" . }}-admission namespace: {{ include "hami-vgpu.namespace" . }} {{- end }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml index d2eb41de..857c6a8d 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml @@ -1,5 +1,4 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) }} -{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} apiVersion: v1 kind: ServiceAccount metadata: @@ -12,4 +11,3 @@ metadata: {{- include "hami-vgpu.labels" . | nindent 4 }} app.kubernetes.io/component: admission-webhook {{- end }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/role.yaml b/packages/system/hami/charts/hami/templates/scheduler/role.yaml index 6e681967..90987a4f 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/role.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/role.yaml @@ -1,4 +1,3 @@ -{{- if not .Values.dra.enabled -}} apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: @@ -10,5 +9,4 @@ metadata: rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] - verbs: ["create", "list", "watch", "get", "update", "patch"] -{{- end }} \ No newline at end of file + verbs: ["create", "list", "watch", "get", "update", "patch"] \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml index 934f8223..96e175a1 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml @@ -1,4 +1,3 @@ -{{- if not .Values.dra.enabled -}} apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: @@ -30,4 +29,3 @@ subjects: name: {{ include "hami-vgpu.mock-device-plugin" . }} namespace: {{ include "hami-vgpu.namespace" . }} {{- end -}} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/service.yaml b/packages/system/hami/charts/hami/templates/scheduler/service.yaml index 3e18bc98..70378c5c 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/service.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/service.yaml @@ -1,4 +1,3 @@ -{{- if not .Values.dra.enabled }} apiVersion: v1 kind: Service metadata: @@ -32,5 +31,4 @@ spec: protocol: TCP selector: app.kubernetes.io/component: hami-scheduler - {{- include "hami-vgpu.selectorLabels" . | nindent 4 }} -{{- end }} \ No newline at end of file + {{- include "hami-vgpu.selectorLabels" . | nindent 4 }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml b/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml index 929bf59a..0435c003 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml @@ -1,4 +1,3 @@ -{{- if not .Values.dra.enabled -}} apiVersion: v1 kind: ServiceAccount metadata: @@ -15,4 +14,3 @@ metadata: name: {{ include "hami-vgpu.mock-device-plugin" . }} namespace: {{ include "hami-vgpu.namespace" . }} {{- end -}} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml b/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml index 148feb5c..7296eb4e 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} +{{- if .Values.scheduler.admissionWebhook.enabled -}} apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: diff --git a/packages/system/hami/charts/hami/values.yaml b/packages/system/hami/charts/hami/values.yaml index 9fbeca70..02feeada 100644 --- a/packages/system/hami/charts/hami/values.yaml +++ b/packages/system/hami/charts/hami/values.yaml @@ -392,24 +392,6 @@ mockDevicePlugin: ## - myRegistryKeySecretName ## pullSecrets: [] -# If this option is enabled, the DRA will be installed and the scheduler extender will not be installed. -dra: - enabled: false - -# Configuration for the hami-dra subchart, which includes DRA drivers. -# This is part of config for DRA, please refer to the DRA documentation for more details. -hami-dra: - monitor: - enabled: true - drivers: - nvidia: - enabled: true - # If you are using gpu driver on host, you need to set this to false - containerDriver: true - image: - repository: projecthami/k8s-dra-driver - tag: "v0.0.1-dev" - devices: amd: customresources: diff --git a/packages/system/hami/values.yaml b/packages/system/hami/values.yaml index 1155e2cb..1d1407b7 100644 --- a/packages/system/hami/values.yaml +++ b/packages/system/hami/values.yaml @@ -1,6 +1,10 @@ hami: devicePlugin: runtimeClassName: nvidia + updateStrategy: + type: OnDelete + # See upstream charts/hami/values.yaml for nodeConfiguration format. + # Each entry: {"name": "node-name", "operatingmode": "hami-core", "devicesplitcount": N, ...} nodeConfiguration: config: | { From 37d5ff0c6f0566ca18495e871c5ad626f7976ca1 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Fri, 24 Apr 2026 14:07:17 +0300 Subject: [PATCH 065/130] fix(hami): align templates with project patterns and clean dead code Unconditionally emit values in hami.yaml matching the project pattern. Remove duplicate test case and add coverage for omitted valuesOverride key. Delete dead PSP template and RBAC rules (policy/v1beta1 removed in K8s 1.25). Override kube-scheduler image registry to registry.k8s.io to avoid Chinese registry for international users. Signed-off-by: Arsolitt --- .../templates/helmreleases/hami.yaml | 4 +- .../tests/gpu_operator_hami_test.yaml | 29 ++++++-------- .../scheduler/job-patch/clusterrole.yaml | 7 ---- .../templates/scheduler/job-patch/psp.yaml | 38 ------------------- packages/system/hami/charts/hami/values.yaml | 3 -- packages/system/hami/values.yaml | 5 +++ 6 files changed, 18 insertions(+), 68 deletions(-) delete mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml diff --git a/packages/apps/kubernetes/templates/helmreleases/hami.yaml b/packages/apps/kubernetes/templates/helmreleases/hami.yaml index 1f7a5358..ac4714da 100644 --- a/packages/apps/kubernetes/templates/helmreleases/hami.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/hami.yaml @@ -32,10 +32,8 @@ spec: force: true remediation: retries: -1 - {{- with .Values.addons.hami.valuesOverride }} values: - {{- toYaml . | nindent 4 }} - {{- end }} + {{- if .Values.addons.hami.valuesOverride }}{{ toYaml .Values.addons.hami.valuesOverride | nindent 4 }}{{ end }} dependsOn: {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }} diff --git a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml index 11de743a..c3ec30f2 100644 --- a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml +++ b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml @@ -30,6 +30,18 @@ tests: path: spec.values value: null + - it: should apply hami defaults when valuesOverride key is omitted + set: + addons: + gpuOperator: + enabled: true + hami: + enabled: true + asserts: + - equal: + path: spec.values.gpu-operator.devicePlugin.enabled + value: false + - it: should allow user overrides to merge with hami defaults set: addons: @@ -50,23 +62,6 @@ tests: path: spec.values.gpu-operator.driver.enabled value: false - - it: should let user override devicePlugin back to true if needed - set: - addons: - gpuOperator: - enabled: true - valuesOverride: - gpu-operator: - devicePlugin: - enabled: true - hami: - enabled: true - valuesOverride: {} - asserts: - - equal: - path: spec.values.gpu-operator.devicePlugin.enabled - value: true - - it: should let user explicitly override devicePlugin.enabled to true with hami enabled set: addons: diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml index 412b1094..77e891cc 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml @@ -18,11 +18,4 @@ rules: verbs: - get - update -{{- if .Values.podSecurityPolicy.enabled }} - - apiGroups: ['extensions'] - resources: ['podsecuritypolicies'] - verbs: ['use'] - resourceNames: - - {{ include "hami-vgpu.fullname" . }}-admission -{{- end }} {{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml deleted file mode 100644 index 1e9cd7da..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml +++ /dev/null @@ -1,38 +0,0 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} -{{- if .Values.podSecurityPolicy.enabled }} -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - name: {{ include "hami-vgpu.fullname" . }}-admission - annotations: - "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded - labels: - {{- include "hami-vgpu.labels" . | nindent 4 }} - app.kubernetes.io/component: admission-webhook -spec: - allowPrivilegeEscalation: false - fsGroup: - ranges: - - max: 65535 - min: 1 - rule: MustRunAs - requiredDropCapabilities: - - ALL - runAsUser: - rule: MustRunAsNonRoot - seLinux: - rule: RunAsAny - supplementalGroups: - ranges: - - max: 65535 - min: 1 - rule: MustRunAs - volumes: - - configMap - - emptyDir - - projected - - secret - - downwardAPI -{{- end }} -{{- end }} diff --git a/packages/system/hami/charts/hami/values.yaml b/packages/system/hami/charts/hami/values.yaml index 02feeada..01caca0b 100644 --- a/packages/system/hami/charts/hami/values.yaml +++ b/packages/system/hami/charts/hami/values.yaml @@ -55,9 +55,6 @@ kunlunResourceVMemoryName: "kunlunxin.com/vxpu-memory" schedulerName: "hami-scheduler" -podSecurityPolicy: - enabled: false - scheduler: # @param nodeName defines the node name and the nvidia-vgpu-scheduler-scheduler will schedule to the node. # if we install the nvidia-vgpu-scheduler-scheduler as default scheduler, we need to remove the k8s default diff --git a/packages/system/hami/values.yaml b/packages/system/hami/values.yaml index 1d1407b7..f45d2998 100644 --- a/packages/system/hami/values.yaml +++ b/packages/system/hami/values.yaml @@ -1,4 +1,9 @@ hami: + scheduler: + kubeScheduler: + image: + registry: registry.k8s.io + repository: kube-scheduler devicePlugin: runtimeClassName: nvidia updateStrategy: From 6a9c310a4bfdb8bab3cd4f52724205adfbd2219e Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Fri, 24 Apr 2026 15:31:24 +0300 Subject: [PATCH 066/130] fix(hami): conditional values emission and cosmetic template cleanup Only emit values key in hami.yaml when valuesOverride has content, matching gpu-operator pattern. Add test verifying empty valuesOverride does not produce spec.values. Fix trailing whitespace and missing newlines in vendored chart templates. Signed-off-by: Arsolitt --- .../kubernetes/templates/helmreleases/hami.yaml | 4 +++- packages/apps/kubernetes/tests/hami_test.yaml | 15 +++++++++++++++ .../hami/charts/hami/templates/_commons.tpl | 2 +- .../hami/templates/device-plugin/configmap.yaml | 2 +- .../hami/templates/device-plugin/monitorrole.yaml | 3 +-- .../templates/device-plugin/monitorservice.yaml | 2 +- .../templates/scheduler/clusterrolebinding.yaml | 2 +- .../hami/templates/scheduler/deployment.yaml | 2 +- .../templates/scheduler/device-configmap.yaml | 2 +- .../charts/hami/templates/scheduler/role.yaml | 2 +- .../charts/hami/templates/scheduler/service.yaml | 2 +- .../charts/hami/templates/scheduler/webhook.yaml | 2 +- 12 files changed, 28 insertions(+), 12 deletions(-) diff --git a/packages/apps/kubernetes/templates/helmreleases/hami.yaml b/packages/apps/kubernetes/templates/helmreleases/hami.yaml index ac4714da..8733e675 100644 --- a/packages/apps/kubernetes/templates/helmreleases/hami.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/hami.yaml @@ -32,8 +32,10 @@ spec: force: true remediation: retries: -1 + {{- if .Values.addons.hami.valuesOverride }} values: - {{- if .Values.addons.hami.valuesOverride }}{{ toYaml .Values.addons.hami.valuesOverride | nindent 4 }}{{ end }} + {{- toYaml .Values.addons.hami.valuesOverride | nindent 4 }} + {{- end }} dependsOn: {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }} diff --git a/packages/apps/kubernetes/tests/hami_test.yaml b/packages/apps/kubernetes/tests/hami_test.yaml index e9d32236..bdbe990f 100644 --- a/packages/apps/kubernetes/tests/hami_test.yaml +++ b/packages/apps/kubernetes/tests/hami_test.yaml @@ -118,6 +118,21 @@ tests: name: test-gpu-operator namespace: test-ns + - it: should not render spec.values when valuesOverride is empty + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - hasDocuments: + count: 1 + - notExists: + path: spec.values + - it: should pass through valuesOverride set: addons: diff --git a/packages/system/hami/charts/hami/templates/_commons.tpl b/packages/system/hami/charts/hami/templates/_commons.tpl index 78a262c9..b68018e4 100644 --- a/packages/system/hami/charts/hami/templates/_commons.tpl +++ b/packages/system/hami/charts/hami/templates/_commons.tpl @@ -46,4 +46,4 @@ imagePullSecrets: - name: {{ . }} {{- end }} {{- end }} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml b/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml index db2ddacb..74631e24 100644 --- a/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml +++ b/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml @@ -10,4 +10,4 @@ metadata: data: config.json: | {{- .Values.devicePlugin.nodeConfiguration.config | nindent 4 }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml index 377f6a86..6ac757f2 100644 --- a/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml @@ -25,5 +25,4 @@ rules: - list - patch {{- end -}} - - + diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml index 0fb156cc..104664ea 100644 --- a/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml @@ -26,4 +26,4 @@ spec: selector: app.kubernetes.io/component: hami-device-plugin {{- include "hami-vgpu.selectorLabels" . | nindent 4 }} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml index aa0f3c66..a81d425c 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml @@ -44,4 +44,4 @@ roleRef: subjects: - kind: ServiceAccount name: {{ include "hami-vgpu.scheduler" . }} - namespace: {{ include "hami-vgpu.namespace" . }} \ No newline at end of file + namespace: {{ include "hami-vgpu.namespace" . }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml b/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml index 12911a42..1d89e189 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml @@ -222,4 +222,4 @@ spec: {{- end }} {{- if .Values.scheduler.nodeName }} nodeName: {{ .Values.scheduler.nodeName }} - {{- end }} \ No newline at end of file + {{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml b/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml index 8738a06f..873b813b 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml @@ -405,4 +405,4 @@ data: memory: 12288 aiCore: 4 aiCPU: 4 - {{ end }} \ No newline at end of file + {{ end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/role.yaml b/packages/system/hami/charts/hami/templates/scheduler/role.yaml index 90987a4f..5f7d7e44 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/role.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/role.yaml @@ -9,4 +9,4 @@ metadata: rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] - verbs: ["create", "list", "watch", "get", "update", "patch"] \ No newline at end of file + verbs: ["create", "list", "watch", "get", "update", "patch"] diff --git a/packages/system/hami/charts/hami/templates/scheduler/service.yaml b/packages/system/hami/charts/hami/templates/scheduler/service.yaml index 70378c5c..d7538fed 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/service.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/service.yaml @@ -31,4 +31,4 @@ spec: protocol: TCP selector: app.kubernetes.io/component: hami-scheduler - {{- include "hami-vgpu.selectorLabels" . | nindent 4 }} \ No newline at end of file + {{- include "hami-vgpu.selectorLabels" . | nindent 4 }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml b/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml index 7296eb4e..db9f8029 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml @@ -54,4 +54,4 @@ webhooks: scope: '*' sideEffects: None timeoutSeconds: 10 -{{- end }} \ No newline at end of file +{{- end }} From 1131e2f113a8fda8402bf9c92014bff9aaf29f25 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Fri, 24 Apr 2026 15:33:07 +0300 Subject: [PATCH 067/130] docs(hami): reference upstream repo for nodeConfiguration format Signed-off-by: Arsolitt --- packages/system/hami/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/system/hami/values.yaml b/packages/system/hami/values.yaml index f45d2998..64b372a6 100644 --- a/packages/system/hami/values.yaml +++ b/packages/system/hami/values.yaml @@ -8,7 +8,7 @@ hami: runtimeClassName: nvidia updateStrategy: type: OnDelete - # See upstream charts/hami/values.yaml for nodeConfiguration format. + # See https://github.com/Project-HAMi/HAMi/blob/v2.8.1/deployments/charts/hami/values.yaml # Each entry: {"name": "node-name", "operatingmode": "hami-core", "devicesplitcount": N, ...} nodeConfiguration: config: | From 3eeda2ba3596851393fd62d3d352cdd1017be267 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Fri, 24 Apr 2026 16:12:34 +0300 Subject: [PATCH 068/130] chore(kubernetes): regenerate code after hami addon addition Signed-off-by: Arsolitt --- .../kubernetes/zz_generated.deepcopy.go | 17 +++++++++++++++++ api/backups/v1alpha1/zz_generated.deepcopy.go | 5 +++++ 2 files changed, 22 insertions(+) diff --git a/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go b/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go index 2cbe3ba1..3f1cb5ce 100644 --- a/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go +++ b/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go @@ -49,6 +49,7 @@ func (in *Addons) DeepCopyInto(out *Addons) { in.Fluxcd.DeepCopyInto(&out.Fluxcd) out.GatewayAPI = in.GatewayAPI in.GpuOperator.DeepCopyInto(&out.GpuOperator) + in.Hami.DeepCopyInto(&out.Hami) in.IngressNginx.DeepCopyInto(&out.IngressNginx) in.MonitoringAgents.DeepCopyInto(&out.MonitoringAgents) in.Velero.DeepCopyInto(&out.Velero) @@ -260,6 +261,22 @@ func (in *GatewayAPIAddon) DeepCopy() *GatewayAPIAddon { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HAMiAddon) DeepCopyInto(out *HAMiAddon) { + *out = *in + in.ValuesOverride.DeepCopyInto(&out.ValuesOverride) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HAMiAddon. +func (in *HAMiAddon) DeepCopy() *HAMiAddon { + if in == nil { + return nil + } + out := new(HAMiAddon) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IngressNginxAddon) DeepCopyInto(out *IngressNginxAddon) { *out = *in diff --git a/api/backups/v1alpha1/zz_generated.deepcopy.go b/api/backups/v1alpha1/zz_generated.deepcopy.go index 89f6171f..61b8f839 100644 --- a/api/backups/v1alpha1/zz_generated.deepcopy.go +++ b/api/backups/v1alpha1/zz_generated.deepcopy.go @@ -620,6 +620,11 @@ func (in *RestoreJobSpec) DeepCopyInto(out *RestoreJobSpec) { *out = new(v1.TypedLocalObjectReference) (*in).DeepCopyInto(*out) } + if in.Options != nil { + in, out := &in.Options, &out.Options + *out = new(runtime.RawExtension) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestoreJobSpec. From 500816b71b1ad3f916e11ff3ddb1fd6dee9fcc76 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Fri, 24 Apr 2026 16:44:46 +0300 Subject: [PATCH 069/130] refactor(ingress): move CiliumLoadBalancerIPPool to tenant chart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pool is now rendered from packages/apps/tenant/templates/cilium-lb-pool.yaml instead of packages/extra/ingress/templates/cilium-lb-pool.yaml. Cilium LB IPAM forbids overlapping CIDRs across pools regardless of serviceSelector, so both the ingress-loadBalancer path and the upcoming per-tenant Gateway in #2470 cannot each own their own pool on the same publishing.externalIPs range. The tenant chart is the natural per-tenant owner — it already creates the Namespace, the cozystack-values Secret, and the HelmReleases for both ingress and gateway. The new pool uses a namespace-only serviceSelector (io.kubernetes.service.namespace: ), which matches any LoadBalancer Service in the tenant namespace. The metadata.name changed from -ingress to -exposure to reflect that the pool is not ingress-specific. Only the ingress-loadBalancer signal is wired in this commit (_cluster.expose-mode=loadBalancer plus .Values.ingress=true on the publishing tenant). The gateway branch is added in #2470 on top of this commit — it rebases, drops its own packages/extra/gateway/templates/cilium-lb-pool.yaml, and adds an OR branch for .Values.gateway in the tenant template. Pool-rendering unit tests moved from packages/extra/ingress/tests/ to packages/apps/tenant/tests/. The ingress chart tests keep the Service-level asserts. packages/apps/tenant/Makefile gains a test target so hack/helm-unit-tests.sh picks up the new suite. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- packages/apps/tenant/Makefile | 3 + .../tenant}/templates/cilium-lb-pool.yaml | 21 +-- packages/apps/tenant/tests/exposure_test.yaml | 141 ++++++++++++++++++ .../extra/ingress/tests/exposure_test.yaml | 104 +------------ 4 files changed, 158 insertions(+), 111 deletions(-) rename packages/{extra/ingress => apps/tenant}/templates/cilium-lb-pool.yaml (54%) create mode 100644 packages/apps/tenant/tests/exposure_test.yaml diff --git a/packages/apps/tenant/Makefile b/packages/apps/tenant/Makefile index 3fe3810d..2f51d836 100644 --- a/packages/apps/tenant/Makefile +++ b/packages/apps/tenant/Makefile @@ -3,3 +3,6 @@ include ../../../hack/package.mk generate: cozyvalues-gen -m 'tenant' -v values.yaml -s values.schema.json -r README.md -g ../../../api/apps/v1alpha1/tenant/types.go ../../../hack/update-crd.sh + +test: + helm unittest . diff --git a/packages/extra/ingress/templates/cilium-lb-pool.yaml b/packages/apps/tenant/templates/cilium-lb-pool.yaml similarity index 54% rename from packages/extra/ingress/templates/cilium-lb-pool.yaml rename to packages/apps/tenant/templates/cilium-lb-pool.yaml index 048eb58b..63dbcaca 100644 --- a/packages/extra/ingress/templates/cilium-lb-pool.yaml +++ b/packages/apps/tenant/templates/cilium-lb-pool.yaml @@ -1,25 +1,28 @@ -{{- $exposeIngress := (index .Values._cluster "expose-ingress") | default "tenant-root" }} {{- $exposeMode := (index .Values._cluster "expose-mode") | default "externalIPs" }} -{{- $exposeExternalIPs := (index .Values._cluster "expose-external-ips") | default "" | nospace }} -{{- $exposeIPsList := list }} -{{- range splitList "," $exposeExternalIPs }} +{{- $exposeIngress := (index .Values._cluster "expose-ingress") | default "tenant-root" }} +{{- $exposeIPs := (index .Values._cluster "expose-external-ips") | default "" | nospace }} +{{- $ipsList := list }} +{{- range splitList "," $exposeIPs }} {{- $ip := . | trim }} {{- if $ip }} - {{- $exposeIPsList = append $exposeIPsList $ip }} + {{- $ipsList = append $ipsList $ip }} {{- end }} {{- end }} -{{- if and (eq $exposeMode "loadBalancer") (eq $exposeIngress .Release.Namespace) $exposeIPsList }} +{{- $isPublishingIngressLB := and + (eq $exposeMode "loadBalancer") + (eq $exposeIngress .Release.Namespace) + .Values.ingress }} +{{- if and $isPublishingIngressLB $ipsList }} apiVersion: cilium.io/v2 kind: CiliumLoadBalancerIPPool metadata: - name: {{ trimPrefix "tenant-" .Release.Namespace }}-ingress + name: {{ trimPrefix "tenant-" .Release.Namespace }}-exposure spec: blocks: - {{- range $exposeIPsList }} + {{- range $ipsList }} - cidr: {{ . }}{{ if not (contains "/" .) }}/{{ if contains ":" . }}128{{ else }}32{{ end }}{{ end }} {{- end }} serviceSelector: matchLabels: "io.kubernetes.service.namespace": {{ .Release.Namespace | quote }} - "app.kubernetes.io/name": ingress-nginx {{- end }} diff --git a/packages/apps/tenant/tests/exposure_test.yaml b/packages/apps/tenant/tests/exposure_test.yaml new file mode 100644 index 00000000..e86fab5b --- /dev/null +++ b/packages/apps/tenant/tests/exposure_test.yaml @@ -0,0 +1,141 @@ +suite: tenant CiliumLoadBalancerIPPool rendering for publishing.exposure=loadBalancer +templates: + - templates/cilium-lb-pool.yaml + +release: + name: tenant-root + namespace: tenant-root + +tests: + - it: default exposure (externalIPs) renders no pool + set: + ingress: true + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10,192.0.2.11" + asserts: + - hasDocuments: + count: 0 + + - it: loadBalancer mode in publishing tenant with ingress=true renders v2 pool with namespace-only selector + set: + ingress: true + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10,192.0.2.11" + expose-mode: loadBalancer + asserts: + - hasDocuments: + count: 1 + - equal: + path: apiVersion + value: cilium.io/v2 + - equal: + path: kind + value: CiliumLoadBalancerIPPool + - equal: + path: metadata.name + value: root-exposure + - equal: + path: spec.blocks + value: + - cidr: 192.0.2.10/32 + - cidr: 192.0.2.11/32 + - equal: + path: spec.serviceSelector.matchLabels["io.kubernetes.service.namespace"] + value: tenant-root + - notExists: + path: spec.serviceSelector.matchLabels["app.kubernetes.io/name"] + + - it: loadBalancer mode with IPv6 emits /128 CIDR + set: + ingress: true + _cluster: + expose-ingress: tenant-root + expose-external-ips: "2001:db8::1" + expose-mode: loadBalancer + asserts: + - equal: + path: spec.blocks + value: + - cidr: 2001:db8::1/128 + + - it: loadBalancer mode with mixed IPv4 and IPv6 emits correct CIDR per family + set: + ingress: true + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10,2001:db8::1" + expose-mode: loadBalancer + asserts: + - equal: + path: spec.blocks + value: + - cidr: 192.0.2.10/32 + - cidr: 2001:db8::1/128 + + - it: loadBalancer mode accepts pre-CIDR input without double-suffixing + set: + ingress: true + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10/32,2001:db8::1/128" + expose-mode: loadBalancer + asserts: + - equal: + path: spec.blocks + value: + - cidr: 192.0.2.10/32 + - cidr: 2001:db8::1/128 + + - it: loadBalancer mode filters out empty entries from externalIPs (trailing, leading, repeated commas) + set: + ingress: true + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10,,192.0.2.11," + expose-mode: loadBalancer + asserts: + - hasDocuments: + count: 1 + - equal: + path: spec.blocks + value: + - cidr: 192.0.2.10/32 + - cidr: 192.0.2.11/32 + + - it: loadBalancer mode with ingress=false in publishing tenant renders no pool + set: + ingress: false + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10" + expose-mode: loadBalancer + asserts: + - hasDocuments: + count: 0 + + - it: loadBalancer mode in a non-publishing tenant renders no pool + set: + ingress: true + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10" + expose-mode: loadBalancer + release: + name: tenant-u1 + namespace: tenant-u1 + asserts: + - hasDocuments: + count: 0 + + - it: loadBalancer mode in publishing tenant with empty externalIPs renders no pool + set: + ingress: true + _cluster: + expose-ingress: tenant-root + expose-external-ips: "" + expose-mode: loadBalancer + asserts: + - hasDocuments: + count: 0 diff --git a/packages/extra/ingress/tests/exposure_test.yaml b/packages/extra/ingress/tests/exposure_test.yaml index 323c8e68..1c76b3da 100644 --- a/packages/extra/ingress/tests/exposure_test.yaml +++ b/packages/extra/ingress/tests/exposure_test.yaml @@ -1,14 +1,13 @@ suite: ingress exposure modes templates: - templates/nginx-ingress.yaml - - templates/cilium-lb-pool.yaml release: name: ingress namespace: tenant-root tests: - - it: default exposure (externalIPs) renders ClusterIP Service with spec.externalIPs and no CiliumLoadBalancerIPPool + - it: default exposure (externalIPs) renders ClusterIP Service with spec.externalIPs set: _cluster: expose-ingress: tenant-root @@ -31,9 +30,6 @@ tests: - template: templates/nginx-ingress.yaml notExists: path: spec.values.ingress-nginx.controller.service.labels - - template: templates/cilium-lb-pool.yaml - hasDocuments: - count: 0 - it: legacy config without expose-mode falls back to externalIPs behavior set: @@ -45,9 +41,6 @@ tests: equal: path: spec.values.ingress-nginx.controller.service.type value: ClusterIP - - template: templates/cilium-lb-pool.yaml - hasDocuments: - count: 0 - it: externalIPs mode in a namespace other than publishing.ingressName renders LoadBalancer fallback without externalIPs set: @@ -68,11 +61,8 @@ tests: - template: templates/nginx-ingress.yaml notExists: path: spec.values.ingress-nginx.controller.service.externalIPs - - template: templates/cilium-lb-pool.yaml - hasDocuments: - count: 0 - - it: loadBalancer mode renders LoadBalancer Service with lb-pool label and a v2 CiliumLoadBalancerIPPool + - it: loadBalancer mode renders LoadBalancer Service without externalIPs on the Service set: _cluster: expose-ingress: tenant-root @@ -93,62 +83,6 @@ tests: - template: templates/nginx-ingress.yaml notExists: path: spec.values.ingress-nginx.controller.service.externalIPs - - template: templates/cilium-lb-pool.yaml - hasDocuments: - count: 1 - - template: templates/cilium-lb-pool.yaml - equal: - path: apiVersion - value: cilium.io/v2 - - template: templates/cilium-lb-pool.yaml - equal: - path: kind - value: CiliumLoadBalancerIPPool - - template: templates/cilium-lb-pool.yaml - equal: - path: metadata.name - value: root-ingress - - template: templates/cilium-lb-pool.yaml - equal: - path: spec.blocks - value: - - cidr: 192.0.2.10/32 - - cidr: 192.0.2.11/32 - - template: templates/cilium-lb-pool.yaml - equal: - path: spec.serviceSelector.matchLabels["io.kubernetes.service.namespace"] - value: tenant-root - - template: templates/cilium-lb-pool.yaml - equal: - path: spec.serviceSelector.matchLabels["app.kubernetes.io/name"] - value: ingress-nginx - - - it: loadBalancer mode with IPv6 address emits /128 CIDR - set: - _cluster: - expose-ingress: tenant-root - expose-external-ips: "2001:db8::1" - expose-mode: loadBalancer - asserts: - - template: templates/cilium-lb-pool.yaml - equal: - path: spec.blocks - value: - - cidr: 2001:db8::1/128 - - - it: loadBalancer mode with mixed IPv4 and IPv6 emits correct CIDR per family - set: - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10,2001:db8::1" - expose-mode: loadBalancer - asserts: - - template: templates/cilium-lb-pool.yaml - equal: - path: spec.blocks - value: - - cidr: 192.0.2.10/32 - - cidr: 2001:db8::1/128 - it: loadBalancer mode without externalIPs fails chart render with explicit message set: @@ -206,26 +140,6 @@ tests: - template: templates/nginx-ingress.yaml notExists: path: spec.values.ingress-nginx.controller.service.externalIPs - - template: templates/cilium-lb-pool.yaml - hasDocuments: - count: 0 - - - it: loadBalancer mode filters out empty entries from externalIPs (trailing comma, leading comma) - set: - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10,,192.0.2.11," - expose-mode: loadBalancer - asserts: - - template: templates/cilium-lb-pool.yaml - hasDocuments: - count: 1 - - template: templates/cilium-lb-pool.yaml - equal: - path: spec.blocks - value: - - cidr: 192.0.2.10/32 - - cidr: 192.0.2.11/32 - it: loadBalancer mode with only-empty externalIPs fails chart render (comma-only input) set: @@ -249,17 +163,3 @@ tests: path: spec.values.ingress-nginx.controller.service.externalIPs value: - 192.0.2.10 - - - it: loadBalancer mode accepts pre-CIDR input without double-suffixing - set: - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10/32,2001:db8::1/128" - expose-mode: loadBalancer - asserts: - - template: templates/cilium-lb-pool.yaml - equal: - path: spec.blocks - value: - - cidr: 192.0.2.10/32 - - cidr: 2001:db8::1/128 From 27225f9e8337ae9f11473273aa1939b04cd68a9e Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Mon, 27 Apr 2026 12:46:20 +0300 Subject: [PATCH 072/130] fix(monitoring): use node-level GPU metrics instead of namespace-level DCGM exporter metrics carry the exporter's own namespace (cozy-gpu-operator), not the workload namespace. Recording rules that filtered namespace!~"cozy-.*" silently dropped all DCGM series, producing empty dashboard panels. Replace namespace-level hardware aggregations with node-level equivalents (grouped by Hostname), keep namespace-level allocation rules that use kube_pod_container_resource_requests (which carries the real workload namespace), and rename pod-level efficiency rules to gpu-level since DCGM cannot attribute hardware metrics to individual pods. Signed-off-by: Arsolitt --- dashboards/gpu/gpu-efficiency.json | 68 +++--- dashboards/gpu/gpu-performance.json | 12 +- dashboards/gpu/gpu-tenants.json | 202 +++--------------- .../alerts/gpu-recording.rules.yaml | 94 ++++---- 4 files changed, 110 insertions(+), 266 deletions(-) diff --git a/dashboards/gpu/gpu-efficiency.json b/dashboards/gpu/gpu-efficiency.json index 3bd59bc0..f351c964 100644 --- a/dashboards/gpu/gpu-efficiency.json +++ b/dashboards/gpu/gpu-efficiency.json @@ -35,12 +35,12 @@ "id": 2, "targets": [ { - "expr": "avg(pod:tensor_saturation:avg5m{namespace=~\"$namespace\"}) * 100", + "expr": "avg(gpu:tensor_saturation:avg5m) * 100", "refId": "A" } ], "title": "Avg Tensor Saturation", - "description": "Mean tensor core saturation across selected namespaces. \u003c10% means GPUs are used inefficiently (tenants could move to CPU or optimize their code).", + "description": "Mean tensor core saturation across all GPUs. \u003c10% means GPUs are used inefficiently (workloads could move to CPU or optimize their code). Cluster-wide — DCGM metrics cannot be attributed to workload namespaces.", "transparent": false, "datasource": { "type": "prometheus", @@ -103,12 +103,12 @@ "id": 3, "targets": [ { - "expr": "avg(pod:util_per_watt:avg5m{namespace=~\"$namespace\"})", + "expr": "avg(gpu:util_per_watt:avg5m)", "refId": "A" } ], "title": "Avg Utilization per Watt", - "description": "NVML utilization % per watt across selected namespaces. Higher value = more efficient workload.", + "description": "NVML utilization % per watt across all GPUs. Higher value = more efficient workload. Cluster-wide — DCGM metrics cannot be attributed to workload namespaces.", "transparent": false, "datasource": { "type": "prometheus", @@ -242,13 +242,13 @@ "id": 11, "targets": [ { - "expr": "DCGM_FI_DEV_GPU_UTIL{namespace=~\"$namespace\", namespace!=\"\"}", - "legendFormat": "{{namespace}}/{{pod}}", + "expr": "DCGM_FI_DEV_GPU_UTIL", + "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } ], "title": "NVML GPU Utilization", - "description": "Classic utilization metric. Shows activity of any engine (SM, copy, encoder).", + "description": "Classic utilization metric. Shows activity of any engine (SM, copy, encoder). Cluster-wide — DCGM namespace is the exporter's own namespace, not the workload namespace.", "transparent": false, "datasource": { "type": "prometheus", @@ -296,13 +296,13 @@ "id": 12, "targets": [ { - "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace=~\"$namespace\", namespace!=\"\"} * 100", - "legendFormat": "{{namespace}}/{{pod}}", + "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE * 100", + "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } ], "title": "Tensor Pipe Active", - "description": "Real tensor core load (HMMA). For AI/LLM inference it should be ≥20%, otherwise the workload is unoptimized.", + "description": "Real tensor core load (HMMA). For AI/LLM inference it should be ≥20%, otherwise the workload is unoptimized. Cluster-wide — DCGM namespace is the exporter's own namespace, not the workload namespace.", "transparent": false, "datasource": { "type": "prometheus", @@ -348,7 +348,7 @@ { "type": "row", "collapsed": false, - "title": "Per-pod ranking", + "title": "Per-GPU ranking", "gridPos": { "h": 1, "w": 24, @@ -363,15 +363,15 @@ "id": 21, "targets": [ { - "expr": "topk(20, pod:tensor_saturation:avg5m{namespace=~\"$namespace\"} * 100)", + "expr": "topk(20, gpu:tensor_saturation:avg5m * 100)", "instant": true, "range": false, "format": "table", "refId": "A" } ], - "title": "Tensor Saturation per pod (5m avg)", - "description": "Who is exercising tensor cores and who is not. Sorted descending.", + "title": "Tensor Saturation per GPU (5m avg)", + "description": "Which GPUs are exercising tensor cores and which are not. Sorted descending. Grouped by Hostname/gpu/UUID — DCGM metrics cannot be attributed to workload namespaces.", "transparent": false, "datasource": { "type": "prometheus", @@ -390,20 +390,19 @@ "options": { "excludeByName": { "DCGM_FI_DRIVER_VERSION": true, - "Hostname": true, "Time": true, - "UUID": true, "__name__": true, "cluster": true, "container": true, "device": true, "endpoint": true, - "gpu": true, "gpu_driver_version": true, "instance": true, "job": true, "modelName": true, + "namespace": true, "pci_bus_id": true, + "pod": true, "prometheus": true, "service": true, "tenant": true, @@ -412,14 +411,16 @@ "unit": true }, "indexByName": { - "Value": 2, - "namespace": 0, - "pod": 1 + "Hostname": 0, + "UUID": 2, + "Value": 3, + "gpu": 1 }, "renameByName": { + "Hostname": "Node", + "UUID": "UUID", "Value": "Saturation", - "namespace": "Namespace", - "pod": "Pod" + "gpu": "GPU" } } } @@ -501,15 +502,15 @@ "id": 22, "targets": [ { - "expr": "topk(20, pod:util_per_watt:avg5m{namespace=~\"$namespace\"})", + "expr": "topk(20, gpu:util_per_watt:avg5m)", "instant": true, "range": false, "format": "table", "refId": "A" } ], - "title": "Utilization / Watt per pod (5m avg)", - "description": "How efficiently each pod spends watts. Low value = poor optimization.", + "title": "Utilization / Watt per GPU (5m avg)", + "description": "How efficiently each GPU spends watts. Low value = poor optimization. Grouped by Hostname/gpu/UUID — DCGM metrics cannot be attributed to workload namespaces.", "transparent": false, "datasource": { "type": "prometheus", @@ -528,20 +529,19 @@ "options": { "excludeByName": { "DCGM_FI_DRIVER_VERSION": true, - "Hostname": true, "Time": true, - "UUID": true, "__name__": true, "cluster": true, "container": true, "device": true, "endpoint": true, - "gpu": true, "gpu_driver_version": true, "instance": true, "job": true, "modelName": true, + "namespace": true, "pci_bus_id": true, + "pod": true, "prometheus": true, "service": true, "tenant": true, @@ -550,14 +550,16 @@ "unit": true }, "indexByName": { - "Value": 2, - "namespace": 0, - "pod": 1 + "Hostname": 0, + "UUID": 2, + "Value": 3, + "gpu": 1 }, "renameByName": { + "Hostname": "Node", + "UUID": "UUID", "Value": "Util/Watt", - "namespace": "Namespace", - "pod": "Pod" + "gpu": "GPU" } } } diff --git a/dashboards/gpu/gpu-performance.json b/dashboards/gpu/gpu-performance.json index e581a76b..f9951577 100644 --- a/dashboards/gpu/gpu-performance.json +++ b/dashboards/gpu/gpu-performance.json @@ -273,7 +273,7 @@ "id": 11, "targets": [ { - "expr": "DCGM_FI_DEV_GPU_UTIL{Hostname=~\"$Hostname\", namespace=~\"$namespace\"}", + "expr": "DCGM_FI_DEV_GPU_UTIL{Hostname=~\"$Hostname\"}", "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", "refId": "A" } @@ -327,7 +327,7 @@ "id": 12, "targets": [ { - "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 100", + "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{Hostname=~\"$Hostname\"} * 100", "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", "refId": "A" } @@ -381,7 +381,7 @@ "id": 13, "targets": [ { - "expr": "DCGM_FI_PROF_GR_ENGINE_ACTIVE{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 100", + "expr": "DCGM_FI_PROF_GR_ENGINE_ACTIVE{Hostname=~\"$Hostname\"} * 100", "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", "refId": "A" } @@ -435,7 +435,7 @@ "id": 14, "targets": [ { - "expr": "DCGM_FI_DEV_MEM_COPY_UTIL{Hostname=~\"$Hostname\", namespace=~\"$namespace\"}", + "expr": "DCGM_FI_DEV_MEM_COPY_UTIL{Hostname=~\"$Hostname\"}", "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", "refId": "A" } @@ -502,7 +502,7 @@ "id": 21, "targets": [ { - "expr": "DCGM_FI_DEV_FB_USED{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 1048576", + "expr": "DCGM_FI_DEV_FB_USED{Hostname=~\"$Hostname\"} * 1048576", "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", "refId": "A" } @@ -553,7 +553,7 @@ "id": 22, "targets": [ { - "expr": "DCGM_FI_DEV_FB_FREE{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 1048576", + "expr": "DCGM_FI_DEV_FB_FREE{Hostname=~\"$Hostname\"} * 1048576", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } diff --git a/dashboards/gpu/gpu-tenants.json b/dashboards/gpu/gpu-tenants.json index d0265c0d..e2c400a8 100644 --- a/dashboards/gpu/gpu-tenants.json +++ b/dashboards/gpu/gpu-tenants.json @@ -147,12 +147,12 @@ "id": 4, "targets": [ { - "expr": "avg(namespace:gpu_util:avg{namespace=~\"$namespace\"})", + "expr": "cluster:gpu_util:avg", "refId": "A" } ], - "title": "Avg tenant util", - "description": "Average NVML utilization across all tenant GPUs.", + "title": "Avg cluster GPU util", + "description": "Average NVML utilization across all GPUs in the cluster. Cluster-wide — DCGM hardware metrics cannot be attributed to workload namespaces.", "transparent": false, "datasource": { "type": "prometheus", @@ -211,12 +211,12 @@ "id": 5, "targets": [ { - "expr": "sum(namespace:power_watts:sum{namespace=~\"$namespace\"})", + "expr": "cluster:gpu_power_watts:sum", "refId": "A" } ], - "title": "Total tenant power", - "description": "Live power draw summed across tenant workloads.", + "title": "Total GPU power", + "description": "Live power draw summed across all GPUs in the cluster. Cluster-wide — DCGM hardware metrics cannot be attributed to workload namespaces.", "transparent": false, "datasource": { "type": "prometheus", @@ -266,12 +266,12 @@ "id": 6, "targets": [ { - "expr": "sum(sum_over_time(namespace:power_watts:sum{namespace=~\"$namespace\"}[24h:1m])) / 60 / 1000", + "expr": "sum(sum_over_time(node:power_watts:sum[24h:1m])) / 60 / 1000", "refId": "A" } ], "title": "Energy (last 24h)", - "description": "Integrated power draw over 24h. Feed into billing/chargeback as kWh.", + "description": "Integrated cluster-wide GPU power draw over 24h. Cluster-wide — DCGM hardware metrics cannot be attributed to workload namespaces.", "transparent": false, "datasource": { "type": "prometheus", @@ -452,13 +452,13 @@ "id": 12, "targets": [ { - "expr": "namespace:fb_used_bytes:sum{namespace=~\"$namespace\"}", - "legendFormat": "{{namespace}}", + "expr": "node:fb_used_bytes:sum", + "legendFormat": "{{Hostname}}", "refId": "A" } ], - "title": "VRAM used per namespace", - "description": "FB memory actively used by tenant workloads.", + "title": "VRAM used per node", + "description": "FB memory actively used per node. DCGM hardware metrics cannot be attributed to workload namespaces — shown per-node instead.", "transparent": false, "datasource": { "type": "prometheus", @@ -513,7 +513,7 @@ { "type": "row", "collapsed": false, - "title": "Utilization", + "title": "Utilization (per-node)", "gridPos": { "h": 1, "w": 24, @@ -528,13 +528,13 @@ "id": 21, "targets": [ { - "expr": "namespace:gpu_util:avg{namespace=~\"$namespace\"}", - "legendFormat": "{{namespace}}", + "expr": "node:gpu_util:avg", + "legendFormat": "{{node}}", "refId": "A" } ], - "title": "NVML utilization per namespace", - "description": "Per-tenant NVML utilization. Stacked to show fleet-wide pressure over time.", + "title": "NVML utilization per node", + "description": "Per-node NVML utilization. Stacked to show fleet-wide pressure over time. DCGM hardware metrics cannot be attributed to workload namespaces — shown per-node instead.", "transparent": false, "datasource": { "type": "prometheus", @@ -584,13 +584,13 @@ "id": 22, "targets": [ { - "expr": "namespace:tensor_active:avg{namespace=~\"$namespace\"} * 100", - "legendFormat": "{{namespace}}", + "expr": "node:tensor_active:avg * 100", + "legendFormat": "{{node}}", "refId": "A" } ], - "title": "Tensor saturation per namespace", - "description": "Real tensor core load per tenant. Useful to find AI tenants that claim GPUs but leave tensor cores idle.", + "title": "Tensor saturation per node", + "description": "Real tensor core load per node. Useful to spot nodes with idle tensor cores. DCGM hardware metrics cannot be attributed to workload namespaces — shown per-node instead.", "transparent": false, "datasource": { "type": "prometheus", @@ -653,13 +653,13 @@ "id": 31, "targets": [ { - "expr": "namespace:power_watts:sum{namespace=~\"$namespace\"}", - "legendFormat": "{{namespace}}", + "expr": "node:power_watts:sum", + "legendFormat": "{{node}}", "refId": "A" } ], - "title": "Power draw per namespace", - "description": "Per-tenant power draw. Stacked to see cluster-wide energy burn over time.", + "title": "Power draw per node", + "description": "Per-node GPU power draw. Stacked to see cluster-wide energy burn over time. DCGM hardware metrics cannot be attributed to workload namespaces — shown per-node instead.", "transparent": false, "datasource": { "type": "prometheus", @@ -783,38 +783,10 @@ "range": false, "format": "table", "refId": "gpus" - }, - { - "expr": "namespace:gpu_util:avg{namespace=~\"$namespace\"}", - "instant": true, - "range": false, - "format": "table", - "refId": "util" - }, - { - "expr": "namespace:tensor_active:avg{namespace=~\"$namespace\"} * 100", - "instant": true, - "range": false, - "format": "table", - "refId": "tensor" - }, - { - "expr": "namespace:power_watts:sum{namespace=~\"$namespace\"}", - "instant": true, - "range": false, - "format": "table", - "refId": "power" - }, - { - "expr": "namespace:fb_used_bytes:sum{namespace=~\"$namespace\"}", - "instant": true, - "range": false, - "format": "table", - "refId": "vram" } ], - "title": "Top consumers", - "description": "Live ranking of tenants by power draw. Columns joined from namespace recording rules.", + "title": "Top consumers (allocation)", + "description": "Live ranking of tenants by GPU allocation. Hardware metrics (util, tensor, power, VRAM) cannot be attributed to workload namespaces — see Fleet and Performance dashboards for per-node/per-GPU hardware views.", "transparent": false, "datasource": { "type": "prometheus", @@ -828,13 +800,6 @@ }, "repeatDirection": "h", "transformations": [ - { - "id": "joinByField", - "options": { - "byField": "namespace", - "mode": "outer" - } - }, { "id": "organize", "options": { @@ -914,18 +879,10 @@ }, "indexByName": { "Value #gpus": 1, - "Value #power": 4, - "Value #tensor": 3, - "Value #util": 2, - "Value #vram": 5, "namespace": 0 }, "renameByName": { "Value #gpus": "GPUs", - "Value #power": "Power", - "Value #tensor": "Tensor %", - "Value #util": "Util %", - "Value #vram": "VRAM", "namespace": "Namespace" } } @@ -937,7 +894,7 @@ "showTypeIcons": false, "sortBy": [ { - "displayName": "Power", + "displayName": "GPUs", "desc": true } ], @@ -946,101 +903,6 @@ "reducer": [] }, "cellHeight": "sm" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Util %" - }, - "properties": [ - { - "id": "unit", - "value": "percent" - }, - { - "id": "min", - "value": 0 - }, - { - "id": "max", - "value": 100 - }, - { - "id": "decimals", - "value": 1 - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "gauge" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Tensor %" - }, - "properties": [ - { - "id": "unit", - "value": "percent" - }, - { - "id": "min", - "value": 0 - }, - { - "id": "max", - "value": 100 - }, - { - "id": "decimals", - "value": 1 - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "gauge" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Power" - }, - "properties": [ - { - "id": "unit", - "value": "watt" - }, - { - "id": "decimals", - "value": 0 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "VRAM" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - } - ] } }, { @@ -1125,13 +987,13 @@ "id": 52, "targets": [ { - "expr": "sum_over_time(namespace:power_watts:sum{namespace=~\"$namespace\"}[24h:1m]) / 60 / 1000", - "legendFormat": "{{namespace}}", + "expr": "sum_over_time(node:power_watts:sum[24h:1m]) / 60 / 1000", + "legendFormat": "{{node}}", "refId": "A" } ], - "title": "Energy per namespace (last 24h)", - "description": "kWh consumed per tenant over the last 24h. Integrated from 1-minute power samples.", + "title": "Energy per node (last 24h)", + "description": "kWh consumed per node over the last 24h. Integrated from 1-minute power samples. DCGM hardware metrics cannot be attributed to workload namespaces — shown per-node instead.", "transparent": false, "datasource": { "type": "prometheus", diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml index 093a2939..b046edf5 100644 --- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml +++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml @@ -42,46 +42,23 @@ spec: # excluding infra pods would under-report raw capacity and # break capacity-planning math. # - # * cluster:gpu_util:avg DOES filter infra namespaces because it - # is a tenant-oriented KPI. Admins want to see the mean - # utilization of tenant workloads, not have it diluted by - # GPU Operator DaemonSet pods whose containers hold a GPU - # handle but do no useful compute. - # - # Do not "normalize" by adding the filter to the hardware rules — - # downstream consumers (capacity planning, billing) rely on the - # raw totals. + # No namespace filter — DCGM exporter metrics carry the exporter's + # own namespace, not the workload namespace. Filtering by namespace + # here would silently drop all series. - record: cluster:gpu_util:avg - expr: avg(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}) + expr: avg(DCGM_FI_DEV_GPU_UTIL) - record: cluster:gpu_power_watts:sum expr: sum(DCGM_FI_DEV_POWER_USAGE) - - name: gpu.recording.namespace.1m + - name: gpu.recording.node.1m interval: 1m params: {} rules: - # Kube-requested GPU count per namespace — billable view, includes - # Pending pods. Use this for GPU-hour reporting via - # sum_over_time(...[1h:1m])/60. - # - # Filters cozy-*/kube-* so the namespace set here matches every - # other namespace:* rule below. This keeps the $namespace variable - # in the tenant dashboard consistent with panels that read filtered - # rules — otherwise picking a system ns from the dropdown leaves - # half the panels empty while others show values. - # - # Trade-off: per-namespace GPU accounting for system workloads is - # no longer available here. If it's ever needed (e.g. operator - # pods pinning tenant GPUs), add a separate rule like - # system:gpu_count:allocated rather than widening this one — the - # "billable tenant view" invariant is relied on by tenants.json - # and keeping it narrow protects consumers from the asymmetry - # that triggered this filter in the first place. - # - # Note: sum(namespace:gpu_count:allocated) no longer equals - # cluster:gpu_count:allocated — the cluster-level rule intentionally - # keeps system pods to stay aligned with cluster:gpu_count:total - # when computing :free. + # Kube-requested GPU count per namespace — billable/allocation view. + # This is the only namespace-level rule that works correctly because + # kube_pod_container_resource_requests carries the real workload + # namespace, unlike DCGM exporter metrics which carry the exporter's + # own namespace (cozy-gpu-operator). - record: namespace:gpu_count:allocated expr: | sum by (namespace) ( @@ -90,14 +67,18 @@ spec: kube_pod_status_phase{phase=~"Pending|Running"} == 1 ) ) - - record: namespace:gpu_util:avg - expr: avg by (namespace) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}) - - record: namespace:tensor_active:avg - expr: avg by (namespace) (DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace!="", namespace!~"cozy-.*|kube-.*"}) - - record: namespace:fb_used_bytes:sum - expr: sum by (namespace) (DCGM_FI_DEV_FB_USED{namespace!="", namespace!~"cozy-.*|kube-.*"}) * 1048576 - - record: namespace:power_watts:sum - expr: sum by (namespace) (DCGM_FI_DEV_POWER_USAGE{namespace!="", namespace!~"cozy-.*|kube-.*"}) + # Hardware metrics aggregated per node (Hostname). DCGM exporter + # metrics carry a capital-H "Hostname" label identifying the node + # but no usable workload namespace — so hardware telemetry is + # aggregated at node/GPU granularity, not namespace. + - record: node:gpu_util:avg + expr: avg by (Hostname) (DCGM_FI_DEV_GPU_UTIL) + - record: node:tensor_active:avg + expr: avg by (Hostname) (DCGM_FI_PROF_PIPE_TENSOR_ACTIVE) + - record: node:fb_used_bytes:sum + expr: sum by (Hostname) (DCGM_FI_DEV_FB_USED) * 1048576 + - record: node:power_watts:sum + expr: sum by (Hostname) (DCGM_FI_DEV_POWER_USAGE) # Known gap: there is no lower-bound sanity alert for under-reporting. # If DCGM ever switches POWER/THERMAL_VIOLATION counter units the other @@ -113,27 +94,26 @@ spec: interval: 1m params: {} rules: - # Tensor hardware saturation — the honest "am I using the GPU" metric - # for AI/LLM workloads. Unlike NVML, idle tensor cores are visible. - # Stored as a 0..1 ratio to stay consistent with namespace:tensor_active:avg - # and DCGM's native units. Consumers multiply by 100 at display time. - - record: pod:tensor_saturation:avg5m + # Tensor hardware saturation per GPU — the honest "am I using the GPU" + # metric for AI/LLM workloads. Aggregated at GPU level (Hostname + + # gpu + UUID) because DCGM metrics don't carry workload namespace. + # Stored as a 0..1 ratio; consumers multiply by 100 at display time. + - record: gpu:tensor_saturation:avg5m expr: | - avg by (Hostname, gpu, UUID, namespace, pod) ( - avg_over_time(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]) + avg by (Hostname, gpu, UUID) ( + avg_over_time(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE[5m]) ) - # Power efficiency — utilization per watt, reveals unoptimized clients. - # Explicit on(...) pins the bigop matching set to the labels we - # group by — protects against empty results if dcgm-exporter - # relabeling ever diverges between the two metrics. - - record: pod:util_per_watt:avg5m + # Power efficiency per GPU — utilization per watt, reveals + # unoptimized workloads. Explicit on(...) pins the matching set + # to GPU-identifying labels. + - record: gpu:util_per_watt:avg5m expr: | - avg by (Hostname, gpu, UUID, namespace, pod) ( - avg_over_time(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]) - / on (Hostname, gpu, UUID, namespace, pod) + avg by (Hostname, gpu, UUID) ( + avg_over_time(DCGM_FI_DEV_GPU_UTIL[5m]) + / on (Hostname, gpu, UUID) clamp_min( - avg_over_time(DCGM_FI_DEV_POWER_USAGE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]), + avg_over_time(DCGM_FI_DEV_POWER_USAGE[5m]), 1 ) ) From f866c71b684c742ac47e66ceedcca1d7158d6363 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Mon, 27 Apr 2026 12:51:31 +0300 Subject: [PATCH 073/130] fix(gpu-operator): add node relabel to example serviceMonitor values Signed-off-by: Arsolitt --- .../system/gpu-operator/examples/values-native-talos.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/system/gpu-operator/examples/values-native-talos.yaml b/packages/system/gpu-operator/examples/values-native-talos.yaml index 5a3d7786..ef7b2891 100644 --- a/packages/system/gpu-operator/examples/values-native-talos.yaml +++ b/packages/system/gpu-operator/examples/values-native-talos.yaml @@ -44,5 +44,9 @@ spec: # documented overhead concerns below 1s; 15s is safe. interval: "15s" honorLabels: true + relabelings: + - sourceLabels: [__meta_kubernetes_pod_node_name] + targetLabel: node + action: replace config: name: dcgm-custom-metrics From d8d870cc2af73925561ae2f9aa936613b47ad0fe Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Mon, 27 Apr 2026 13:24:27 +0300 Subject: [PATCH 074/130] fix(hami): use RollingUpdate strategy for device plugin DaemonSet OnDelete requires manual pod deletion to apply updates. RollingUpdate with maxUnavailable constraint matches upstream default and is consistent with all other system packages. Signed-off-by: Arsolitt --- packages/system/hami/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/system/hami/values.yaml b/packages/system/hami/values.yaml index 64b372a6..76e8d631 100644 --- a/packages/system/hami/values.yaml +++ b/packages/system/hami/values.yaml @@ -7,7 +7,7 @@ hami: devicePlugin: runtimeClassName: nvidia updateStrategy: - type: OnDelete + type: RollingUpdate # See https://github.com/Project-HAMi/HAMi/blob/v2.8.1/deployments/charts/hami/values.yaml # Each entry: {"name": "node-name", "operatingmode": "hami-core", "devicesplitcount": N, ...} nodeConfiguration: From 36852548e576e153981eab0146614442b5bc9a15 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Mon, 27 Apr 2026 13:29:15 +0300 Subject: [PATCH 075/130] fix(hami): correct label indentation in device-plugin monitorservice Use nindent instead of indent with leading whitespace to prevent broken YAML rendering when devicePlugin.service.labels is set. Signed-off-by: Arsolitt --- .../charts/hami/templates/device-plugin/monitorservice.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml index 104664ea..88f1b214 100644 --- a/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml @@ -7,8 +7,8 @@ metadata: labels: app.kubernetes.io/component: hami-device-plugin {{- include "hami-vgpu.labels" . | nindent 4 }} - {{- if .Values.devicePlugin.service.labels }} # Use devicePlugin instead of scheduler - {{ toYaml .Values.devicePlugin.service.labels | indent 4 }} + {{- with .Values.devicePlugin.service.labels }} + {{- toYaml . | nindent 4 }} {{- end }} {{- if .Values.devicePlugin.service.annotations }} # Use devicePlugin instead of scheduler annotations: {{ toYaml .Values.devicePlugin.service.annotations | nindent 4 }} From f0e033ebcb2fc727bc8fb473507de34edacbf898 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Mon, 27 Apr 2026 13:44:08 +0300 Subject: [PATCH 076/130] style(kubernetes): use with instead of if for hami valuesOverride Signed-off-by: Arsolitt --- packages/apps/kubernetes/templates/helmreleases/hami.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apps/kubernetes/templates/helmreleases/hami.yaml b/packages/apps/kubernetes/templates/helmreleases/hami.yaml index 8733e675..1f7a5358 100644 --- a/packages/apps/kubernetes/templates/helmreleases/hami.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/hami.yaml @@ -32,9 +32,9 @@ spec: force: true remediation: retries: -1 - {{- if .Values.addons.hami.valuesOverride }} + {{- with .Values.addons.hami.valuesOverride }} values: - {{- toYaml .Values.addons.hami.valuesOverride | nindent 4 }} + {{- toYaml . | nindent 4 }} {{- end }} dependsOn: From c53f10475025c6675e5270f359f97f619db7c80f Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Mon, 27 Apr 2026 13:44:12 +0300 Subject: [PATCH 077/130] docs(hami): clarify that parameter defaults come from upstream chart Signed-off-by: Arsolitt --- packages/system/hami/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/system/hami/README.md b/packages/system/hami/README.md index 283f9e80..d648364d 100644 --- a/packages/system/hami/README.md +++ b/packages/system/hami/README.md @@ -72,6 +72,8 @@ resources: ## Parameters +Default values shown below are inherited from the upstream HAMi chart and may change with upstream updates. + | Name | Description | Default | | --- | --- | --- | | `hami.devicePlugin.runtimeClassName` | RuntimeClass for device plugin pods | `nvidia` | From 9ba5e587810f44d522d5876e38f12f87c3dde3bb Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Mon, 27 Apr 2026 13:44:17 +0300 Subject: [PATCH 078/130] fix(kubernetes): avoid rendering empty values key in gpu-operator HelmRelease Wrap the values: block in a conditional so the key is omitted entirely when no defaults or overrides produce content. Previously the template always emitted values: null, triggering unnecessary FluxCD reconciliation. Signed-off-by: Arsolitt --- .../apps/kubernetes/templates/helmreleases/gpu-operator.yaml | 4 +++- packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml index 7a5ecaf1..15849495 100644 --- a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml @@ -40,8 +40,10 @@ spec: {{- $defaults := fromYaml (include "cozystack.defaultGpuOperatorValues" .) }} {{- $overrides := deepCopy (default (dict) .Values.addons.gpuOperator.valuesOverride) }} {{- $merged := mergeOverwrite (default (dict) $defaults) $overrides }} + {{- if $merged }} values: - {{- if $merged }}{{ toYaml $merged | nindent 4 }}{{ end }} + {{- toYaml $merged | nindent 4 }} + {{- end }} dependsOn: {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }} diff --git a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml index c3ec30f2..4fa754fb 100644 --- a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml +++ b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml @@ -26,9 +26,8 @@ tests: enabled: false valuesOverride: {} asserts: - - equal: + - notExists: path: spec.values - value: null - it: should apply hami defaults when valuesOverride key is omitted set: From 7443e223453032d0a907311f3b5865889b713299 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 27 Apr 2026 17:18:45 +0500 Subject: [PATCH 079/130] build(linstor): include linstor-gui in root image build target The linstor-gui package was added in #2382 with its own per-package Makefile and Dockerfile, but the root Makefile's `build:` target was not updated to invoke it. As a result `ghcr.io/cozystack/cozystack/ linstor-gui` has never been published (registry returns NAME_UNKNOWN) and the chart's `image.tag` was never digest-pinned. Any cluster deploying the chart hits ImagePullBackOff. Wire the package into the root build alongside the other system images. The next CI build will publish the image and the per-package Makefile will rewrite values.yaml `image.repository`/`image.tag` to a digest-pinned reference automatically. Assisted-By: Claude Signed-off-by: Myasnikov Daniil --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 0e94c5e6..5041ac64 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,7 @@ build: build-deps make -C packages/system/lineage-controller-webhook image make -C packages/system/cilium image make -C packages/system/linstor image + make -C packages/system/linstor-gui image make -C packages/system/kubeovn-webhook image make -C packages/system/kubeovn-plunger image make -C packages/system/dashboard image From a9a66bf066360c340203508fd6072977ff4eb8ea Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 27 Apr 2026 17:31:07 +0500 Subject: [PATCH 080/130] build(linstor): use shared BUILDX_ARGS for linstor-gui image build The per-package Makefile added in #2382 hardcoded buildx flags (--provenance, --builder, --platform=linux/amd64,linux/arm64, --push, --load, --label) instead of using the shared $(BUILDX_ARGS) macro from hack/common-envs.mk. This broke CI: the runner's default docker driver does not support multi-platform builds, and the hardcoded multi-arch platform list crashed `make build` with "Multi-platform build is not supported for the docker driver." Replace the hardcoded flags with $(BUILDX_ARGS) to match every other package (e.g. linstor, dashboard, cilium). $(BUILDX_ARGS) injects --push, --load, --label, --provenance=false, and only sets --builder or --platform when the operator explicitly exports BUILDER/PLATFORM. Assisted-By: Claude Signed-off-by: Myasnikov Daniil --- packages/system/linstor-gui/Makefile | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/system/linstor-gui/Makefile b/packages/system/linstor-gui/Makefile index e202f4cc..b408491d 100644 --- a/packages/system/linstor-gui/Makefile +++ b/packages/system/linstor-gui/Makefile @@ -12,18 +12,13 @@ image: image-linstor-gui image-linstor-gui: docker buildx build images/linstor-gui \ - --provenance false \ - --builder=$(BUILDER) \ - --platform=linux/amd64,linux/arm64 \ --build-arg LINSTOR_GUI_VERSION=$(LINSTOR_GUI_VERSION) \ --tag $(REGISTRY)/linstor-gui:$(call settag,$(LINSTOR_GUI_VERSION)) \ --tag $(REGISTRY)/linstor-gui:$(call settag,$(LINSTOR_GUI_VERSION)-$(TAG)) \ --cache-from type=registry,ref=$(REGISTRY)/linstor-gui:latest \ --cache-to type=inline \ --metadata-file images/linstor-gui.json \ - --push=$(PUSH) \ - --label "org.opencontainers.image.source=https://github.com/cozystack/cozystack" \ - --load=$(LOAD) + $(BUILDX_ARGS) REPOSITORY="$(REGISTRY)/linstor-gui" \ yq -i '.image.repository = strenv(REPOSITORY)' values.yaml TAG="$(call settag,$(LINSTOR_GUI_VERSION))@$$(yq e '."containerimage.digest"' images/linstor-gui.json -o json -r)" \ From 073fb1630d67ace6655deb262d1734fab98e8ce4 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Mon, 27 Apr 2026 11:32:17 +0200 Subject: [PATCH 081/130] feat(linstor): bump linstor-csi to v1.10.6 with Protocol-C dual-attach fix Bump linstor-csi to v1.10.6 and add an out-of-tree patch that overrides DrbdOptions/Net/protocol to C on the resource-definition whenever a volume is attached to a second node with allow-two-primaries (the live-migration flow), and reverts it on detach. Without the patch, any KubeVirt live migration of a VM whose volume sits on a Protocol-A/B resource (for example a "replicated-async" StorageClass) ends up in a permanent evacuation loop: drbdadm adjust on the satellite rejects allow-two-primaries with "Protocol C required" (errno 139), the migration fails, and KubeVirt retries until manual intervention. The override is set at the resource-definition level so it covers every connection, including diskless TieBreaker peers, and is tagged with an Aux marker so Detach reverts only the override we installed. The 001 relocate-after-clone patch was regenerated against v1.10.6 (context shift only, no logic change). Verified end-to-end on dev5: live migration of a KubeVirt VM whose PVC sits on a Protocol-A resource-group now succeeds in a single Attach. Patch is upstreamed as draft PR piraeusdatastore/linstor-csi#435. Co-Authored-By: Claude Signed-off-by: Andrei Kvapil --- packages/system/linstor/Makefile | 2 +- .../linstor/images/linstor-csi/Dockerfile | 2 +- .../001-relocate-after-clone-restore.diff | 62 +------- ...2-protocol-c-override-for-dual-attach.diff | 132 ++++++++++++++++++ 4 files changed, 140 insertions(+), 58 deletions(-) create mode 100644 packages/system/linstor/images/linstor-csi/patches/002-protocol-c-override-for-dual-attach.diff diff --git a/packages/system/linstor/Makefile b/packages/system/linstor/Makefile index 2ad4def9..8d566678 100644 --- a/packages/system/linstor/Makefile +++ b/packages/system/linstor/Makefile @@ -5,7 +5,7 @@ include ../../../hack/common-envs.mk include ../../../hack/package.mk LINSTOR_VERSION ?= 1.33.2 -LINSTOR_CSI_VERSION ?= v1.10.5 +LINSTOR_CSI_VERSION ?= v1.10.6 image: image-piraeus-server image-linstor-csi diff --git a/packages/system/linstor/images/linstor-csi/Dockerfile b/packages/system/linstor/images/linstor-csi/Dockerfile index 6cfd44aa..df5d179d 100644 --- a/packages/system/linstor/images/linstor-csi/Dockerfile +++ b/packages/system/linstor/images/linstor-csi/Dockerfile @@ -1,6 +1,6 @@ FROM golang:1.25 AS builder -ARG VERSION=v1.10.5 +ARG VERSION=v1.10.6 ARG LINSTOR_WAIT_UNTIL_VERSION=v0.3.1 ARG TARGETARCH ARG TARGETOS diff --git a/packages/system/linstor/images/linstor-csi/patches/001-relocate-after-clone-restore.diff b/packages/system/linstor/images/linstor-csi/patches/001-relocate-after-clone-restore.diff index ff0aec0c..0e272670 100644 --- a/packages/system/linstor/images/linstor-csi/patches/001-relocate-after-clone-restore.diff +++ b/packages/system/linstor/images/linstor-csi/patches/001-relocate-after-clone-restore.diff @@ -1,25 +1,7 @@ diff --git a/pkg/client/linstor.go b/pkg/client/linstor.go -index f544493..98e7fde 100644 +index ac4651d..46d565d 100644 --- a/pkg/client/linstor.go +++ b/pkg/client/linstor.go -@@ -181,7 +181,7 @@ func LogLevel(s string) func(*Linstor) error { - func (s *Linstor) ListAllWithStatus(ctx context.Context) ([]volume.VolumeStatus, error) { - var vols []volume.VolumeStatus - -- resourcesByName := make(map[string][]lapi.Resource) -+ resourcesByName := make(map[string][]lapi.ResourceWithVolumes) - - resDefs, err := s.client.ResourceDefinitions.GetAll(ctx, lapi.RDGetAllRequest{WithVolumeDefinitions: true}) - if err != nil { -@@ -194,7 +194,7 @@ func (s *Linstor) ListAllWithStatus(ctx context.Context) ([]volume.VolumeStatus, - } - - for i := range allResources { -- resourcesByName[allResources[i].Name] = append(resourcesByName[allResources[i].Name], allResources[i].Resource) -+ resourcesByName[allResources[i].Name] = append(resourcesByName[allResources[i].Name], allResources[i]) - } - - for _, rd := range resDefs { @@ -462,6 +462,14 @@ func (s *Linstor) Clone(ctx context.Context, vol, src *volume.Info, params *volu return err } @@ -35,7 +17,7 @@ index f544493..98e7fde 100644 logger.Debug("reconcile extra properties") err = s.client.ResourceDefinitions.Modify(ctx, vol.ID, lapi.GenericPropsModify{OverrideProps: vol.Properties}) -@@ -1280,6 +1288,14 @@ func (s *Linstor) VolFromSnap(ctx context.Context, snap *volume.Snapshot, vol *v +@@ -1297,6 +1305,14 @@ func (s *Linstor) VolFromSnap(ctx context.Context, snap *volume.Snapshot, vol *v return err } @@ -50,7 +32,7 @@ index f544493..98e7fde 100644 logger.Debug("reconcile extra properties") err = s.client.ResourceDefinitions.Modify(ctx, vol.ID, lapi.GenericPropsModify{OverrideProps: vol.Properties}) -@@ -1470,9 +1486,8 @@ func (s *Linstor) reconcileSnapshotResources(ctx context.Context, snapshot *volu +@@ -1487,9 +1503,8 @@ func (s *Linstor) reconcileSnapshotResources(ctx context.Context, snapshot *volu return fmt.Errorf("snapshot '%s' not deployed on any node", snap.Name) } @@ -62,7 +44,7 @@ index f544493..98e7fde 100644 for _, snapNode := range snap.Nodes { if err := s.NodeAvailable(ctx, snapNode); err != nil { logger.WithField("selected node candidate", snapNode).WithError(err).Debug("node is not available") -@@ -1480,13 +1495,23 @@ func (s *Linstor) reconcileSnapshotResources(ctx context.Context, snapshot *volu +@@ -1497,13 +1512,23 @@ func (s *Linstor) reconcileSnapshotResources(ctx context.Context, snapshot *volu } if slices.Contains(preferredNodes, snapNode) { @@ -92,7 +74,7 @@ index f544493..98e7fde 100644 } if selectedNode == "" { -@@ -1679,6 +1704,114 @@ func (s *Linstor) reconcileResourcePlacement(ctx context.Context, vol *volume.In +@@ -1696,6 +1721,114 @@ func (s *Linstor) reconcileResourcePlacement(ctx context.Context, vol *volume.In return nil } @@ -207,40 +189,8 @@ index f544493..98e7fde 100644 // FindSnapsByID searches the snapshot in the backend func (s *Linstor) FindSnapsByID(ctx context.Context, id string) ([]*volume.Snapshot, error) { snapshotId, err := volume.ParseSnapshotId(id) -@@ -2173,7 +2306,7 @@ func (s *Linstor) Status(ctx context.Context, volId string) ([]string, *csi.Volu - "volume": volId, - }).Debug("getting assignments") - -- ress, err := s.client.Resources.GetAll(ctx, volId) -+ ress, err := s.client.Resources.GetResourceView(ctx, &lapi.ListOpts{Resource: []string{volId}}) - if err != nil { - return nil, nil, fmt.Errorf("failed to list resources for '%s': %w", volId, err) - } -@@ -2261,13 +2394,20 @@ func GetSnapshotRemoteAndReadiness(snap *lapi.Snapshot) (string, bool, error) { - return "", slices.Contains(snap.Flags, lapiconsts.FlagSuccessful), nil - } - --func NodesAndConditionFromResources(ress []lapi.Resource) ([]string, *csi.VolumeCondition) { -+func NodesAndConditionFromResources(ress []lapi.ResourceWithVolumes) ([]string, *csi.VolumeCondition) { - var allNodes, abnormalNodes []string - - for i := range ress { - res := &ress[i] - -- allNodes = append(allNodes, res.NodeName) -+ // A resource is a CSI publish target if any of its volumes were created -+ // by ControllerPublishVolume, identified by the temporary-diskless-attach property. -+ if slices.ContainsFunc(res.Volumes, func(v lapi.Volume) bool { -+ createdFor, ok := v.Props[linstor.PropertyCreatedFor] -+ return ok && createdFor == linstor.CreatedForTemporaryDisklessAttach -+ }) { -+ allNodes = append(allNodes, res.NodeName) -+ } - - if res.State == nil { - abnormalNodes = append(abnormalNodes, res.NodeName) diff --git a/pkg/volume/parameter.go b/pkg/volume/parameter.go -index 39acd95..aed18ab 100644 +index 39acd95..54d1dfb 100644 --- a/pkg/volume/parameter.go +++ b/pkg/volume/parameter.go @@ -50,6 +50,7 @@ const ( diff --git a/packages/system/linstor/images/linstor-csi/patches/002-protocol-c-override-for-dual-attach.diff b/packages/system/linstor/images/linstor-csi/patches/002-protocol-c-override-for-dual-attach.diff new file mode 100644 index 00000000..279c636b --- /dev/null +++ b/packages/system/linstor/images/linstor-csi/patches/002-protocol-c-override-for-dual-attach.diff @@ -0,0 +1,132 @@ +diff --git a/pkg/client/linstor.go b/pkg/client/linstor.go +index ac4651d..9d75c79 100644 +--- a/pkg/client/linstor.go ++++ b/pkg/client/linstor.go +@@ -653,11 +653,35 @@ func (s *Linstor) Attach(ctx context.Context, volId, node string, rwxBlock bool) + } + + if otherResInUse > 0 && rwxBlock { +- rdPropsModify := lapi.GenericPropsModify{OverrideProps: map[string]string{ ++ rdProps := map[string]string{ + linstor.PropertyAllowTwoPrimaries: "yes", +- }} ++ } + +- err = s.client.ResourceDefinitions.Modify(ctx, volId, rdPropsModify) ++ // DRBD requires Protocol C whenever allow-two-primaries is enabled. ++ // If the resource is configured with Protocol A or B (typically ++ // through its resource-group), drbdadm adjust on the satellites ++ // fails with "Protocol C required" (errno 139) and live migration ++ // cannot proceed. Override Protocol to C on the resource-definition ++ // itself so it applies to every connection (including diskless ++ // peers, e.g. a TieBreaker). The marker lets Detach revert exactly ++ // the override we installed without disturbing operator-set props. ++ proto, protoErr := s.getEffectiveDrbdProtocol(ctx, volId) ++ if protoErr != nil { ++ s.log.WithError(protoErr).WithField("volume", volId). ++ Warn("failed to determine effective DRBD protocol; skipping Protocol=C override for dual-attach") ++ } else if proto == "A" || proto == "B" { ++ s.log.WithFields(logrus.Fields{ ++ "volume": volId, ++ "protocol": proto, ++ }).Info("installing Protocol=C override on resource-definition for dual-attach") ++ ++ rdProps[linstor.PropertyDrbdNetProtocol] = "C" ++ rdProps[linstor.PropertyCsiProtocolOverride] = "yes" ++ } ++ ++ err = s.client.ResourceDefinitions.Modify(ctx, volId, lapi.GenericPropsModify{ ++ OverrideProps: rdProps, ++ }) + if err != nil { + return "", err + } +@@ -774,11 +798,26 @@ func (s *Linstor) Detach(ctx context.Context, volId, node string) error { + } + + if resInUse == 1 { +- rdPropsModify := lapi.GenericPropsModify{DeleteProps: []string{ +- linstor.PropertyAllowTwoPrimaries, +- }} ++ deleteProps := []string{linstor.PropertyAllowTwoPrimaries} ++ ++ // Drop the Protocol=C override only if Attach installed it (gated by ++ // our marker), so an operator-set Protocol property on the ++ // resource-definition is preserved. ++ rd, getErr := s.client.ResourceDefinitions.Get(ctx, volId) ++ if getErr != nil { ++ log.WithError(getErr).Warn("failed to fetch resource-definition props; skipping Protocol=C override removal") ++ } else if rd.Props[linstor.PropertyCsiProtocolOverride] != "" { ++ log.Info("removing Protocol=C override from resource-definition") + +- err = s.client.ResourceDefinitions.Modify(ctx, volId, rdPropsModify) ++ deleteProps = append(deleteProps, ++ linstor.PropertyDrbdNetProtocol, ++ linstor.PropertyCsiProtocolOverride, ++ ) ++ } ++ ++ err = s.client.ResourceDefinitions.Modify(ctx, volId, lapi.GenericPropsModify{ ++ DeleteProps: deleteProps, ++ }) + if err != nil { + return nil404(err) + } +@@ -805,6 +844,33 @@ func (s *Linstor) Detach(ctx context.Context, volId, node string) error { + return nil404(s.client.Resources.Delete(ctx, volId, node)) + } + ++// getEffectiveDrbdProtocol returns the DRBD network protocol ("A", "B" or "C") ++// that LINSTOR will hand to drbdadm for the given resource. Resource-definition ++// properties take precedence; otherwise the value is inherited from the ++// resource-group. An empty string is returned when neither level sets the ++// property (LINSTOR's compiled-in default is "C"). ++func (s *Linstor) getEffectiveDrbdProtocol(ctx context.Context, volId string) (string, error) { ++ rd, err := s.client.ResourceDefinitions.Get(ctx, volId) ++ if err != nil { ++ return "", err ++ } ++ ++ if v, ok := rd.Props[linstor.PropertyDrbdNetProtocol]; ok { ++ return v, nil ++ } ++ ++ if rd.ResourceGroupName == "" { ++ return "", nil ++ } ++ ++ rg, err := s.client.ResourceGroups.Get(ctx, rd.ResourceGroupName) ++ if err != nil { ++ return "", err ++ } ++ ++ return rg.Props[linstor.PropertyDrbdNetProtocol], nil ++} ++ + // CapacityBytes returns the amount of free space in the storage pool specified by the params and topology. + func (s *Linstor) CapacityBytes(ctx context.Context, storagePools []string, overProvision *float64, segments map[string]string) (int64, error) { + log := s.log.WithField("storage-pools", storagePools).WithField("segments", segments) +diff --git a/pkg/linstor/const.go b/pkg/linstor/const.go +index 9a5f79c..c8bc9c3 100644 +--- a/pkg/linstor/const.go ++++ b/pkg/linstor/const.go +@@ -44,6 +44,19 @@ const ( + // PropertyAllowTwoPrimaries is DRBD option to allow second primary. Mainly used for live-migration. + PropertyAllowTwoPrimaries = lc.NamespcDrbdNetOptions + "/allow-two-primaries" + ++ // PropertyDrbdNetProtocol is the DRBD network replication protocol option ++ // (A = async, B = semi-sync, C = sync). DRBD requires Protocol C whenever ++ // allow-two-primaries is enabled, otherwise drbdadm adjust fails with ++ // "Protocol C required" (errno 139). ++ PropertyDrbdNetProtocol = lc.NamespcDrbdNetOptions + "/protocol" ++ ++ // PropertyCsiProtocolOverride marks a resource-connection where this driver ++ // has installed a temporary Protocol=C override to make a Protocol-A/B ++ // resource compatible with allow-two-primaries during live migration. The ++ // marker lets us safely remove only the overrides we set, without touching ++ // connection-level overrides installed by the operator. ++ PropertyCsiProtocolOverride = lc.NamespcAuxiliary + "/csi-protocol-override" ++ + // CreatedForTemporaryDisklessAttach marks a resource as temporary, i.e. it should be removed after it is no longer + // needed. + CreatedForTemporaryDisklessAttach = "temporary-diskless-attach" From 80631bc9160a4567cce8facdf781799f15455497 Mon Sep 17 00:00:00 2001 From: mattia-eleuteri Date: Mon, 27 Apr 2026 16:29:05 +0200 Subject: [PATCH 082/130] [vm-instance] Set wholeIP annotation conditionally on externalMethod MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Render `networking.cozystack.io/wholeIP: "false"` on the Service when `externalMethod: PortList` is configured (was always `"true"` before). Combined with cozy-proxy v0.3.0+ which adds per-port filtering for "false"-annotated services, this makes `externalMethod: PortList` behave as documented: only ports listed in `externalPorts` are reachable from the LoadBalancer IP. Backward-compatible: existing services with `externalMethod: WholeIP` continue to set `wholeIP: "true"` and behave identically. cozy-proxy versions older than v0.3.0 ignore Services with `wholeIP: "false"`, which means PortList Services on older cozy-proxy will lose their egress IP preservation — but that path was already non-functional ingress-wise, so this is not a regression for users actually relying on PortList. Signed-off-by: mattia-eleuteri --- docs/changelogs/v1.3.1.md | 11 +++++++++++ packages/apps/vm-instance/templates/service.yaml | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 docs/changelogs/v1.3.1.md diff --git a/docs/changelogs/v1.3.1.md b/docs/changelogs/v1.3.1.md new file mode 100644 index 00000000..7f9ecfab --- /dev/null +++ b/docs/changelogs/v1.3.1.md @@ -0,0 +1,11 @@ + + +## Bug Fixes + +* **[vm-instance] Fix `externalMethod: PortList` not filtering ingress ports**: The `vm-instance` chart now sets `networking.cozystack.io/wholeIP: "false"` on the rendered Service when `externalMethod: PortList` is configured, signaling cozy-proxy to install per-port ingress filtering based on `Service.spec.ports`. Previously the annotation was always `"true"`, which kept cozy-proxy in whole-IP passthrough mode and made `PortList` non-functional. **Requires cozy-proxy v0.3.0 or later** (bundled with this chart). ([@mattia-eleuteri](https://github.com/mattia-eleuteri) in #). + +--- + +**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.3.0...v1.3.1 diff --git a/packages/apps/vm-instance/templates/service.yaml b/packages/apps/vm-instance/templates/service.yaml index b7e3a420..4607ffcf 100644 --- a/packages/apps/vm-instance/templates/service.yaml +++ b/packages/apps/vm-instance/templates/service.yaml @@ -9,7 +9,7 @@ metadata: {{- if .Values.external }} service.kubernetes.io/service-proxy-name: "cozy-proxy" annotations: - networking.cozystack.io/wholeIP: "true" + networking.cozystack.io/wholeIP: {{ ternary "true" "false" (eq .Values.externalMethod "WholeIP") | quote }} {{- end }} spec: type: {{ ternary "LoadBalancer" "ClusterIP" .Values.external }} From ed01319aa199272714444a21145cc04a38146f65 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Tue, 28 Apr 2026 03:18:25 +0300 Subject: [PATCH 083/130] feat(dashboard): replace UI with new cozystack-ui (keeping BFF) Replace old openapi-ui with new cozystack-ui that proxies API requests through BFF. Changes: - New UI image: 999669/cozystack-ui:latest@sha256:74e39ad4 - BFF container: kept unchanged for K8s API authentication - New UI includes TypeScript fixes and improved backup pages - nginx configured to proxy /api and /apis to localhost:64231 (BFF) Signed-off-by: IvanHunters --- packages/system/dashboard/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/system/dashboard/values.yaml b/packages/system/dashboard/values.yaml index a5188825..131d71e7 100644 --- a/packages/system/dashboard/values.yaml +++ b/packages/system/dashboard/values.yaml @@ -1,5 +1,5 @@ openapiUI: - image: ghcr.io/cozystack/cozystack/openapi-ui:v1.3.0@sha256:0fa79c373a62840a617ff1ca1b0e31931c13a6cf7b0bb0ff0dc191f047a465a3 + image: 999669/cozystack-ui:latest@sha256:74e39ad4f199fe5bc728793acbfb1d6d4fc76e32894521668a486306d498b71d openapiUIK8sBff: image: ghcr.io/cozystack/cozystack/openapi-ui-k8s-bff:v1.3.0@sha256:1b3ea6d4c7dbbe6a8def3b2807fffdfab2ac4afc39d7a846e57dd491fa168f92 tokenProxy: From ddcc1f2ad651fb47265ba623fe2b56b46c54039f Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Tue, 28 Apr 2026 03:02:59 +0300 Subject: [PATCH 084/130] feat(dashboard): update UI image to amd64 and remove old UI build - Update openapiUI image to amd64 build (sha256:9d8fc5c1...) - Remove old openapi-ui and openapi-ui-k8s-bff build targets from Makefile - Remove old UI Dockerfiles and related files - Add RBAC permissions for cozystack.io/applicationdefinitions to fix 403 errors The new cozystack-ui requires read access to applicationdefinitions API group for marketplace and application catalog functionality. Signed-off-by: IvanHunters --- packages/system/dashboard/Makefile | 34 +------- packages/system/dashboard/templates/rbac.yaml | 8 ++ packages/system/dashboard/templates/web.yaml | 78 ------------------- packages/system/dashboard/values.yaml | 4 +- 4 files changed, 10 insertions(+), 114 deletions(-) diff --git a/packages/system/dashboard/Makefile b/packages/system/dashboard/Makefile index a9f680a3..b136c864 100644 --- a/packages/system/dashboard/Makefile +++ b/packages/system/dashboard/Makefile @@ -5,7 +5,7 @@ include ../../../hack/common-envs.mk include ../../../hack/package.mk update: update-crd update-dockerfiles -image: image-openapi-ui image-openapi-ui-k8s-bff image-token-proxy update-tenant-text +image: image-token-proxy update-tenant-text update-dockerfiles: @@ -18,38 +18,6 @@ update-crd: rm -f crds/_helpers.tpl sed -i '/{{/d' crds/*.yml crds/*.yaml -image-openapi-ui: - docker buildx build images/openapi-ui \ - --provenance false \ - --builder=$(BUILDER) \ - --platform=linux/amd64 \ - --tag $(REGISTRY)/openapi-ui:$(call settag,$(TAG)) \ - --cache-from type=registry,ref=$(REGISTRY)/openapi-ui:latest \ - --cache-to type=inline \ - --metadata-file images/openapi-ui.json \ - --push=$(PUSH) \ - --label "org.opencontainers.image.source=https://github.com/cozystack/cozystack" \ - --load=$(LOAD) - IMAGE="$(REGISTRY)/openapi-ui:$(call settag,$(TAG))@$$(yq e '."containerimage.digest"' images/openapi-ui.json -r)" \ - yq -i '.openapiUI.image = strenv(IMAGE)' values.yaml - rm -f images/openapi-ui.json - -image-openapi-ui-k8s-bff: - docker buildx build images/openapi-ui-k8s-bff \ - --provenance false \ - --builder=$(BUILDER) \ - --platform=linux/amd64 \ - --tag $(REGISTRY)/openapi-ui-k8s-bff:$(call settag,$(TAG)) \ - --cache-from type=registry,ref=$(REGISTRY)/openapi-ui-k8s-bff:latest \ - --cache-to type=inline \ - --metadata-file images/openapi-ui-k8s-bff.json \ - --push=$(PUSH) \ - --label "org.opencontainers.image.source=https://github.com/cozystack/cozystack" \ - --load=$(LOAD) - IMAGE="$(REGISTRY)/openapi-ui-k8s-bff:$(call settag,$(TAG))@$$(yq e '."containerimage.digest"' images/openapi-ui-k8s-bff.json -r)" \ - yq -i '.openapiUIK8sBff.image = strenv(IMAGE)' values.yaml - rm -f images/openapi-ui-k8s-bff.json - image-token-proxy: docker buildx build images/token-proxy \ --provenance false \ diff --git a/packages/system/dashboard/templates/rbac.yaml b/packages/system/dashboard/templates/rbac.yaml index eb687a3e..c64e851f 100644 --- a/packages/system/dashboard/templates/rbac.yaml +++ b/packages/system/dashboard/templates/rbac.yaml @@ -4,6 +4,14 @@ kind: ClusterRole metadata: name: cozystack-dashboard-readonly rules: +- apiGroups: + - cozystack.io + resources: + - applicationdefinitions + verbs: + - get + - list + - watch - apiGroups: - dashboard.cozystack.io resources: diff --git a/packages/system/dashboard/templates/web.yaml b/packages/system/dashboard/templates/web.yaml index 7799afbf..1fd5e850 100644 --- a/packages/system/dashboard/templates/web.yaml +++ b/packages/system/dashboard/templates/web.yaml @@ -38,84 +38,6 @@ spec: topologyKey: kubernetes.io/hostname weight: 100 containers: - - env: - - name: BASE_API_GROUP - value: dashboard.cozystack.io - - name: BASE_API_VERSION - value: v1alpha1 - - name: BASE_NAMESPACE_FULL_PATH - value: "/apis/core.cozystack.io/v1alpha1/tenantnamespaces" - - name: BASE_NAVIGATION_RESOURCE_PLURAL - value: navigations - - name: BASE_NAVIGATION_RESOURCE_NAME - value: navigation - - name: BASE_FRONTEND_PREFIX - value: /openapi-ui - - name: BASE_FACTORY_NAMESPACED_API_KEY - value: base-factory-namespaced-api - - name: BASE_FACTORY_CLUSTERSCOPED_API_KEY - value: base-factory-clusterscoped-api - - name: BASE_FACTORY_NAMESPACED_BUILTIN_KEY - value: base-factory-namespaced-builtin - - name: BASE_FACTORY_CLUSTERSCOPED_BUILTIN_KEY - value: base-factory-clusterscoped-builtin - - name: BASE_NAMESPACE_FACTORY_KEY - value: base-factory-clusterscoped-builtin - - name: BASE_ALLOWED_AUTH_HEADERS - value: user-agent,accept,content-type,origin,referer,accept-encoding,cookie,authorization - - name: LOGGER - value: "true" - - name: LOGGER_WITH_HEADERS - value: "false" - - name: PORT - value: "64231" - image: {{ .Values.openapiUIK8sBff.image | quote }} - imagePullPolicy: IfNotPresent - livenessProbe: - failureThreshold: 3 - httpGet: - path: /healthcheck - port: 64231 - scheme: HTTP - initialDelaySeconds: 3 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 2 - startupProbe: - httpGet: - path: /healthcheck - port: 64231 - scheme: HTTP - failureThreshold: 30 - periodSeconds: 2 - name: bff - ports: - - containerPort: 64231 - name: http-bff - protocol: TCP - resources: - limits: - cpu: 1 - memory: 1Gi - requests: - cpu: 100m - ephemeral-storage: 50Mi - memory: 128Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - add: [] - drop: - - ALL - privileged: false - readOnlyRootFilesystem: false - runAsGroup: 0 - runAsNonRoot: true - runAsUser: 101 - seccompProfile: - type: RuntimeDefault - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - env: - name: BASEPREFIX value: /openapi-ui diff --git a/packages/system/dashboard/values.yaml b/packages/system/dashboard/values.yaml index 131d71e7..65233c34 100644 --- a/packages/system/dashboard/values.yaml +++ b/packages/system/dashboard/values.yaml @@ -1,6 +1,4 @@ openapiUI: - image: 999669/cozystack-ui:latest@sha256:74e39ad4f199fe5bc728793acbfb1d6d4fc76e32894521668a486306d498b71d -openapiUIK8sBff: - image: ghcr.io/cozystack/cozystack/openapi-ui-k8s-bff:v1.3.0@sha256:1b3ea6d4c7dbbe6a8def3b2807fffdfab2ac4afc39d7a846e57dd491fa168f92 + image: 999669/cozystack-ui:latest@sha256:9d8fc5c1a16b5d61e6e6bc9a02621c5d7681de960bb7054a5908a61ab4310799 tokenProxy: image: ghcr.io/cozystack/cozystack/token-proxy:v1.3.0@sha256:2e280991e07853ea48f97b0a42946afffa10d03d6a83d41099ed83e6ffc94fdc From 4de9c50e2f63a846a62660a15915a9636b2f71b4 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Tue, 28 Apr 2026 03:26:35 +0300 Subject: [PATCH 085/130] fix(dashboard): update UI image with VNC WebSocket fix Update to new image digest with dynamic WebSocket URL fix for VNC connections. VNC now uses window.location instead of hardcoded localhost:8001. Image: 999669/cozystack-ui:latest@sha256:ea4e832c2... Signed-off-by: IvanHunters --- packages/system/dashboard/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/system/dashboard/values.yaml b/packages/system/dashboard/values.yaml index 65233c34..dc0384b0 100644 --- a/packages/system/dashboard/values.yaml +++ b/packages/system/dashboard/values.yaml @@ -1,4 +1,4 @@ openapiUI: - image: 999669/cozystack-ui:latest@sha256:9d8fc5c1a16b5d61e6e6bc9a02621c5d7681de960bb7054a5908a61ab4310799 + image: 999669/cozystack-ui:latest@sha256:ea4e832c277ecf16dde99b4bd82ac5334d03d3c9cb0572fe896e2f128a6e6dc1 tokenProxy: image: ghcr.io/cozystack/cozystack/token-proxy:v1.3.0@sha256:2e280991e07853ea48f97b0a42946afffa10d03d6a83d41099ed83e6ffc94fdc From b0afc9a07c8a842915af1fa98ff1e7d17ccb39c8 Mon Sep 17 00:00:00 2001 From: mattia-eleuteri Date: Tue, 28 Apr 2026 08:37:13 +0200 Subject: [PATCH 086/130] [vm-instance] Add externalAllowICMP knob, drop in-PR changelog - Add `externalAllowICMP` value (default true) propagated as `networking.cozystack.io/allowICMP` annotation on the rendered Service when `externalMethod: PortList`. The cozy-proxy companion (released as part of cozystack/cozy-proxy#11 + #12) drops ICMP by default in port-filter mode, which breaks ping and PMTU discovery; defaulting the chart to "true" preserves user expectations while still allowing operators to opt out by setting `externalAllowICMP: false`. - Remove the v1.3.1.md changelog entry. Project convention is to add changelogs in a dedicated "docs: add changelog for vX.Y.Z" commit at release time, not as part of feature/fix PRs. Signed-off-by: mattia-eleuteri --- api/apps/v1alpha1/vminstance/types.go | 3 ++ docs/changelogs/v1.3.1.md | 11 ---- packages/apps/vm-instance/README.md | 51 ++++++++++--------- .../apps/vm-instance/templates/service.yaml | 3 ++ packages/apps/vm-instance/values.schema.json | 5 ++ packages/apps/vm-instance/values.yaml | 3 ++ .../vm-instance-rd/cozyrds/vm-instance.yaml | 4 +- 7 files changed, 42 insertions(+), 38 deletions(-) delete mode 100644 docs/changelogs/v1.3.1.md diff --git a/api/apps/v1alpha1/vminstance/types.go b/api/apps/v1alpha1/vminstance/types.go index 2bb059c6..9c48724a 100644 --- a/api/apps/v1alpha1/vminstance/types.go +++ b/api/apps/v1alpha1/vminstance/types.go @@ -26,6 +26,9 @@ type ConfigSpec struct { // Ports to forward from outside the cluster. // +kubebuilder:default:={22} ExternalPorts []int `json:"externalPorts,omitempty"` + // Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect. + // +kubebuilder:default:=true + ExternalAllowICMP bool `json:"externalAllowICMP"` // Requested running state of the VirtualMachineInstance // +kubebuilder:default:="Always" RunStrategy RunStrategy `json:"runStrategy"` diff --git a/docs/changelogs/v1.3.1.md b/docs/changelogs/v1.3.1.md deleted file mode 100644 index 7f9ecfab..00000000 --- a/docs/changelogs/v1.3.1.md +++ /dev/null @@ -1,11 +0,0 @@ - - -## Bug Fixes - -* **[vm-instance] Fix `externalMethod: PortList` not filtering ingress ports**: The `vm-instance` chart now sets `networking.cozystack.io/wholeIP: "false"` on the rendered Service when `externalMethod: PortList` is configured, signaling cozy-proxy to install per-port ingress filtering based on `Service.spec.ports`. Previously the annotation was always `"true"`, which kept cozy-proxy in whole-IP passthrough mode and made `PortList` non-functional. **Requires cozy-proxy v0.3.0 or later** (bundled with this chart). ([@mattia-eleuteri](https://github.com/mattia-eleuteri) in #). - ---- - -**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.3.0...v1.3.1 diff --git a/packages/apps/vm-instance/README.md b/packages/apps/vm-instance/README.md index a2b6603e..9d6a52b2 100644 --- a/packages/apps/vm-instance/README.md +++ b/packages/apps/vm-instance/README.md @@ -36,31 +36,32 @@ virtctl ssh @ ### Common parameters -| Name | Description | Type | Value | -| ------------------- | --------------------------------------------------------------------------------------------------------------------------------- | ---------- | ----------- | -| `external` | Enable external access from outside the cluster. | `bool` | `false` | -| `externalMethod` | Method to pass through traffic to the VM. | `string` | `PortList` | -| `externalPorts` | Ports to forward from outside the cluster. | `[]int` | `[22]` | -| `runStrategy` | Requested running state of the VirtualMachineInstance | `string` | `Always` | -| `instanceType` | Virtual Machine instance type. | `string` | `u1.medium` | -| `instanceProfile` | Virtual Machine preferences profile. | `string` | `ubuntu` | -| `disks` | List of disks to attach. | `[]object` | `[]` | -| `disks[i].name` | Disk name. | `string` | `""` | -| `disks[i].bus` | Disk bus type (e.g. "sata"). | `string` | `""` | -| `networks` | Networks to attach the VM to. | `[]object` | `[]` | -| `networks[i].name` | Network attachment name. | `string` | `""` | -| `subnets` | Deprecated: use networks instead. | `[]object` | `[]` | -| `subnets[i].name` | Network attachment name. | `string` | `""` | -| `gpus` | List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). | `[]object` | `[]` | -| `gpus[i].name` | The name of the GPU resource to attach. | `string` | `""` | -| `cpuModel` | Model specifies the CPU model inside the VMI. List of available models https://github.com/libvirt/libvirt/tree/master/src/cpu_map | `string` | `""` | -| `resources` | Resource configuration for the virtual machine. | `object` | `{}` | -| `resources.cpu` | Number of CPU cores allocated. | `quantity` | `""` | -| `resources.memory` | Amount of memory allocated. | `quantity` | `""` | -| `resources.sockets` | Number of CPU sockets (vCPU topology). | `quantity` | `""` | -| `sshKeys` | List of SSH public keys for authentication. | `[]string` | `[]` | -| `cloudInit` | Cloud-init user data. | `string` | `""` | -| `cloudInitSeed` | Seed string to generate SMBIOS UUID for the VM. | `string` | `""` | +| Name | Description | Type | Value | +| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ----------- | +| `external` | Enable external access from outside the cluster. | `bool` | `false` | +| `externalMethod` | Method to pass through traffic to the VM. | `string` | `PortList` | +| `externalPorts` | Ports to forward from outside the cluster. | `[]int` | `[22]` | +| `externalAllowICMP` | Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect. | `bool` | `true` | +| `runStrategy` | Requested running state of the VirtualMachineInstance | `string` | `Always` | +| `instanceType` | Virtual Machine instance type. | `string` | `u1.medium` | +| `instanceProfile` | Virtual Machine preferences profile. | `string` | `ubuntu` | +| `disks` | List of disks to attach. | `[]object` | `[]` | +| `disks[i].name` | Disk name. | `string` | `""` | +| `disks[i].bus` | Disk bus type (e.g. "sata"). | `string` | `""` | +| `networks` | Networks to attach the VM to. | `[]object` | `[]` | +| `networks[i].name` | Network attachment name. | `string` | `""` | +| `subnets` | Deprecated: use networks instead. | `[]object` | `[]` | +| `subnets[i].name` | Network attachment name. | `string` | `""` | +| `gpus` | List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). | `[]object` | `[]` | +| `gpus[i].name` | The name of the GPU resource to attach. | `string` | `""` | +| `cpuModel` | Model specifies the CPU model inside the VMI. List of available models https://github.com/libvirt/libvirt/tree/master/src/cpu_map | `string` | `""` | +| `resources` | Resource configuration for the virtual machine. | `object` | `{}` | +| `resources.cpu` | Number of CPU cores allocated. | `quantity` | `""` | +| `resources.memory` | Amount of memory allocated. | `quantity` | `""` | +| `resources.sockets` | Number of CPU sockets (vCPU topology). | `quantity` | `""` | +| `sshKeys` | List of SSH public keys for authentication. | `[]string` | `[]` | +| `cloudInit` | Cloud-init user data. | `string` | `""` | +| `cloudInitSeed` | Seed string to generate SMBIOS UUID for the VM. | `string` | `""` | ## U Series diff --git a/packages/apps/vm-instance/templates/service.yaml b/packages/apps/vm-instance/templates/service.yaml index 4607ffcf..cfeffa81 100644 --- a/packages/apps/vm-instance/templates/service.yaml +++ b/packages/apps/vm-instance/templates/service.yaml @@ -10,6 +10,9 @@ metadata: service.kubernetes.io/service-proxy-name: "cozy-proxy" annotations: networking.cozystack.io/wholeIP: {{ ternary "true" "false" (eq .Values.externalMethod "WholeIP") | quote }} + {{- if eq .Values.externalMethod "PortList" }} + networking.cozystack.io/allowICMP: {{ ternary "true" "false" (ne .Values.externalAllowICMP false) | quote }} + {{- end }} {{- end }} spec: type: {{ ternary "LoadBalancer" "ClusterIP" .Values.external }} diff --git a/packages/apps/vm-instance/values.schema.json b/packages/apps/vm-instance/values.schema.json index 34f7f634..01bd30a9 100644 --- a/packages/apps/vm-instance/values.schema.json +++ b/packages/apps/vm-instance/values.schema.json @@ -26,6 +26,11 @@ "type": "integer" } }, + "externalAllowICMP": { + "description": "Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect.", + "type": "boolean", + "default": true + }, "runStrategy": { "description": "Requested running state of the VirtualMachineInstance", "type": "string", diff --git a/packages/apps/vm-instance/values.yaml b/packages/apps/vm-instance/values.yaml index f07b75d3..92e399c2 100644 --- a/packages/apps/vm-instance/values.yaml +++ b/packages/apps/vm-instance/values.yaml @@ -31,6 +31,9 @@ externalMethod: PortList externalPorts: - 22 +## @param {bool} externalAllowICMP - Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect. +externalAllowICMP: true + ## @enum {string} RunStrategy - Requested running state of the VirtualMachineInstance ## @value Always - VMI should always be running ## @value Halted - VMI should never be running diff --git a/packages/system/vm-instance-rd/cozyrds/vm-instance.yaml b/packages/system/vm-instance-rd/cozyrds/vm-instance.yaml index e93fa86b..e56bd9dd 100644 --- a/packages/system/vm-instance-rd/cozyrds/vm-instance.yaml +++ b/packages/system/vm-instance-rd/cozyrds/vm-instance.yaml @@ -8,7 +8,7 @@ spec: singular: vminstance plural: vminstances openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"externalMethod":{"description":"Method to pass through traffic to the VM.","type":"string","default":"PortList","enum":["PortList","WholeIP"]},"externalPorts":{"description":"Ports to forward from outside the cluster.","type":"array","default":[22],"items":{"type":"integer"}},"runStrategy":{"description":"Requested running state of the VirtualMachineInstance","type":"string","default":"Always","enum":["Always","Halted","Manual","RerunOnFailure","Once"]},"instanceType":{"description":"Virtual Machine instance type.","type":"string","default":"u1.medium"},"instanceProfile":{"description":"Virtual Machine preferences profile.","type":"string","default":"ubuntu","enum":["alpine","centos.7","centos.7.desktop","centos.stream10","centos.stream10.desktop","centos.stream8","centos.stream8.desktop","centos.stream8.dpdk","centos.stream9","centos.stream9.desktop","centos.stream9.dpdk","cirros","fedora","fedora.arm64","opensuse.leap","opensuse.tumbleweed","rhel.10","rhel.10.arm64","rhel.7","rhel.7.desktop","rhel.8","rhel.8.desktop","rhel.8.dpdk","rhel.9","rhel.9.arm64","rhel.9.desktop","rhel.9.dpdk","rhel.9.realtime","sles","ubuntu","windows.10","windows.10.virtio","windows.11","windows.11.virtio","windows.2k16","windows.2k16.virtio","windows.2k19","windows.2k19.virtio","windows.2k22","windows.2k22.virtio","windows.2k25","windows.2k25.virtio",""]},"disks":{"description":"List of disks to attach.","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"bus":{"description":"Disk bus type (e.g. \"sata\").","type":"string"},"name":{"description":"Disk name.","type":"string"}}}},"networks":{"description":"Networks to attach the VM to.","type":"array","default":[],"items":{"type":"object","properties":{"name":{"description":"Network attachment name.","type":"string"}}}},"subnets":{"description":"Deprecated: use networks instead.","type":"array","default":[],"items":{"type":"object","properties":{"name":{"description":"Network attachment name.","type":"string"}}}},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"name":{"description":"The name of the GPU resource to attach.","type":"string"}}}},"cpuModel":{"description":"Model specifies the CPU model inside the VMI. List of available models https://github.com/libvirt/libvirt/tree/master/src/cpu_map","type":"string","default":""},"resources":{"description":"Resource configuration for the virtual machine.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"sockets":{"description":"Number of CPU sockets (vCPU topology).","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"sshKeys":{"description":"List of SSH public keys for authentication.","type":"array","default":[],"items":{"type":"string"}},"cloudInit":{"description":"Cloud-init user data.","type":"string","default":""},"cloudInitSeed":{"description":"Seed string to generate SMBIOS UUID for the VM.","type":"string","default":""}}} + {"title":"Chart Values","type":"object","properties":{"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"externalMethod":{"description":"Method to pass through traffic to the VM.","type":"string","default":"PortList","enum":["PortList","WholeIP"]},"externalPorts":{"description":"Ports to forward from outside the cluster.","type":"array","default":[22],"items":{"type":"integer"}},"externalAllowICMP":{"description":"Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect.","type":"boolean","default":true},"runStrategy":{"description":"Requested running state of the VirtualMachineInstance","type":"string","default":"Always","enum":["Always","Halted","Manual","RerunOnFailure","Once"]},"instanceType":{"description":"Virtual Machine instance type.","type":"string","default":"u1.medium"},"instanceProfile":{"description":"Virtual Machine preferences profile.","type":"string","default":"ubuntu","enum":["alpine","centos.7","centos.7.desktop","centos.stream10","centos.stream10.desktop","centos.stream8","centos.stream8.desktop","centos.stream8.dpdk","centos.stream9","centos.stream9.desktop","centos.stream9.dpdk","cirros","fedora","fedora.arm64","opensuse.leap","opensuse.tumbleweed","rhel.10","rhel.10.arm64","rhel.7","rhel.7.desktop","rhel.8","rhel.8.desktop","rhel.8.dpdk","rhel.9","rhel.9.arm64","rhel.9.desktop","rhel.9.dpdk","rhel.9.realtime","sles","ubuntu","windows.10","windows.10.virtio","windows.11","windows.11.virtio","windows.2k16","windows.2k16.virtio","windows.2k19","windows.2k19.virtio","windows.2k22","windows.2k22.virtio","windows.2k25","windows.2k25.virtio",""]},"disks":{"description":"List of disks to attach.","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"bus":{"description":"Disk bus type (e.g. \"sata\").","type":"string"},"name":{"description":"Disk name.","type":"string"}}}},"networks":{"description":"Networks to attach the VM to.","type":"array","default":[],"items":{"type":"object","properties":{"name":{"description":"Network attachment name.","type":"string"}}}},"subnets":{"description":"Deprecated: use networks instead.","type":"array","default":[],"items":{"type":"object","properties":{"name":{"description":"Network attachment name.","type":"string"}}}},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"name":{"description":"The name of the GPU resource to attach.","type":"string"}}}},"cpuModel":{"description":"Model specifies the CPU model inside the VMI. List of available models https://github.com/libvirt/libvirt/tree/master/src/cpu_map","type":"string","default":""},"resources":{"description":"Resource configuration for the virtual machine.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"sockets":{"description":"Number of CPU sockets (vCPU topology).","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"sshKeys":{"description":"List of SSH public keys for authentication.","type":"array","default":[],"items":{"type":"string"}},"cloudInit":{"description":"Cloud-init user data.","type":"string","default":""},"cloudInitSeed":{"description":"Seed string to generate SMBIOS UUID for the VM.","type":"string","default":""}}} release: prefix: vm-instance- labels: @@ -26,7 +26,7 @@ spec: tags: - compute icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODdfMzQ1NCkiLz4KPGcgY2xpcC1wYXRoPSJ1cmwoI2NsaXAwXzY4N18zNDU0KSI+CjxwYXRoIGQ9Ik04OS41MDM5IDExMS43MDdINTQuNDk3QzU0LjE3MjcgMTExLjcwNyA1NC4wMTA4IDExMS4yMjEgNTQuMzM0OSAxMTEuMDU5TDU3LjI1MjIgMTA4Ljk1MkM2MC4zMzE0IDEwNi42ODMgNjEuOTUyMiAxMDIuNjMxIDYwLjk3OTcgOTguNzQxMkg4My4wMjFDODIuMDQ4NSAxMDIuNjMxIDgzLjY2OTMgMTA2LjY4MyA4Ni43NDg1IDEwOC45NTJMODkuNjY1OCAxMTEuMDU5Qzg5Ljk5IDExMS4yMjEgODkuODI3OSAxMTEuNzA3IDg5LjUwMzkgMTExLjcwN1oiIGZpbGw9IiNCMEI2QkIiLz4KPHBhdGggZD0iTTExMy4zMjggOTguNzQxSDMwLjY3MjVDMjcuNTkzMSA5OC43NDEgMjUgOTYuMTQ4IDI1IDkzLjA2ODdWMzMuMTAzMkMyNSAzMC4wMjM5IDI3LjU5MzEgMjcuNDMwNyAzMC42NzI1IDI3LjQzMDdIMTEzLjMyOEMxMTYuNDA3IDI3LjQzMDcgMTE5IDMwLjAyMzcgMTE5IDMzLjEwMzJWOTMuMDY4N0MxMTkgOTYuMTQ4IDExNi40MDcgOTguNzQxIDExMy4zMjggOTguNzQxWiIgZmlsbD0iI0U4RURFRSIvPgo8cGF0aCBkPSJNMTE5IDg0LjE1NDlIMjVWMzMuMTAzMkMyNSAzMC4wMjM5IDI3LjU5MzEgMjcuNDMwNyAzMC42NzI1IDI3LjQzMDdIMTEzLjMyOEMxMTYuNDA3IDI3LjQzMDcgMTE5IDMwLjAyMzcgMTE5IDMzLjEwMzJMMTE5IDg0LjE1NDlaIiBmaWxsPSIjMDBCM0ZGIi8+CjxwYXRoIGQ9Ik05MC42Mzc0IDExNi41NjlINTMuMzYxNkM1Mi4wNjUxIDExNi41NjkgNTAuOTMwNyAxMTUuNDM1IDUwLjkzMDcgMTE0LjEzOEM1MC45MzA3IDExMi44NDEgNTIuMDY1MSAxMTEuNzA3IDUzLjM2MTYgMTExLjcwN0g5MC42Mzc0QzkxLjkzMzkgMTExLjcwNyA5My4wNjg0IDExMi44NDEgOTMuMDY4NCAxMTQuMTM4QzkzLjA2ODQgMTE1LjQzNSA5MS45MzM5IDExNi41NjkgOTAuNjM3NCAxMTYuNTY5WiIgZmlsbD0iI0U4RURFRSIvPgo8L2c+CjxwYXRoIGQ9Ik03Mi41Mjc1IDUzLjgzNjdDNzIuNDQzMSA1My44MzUxIDcyLjM2MDUgNTMuODEyMiA3Mi4yODczIDUzLjc3MDFMNTYuNDY5OSA0NC43OTM0QzU2LjM5ODMgNDQuNzUxOSA1Ni4zMzg4IDQ0LjY5MjMgNTYuMjk3MyA0NC42MjA3QzU2LjI1NTkgNDQuNTQ5IDU2LjIzMzggNDQuNDY3OCA1Ni4yMzM0IDQ0LjM4NUM1Ni4yMzM0IDQ0LjIxNjkgNTYuMzI1OCA0NC4wNjE3IDU2LjQ2OTkgNDMuOTc4NUw3Mi4xOTEyIDM1LjA2MDlDNzIuMjYzNyAzNS4wMjEgNzIuMzQ1IDM1IDcyLjQyNzcgMzVDNzIuNTEwNSAzNSA3Mi41OTE4IDM1LjAyMSA3Mi42NjQzIDM1LjA2MDlMODguNDg3MiA0NC4wMzk1Qzg4LjU1OTEgNDQuMDgwMSA4OC42MTg4IDQ0LjEzOTIgODguNjYgNDQuMjEwN0M4OC43MDEzIDQ0LjI4MjIgODguNzIyNyA0NC4zNjM1IDg4LjcyMTkgNDQuNDQ2Qzg4LjcyMjUgNDQuNTI4NSA4OC43MDEgNDQuNjA5NyA4OC42NTk4IDQ0LjY4MTJDODguNjE4NSA0NC43NTI2IDg4LjU1ODkgNDQuODExOCA4OC40ODcyIDQ0Ljg1MjVMNzIuNzcxNCA1My43NjgzQzcyLjY5NzIgNTMuODExNCA3Mi42MTMzIDUzLjgzNDkgNzIuNTI3NSA1My44MzY3IiBmaWxsPSJ3aGl0ZSIvPgo8cGF0aCBvcGFjaXR5PSIwLjciIGQ9Ik03MC4yNTUzIDc1LjY1MTdDNzAuMTcxIDc1LjY1MzUgNzAuMDg3OCA3NS42MzE3IDcwLjAxNTEgNzUuNTg4OEw1NC4yNDU4IDY2LjY0MTdDNTQuMTcxNSA2Ni42MDI0IDU0LjEwOTUgNjYuNTQzNiA1NC4wNjYxIDY2LjQ3MTZDNTQuMDIyOCA2Ni4zOTk3IDU0IDY2LjMxNzMgNTQgNjYuMjMzM1Y0OC4yNzhDNTQgNDguMTA4IDU0LjA5MjQgNDcuOTU0NiA1NC4yNDM5IDQ3Ljg2OTZDNTQuMzE3MiA0Ny44MjcxIDU0LjQwMDQgNDcuODA0NyA1NC40ODUxIDQ3LjgwNDdDNTQuNTY5NyA0Ny44MDQ3IDU0LjY1MjkgNDcuODI3MSA1NC43MjYyIDQ3Ljg2OTZMNzAuNDkzNyA1Ni44MTMxQzcwLjU2NDIgNTYuODU2NSA3MC42MjI1IDU2LjkxNyA3MC42NjMyIDU2Ljk4OTFDNzAuNzAzOSA1Ny4wNjEyIDcwLjcyNTcgNTcuMTQyNCA3MC43MjY1IDU3LjIyNTFWNzUuMTgwNUM3MC43MjU5IDc1LjI2MjggNzAuNzA0MiA3NS4zNDM2IDcwLjY2MzUgNzUuNDE1MUM3MC42MjI3IDc1LjQ4NjYgNzAuNTY0MiA3NS41NDY0IDcwLjQ5MzcgNzUuNTg4OEM3MC40MjA2IDc1LjYyOTEgNzAuMzM4NyA3NS42NTA3IDcwLjI1NTMgNzUuNjUxNyIgZmlsbD0id2hpdGUiLz4KPHBhdGggb3BhY2l0eT0iMC40IiBkPSJNNzQuNzE5OCA3NS42NTExQzc0LjYzMzMgNzUuNjUxMiA3NC41NDgyIDc1LjYyOTYgNzQuNDcyMiA3NS41ODgzQzc0LjQwMTYgNzUuNTQ2MSA3NC4zNDMyIDc1LjQ4NjIgNzQuMzAyNyA3NS40MTQ3Qzc0LjI2MjMgNzUuMzQzMSA3NC4yNDExIDc1LjI2MjIgNzQuMjQxMiA3NS4xOFY1Ny4zMzczQzc0LjI0MTIgNTcuMTcxIDc0LjMzMzYgNTcuMDE1OCA3NC40NzIyIDU2LjkyOUw5MC4yMzk3IDQ3Ljk4NTVDOTAuMzExOSA0Ny45NDM4IDkwLjM5MzggNDcuOTIxOSA5MC40NzcxIDQ3LjkyMTlDOTAuNTYwNSA0Ny45MjE5IDkwLjY0MjQgNDcuOTQzOCA5MC43MTQ2IDQ3Ljk4NTVDOTAuNzg3NiA0OC4wMjU1IDkwLjg0ODUgNDguMDg0MiA5MC44OTExIDQ4LjE1NTdDOTAuOTMzNyA0OC4yMjcyIDkwLjk1NjMgNDguMzA4OCA5MC45NTY2IDQ4LjM5MlY2Ni4yMzI4QzkwLjk1NyA2Ni4zMTY0IDkwLjkzNDcgNjYuMzk4NSA5MC44OTIxIDY2LjQ3MDRDOTAuODQ5NSA2Ni41NDI0IDkwLjc4ODEgNjYuNjAxNCA5MC43MTQ2IDY2LjY0MTFMNzQuOTUyNiA3NS41ODgzQzc0Ljg4MjUgNzUuNjMwNyA3NC44MDE4IDc1LjY1MjUgNzQuNzE5OCA3NS42NTExIiBmaWxsPSJ3aGl0ZSIvPgo8ZGVmcz4KPGxpbmVhckdyYWRpZW50IGlkPSJwYWludDBfbGluZWFyXzY4N18zNDU0IiB4MT0iMTYxIiB5MT0iMTgwIiB4Mj0iMy41OTI4NGUtMDciIHkyPSI0Ljk5OTk4IiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSI+CjxzdG9wLz4KPHN0b3Agb2Zmc2V0PSIxIiBzdG9wLWNvbG9yPSIjNTk1NjU2Ii8+CjwvbGluZWFyR3JhZGllbnQ+CjxjbGlwUGF0aCBpZD0iY2xpcDBfNjg3XzM0NTQiPgo8cmVjdCB3aWR0aD0iOTQiIGhlaWdodD0iOTQiIGZpbGw9IndoaXRlIiB0cmFuc2Zvcm09InRyYW5zbGF0ZSgyNSAyNSkiLz4KPC9jbGlwUGF0aD4KPC9kZWZzPgo8L3N2Zz4K - keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "external"], ["spec", "externalMethod"], ["spec", "externalPorts"], ["spec", "runStrategy"], ["spec", "instanceType"], ["spec", "instanceProfile"], ["spec", "disks"], ["spec", "networks"], ["spec", "subnets"], ["spec", "gpus"], ["spec", "cpuModel"], ["spec", "resources"], ["spec", "sshKeys"], ["spec", "cloudInit"], ["spec", "cloudInitSeed"]] + keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "external"], ["spec", "externalMethod"], ["spec", "externalPorts"], ["spec", "externalAllowICMP"], ["spec", "runStrategy"], ["spec", "instanceType"], ["spec", "instanceProfile"], ["spec", "disks"], ["spec", "networks"], ["spec", "subnets"], ["spec", "gpus"], ["spec", "cpuModel"], ["spec", "resources"], ["spec", "sshKeys"], ["spec", "cloudInit"], ["spec", "cloudInitSeed"]] secrets: exclude: [] include: [] From bb51c88f7817c1d862fdb7947deef7798f717d82 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 11:18:59 +0300 Subject: [PATCH 087/130] fix(monitoring): remove unused namespace variable from GPU efficiency dashboard Address review feedback from coderabbitai on dashboards/gpu/gpu-efficiency.json:839 Signed-off-by: Arsolitt --- dashboards/gpu/gpu-efficiency.json | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/dashboards/gpu/gpu-efficiency.json b/dashboards/gpu/gpu-efficiency.json index f351c964..28a24568 100644 --- a/dashboards/gpu/gpu-efficiency.json +++ b/dashboards/gpu/gpu-efficiency.json @@ -812,30 +812,6 @@ "auto": false, "auto_min": "10s", "auto_count": 30 - }, - { - "type": "query", - "name": "namespace", - "label": "Namespace", - "skipUrlSync": false, - "query": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "multi": true, - "allowCustomValue": true, - "refresh": 2, - "sort": 1, - "includeAll": true, - "auto": false, - "auto_min": "10s", - "auto_count": 30 } ] }, From 452bff45675c55e0c8e01d290dfbb3b1e7f59196 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 11:19:56 +0300 Subject: [PATCH 088/130] fix(monitoring): remove unused namespace variable from GPU performance dashboard Address review feedback from coderabbitai on dashboards/gpu/gpu-performance.json:277 Signed-off-by: Arsolitt --- dashboards/gpu/gpu-performance.json | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/dashboards/gpu/gpu-performance.json b/dashboards/gpu/gpu-performance.json index f9951577..aed9c69a 100644 --- a/dashboards/gpu/gpu-performance.json +++ b/dashboards/gpu/gpu-performance.json @@ -950,30 +950,6 @@ "auto": false, "auto_min": "10s", "auto_count": 30 - }, - { - "type": "query", - "name": "namespace", - "label": "Namespace", - "skipUrlSync": false, - "query": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "multi": true, - "allowCustomValue": true, - "refresh": 2, - "sort": 1, - "includeAll": true, - "auto": false, - "auto_min": "10s", - "auto_count": 30 } ] }, From 4c697982b284a9d1f1b47d9cbbdbeab6314d253d Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 11:20:19 +0300 Subject: [PATCH 089/130] fix(monitoring): use Hostname label in GPU tenants dashboard legends Address review feedback from coderabbitai and gemini-code-assist on dashboards/gpu/gpu-tenants.json:532 Signed-off-by: Arsolitt --- dashboards/gpu/gpu-tenants.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dashboards/gpu/gpu-tenants.json b/dashboards/gpu/gpu-tenants.json index e2c400a8..6194a15e 100644 --- a/dashboards/gpu/gpu-tenants.json +++ b/dashboards/gpu/gpu-tenants.json @@ -529,7 +529,7 @@ "targets": [ { "expr": "node:gpu_util:avg", - "legendFormat": "{{node}}", + "legendFormat": "{{Hostname}}", "refId": "A" } ], @@ -585,7 +585,7 @@ "targets": [ { "expr": "node:tensor_active:avg * 100", - "legendFormat": "{{node}}", + "legendFormat": "{{Hostname}}", "refId": "A" } ], @@ -654,7 +654,7 @@ "targets": [ { "expr": "node:power_watts:sum", - "legendFormat": "{{node}}", + "legendFormat": "{{Hostname}}", "refId": "A" } ], @@ -988,7 +988,7 @@ "targets": [ { "expr": "sum_over_time(node:power_watts:sum[24h:1m]) / 60 / 1000", - "legendFormat": "{{node}}", + "legendFormat": "{{Hostname}}", "refId": "A" } ], From bbf338a57d4dca0286bef4dab991ccbce6a2caa2 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 11:20:41 +0300 Subject: [PATCH 090/130] fix(gpu-operator): fail fast on missing artifacts in driver-compat example Address review feedback from coderabbitai on packages/system/gpu-operator/examples/nvidia-driver-compat.yaml:77 Signed-off-by: Arsolitt --- .../examples/nvidia-driver-compat.yaml | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml b/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml index 7706b221..fc9d5981 100644 --- a/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml +++ b/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml @@ -61,16 +61,21 @@ spec: mkdir -p "$DST/bin" - if [ -f "$GLIBC_LIB/libnvidia-ml.so.1" ]; then - cp -f "$GLIBC_LIB/libnvidia-ml.so.1" "$DST/libnvidia-ml.so.1" - echo "Copied libnvidia-ml.so.1" - fi + [ -f "$GLIBC_LIB/libnvidia-ml.so.1" ] || { + echo "missing $GLIBC_LIB/libnvidia-ml.so.1" >&2 + exit 1 + } + [ -f "$SRC_BIN/nvidia-smi" ] || { + echo "missing $SRC_BIN/nvidia-smi" >&2 + exit 1 + } - if [ -f "$SRC_BIN/nvidia-smi" ]; then - cp -f "$SRC_BIN/nvidia-smi" "$DST/bin/nvidia-smi" - chmod +x "$DST/bin/nvidia-smi" - echo "Copied nvidia-smi" - fi + cp -f "$GLIBC_LIB/libnvidia-ml.so.1" "$DST/libnvidia-ml.so.1" + echo "Copied libnvidia-ml.so.1" + + cp -f "$SRC_BIN/nvidia-smi" "$DST/bin/nvidia-smi" + chmod +x "$DST/bin/nvidia-smi" + echo "Copied nvidia-smi" mkdir -p /host/run/nvidia/validations touch /host/run/nvidia/validations/.driver-ctr-ready From 5718740ae3edddd9112e483426ebfebba423ed92 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 11:20:57 +0300 Subject: [PATCH 091/130] docs(gpu-operator): correct gpu-quotas dashboard dependencies in README Address review feedback from coderabbitai on packages/system/gpu-operator/examples/README.md:85 Signed-off-by: Arsolitt --- packages/system/gpu-operator/examples/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/system/gpu-operator/examples/README.md b/packages/system/gpu-operator/examples/README.md index f9094549..b6f170c8 100644 --- a/packages/system/gpu-operator/examples/README.md +++ b/packages/system/gpu-operator/examples/README.md @@ -81,7 +81,7 @@ What each dashboard needs on top of the upstream DCGM Exporter | `gpu-performance` | Per-node, per-GPU deep dive | `DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION` | | `gpu-efficiency` | Per-workload util vs tensor active | `DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION` (via `gpu:*_throttle_fraction:rate5m` recording rules) | | `gpu-fleet` | Cluster-wide admin inventory | `DCGM_FI_DEV_POWER_MGMT_LIMIT` (for the TDP vs draw panel) | -| `gpu-quotas` | Kube-quota vs live usage | `kube_pod_container_resource_requests`, `kube_pod_status_phase`, `DCGM_FI_DEV_GPU_UTIL` (via `namespace:gpu_count:allocated` / `cluster:gpu_count:*` recording rules) | +| `gpu-quotas` | Kube-quota vs live usage | `kube_pod_container_resource_requests`, `kube_pod_status_phase`, `kube_node_status_allocatable` (via `namespace:gpu_count:allocated` / `cluster:gpu_count:*` recording rules) | | `gpu-tenants` | Per-namespace tenant view | nothing (works on default counters) | `DCGM_FI_PROF_PIPE_TENSOR_ACTIVE` and `DCGM_FI_PROF_GR_ENGINE_ACTIVE` From aba5ae3fcd0f4f4fdceae05f24e9371550076731 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 11:21:12 +0300 Subject: [PATCH 092/130] fix(monitoring): prevent many-to-many match in util-per-watt recording rule Address review feedback from coderabbitai on packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml:119 Signed-off-by: Arsolitt --- .../alerts/gpu-recording.rules.yaml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml index b046edf5..5e0f9ea0 100644 --- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml +++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml @@ -109,13 +109,15 @@ spec: # to GPU-identifying labels. - record: gpu:util_per_watt:avg5m expr: | - avg by (Hostname, gpu, UUID) ( + max by (Hostname, gpu, UUID) ( avg_over_time(DCGM_FI_DEV_GPU_UTIL[5m]) - / on (Hostname, gpu, UUID) - clamp_min( - avg_over_time(DCGM_FI_DEV_POWER_USAGE[5m]), - 1 - ) + ) + / on (Hostname, gpu, UUID) + clamp_min( + max by (Hostname, gpu, UUID) ( + avg_over_time(DCGM_FI_DEV_POWER_USAGE[5m]) + ), + 1 ) # Fraction of time power-throttled (TDP cap) — 1.0 = fully throttled. From 8e6266703da78b28109ec34ed6fd030612dc8c68 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 11:43:30 +0300 Subject: [PATCH 093/130] Revert "chore: ignore CLAUDE.local.md" This reverts commit 11f7d3589b3284fca96d0df21a02a5e967dadce7. Signed-off-by: Arsolitt --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 64470222..61003de5 100644 --- a/.gitignore +++ b/.gitignore @@ -84,4 +84,3 @@ tmp/ # build revision marker (generated by make image-packages) packages/core/platform/.build-revision .claude/ -CLAUDE.local.md From 31de9989f62b2f286fc3e6bf58ca85b1b96e54ad Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 11:44:04 +0300 Subject: [PATCH 094/130] fix(gpu-operator): add DCGM_FI_DRIVER_VERSION to custom metrics CSV Signed-off-by: Arsolitt --- packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml b/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml index 8e57788b..5b82fd44 100644 --- a/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml +++ b/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml @@ -16,6 +16,9 @@ data: # If line starts with a '#' it is considered a comment # DCGM FIELD, Prometheus metric type, help message + # Identity + DCGM_FI_DRIVER_VERSION, label, Driver version. + # Clocks DCGM_FI_DEV_SM_CLOCK, gauge, SM clock frequency (in MHz). DCGM_FI_DEV_MEM_CLOCK, gauge, Memory clock frequency (in MHz). From cacd3714bd24d7a3ec7182b8ec7b1057041fcf0a Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 11:44:08 +0300 Subject: [PATCH 095/130] fix(monitoring): include phase label in GPU limits query for consistency Signed-off-by: Arsolitt --- dashboards/gpu/gpu-quotas.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json index ce1e387e..62928b56 100644 --- a/dashboards/gpu/gpu-quotas.json +++ b/dashboards/gpu/gpu-quotas.json @@ -430,7 +430,7 @@ "refId": "requested" }, { - "expr": "kube_pod_container_resource_limits{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left() (kube_pod_status_phase{phase=~\"Running|Pending\"} == 1)", + "expr": "kube_pod_container_resource_limits{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left(phase) (kube_pod_status_phase{phase=~\"Running|Pending\"} == 1)", "instant": true, "range": false, "format": "table", From b0784c0d33d55fefeaaf60cdeabca4bebec1c069 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 11:44:11 +0300 Subject: [PATCH 096/130] fix(monitoring): generalize GPU temperature description in fleet dashboard Signed-off-by: Arsolitt --- dashboards/gpu/gpu-fleet.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboards/gpu/gpu-fleet.json b/dashboards/gpu/gpu-fleet.json index a7d3f085..60c12bfb 100644 --- a/dashboards/gpu/gpu-fleet.json +++ b/dashboards/gpu/gpu-fleet.json @@ -860,7 +860,7 @@ } ], "title": "Max GPU temperature", - "description": "Hottest GPU in the fleet right now. A10 sustained target \u003c83°C.", + "description": "Hottest GPU in the fleet right now. Adjust thresholds to match your GPU model specifications.", "transparent": false, "datasource": { "type": "prometheus", From ed6f9bbd1da5d0daeea5fbe38e5aa42e3a528496 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 12:02:18 +0300 Subject: [PATCH 097/130] fix(monitoring): regenerate gpu-quotas dashboard from SDK Align dashboard JSON with the SDK source of truth. Signed-off-by: Arsolitt --- dashboards/gpu/gpu-quotas.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json index 62928b56..ce1e387e 100644 --- a/dashboards/gpu/gpu-quotas.json +++ b/dashboards/gpu/gpu-quotas.json @@ -430,7 +430,7 @@ "refId": "requested" }, { - "expr": "kube_pod_container_resource_limits{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left(phase) (kube_pod_status_phase{phase=~\"Running|Pending\"} == 1)", + "expr": "kube_pod_container_resource_limits{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left() (kube_pod_status_phase{phase=~\"Running|Pending\"} == 1)", "instant": true, "range": false, "format": "table", From 84f506116f452889011c9d31c0cd48628265d448 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 12:07:31 +0300 Subject: [PATCH 098/130] docs(gpu-operator): clarify violation counter unit ambiguity in DCGM CSV Signed-off-by: Arsolitt --- .../gpu-operator/examples/dcgm-custom-metrics.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml b/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml index 5b82fd44..ef2b0b88 100644 --- a/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml +++ b/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml @@ -66,12 +66,12 @@ data: DCGM_FI_DEV_ROW_REMAP_FAILURE, gauge, Whether remapping of rows has failed. # Throttle / violation counters (crucial for SLA and tenant monitoring) - DCGM_FI_DEV_POWER_VIOLATION, counter, Throttling duration due to power constraints (in us). - DCGM_FI_DEV_THERMAL_VIOLATION, counter, Throttling duration due to thermal constraints (in us). - DCGM_FI_DEV_SYNC_BOOST_VIOLATION, counter, Throttling duration due to sync-boost constraints (in us). - DCGM_FI_DEV_BOARD_LIMIT_VIOLATION, counter, Throttling duration due to board limit constraints (in us). - DCGM_FI_DEV_LOW_UTIL_VIOLATION, counter, Throttling duration due to low utilization (in us). - DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (in us). + DCGM_FI_DEV_POWER_VIOLATION, counter, Throttling duration due to power constraints (us per docs; ns on DCGM 3.x). + DCGM_FI_DEV_THERMAL_VIOLATION, counter, Throttling duration due to thermal constraints (us per docs; ns on DCGM 3.x). + DCGM_FI_DEV_SYNC_BOOST_VIOLATION, counter, Throttling duration due to sync-boost constraints (us per docs; ns on DCGM 3.x). + DCGM_FI_DEV_BOARD_LIMIT_VIOLATION, counter, Throttling duration due to board limit constraints (us per docs; ns on DCGM 3.x). + DCGM_FI_DEV_LOW_UTIL_VIOLATION, counter, Throttling duration due to low utilization (us per docs; ns on DCGM 3.x). + DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (us per docs; ns on DCGM 3.x). # NVLink — DCGM silently drops this metric on GPUs without NVLink, so # enabling it unconditionally is safe and keeps this file reusable From 2a6653e11ae6a97d992f80e4ae0731995deea08a Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 12:08:15 +0300 Subject: [PATCH 099/130] docs(monitoring): add panel descriptions to GPU quotas dashboard Regenerated from SDK source. Signed-off-by: Arsolitt --- dashboards/gpu/gpu-quotas.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json index ce1e387e..865fb95c 100644 --- a/dashboards/gpu/gpu-quotas.json +++ b/dashboards/gpu/gpu-quotas.json @@ -38,6 +38,7 @@ } ], "title": "GPU allocatable", + "description": "Total GPU capacity the cluster can schedule to pods.", "transparent": false, "datasource": { "type": "prometheus", @@ -91,6 +92,7 @@ } ], "title": "GPU requested", + "description": "Sum of GPU requests across all pods cluster-wide, including system namespaces.", "transparent": false, "datasource": { "type": "prometheus", @@ -148,6 +150,7 @@ } ], "title": "Allocation ratio", + "description": "Percentage of allocatable GPUs currently requested by pods.", "transparent": false, "datasource": { "type": "prometheus", @@ -213,6 +216,7 @@ } ], "title": "Pending pods (GPU)", + "description": "Pods requesting GPUs that are stuck in Pending state — indicates capacity shortage.", "transparent": false, "datasource": { "type": "prometheus", @@ -284,6 +288,7 @@ } ], "title": "GPU requested per namespace", + "description": "GPU allocation breakdown by namespace — spot top consumers at a glance.", "transparent": false, "datasource": { "type": "prometheus", @@ -352,6 +357,7 @@ } ], "title": "GPU allocated over time", + "description": "Requested vs allocatable GPUs over time — shows allocation pressure trends.", "transparent": false, "datasource": { "type": "prometheus", @@ -438,6 +444,7 @@ } ], "title": "Pods requesting GPU", + "description": "Per-pod GPU requests and limits with scheduling status — Running or Pending.", "transparent": false, "datasource": { "type": "prometheus", From d20285836cede23c3d7cfe091e5444359bac4320 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Tue, 28 Apr 2026 11:14:27 +0200 Subject: [PATCH 100/130] feat(cozy-proxy): bump to v0.3.0 Pulls in the per-port filtering and allowICMP support that the companion vm-instance chart fix in #2501 relies on. cozy-proxy v0.3.0 also tightens the selector to the standard service.kubernetes.io/service-proxy-name=cozy-proxy label and switches the default ingress mode to port-filter; both are already covered by the vm-instance chart (label landed in #2357, wholeIP/allowICMP wired explicitly in #2501), so VM workloads upgrade transparently. Out-of-tree consumers using cozy-proxy annotations directly (without the label, or relying on the absent-annotation passthrough default) are called out in the upstream v0.3.0 release notes: https://github.com/cozystack/cozy-proxy/releases/tag/v0.3.0 Signed-off-by: Andrei Kvapil --- packages/system/cozy-proxy/charts/cozy-proxy/Chart.yaml | 4 ++-- packages/system/cozy-proxy/charts/cozy-proxy/values.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/system/cozy-proxy/charts/cozy-proxy/Chart.yaml b/packages/system/cozy-proxy/charts/cozy-proxy/Chart.yaml index 13352680..58f166f6 100644 --- a/packages/system/cozy-proxy/charts/cozy-proxy/Chart.yaml +++ b/packages/system/cozy-proxy/charts/cozy-proxy/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: cozy-proxy description: A simple kube-proxy addon for 1:1 NAT services in Kubernetes using an NFT backend type: application -version: 0.2.0 -appVersion: 0.2.0 +version: 0.3.0 +appVersion: 0.3.0 diff --git a/packages/system/cozy-proxy/charts/cozy-proxy/values.yaml b/packages/system/cozy-proxy/charts/cozy-proxy/values.yaml index 8cde5bed..e143e926 100644 --- a/packages/system/cozy-proxy/charts/cozy-proxy/values.yaml +++ b/packages/system/cozy-proxy/charts/cozy-proxy/values.yaml @@ -1,6 +1,6 @@ image: repository: ghcr.io/cozystack/cozystack/cozy-proxy - tag: v0.2.0 + tag: v0.3.0 pullPolicy: IfNotPresent daemonset: From b2a8cca3bb8960ab2f058051deb03f0b7cab0dca Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 12:15:32 +0300 Subject: [PATCH 101/130] fix(monitoring): filter zero-valued series from active tenants count Address review feedback from coderabbitai on dashboards/gpu/gpu-tenants.json:39 Signed-off-by: Arsolitt --- dashboards/gpu/gpu-tenants.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboards/gpu/gpu-tenants.json b/dashboards/gpu/gpu-tenants.json index 6194a15e..7fef321d 100644 --- a/dashboards/gpu/gpu-tenants.json +++ b/dashboards/gpu/gpu-tenants.json @@ -35,7 +35,7 @@ "id": 2, "targets": [ { - "expr": "count(namespace:gpu_count:allocated{namespace=~\"$namespace\"})", + "expr": "count(namespace:gpu_count:allocated{namespace=~\"$namespace\"} \u003e 0)", "refId": "A" } ], From 7257b6aed4bb2b6499168eab467a400196f2e625 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Tue, 28 Apr 2026 12:26:57 +0300 Subject: [PATCH 102/130] fix(api): address review feedback on TenantNamespace RBAC Address review feedback from sircthulhu and CodeRabbit: 1. Return Forbidden instead of NotFound for unauthorized access - Get() now returns 403 Forbidden to follow standard K8s RBAC behavior - Previously returned 404 NotFound for security-by-obscurity - Updated test expectations to match new behavior 2. Propagate field and label selectors in Watch handler - Pass opts.FieldSelector and opts.LabelSelector to upstream Watch - Add defensive filtering before authorization to prevent RBAC bypass - Fixes potential issue with resourceNames restrictions 3. Refactor subject-matching logic to eliminate duplication - Extract matchesSubject() helper for Group/User/ServiceAccount checks - Remove duplicated code from filterAccessible and hasAccessToNamespace - Consolidates ServiceAccount namespace fallback logic All tests pass successfully. Signed-off-by: IvanHunters --- pkg/registry/core/tenantnamespace/rest.go | 94 +++++++++++-------- .../core/tenantnamespace/rest_test.go | 6 +- 2 files changed, 56 insertions(+), 44 deletions(-) diff --git a/pkg/registry/core/tenantnamespace/rest.go b/pkg/registry/core/tenantnamespace/rest.go index 7724c0e1..223a6925 100644 --- a/pkg/registry/core/tenantnamespace/rest.go +++ b/pkg/registry/core/tenantnamespace/rest.go @@ -17,6 +17,8 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metainternal "k8s.io/apimachinery/pkg/apis/meta/internalversion" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" @@ -130,8 +132,8 @@ func (r *REST) Get( return nil, err } if !hasAccess { - // Return NotFound instead of Forbidden to prevent enumeration - return nil, apierrors.NewNotFound(r.gvr.GroupResource(), name) + // Return Forbidden to follow standard K8s RBAC behavior + return nil, apierrors.NewForbidden(r.gvr.GroupResource(), name, fmt.Errorf("access denied")) } ns := &corev1.Namespace{} @@ -155,10 +157,20 @@ func (r *REST) Get( func (r *REST) Watch(ctx context.Context, opts *metainternal.ListOptions) (watch.Interface, error) { nsList := &corev1.NamespaceList{} - nsWatch, err := r.w.Watch(ctx, nsList, &client.ListOptions{Raw: &metav1.ListOptions{ + + // Build upstream watch options with field and label selectors + rawOpts := &metav1.ListOptions{ Watch: true, ResourceVersion: opts.ResourceVersion, - }}) + } + if opts.FieldSelector != nil { + rawOpts.FieldSelector = opts.FieldSelector.String() + } + if opts.LabelSelector != nil { + rawOpts.LabelSelector = opts.LabelSelector.String() + } + + nsWatch, err := r.w.Watch(ctx, nsList, &client.ListOptions{Raw: rawOpts}) if err != nil { return nil, err } @@ -200,6 +212,18 @@ func (r *REST) Watch(ctx context.Context, opts *metainternal.ListOptions) (watch continue } + // Apply defensive filtering for field and label selectors + if opts.FieldSelector != nil { + if !opts.FieldSelector.Matches(fields.Set{"metadata.name": ns.Name}) { + continue + } + } + if opts.LabelSelector != nil { + if !opts.LabelSelector.Matches(labels.Set(ns.Labels)) { + continue + } + } + // Check if user has access to this namespace hasAccess, err := r.hasAccessToNamespace(ctx, ns.Name) if err != nil { @@ -331,6 +355,26 @@ func (r *REST) makeList(src *corev1.NamespaceList, allowed []string) *corev1alph return out } +// matchesSubject checks if a RoleBinding subject matches the user's identity. +// It handles Group, User, and ServiceAccount subjects with proper namespace fallback. +func matchesSubject(subj rbacv1.Subject, bindingNamespace, username string, groups map[string]struct{}) bool { + switch subj.Kind { + case "Group": + _, ok := groups[subj.Name] + return ok + case "User": + return subj.Name == username + case "ServiceAccount": + saNamespace := subj.Namespace + if saNamespace == "" { + saNamespace = bindingNamespace + } + return username == fmt.Sprintf("system:serviceaccount:%s:%s", saNamespace, subj.Name) + default: + return false + } +} + func (r *REST) filterAccessible( ctx context.Context, names []string, @@ -369,26 +413,9 @@ func (r *REST) filterAccessible( subjectLoop: for j := range rbs.Items[i].Subjects { subj := rbs.Items[i].Subjects[j] - switch subj.Kind { - case "Group": - if _, ok = groups[subj.Name]; ok { - allowedNameSet[rbs.Items[i].Namespace] = struct{}{} - break subjectLoop - } - case "User": - if subj.Name == u.GetName() { - allowedNameSet[rbs.Items[i].Namespace] = struct{}{} - break subjectLoop - } - case "ServiceAccount": - saNamespace := subj.Namespace - if saNamespace == "" { - saNamespace = rbs.Items[i].Namespace - } - if u.GetName() == fmt.Sprintf("system:serviceaccount:%s:%s", saNamespace, subj.Name) { - allowedNameSet[rbs.Items[i].Namespace] = struct{}{} - break subjectLoop - } + if matchesSubject(subj, rbs.Items[i].Namespace, u.GetName(), groups) { + allowedNameSet[rbs.Items[i].Namespace] = struct{}{} + break subjectLoop } } } @@ -434,23 +461,8 @@ func (r *REST) hasAccessToNamespace( for i := range rbs.Items { for j := range rbs.Items[i].Subjects { subj := rbs.Items[i].Subjects[j] - switch subj.Kind { - case "Group": - if _, ok := groups[subj.Name]; ok { - return true, nil - } - case "User": - if subj.Name == u.GetName() { - return true, nil - } - case "ServiceAccount": - saNamespace := subj.Namespace - if saNamespace == "" { - saNamespace = rbs.Items[i].Namespace - } - if u.GetName() == fmt.Sprintf("system:serviceaccount:%s:%s", saNamespace, subj.Name) { - return true, nil - } + if matchesSubject(subj, rbs.Items[i].Namespace, u.GetName(), groups) { + return true, nil } } } diff --git a/pkg/registry/core/tenantnamespace/rest_test.go b/pkg/registry/core/tenantnamespace/rest_test.go index eb678949..e1fb1e85 100644 --- a/pkg/registry/core/tenantnamespace/rest_test.go +++ b/pkg/registry/core/tenantnamespace/rest_test.go @@ -462,9 +462,9 @@ func TestGet_WithoutAccess(t *testing.T) { t.Errorf("expected nil object, got %v", obj) } - // Verify it returns NotFound (not Forbidden) to prevent enumeration - if !apierrors.IsNotFound(err) { - t.Errorf("expected NotFound error, got %v", err) + // Verify it returns Forbidden to follow standard K8s RBAC behavior + if !apierrors.IsForbidden(err) { + t.Errorf("expected Forbidden error, got %v", err) } } From 61ed7ad89c0bff66236e6884e6b6cd6038383718 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Tue, 28 Apr 2026 16:33:37 +0500 Subject: [PATCH 103/130] fix(api): address review feedback on TenantNamespace watch path - Hoist user identity extraction out of the Watch goroutine; reuse a cached username and groups map across events instead of re-fetching them per event. Watch now returns Unauthorized up front when no user is present in the context, rather than failing silently per event. - Switch the per-event access-check error log to structured klog.ErrorS to comply with the project Go style guide. - Strengthen TestGet_WithAccess to assert the concrete *TenantNamespace type plus Name, Kind, and APIVersion, so type or metadata regressions fail fast. Assisted-By: Claude Signed-off-by: Myasnikov Daniil --- pkg/registry/core/tenantnamespace/rest.go | 35 +++++++++++++++---- .../core/tenantnamespace/rest_test.go | 16 +++++++-- 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/pkg/registry/core/tenantnamespace/rest.go b/pkg/registry/core/tenantnamespace/rest.go index 223a6925..4ee97eb0 100644 --- a/pkg/registry/core/tenantnamespace/rest.go +++ b/pkg/registry/core/tenantnamespace/rest.go @@ -156,6 +156,18 @@ func (r *REST) Get( // ----------------------------------------------------------------------------- func (r *REST) Watch(ctx context.Context, opts *metainternal.ListOptions) (watch.Interface, error) { + // Extract user identity once for the lifetime of the watch — it does not + // change between events and rebuilding it per event is wasteful. + u, ok := request.UserFrom(ctx) + if !ok { + return nil, apierrors.NewUnauthorized("user missing in context") + } + username := u.GetName() + groups := make(map[string]struct{}) + for _, group := range u.GetGroups() { + groups[group] = struct{}{} + } + nsList := &corev1.NamespaceList{} // Build upstream watch options with field and label selectors @@ -224,10 +236,11 @@ func (r *REST) Watch(ctx context.Context, opts *metainternal.ListOptions) (watch } } - // Check if user has access to this namespace - hasAccess, err := r.hasAccessToNamespace(ctx, ns.Name) + // Check if user has access to this namespace using the cached + // identity — avoids re-extracting user/groups on every event. + hasAccess, err := r.hasAccessToNamespaceForUser(ctx, ns.Name, username, groups) if err != nil { - klog.Errorf("Failed to check access for namespace %s in watch: %v", ns.Name, err) + klog.ErrorS(err, "Failed to check access for namespace in watch", "namespace", ns.Name) continue } if !hasAccess { @@ -437,12 +450,22 @@ func (r *REST) hasAccessToNamespace( if !ok { return false, fmt.Errorf("user missing in context") } - - // Check privileged groups groups := make(map[string]struct{}) for _, group := range u.GetGroups() { groups[group] = struct{}{} } + return r.hasAccessToNamespaceForUser(ctx, namespace, u.GetName(), groups) +} + +// hasAccessToNamespaceForUser is the inner check that does not re-extract user +// identity from context. Use this in hot paths (e.g. the Watch loop) where the +// caller has already cached the user name and groups. +func (r *REST) hasAccessToNamespaceForUser( + ctx context.Context, + namespace, username string, + groups map[string]struct{}, +) (bool, error) { + // Check privileged groups if _, ok := groups["system:masters"]; ok { return true, nil } @@ -461,7 +484,7 @@ func (r *REST) hasAccessToNamespace( for i := range rbs.Items { for j := range rbs.Items[i].Subjects { subj := rbs.Items[i].Subjects[j] - if matchesSubject(subj, rbs.Items[i].Namespace, u.GetName(), groups) { + if matchesSubject(subj, rbs.Items[i].Namespace, username, groups) { return true, nil } } diff --git a/pkg/registry/core/tenantnamespace/rest_test.go b/pkg/registry/core/tenantnamespace/rest_test.go index e1fb1e85..52d2a075 100644 --- a/pkg/registry/core/tenantnamespace/rest_test.go +++ b/pkg/registry/core/tenantnamespace/rest_test.go @@ -15,6 +15,8 @@ import ( "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/endpoints/request" "sigs.k8s.io/controller-runtime/pkg/client/fake" + + corev1alpha1 "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1" ) func TestMakeListSortsAlphabetically(t *testing.T) { @@ -408,8 +410,18 @@ func TestGet_WithAccess(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - if obj == nil { - t.Fatal("expected object, got nil") + tn, ok := obj.(*corev1alpha1.TenantNamespace) + if !ok { + t.Fatalf("expected *TenantNamespace, got %T", obj) + } + if tn.Name != "tenant-test" { + t.Errorf("expected name %q, got %q", "tenant-test", tn.Name) + } + if tn.Kind != "TenantNamespace" { + t.Errorf("expected Kind=TenantNamespace, got %q", tn.Kind) + } + if tn.APIVersion != corev1alpha1.SchemeGroupVersion.String() { + t.Errorf("expected APIVersion=%q, got %q", corev1alpha1.SchemeGroupVersion.String(), tn.APIVersion) } } From 1dc02d44f4ea15253ffd799e85a54ed79ae87fd1 Mon Sep 17 00:00:00 2001 From: Timofei Larkin Date: Mon, 27 Apr 2026 10:27:46 +0300 Subject: [PATCH 104/130] refactor(lineage-controller-webhook): align with cozystack-api shape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Collapse the chart to a single Deployment shape that mirrors cozystack-api: 2 replicas, soft (preferred) nodeAffinity to node-role.kubernetes.io/control-plane via Exists, permissive tolerations, soft podAntiAffinity on hostname, an unconditional PodDisruptionBudget with maxUnavailable: 1, and a Service with spec.trafficDistribution: PreferClose. The same shape works on Talos / kubeadm / k3s and on managed Kubernetes / Cozy-in-Cozy tenant clusters without per-distro overrides — fixing #2417 by making the soft control-plane affinity gracefully fall back to worker scheduling when no control-plane nodes are visible. Drop the DaemonSet path entirely. The previous PR's deployment.enabled toggle, the workload-kind switch in templates/workload.yaml, the fail-guard for localK8sAPIEndpoint+nodeAffinity=[], the conditional PDB, and the values-shape knobs for nodeAffinity and tolerations all go away as a consequence. Override the standard Deployment fields through the usual component-values mechanism if a non-default topology is ever needed. Mark localK8sAPIEndpoint.enabled deprecated and flip the default to false. The flag injects KUBERNETES_SERVICE_HOST=status.hostIP, which is only valid when the pod is actually scheduled on an apiserver- bearing node. With the new soft control-plane affinity, the pod can land off-control-plane and crash-loop. The latency motivation for the flag is real but pending separate webhook performance work; once addressed, the flag can be removed. Revert all changes to packages/core/platform/images/migrations/migrations/20. The earlier ds/...-or-deploy/... fallback was over-engineered: per run-migrations.sh, migration 20 fires only when CURRENT < 20 (i.e. direct upgrades from pre-0.37 to 1.3+), and that path is unsupported anyway. The original ds/... rollout-status line is dead code on every supported install and upgrade path. Add helm-unittest coverage for: the default Deployment shape (replicas, soft nodeAffinity, soft podAntiAffinity, tolerations); no env vars at the default localK8sAPIEndpoint.enabled=false; PDB rendered unconditionally with maxUnavailable: 1, including at replicas=1 (the no-op case); replicas value drives spec.replicas; and that localK8sAPIEndpoint.enabled=true does inject the env vars. The Service test continues to assert trafficDistribution: PreferClose with no internalTrafficPolicy. Add a slim README documenting the topology, parameters, and the deprecation note for localK8sAPIEndpoint. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Timofei Larkin --- .../platform/templates/bundles/system.yaml | 3 - .../lineage-controller-webhook/Makefile | 5 + .../lineage-controller-webhook/README.md | 67 +++++++++++++ .../templates/deployment.yaml | 49 --------- .../templates/pdb.yaml | 4 +- .../templates/service.yaml | 2 +- .../{daemonset.yaml => workload.yaml} | 27 +++-- .../tests/service_test.yaml | 30 ++++++ .../tests/workload_test.yaml | 99 +++++++++++++++++++ .../lineage-controller-webhook/values.yaml | 18 ++-- 10 files changed, 230 insertions(+), 74 deletions(-) create mode 100644 packages/system/lineage-controller-webhook/README.md delete mode 100644 packages/system/lineage-controller-webhook/templates/deployment.yaml rename packages/system/lineage-controller-webhook/templates/{daemonset.yaml => workload.yaml} (67%) create mode 100644 packages/system/lineage-controller-webhook/tests/service_test.yaml create mode 100644 packages/system/lineage-controller-webhook/tests/workload_test.yaml diff --git a/packages/core/platform/templates/bundles/system.yaml b/packages/core/platform/templates/bundles/system.yaml index 43ea7266..6c587c82 100644 --- a/packages/core/platform/templates/bundles/system.yaml +++ b/packages/core/platform/templates/bundles/system.yaml @@ -105,9 +105,6 @@ {{- /* cozystack-api DaemonSet */ -}} {{- $apiValues := dict "cozystackAPI" (dict "nodeSelector" $genericNodeSelector) -}} {{- $_ := set $cozystackEngineComponents "cozystack-api" (dict "values" $apiValues) -}} -{{- /* lineage-controller-webhook DaemonSet */ -}} -{{- $lineageValues := dict "lineageControllerWebhook" (dict "nodeSelector" $genericNodeSelector) -}} -{{- $_ := set $cozystackEngineComponents "lineage-controller-webhook" (dict "values" $lineageValues) -}} {{- end -}} {{- if .Values.authentication.oidc.enabled }} {{include "cozystack.platform.package" (list "cozystack.cozystack-engine" "oidc" $ $cozystackEngineComponents) }} diff --git a/packages/system/lineage-controller-webhook/Makefile b/packages/system/lineage-controller-webhook/Makefile index d5bada31..cdc1cf01 100644 --- a/packages/system/lineage-controller-webhook/Makefile +++ b/packages/system/lineage-controller-webhook/Makefile @@ -4,8 +4,13 @@ NAMESPACE=cozy-system include ../../../hack/common-envs.mk include ../../../hack/package.mk +.PHONY: test + image: image-lineage-controller-webhook +test: + helm unittest . + image-lineage-controller-webhook: docker buildx build -f images/lineage-controller-webhook/Dockerfile ../../.. \ --tag $(REGISTRY)/lineage-controller-webhook:$(call settag,$(TAG)) \ diff --git a/packages/system/lineage-controller-webhook/README.md b/packages/system/lineage-controller-webhook/README.md new file mode 100644 index 00000000..1e43c9c8 --- /dev/null +++ b/packages/system/lineage-controller-webhook/README.md @@ -0,0 +1,67 @@ +# lineage-controller-webhook + +Cozystack system package for the **lineage controller webhook** — a mutating +admission webhook that stamps "lineage" labels onto tenant workloads, linking +each resource back to the Cozystack `Application` that ultimately owns it. + +The webhook intercepts CREATE and UPDATE on `pods`, `secrets`, `services`, +`persistentvolumeclaims`, `ingresses.networking.k8s.io`, and +`workloadmonitors.cozystack.io` outside system namespaces. For each request it +walks the ownership graph upward (Kubernetes `ownerReferences`, then the +`HelmRelease` Flux installed the resource with, then the Cozystack +`Application`-derived CRD that produced the HelmRelease) and writes the +discovered application's group, kind and name as labels +(`apps.cozystack.io/application.{group,kind,name}`) on the incoming object. +Those labels let the aggregated API server, the Cozystack dashboard, and other +lineage-aware consumers reason about which application a resource belongs to. + +The webhook serves TLS on port 9443 with a cert-manager issued certificate, and +runs with `failurePolicy: Fail` — every CREATE/UPDATE on the resources above +is gated on the webhook being reachable. + +## Topology + +The chart ships a single shape, modelled on `cozystack-api`: + +- **Deployment** with two replicas (override via `replicas`). +- **Soft `nodeAffinity`** preferring `node-role.kubernetes.io/control-plane` + (`Exists`, so it matches both Talos's empty value and k3s/kubeadm's `"true"`). + Soft means: the pod lands on a control-plane node when one is reachable, and + on any worker otherwise — no override needed for managed Kubernetes, + Cozy-in-Cozy tenants, or any other cluster where control-plane nodes aren't + visible. +- **Permissive `tolerations`** (`[{operator: Exists}]`) so the pod can land on + tainted control-plane nodes when the soft affinity is satisfiable. +- **Soft `podAntiAffinity`** on `kubernetes.io/hostname` so replicas spread + across nodes when possible (best-effort). +- **`PodDisruptionBudget`** with `maxUnavailable: 1`, unconditional. At + `replicas: 1` it's a useful no-op (allows full drain); at `replicas >= 2` + it caps disruption to one pod. +- **`Service` with `spec.trafficDistribution: PreferClose`** so the apiserver + prefers a webhook endpoint on its own node when one exists, and transparently + falls over to a remote endpoint otherwise. Requires Kubernetes ≥ 1.31; on + older clusters the field is silently ignored and traffic uses default + cluster-wide distribution. + +## Parameters + +All keys live under the top-level `lineageControllerWebhook:` map. + +| Name | Description | Value | +| ----------------------------- | --------------------------------------------------------- | -------------------------------------------------------------------------------- | +| `image` | Container image (digest-pinned) | `ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.3.0@sha256:e898…6fb0` | +| `debug` | Enable `--zap-log-level=debug` instead of `info` | `false` | +| `replicas` | Deployment replica count | `2` | +| `localK8sAPIEndpoint.enabled` | **Deprecated.** See note below. | `false` | + +### `localK8sAPIEndpoint.enabled` (deprecated) + +When enabled, this injects `KUBERNETES_SERVICE_HOST=status.hostIP` and +`KUBERNETES_SERVICE_PORT=6443` so the webhook talks to the kube-apiserver on +its own node. It was originally added to avoid latency on the +webhook-to-apiserver path, but it is only valid when the pod is actually +scheduled on an apiserver-bearing node — which the chart's soft control-plane +affinity no longer guarantees. With this flag enabled and the pod scheduled +off a control-plane node, the controller will crash-loop dialing a non- +apiserver hostIP. Slated for removal once the latency motivation is addressed +in the webhook itself; **leave disabled**. diff --git a/packages/system/lineage-controller-webhook/templates/deployment.yaml b/packages/system/lineage-controller-webhook/templates/deployment.yaml deleted file mode 100644 index 87717fd3..00000000 --- a/packages/system/lineage-controller-webhook/templates/deployment.yaml +++ /dev/null @@ -1,49 +0,0 @@ -{{- if .Values.lineageControllerWebhook.deployment.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: lineage-controller-webhook - labels: - app: lineage-controller-webhook -spec: - replicas: {{ .Values.lineageControllerWebhook.deployment.replicas }} - selector: - matchLabels: - app: lineage-controller-webhook - template: - metadata: - labels: - app: lineage-controller-webhook - spec: - serviceAccountName: lineage-controller-webhook - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - topologyKey: kubernetes.io/hostname - labelSelector: - matchLabels: - app: lineage-controller-webhook - containers: - - name: lineage-controller-webhook - image: "{{ .Values.lineageControllerWebhook.image }}" - args: - {{- if .Values.lineageControllerWebhook.debug }} - - --zap-log-level=debug - {{- else }} - - --zap-log-level=info - {{- end }} - ports: - - name: webhook - containerPort: 9443 - volumeMounts: - - name: webhook-certs - mountPath: /tmp/k8s-webhook-server/serving-certs - readOnly: true - volumes: - - name: webhook-certs - secret: - secretName: lineage-controller-webhook-cert - defaultMode: 0400 -{{- end }} \ No newline at end of file diff --git a/packages/system/lineage-controller-webhook/templates/pdb.yaml b/packages/system/lineage-controller-webhook/templates/pdb.yaml index b0a9d4e4..0786c8ff 100644 --- a/packages/system/lineage-controller-webhook/templates/pdb.yaml +++ b/packages/system/lineage-controller-webhook/templates/pdb.yaml @@ -1,11 +1,9 @@ -{{- if .Values.lineageControllerWebhook.deployment.enabled }} apiVersion: policy/v1 kind: PodDisruptionBudget metadata: name: lineage-controller-webhook spec: - minAvailable: {{ if gt (int .Values.lineageControllerWebhook.deployment.replicas) 1 }}1{{ else }}0{{ end }} + maxUnavailable: 1 selector: matchLabels: app: lineage-controller-webhook -{{- end }} \ No newline at end of file diff --git a/packages/system/lineage-controller-webhook/templates/service.yaml b/packages/system/lineage-controller-webhook/templates/service.yaml index c5df90fb..fbb3ad54 100644 --- a/packages/system/lineage-controller-webhook/templates/service.yaml +++ b/packages/system/lineage-controller-webhook/templates/service.yaml @@ -5,7 +5,7 @@ metadata: labels: app: lineage-controller-webhook spec: - internalTrafficPolicy: Local + trafficDistribution: PreferClose type: ClusterIP ports: - port: 443 diff --git a/packages/system/lineage-controller-webhook/templates/daemonset.yaml b/packages/system/lineage-controller-webhook/templates/workload.yaml similarity index 67% rename from packages/system/lineage-controller-webhook/templates/daemonset.yaml rename to packages/system/lineage-controller-webhook/templates/workload.yaml index 536ea24a..097919c6 100644 --- a/packages/system/lineage-controller-webhook/templates/daemonset.yaml +++ b/packages/system/lineage-controller-webhook/templates/workload.yaml @@ -1,11 +1,11 @@ -{{- if not .Values.lineageControllerWebhook.deployment.enabled }} apiVersion: apps/v1 -kind: DaemonSet +kind: Deployment metadata: name: lineage-controller-webhook labels: app: lineage-controller-webhook spec: + replicas: {{ .Values.lineageControllerWebhook.replicas }} selector: matchLabels: app: lineage-controller-webhook @@ -14,13 +14,25 @@ spec: labels: app: lineage-controller-webhook spec: - {{- with .Values.lineageControllerWebhook.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} + serviceAccountName: lineage-controller-webhook tolerations: - operator: Exists - serviceAccountName: lineage-controller-webhook + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app: lineage-controller-webhook containers: - name: lineage-controller-webhook image: "{{ .Values.lineageControllerWebhook.image }}" @@ -52,4 +64,3 @@ spec: secret: secretName: lineage-controller-webhook-cert defaultMode: 0400 -{{- end }} \ No newline at end of file diff --git a/packages/system/lineage-controller-webhook/tests/service_test.yaml b/packages/system/lineage-controller-webhook/tests/service_test.yaml new file mode 100644 index 00000000..3cdbddd8 --- /dev/null +++ b/packages/system/lineage-controller-webhook/tests/service_test.yaml @@ -0,0 +1,30 @@ +suite: lineage-controller-webhook service +templates: + - templates/service.yaml + +release: + name: lineage-controller-webhook + namespace: cozy-system + +tests: + - it: renders a ClusterIP service on 443 -> 9443 with PreferClose traffic distribution + asserts: + - isKind: + of: Service + - equal: + path: spec.type + value: ClusterIP + - equal: + path: spec.trafficDistribution + value: PreferClose + - notExists: + path: spec.internalTrafficPolicy + - equal: + path: spec.ports[0].port + value: 443 + - equal: + path: spec.ports[0].targetPort + value: 9443 + - equal: + path: spec.selector.app + value: lineage-controller-webhook diff --git a/packages/system/lineage-controller-webhook/tests/workload_test.yaml b/packages/system/lineage-controller-webhook/tests/workload_test.yaml new file mode 100644 index 00000000..bcb24014 --- /dev/null +++ b/packages/system/lineage-controller-webhook/tests/workload_test.yaml @@ -0,0 +1,99 @@ +suite: lineage-controller-webhook workload + PDB +templates: + - templates/workload.yaml + - templates/pdb.yaml + +release: + name: lineage-controller-webhook + namespace: cozy-system + +tests: + # ---- Default Deployment shape ---------------------------------------------- + + - it: defaults render a 2-replica Deployment with soft control-plane nodeAffinity, soft podAntiAffinity, and Exists tolerations + template: templates/workload.yaml + asserts: + - isKind: + of: Deployment + - equal: + path: spec.replicas + value: 2 + - equal: + path: spec.template.spec.tolerations + value: + - operator: Exists + - equal: + path: spec.template.spec.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution + value: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - notExists: + path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution + - equal: + path: spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.topologyKey + value: kubernetes.io/hostname + + - it: defaults inject no localK8sAPIEndpoint env vars (knob is opt-in) + template: templates/workload.yaml + asserts: + - notExists: + path: spec.template.spec.containers[0].env + + # ---- PDB -------------------------------------------------------------------- + + - it: PDB renders unconditionally with maxUnavailable=1 at the default replica count + template: templates/pdb.yaml + asserts: + - isKind: + of: PodDisruptionBudget + - equal: + path: spec.maxUnavailable + value: 1 + - notExists: + path: spec.minAvailable + + - it: PDB still renders at replicas=1 (no-op, but consistent shape) + set: + lineageControllerWebhook.replicas: 1 + template: templates/pdb.yaml + asserts: + - isKind: + of: PodDisruptionBudget + - equal: + path: spec.maxUnavailable + value: 1 + + # ---- Replica override ------------------------------------------------------- + + - it: replicas value drives spec.replicas + set: + lineageControllerWebhook.replicas: 5 + template: templates/workload.yaml + asserts: + - equal: + path: spec.replicas + value: 5 + + # ---- Deprecated localK8sAPIEndpoint knob ----------------------------------- + + - it: localK8sAPIEndpoint.enabled=true injects KUBERNETES_SERVICE_HOST and PORT env vars + set: + lineageControllerWebhook.localK8sAPIEndpoint.enabled: true + template: templates/workload.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: KUBERNETES_SERVICE_HOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + - contains: + path: spec.template.spec.containers[0].env + content: + name: KUBERNETES_SERVICE_PORT + value: "6443" diff --git a/packages/system/lineage-controller-webhook/values.yaml b/packages/system/lineage-controller-webhook/values.yaml index 2b84f4e1..e56d332d 100644 --- a/packages/system/lineage-controller-webhook/values.yaml +++ b/packages/system/lineage-controller-webhook/values.yaml @@ -1,15 +1,13 @@ lineageControllerWebhook: image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.3.0@sha256:e8984709686a5eaf19b89da378d7b8c688ea5607e0783a88d9c9e4ccfca96fb0 debug: false + replicas: 2 + # DEPRECATED. Injects KUBERNETES_SERVICE_HOST=status.hostIP and + # KUBERNETES_SERVICE_PORT=6443 so the webhook talks to the apiserver on its + # own node — only valid when the pod is actually scheduled on an apiserver- + # bearing node. Now that the chart's nodeAffinity to control-plane is soft + # (preferred, not required), the pod can land off-control-plane and crash- + # loop dialing a non-apiserver hostIP. Slated for removal once webhook + # performance work obviates the locality motivation. Leave disabled. localK8sAPIEndpoint: - # incompatable with Deployment mode - enabled: true - # nodeSelector for the DaemonSet - # Talos uses empty value: "node-role.kubernetes.io/control-plane": "" - # Generic k8s (k3s, kubeadm) uses: "node-role.kubernetes.io/control-plane": "true" - nodeSelector: - node-role.kubernetes.io/control-plane: "" - # if enabled, replace DaemonSet by Deployment - deployment: enabled: false - replicas: 1 From 3a8359fa73f5cd96d8363c48d31414d8cadf3447 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 18:59:43 +0300 Subject: [PATCH 105/130] chore(kubernetes): regenerate deepcopy after merge Run `make generate` to pick up the Images type added in main. Signed-off-by: Arsolitt --- .../v1alpha1/kubernetes/zz_generated.deepcopy.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go b/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go index 3f1cb5ce..e021fbaa 100644 --- a/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go +++ b/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go @@ -136,6 +136,7 @@ func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) { } in.Addons.DeepCopyInto(&out.Addons) in.ControlPlane.DeepCopyInto(&out.ControlPlane) + out.Images = in.Images } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigSpec. @@ -277,6 +278,21 @@ func (in *HAMiAddon) DeepCopy() *HAMiAddon { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Images) DeepCopyInto(out *Images) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Images. +func (in *Images) DeepCopy() *Images { + if in == nil { + return nil + } + out := new(Images) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IngressNginxAddon) DeepCopyInto(out *IngressNginxAddon) { *out = *in From 80b86b0c1a39266488b84473724a54b0d3ad7d9c Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 19:02:35 +0300 Subject: [PATCH 106/130] chore(kubernetes): regenerate CRD after merge conflict resolution Per-package `make generate` corrects keysOrder placement and openAPISchema formatting in the kubernetes CRD definition. Signed-off-by: Arsolitt --- api/apps/v1alpha1/kubernetes/types.go | 2 +- packages/system/kubernetes-rd/cozyrds/kubernetes.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/apps/v1alpha1/kubernetes/types.go b/api/apps/v1alpha1/kubernetes/types.go index e0143445..88a3c97e 100644 --- a/api/apps/v1alpha1/kubernetes/types.go +++ b/api/apps/v1alpha1/kubernetes/types.go @@ -6,7 +6,7 @@ package kubernetes import ( resource "k8s.io/apimachinery/pkg/api/resource" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" k8sRuntime "k8s.io/apimachinery/pkg/runtime" ) diff --git a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml index b4ada238..f81eaff0 100644 --- a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml +++ b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml @@ -22,7 +22,7 @@ spec: singular: kubernetes plural: kuberneteses openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"ephemeralStorage":"20Gi","gpus":[],"instanceType":"u1.medium","maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"]}},"additionalProperties":{"type":"object","required":["ephemeralStorage","instanceType","maxReplicas","minReplicas","resources"],"properties":{"ephemeralStorage":{"description":"Ephemeral storage size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string"}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium"},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `.` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","hami","ingressNginx","monitoringAgents","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"hami":{"description":"HAMi GPU virtualization middleware.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable HAMi (requires GPU Operator).","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"large","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"images":{"description":"Optional image overrides for air-gapped or rate-limited registries.","type":"object","default":{},"properties":{"waitForKubeconfig":{"description":"Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.","type":"string","default":""}}}}} + {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"ephemeralStorage":"20Gi","gpus":[],"instanceType":"u1.medium","maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"]}},"additionalProperties":{"type":"object","required":["ephemeralStorage","instanceType","maxReplicas","minReplicas","resources"],"properties":{"ephemeralStorage":{"description":"Ephemeral storage size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string"}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium"},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `.` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","hami","ingressNginx","monitoringAgents","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"hami":{"description":"HAMi GPU virtualization middleware.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable HAMi (requires GPU Operator).","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"large","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"images":{"description":"Optional image overrides for air-gapped or rate-limited registries.","type":"object","default":{},"properties":{"waitForKubeconfig":{"description":"Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.","type":"string","default":""}}}}} release: prefix: kubernetes- labels: @@ -40,7 +40,7 @@ spec: tags: - compute icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODFfMjg0NSkiLz4KPHBhdGggZD0iTTcxLjk5NjggMTlDNzAuMzAzOSAxOS4wMDAyIDY4LjkzMTIgMjAuNTMyMiA2OC45MzE0IDIyLjQyMjFDNjguOTMxNCAyMi40NTExIDY4LjkzNzMgMjIuNDc4OCA2OC45Mzc5IDIyLjUwNzZDNjguOTM1NCAyMi43NjQ0IDY4LjkyMzEgMjMuMDczNyA2OC45MzE0IDIzLjI5NzNDNjguOTcxNyAyNC4zODczIDY5LjIwODIgMjUuMjIxNiA2OS4zNTA2IDI2LjIyNThDNjkuNjA4NCAyOC4zNzUyIDY5LjgyNDUgMzAuMTU2OSA2OS42OTEyIDMxLjgxM0M2OS41NjE1IDMyLjQzNzUgNjkuMTAzNyAzMy4wMDg2IDY4LjY5NTYgMzMuNDA1Nkw2OC42MjM1IDM0LjcwODZDNjYuNzgzOSAzNC44NjE3IDY0LjkzMTkgMzUuMTQyMSA2My4wODIxIDM1LjU2NDFDNTUuMTIyNiAzNy4zNzk4IDQ4LjI2OTUgNDEuNDk5MSA0My4wNTIgNDcuMDYwOUM0Mi43MTM0IDQ2LjgyODggNDIuMTIxMSA0Ni40MDE5IDQxLjk0NSA0Ni4yNzEyQzQxLjM5NzcgNDYuMzQ1NCA0MC44NDQ1IDQ2LjUxNTEgNDAuMTI0MSA0Ni4wOTM1QzM4Ljc1MjIgNDUuMTY1NyAzNy41MDI4IDQzLjg4NTEgMzUuOTkxIDQyLjM0MjRDMzUuMjk4MiA0MS42MDQ0IDM0Ljc5NjYgNDAuOTAxOCAzMy45NzM1IDQwLjE5MDRDMzMuNzg2NiA0MC4wMjg5IDMzLjUwMTQgMzkuODEwNCAzMy4yOTIzIDM5LjY0NDJDMzIuNjQ4OSAzOS4xMjg4IDMxLjg5IDM4Ljg2IDMxLjE1NyAzOC44MzQ4QzMwLjIxNDcgMzguODAyNCAyOS4zMDc1IDM5LjE3MjUgMjguNzEzOCAzOS45MjA2QzI3LjY1ODQgNDEuMjUwNiAyNy45OTYzIDQzLjI4MzMgMjkuNDY3MSA0NC40NjE0QzI5LjQ4MiA0NC40NzM0IDI5LjQ5NzkgNDQuNDgyNyAyOS41MTI5IDQ0LjQ5NDNDMjkuNzE1IDQ0LjY1ODkgMjkuOTYyNSA0NC44Njk4IDMwLjE0ODMgNDUuMDA3NkMzMS4wMjE3IDQ1LjY1NTUgMzEuODE5NSA0NS45ODcyIDMyLjY4OTcgNDYuNTAxNUMzNC41MjMxIDQ3LjYzOTEgMzYuMDQzIDQ4LjU4MjMgMzcuMjQ4NiA0OS43MTk2QzM3LjcxOTQgNTAuMjIzNyAzNy44MDE2IDUxLjExMjIgMzcuODY0MyA1MS40OTY0TDM4Ljg0NjggNTIuMzc4MkMzMy41ODcyIDYwLjMzMDggMzEuMTUzIDcwLjE1MzkgMzIuNTkxNSA4MC4xNjI3TDMxLjMwNzcgODAuNTM3OEMzMC45NjkzIDgwLjk3NjggMzAuNDkxMiA4MS42Njc2IDI5Ljk5MTEgODEuODczOEMyOC40MTM4IDgyLjM3MjkgMjYuNjM4NyA4Mi41NTYyIDI0LjQ5NTYgODIuNzgxOUMyMy40ODk0IDgyLjg2NiAyMi42MjEzIDgyLjgxNTggMjEuNTU0NiA4My4wMTg4QzIxLjMxOTggODMuMDYzNSAyMC45OTI3IDgzLjE0OTEgMjAuNzM1OCA4My4yMDk3QzIwLjcyNjkgODMuMjExNiAyMC43MTg2IDgzLjIxNDIgMjAuNzA5NiA4My4yMTYyQzIwLjY5NTYgODMuMjE5NSAyMC42NzcyIDgzLjIyNjMgMjAuNjYzOCA4My4yMjk0QzE4Ljg1NyA4My42NjggMTcuNjk2MyA4NS4zMzY1IDE4LjA2OTkgODYuOTgwNUMxOC40NDM3IDg4LjYyNDggMjAuMjA4NiA4OS42MjQ4IDIyLjAyNjIgODkuMjMxMkMyMi4wMzkzIDg5LjIyODIgMjIuMDU4NCA4OS4yMjc3IDIyLjA3MiA4OS4yMjQ2QzIyLjA5MjYgODkuMjE5OSAyMi4xMTA2IDg5LjIwOTkgMjIuMTMxIDg5LjIwNDlDMjIuMzg0NCA4OS4xNDkgMjIuNzAxOSA4OS4wODY4IDIyLjkyMzYgODkuMDI3MkMyMy45NzIzIDg4Ljc0NTEgMjQuNzMxOCA4OC4zMzA2IDI1LjY3NDYgODcuOTY3N0MyNy43MDI5IDg3LjIzNjggMjkuMzgyOCA4Ni42MjYyIDMxLjAxOTUgODYuMzg4M0MzMS43MDMgODYuMzM0NSAzMi40MjMyIDg2LjgxMiAzMi43ODE0IDg3LjAxMzRMMzQuMTE3NyA4Ni43ODMxQzM3LjE5MjYgOTYuMzYxMyA0My42MzY2IDEwNC4xMDMgNTEuNzk2MyAxMDguOTYxTDUxLjIzOTYgMTEwLjMwM0M1MS40NDAzIDExMC44MjQgNTEuNjYxNiAxMTEuNTMgNTEuNTEyMSAxMTIuMDQ1QzUwLjkxNzEgMTEzLjU5NSA0OS44OTggMTE1LjIzMSA0OC43Mzc0IDExNy4wNTVDNDguMTc1NSAxMTcuODk4IDQ3LjYwMDQgMTE4LjU1MiA0Ny4wOTM0IDExOS41MTZDNDYuOTcyIDExOS43NDcgNDYuODE3NSAxMjAuMTAyIDQ2LjcwMDQgMTIwLjM0NkM0NS45MTI1IDEyMi4wMzkgNDYuNDkwNCAxMjMuOTkgNDguMDAzOCAxMjQuNzIyQzQ5LjUyNjggMTI1LjQ1OSA1MS40MTcxIDEyNC42ODIgNTIuMjM1MiAxMjIuOTg1QzUyLjIzNjQgMTIyLjk4MiA1Mi4yNDA2IDEyMi45OCA1Mi4yNDE3IDEyMi45NzhDNTIuMjQyNiAxMjIuOTc2IDUyLjI0MDkgMTIyLjk3MyA1Mi4yNDE3IDEyMi45NzFDNTIuMzU4MiAxMjIuNzMxIDUyLjUyMzMgMTIyLjQxNSA1Mi42MjE2IDEyMi4xODhDNTMuMDU2IDEyMS4xODkgNTMuMjAwNSAxMjAuMzMyIDUzLjUwNTkgMTE5LjM2NUM1NC4zMTcgMTE3LjMxOCA1NC43NjI2IDExNS4xNyA1NS44NzkxIDExMy44MzJDNTYuMTg0OSAxMTMuNDY2IDU2LjY4MzMgMTEzLjMyNSA1Ny4yMDAxIDExMy4xODZMNTcuODk0NCAxMTEuOTIyQzY1LjAwOCAxMTQuNjY1IDcyLjk3MDUgMTE1LjQwMiA4MC45MjQ1IDExMy41ODdDODIuNzM5MSAxMTMuMTczIDg0LjQ5MDggMTEyLjYzNyA4Ni4xODQzIDExMS45OTRDODYuMzc5NCAxMTIuMzQyIDg2Ljc0MiAxMTMuMDExIDg2LjgzOTMgMTEzLjE3OUM4Ny4zNjQ0IDExMy4zNTEgODcuOTM3NyAxMTMuNDM5IDg4LjQwNDcgMTE0LjEzM0M4OS4yNDAxIDExNS41NjcgODkuODExNCAxMTcuMjYzIDkwLjUwNzMgMTE5LjMxMkM5MC44MTI4IDEyMC4yNzkgOTAuOTYzOCAxMjEuMTM2IDkxLjM5ODEgMTIyLjEzNkM5MS40OTcxIDEyMi4zNjMgOTEuNjYxNCAxMjIuNjg0IDkxLjc3OCAxMjIuOTI1QzkyLjU5NDQgMTI0LjYyOCA5NC40OTA3IDEyNS40MDcgOTYuMDE1OSAxMjQuNjY5Qzk3LjUyOTIgMTIzLjkzNyA5OC4xMDc3IDEyMS45ODYgOTcuMzE5NCAxMjAuMjkzQzk3LjIwMjMgMTIwLjA0OSA5Ny4wNDEyIDExOS42OTUgOTYuOTE5OCAxMTkuNDY0Qzk2LjQxMjcgMTE4LjQ5OSA5NS44Mzc3IDExNy44NTIgOTUuMjc1OCAxMTcuMDA5Qzk0LjExNTIgMTE1LjE4NSA5My4xNTI2IDExMy42NyA5Mi41NTc1IDExMi4xMkM5Mi4zMDg3IDExMS4zMiA5Mi41OTk1IDExMC44MjMgOTIuNzkzMyAxMTAuMzAzQzkyLjY3NzIgMTEwLjE3IDkyLjQyODggMTA5LjQxNCA5Mi4yODI0IDEwOS4wNTlDMTAwLjc2MiAxMDQuMDI5IDEwNy4wMTcgOTUuOTk4NSAxMDkuOTU1IDg2LjcyMzlDMTEwLjM1MSA4Ni43ODY1IDExMS4wNDEgODYuOTA5MSAxMTEuMjY1IDg2Ljk1NDJDMTExLjcyNiA4Ni42NDg3IDExMi4xNDkgODYuMjUwMSAxMTIuOTgxIDg2LjMxNTlDMTE0LjYxNyA4Ni41NTM3IDExNi4yOTcgODcuMTY0NSAxMTguMzI2IDg3Ljg5NTNDMTE5LjI2OCA4OC4yNTgxIDEyMC4wMjggODguNjc5MyAxMjEuMDc3IDg4Ljk2MTRDMTIxLjI5OCA4OS4wMjEgMTIxLjYxNiA4OS4wNzY2IDEyMS44NjkgODkuMTMyNUMxMjEuODg5IDg5LjEzNzUgMTIxLjkwOCA4OS4xNDc1IDEyMS45MjggODkuMTUyMkMxMjEuOTQyIDg5LjE1NTMgMTIxLjk2MSA4OS4xNTU4IDEyMS45NzQgODkuMTU4OEMxMjMuNzkyIDg5LjU1MiAxMjUuNTU3IDg4LjU1MjYgMTI1LjkzIDg2LjkwODFDMTI2LjMwMyA4NS4yNjQxIDEyNS4xNDMgODMuNTk1MiAxMjMuMzM2IDgzLjE1N0MxMjMuMDc0IDgzLjA5NyAxMjIuNzAxIDgyLjk5NSAxMjIuNDQ2IDgyLjk0NjVDMTIxLjM3OSA4Mi43NDM1IDEyMC41MTEgODIuNzkzNSAxMTkuNTA1IDgyLjcwOTVDMTE3LjM2MSA4Mi40ODM5IDExNS41ODYgODIuMzAwNCAxMTQuMDA5IDgxLjgwMTRDMTEzLjM2NiA4MS41NTA3IDExMi45MDggODAuNzgxOSAxMTIuNjg2IDgwLjQ2NTVMMTExLjQ0OCA4MC4xMDM1QzExMi4wOSA3NS40MzggMTExLjkxNyA3MC41ODI1IDExMC44MDYgNjUuNzI0M0MxMDkuNjg1IDYwLjgyMDggMTA3LjcwNCA1Ni4zMzYxIDEwNS4wNjIgNTIuMzg0OEMxMDUuMzc5IDUyLjA5NDggMTA1Ljk3OSA1MS41NjEyIDEwNi4xNDkgNTEuNDA0M0MxMDYuMTk5IDUwLjg1MTcgMTA2LjE1NiA1MC4yNzIyIDEwNi43MjUgNDkuNjYwM0MxMDcuOTMxIDQ4LjUyMyAxMDkuNDUxIDQ3LjU3OTkgMTExLjI4NCA0Ni40NDIzQzExMi4xNTQgNDUuOTI3OSAxMTIuOTU5IDQ1LjU5NjQgMTEzLjgzMiA0NC45NDg0QzExNC4wMyA0NC44MDE5IDExNC4yOTkgNDQuNTY5OSAxMTQuNTA3IDQ0LjQwMjJDMTE1Ljk3NyA0My4yMjM3IDExNi4zMTYgNDEuMTkxMSAxMTUuMjYgMzkuODYxNEMxMTQuMjA0IDM4LjUzMTcgMTEyLjE1OSAzOC40MDY1IDExMC42ODggMzkuNTg1QzExMC40NzkgMzkuNzUxNiAxMTAuMTk1IDM5Ljk2ODggMTEwLjAwNyA0MC4xMzEyQzEwOS4xODQgNDAuODQyNiAxMDguNjc2IDQxLjU0NTIgMTA3Ljk4MyA0Mi4yODMyQzEwNi40NzEgNDMuODI1OSAxMDUuMjIyIDQ1LjExMyAxMDMuODUgNDYuMDQwOUMxMDMuMjU1IDQ2LjM4ODUgMTAyLjM4NSA0Ni4yNjgyIDEwMS45OSA0Ni4yNDQ5TDEwMC44MjQgNDcuMDgwNkM5NC4xNzUzIDQwLjA3NjMgODUuMTIzNSAzNS41OTgyIDc1LjM3NjYgMzQuNzI4M0M3NS4zNDk0IDM0LjMxNzkgNzUuMzEzNyAzMy41NzYxIDc1LjMwNDYgMzMuMzUyOUM3NC45MDU2IDMyLjk2OTMgNzQuNDIzNSAzMi42NDE4IDc0LjMwMjQgMzEuODEzQzc0LjE2OTEgMzAuMTU2OSA3NC4zOTE3IDI4LjM3NTIgNzQuNjQ5NiAyNi4yMjU4Qzc0Ljc5MTkgMjUuMjIxNiA3NS4wMjg0IDI0LjM4NzMgNzUuMDY4OCAyMy4yOTczQzc1LjA3OCAyMy4wNDk1IDc1LjA2MzIgMjIuNjkgNzUuMDYyMiAyMi40MjIxQzc1LjA2MiAyMC41MzIyIDczLjY4OTggMTguOTk5OCA3MS45OTY4IDE5Wk02OC4xNTg1IDQyLjg4ODZMNjcuMjQ4IDU5LjA0NDdMNjcuMTgyNSA1OS4wNzc2QzY3LjEyMTQgNjAuNTIyOSA2NS45Mzc1IDYxLjY3NyA2NC40ODM5IDYxLjY3N0M2My44ODg0IDYxLjY3NyA2My4zMzg4IDYxLjQ4NDkgNjIuODkyMiA2MS4xNTcxTDYyLjg2NiA2MS4xNzAzTDQ5LjY4MDcgNTEuNzc5NEM1My43MzMxIDQ3Ljc3NTkgNTguOTE2NCA0NC44MTcyIDY0Ljg5IDQzLjQ1NDZDNjUuOTgxMiA0My4yMDU2IDY3LjA3MTkgNDMuMDIwOSA2OC4xNTg1IDQyLjg4ODZaTTc1Ljg0MTcgNDIuODg4NkM4Mi44MTU5IDQzLjc1MDQgODkuMjY1NyA0Ni45MjMyIDk0LjIwODEgNTEuNzg2TDgxLjEwOCA2MS4xMTc2TDgxLjA2MjEgNjEuMDk3OUM3OS44OTk0IDYxLjk1MTIgNzguMjYxMSA2MS43Mzk0IDc3LjM1NDggNjAuNTk3OEM3Ni45ODM1IDYwLjEzMDEgNzYuNzg4NyA1OS41ODAxIDc2Ljc2NTMgNTkuMDI0OUw3Ni43NTIyIDU5LjAxODRMNzUuODQxNyA0Mi44ODg2Wk00NC44OTkxIDU3LjgxNEw1Ni45MzgyIDY4LjYzM0w1Ni45MjUxIDY4LjY5ODhDNTguMDExNyA2OS42NDc5IDU4LjE3MiA3MS4yOTQ5IDU3LjI2NTcgNzIuNDM2OEM1Ni44OTQ0IDcyLjkwNDUgNTYuMzk3NSA3My4yMTgyIDU1Ljg2MzkgNzMuMzY0N0w1NS44NTA4IDczLjQxNzNMNDAuNDE4OCA3Ny44OTIzQzM5LjYzMzQgNzAuNjc2NSA0MS4zMjYxIDYzLjY2MjEgNDQuODk5MSA1Ny44MTRaTTk5LjAwOTQgNTcuODIwNkMxMDAuNzk4IDYwLjczMzYgMTAyLjE1MyA2My45ODcxIDEwMi45NTkgNjcuNTE0M0MxMDMuNzU2IDcwLjk5OTEgMTAzLjk1NiA3NC40Nzc4IDEwMy42MjcgNzcuODM5N0w4OC4xMTY2IDczLjM1MTVMODguMTAzNSA3My4yODU3Qzg2LjcxNDUgNzIuOTA0MyA4NS44NjA5IDcxLjQ4NDggODYuMTg0MyA3MC4wNjExQzg2LjMxNjggNjkuNDc3OCA4Ni42MjQ5IDY4Ljk4NDQgODcuMDQyMyA2OC42MTk4TDg3LjAzNTggNjguNTg2OUw5OS4wMDk0IDU3LjgyMDZaTTY5LjUyNzQgNjkuNDY4OEg3NC40NTk2TDc3LjUyNTEgNzMuMzE4Nkw3Ni40MjQ3IDc4LjEyMjZMNzEuOTk2OCA4MC4yNjE0TDY3LjU1NTggNzguMTE2MUw2Ni40NTU0IDczLjMxMkw2OS41Mjc0IDY5LjQ2ODhaTTg1LjMzOTMgODIuNjQzN0M4NS41NDg5IDgyLjYzMzEgODUuNzU3NiA4Mi42NTIgODUuOTYxNiA4Mi42ODk4TDg1Ljk4NzggODIuNjU2OUwxMDEuOTUgODUuMzY4MkM5OS42MTQyIDkxLjk2MjQgOTUuMTQ0IDk3LjY3NSA4OS4xNzExIDEwMS40OThMODIuOTc0NyA4Ni40NjA2TDgyLjk5NDQgODYuNDM0M0M4Mi40MjUyIDg1LjEwNTUgODIuOTk0OCA4My41NDcyIDg0LjMwNDQgODIuOTEzNUM4NC42Mzk3IDgyLjc1MTMgODQuOTkgODIuNjYxNCA4NS4zMzkzIDgyLjY0MzdaTTU4LjUyOTggODIuNzA5NUM1OS43NDggODIuNzI2NyA2MC44NDA2IDgzLjU3NjEgNjEuMTIzNyA4NC44MjJDNjEuMjU2MiA4NS40MDUyIDYxLjE5MTcgODUuOTgzMSA2MC45NzMgODYuNDkzNUw2MS4wMTg5IDg2LjU1MjdMNTQuODg4IDEwMS40MzlDNDkuMTU1OSA5Ny43NDMyIDQ0LjU5MDQgOTIuMjA5OSA0Mi4xNDgxIDg1LjQyMDhMNTcuOTczMSA4Mi43MjI3TDU3Ljk5OTMgODIuNzU1NkM1OC4xNzYzIDgyLjcyMjkgNTguMzU1OCA4Mi43MDcxIDU4LjUyOTggODIuNzA5NVpNNzEuODk4NiA4OS4yMzEyQzcyLjMyMjkgODkuMjE1NSA3Mi43NTM0IDg5LjMwMyA3My4xNjI3IDg5LjUwMUM3My42OTkyIDg5Ljc2MDYgNzQuMTEzNiA5MC4xNjkyIDc0LjM3NDUgOTAuNjU5Mkg3NC40MzM0TDgyLjIzNDYgMTA0LjgyMUM4MS4yMjIxIDEwNS4xNjIgODAuMTgxMyAxMDUuNDU0IDc5LjExNjcgMTA1LjY5N0M3My4xNTA1IDEwNy4wNTggNjcuMjAzMiAxMDYuNjQ1IDYxLjgxOCAxMDQuODAyTDY5LjU5OTUgOTAuNjY1OEg2OS42MTI2QzcwLjA3OTUgODkuNzg4OCA3MC45NjUgODkuMjY1NiA3MS44OTg2IDg5LjIzMTJaIiBmaWxsPSJ3aGl0ZSIgc3Ryb2tlPSJ3aGl0ZSIgc3Ryb2tlLXdpZHRoPSIwLjI1Ii8+CjxkZWZzPgo8bGluZWFyR3JhZGllbnQgaWQ9InBhaW50MF9saW5lYXJfNjgxXzI4NDUiIHgxPSIxMCIgeTE9IjE1LjUiIHgyPSIxNDQiIHkyPSIxMzEuNSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiPgo8c3RvcCBzdG9wLWNvbG9yPSIjNEQ4N0ZGIi8+CjxzdG9wIG9mZnNldD0iMSIgc3RvcC1jb2xvcj0iIzA1NDdEMCIvPgo8L2xpbmVhckdyYWRpZW50Pgo8L2RlZnM+Cjwvc3ZnPgo= - keysOrder: [["apiVersion"],["appVersion"],["kind"],["metadata"],["metadata","name"],["spec","storageClass"],["spec","nodeGroups"],["spec","nodeGroups","md0"],["spec","nodeGroups","md0","minReplicas"],["spec","nodeGroups","md0","maxReplicas"],["spec","nodeGroups","md0","instanceType"],["spec","nodeGroups","md0","ephemeralStorage"],["spec","nodeGroups","md0","roles"],["spec","nodeGroups","md0","resources"],["spec","nodeGroups","md0","gpus"],["spec","version"],["spec","host"],["spec","addons"],["spec","addons","certManager"],["spec","addons","certManager","enabled"],["spec","addons","certManager","valuesOverride"],["spec","addons","cilium"],["spec","addons","cilium","valuesOverride"],["spec","addons","gatewayAPI"],["spec","addons","gatewayAPI","enabled"],["spec","addons","ingressNginx"],["spec","addons","ingressNginx","enabled"],["spec","addons","ingressNginx","exposeMethod"],["spec","addons","ingressNginx","hosts"],["spec","addons","ingressNginx","valuesOverride"],["spec","addons","gpuOperator"],["spec","addons","gpuOperator","enabled"],["spec","addons","gpuOperator","valuesOverride"],["spec","addons","fluxcd"],["spec","addons","fluxcd","enabled"],["spec","addons","fluxcd","valuesOverride"],["spec","addons","monitoringAgents"],["spec","addons","monitoringAgents","enabled"],["spec","addons","monitoringAgents","valuesOverride"],["spec","addons","verticalPodAutoscaler"],["spec","addons","verticalPodAutoscaler","valuesOverride"],["spec","addons","velero"],["spec","addons","velero","enabled"],["spec","addons","velero","valuesOverride"],["spec","addons","coredns"],["spec","addons","coredns","valuesOverride"],["spec","controlPlane"],["spec","controlPlane","replicas"],["spec","controlPlane","apiServer"],["spec","controlPlane","apiServer","resources"],["spec","controlPlane","apiServer","resourcesPreset"],["spec","controlPlane","controllerManager"],["spec","controlPlane","controllerManager","resources"],["spec","controlPlane","controllerManager","resourcesPreset"],["spec","controlPlane","scheduler"],["spec","controlPlane","scheduler","resources"],["spec","controlPlane","scheduler","resourcesPreset"],["spec","controlPlane","konnectivity"],["spec","controlPlane","konnectivity","server"],["spec","controlPlane","konnectivity","server","resources"],["spec","controlPlane","konnectivity","server","resourcesPreset"],["spec","addons","hami"],["spec","addons","hami","enabled"],["spec","addons","hami","valuesOverride"],["spec","images"],["spec","images","waitForKubeconfig"]] + keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "nodeGroups"], ["spec", "nodeGroups", "md0"], ["spec", "nodeGroups", "md0", "minReplicas"], ["spec", "nodeGroups", "md0", "maxReplicas"], ["spec", "nodeGroups", "md0", "instanceType"], ["spec", "nodeGroups", "md0", "ephemeralStorage"], ["spec", "nodeGroups", "md0", "roles"], ["spec", "nodeGroups", "md0", "resources"], ["spec", "nodeGroups", "md0", "gpus"], ["spec", "version"], ["spec", "host"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "hami"], ["spec", "addons", "hami", "enabled"], ["spec", "addons", "hami", "valuesOverride"], ["spec", "addons", "fluxcd"], ["spec", "addons", "fluxcd", "enabled"], ["spec", "addons", "fluxcd", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"], ["spec", "images"], ["spec", "images", "waitForKubeconfig"]] secrets: exclude: [] include: From 805b3b17cbc4d89206203bc24b45c1412c29a2c8 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 19:11:26 +0300 Subject: [PATCH 107/130] fix(kubernetes): add CI values to GPU Operator HAMi tests Tests need _namespace.etcd from values-ci.yaml after the merge introduced an etcd-namespace guard on the gpu-operator HelmRelease condition. Signed-off-by: Arsolitt --- packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml index 4fa754fb..44528470 100644 --- a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml +++ b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml @@ -1,6 +1,8 @@ suite: GPU Operator HelmRelease HAMi integration tests templates: - templates/helmreleases/gpu-operator.yaml +values: + - values-ci.yaml tests: - it: should disable devicePlugin when hami is enabled set: From c7c4f724afd86f11d40bc1864175c81c1a4b17f7 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Tue, 28 Apr 2026 23:35:46 +0300 Subject: [PATCH 108/130] fix(dashboard): add VNC access permissions and update UI image - Add RBAC permissions for VNC console access to virtual machines - Update cozystack-ui image to latest version with VNC fixes Signed-off-by: IvanHunters --- packages/system/dashboard/templates/rbac.yaml | 14 ++++++++++++++ packages/system/dashboard/values.yaml | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/system/dashboard/templates/rbac.yaml b/packages/system/dashboard/templates/rbac.yaml index c64e851f..b5d2b642 100644 --- a/packages/system/dashboard/templates/rbac.yaml +++ b/packages/system/dashboard/templates/rbac.yaml @@ -36,6 +36,20 @@ rules: - get - list - watch +- apiGroups: + - kubevirt.io + resources: + - virtualmachineinstances + verbs: + - get + - list + - watch +- apiGroups: + - subresources.kubevirt.io + resources: + - virtualmachineinstances/vnc + verbs: + - get --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/packages/system/dashboard/values.yaml b/packages/system/dashboard/values.yaml index dc0384b0..dbe330ea 100644 --- a/packages/system/dashboard/values.yaml +++ b/packages/system/dashboard/values.yaml @@ -1,4 +1,4 @@ openapiUI: - image: 999669/cozystack-ui:latest@sha256:ea4e832c277ecf16dde99b4bd82ac5334d03d3c9cb0572fe896e2f128a6e6dc1 + image: 999669/cozystack-ui:latest@sha256:a93a92a8f7b8d7c4842876215f737d782ad373ff8f4ceb89d7b536945184341e tokenProxy: image: ghcr.io/cozystack/cozystack/token-proxy:v1.3.0@sha256:2e280991e07853ea48f97b0a42946afffa10d03d6a83d41099ed83e6ffc94fdc From 8b037d3045cb22b7bc91eeb58fdcf8f2f007916f Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Tue, 28 Apr 2026 23:38:58 +0300 Subject: [PATCH 109/130] refactor(dashboard): migrate to new cozystack-ui console MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove old openapi-ui implementation and adopt new React-based console: **Removed:** - images/openapi-ui/ - old PRO-Robotech based UI - images/openapi-ui-k8s-bff/ - unused backend-for-frontend - templates/nginx.yaml - separate nginx proxy (replaced by built-in nginx) - templates/configmap.yaml - old UI configuration **Changes:** - Rename web → console throughout all manifests - Rename incloud-web-* → cozy-dashboard-* for all resources - Remove all environment variables (new UI requires no configuration) - Update gatekeeper upstream to point directly to console service - Simplify Makefile (only token-proxy image build remains) - Update values.yaml: openapiUI → console **Architecture changes:** Before: gatekeeper → nginx → web (openapi-ui) After: gatekeeper → console (React SPA with built-in nginx) The new console includes nginx that proxies /api and /apis to K8s API, eliminating the need for a separate nginx deployment. Authentication flow remains unchanged - gatekeeper adds Authorization header which is passed through to K8s API. Breaking change: Resource names changed from incloud-web-* to cozy-dashboard-*. Existing deployments will need migration. Signed-off-by: IvanHunters --- packages/system/dashboard/Makefile | 17 +- .../images/openapi-ui-k8s-bff/Dockerfile | 22 -- .../dashboard/images/openapi-ui/Dockerfile | 69 ------ .../flatmap-unresolved-placeholder.diff | 17 -- .../patches/formlistinput-allow-empty.diff | 37 ---- .../secret-copy-preserve-newlines.diff | 29 --- .../patches/tenantmodules.diff | 50 ----- .../system/dashboard/templates/configmap.yaml | 24 --- .../{nginx-sa.yaml => console-sa.yaml} | 2 +- .../{nginx-svc.yaml => console-svc.yaml} | 8 +- .../templates/{nginx.yaml => console.yaml} | 39 ++-- .../dashboard/templates/gatekeeper.yaml | 4 +- .../dashboard/templates/nginx-config.yaml | 97 --------- .../system/dashboard/templates/web-sa.yaml | 4 - .../system/dashboard/templates/web-svc.yaml | 18 -- packages/system/dashboard/templates/web.yaml | 196 ------------------ packages/system/dashboard/values.yaml | 2 +- 17 files changed, 26 insertions(+), 609 deletions(-) delete mode 100644 packages/system/dashboard/images/openapi-ui-k8s-bff/Dockerfile delete mode 100644 packages/system/dashboard/images/openapi-ui/Dockerfile delete mode 100644 packages/system/dashboard/images/openapi-ui/openapi-k8s-toolkit/patches/flatmap-unresolved-placeholder.diff delete mode 100644 packages/system/dashboard/images/openapi-ui/openapi-k8s-toolkit/patches/formlistinput-allow-empty.diff delete mode 100644 packages/system/dashboard/images/openapi-ui/openapi-k8s-toolkit/patches/secret-copy-preserve-newlines.diff delete mode 100644 packages/system/dashboard/images/openapi-ui/openapi-k8s-toolkit/patches/tenantmodules.diff delete mode 100644 packages/system/dashboard/templates/configmap.yaml rename packages/system/dashboard/templates/{nginx-sa.yaml => console-sa.yaml} (59%) rename packages/system/dashboard/templates/{nginx-svc.yaml => console-svc.yaml} (52%) rename packages/system/dashboard/templates/{nginx.yaml => console.yaml} (72%) delete mode 100644 packages/system/dashboard/templates/nginx-config.yaml delete mode 100644 packages/system/dashboard/templates/web-sa.yaml delete mode 100644 packages/system/dashboard/templates/web-svc.yaml delete mode 100644 packages/system/dashboard/templates/web.yaml diff --git a/packages/system/dashboard/Makefile b/packages/system/dashboard/Makefile index b136c864..b46252a3 100644 --- a/packages/system/dashboard/Makefile +++ b/packages/system/dashboard/Makefile @@ -4,19 +4,7 @@ export NAMESPACE=cozy-$(NAME) include ../../../hack/common-envs.mk include ../../../hack/package.mk -update: update-crd update-dockerfiles -image: image-token-proxy update-tenant-text - - -update-dockerfiles: - @echo Update dockerfiles manually - -update-crd: - rm -rf crds - mkdir -p crds - wget -O- https://github.com/PRO-Robotech/helmfile-manifests/archive/refs/heads/main.tar.gz | tar -C crds -xzvf- helmfile-manifests-main/charts/incloud-main/incloud-web-1.0.0/incloud-web/templates --strip-components=6 - rm -f crds/_helpers.tpl - sed -i '/{{/d' crds/*.yml crds/*.yaml +image: image-token-proxy image-token-proxy: docker buildx build images/token-proxy \ @@ -33,6 +21,3 @@ image-token-proxy: IMAGE="$(REGISTRY)/token-proxy:$(call settag,$(TAG))@$$(yq e '."containerimage.digest"' images/token-proxy.json -r)" \ yq -i '.tokenProxy.image = strenv(IMAGE)' values.yaml rm -f images/token-proxy.json - -update-tenant-text: - sed -i 's|\($$tenantText := "\)[^"]\+|\1$(TAG)|' templates/configmap.yaml diff --git a/packages/system/dashboard/images/openapi-ui-k8s-bff/Dockerfile b/packages/system/dashboard/images/openapi-ui-k8s-bff/Dockerfile deleted file mode 100644 index ea0c900d..00000000 --- a/packages/system/dashboard/images/openapi-ui-k8s-bff/Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -# imported from https://github.com/PRO-Robotech/openapi-ui-k8s-bff -ARG NODE_VERSION=20.18.1 -FROM node:${NODE_VERSION}-alpine AS builder -WORKDIR /src - -# release/1.4.0 -ARG COMMIT_REF=92e4b618eb9ad17b19827b5a2b7ceab33e8cf534 -RUN wget -O- https://github.com/PRO-Robotech/openapi-ui-k8s-bff/archive/${COMMIT_REF}.tar.gz | tar xzf - --strip-components=1 - -ENV PATH=/src/node_modules/.bin:$PATH -RUN npm install -RUN npm run build - -FROM node:${NODE_VERSION}-alpine -WORKDIR /app -COPY --from=builder /src/package*.json /app/ -COPY --from=builder /src/node_modules /app/node_modules -COPY --from=builder /src/src/swagger/swagger-output.json /app/dist/swagger/swagger-output.json -COPY --from=builder /src/dist /app/dist -EXPOSE 8080 -USER 1001 -CMD [ "node", "/app/dist/index.js"] diff --git a/packages/system/dashboard/images/openapi-ui/Dockerfile b/packages/system/dashboard/images/openapi-ui/Dockerfile deleted file mode 100644 index 82cc6db0..00000000 --- a/packages/system/dashboard/images/openapi-ui/Dockerfile +++ /dev/null @@ -1,69 +0,0 @@ -ARG NODE_VERSION=20.18.1 - -# openapi-k8s-toolkit -# imported from https://github.com/PRO-Robotech/openapi-k8s-toolkit -FROM node:${NODE_VERSION}-alpine AS openapi-k8s-toolkit-builder -RUN apk add git -WORKDIR /src -# release/1.4.0 -ARG COMMIT=d6b9e4ad0d1eb9d3730f7f0c664792c8dda3214d -RUN wget -O- https://github.com/PRO-Robotech/openapi-k8s-toolkit/archive/${COMMIT}.tar.gz | tar -xzvf- --strip-components=1 - -COPY openapi-k8s-toolkit/patches /patches -RUN git apply /patches/*.diff - -RUN npm install -RUN npm install --build-from-source @swc/core -RUN npm run build - - -# openapi-ui -# imported from https://github.com/PRO-Robotech/openapi-ui -FROM node:${NODE_VERSION}-alpine AS builder -#RUN apk add git -WORKDIR /src - -# release/1.4.0 -ARG COMMIT_REF=6addca6939264ef2e39801baa88c1460cc1aa53e -RUN wget -O- https://github.com/PRO-Robotech/openapi-ui/archive/${COMMIT_REF}.tar.gz | tar xzf - --strip-components=1 - -#COPY openapi-ui/patches /patches -#RUN git apply /patches/*.diff - -ENV PATH=/src/node_modules/.bin:$PATH - -RUN npm install - -# add patched openapi-k8s-toolkit -RUN rm -rf node_modules/@prorobotech/openapi-k8s-toolkit/dist -COPY --from=openapi-k8s-toolkit-builder /src/dist node_modules/@prorobotech/openapi-k8s-toolkit/dist - -RUN npm run build - -FROM node:${NODE_VERSION}-alpine AS builder2 -WORKDIR /src -ENV PATH=/src/node_modules/.bin:$PATH - -COPY --from=builder /src/server/package.json ./ -COPY --from=builder /src/server/package-lock.json ./ -RUN npm install -COPY --from=builder /src/server server -COPY --from=builder /src/tsconfig.server.json ./ -COPY --from=builder /src/build /src/build -RUN npm run server:build - -FROM node:${NODE_VERSION}-alpine -WORKDIR /app -COPY --from=builder2 /src/node_modules /app/node_modules -COPY --from=builder2 /src/build /app/build -EXPOSE 8080 -RUN sed -i -e 's|OpenAPI UI|Cozystack|g' build/index.html -# Fix Factory component: return null while loading instead of showing "Factory Not Found" 404 -RUN APP_JS=$(find build -name "App-react.js" -type f | head -1) && \ - if [ -n "$APP_JS" ]; then \ - sed -i 's|const { data: factoryData } = useK8sSmartResource({|const { data: factoryData, isLoading: factoryIsLoading } = useK8sSmartResource({|' "$APP_JS" && \ - sed -i '/Factory Not Found/s/return /return factoryIsLoading ? null : /' "$APP_JS" && \ - echo "Factory loading patch applied to $APP_JS"; \ - fi -USER 1001 -CMD ["node", "/app/build/index.js"] diff --git a/packages/system/dashboard/images/openapi-ui/openapi-k8s-toolkit/patches/flatmap-unresolved-placeholder.diff b/packages/system/dashboard/images/openapi-ui/openapi-k8s-toolkit/patches/flatmap-unresolved-placeholder.diff deleted file mode 100644 index e98861a3..00000000 --- a/packages/system/dashboard/images/openapi-ui/openapi-k8s-toolkit/patches/flatmap-unresolved-placeholder.diff +++ /dev/null @@ -1,17 +0,0 @@ -diff --git a/src/components/molecules/EnrichedTable/organisms/EnrichedTableProvider/utils.ts b/src/components/molecules/EnrichedTable/organisms/EnrichedTableProvider/utils.ts ---- a/src/components/molecules/EnrichedTable/organisms/EnrichedTableProvider/utils.ts -+++ b/src/components/molecules/EnrichedTable/organisms/EnrichedTableProvider/utils.ts -@@ -185,6 +185,11 @@ - return `['${escaped}']` - }) - } -- const jpQueryResult = jp.query(el, `$${resolvedJsonPath}`) -- fieldValue = Array.isArray(jpQueryResult) && jpQueryResult.length === 1 ? jpQueryResult[0] : jpQueryResult -+ if (/_flatMap[^\]]+_Key/.test(resolvedJsonPath)) { -+ // Placeholder was not resolved (row not yet expanded or key missing) — skip query -+ fieldValue = null -+ } else { -+ const jpQueryResult = jp.query(el, `$${resolvedJsonPath}`) -+ fieldValue = Array.isArray(jpQueryResult) && jpQueryResult.length === 1 ? jpQueryResult[0] : jpQueryResult -+ } - } diff --git a/packages/system/dashboard/images/openapi-ui/openapi-k8s-toolkit/patches/formlistinput-allow-empty.diff b/packages/system/dashboard/images/openapi-ui/openapi-k8s-toolkit/patches/formlistinput-allow-empty.diff deleted file mode 100644 index 1c83df7c..00000000 --- a/packages/system/dashboard/images/openapi-ui/openapi-k8s-toolkit/patches/formlistinput-allow-empty.diff +++ /dev/null @@ -1,37 +0,0 @@ -diff --git a/src/localTypes/formExtensions.ts b/src/localTypes/formExtensions.ts ---- a/src/localTypes/formExtensions.ts -+++ b/src/localTypes/formExtensions.ts -@@ -59,2 +59,4 @@ - relatedValuePath?: string -+ allowEmpty?: boolean -+ persistType?: 'str' | 'number' | 'arr' | 'obj' - } -diff --git a/src/components/molecules/BlackholeForm/molecules/FormListInput/FormListInput.tsx b/src/components/molecules/BlackholeForm/molecules/FormListInput/FormListInput.tsx ---- a/src/components/molecules/BlackholeForm/molecules/FormListInput/FormListInput.tsx -+++ b/src/components/molecules/BlackholeForm/molecules/FormListInput/FormListInput.tsx -@@ -149,3 +149,10 @@ - }, [relatedPath, form, arrName, fixedName, relatedFieldValue, onValuesChangeCallBack, isTouchedPeristed]) - -+ // When allowEmpty is set, auto-persist the field so the BFF preserves empty values -+ useEffect(() => { -+ if (customProps.allowEmpty) { -+ persistedControls.onPersistMark(persistName || name, customProps.persistType ?? 'str') -+ } -+ }, [customProps.allowEmpty, customProps.persistType, persistedControls, persistName, name]) -+ - const uri = prepareTemplate({ -@@ -267,5 +274,14 @@ - validateTrigger="onBlur" - hasFeedback={designNewLayout ? { icons: feedbackIcons } : true} - style={{ flex: 1 }} -+ normalize={(value: unknown) => { -+ if (customProps.allowEmpty && (value === undefined || value === null)) { -+ if (customProps.persistType === 'number') return 0 -+ if (customProps.persistType === 'arr') return [] -+ if (customProps.persistType === 'obj') return {} -+ return '' -+ } -+ return value -+ }} - > -