diff --git a/.github/workflows/codegen-drift.yml b/.github/workflows/codegen-drift.yml deleted file mode 100644 index a26caf5e..00000000 --- a/.github/workflows/codegen-drift.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: Codegen Drift Check - -on: - pull_request: - types: [opened, synchronize, reopened] - paths: - - 'api/**' - - 'pkg/apis/**' - - 'pkg/generated/**' - - 'internal/crdinstall/manifests/**' - - 'packages/system/cozystack-controller/definitions/**' - - 'packages/system/application-definition-crd/definition/**' - - 'packages/system/backup-controller/definitions/**' - - 'packages/system/backupstrategy-controller/definitions/**' - - 'hack/update-codegen.sh' - - 'hack/boilerplate.go.txt' - - 'Makefile' - - 'go.mod' - - 'go.sum' - - '.github/workflows/codegen-drift.yml' - -concurrency: - group: codegen-drift-${{ github.workflow }}-${{ github.event.pull_request.number }} - cancel-in-progress: true - -jobs: - codegen-drift: - name: Verify generated code is up to date - runs-on: ubuntu-22.04 - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version-file: go.mod - cache: true - - - name: Pre-fetch k8s.io/code-generator module - # hack/update-codegen.sh sources kube_codegen.sh from the Go module cache. - # The module is not declared in go.mod, so fetch it explicitly at the - # version pinned in the script. - run: | - version=$(grep -oP 'code-generator@\Kv[0-9.]+' hack/update-codegen.sh) - tmpdir=$(mktemp -d) - cd "$tmpdir" - go mod init codegen-fetch - go get "k8s.io/code-generator@${version}" - - - name: Run make generate - run: make generate - - - name: Fail on drift - run: | - if [ -n "$(git status --porcelain)" ]; then - echo "::error::'make generate' produced changes. Run 'make generate' locally and commit the result." - git status --short - git diff --color=always - exit 1 - fi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6492b92d..ac0f7e30 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,17 +1,6 @@ repos: - repo: local hooks: - - id: run-make-generate-root - name: Run 'make generate' at repo root - entry: | - flock -x .git/pre-commit.lock sh -c ' - echo "Running make generate at repo root" - make generate || exit $? - git diff --color=always | cat - ' - language: system - files: ^(api/|pkg/apis/|pkg/generated/|internal/crdinstall/manifests/|packages/system/cozystack-controller/definitions/|packages/system/application-definition-crd/definition/|packages/system/backup-controller/definitions/|packages/system/backupstrategy-controller/definitions/|hack/update-codegen\.sh$|hack/boilerplate\.go\.txt$|Makefile$) - pass_filenames: false - id: run-make-generate name: Run 'make generate' in all app directories entry: | diff --git a/Makefile b/Makefile index acc9b6f6..10a55314 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,6 @@ build: build-deps make -C packages/system/lineage-controller-webhook image make -C packages/system/cilium image make -C packages/system/linstor image - make -C packages/system/linstor-gui image make -C packages/system/kubeovn-webhook image make -C packages/system/kubeovn-plunger image make -C packages/system/dashboard image diff --git a/api/apps/v1alpha1/kubernetes/types.go b/api/apps/v1alpha1/kubernetes/types.go index 88a3c97e..ea4a2b64 100644 --- a/api/apps/v1alpha1/kubernetes/types.go +++ b/api/apps/v1alpha1/kubernetes/types.go @@ -69,9 +69,6 @@ type Addons struct { // NVIDIA GPU Operator. // +kubebuilder:default:={} GpuOperator GPUOperatorAddon `json:"gpuOperator"` - // HAMi GPU virtualization middleware. - // +kubebuilder:default:={} - Hami HAMiAddon `json:"hami"` // Ingress-NGINX controller. // +kubebuilder:default:={} IngressNginx IngressNginxAddon `json:"ingressNginx"` @@ -163,15 +160,6 @@ type GatewayAPIAddon struct { Enabled bool `json:"enabled"` } -type HAMiAddon struct { - // Enable HAMi (requires GPU Operator). - // +kubebuilder:default:=false - Enabled bool `json:"enabled"` - // Custom Helm values overrides. - // +kubebuilder:default:={} - ValuesOverride k8sRuntime.RawExtension `json:"valuesOverride"` -} - type Images struct { // Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag. // +kubebuilder:default:="" diff --git a/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go b/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go index e021fbaa..2cbe3ba1 100644 --- a/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go +++ b/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go @@ -49,7 +49,6 @@ func (in *Addons) DeepCopyInto(out *Addons) { in.Fluxcd.DeepCopyInto(&out.Fluxcd) out.GatewayAPI = in.GatewayAPI in.GpuOperator.DeepCopyInto(&out.GpuOperator) - in.Hami.DeepCopyInto(&out.Hami) in.IngressNginx.DeepCopyInto(&out.IngressNginx) in.MonitoringAgents.DeepCopyInto(&out.MonitoringAgents) in.Velero.DeepCopyInto(&out.Velero) @@ -136,7 +135,6 @@ func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) { } in.Addons.DeepCopyInto(&out.Addons) in.ControlPlane.DeepCopyInto(&out.ControlPlane) - out.Images = in.Images } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigSpec. @@ -262,37 +260,6 @@ func (in *GatewayAPIAddon) DeepCopy() *GatewayAPIAddon { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *HAMiAddon) DeepCopyInto(out *HAMiAddon) { - *out = *in - in.ValuesOverride.DeepCopyInto(&out.ValuesOverride) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HAMiAddon. -func (in *HAMiAddon) DeepCopy() *HAMiAddon { - if in == nil { - return nil - } - out := new(HAMiAddon) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Images) DeepCopyInto(out *Images) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Images. -func (in *Images) DeepCopy() *Images { - if in == nil { - return nil - } - out := new(Images) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IngressNginxAddon) DeepCopyInto(out *IngressNginxAddon) { *out = *in diff --git a/api/apps/v1alpha1/postgresql/types.go b/api/apps/v1alpha1/postgresql/types.go index a2ff77a8..fa85d51f 100644 --- a/api/apps/v1alpha1/postgresql/types.go +++ b/api/apps/v1alpha1/postgresql/types.go @@ -92,9 +92,6 @@ type Bootstrap struct { // Timestamp (RFC3339) for point-in-time recovery; empty means latest. // +kubebuilder:default:="" RecoveryTime string `json:"recoveryTime,omitempty"` - // Barman server name (S3 path prefix) used by the original cluster when writing backups. Set this only when the original cluster had an explicit barmanObjectStore.serverName that differed from its Kubernetes resource name. - // +kubebuilder:default:="" - ServerName string `json:"serverName,omitempty"` } type Database struct { diff --git a/api/apps/v1alpha1/vminstance/types.go b/api/apps/v1alpha1/vminstance/types.go index 9c48724a..2bb059c6 100644 --- a/api/apps/v1alpha1/vminstance/types.go +++ b/api/apps/v1alpha1/vminstance/types.go @@ -26,9 +26,6 @@ type ConfigSpec struct { // Ports to forward from outside the cluster. // +kubebuilder:default:={22} ExternalPorts []int `json:"externalPorts,omitempty"` - // Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect. - // +kubebuilder:default:=true - ExternalAllowICMP bool `json:"externalAllowICMP"` // Requested running state of the VirtualMachineInstance // +kubebuilder:default:="Always" RunStrategy RunStrategy `json:"runStrategy"` diff --git a/api/backups/v1alpha1/zz_generated.deepcopy.go b/api/backups/v1alpha1/zz_generated.deepcopy.go index 61b8f839..89f6171f 100644 --- a/api/backups/v1alpha1/zz_generated.deepcopy.go +++ b/api/backups/v1alpha1/zz_generated.deepcopy.go @@ -620,11 +620,6 @@ func (in *RestoreJobSpec) DeepCopyInto(out *RestoreJobSpec) { *out = new(v1.TypedLocalObjectReference) (*in).DeepCopyInto(*out) } - if in.Options != nil { - in, out := &in.Options, &out.Options - *out = new(runtime.RawExtension) - (*in).DeepCopyInto(*out) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestoreJobSpec. diff --git a/dashboards/gpu/gpu-efficiency.json b/dashboards/gpu/gpu-efficiency.json deleted file mode 100644 index 28a24568..00000000 --- a/dashboards/gpu/gpu-efficiency.json +++ /dev/null @@ -1,819 +0,0 @@ -{ - "uid": "gpu-efficiency", - "title": "GPU Efficiency Score", - "description": "Tensor saturation, util/watt and throttling — reveals inefficient GPU workloads", - "tags": [ - "gpu", - "efficiency", - "finops" - ], - "timezone": "browser", - "editable": true, - "graphTooltip": 1, - "time": { - "from": "now-1h", - "to": "now" - }, - "fiscalYearStartMonth": 0, - "schemaVersion": 42, - "panels": [ - { - "type": "row", - "collapsed": false, - "title": "Overall efficiency metrics", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [] - }, - { - "type": "stat", - "id": 2, - "targets": [ - { - "expr": "avg(gpu:tensor_saturation:avg5m) * 100", - "refId": "A" - } - ], - "title": "Avg Tensor Saturation", - "description": "Mean tensor core saturation across all GPUs. \u003c10% means GPUs are used inefficiently (workloads could move to CPU or optimize their code). Cluster-wide — DCGM metrics cannot be attributed to workload namespaces.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "area", - "colorMode": "background", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "red" - }, - { - "value": 10, - "color": "orange" - }, - { - "value": 30, - "color": "yellow" - }, - { - "value": 60, - "color": "green" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 3, - "targets": [ - { - "expr": "avg(gpu:util_per_watt:avg5m)", - "refId": "A" - } - ], - "title": "Avg Utilization per Watt", - "description": "NVML utilization % per watt across all GPUs. Higher value = more efficient workload. Cluster-wide — DCGM metrics cannot be attributed to workload namespaces.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 8, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "area", - "colorMode": "background", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "none", - "decimals": 3, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "red" - }, - { - "value": 0.5, - "color": "yellow" - }, - { - "value": 1, - "color": "green" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 4, - "targets": [ - { - "expr": "avg(gpu:power_throttle_fraction:rate5m)", - "refId": "A" - } - ], - "title": "Avg Power Throttling", - "description": "Fraction of time GPUs hit the TDP cap and lose performance. \u003e5% means tenants underutilize billed FLOPS. Cluster-wide — rule aggregates by (Hostname, gpu, UUID) and drops namespace label.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 16, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "area", - "colorMode": "background", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "percentunit", - "decimals": 2, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - }, - { - "value": 0.05, - "color": "yellow" - }, - { - "value": 0.2, - "color": "red" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "row", - "collapsed": false, - "title": "NVML vs Tensor (mismatch detector)", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 6 - }, - "id": 10, - "panels": [] - }, - { - "type": "timeseries", - "id": 11, - "targets": [ - { - "expr": "DCGM_FI_DEV_GPU_UTIL", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" - } - ], - "title": "NVML GPU Utilization", - "description": "Classic utilization metric. Shows activity of any engine (SM, copy, encoder). Cluster-wide — DCGM namespace is the exporter's own namespace, not the workload namespace.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 7 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "id": 12, - "targets": [ - { - "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE * 100", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" - } - ], - "title": "Tensor Pipe Active", - "description": "Real tensor core load (HMMA). For AI/LLM inference it should be ≥20%, otherwise the workload is unoptimized. Cluster-wide — DCGM namespace is the exporter's own namespace, not the workload namespace.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 7 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [] - } - }, - { - "type": "row", - "collapsed": false, - "title": "Per-GPU ranking", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 15 - }, - "id": 20, - "panels": [] - }, - { - "type": "table", - "id": 21, - "targets": [ - { - "expr": "topk(20, gpu:tensor_saturation:avg5m * 100)", - "instant": true, - "range": false, - "format": "table", - "refId": "A" - } - ], - "title": "Tensor Saturation per GPU (5m avg)", - "description": "Which GPUs are exercising tensor cores and which are not. Sorted descending. Grouped by Hostname/gpu/UUID — DCGM metrics cannot be attributed to workload namespaces.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "repeatDirection": "h", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "DCGM_FI_DRIVER_VERSION": true, - "Time": true, - "__name__": true, - "cluster": true, - "container": true, - "device": true, - "endpoint": true, - "gpu_driver_version": true, - "instance": true, - "job": true, - "modelName": true, - "namespace": true, - "pci_bus_id": true, - "pod": true, - "prometheus": true, - "service": true, - "tenant": true, - "tier": true, - "uid": true, - "unit": true - }, - "indexByName": { - "Hostname": 0, - "UUID": 2, - "Value": 3, - "gpu": 1 - }, - "renameByName": { - "Hostname": "Node", - "UUID": "UUID", - "Value": "Saturation", - "gpu": "GPU" - } - } - } - ], - "options": { - "frameIndex": 0, - "showHeader": true, - "showTypeIcons": false, - "sortBy": [ - { - "displayName": "Saturation", - "desc": true - } - ], - "footer": { - "show": false, - "reducer": [] - }, - "cellHeight": "sm" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Saturation" - }, - "properties": [ - { - "id": "unit", - "value": "percent" - }, - { - "id": "min", - "value": 0 - }, - { - "id": "max", - "value": 100 - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "gauge" - } - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "red" - }, - { - "color": "orange", - "value": 10 - }, - { - "color": "yellow", - "value": 30 - }, - { - "color": "green", - "value": 60 - } - ] - } - } - ] - } - ] - } - }, - { - "type": "table", - "id": 22, - "targets": [ - { - "expr": "topk(20, gpu:util_per_watt:avg5m)", - "instant": true, - "range": false, - "format": "table", - "refId": "A" - } - ], - "title": "Utilization / Watt per GPU (5m avg)", - "description": "How efficiently each GPU spends watts. Low value = poor optimization. Grouped by Hostname/gpu/UUID — DCGM metrics cannot be attributed to workload namespaces.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "repeatDirection": "h", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "DCGM_FI_DRIVER_VERSION": true, - "Time": true, - "__name__": true, - "cluster": true, - "container": true, - "device": true, - "endpoint": true, - "gpu_driver_version": true, - "instance": true, - "job": true, - "modelName": true, - "namespace": true, - "pci_bus_id": true, - "pod": true, - "prometheus": true, - "service": true, - "tenant": true, - "tier": true, - "uid": true, - "unit": true - }, - "indexByName": { - "Hostname": 0, - "UUID": 2, - "Value": 3, - "gpu": 1 - }, - "renameByName": { - "Hostname": "Node", - "UUID": "UUID", - "Value": "Util/Watt", - "gpu": "GPU" - } - } - } - ], - "options": { - "frameIndex": 0, - "showHeader": true, - "showTypeIcons": false, - "sortBy": [ - { - "displayName": "Util/Watt", - "desc": true - } - ], - "footer": { - "show": false, - "reducer": [] - }, - "cellHeight": "sm" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Util/Watt" - }, - "properties": [ - { - "id": "unit", - "value": "none" - }, - { - "id": "decimals", - "value": 3 - }, - { - "id": "min", - "value": 0 - }, - { - "id": "max", - "value": 1.5 - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "gauge" - } - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "red" - }, - { - "color": "yellow", - "value": 0.5 - }, - { - "color": "green", - "value": 1 - } - ] - } - } - ] - } - ] - } - }, - { - "type": "row", - "collapsed": false, - "title": "Throttling", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 24 - }, - "id": 30, - "panels": [] - }, - { - "type": "timeseries", - "id": 31, - "targets": [ - { - "expr": "gpu:power_throttle_fraction:rate5m", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" - } - ], - "title": "Power throttle fraction per GPU", - "description": "Fraction of time the GPU was power-throttled. 1.0 = always throttled. Cluster-wide — rule aggregates by (Hostname, gpu, UUID) and drops namespace label.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 25 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "percentunit", - "min": 0, - "max": 1, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - }, - { - "value": 0.05, - "color": "yellow" - }, - { - "value": 0.2, - "color": "red" - } - ] - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 25, - "showPoints": "never" - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "id": 32, - "targets": [ - { - "expr": "gpu:thermal_throttle_fraction:rate5m", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" - } - ], - "title": "Thermal throttle fraction per GPU", - "description": "Fraction of time the GPU was thermal-throttled. Cluster-wide — rule aggregates by (Hostname, gpu, UUID) and drops namespace label.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 25 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "percentunit", - "min": 0, - "max": 1, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - }, - { - "value": 0.05, - "color": "yellow" - }, - { - "value": 0.2, - "color": "red" - } - ] - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 25, - "showPoints": "never" - } - }, - "overrides": [] - } - } - ], - "templating": { - "list": [ - { - "type": "datasource", - "name": "ds_prometheus", - "label": "Prometheus", - "skipUrlSync": false, - "query": "prometheus", - "current": { - "selected": false, - "text": "default", - "value": "default" - }, - "multi": false, - "allowCustomValue": true, - "includeAll": false, - "regex": "", - "auto": false, - "auto_min": "10s", - "auto_count": 30 - } - ] - }, - "annotations": {} -} diff --git a/dashboards/gpu/gpu-fleet.json b/dashboards/gpu/gpu-fleet.json deleted file mode 100644 index 60c12bfb..00000000 --- a/dashboards/gpu/gpu-fleet.json +++ /dev/null @@ -1,1157 +0,0 @@ -{ - "uid": "gpu-fleet", - "title": "GPU Fleet Overview", - "description": "Cluster-wide GPU inventory, capacity, utilization and health — admin view across the whole fleet.", - "tags": [ - "gpu", - "fleet", - "admin" - ], - "timezone": "browser", - "editable": true, - "graphTooltip": 1, - "time": { - "from": "now-6h", - "to": "now" - }, - "fiscalYearStartMonth": 0, - "schemaVersion": 42, - "panels": [ - { - "type": "row", - "collapsed": false, - "title": "Cluster snapshot", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [] - }, - { - "type": "stat", - "id": 2, - "targets": [ - { - "expr": "cluster:gpu_count:total", - "refId": "A" - } - ], - "title": "Total GPUs", - "description": "Physical GPUs discovered by DCGM across all nodes.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "value", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "blue" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 3, - "targets": [ - { - "expr": "cluster:gpu_count:allocated", - "refId": "A" - } - ], - "title": "Allocated", - "description": "Sum of kube GPU requests across all pods. Billable view — includes Pending pods.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "background", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "blue" - }, - { - "value": 1, - "color": "green" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 4, - "targets": [ - { - "expr": "cluster:gpu_count:free", - "refId": "A" - } - ], - "title": "Free", - "description": "Unallocated capacity available for new tenants.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "background", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "red" - }, - { - "value": 1, - "color": "yellow" - }, - { - "value": 2, - "color": "green" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 5, - "targets": [ - { - "expr": "count(count by (Hostname) (DCGM_FI_DEV_GPU_UTIL))", - "refId": "A" - } - ], - "title": "Nodes with GPU", - "description": "Nodes reporting at least one GPU to DCGM. Counted via DCGM_FI_DEV_GPU_UTIL because kube-state-metrics does not expose node labels by default.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "value", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "blue" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 6, - "targets": [ - { - "expr": "cluster:gpu_util:avg", - "refId": "A" - } - ], - "title": "Average utilization", - "description": "Legacy NVML utilization across all tenant GPUs.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 16, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "area", - "colorMode": "value", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "blue" - }, - { - "value": 30, - "color": "green" - }, - { - "value": 80, - "color": "orange" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 7, - "targets": [ - { - "expr": "cluster:gpu_power_watts:sum", - "refId": "A" - } - ], - "title": "Total power", - "description": "Live power draw summed across the fleet.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 20, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "area", - "colorMode": "value", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "watt", - "decimals": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "row", - "collapsed": false, - "title": "Per-node GPU activity", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 10, - "panels": [] - }, - { - "type": "bargauge", - "id": 11, - "targets": [ - { - "expr": "avg by (Hostname) (DCGM_FI_DEV_GPU_UTIL)", - "legendFormat": "{{Hostname}}", - "refId": "A" - } - ], - "title": "GPU utilization per node", - "description": "Average NVML utilization per node — spot idle or saturated hosts at a glance.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 6 - }, - "repeatDirection": "h", - "options": { - "displayMode": "gradient", - "valueMode": "color", - "namePlacement": "auto", - "showUnfilled": true, - "sizing": "auto", - "minVizWidth": 8, - "minVizHeight": 16, - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false, - "calcs": [] - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "maxVizHeight": 300, - "orientation": "horizontal" - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "blue" - }, - { - "value": 30, - "color": "green" - }, - { - "value": 80, - "color": "orange" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "bargauge", - "id": 12, - "targets": [ - { - "expr": "sum by (Hostname) (DCGM_FI_DEV_POWER_USAGE) / clamp_min(sum by (Hostname) (DCGM_FI_DEV_POWER_MGMT_LIMIT), 1) * 100", - "legendFormat": "{{Hostname}}", - "refId": "A" - } - ], - "title": "Power draw per node (% of TDP)", - "description": "GPU power usage as a fraction of the node's combined TDP cap. Works across GPU SKUs without per-model thresholds. Requires DCGM_FI_DEV_POWER_MGMT_LIMIT from custom DCGM CSV (see packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml).", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 6 - }, - "repeatDirection": "h", - "options": { - "displayMode": "gradient", - "valueMode": "color", - "namePlacement": "auto", - "showUnfilled": true, - "sizing": "auto", - "minVizWidth": 8, - "minVizHeight": 16, - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false, - "calcs": [] - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "maxVizHeight": 300, - "orientation": "horizontal" - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - }, - { - "value": 60, - "color": "yellow" - }, - { - "value": 80, - "color": "orange" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "row", - "collapsed": false, - "title": "Usage trends", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 20, - "panels": [] - }, - { - "type": "timeseries", - "id": 21, - "targets": [ - { - "expr": "avg by (Hostname) (DCGM_FI_DEV_GPU_UTIL)", - "legendFormat": "{{Hostname}}", - "refId": "A" - } - ], - "title": "GPU utilization per node (trend)", - "description": "Per-node NVML utilization over time — shows shifting workload across hosts.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 15 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 20, - "showPoints": "never" - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "id": 22, - "targets": [ - { - "expr": "sum by (Hostname) (DCGM_FI_DEV_POWER_USAGE)", - "legendFormat": "{{Hostname}}", - "refId": "A" - } - ], - "title": "Power draw per node (trend)", - "description": "Per-node GPU power consumption over time.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 15 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "watt", - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 20, - "showPoints": "never" - } - }, - "overrides": [] - } - }, - { - "type": "row", - "collapsed": false, - "title": "Health", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 30, - "panels": [] - }, - { - "type": "stat", - "id": 31, - "targets": [ - { - "expr": "count((max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS)) \u003e 0) or vector(0)", - "refId": "A" - } - ], - "title": "GPUs with XID errors", - "description": "Count of GPUs reporting a non-zero XID code. XID = NVIDIA hardware/driver error — any non-zero value needs investigation.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 0, - "y": 24 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "background", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - }, - { - "value": 1, - "color": "red" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 32, - "targets": [ - { - "expr": "avg(gpu:power_throttle_fraction:rate5m)", - "refId": "A" - } - ], - "title": "Power throttling (cluster avg)", - "description": "Fraction of time the fleet spent power-throttled. \u003e5% = billed FLOPS are being underdelivered.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 6, - "y": 24 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "background", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "percentunit", - "decimals": 2, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - }, - { - "value": 0.05, - "color": "yellow" - }, - { - "value": 0.2, - "color": "red" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 33, - "targets": [ - { - "expr": "avg(gpu:thermal_throttle_fraction:rate5m)", - "refId": "A" - } - ], - "title": "Thermal throttling (cluster avg)", - "description": "Fraction of time the fleet spent thermal-throttled. Indicates cooling/datacenter issues.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 12, - "y": 24 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "background", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "percentunit", - "decimals": 2, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - }, - { - "value": 0.05, - "color": "yellow" - }, - { - "value": 0.2, - "color": "red" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 34, - "targets": [ - { - "expr": "max(DCGM_FI_DEV_GPU_TEMP)", - "refId": "A" - } - ], - "title": "Max GPU temperature", - "description": "Hottest GPU in the fleet right now. Adjust thresholds to match your GPU model specifications.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 18, - "y": 24 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "background", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "celsius", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - }, - { - "value": 75, - "color": "yellow" - }, - { - "value": 85, - "color": "red" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "id": 35, - "targets": [ - { - "expr": "DCGM_FI_DEV_GPU_TEMP", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" - } - ], - "title": "Temperature per GPU", - "description": "Per-GPU temperature trace. Sustained red zone = cooling issue or aggressive workload.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 29 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "celsius", - "min": 20, - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - }, - { - "value": 75, - "color": "yellow" - }, - { - "value": 85, - "color": "red" - } - ] - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "id": 36, - "targets": [ - { - "expr": "gpu:power_throttle_fraction:rate5m", - "legendFormat": "power {{Hostname}}/{{gpu}}", - "refId": "A" - }, - { - "expr": "gpu:thermal_throttle_fraction:rate5m", - "legendFormat": "thermal {{Hostname}}/{{gpu}}", - "refId": "B" - } - ], - "title": "Throttle fraction per GPU", - "description": "Red = power throttle, blue = thermal throttle. Both on same chart to compare causes.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 29 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "percentunit", - "min": 0, - "max": 1, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 25, - "showPoints": "never" - } - }, - "overrides": [] - } - }, - { - "type": "row", - "collapsed": false, - "title": "Hardware inventory", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 37 - }, - "id": 40, - "panels": [] - }, - { - "type": "table", - "id": 41, - "targets": [ - { - "expr": "group by (Hostname, modelName, gpu, UUID) (DCGM_FI_DEV_GPU_UTIL)", - "instant": true, - "range": false, - "format": "table", - "refId": "A" - } - ], - "title": "GPU inventory", - "description": "One row per physical GPU. Model / index / UUID sourced from DCGM labels.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 38 - }, - "repeatDirection": "h", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Value": true, - "__name__": true, - "cluster": true, - "container": true, - "device": true, - "endpoint": true, - "instance": true, - "job": true, - "pci_bus_id": true, - "prometheus": true, - "service": true, - "tenant": true, - "tier": true, - "uid": true, - "unit": true - }, - "indexByName": { - "Hostname": 0, - "UUID": 3, - "gpu": 2, - "modelName": 1 - }, - "renameByName": { - "Hostname": "Node", - "UUID": "UUID", - "gpu": "Index", - "modelName": "GPU Model" - } - } - } - ], - "options": { - "frameIndex": 0, - "showHeader": true, - "showTypeIcons": false, - "footer": { - "show": false, - "reducer": [] - }, - "cellHeight": "sm" - } - } - ], - "templating": { - "list": [ - { - "type": "datasource", - "name": "ds_prometheus", - "label": "Prometheus", - "skipUrlSync": false, - "query": "prometheus", - "current": { - "selected": false, - "text": "default", - "value": "default" - }, - "multi": false, - "allowCustomValue": true, - "includeAll": false, - "regex": "", - "auto": false, - "auto_min": "10s", - "auto_count": 30 - } - ] - }, - "annotations": {} -} diff --git a/dashboards/gpu/gpu-performance.json b/dashboards/gpu/gpu-performance.json deleted file mode 100644 index aed9c69a..00000000 --- a/dashboards/gpu/gpu-performance.json +++ /dev/null @@ -1,957 +0,0 @@ -{ - "uid": "gpu-performance", - "title": "GPU Performance", - "tags": [ - "gpu", - "dcgm" - ], - "timezone": "browser", - "editable": true, - "graphTooltip": 1, - "time": { - "from": "now-1h", - "to": "now" - }, - "fiscalYearStartMonth": 0, - "schemaVersion": 42, - "panels": [ - { - "type": "row", - "collapsed": false, - "title": "Overview", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [] - }, - { - "type": "stat", - "id": 2, - "targets": [ - { - "expr": "cluster:gpu_count:total", - "refId": "A" - } - ], - "title": "Total GPUs", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "value", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "blue" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 3, - "targets": [ - { - "expr": "cluster:gpu_count:allocated", - "refId": "A" - } - ], - "title": "Allocated", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "value", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - }, - { - "value": 1, - "color": "yellow" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 4, - "targets": [ - { - "expr": "cluster:gpu_util:avg", - "refId": "A" - } - ], - "title": "Average utilization", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "area", - "colorMode": "value", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "blue" - }, - { - "value": 30, - "color": "green" - }, - { - "value": 80, - "color": "orange" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 5, - "targets": [ - { - "expr": "cluster:gpu_power_watts:sum", - "refId": "A" - } - ], - "title": "Power draw", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "area", - "colorMode": "value", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "watt", - "decimals": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "row", - "collapsed": false, - "title": "Utilization", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 10, - "panels": [] - }, - { - "type": "timeseries", - "id": 11, - "targets": [ - { - "expr": "DCGM_FI_DEV_GPU_UTIL{Hostname=~\"$Hostname\"}", - "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", - "refId": "A" - } - ], - "title": "GPU Utilization (NVML)", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 6 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "id": 12, - "targets": [ - { - "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{Hostname=~\"$Hostname\"} * 100", - "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", - "refId": "A" - } - ], - "title": "Tensor Pipe Active (realistic load for LLM/AI)", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 6 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "id": 13, - "targets": [ - { - "expr": "DCGM_FI_PROF_GR_ENGINE_ACTIVE{Hostname=~\"$Hostname\"} * 100", - "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", - "refId": "A" - } - ], - "title": "Graphics Engine Active", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 14 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "id": 14, - "targets": [ - { - "expr": "DCGM_FI_DEV_MEM_COPY_UTIL{Hostname=~\"$Hostname\"}", - "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", - "refId": "A" - } - ], - "title": "Memory Copy Utilization", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 14 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false - } - }, - "overrides": [] - } - }, - { - "type": "row", - "collapsed": false, - "title": "Memory", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 22 - }, - "id": 20, - "panels": [] - }, - { - "type": "timeseries", - "id": 21, - "targets": [ - { - "expr": "DCGM_FI_DEV_FB_USED{Hostname=~\"$Hostname\"} * 1048576", - "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", - "refId": "A" - } - ], - "title": "VRAM Used", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 23 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "bytes", - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 25, - "showPoints": "never" - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "id": 22, - "targets": [ - { - "expr": "DCGM_FI_DEV_FB_FREE{Hostname=~\"$Hostname\"} * 1048576", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" - } - ], - "title": "VRAM Free", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 23 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "min" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "bytes", - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [] - } - }, - { - "type": "row", - "collapsed": false, - "title": "Power \u0026 Temperature", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 31 - }, - "id": 30, - "panels": [] - }, - { - "type": "timeseries", - "id": 31, - "targets": [ - { - "expr": "DCGM_FI_DEV_POWER_USAGE{Hostname=~\"$Hostname\"}", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" - } - ], - "title": "Power Usage per GPU", - "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 32 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "watt", - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "id": 32, - "targets": [ - { - "expr": "DCGM_FI_DEV_GPU_TEMP{Hostname=~\"$Hostname\"}", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" - } - ], - "title": "GPU Temperature", - "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "celsius", - "min": 20, - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - }, - { - "value": 75, - "color": "yellow" - }, - { - "value": 85, - "color": "red" - } - ] - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [] - } - }, - { - "type": "row", - "collapsed": false, - "title": "Health", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 40 - }, - "id": 40, - "panels": [] - }, - { - "type": "stat", - "id": 41, - "targets": [ - { - "expr": "max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS{Hostname=~\"$Hostname\"})", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" - } - ], - "title": "XID errors (latest)", - "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 41 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "background", - "justifyMode": "auto", - "textMode": "value_and_name", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - }, - { - "value": 1, - "color": "red" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "id": 42, - "targets": [ - { - "expr": "rate(DCGM_FI_DEV_POWER_VIOLATION{Hostname=~\"$Hostname\"}[5m])", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" - } - ], - "title": "Power Violation (µs/s throttled due to power)", - "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 8, - "y": 41 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "µs", - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "id": 43, - "targets": [ - { - "expr": "rate(DCGM_FI_DEV_THERMAL_VIOLATION{Hostname=~\"$Hostname\"}[5m])", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" - } - ], - "title": "Thermal Violation (µs/s throttled due to thermals)", - "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 16, - "y": 41 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "µs", - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [] - } - } - ], - "templating": { - "list": [ - { - "type": "datasource", - "name": "ds_prometheus", - "label": "Prometheus", - "skipUrlSync": false, - "query": "prometheus", - "current": { - "selected": false, - "text": "default", - "value": "default" - }, - "multi": false, - "allowCustomValue": true, - "includeAll": false, - "regex": "", - "auto": false, - "auto_min": "10s", - "auto_count": 30 - }, - { - "type": "query", - "name": "Hostname", - "label": "Host", - "skipUrlSync": false, - "query": "label_values(DCGM_FI_DEV_GPU_UTIL, Hostname)", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "multi": true, - "allowCustomValue": true, - "refresh": 2, - "sort": 1, - "includeAll": true, - "auto": false, - "auto_min": "10s", - "auto_count": 30 - } - ] - }, - "annotations": {} -} diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json deleted file mode 100644 index 865fb95c..00000000 --- a/dashboards/gpu/gpu-quotas.json +++ /dev/null @@ -1,637 +0,0 @@ -{ - "uid": "gpu-quotas", - "title": "GPU Quotas \u0026 Allocation", - "tags": [ - "gpu", - "quotas" - ], - "timezone": "browser", - "editable": true, - "graphTooltip": 1, - "time": { - "from": "now-6h", - "to": "now" - }, - "fiscalYearStartMonth": 0, - "schemaVersion": 42, - "panels": [ - { - "type": "row", - "collapsed": false, - "title": "Allocation overview", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [] - }, - { - "type": "stat", - "id": 2, - "targets": [ - { - "expr": "sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"})", - "refId": "A" - } - ], - "title": "GPU allocatable", - "description": "Total GPU capacity the cluster can schedule to pods.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "value", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "blue" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 3, - "targets": [ - { - "expr": "cluster:gpu_count:allocated", - "refId": "A" - } - ], - "title": "GPU requested", - "description": "Sum of GPU requests across all pods cluster-wide, including system namespaces.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "value", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - }, - { - "value": 1, - "color": "yellow" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "gauge", - "id": 4, - "targets": [ - { - "expr": "cluster:gpu_count:allocated / sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"}) * 100", - "refId": "A" - } - ], - "title": "Allocation ratio", - "description": "Percentage of allocatable GPUs currently requested by pods.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto", - "minVizWidth": 75, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "minVizHeight": 75, - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "blue" - }, - { - "value": 40, - "color": "green" - }, - { - "value": 80, - "color": "yellow" - }, - { - "value": 95, - "color": "red" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 5, - "targets": [ - { - "expr": "count((sum by (namespace, pod) (kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}) \u003e 0) * on(namespace, pod) group_left() (kube_pod_status_phase{phase=\"Pending\"} == 1)) or vector(0)", - "refId": "A" - } - ], - "title": "Pending pods (GPU)", - "description": "Pods requesting GPUs that are stuck in Pending state — indicates capacity shortage.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "background", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - }, - { - "value": 1, - "color": "red" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "row", - "collapsed": false, - "title": "Per namespace", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 10, - "panels": [] - }, - { - "type": "bargauge", - "id": 11, - "targets": [ - { - "expr": "sum by (namespace) (namespace:gpu_count:allocated{namespace=~\"$namespace\"})", - "legendFormat": "{{namespace}}", - "refId": "A" - } - ], - "title": "GPU requested per namespace", - "description": "GPU allocation breakdown by namespace — spot top consumers at a glance.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 6 - }, - "repeatDirection": "h", - "options": { - "displayMode": "gradient", - "valueMode": "color", - "namePlacement": "auto", - "showUnfilled": true, - "sizing": "auto", - "minVizWidth": 8, - "minVizHeight": 16, - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false, - "calcs": [] - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "maxVizHeight": 300, - "orientation": "horizontal" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "min": 0, - "max": 8, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "id": 12, - "targets": [ - { - "expr": "sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"})", - "legendFormat": "Allocatable (total)", - "refId": "A" - }, - { - "expr": "sum(namespace:gpu_count:allocated{namespace=~\"$namespace\"})", - "legendFormat": "Requested", - "refId": "B" - } - ], - "title": "GPU allocated over time", - "description": "Requested vs allocatable GPUs over time — shows allocation pressure trends.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 6 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "custom": { - "drawStyle": "line", - "lineInterpolation": "stepAfter", - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Allocatable (total)" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - } - ] - } - }, - { - "type": "row", - "collapsed": false, - "title": "Pods with GPU", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 20, - "panels": [] - }, - { - "type": "table", - "id": 21, - "targets": [ - { - "expr": "kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left(phase) (kube_pod_status_phase{phase=~\"Running|Pending\"} == 1)", - "instant": true, - "range": false, - "format": "table", - "refId": "requested" - }, - { - "expr": "kube_pod_container_resource_limits{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left() (kube_pod_status_phase{phase=~\"Running|Pending\"} == 1)", - "instant": true, - "range": false, - "format": "table", - "refId": "limits" - } - ], - "title": "Pods requesting GPU", - "description": "Per-pod GPU requests and limits with scheduling status — Running or Pending.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 15 - }, - "repeatDirection": "h", - "transformations": [ - { - "id": "joinByField", - "options": { - "byField": "pod", - "mode": "outer" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Time 1": true, - "Time 2": true, - "__name__": true, - "__name__ 1": true, - "__name__ 2": true, - "cluster": true, - "cluster 1": true, - "cluster 2": true, - "container 2": true, - "endpoint": true, - "endpoint 1": true, - "endpoint 2": true, - "instance": true, - "instance 1": true, - "instance 2": true, - "job": true, - "job 1": true, - "job 2": true, - "namespace 2": true, - "node 2": true, - "prometheus": true, - "prometheus 1": true, - "prometheus 2": true, - "resource": true, - "resource 1": true, - "resource 2": true, - "service": true, - "service 1": true, - "service 2": true, - "tenant": true, - "tenant 1": true, - "tenant 2": true, - "tier": true, - "tier 1": true, - "tier 2": true, - "uid": true, - "uid 1": true, - "uid 2": true, - "unit": true, - "unit 1": true, - "unit 2": true - }, - "indexByName": { - "Value #limits": 5, - "Value #requested": 4, - "container 1": 2, - "namespace 1": 0, - "node 1": 3, - "phase": 6, - "pod": 1 - }, - "renameByName": { - "Value #limits": "Limit", - "Value #requested": "Req", - "container 1": "Container", - "namespace 1": "Namespace", - "node 1": "Node", - "phase": "Status" - } - } - } - ], - "options": { - "frameIndex": 0, - "showHeader": true, - "showTypeIcons": false, - "footer": { - "show": false, - "reducer": [] - }, - "cellHeight": "sm" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Status" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "mode": "basic", - "type": "color-background" - } - }, - { - "id": "mappings", - "value": [ - { - "options": { - "Failed": { - "color": "red", - "index": 2 - }, - "Pending": { - "color": "orange", - "index": 1 - }, - "Running": { - "color": "green", - "index": 0 - } - }, - "type": "value" - } - ] - } - ] - } - ] - } - } - ], - "templating": { - "list": [ - { - "type": "datasource", - "name": "ds_prometheus", - "label": "Prometheus", - "skipUrlSync": false, - "query": "prometheus", - "current": { - "selected": false, - "text": "default", - "value": "default" - }, - "multi": false, - "allowCustomValue": true, - "includeAll": false, - "regex": "", - "auto": false, - "auto_min": "10s", - "auto_count": 30 - }, - { - "type": "query", - "name": "namespace", - "label": "Namespace", - "skipUrlSync": false, - "query": "label_values(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}, namespace)", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "multi": true, - "allowCustomValue": true, - "refresh": 2, - "sort": 1, - "includeAll": true, - "auto": false, - "auto_min": "10s", - "auto_count": 30 - } - ] - }, - "annotations": {} -} diff --git a/dashboards/gpu/gpu-tenants.json b/dashboards/gpu/gpu-tenants.json deleted file mode 100644 index 7fef321d..00000000 --- a/dashboards/gpu/gpu-tenants.json +++ /dev/null @@ -1,1098 +0,0 @@ -{ - "uid": "gpu-tenants", - "title": "GPU Tenant Usage", - "description": "Per-tenant GPU consumption: live allocation, utilization, tensor saturation, power, and 24h GPU-hours / kWh — admin view across all namespaces.", - "tags": [ - "gpu", - "tenants", - "admin" - ], - "timezone": "browser", - "editable": true, - "graphTooltip": 1, - "time": { - "from": "now-24h", - "to": "now" - }, - "fiscalYearStartMonth": 0, - "schemaVersion": 42, - "panels": [ - { - "type": "row", - "collapsed": false, - "title": "Tenant fleet snapshot", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [] - }, - { - "type": "stat", - "id": 2, - "targets": [ - { - "expr": "count(namespace:gpu_count:allocated{namespace=~\"$namespace\"} \u003e 0)", - "refId": "A" - } - ], - "title": "Active tenants", - "description": "Namespaces with at least one GPU request (Pending + Running).", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "value", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "blue" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 3, - "targets": [ - { - "expr": "sum(namespace:gpu_count:allocated{namespace=~\"$namespace\"})", - "refId": "A" - } - ], - "title": "Allocated GPUs", - "description": "Sum of kube GPU requests across tenants. Billable view — includes Pending pods.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "background", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "blue" - }, - { - "value": 1, - "color": "green" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 4, - "targets": [ - { - "expr": "cluster:gpu_util:avg", - "refId": "A" - } - ], - "title": "Avg cluster GPU util", - "description": "Average NVML utilization across all GPUs in the cluster. Cluster-wide — DCGM hardware metrics cannot be attributed to workload namespaces.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "area", - "colorMode": "value", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "blue" - }, - { - "value": 30, - "color": "green" - }, - { - "value": 80, - "color": "orange" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 5, - "targets": [ - { - "expr": "cluster:gpu_power_watts:sum", - "refId": "A" - } - ], - "title": "Total GPU power", - "description": "Live power draw summed across all GPUs in the cluster. Cluster-wide — DCGM hardware metrics cannot be attributed to workload namespaces.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "area", - "colorMode": "value", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "watt", - "decimals": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 6, - "targets": [ - { - "expr": "sum(sum_over_time(node:power_watts:sum[24h:1m])) / 60 / 1000", - "refId": "A" - } - ], - "title": "Energy (last 24h)", - "description": "Integrated cluster-wide GPU power draw over 24h. Cluster-wide — DCGM hardware metrics cannot be attributed to workload namespaces.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 16, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "value", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "kwatth", - "decimals": 1, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 7, - "targets": [ - { - "expr": "sum(sum_over_time(namespace:gpu_count:allocated{namespace=~\"$namespace\"}[24h:1m])) / 60", - "refId": "A" - } - ], - "title": "GPU-hours (last 24h)", - "description": "Sum of (GPU count × time requested) across all tenants over the last 24h. Integrated from kube requests, so Pending pods count.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 20, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "value", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 1, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "blue" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "row", - "collapsed": false, - "title": "Current allocation", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 10, - "panels": [] - }, - { - "type": "bargauge", - "id": 11, - "targets": [ - { - "expr": "namespace:gpu_count:allocated{namespace=~\"$namespace\"}", - "legendFormat": "{{namespace}}", - "refId": "A" - } - ], - "title": "GPUs per namespace", - "description": "Kube GPU requests per tenant (billable). Includes Pending pods.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 6 - }, - "repeatDirection": "h", - "options": { - "displayMode": "gradient", - "valueMode": "color", - "namePlacement": "auto", - "showUnfilled": true, - "sizing": "auto", - "minVizWidth": 8, - "minVizHeight": 16, - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false, - "calcs": [] - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "maxVizHeight": 300, - "orientation": "horizontal" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "bargauge", - "id": 12, - "targets": [ - { - "expr": "node:fb_used_bytes:sum", - "legendFormat": "{{Hostname}}", - "refId": "A" - } - ], - "title": "VRAM used per node", - "description": "FB memory actively used per node. DCGM hardware metrics cannot be attributed to workload namespaces — shown per-node instead.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 6 - }, - "repeatDirection": "h", - "options": { - "displayMode": "gradient", - "valueMode": "color", - "namePlacement": "auto", - "showUnfilled": true, - "sizing": "auto", - "minVizWidth": 8, - "minVizHeight": 16, - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false, - "calcs": [] - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "maxVizHeight": 300, - "orientation": "horizontal" - }, - "fieldConfig": { - "defaults": { - "unit": "bytes", - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "row", - "collapsed": false, - "title": "Utilization (per-node)", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 20, - "panels": [] - }, - { - "type": "timeseries", - "id": 21, - "targets": [ - { - "expr": "node:gpu_util:avg", - "legendFormat": "{{Hostname}}", - "refId": "A" - } - ], - "title": "NVML utilization per node", - "description": "Per-node NVML utilization. Stacked to show fleet-wide pressure over time. DCGM hardware metrics cannot be attributed to workload namespaces — shown per-node instead.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 15 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 40, - "showPoints": "never", - "stacking": { - "mode": "normal" - } - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "id": 22, - "targets": [ - { - "expr": "node:tensor_active:avg * 100", - "legendFormat": "{{Hostname}}", - "refId": "A" - } - ], - "title": "Tensor saturation per node", - "description": "Real tensor core load per node. Useful to spot nodes with idle tensor cores. DCGM hardware metrics cannot be attributed to workload namespaces — shown per-node instead.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 15 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 40, - "showPoints": "never", - "stacking": { - "mode": "normal" - } - } - }, - "overrides": [] - } - }, - { - "type": "row", - "collapsed": false, - "title": "Power \u0026 allocation trend", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 30, - "panels": [] - }, - { - "type": "timeseries", - "id": 31, - "targets": [ - { - "expr": "node:power_watts:sum", - "legendFormat": "{{Hostname}}", - "refId": "A" - } - ], - "title": "Power draw per node", - "description": "Per-node GPU power draw. Stacked to see cluster-wide energy burn over time. DCGM hardware metrics cannot be attributed to workload namespaces — shown per-node instead.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "watt", - "min": 0, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 40, - "showPoints": "never", - "stacking": { - "mode": "normal" - } - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "id": 32, - "targets": [ - { - "expr": "namespace:gpu_count:allocated{namespace=~\"$namespace\"}", - "legendFormat": "{{namespace}}", - "refId": "A" - } - ], - "title": "Allocated GPUs per namespace", - "description": "Kube-requested GPU count per tenant over time. Step-after interpolation reflects discrete allocation changes.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "min": 0, - "custom": { - "drawStyle": "line", - "lineInterpolation": "stepAfter", - "fillOpacity": 40, - "showPoints": "never", - "stacking": { - "mode": "normal" - } - } - }, - "overrides": [] - } - }, - { - "type": "row", - "collapsed": false, - "title": "Top consumers (live)", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 32 - }, - "id": 40, - "panels": [] - }, - { - "type": "table", - "id": 41, - "targets": [ - { - "expr": "namespace:gpu_count:allocated{namespace=~\"$namespace\"}", - "instant": true, - "range": false, - "format": "table", - "refId": "gpus" - } - ], - "title": "Top consumers (allocation)", - "description": "Live ranking of tenants by GPU allocation. Hardware metrics (util, tensor, power, VRAM) cannot be attributed to workload namespaces — see Fleet and Performance dashboards for per-node/per-GPU hardware views.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 33 - }, - "repeatDirection": "h", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Time 1": true, - "Time 2": true, - "Time 3": true, - "Time 4": true, - "Time 5": true, - "__name__": true, - "__name__ 1": true, - "__name__ 2": true, - "__name__ 3": true, - "__name__ 4": true, - "__name__ 5": true, - "cluster": true, - "cluster 1": true, - "cluster 2": true, - "cluster 3": true, - "cluster 4": true, - "cluster 5": true, - "endpoint": true, - "endpoint 1": true, - "endpoint 2": true, - "endpoint 3": true, - "endpoint 4": true, - "endpoint 5": true, - "instance": true, - "instance 1": true, - "instance 2": true, - "instance 3": true, - "instance 4": true, - "instance 5": true, - "job": true, - "job 1": true, - "job 2": true, - "job 3": true, - "job 4": true, - "job 5": true, - "prometheus": true, - "prometheus 1": true, - "prometheus 2": true, - "prometheus 3": true, - "prometheus 4": true, - "prometheus 5": true, - "service": true, - "service 1": true, - "service 2": true, - "service 3": true, - "service 4": true, - "service 5": true, - "tenant": true, - "tenant 1": true, - "tenant 2": true, - "tenant 3": true, - "tenant 4": true, - "tenant 5": true, - "tier": true, - "tier 1": true, - "tier 2": true, - "tier 3": true, - "tier 4": true, - "tier 5": true, - "uid": true, - "uid 1": true, - "uid 2": true, - "uid 3": true, - "uid 4": true, - "uid 5": true, - "unit": true, - "unit 1": true, - "unit 2": true, - "unit 3": true, - "unit 4": true, - "unit 5": true - }, - "indexByName": { - "Value #gpus": 1, - "namespace": 0 - }, - "renameByName": { - "Value #gpus": "GPUs", - "namespace": "Namespace" - } - } - } - ], - "options": { - "frameIndex": 0, - "showHeader": true, - "showTypeIcons": false, - "sortBy": [ - { - "displayName": "GPUs", - "desc": true - } - ], - "footer": { - "show": false, - "reducer": [] - }, - "cellHeight": "sm" - } - }, - { - "type": "row", - "collapsed": false, - "title": "Historical usage (last 24h)", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 43 - }, - "id": 50, - "panels": [] - }, - { - "type": "bargauge", - "id": 51, - "targets": [ - { - "expr": "sum_over_time(namespace:gpu_count:allocated{namespace=~\"$namespace\"}[24h:1m]) / 60", - "legendFormat": "{{namespace}}", - "refId": "A" - } - ], - "title": "GPU-hours per namespace (last 24h)", - "description": "Billable GPU-hours per tenant over the last 24h window.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 44 - }, - "repeatDirection": "h", - "options": { - "displayMode": "gradient", - "valueMode": "color", - "namePlacement": "auto", - "showUnfilled": true, - "sizing": "auto", - "minVizWidth": 8, - "minVizHeight": 16, - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false, - "calcs": [] - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "maxVizHeight": 300, - "orientation": "horizontal" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "blue" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "bargauge", - "id": 52, - "targets": [ - { - "expr": "sum_over_time(node:power_watts:sum[24h:1m]) / 60 / 1000", - "legendFormat": "{{Hostname}}", - "refId": "A" - } - ], - "title": "Energy per node (last 24h)", - "description": "kWh consumed per node over the last 24h. Integrated from 1-minute power samples. DCGM hardware metrics cannot be attributed to workload namespaces — shown per-node instead.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 44 - }, - "repeatDirection": "h", - "options": { - "displayMode": "gradient", - "valueMode": "color", - "namePlacement": "auto", - "showUnfilled": true, - "sizing": "auto", - "minVizWidth": 8, - "minVizHeight": 16, - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false, - "calcs": [] - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "maxVizHeight": 300, - "orientation": "horizontal" - }, - "fieldConfig": { - "defaults": { - "unit": "kwatth", - "decimals": 2, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - } - ] - } - }, - "overrides": [] - } - } - ], - "templating": { - "list": [ - { - "type": "datasource", - "name": "ds_prometheus", - "label": "Prometheus", - "skipUrlSync": false, - "query": "prometheus", - "current": { - "selected": false, - "text": "default", - "value": "default" - }, - "multi": false, - "allowCustomValue": true, - "includeAll": false, - "regex": "", - "auto": false, - "auto_min": "10s", - "auto_count": 30 - }, - { - "type": "query", - "name": "namespace", - "label": "Namespace", - "skipUrlSync": false, - "query": "label_values(namespace:gpu_count:allocated, namespace)", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "multi": true, - "allowCustomValue": true, - "refresh": 2, - "sort": 1, - "includeAll": true, - "auto": false, - "auto_min": "10s", - "auto_count": 30 - } - ] - }, - "annotations": {} -} diff --git a/docs/changelogs/v1.3.1.md b/docs/changelogs/v1.3.1.md deleted file mode 100644 index 0f149d0d..00000000 --- a/docs/changelogs/v1.3.1.md +++ /dev/null @@ -1,37 +0,0 @@ - - -# v1.3.1 (2026-04-28) - -Patch release covering a TenantNamespace IDOR fix in the API, a destructive `post-upgrade` hook removed from the etcd chart, kamaji controller stability, a `linstor-csi` bump that fixes live migration on Protocol-A/B DRBD resources, the missing `linstor-gui` build wiring, and a velero RBAC fix that unblocked installs on bundles without Velero. - -## Security - -* **fix(api): prevent IDOR in TenantNamespace Get and Watch handlers**: Two IDOR (Insecure Direct Object Reference) vulnerabilities allowed authenticated users to read TenantNamespace metadata they had no RoleBinding for. The `Get` and `Watch` handlers now go through a new `hasAccessToNamespace()` helper that lists RoleBindings scoped only to the target namespace (orders of magnitude faster than the previous all-cluster scan), returns `NotFound` instead of leaking existence on unauthorized access, and applies the same check on the `Watch` filter path. Includes regression tests for the unauthorized paths. ([**@IvanHunters**](https://github.com/IvanHunters) in #2471, backport #2524) - -## Features - -* **feat(linstor): bump linstor-csi to v1.10.6 with Protocol-C dual-attach fix**: Live migration of KubeVirt VMs on Protocol-A/B (async) DRBD volumes no longer fails with `Protocol C required`. `linstor-csi` v1.10.6 now installs a `Protocol=C` override on the resource-definition during dual-attach and reverts it on detach, so `replicated-async` StorageClasses and other Protocol-A/B resource groups support live migration without manual `drbdadm adjust` intervention. ([**@kvaps**](https://github.com/kvaps) in #2496, backport #2505) - -## Fixes - -* **fix(backups): move velero-configmap Role to velero chart**: The `backupstrategy-controller` (a default package) declared a Role/RoleBinding scoped to the `cozy-velero` namespace for managing `ResourceModifier` ConfigMaps. On bundles where Velero was not enabled, that namespace did not exist and the HelmRelease failed with `namespaces "cozy-velero" not found`, blocking installation. The Role/RoleBinding now lives in the velero chart, so it is created only when velero is actually deployed. ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2459, backport #2467) - -* **fix(etcd): remove destructive post-upgrade cert-regeneration hook**: The etcd chart ran a `post-upgrade` Helm hook on every upgrade that deleted etcd TLS Secrets (`etcd-ca-tls`, `etcd-peer-ca-tls`, `etcd-client-tls`, `etcd-peer-tls`, `etcd-server-tls`) and then deleted etcd pods, forcing cert-manager to re-issue the entire etcd CA chain. On clusters with Kamaji-managed tenant control planes this put every tenant `kube-apiserver` into CrashLoopBackOff until each DataStore was manually re-reconciled. The hook was a one-shot `2.6.0 → 2.6.1` migration that became a permanent footgun once chart versioning moved to `0.0.0+` (always `< 2.6.1` per semver) and after the underlying `rotationPolicy: Always` issue was fixed in `47d81f70`. The hook is now removed entirely. ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2462, backport #2511) - -* **fix(kamaji): increase memory limits and add startup probe**: The kamaji controller frequently entered CrashLoopBackOff due to OOMKills (exit 137) within ~20–25 seconds of startup, with the readiness probe failing while the controller was still finishing initialization. Memory limit raised from 500Mi to 512Mi, request from 100Mi to 256Mi, and a 60-second startup probe (12 attempts × 5s periods) is added so the controller has room to boot before liveness/readiness probes engage. ([**@IvanHunters**](https://github.com/IvanHunters) in #2421, backport #2491) - -## Build - -* **build(linstor): include linstor-gui in root image build target**: The `linstor-gui` package (added in #2382) was never wired into the root `Makefile`'s `build:` target, so CI never built or published the image. `ghcr.io/cozystack/cozystack/linstor-gui` returned `NAME_UNKNOWN` and `values.yaml` stayed pinned to `tag: 2.3.0` without a digest. The missing build line is added so the next CI run publishes the image and the per-package `Makefile` digest-pins `values.yaml` automatically. ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2498, backport #2518) - -## Contributors - -Thanks to everyone who contributed to this patch release: - -* [**@IvanHunters**](https://github.com/IvanHunters) -* [**@kvaps**](https://github.com/kvaps) -* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) - -**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.3.0...v1.3.1 diff --git a/hack/check-gpu-recording-rules.bats b/hack/check-gpu-recording-rules.bats deleted file mode 100644 index bae6785e..00000000 --- a/hack/check-gpu-recording-rules.bats +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env bats -# ----------------------------------------------------------------------------- -# Cross-validation between GPU recording rules, the dashboards that consume -# them, and the DCGM Exporter metric set the cluster actually scrapes. Catches: -# -# 1. dangling references — a dashboard query mentions a recording rule name -# that doesn't exist in gpu-recording.rules.yaml. This is the bug the -# pre-merge review caught: gpu-efficiency.json shipped panels keyed on -# pod:tensor_saturation:avg5m without the rule being defined, so the -# panel showed "No data" everywhere. -# -# 2. typos in rule names — same bug class, manifested as a single-character -# difference between rule and reference. -# -# 3. undeclared DCGM metrics — a dashboard query or recording rule mentions -# a DCGM_FI_* metric that is neither in the upstream default CSV nor in -# the project's custom CSV (dcgm-custom-metrics.yaml), meaning DCGM -# Exporter would never emit it and the panel silently shows "No data". -# Example regression: gpu-fleet.json shipped a TDP panel referencing -# DCGM_FI_DEV_POWER_MGMT_LIMIT before the custom CSV declared it. -# -# Scope: only dashboards listed in packages/system/monitoring/dashboards-infra.list -# under the "gpu/" prefix (i.e. shipped to production), not every JSON file in -# dashboards/gpu/. Untracked drafts stay out of scope on purpose — adding one -# to dashboards-infra.list is what brings it under the test. -# -# Reverse direction (rule defined but never consumed) is intentionally NOT -# enforced: some rules exist for ad-hoc PromQL or upcoming dashboards. Treat -# unused rules as an editorial concern, not a regression. -# -# Title syntax constraints from cozytest.sh's awk parser: -# - Titles delimited by ASCII double quotes; embedded quotes truncate. -# - Only [A-Za-z0-9] from the title survives into the function name; titles -# differing only in punctuation collapse to the same function. -# -# Run with: hack/cozytest.sh hack/check-gpu-recording-rules.bats -# ----------------------------------------------------------------------------- - -REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" -RULES_FILE="$REPO_ROOT/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml" -DASHBOARDS_LIST="$REPO_ROOT/packages/system/monitoring/dashboards-infra.list" -DASHBOARDS_DIR="$REPO_ROOT/dashboards" -DCGM_DEFAULT_CSV="$REPO_ROOT/hack/dcgm-default-counters.csv" -DCGM_CUSTOM_CSV="$REPO_ROOT/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml" - -# Extract the set of "- record: NAME" entries from the rules YAML. -# Outputs one rule name per line, sorted and deduplicated. -extract_rules() { - awk '/^[[:space:]]*-[[:space:]]*record:[[:space:]]/ { - sub(/^[[:space:]]*-[[:space:]]*record:[[:space:]]*/, "") - sub(/[[:space:]]*$/, "") - print - }' "$RULES_FILE" | sort -u -} - -# Extract the set of recording-rule references from a dashboard JSON. -# A recording-rule reference is matched by the pattern -# :(:)+ -# where each is [a-z0-9_]. Raw DCGM metrics (DCGM_FI_*), -# kube-state-metrics (kube_*) and similar uppercase / single-word metric -# names do not match because the leading segment must be lowercase and the -# whole expression must contain at least two ':' characters. -extract_refs() { - json_file=$1 - # Prometheus convention allows 2-segment rule names (level:metric); this - # regex is tuned to the 3+ segment convention used in this repo - # (level:metric:op — e.g. cluster:gpu_count:total). Update if future - # rules use 2 segments, otherwise they will be silently skipped. - grep -hoE '[a-z][a-z0-9_]*:[a-z0-9_]+:[a-z0-9_]+' "$json_file" | sort -u -} - -# Resolve "gpu/foo" -> "$DASHBOARDS_DIR/gpu/foo.json" -list_tracked_gpu_dashboards() { - awk '/^gpu\// { print $0 ".json" }' "$DASHBOARDS_LIST" -} - -# Extract the set of DCGM_FI_* metric names declared in a CSV file. Handles -# both the upstream-style default CSV (unindented) and the ConfigMap-style -# custom CSV (YAML-indented). A declaration line starts — after any leading -# whitespace — with "DCGM_FI_," ; comment lines begin with "#" and are -# skipped. Uses POSIX awk's match()+RSTART/RLENGTH so no GNU extensions -# are required. -extract_csv_metrics() { - file=$1 - awk ' - { - line = $0 - sub(/^[[:space:]]+/, "", line) - if (line ~ /^#/) next - if (match(line, /^DCGM_FI_[A-Z0-9_]+/)) { - print substr(line, RSTART, RLENGTH) - } - } - ' "$file" | sort -u -} - -# Extract the set of DCGM_FI_* metric references from a text file (dashboard -# JSON or rules YAML). A DCGM metric name has at least two underscore-delimited -# segments after the "DCGM_FI_" prefix (e.g. DCGM_FI_DEV_GPU_UTIL, DCGM_FI_PROF_ -# PIPE_TENSOR_ACTIVE, DCGM_FI_DRIVER_VERSION). Requiring two segments keeps -# the matcher from latching onto glob stubs like "DCGM_FI_DEV_*_VIOLATION" that -# appear in comments. -extract_dcgm_refs() { - file=$1 - grep -hoE 'DCGM_FI_[A-Z0-9]+(_[A-Z0-9]+)+' "$file" | sort -u -} - -@test "every recording rule reference in tracked GPU dashboards has a matching record" { - TMP=$(mktemp -d) - trap 'rm -rf "$TMP"' EXIT - - extract_rules > "$TMP/rules.txt" - [ -s "$TMP/rules.txt" ] || { echo "no recording rules extracted from $RULES_FILE" >&2; exit 1; } - - list_tracked_gpu_dashboards > "$TMP/dashboards.txt" - [ -s "$TMP/dashboards.txt" ] || { echo "no gpu/* dashboards listed in $DASHBOARDS_LIST" >&2; exit 1; } - - failed=0 - while IFS= read -r dashboard_rel; do - dashboard="$DASHBOARDS_DIR/$dashboard_rel" - if [ ! -f "$dashboard" ]; then - echo "ERROR: dashboard listed but file missing: $dashboard" >&2 - failed=1 - continue - fi - - extract_refs "$dashboard" > "$TMP/refs.txt" - # comm -23: lines unique to refs.txt (referenced but not defined) - # Both inputs must be sorted; extract_* helpers already sort. - comm -23 "$TMP/refs.txt" "$TMP/rules.txt" > "$TMP/missing.txt" - if [ -s "$TMP/missing.txt" ]; then - echo "ERROR: $dashboard_rel references undefined recording rules:" >&2 - sed 's/^/ - /' "$TMP/missing.txt" >&2 - failed=1 - fi - done < "$TMP/dashboards.txt" - - [ "$failed" -eq 0 ] -} - -@test "every DCGM metric referenced in tracked dashboards and rules is declared" { - TMP=$(mktemp -d) - trap 'rm -rf "$TMP"' EXIT - - [ -f "$DCGM_DEFAULT_CSV" ] || { echo "missing $DCGM_DEFAULT_CSV" >&2; exit 1; } - [ -f "$DCGM_CUSTOM_CSV" ] || { echo "missing $DCGM_CUSTOM_CSV" >&2; exit 1; } - - { - extract_csv_metrics "$DCGM_DEFAULT_CSV" - extract_csv_metrics "$DCGM_CUSTOM_CSV" - } | sort -u > "$TMP/declared.txt" - [ -s "$TMP/declared.txt" ] || { echo "no DCGM metrics extracted from CSVs" >&2; exit 1; } - - list_tracked_gpu_dashboards > "$TMP/dashboards.txt" - [ -s "$TMP/dashboards.txt" ] || { echo "no gpu/* dashboards listed in $DASHBOARDS_LIST" >&2; exit 1; } - - failed=0 - - # Dashboard coverage — every dashboard's DCGM references must resolve. - while IFS= read -r dashboard_rel; do - dashboard="$DASHBOARDS_DIR/$dashboard_rel" - [ -f "$dashboard" ] || continue # handled by the existence test - extract_dcgm_refs "$dashboard" > "$TMP/refs.txt" - [ -s "$TMP/refs.txt" ] || continue # dashboard relies entirely on recording rules - comm -23 "$TMP/refs.txt" "$TMP/declared.txt" > "$TMP/missing.txt" - if [ -s "$TMP/missing.txt" ]; then - echo "ERROR: $dashboard_rel references DCGM metrics not declared in any CSV:" >&2 - sed 's/^/ - /' "$TMP/missing.txt" >&2 - failed=1 - fi - done < "$TMP/dashboards.txt" - - # Rules coverage — recording rules consume DCGM directly, so their set - # must be declared too, otherwise derived series on every dashboard - # collapse to empty. - extract_dcgm_refs "$RULES_FILE" > "$TMP/rule-refs.txt" - if [ -s "$TMP/rule-refs.txt" ]; then - comm -23 "$TMP/rule-refs.txt" "$TMP/declared.txt" > "$TMP/rule-missing.txt" - if [ -s "$TMP/rule-missing.txt" ]; then - echo "ERROR: gpu-recording.rules.yaml references DCGM metrics not declared in any CSV:" >&2 - sed 's/^/ - /' "$TMP/rule-missing.txt" >&2 - failed=1 - fi - fi - - [ "$failed" -eq 0 ] -} - -@test "every tracked GPU dashboard listed in dashboards-infra.list exists on disk" { - TMP=$(mktemp -d) - trap 'rm -rf "$TMP"' EXIT - - list_tracked_gpu_dashboards > "$TMP/dashboards.txt" - [ -s "$TMP/dashboards.txt" ] || { echo "no gpu/* dashboards listed in $DASHBOARDS_LIST" >&2; exit 1; } - - failed=0 - while IFS= read -r dashboard_rel; do - dashboard="$DASHBOARDS_DIR/$dashboard_rel" - if [ ! -f "$dashboard" ]; then - echo "ERROR: $dashboard_rel listed in dashboards-infra.list but $dashboard does not exist" >&2 - failed=1 - fi - done < "$TMP/dashboards.txt" - - [ "$failed" -eq 0 ] -} diff --git a/hack/dcgm-default-counters.csv b/hack/dcgm-default-counters.csv deleted file mode 100644 index b5e94540..00000000 --- a/hack/dcgm-default-counters.csv +++ /dev/null @@ -1,104 +0,0 @@ -# Snapshot of the upstream DCGM Exporter default-counters.csv used as a -# fixture by hack/check-gpu-recording-rules.bats. The test verifies that -# every DCGM_FI_* metric referenced by a tracked GPU dashboard is either -# declared here (upstream defaults) or in -# packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml -# (the project's custom CSV). -# -# Source: https://github.com/NVIDIA/dcgm-exporter/blob/4.1.1-4.0.4/etc/default-counters.csv -# Pinned to the DCGM Exporter image tag shipped by the gpu-operator -# chart under packages/system/gpu-operator/charts/gpu-operator/values.yaml -# (dcgmExporter.version = 4.1.1-4.0.4-ubuntu22.04). When that image is -# bumped, refresh this file from the matching tag in the NVIDIA/dcgm-exporter -# repo and re-run `./hack/cozytest.sh hack/check-gpu-recording-rules.bats`. - -# Format -# If line starts with a '#' it is considered a comment -# DCGM FIELD, Prometheus metric type, help message - -# Clocks -DCGM_FI_DEV_SM_CLOCK, gauge, SM clock frequency (in MHz). -DCGM_FI_DEV_MEM_CLOCK, gauge, Memory clock frequency (in MHz). - -# Temperature -DCGM_FI_DEV_MEMORY_TEMP, gauge, Memory temperature (in C). -DCGM_FI_DEV_GPU_TEMP, gauge, GPU temperature (in C). - -# Power -DCGM_FI_DEV_POWER_USAGE, gauge, Power draw (in W). -DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION, counter, Total energy consumption since boot (in mJ). - -# PCIE -# DCGM_FI_PROF_PCIE_TX_BYTES, counter, Total number of bytes transmitted through PCIe TX via NVML. -# DCGM_FI_PROF_PCIE_RX_BYTES, counter, Total number of bytes received through PCIe RX via NVML. -DCGM_FI_DEV_PCIE_REPLAY_COUNTER, counter, Total number of PCIe retries. - -# Utilization (the sample period varies depending on the product) -DCGM_FI_DEV_GPU_UTIL, gauge, GPU utilization (in %). -DCGM_FI_DEV_MEM_COPY_UTIL, gauge, Memory utilization (in %). -DCGM_FI_DEV_ENC_UTIL, gauge, Encoder utilization (in %). -DCGM_FI_DEV_DEC_UTIL , gauge, Decoder utilization (in %). - -# Errors and violations -DCGM_FI_DEV_XID_ERRORS, gauge, Value of the last XID error encountered. -# DCGM_FI_DEV_POWER_VIOLATION, counter, Throttling duration due to power constraints (in us). -# DCGM_FI_DEV_THERMAL_VIOLATION, counter, Throttling duration due to thermal constraints (in us). -# DCGM_FI_DEV_SYNC_BOOST_VIOLATION, counter, Throttling duration due to sync-boost constraints (in us). -# DCGM_FI_DEV_BOARD_LIMIT_VIOLATION, counter, Throttling duration due to board limit constraints (in us). -# DCGM_FI_DEV_LOW_UTIL_VIOLATION, counter, Throttling duration due to low utilization (in us). -# DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (in us). - -# Memory usage -DCGM_FI_DEV_FB_FREE, gauge, Framebuffer memory free (in MiB). -DCGM_FI_DEV_FB_USED, gauge, Framebuffer memory used (in MiB). - -# ECC -# DCGM_FI_DEV_ECC_SBE_VOL_TOTAL, counter, Total number of single-bit volatile ECC errors. -# DCGM_FI_DEV_ECC_DBE_VOL_TOTAL, counter, Total number of double-bit volatile ECC errors. -# DCGM_FI_DEV_ECC_SBE_AGG_TOTAL, counter, Total number of single-bit persistent ECC errors. -# DCGM_FI_DEV_ECC_DBE_AGG_TOTAL, counter, Total number of double-bit persistent ECC errors. - -# Retired pages -# DCGM_FI_DEV_RETIRED_SBE, counter, Total number of retired pages due to single-bit errors. -# DCGM_FI_DEV_RETIRED_DBE, counter, Total number of retired pages due to double-bit errors. -# DCGM_FI_DEV_RETIRED_PENDING, counter, Total number of pages pending retirement. - -# NVLink -# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_TOTAL, counter, Total number of NVLink flow-control CRC errors. -# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_TOTAL, counter, Total number of NVLink data CRC errors. -# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_TOTAL, counter, Total number of NVLink retries. -# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_TOTAL, counter, Total number of NVLink recovery errors. -DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL, counter, Total number of NVLink bandwidth counters for all lanes. -# DCGM_FI_DEV_NVLINK_BANDWIDTH_L0, counter, The number of bytes of active NVLink rx or tx data including both header and payload. - -# VGPU License status -DCGM_FI_DEV_VGPU_LICENSE_STATUS, gauge, vGPU License status - -# Remapped rows -DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for uncorrectable errors -DCGM_FI_DEV_CORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for correctable errors -DCGM_FI_DEV_ROW_REMAP_FAILURE, gauge, Whether remapping of rows has failed - -# Static configuration information. These appear as labels on the other metrics -DCGM_FI_DRIVER_VERSION, label, Driver Version -# DCGM_FI_NVML_VERSION, label, NVML Version -# DCGM_FI_DEV_BRAND, label, Device Brand -# DCGM_FI_DEV_SERIAL, label, Device Serial Number -# DCGM_FI_DEV_OEM_INFOROM_VER, label, OEM inforom version -# DCGM_FI_DEV_ECC_INFOROM_VER, label, ECC inforom version -# DCGM_FI_DEV_POWER_INFOROM_VER, label, Power management object inforom version -# DCGM_FI_DEV_INFOROM_IMAGE_VER, label, Inforom image version -# DCGM_FI_DEV_VBIOS_VERSION, label, VBIOS version of the device - -# Datacenter Profiling (DCP) metrics -# NOTE: supported on Nvidia datacenter Volta GPUs and newer -DCGM_FI_PROF_GR_ENGINE_ACTIVE, gauge, Ratio of time the graphics engine is active. -# DCGM_FI_PROF_SM_ACTIVE, gauge, The ratio of cycles an SM has at least 1 warp assigned. -# DCGM_FI_PROF_SM_OCCUPANCY, gauge, The ratio of number of warps resident on an SM. -DCGM_FI_PROF_PIPE_TENSOR_ACTIVE, gauge, Ratio of cycles the tensor (HMMA) pipe is active. -DCGM_FI_PROF_DRAM_ACTIVE, gauge, Ratio of cycles the device memory interface is active sending or receiving data. -# DCGM_FI_PROF_PIPE_FP64_ACTIVE, gauge, Ratio of cycles the fp64 pipes are active. -# DCGM_FI_PROF_PIPE_FP32_ACTIVE, gauge, Ratio of cycles the fp32 pipes are active. -# DCGM_FI_PROF_PIPE_FP16_ACTIVE, gauge, Ratio of cycles the fp16 pipes are active. -DCGM_FI_PROF_PCIE_TX_BYTES, gauge, The rate of data transmitted over the PCIe bus - including both protocol headers and data payloads - in bytes per second. -DCGM_FI_PROF_PCIE_RX_BYTES, gauge, The rate of data received over the PCIe bus - including both protocol headers and data payloads - in bytes per second. diff --git a/packages/apps/kubernetes/README.md b/packages/apps/kubernetes/README.md index bed1117a..ecbbc31b 100644 --- a/packages/apps/kubernetes/README.md +++ b/packages/apps/kubernetes/README.md @@ -128,9 +128,6 @@ See the reference for components utilized in this service: | `addons.gpuOperator` | NVIDIA GPU Operator. | `object` | `{}` | | `addons.gpuOperator.enabled` | Enable GPU Operator. | `bool` | `false` | | `addons.gpuOperator.valuesOverride` | Custom Helm values overrides. | `object` | `{}` | -| `addons.hami` | HAMi GPU virtualization middleware. | `object` | `{}` | -| `addons.hami.enabled` | Enable HAMi (requires GPU Operator). | `bool` | `false` | -| `addons.hami.valuesOverride` | Custom Helm values overrides. | `object` | `{}` | | `addons.fluxcd` | FluxCD GitOps operator. | `object` | `{}` | | `addons.fluxcd.enabled` | Enable FluxCD. | `bool` | `false` | | `addons.fluxcd.valuesOverride` | Custom Helm values overrides. | `object` | `{}` | diff --git a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml index 655dc868..e243bfad 100644 --- a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml @@ -1,11 +1,3 @@ -{{- define "cozystack.defaultGpuOperatorValues" -}} -{{- if .Values.addons.hami.enabled }} -gpu-operator: - devicePlugin: - enabled: false -{{- end }} -{{- end }} - {{- if and .Values.addons.gpuOperator.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease @@ -37,12 +29,9 @@ spec: force: true remediation: retries: -1 - {{- $defaults := fromYaml (include "cozystack.defaultGpuOperatorValues" .) }} - {{- $overrides := deepCopy (default (dict) .Values.addons.gpuOperator.valuesOverride) }} - {{- $merged := mergeOverwrite (default (dict) $defaults) $overrides }} - {{- if $merged }} + {{- with .Values.addons.gpuOperator.valuesOverride }} values: - {{- toYaml $merged | nindent 4 }} + {{- toYaml . | nindent 4 }} {{- end }} dependsOn: diff --git a/packages/apps/kubernetes/templates/helmreleases/hami.yaml b/packages/apps/kubernetes/templates/helmreleases/hami.yaml deleted file mode 100644 index f1538c7c..00000000 --- a/packages/apps/kubernetes/templates/helmreleases/hami.yaml +++ /dev/null @@ -1,49 +0,0 @@ -{{- if and .Values.addons.hami.enabled .Values._namespace.etcd }} -{{- if not .Values.addons.gpuOperator.enabled }} -{{- fail "addons.hami requires addons.gpuOperator to be enabled" }} -{{- end }} -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: {{ .Release.Name }}-hami - labels: - cozystack.io/repository: system - cozystack.io/target-cluster-name: {{ .Release.Name }} - sharding.fluxcd.io/key: tenants -spec: - releaseName: hami - chartRef: - kind: ExternalArtifact - name: cozystack-kubernetes-application-kubevirt-kubernetes-hami - namespace: cozy-system - kubeConfig: - secretRef: - name: {{ .Release.Name }}-admin-kubeconfig - key: super-admin.svc - targetNamespace: cozy-hami - storageNamespace: cozy-hami - interval: 5m - timeout: 10m - install: - createNamespace: true - remediation: - retries: -1 - upgrade: - force: true - remediation: - retries: -1 - {{- with .Values.addons.hami.valuesOverride }} - values: - {{- toYaml . | nindent 4 }} - {{- end }} - - dependsOn: - {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }} - - name: {{ .Release.Name }} - namespace: {{ .Release.Namespace }} - {{- end }} - - name: {{ .Release.Name }}-cilium - namespace: {{ .Release.Namespace }} - - name: {{ .Release.Name }}-gpu-operator - namespace: {{ .Release.Namespace }} -{{- end }} diff --git a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml deleted file mode 100644 index 44528470..00000000 --- a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml +++ /dev/null @@ -1,99 +0,0 @@ -suite: GPU Operator HelmRelease HAMi integration tests -templates: - - templates/helmreleases/gpu-operator.yaml -values: - - values-ci.yaml -tests: - - it: should disable devicePlugin when hami is enabled - set: - addons: - gpuOperator: - enabled: true - valuesOverride: {} - hami: - enabled: true - valuesOverride: {} - asserts: - - equal: - path: spec.values.gpu-operator.devicePlugin.enabled - value: false - - - it: should not have values when hami is disabled and no overrides - set: - addons: - gpuOperator: - enabled: true - valuesOverride: {} - hami: - enabled: false - valuesOverride: {} - asserts: - - notExists: - path: spec.values - - - it: should apply hami defaults when valuesOverride key is omitted - set: - addons: - gpuOperator: - enabled: true - hami: - enabled: true - asserts: - - equal: - path: spec.values.gpu-operator.devicePlugin.enabled - value: false - - - it: should allow user overrides to merge with hami defaults - set: - addons: - gpuOperator: - enabled: true - valuesOverride: - gpu-operator: - driver: - enabled: false - hami: - enabled: true - valuesOverride: {} - asserts: - - equal: - path: spec.values.gpu-operator.devicePlugin.enabled - value: false - - equal: - path: spec.values.gpu-operator.driver.enabled - value: false - - - it: should let user explicitly override devicePlugin.enabled to true with hami enabled - set: - addons: - gpuOperator: - enabled: true - valuesOverride: - gpu-operator: - devicePlugin: - enabled: true - driver: - enabled: false - hami: - enabled: true - valuesOverride: {} - asserts: - - equal: - path: spec.values.gpu-operator.devicePlugin.enabled - value: true - - equal: - path: spec.values.gpu-operator.driver.enabled - value: false - - - it: should not render when gpuOperator is disabled - set: - addons: - gpuOperator: - enabled: false - valuesOverride: {} - hami: - enabled: false - valuesOverride: {} - asserts: - - hasDocuments: - count: 0 diff --git a/packages/apps/kubernetes/tests/hami_test.yaml b/packages/apps/kubernetes/tests/hami_test.yaml deleted file mode 100644 index 27f7c75b..00000000 --- a/packages/apps/kubernetes/tests/hami_test.yaml +++ /dev/null @@ -1,153 +0,0 @@ -suite: HAMi HelmRelease tests -templates: - - templates/helmreleases/hami.yaml -values: - - values-ci.yaml -tests: - - it: should not render when hami is disabled - set: - addons: - hami: - enabled: false - valuesOverride: {} - gpuOperator: - enabled: true - valuesOverride: {} - asserts: - - hasDocuments: - count: 0 - - - it: should render HelmRelease when hami is enabled - set: - addons: - hami: - enabled: true - valuesOverride: {} - gpuOperator: - enabled: true - valuesOverride: {} - asserts: - - hasDocuments: - count: 1 - - isKind: - of: HelmRelease - - - it: should fail when gpuOperator is not enabled - set: - addons: - hami: - enabled: true - valuesOverride: {} - gpuOperator: - enabled: false - valuesOverride: {} - asserts: - - failedTemplate: - errorMessage: "addons.hami requires addons.gpuOperator to be enabled" - - - it: should have correct metadata labels - set: - addons: - hami: - enabled: true - valuesOverride: {} - gpuOperator: - enabled: true - valuesOverride: {} - asserts: - - equal: - path: metadata.labels["cozystack.io/repository"] - value: system - - equal: - path: metadata.labels["sharding.fluxcd.io/key"] - value: tenants - - - it: should use ExternalArtifact chartRef - set: - addons: - hami: - enabled: true - valuesOverride: {} - gpuOperator: - enabled: true - valuesOverride: {} - asserts: - - equal: - path: spec.chartRef.kind - value: ExternalArtifact - - equal: - path: spec.chartRef.namespace - value: cozy-system - - - it: should target cozy-hami namespace - set: - addons: - hami: - enabled: true - valuesOverride: {} - gpuOperator: - enabled: true - valuesOverride: {} - asserts: - - equal: - path: spec.targetNamespace - value: cozy-hami - - equal: - path: spec.storageNamespace - value: cozy-hami - - - it: should depend on gpu-operator and cilium - release: - name: test - namespace: test-ns - set: - addons: - hami: - enabled: true - valuesOverride: {} - gpuOperator: - enabled: true - valuesOverride: {} - asserts: - - contains: - path: spec.dependsOn - content: - name: test-cilium - namespace: test-ns - - contains: - path: spec.dependsOn - content: - name: test-gpu-operator - namespace: test-ns - - - it: should not render spec.values when valuesOverride is empty - set: - addons: - hami: - enabled: true - valuesOverride: {} - gpuOperator: - enabled: true - valuesOverride: {} - asserts: - - hasDocuments: - count: 1 - - notExists: - path: spec.values - - - it: should pass through valuesOverride - set: - addons: - hami: - enabled: true - valuesOverride: - hami: - devicePlugin: - deviceSplitCount: 5 - gpuOperator: - enabled: true - valuesOverride: {} - asserts: - - equal: - path: spec.values.hami.devicePlugin.deviceSplitCount - value: 5 diff --git a/packages/apps/kubernetes/values.schema.json b/packages/apps/kubernetes/values.schema.json index a43deaec..81e9e2b8 100644 --- a/packages/apps/kubernetes/values.schema.json +++ b/packages/apps/kubernetes/values.schema.json @@ -149,7 +149,6 @@ "fluxcd", "gatewayAPI", "gpuOperator", - "hami", "ingressNginx", "monitoringAgents", "velero", @@ -269,28 +268,6 @@ } } }, - "hami": { - "description": "HAMi GPU virtualization middleware.", - "type": "object", - "default": {}, - "required": [ - "enabled", - "valuesOverride" - ], - "properties": { - "enabled": { - "description": "Enable HAMi (requires GPU Operator).", - "type": "boolean", - "default": false - }, - "valuesOverride": { - "description": "Custom Helm values overrides.", - "type": "object", - "default": {}, - "x-kubernetes-preserve-unknown-fields": true - } - } - }, "ingressNginx": { "description": "Ingress-NGINX controller.", "type": "object", diff --git a/packages/apps/kubernetes/values.yaml b/packages/apps/kubernetes/values.yaml index d609476c..68ba07b0 100644 --- a/packages/apps/kubernetes/values.yaml +++ b/packages/apps/kubernetes/values.yaml @@ -94,10 +94,6 @@ host: "" ## @field {bool} enabled - Enable FluxCD. ## @field {object} valuesOverride - Custom Helm values overrides. -## @typedef {struct} HAMiAddon - HAMi GPU virtualization middleware. -## @field {bool} enabled - Enable HAMi (requires GPU Operator). -## @field {object} valuesOverride - Custom Helm values overrides. - ## @typedef {struct} MonitoringAgentsAddon - Monitoring agents (Fluent Bit, VMAgents). ## @field {bool} enabled - Enable monitoring agents. ## @field {object} valuesOverride - Custom Helm values overrides. @@ -118,7 +114,6 @@ host: "" ## @field {GatewayAPIAddon} gatewayAPI - Gateway API addon. ## @field {IngressNginxAddon} ingressNginx - Ingress-NGINX controller. ## @field {GPUOperatorAddon} gpuOperator - NVIDIA GPU Operator. -## @field {HAMiAddon} hami - HAMi GPU virtualization middleware. ## @field {FluxCDAddon} fluxcd - FluxCD GitOps operator. ## @field {MonitoringAgentsAddon} monitoringAgents - Monitoring agents. ## @field {VerticalPodAutoscalerAddon} verticalPodAutoscaler - Vertical Pod Autoscaler. @@ -142,9 +137,6 @@ addons: gpuOperator: enabled: false valuesOverride: {} - hami: - enabled: false - valuesOverride: {} fluxcd: enabled: false valuesOverride: {} diff --git a/packages/apps/postgres/README.md b/packages/apps/postgres/README.md index 4cda7284..5a550f7a 100644 --- a/packages/apps/postgres/README.md +++ b/packages/apps/postgres/README.md @@ -133,13 +133,12 @@ See: ### Bootstrap (recovery) parameters -| Name | Description | Type | Value | -| ------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------- | -| `bootstrap` | Bootstrap configuration. | `object` | `{}` | -| `bootstrap.enabled` | Whether to restore from a backup. | `bool` | `false` | -| `bootstrap.recoveryTime` | Timestamp (RFC3339) for point-in-time recovery; empty means latest. | `string` | `""` | -| `bootstrap.oldName` | Previous cluster name before deletion. | `string` | `""` | -| `bootstrap.serverName` | Barman server name (S3 path prefix) used by the original cluster when writing backups. Set this only when the original cluster had an explicit barmanObjectStore.serverName that differed from its Kubernetes resource name. | `string` | `""` | +| Name | Description | Type | Value | +| ------------------------ | ------------------------------------------------------------------- | -------- | ------- | +| `bootstrap` | Bootstrap configuration. | `object` | `{}` | +| `bootstrap.enabled` | Whether to restore from a backup. | `bool` | `false` | +| `bootstrap.recoveryTime` | Timestamp (RFC3339) for point-in-time recovery; empty means latest. | `string` | `""` | +| `bootstrap.oldName` | Previous cluster name before deletion. | `string` | `""` | ## Parameter examples and reference diff --git a/packages/apps/postgres/templates/db.yaml b/packages/apps/postgres/templates/db.yaml index 2cdaec1d..5c40b2c7 100644 --- a/packages/apps/postgres/templates/db.yaml +++ b/packages/apps/postgres/templates/db.yaml @@ -32,9 +32,6 @@ spec: - name: {{ .Values.bootstrap.oldName }} barmanObjectStore: destinationPath: {{ .Values.backup.destinationPath }} - {{- if .Values.bootstrap.serverName }} - serverName: {{ .Values.bootstrap.serverName }} - {{- end }} endpointURL: {{ .Values.backup.endpointURL }} s3Credentials: accessKeyId: diff --git a/packages/apps/postgres/values.schema.json b/packages/apps/postgres/values.schema.json index 98e29822..b2a4aeba 100644 --- a/packages/apps/postgres/values.schema.json +++ b/packages/apps/postgres/values.schema.json @@ -254,11 +254,6 @@ "description": "Timestamp (RFC3339) for point-in-time recovery; empty means latest.", "type": "string", "default": "" - }, - "serverName": { - "description": "Barman server name (S3 path prefix) used by the original cluster when writing backups. Set this only when the original cluster had an explicit barmanObjectStore.serverName that differed from its Kubernetes resource name.", - "type": "string", - "default": "" } } } diff --git a/packages/apps/postgres/values.yaml b/packages/apps/postgres/values.yaml index 2ceaa9cf..b8f07f63 100644 --- a/packages/apps/postgres/values.yaml +++ b/packages/apps/postgres/values.yaml @@ -154,7 +154,6 @@ backup: ## @field {bool} enabled - Whether to restore from a backup. ## @field {string} [recoveryTime] - Timestamp (RFC3339) for point-in-time recovery; empty means latest. ## @field {string} oldName - Previous cluster name before deletion. -## @field {string} [serverName] - Barman server name (S3 path prefix) used by the original cluster when writing backups. Set this only when the original cluster had an explicit barmanObjectStore.serverName that differed from its Kubernetes resource name. ## @param {Bootstrap} bootstrap - Bootstrap configuration. bootstrap: @@ -162,4 +161,3 @@ bootstrap: # example: 2020-11-26 15:22:00.00000+00 recoveryTime: "" oldName: "" - serverName: "" diff --git a/packages/apps/tenant/Makefile b/packages/apps/tenant/Makefile index 2f51d836..3fe3810d 100644 --- a/packages/apps/tenant/Makefile +++ b/packages/apps/tenant/Makefile @@ -3,6 +3,3 @@ include ../../../hack/package.mk generate: cozyvalues-gen -m 'tenant' -v values.yaml -s values.schema.json -r README.md -g ../../../api/apps/v1alpha1/tenant/types.go ../../../hack/update-crd.sh - -test: - helm unittest . diff --git a/packages/apps/tenant/templates/cilium-lb-pool.yaml b/packages/apps/tenant/templates/cilium-lb-pool.yaml deleted file mode 100644 index 63dbcaca..00000000 --- a/packages/apps/tenant/templates/cilium-lb-pool.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{- $exposeMode := (index .Values._cluster "expose-mode") | default "externalIPs" }} -{{- $exposeIngress := (index .Values._cluster "expose-ingress") | default "tenant-root" }} -{{- $exposeIPs := (index .Values._cluster "expose-external-ips") | default "" | nospace }} -{{- $ipsList := list }} -{{- range splitList "," $exposeIPs }} - {{- $ip := . | trim }} - {{- if $ip }} - {{- $ipsList = append $ipsList $ip }} - {{- end }} -{{- end }} -{{- $isPublishingIngressLB := and - (eq $exposeMode "loadBalancer") - (eq $exposeIngress .Release.Namespace) - .Values.ingress }} -{{- if and $isPublishingIngressLB $ipsList }} -apiVersion: cilium.io/v2 -kind: CiliumLoadBalancerIPPool -metadata: - name: {{ trimPrefix "tenant-" .Release.Namespace }}-exposure -spec: - blocks: - {{- range $ipsList }} - - cidr: {{ . }}{{ if not (contains "/" .) }}/{{ if contains ":" . }}128{{ else }}32{{ end }}{{ end }} - {{- end }} - serviceSelector: - matchLabels: - "io.kubernetes.service.namespace": {{ .Release.Namespace | quote }} -{{- end }} diff --git a/packages/apps/tenant/tests/exposure_test.yaml b/packages/apps/tenant/tests/exposure_test.yaml deleted file mode 100644 index e86fab5b..00000000 --- a/packages/apps/tenant/tests/exposure_test.yaml +++ /dev/null @@ -1,141 +0,0 @@ -suite: tenant CiliumLoadBalancerIPPool rendering for publishing.exposure=loadBalancer -templates: - - templates/cilium-lb-pool.yaml - -release: - name: tenant-root - namespace: tenant-root - -tests: - - it: default exposure (externalIPs) renders no pool - set: - ingress: true - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10,192.0.2.11" - asserts: - - hasDocuments: - count: 0 - - - it: loadBalancer mode in publishing tenant with ingress=true renders v2 pool with namespace-only selector - set: - ingress: true - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10,192.0.2.11" - expose-mode: loadBalancer - asserts: - - hasDocuments: - count: 1 - - equal: - path: apiVersion - value: cilium.io/v2 - - equal: - path: kind - value: CiliumLoadBalancerIPPool - - equal: - path: metadata.name - value: root-exposure - - equal: - path: spec.blocks - value: - - cidr: 192.0.2.10/32 - - cidr: 192.0.2.11/32 - - equal: - path: spec.serviceSelector.matchLabels["io.kubernetes.service.namespace"] - value: tenant-root - - notExists: - path: spec.serviceSelector.matchLabels["app.kubernetes.io/name"] - - - it: loadBalancer mode with IPv6 emits /128 CIDR - set: - ingress: true - _cluster: - expose-ingress: tenant-root - expose-external-ips: "2001:db8::1" - expose-mode: loadBalancer - asserts: - - equal: - path: spec.blocks - value: - - cidr: 2001:db8::1/128 - - - it: loadBalancer mode with mixed IPv4 and IPv6 emits correct CIDR per family - set: - ingress: true - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10,2001:db8::1" - expose-mode: loadBalancer - asserts: - - equal: - path: spec.blocks - value: - - cidr: 192.0.2.10/32 - - cidr: 2001:db8::1/128 - - - it: loadBalancer mode accepts pre-CIDR input without double-suffixing - set: - ingress: true - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10/32,2001:db8::1/128" - expose-mode: loadBalancer - asserts: - - equal: - path: spec.blocks - value: - - cidr: 192.0.2.10/32 - - cidr: 2001:db8::1/128 - - - it: loadBalancer mode filters out empty entries from externalIPs (trailing, leading, repeated commas) - set: - ingress: true - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10,,192.0.2.11," - expose-mode: loadBalancer - asserts: - - hasDocuments: - count: 1 - - equal: - path: spec.blocks - value: - - cidr: 192.0.2.10/32 - - cidr: 192.0.2.11/32 - - - it: loadBalancer mode with ingress=false in publishing tenant renders no pool - set: - ingress: false - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10" - expose-mode: loadBalancer - asserts: - - hasDocuments: - count: 0 - - - it: loadBalancer mode in a non-publishing tenant renders no pool - set: - ingress: true - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10" - expose-mode: loadBalancer - release: - name: tenant-u1 - namespace: tenant-u1 - asserts: - - hasDocuments: - count: 0 - - - it: loadBalancer mode in publishing tenant with empty externalIPs renders no pool - set: - ingress: true - _cluster: - expose-ingress: tenant-root - expose-external-ips: "" - expose-mode: loadBalancer - asserts: - - hasDocuments: - count: 0 diff --git a/packages/apps/vm-instance/README.md b/packages/apps/vm-instance/README.md index 9d6a52b2..a2b6603e 100644 --- a/packages/apps/vm-instance/README.md +++ b/packages/apps/vm-instance/README.md @@ -36,32 +36,31 @@ virtctl ssh @ ### Common parameters -| Name | Description | Type | Value | -| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ----------- | -| `external` | Enable external access from outside the cluster. | `bool` | `false` | -| `externalMethod` | Method to pass through traffic to the VM. | `string` | `PortList` | -| `externalPorts` | Ports to forward from outside the cluster. | `[]int` | `[22]` | -| `externalAllowICMP` | Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect. | `bool` | `true` | -| `runStrategy` | Requested running state of the VirtualMachineInstance | `string` | `Always` | -| `instanceType` | Virtual Machine instance type. | `string` | `u1.medium` | -| `instanceProfile` | Virtual Machine preferences profile. | `string` | `ubuntu` | -| `disks` | List of disks to attach. | `[]object` | `[]` | -| `disks[i].name` | Disk name. | `string` | `""` | -| `disks[i].bus` | Disk bus type (e.g. "sata"). | `string` | `""` | -| `networks` | Networks to attach the VM to. | `[]object` | `[]` | -| `networks[i].name` | Network attachment name. | `string` | `""` | -| `subnets` | Deprecated: use networks instead. | `[]object` | `[]` | -| `subnets[i].name` | Network attachment name. | `string` | `""` | -| `gpus` | List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). | `[]object` | `[]` | -| `gpus[i].name` | The name of the GPU resource to attach. | `string` | `""` | -| `cpuModel` | Model specifies the CPU model inside the VMI. List of available models https://github.com/libvirt/libvirt/tree/master/src/cpu_map | `string` | `""` | -| `resources` | Resource configuration for the virtual machine. | `object` | `{}` | -| `resources.cpu` | Number of CPU cores allocated. | `quantity` | `""` | -| `resources.memory` | Amount of memory allocated. | `quantity` | `""` | -| `resources.sockets` | Number of CPU sockets (vCPU topology). | `quantity` | `""` | -| `sshKeys` | List of SSH public keys for authentication. | `[]string` | `[]` | -| `cloudInit` | Cloud-init user data. | `string` | `""` | -| `cloudInitSeed` | Seed string to generate SMBIOS UUID for the VM. | `string` | `""` | +| Name | Description | Type | Value | +| ------------------- | --------------------------------------------------------------------------------------------------------------------------------- | ---------- | ----------- | +| `external` | Enable external access from outside the cluster. | `bool` | `false` | +| `externalMethod` | Method to pass through traffic to the VM. | `string` | `PortList` | +| `externalPorts` | Ports to forward from outside the cluster. | `[]int` | `[22]` | +| `runStrategy` | Requested running state of the VirtualMachineInstance | `string` | `Always` | +| `instanceType` | Virtual Machine instance type. | `string` | `u1.medium` | +| `instanceProfile` | Virtual Machine preferences profile. | `string` | `ubuntu` | +| `disks` | List of disks to attach. | `[]object` | `[]` | +| `disks[i].name` | Disk name. | `string` | `""` | +| `disks[i].bus` | Disk bus type (e.g. "sata"). | `string` | `""` | +| `networks` | Networks to attach the VM to. | `[]object` | `[]` | +| `networks[i].name` | Network attachment name. | `string` | `""` | +| `subnets` | Deprecated: use networks instead. | `[]object` | `[]` | +| `subnets[i].name` | Network attachment name. | `string` | `""` | +| `gpus` | List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). | `[]object` | `[]` | +| `gpus[i].name` | The name of the GPU resource to attach. | `string` | `""` | +| `cpuModel` | Model specifies the CPU model inside the VMI. List of available models https://github.com/libvirt/libvirt/tree/master/src/cpu_map | `string` | `""` | +| `resources` | Resource configuration for the virtual machine. | `object` | `{}` | +| `resources.cpu` | Number of CPU cores allocated. | `quantity` | `""` | +| `resources.memory` | Amount of memory allocated. | `quantity` | `""` | +| `resources.sockets` | Number of CPU sockets (vCPU topology). | `quantity` | `""` | +| `sshKeys` | List of SSH public keys for authentication. | `[]string` | `[]` | +| `cloudInit` | Cloud-init user data. | `string` | `""` | +| `cloudInitSeed` | Seed string to generate SMBIOS UUID for the VM. | `string` | `""` | ## U Series diff --git a/packages/apps/vm-instance/templates/service.yaml b/packages/apps/vm-instance/templates/service.yaml index cfeffa81..b7e3a420 100644 --- a/packages/apps/vm-instance/templates/service.yaml +++ b/packages/apps/vm-instance/templates/service.yaml @@ -9,10 +9,7 @@ metadata: {{- if .Values.external }} service.kubernetes.io/service-proxy-name: "cozy-proxy" annotations: - networking.cozystack.io/wholeIP: {{ ternary "true" "false" (eq .Values.externalMethod "WholeIP") | quote }} - {{- if eq .Values.externalMethod "PortList" }} - networking.cozystack.io/allowICMP: {{ ternary "true" "false" (ne .Values.externalAllowICMP false) | quote }} - {{- end }} + networking.cozystack.io/wholeIP: "true" {{- end }} spec: type: {{ ternary "LoadBalancer" "ClusterIP" .Values.external }} diff --git a/packages/apps/vm-instance/values.schema.json b/packages/apps/vm-instance/values.schema.json index 01bd30a9..34f7f634 100644 --- a/packages/apps/vm-instance/values.schema.json +++ b/packages/apps/vm-instance/values.schema.json @@ -26,11 +26,6 @@ "type": "integer" } }, - "externalAllowICMP": { - "description": "Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect.", - "type": "boolean", - "default": true - }, "runStrategy": { "description": "Requested running state of the VirtualMachineInstance", "type": "string", diff --git a/packages/apps/vm-instance/values.yaml b/packages/apps/vm-instance/values.yaml index 92e399c2..f07b75d3 100644 --- a/packages/apps/vm-instance/values.yaml +++ b/packages/apps/vm-instance/values.yaml @@ -31,9 +31,6 @@ externalMethod: PortList externalPorts: - 22 -## @param {bool} externalAllowICMP - Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect. -externalAllowICMP: true - ## @enum {string} RunStrategy - Requested running state of the VirtualMachineInstance ## @value Always - VMI should always be running ## @value Halted - VMI should never be running diff --git a/packages/core/platform/sources/hami.yaml b/packages/core/platform/sources/hami.yaml deleted file mode 100644 index 0184e405..00000000 --- a/packages/core/platform/sources/hami.yaml +++ /dev/null @@ -1,24 +0,0 @@ ---- -apiVersion: cozystack.io/v1alpha1 -kind: PackageSource -metadata: - name: cozystack.hami -spec: - sourceRef: - kind: OCIRepository - name: cozystack-packages - namespace: cozy-system - path: / - variants: - - name: default - dependsOn: - - cozystack.gpu-operator - components: - - name: hami - path: system/hami - valuesFiles: - - values.yaml - install: - privileged: true - namespace: cozy-hami - releaseName: hami diff --git a/packages/core/platform/sources/kubernetes-application.yaml b/packages/core/platform/sources/kubernetes-application.yaml index 088383ad..9787cb19 100644 --- a/packages/core/platform/sources/kubernetes-application.yaml +++ b/packages/core/platform/sources/kubernetes-application.yaml @@ -52,8 +52,6 @@ spec: path: system/cilium - name: kubernetes-gpu-operator path: system/gpu-operator - - name: kubernetes-hami - path: system/hami - name: kubernetes-vertical-pod-autoscaler path: system/vertical-pod-autoscaler - name: kubernetes-prometheus-operator-crds diff --git a/packages/core/platform/templates/apps.yaml b/packages/core/platform/templates/apps.yaml index 6e1ae25a..7ee0de52 100644 --- a/packages/core/platform/templates/apps.yaml +++ b/packages/core/platform/templates/apps.yaml @@ -30,7 +30,6 @@ stringData: expose-services: {{ .Values.publishing.exposedServices | join "," | quote }} expose-ingress: {{ .Values.publishing.ingressName | quote }} expose-external-ips: {{ .Values.publishing.externalIPs | join "," | quote }} - expose-mode: {{ .Values.publishing.exposure | default "externalIPs" | quote }} cluster-domain: {{ .Values.networking.clusterDomain | quote }} api-server-endpoint: {{ .Values.publishing.apiServerEndpoint | quote }} {{- with .Values.branding }} diff --git a/packages/core/platform/templates/bundles/iaas.yaml b/packages/core/platform/templates/bundles/iaas.yaml index 66ec98b2..ced0a322 100644 --- a/packages/core/platform/templates/bundles/iaas.yaml +++ b/packages/core/platform/templates/bundles/iaas.yaml @@ -10,7 +10,6 @@ {{include "cozystack.platform.package.default" (list "cozystack.kubevirt-cdi" $) }} {{include "cozystack.platform.package.optional.default" (list "cozystack.vm-default-images" $) }} {{include "cozystack.platform.package.optional.default" (list "cozystack.gpu-operator" $) }} -{{include "cozystack.platform.package.optional.default" (list "cozystack.hami" $) }} {{include "cozystack.platform.package.default" (list "cozystack.kamaji" $) }} {{include "cozystack.platform.package.default" (list "cozystack.capi-operator" $) }} {{include "cozystack.platform.package.default" (list "cozystack.capi-provider-bootstrap-kubeadm" $) }} diff --git a/packages/core/platform/templates/bundles/system.yaml b/packages/core/platform/templates/bundles/system.yaml index 6c587c82..43ea7266 100644 --- a/packages/core/platform/templates/bundles/system.yaml +++ b/packages/core/platform/templates/bundles/system.yaml @@ -105,6 +105,9 @@ {{- /* cozystack-api DaemonSet */ -}} {{- $apiValues := dict "cozystackAPI" (dict "nodeSelector" $genericNodeSelector) -}} {{- $_ := set $cozystackEngineComponents "cozystack-api" (dict "values" $apiValues) -}} +{{- /* lineage-controller-webhook DaemonSet */ -}} +{{- $lineageValues := dict "lineageControllerWebhook" (dict "nodeSelector" $genericNodeSelector) -}} +{{- $_ := set $cozystackEngineComponents "lineage-controller-webhook" (dict "values" $lineageValues) -}} {{- end -}} {{- if .Values.authentication.oidc.enabled }} {{include "cozystack.platform.package" (list "cozystack.cozystack-engine" "oidc" $ $cozystackEngineComponents) }} diff --git a/packages/core/platform/values.yaml b/packages/core/platform/values.yaml index 605e23bc..f33926db 100644 --- a/packages/core/platform/values.yaml +++ b/packages/core/platform/values.yaml @@ -45,41 +45,6 @@ publishing: - cdi-uploadproxy apiServerEndpoint: "" # example: "https://api.example.org" externalIPs: [] - # Exposure mode for the ingress-nginx Service. When "externalIPs" (current - # default) is selected, the Service is created as ClusterIP with - # Service.spec.externalIPs set from publishing.externalIPs. When - # "loadBalancer" is selected, the Service is type: LoadBalancer and a - # CiliumLoadBalancerIPPool makes those same addresses allocatable via LB IPAM. - # - # Service.spec.externalIPs is deprecated upstream in Kubernetes v1.36 - # (KEP-5707). The AllowServiceExternalIPs feature gate is expected to default - # to false around v1.40 and the implementation removed around v1.43 — switch - # to "loadBalancer" before upgrading past v1.40. - # - # Caveats for the "loadBalancer" mode: - # - publishing.externalIPs must contain at least one non-empty address, - # otherwise the chart render fails with an explicit error (a LoadBalancer - # Service without a pool would sit in forever). - # - The ingress-nginx Service is created with externalTrafficPolicy: Local - # to preserve the client source IP. Traffic arriving on a node that does - # not host an ingress-nginx pod is dropped, so the external IP must be - # routed to a node that runs the ingress pod (floating IP / keepalived / - # upstream router / podAntiAffinity). - # - Cilium does NOT announce the IP on its own unless L2 announcements or - # BGP are enabled in the Cilium values (disabled by default in Cozystack). - # This mode assumes the operator already routes the externalIPs to a - # cluster node; enabling announcements is out of scope for this setting. - # - Switching this value on a running cluster causes the ingress-nginx - # Service to be recreated (the HelmRelease has upgrade.force: true and - # the Service kind changes between ClusterIP and LoadBalancer). Expect a - # brief interruption of ingress traffic during the flip. - # - # Scope: this setting only controls the ingress-nginx Service. Other - # cozystack components that currently write Service.spec.externalIPs directly - # (e.g. the vpn app at packages/apps/vpn/templates/service.yaml) are NOT - # migrated by flipping this value and must be addressed separately before - # the AllowServiceExternalIPs feature gate flips to off in ~v1.40. - exposure: externalIPs # "externalIPs" or "loadBalancer" certificates: solver: http01 # "http01" or "dns01" issuerName: letsencrypt-prod diff --git a/packages/extra/etcd/Makefile b/packages/extra/etcd/Makefile index 2b3ed61e..f37d6e1e 100644 --- a/packages/extra/etcd/Makefile +++ b/packages/extra/etcd/Makefile @@ -5,6 +5,3 @@ include ../../../hack/package.mk generate: cozyvalues-gen -v values.yaml -s values.schema.json -r README.md ../../../hack/update-crd.sh - -test: - helm unittest . diff --git a/packages/extra/etcd/templates/hook/job.yaml b/packages/extra/etcd/templates/hook/job.yaml new file mode 100644 index 00000000..3bf2f84b --- /dev/null +++ b/packages/extra/etcd/templates/hook/job.yaml @@ -0,0 +1,39 @@ +{{- $shouldUpdateCerts := true }} +{{- $configMap := lookup "v1" "ConfigMap" .Release.Namespace "etcd-deployed-version" }} +{{- if $configMap }} + {{- $deployedVersion := index $configMap "data" "version" }} + {{- if $deployedVersion | semverCompare ">= 2.6.1" }} + {{- $shouldUpdateCerts = false }} + {{- end }} +{{- end }} + +{{- if $shouldUpdateCerts }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: etcd-hook + annotations: + helm.sh/hook: post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +spec: + template: + metadata: + labels: + policy.cozystack.io/allow-to-apiserver: "true" + spec: + serviceAccountName: etcd-hook + containers: + - name: kubectl + image: docker.io/alpine/k8s:1.33.4 + command: + - sh + args: + - -exc + - |- + kubectl --namespace={{ .Release.Namespace }} delete secrets etcd-ca-tls etcd-peer-ca-tls + sleep 10 + kubectl --namespace={{ .Release.Namespace }} delete secrets etcd-client-tls etcd-peer-tls etcd-server-tls + kubectl --namespace={{ .Release.Namespace }} delete pods --selector=app.kubernetes.io/instance=etcd,app.kubernetes.io/managed-by=etcd-operator,app.kubernetes.io/name=etcd,cozystack.io/service=etcd + restartPolicy: Never +{{- end }} diff --git a/packages/extra/etcd/templates/hook/role.yaml b/packages/extra/etcd/templates/hook/role.yaml new file mode 100644 index 00000000..327eeadb --- /dev/null +++ b/packages/extra/etcd/templates/hook/role.yaml @@ -0,0 +1,26 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + annotations: + helm.sh/hook: post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + name: etcd-hook +rules: +- apiGroups: + - "" + resources: + - secrets + - pods + verbs: + - get + - list + - watch + - delete +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch diff --git a/packages/extra/etcd/templates/hook/rolebinding.yaml b/packages/extra/etcd/templates/hook/rolebinding.yaml new file mode 100644 index 00000000..0ee0ffd1 --- /dev/null +++ b/packages/extra/etcd/templates/hook/rolebinding.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: etcd-hook + annotations: + helm.sh/hook: post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: etcd-hook +subjects: + - kind: ServiceAccount + name: etcd-hook + namespace: {{ .Release.Namespace | quote }} diff --git a/packages/extra/etcd/templates/hook/serviceaccount.yaml b/packages/extra/etcd/templates/hook/serviceaccount.yaml new file mode 100644 index 00000000..552fb5fc --- /dev/null +++ b/packages/extra/etcd/templates/hook/serviceaccount.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: etcd-hook + annotations: + helm.sh/hook: post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded diff --git a/packages/extra/etcd/templates/version.yaml b/packages/extra/etcd/templates/version.yaml new file mode 100644 index 00000000..cc9375bb --- /dev/null +++ b/packages/extra/etcd/templates/version.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: etcd-deployed-version +data: + version: {{ .Chart.Version }} diff --git a/packages/extra/etcd/tests/no-post-upgrade-hook_test.yaml b/packages/extra/etcd/tests/no-post-upgrade-hook_test.yaml deleted file mode 100644 index 0e4f5aa3..00000000 --- a/packages/extra/etcd/tests/no-post-upgrade-hook_test.yaml +++ /dev/null @@ -1,34 +0,0 @@ -suite: etcd chart does not ship a destructive post-upgrade cert-regeneration hook - -release: - name: etcd - namespace: tenant-root - -templates: - - templates/check-release-name.yaml - - templates/dashboard-resourcemap.yaml - - templates/datastore.yaml - - templates/etcd-defrag.yaml - - templates/hook/job.yaml - - templates/podscrape.yaml - - templates/prometheus-rules.yaml - - templates/version.yaml - -tests: - - it: renders no Job named etcd-hook - documentSelector: - path: metadata.name - value: etcd-hook - skipEmptyTemplates: true - asserts: - - hasDocuments: - count: 0 - - - it: renders no ConfigMap named etcd-deployed-version - documentSelector: - path: metadata.name - value: etcd-deployed-version - skipEmptyTemplates: true - asserts: - - hasDocuments: - count: 0 diff --git a/packages/extra/ingress/Makefile b/packages/extra/ingress/Makefile index 65b53c03..958ce484 100644 --- a/packages/extra/ingress/Makefile +++ b/packages/extra/ingress/Makefile @@ -10,6 +10,3 @@ get-cloudflare-ips: generate: cozyvalues-gen -v values.yaml -s values.schema.json -r README.md ../../../hack/update-crd.sh - -test: - helm unittest . diff --git a/packages/extra/ingress/README.md b/packages/extra/ingress/README.md index c541d8e9..0e786dfb 100644 --- a/packages/extra/ingress/README.md +++ b/packages/extra/ingress/README.md @@ -14,17 +14,3 @@ | `resources.memory` | Memory (RAM) available to each replica. | `quantity` | `""` | | `resourcesPreset` | Default sizing preset used when `resources` is omitted. | `string` | `micro` | - -## Exposure mode - -The ingress Service type is driven by the cluster-wide `publishing.exposure` value in the platform chart, not by any key in this package. Two modes exist: - -- `externalIPs` (default) has three rendered shapes: - - Release namespace matches `publishing.ingressName` AND `publishing.externalIPs` is non-empty → Service is `ClusterIP` with `Service.spec.externalIPs` set from that list and `externalTrafficPolicy: Cluster`. - - Release namespace matches `publishing.ingressName` but `publishing.externalIPs` is empty → Service falls back to `type: LoadBalancer` with `externalTrafficPolicy: Local`. - - Release namespace does not match `publishing.ingressName` (non-root tenants) → Service is `type: LoadBalancer` with `externalTrafficPolicy: Local`. - `Service.spec.externalIPs` is deprecated upstream in Kubernetes v1.36 (KEP-5707); plan migration before v1.40. -- `loadBalancer` — Service is `type: LoadBalancer` with `externalTrafficPolicy: Local`, and a `CiliumLoadBalancerIPPool` makes the addresses in `publishing.externalIPs` allocatable via Cilium LB IPAM. Requires `publishing.externalIPs` to contain at least one non-empty address (render fails otherwise) and assumes the addresses are already routed to a cluster node (floating IP / upstream router). See the inline comment on `publishing.exposure` in the platform chart for full caveats, including the note that switching the value on a running cluster causes the ingress Service to be recreated. - -This setting only migrates ingress-nginx away from `Service.spec.externalIPs`. Other cozystack components that use the same deprecated field (e.g. the `vpn` app) must be migrated separately before Kubernetes v1.40 flips the `AllowServiceExternalIPs` feature gate off. - diff --git a/packages/extra/ingress/templates/nginx-ingress.yaml b/packages/extra/ingress/templates/nginx-ingress.yaml index 8e1f00fb..ca50d276 100644 --- a/packages/extra/ingress/templates/nginx-ingress.yaml +++ b/packages/extra/ingress/templates/nginx-ingress.yaml @@ -1,19 +1,5 @@ {{- $exposeIngress := (index .Values._cluster "expose-ingress") | default "tenant-root" }} {{- $exposeExternalIPs := (index .Values._cluster "expose-external-ips") | default "" | nospace }} -{{- $exposeMode := (index .Values._cluster "expose-mode") | default "externalIPs" }} -{{- $exposeIPsList := list }} -{{- range splitList "," $exposeExternalIPs }} - {{- $ip := . | trim }} - {{- if $ip }} - {{- $exposeIPsList = append $exposeIPsList $ip }} - {{- end }} -{{- end }} -{{- if not (has $exposeMode (list "externalIPs" "loadBalancer")) }} -{{- fail (printf "unknown publishing.exposure mode %q: must be \"externalIPs\" or \"loadBalancer\"" $exposeMode) }} -{{- end }} -{{- if and (eq $exposeMode "loadBalancer") (eq $exposeIngress .Release.Namespace) (not $exposeIPsList) }} -{{- fail "publishing.exposure=loadBalancer requires publishing.externalIPs to contain at least one non-empty address: the CiliumLoadBalancerIPPool has nothing to advertise and the Service would stay in ." }} -{{- end }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -55,12 +41,9 @@ spec: enabled: false {{- end }} service: - {{- if and (eq $exposeIngress .Release.Namespace) (eq $exposeMode "loadBalancer") }} - type: LoadBalancer - externalTrafficPolicy: Local - {{- else if and (eq $exposeIngress .Release.Namespace) $exposeIPsList }} + {{- if and (eq $exposeIngress .Release.Namespace) $exposeExternalIPs }} externalIPs: - {{- toYaml $exposeIPsList | nindent 12 }} + {{- toYaml (splitList "," $exposeExternalIPs) | nindent 12 }} type: ClusterIP externalTrafficPolicy: Cluster {{- else }} diff --git a/packages/extra/ingress/tests/exposure_test.yaml b/packages/extra/ingress/tests/exposure_test.yaml deleted file mode 100644 index 1c76b3da..00000000 --- a/packages/extra/ingress/tests/exposure_test.yaml +++ /dev/null @@ -1,165 +0,0 @@ -suite: ingress exposure modes -templates: - - templates/nginx-ingress.yaml - -release: - name: ingress - namespace: tenant-root - -tests: - - it: default exposure (externalIPs) renders ClusterIP Service with spec.externalIPs - set: - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10,192.0.2.11" - asserts: - - template: templates/nginx-ingress.yaml - equal: - path: spec.values.ingress-nginx.controller.service.type - value: ClusterIP - - template: templates/nginx-ingress.yaml - equal: - path: spec.values.ingress-nginx.controller.service.externalTrafficPolicy - value: Cluster - - template: templates/nginx-ingress.yaml - equal: - path: spec.values.ingress-nginx.controller.service.externalIPs - value: - - 192.0.2.10 - - 192.0.2.11 - - template: templates/nginx-ingress.yaml - notExists: - path: spec.values.ingress-nginx.controller.service.labels - - - it: legacy config without expose-mode falls back to externalIPs behavior - set: - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10" - asserts: - - template: templates/nginx-ingress.yaml - equal: - path: spec.values.ingress-nginx.controller.service.type - value: ClusterIP - - - it: externalIPs mode in a namespace other than publishing.ingressName renders LoadBalancer fallback without externalIPs - set: - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10" - release: - namespace: tenant-other - asserts: - - template: templates/nginx-ingress.yaml - equal: - path: spec.values.ingress-nginx.controller.service.type - value: LoadBalancer - - template: templates/nginx-ingress.yaml - equal: - path: spec.values.ingress-nginx.controller.service.externalTrafficPolicy - value: Local - - template: templates/nginx-ingress.yaml - notExists: - path: spec.values.ingress-nginx.controller.service.externalIPs - - - it: loadBalancer mode renders LoadBalancer Service without externalIPs on the Service - set: - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10,192.0.2.11" - expose-mode: loadBalancer - asserts: - - template: templates/nginx-ingress.yaml - equal: - path: spec.values.ingress-nginx.controller.service.type - value: LoadBalancer - - template: templates/nginx-ingress.yaml - equal: - path: spec.values.ingress-nginx.controller.service.externalTrafficPolicy - value: Local - - template: templates/nginx-ingress.yaml - notExists: - path: spec.values.ingress-nginx.controller.service.labels - - template: templates/nginx-ingress.yaml - notExists: - path: spec.values.ingress-nginx.controller.service.externalIPs - - - it: loadBalancer mode without externalIPs fails chart render with explicit message - set: - _cluster: - expose-ingress: tenant-root - expose-external-ips: "" - expose-mode: loadBalancer - asserts: - - template: templates/nginx-ingress.yaml - failedTemplate: - errorMessage: "publishing.exposure=loadBalancer requires publishing.externalIPs to contain at least one non-empty address: the CiliumLoadBalancerIPPool has nothing to advertise and the Service would stay in ." - - - it: unknown exposure mode is rejected with a clear error (case-sensitive enum) - set: - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10" - expose-mode: LoadBalancer - asserts: - - template: templates/nginx-ingress.yaml - failedTemplate: - errorMessage: "unknown publishing.exposure mode \"LoadBalancer\": must be \"externalIPs\" or \"loadBalancer\"" - - - it: another typo in exposure mode also fails - set: - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10" - expose-mode: loadbalancer - asserts: - - template: templates/nginx-ingress.yaml - failedTemplate: - errorMessage: "unknown publishing.exposure mode \"loadbalancer\": must be \"externalIPs\" or \"loadBalancer\"" - - - it: loadBalancer mode in a namespace other than publishing.ingressName falls back to LoadBalancer Service without pool - set: - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10" - expose-mode: loadBalancer - release: - namespace: tenant-other - asserts: - - template: templates/nginx-ingress.yaml - equal: - path: spec.values.ingress-nginx.controller.service.type - value: LoadBalancer - - template: templates/nginx-ingress.yaml - equal: - path: spec.values.ingress-nginx.controller.service.externalTrafficPolicy - value: Local - - template: templates/nginx-ingress.yaml - notExists: - path: spec.values.ingress-nginx.controller.service.labels - - template: templates/nginx-ingress.yaml - notExists: - path: spec.values.ingress-nginx.controller.service.externalIPs - - - it: loadBalancer mode with only-empty externalIPs fails chart render (comma-only input) - set: - _cluster: - expose-ingress: tenant-root - expose-external-ips: ",," - expose-mode: loadBalancer - asserts: - - template: templates/nginx-ingress.yaml - failedTemplate: - errorMessage: "publishing.exposure=loadBalancer requires publishing.externalIPs to contain at least one non-empty address: the CiliumLoadBalancerIPPool has nothing to advertise and the Service would stay in ." - - - it: externalIPs mode also filters out empty entries (trailing comma) - set: - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10," - asserts: - - template: templates/nginx-ingress.yaml - equal: - path: spec.values.ingress-nginx.controller.service.externalIPs - value: - - 192.0.2.10 diff --git a/packages/system/cozy-proxy/charts/cozy-proxy/Chart.yaml b/packages/system/cozy-proxy/charts/cozy-proxy/Chart.yaml index 58f166f6..13352680 100644 --- a/packages/system/cozy-proxy/charts/cozy-proxy/Chart.yaml +++ b/packages/system/cozy-proxy/charts/cozy-proxy/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: cozy-proxy description: A simple kube-proxy addon for 1:1 NAT services in Kubernetes using an NFT backend type: application -version: 0.3.0 -appVersion: 0.3.0 +version: 0.2.0 +appVersion: 0.2.0 diff --git a/packages/system/cozy-proxy/charts/cozy-proxy/values.yaml b/packages/system/cozy-proxy/charts/cozy-proxy/values.yaml index e143e926..8cde5bed 100644 --- a/packages/system/cozy-proxy/charts/cozy-proxy/values.yaml +++ b/packages/system/cozy-proxy/charts/cozy-proxy/values.yaml @@ -1,6 +1,6 @@ image: repository: ghcr.io/cozystack/cozystack/cozy-proxy - tag: v0.3.0 + tag: v0.2.0 pullPolicy: IfNotPresent daemonset: diff --git a/packages/system/dashboard/.gitignore b/packages/system/dashboard/.gitignore new file mode 100644 index 00000000..3eedf1bd --- /dev/null +++ b/packages/system/dashboard/.gitignore @@ -0,0 +1 @@ +.tmp-cozystack-ui diff --git a/packages/system/dashboard/Makefile b/packages/system/dashboard/Makefile index a9f680a3..6023a366 100644 --- a/packages/system/dashboard/Makefile +++ b/packages/system/dashboard/Makefile @@ -4,51 +4,34 @@ export NAMESPACE=cozy-$(NAME) include ../../../hack/common-envs.mk include ../../../hack/package.mk -update: update-crd update-dockerfiles -image: image-openapi-ui image-openapi-ui-k8s-bff image-token-proxy update-tenant-text +.PHONY: image image-console image-token-proxy +image: image-console image-token-proxy -update-dockerfiles: - @echo Update dockerfiles manually +CONSOLE_REPO ?= https://github.com/cozystack/cozystack-ui.git +CONSOLE_REF ?= dd1ff0bdbd5e6e2fb6614fd1d831c4cb5c8f8025 -update-crd: - rm -rf crds - mkdir -p crds - wget -O- https://github.com/PRO-Robotech/helmfile-manifests/archive/refs/heads/main.tar.gz | tar -C crds -xzvf- helmfile-manifests-main/charts/incloud-main/incloud-web-1.0.0/incloud-web/templates --strip-components=6 - rm -f crds/_helpers.tpl - sed -i '/{{/d' crds/*.yml crds/*.yaml - -image-openapi-ui: - docker buildx build images/openapi-ui \ +image-console: + rm -rf .tmp-cozystack-ui + git clone --no-checkout $(CONSOLE_REPO) .tmp-cozystack-ui + git -C .tmp-cozystack-ui fetch --depth 1 origin $(CONSOLE_REF) + git -C .tmp-cozystack-ui checkout FETCH_HEAD + docker buildx build .tmp-cozystack-ui \ + --file .tmp-cozystack-ui/Containerfile \ --provenance false \ --builder=$(BUILDER) \ --platform=linux/amd64 \ - --tag $(REGISTRY)/openapi-ui:$(call settag,$(TAG)) \ - --cache-from type=registry,ref=$(REGISTRY)/openapi-ui:latest \ + --tag $(REGISTRY)/cozystack-ui:$(call settag,$(TAG)) \ + --cache-from type=registry,ref=$(REGISTRY)/cozystack-ui:latest \ --cache-to type=inline \ - --metadata-file images/openapi-ui.json \ + --metadata-file images/console.json \ + --build-arg APP_VERSION=$(COZYSTACK_VERSION) \ --push=$(PUSH) \ - --label "org.opencontainers.image.source=https://github.com/cozystack/cozystack" \ + --label "org.opencontainers.image.source=https://github.com/cozystack/cozystack-ui" \ --load=$(LOAD) - IMAGE="$(REGISTRY)/openapi-ui:$(call settag,$(TAG))@$$(yq e '."containerimage.digest"' images/openapi-ui.json -r)" \ - yq -i '.openapiUI.image = strenv(IMAGE)' values.yaml - rm -f images/openapi-ui.json - -image-openapi-ui-k8s-bff: - docker buildx build images/openapi-ui-k8s-bff \ - --provenance false \ - --builder=$(BUILDER) \ - --platform=linux/amd64 \ - --tag $(REGISTRY)/openapi-ui-k8s-bff:$(call settag,$(TAG)) \ - --cache-from type=registry,ref=$(REGISTRY)/openapi-ui-k8s-bff:latest \ - --cache-to type=inline \ - --metadata-file images/openapi-ui-k8s-bff.json \ - --push=$(PUSH) \ - --label "org.opencontainers.image.source=https://github.com/cozystack/cozystack" \ - --load=$(LOAD) - IMAGE="$(REGISTRY)/openapi-ui-k8s-bff:$(call settag,$(TAG))@$$(yq e '."containerimage.digest"' images/openapi-ui-k8s-bff.json -r)" \ - yq -i '.openapiUIK8sBff.image = strenv(IMAGE)' values.yaml - rm -f images/openapi-ui-k8s-bff.json + IMAGE="$(REGISTRY)/cozystack-ui:$(call settag,$(TAG))@$$(yq e '."containerimage.digest"' images/console.json -r)" \ + yq -i '.console.image = strenv(IMAGE)' values.yaml + rm -rf .tmp-cozystack-ui images/console.json image-token-proxy: docker buildx build images/token-proxy \ @@ -65,6 +48,3 @@ image-token-proxy: IMAGE="$(REGISTRY)/token-proxy:$(call settag,$(TAG))@$$(yq e '."containerimage.digest"' images/token-proxy.json -r)" \ yq -i '.tokenProxy.image = strenv(IMAGE)' values.yaml rm -f images/token-proxy.json - -update-tenant-text: - sed -i 's|\($$tenantText := "\)[^"]\+|\1$(TAG)|' templates/configmap.yaml diff --git a/packages/system/dashboard/images/openapi-ui-k8s-bff/Dockerfile b/packages/system/dashboard/images/openapi-ui-k8s-bff/Dockerfile deleted file mode 100644 index ea0c900d..00000000 --- a/packages/system/dashboard/images/openapi-ui-k8s-bff/Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -# imported from https://github.com/PRO-Robotech/openapi-ui-k8s-bff -ARG NODE_VERSION=20.18.1 -FROM node:${NODE_VERSION}-alpine AS builder -WORKDIR /src - -# release/1.4.0 -ARG COMMIT_REF=92e4b618eb9ad17b19827b5a2b7ceab33e8cf534 -RUN wget -O- https://github.com/PRO-Robotech/openapi-ui-k8s-bff/archive/${COMMIT_REF}.tar.gz | tar xzf - --strip-components=1 - -ENV PATH=/src/node_modules/.bin:$PATH -RUN npm install -RUN npm run build - -FROM node:${NODE_VERSION}-alpine -WORKDIR /app -COPY --from=builder /src/package*.json /app/ -COPY --from=builder /src/node_modules /app/node_modules -COPY --from=builder /src/src/swagger/swagger-output.json /app/dist/swagger/swagger-output.json -COPY --from=builder /src/dist /app/dist -EXPOSE 8080 -USER 1001 -CMD [ "node", "/app/dist/index.js"] diff --git a/packages/system/dashboard/images/openapi-ui/Dockerfile b/packages/system/dashboard/images/openapi-ui/Dockerfile deleted file mode 100644 index 82cc6db0..00000000 --- a/packages/system/dashboard/images/openapi-ui/Dockerfile +++ /dev/null @@ -1,69 +0,0 @@ -ARG NODE_VERSION=20.18.1 - -# openapi-k8s-toolkit -# imported from https://github.com/PRO-Robotech/openapi-k8s-toolkit -FROM node:${NODE_VERSION}-alpine AS openapi-k8s-toolkit-builder -RUN apk add git -WORKDIR /src -# release/1.4.0 -ARG COMMIT=d6b9e4ad0d1eb9d3730f7f0c664792c8dda3214d -RUN wget -O- https://github.com/PRO-Robotech/openapi-k8s-toolkit/archive/${COMMIT}.tar.gz | tar -xzvf- --strip-components=1 - -COPY openapi-k8s-toolkit/patches /patches -RUN git apply /patches/*.diff - -RUN npm install -RUN npm install --build-from-source @swc/core -RUN npm run build - - -# openapi-ui -# imported from https://github.com/PRO-Robotech/openapi-ui -FROM node:${NODE_VERSION}-alpine AS builder -#RUN apk add git -WORKDIR /src - -# release/1.4.0 -ARG COMMIT_REF=6addca6939264ef2e39801baa88c1460cc1aa53e -RUN wget -O- https://github.com/PRO-Robotech/openapi-ui/archive/${COMMIT_REF}.tar.gz | tar xzf - --strip-components=1 - -#COPY openapi-ui/patches /patches -#RUN git apply /patches/*.diff - -ENV PATH=/src/node_modules/.bin:$PATH - -RUN npm install - -# add patched openapi-k8s-toolkit -RUN rm -rf node_modules/@prorobotech/openapi-k8s-toolkit/dist -COPY --from=openapi-k8s-toolkit-builder /src/dist node_modules/@prorobotech/openapi-k8s-toolkit/dist - -RUN npm run build - -FROM node:${NODE_VERSION}-alpine AS builder2 -WORKDIR /src -ENV PATH=/src/node_modules/.bin:$PATH - -COPY --from=builder /src/server/package.json ./ -COPY --from=builder /src/server/package-lock.json ./ -RUN npm install -COPY --from=builder /src/server server -COPY --from=builder /src/tsconfig.server.json ./ -COPY --from=builder /src/build /src/build -RUN npm run server:build - -FROM node:${NODE_VERSION}-alpine -WORKDIR /app -COPY --from=builder2 /src/node_modules /app/node_modules -COPY --from=builder2 /src/build /app/build -EXPOSE 8080 -RUN sed -i -e 's|OpenAPI UI|Cozystack|g' build/index.html -# Fix Factory component: return null while loading instead of showing "Factory Not Found" 404 -RUN APP_JS=$(find build -name "App-react.js" -type f | head -1) && \ - if [ -n "$APP_JS" ]; then \ - sed -i 's|const { data: factoryData } = useK8sSmartResource({|const { data: factoryData, isLoading: factoryIsLoading } = useK8sSmartResource({|' "$APP_JS" && \ - sed -i '/Factory Not Found/s/return /return factoryIsLoading ? null : /' "$APP_JS" && \ - echo "Factory loading patch applied to $APP_JS"; \ - fi -USER 1001 -CMD ["node", "/app/build/index.js"] diff --git a/packages/system/dashboard/images/openapi-ui/openapi-k8s-toolkit/patches/flatmap-unresolved-placeholder.diff b/packages/system/dashboard/images/openapi-ui/openapi-k8s-toolkit/patches/flatmap-unresolved-placeholder.diff deleted file mode 100644 index e98861a3..00000000 --- a/packages/system/dashboard/images/openapi-ui/openapi-k8s-toolkit/patches/flatmap-unresolved-placeholder.diff +++ /dev/null @@ -1,17 +0,0 @@ -diff --git a/src/components/molecules/EnrichedTable/organisms/EnrichedTableProvider/utils.ts b/src/components/molecules/EnrichedTable/organisms/EnrichedTableProvider/utils.ts ---- a/src/components/molecules/EnrichedTable/organisms/EnrichedTableProvider/utils.ts -+++ b/src/components/molecules/EnrichedTable/organisms/EnrichedTableProvider/utils.ts -@@ -185,6 +185,11 @@ - return `['${escaped}']` - }) - } -- const jpQueryResult = jp.query(el, `$${resolvedJsonPath}`) -- fieldValue = Array.isArray(jpQueryResult) && jpQueryResult.length === 1 ? jpQueryResult[0] : jpQueryResult -+ if (/_flatMap[^\]]+_Key/.test(resolvedJsonPath)) { -+ // Placeholder was not resolved (row not yet expanded or key missing) — skip query -+ fieldValue = null -+ } else { -+ const jpQueryResult = jp.query(el, `$${resolvedJsonPath}`) -+ fieldValue = Array.isArray(jpQueryResult) && jpQueryResult.length === 1 ? jpQueryResult[0] : jpQueryResult -+ } - } diff --git a/packages/system/dashboard/images/openapi-ui/openapi-k8s-toolkit/patches/formlistinput-allow-empty.diff b/packages/system/dashboard/images/openapi-ui/openapi-k8s-toolkit/patches/formlistinput-allow-empty.diff deleted file mode 100644 index 1c83df7c..00000000 --- a/packages/system/dashboard/images/openapi-ui/openapi-k8s-toolkit/patches/formlistinput-allow-empty.diff +++ /dev/null @@ -1,37 +0,0 @@ -diff --git a/src/localTypes/formExtensions.ts b/src/localTypes/formExtensions.ts ---- a/src/localTypes/formExtensions.ts -+++ b/src/localTypes/formExtensions.ts -@@ -59,2 +59,4 @@ - relatedValuePath?: string -+ allowEmpty?: boolean -+ persistType?: 'str' | 'number' | 'arr' | 'obj' - } -diff --git a/src/components/molecules/BlackholeForm/molecules/FormListInput/FormListInput.tsx b/src/components/molecules/BlackholeForm/molecules/FormListInput/FormListInput.tsx ---- a/src/components/molecules/BlackholeForm/molecules/FormListInput/FormListInput.tsx -+++ b/src/components/molecules/BlackholeForm/molecules/FormListInput/FormListInput.tsx -@@ -149,3 +149,10 @@ - }, [relatedPath, form, arrName, fixedName, relatedFieldValue, onValuesChangeCallBack, isTouchedPeristed]) - -+ // When allowEmpty is set, auto-persist the field so the BFF preserves empty values -+ useEffect(() => { -+ if (customProps.allowEmpty) { -+ persistedControls.onPersistMark(persistName || name, customProps.persistType ?? 'str') -+ } -+ }, [customProps.allowEmpty, customProps.persistType, persistedControls, persistName, name]) -+ - const uri = prepareTemplate({ -@@ -267,5 +274,14 @@ - validateTrigger="onBlur" - hasFeedback={designNewLayout ? { icons: feedbackIcons } : true} - style={{ flex: 1 }} -+ normalize={(value: unknown) => { -+ if (customProps.allowEmpty && (value === undefined || value === null)) { -+ if (customProps.persistType === 'number') return 0 -+ if (customProps.persistType === 'arr') return [] -+ if (customProps.persistType === 'obj') return {} -+ return '' -+ } -+ return value -+ }} - > -