From 7797f4956918f1631c52714034ee1690b8fd3fd3 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Thu, 16 Apr 2026 20:20:41 +0300
Subject: [PATCH 001/151] test(api): assert parent HelmRelease Install/Upgrade
 Timeout >= 15m

Adds a failing unit test for convertApplicationToHelmRelease asserting
that Install.Timeout and Upgrade.Timeout are at least 15 minutes. The
default flux helm-controller timeout is too short to cover cold-start
Kamaji control-plane bootstrap (image pull + etcd bootstrap + apiserver
Ready + admin-kubeconfig Secret generation) and causes install
remediation loops.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 .../apps/application/rest_timeout_test.go     | 53 +++++++++++++++++++
 1 file changed, 53 insertions(+)
 create mode 100644 pkg/registry/apps/application/rest_timeout_test.go

diff --git a/pkg/registry/apps/application/rest_timeout_test.go b/pkg/registry/apps/application/rest_timeout_test.go
new file mode 100644
index 00000000..8ae13183
--- /dev/null
+++ b/pkg/registry/apps/application/rest_timeout_test.go
@@ -0,0 +1,53 @@
+package application
+
+import (
+	"testing"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1"
+	"github.com/cozystack/cozystack/pkg/config"
+)
+
+func TestConvertApplicationToHelmRelease_SetsInstallAndUpgradeTimeout(t *testing.T) {
+	r := &REST{
+		releaseConfig: config.ReleaseConfig{
+			Prefix: "kubernetes-",
+			ChartRef: config.ChartRefConfig{
+				Kind:      "HelmChart",
+				Name:      "kubernetes",
+				Namespace: "cozy-system",
+			},
+		},
+	}
+
+	app := &appsv1alpha1.Application{
+		ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "tenant-root"},
+	}
+
+	hr, err := r.convertApplicationToHelmRelease(app)
+	if err != nil {
+		t.Fatalf("convertApplicationToHelmRelease returned error: %v", err)
+	}
+
+	if hr.Spec.Install == nil {
+		t.Fatal("Spec.Install must not be nil")
+	}
+	if hr.Spec.Install.Timeout == nil {
+		t.Fatal("Spec.Install.Timeout must be set to cover async admin-kubeconfig provisioning")
+	}
+	if hr.Spec.Install.Timeout.Duration < 15*time.Minute {
+		t.Errorf("Spec.Install.Timeout must be >= 15m (cold bootstrap budget), got %v", hr.Spec.Install.Timeout.Duration)
+	}
+
+	if hr.Spec.Upgrade == nil {
+		t.Fatal("Spec.Upgrade must not be nil")
+	}
+	if hr.Spec.Upgrade.Timeout == nil {
+		t.Fatal("Spec.Upgrade.Timeout must be set to cover async admin-kubeconfig provisioning")
+	}
+	if hr.Spec.Upgrade.Timeout.Duration < 15*time.Minute {
+		t.Errorf("Spec.Upgrade.Timeout must be >= 15m (cold bootstrap budget), got %v", hr.Spec.Upgrade.Timeout.Duration)
+	}
+}

From e4f279f8e2cb08e271130b4a5193d9e0e9d9d3a7 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Thu, 16 Apr 2026 20:21:15 +0300
Subject: [PATCH 002/151] fix(api): set 15m Install/Upgrade Timeout for parent
 HelmRelease

Parent HelmRelease created by cozystack-api for Kubernetes tenants
contains CAPI/Kamaji resources (Cluster, KamajiControlPlane,
MachineDeployment) that asynchronously provision the
*-admin-kubeconfig Secret. Three Deployments in the same chart
(cluster-autoscaler, kccm, kcsi-controller) mount that Secret
directly, so the helm-wait cannot complete until control-plane
bootstrap finishes.

Default flux helm-controller timeout is too short for a cold-node
first-tenant bootstrap (image pull + etcd bootstrap + apiserver Ready
+ admin-kubeconfig generation routinely exceed it). On timeout,
install.remediation triggers uninstall, which removes the Cluster CR
and restarts the cycle indefinitely.

Bumping Install.Timeout and Upgrade.Timeout to 15m gives realistic
bootstrap headroom while remaining bounded.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 pkg/registry/apps/application/rest.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pkg/registry/apps/application/rest.go b/pkg/registry/apps/application/rest.go
index 0728ea13..77d1d458 100644
--- a/pkg/registry/apps/application/rest.go
+++ b/pkg/registry/apps/application/rest.go
@@ -1509,11 +1509,13 @@ func (r *REST) convertApplicationToHelmRelease(app *appsv1alpha1.Application) (*
 			},
 			Interval: metav1.Duration{Duration: 5 * time.Minute},
 			Install: &helmv2.Install{
+				Timeout: &metav1.Duration{Duration: 15 * time.Minute},
 				Remediation: &helmv2.InstallRemediation{
 					Retries: -1,
 				},
 			},
 			Upgrade: &helmv2.Upgrade{
+				Timeout: &metav1.Duration{Duration: 15 * time.Minute},
 				Remediation: &helmv2.UpgradeRemediation{
 					Retries: -1,
 				},

From 3e26234a1c98f138542471a7f4395532e15c7b46 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Thu, 16 Apr 2026 20:24:20 +0300
Subject: [PATCH 003/151] test(kubernetes): assert admin-kubeconfig wait
 pattern and etcd guard

Adds failing helm unittest suite for packages/apps/kubernetes covering:

- cluster-autoscaler, kccm, and csi controller Deployments mount the
  admin-kubeconfig Secret with optional: true
- each of those Deployments has a wait-for-kubeconfig initContainer
  that mounts the same kubeconfig path
- cluster.yaml renders a helm fail with a descriptive message when the
  tenant has no etcd DataStore (empty _namespace.etcd)

Also wires up a test target in the chart Makefile so helm-unit-tests.sh
picks it up.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 packages/apps/kubernetes/Makefile             |  3 +
 .../tests/admin_kubeconfig_wait_test.yaml     | 85 +++++++++++++++++++
 packages/apps/kubernetes/tests/values-ci.yaml |  9 ++
 3 files changed, 97 insertions(+)
 create mode 100644 packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml
 create mode 100644 packages/apps/kubernetes/tests/values-ci.yaml

diff --git a/packages/apps/kubernetes/Makefile b/packages/apps/kubernetes/Makefile
index 01cf736d..0f9e6d57 100644
--- a/packages/apps/kubernetes/Makefile
+++ b/packages/apps/kubernetes/Makefile
@@ -4,6 +4,9 @@ KUBERNETES_PKG_TAG = $(shell awk '$$1 == "version:" {print $$2}' Chart.yaml)
 include ../../../hack/common-envs.mk
 include ../../../hack/package.mk
 
+test:
+	helm unittest .
+
 generate:
 	cozyvalues-gen -m 'kubernetes' -v values.yaml -s values.schema.json -r README.md -g ../../../api/apps/v1alpha1/kubernetes/types.go
 	../../../hack/update-crd.sh
diff --git a/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml b/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml
new file mode 100644
index 00000000..e2d9f359
--- /dev/null
+++ b/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml
@@ -0,0 +1,85 @@
+suite: admin-kubeconfig wait guards
+
+release:
+  name: test
+  namespace: tenant-root
+
+values:
+  - values-ci.yaml
+
+tests:
+  - it: cluster-autoscaler mounts admin-kubeconfig as optional
+    template: templates/cluster-autoscaler/deployment.yaml
+    documentSelector:
+      path: kind
+      value: Deployment
+    asserts:
+      - equal:
+          path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.secretName
+          value: test-admin-kubeconfig
+      - equal:
+          path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.optional
+          value: true
+
+  - it: cluster-autoscaler waits for admin-kubeconfig via initContainer
+    template: templates/cluster-autoscaler/deployment.yaml
+    documentSelector:
+      path: kind
+      value: Deployment
+    asserts:
+      - equal:
+          path: spec.template.spec.initContainers[0].name
+          value: wait-for-kubeconfig
+      - contains:
+          path: spec.template.spec.initContainers[0].volumeMounts
+          content:
+            name: kubeconfig
+            mountPath: /etc/kubernetes/kubeconfig
+            readOnly: true
+
+  - it: kccm mounts admin-kubeconfig as optional
+    template: templates/kccm/manager.yaml
+    asserts:
+      - equal:
+          path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.secretName
+          value: test-admin-kubeconfig
+      - equal:
+          path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.optional
+          value: true
+
+  - it: kccm waits for admin-kubeconfig via initContainer
+    template: templates/kccm/manager.yaml
+    asserts:
+      - equal:
+          path: spec.template.spec.initContainers[0].name
+          value: wait-for-kubeconfig
+
+  - it: csi controller mounts admin-kubeconfig as optional
+    template: templates/csi/deploy.yaml
+    asserts:
+      - equal:
+          path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.secretName
+          value: test-admin-kubeconfig
+      - equal:
+          path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.optional
+          value: true
+
+  - it: csi controller waits for admin-kubeconfig via initContainer
+    template: templates/csi/deploy.yaml
+    asserts:
+      - equal:
+          path: spec.template.spec.initContainers[0].name
+          value: wait-for-kubeconfig
+
+  - it: cluster.yaml fails render when tenant has no etcd DataStore
+    template: templates/cluster.yaml
+    set:
+      _namespace:
+        etcd: ""
+        monitoring: ""
+        ingress: ""
+        seaweedfs: ""
+        host: ""
+    asserts:
+      - failedTemplate:
+          errorPattern: "requires a parent tenant with etcd enabled"
diff --git a/packages/apps/kubernetes/tests/values-ci.yaml b/packages/apps/kubernetes/tests/values-ci.yaml
new file mode 100644
index 00000000..13365e8c
--- /dev/null
+++ b/packages/apps/kubernetes/tests/values-ci.yaml
@@ -0,0 +1,9 @@
+_namespace:
+  etcd: tenant-root
+  monitoring: ""
+  ingress: ""
+  seaweedfs: ""
+  host: ""
+_cluster:
+  cluster-domain: cozy.local
+nodeGroups: null

From ca33cc4e3c4b032293fceec2b7d3c1eb13a241bb Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Thu, 16 Apr 2026 20:25:42 +0300
Subject: [PATCH 004/151] fix(kubernetes): wait for admin-kubeconfig before
 starting CP-side pods

Three Deployments in the Kubernetes app chart mount the tenant
admin-kubeconfig Secret directly as a volume: cluster-autoscaler,
kccm, and the kcsi controller. That Secret is provisioned
asynchronously by Kamaji after control-plane bootstrap, so on a fresh
install the pods used to hit FailedMount and the parent HelmRelease
ran out of its wait budget.

Mark the Secret volume optional and add a wait-for-kubeconfig
initContainer that polls the mounted path until the Secret appears.
Kubelet remounts the optional Secret within its sync period once
Kamaji publishes it, the init container exits, and the main container
starts cleanly. The Deployment becomes Available without the helm-wait
ever seeing a FailedMount.

Pins a busybox image for the init container via
images/busybox.tag (same format as the other pinned tags in this
chart).

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 packages/apps/kubernetes/images/busybox.tag     |  1 +
 .../cluster-autoscaler/deployment.yaml          | 17 +++++++++++++++++
 .../apps/kubernetes/templates/csi/deploy.yaml   | 17 +++++++++++++++++
 .../apps/kubernetes/templates/kccm/manager.yaml | 17 +++++++++++++++++
 4 files changed, 52 insertions(+)
 create mode 100644 packages/apps/kubernetes/images/busybox.tag

diff --git a/packages/apps/kubernetes/images/busybox.tag b/packages/apps/kubernetes/images/busybox.tag
new file mode 100644
index 00000000..e358c12e
--- /dev/null
+++ b/packages/apps/kubernetes/images/busybox.tag
@@ -0,0 +1 @@
+busybox:1.37.0@sha256:1487d0af5f52b4ba31c7e465126ee2123fe3f2305d638e7827681e7cf6c83d5e
diff --git a/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml b/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml
index a00e0155..47c22f76 100644
--- a/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml
+++ b/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml
@@ -23,6 +23,22 @@ spec:
         - key: node-role.kubernetes.io/control-plane
           operator: Exists
           effect: "NoSchedule"
+      initContainers:
+      - name: wait-for-kubeconfig
+        image: "{{ $.Files.Get "images/busybox.tag" | trim }}"
+        command:
+        - sh
+        - -c
+        - |
+          set -eu
+          until [ -s /etc/kubernetes/kubeconfig/super-admin.svc ]; do
+            echo "waiting for admin kubeconfig to be provisioned by Kamaji..."
+            sleep 5
+          done
+        volumeMounts:
+        - name: kubeconfig
+          mountPath: /etc/kubernetes/kubeconfig
+          readOnly: true
       containers:
       - image: "{{ $.Files.Get "images/cluster-autoscaler.tag" | trim }}"
         name: cluster-autoscaler
@@ -56,6 +72,7 @@ spec:
         name: cloud-config
       - secret:
           secretName: {{ .Release.Name }}-admin-kubeconfig
+          optional: true
         name: kubeconfig
       serviceAccountName: {{ .Release.Name }}-cluster-autoscaler
       terminationGracePeriodSeconds: 10
diff --git a/packages/apps/kubernetes/templates/csi/deploy.yaml b/packages/apps/kubernetes/templates/csi/deploy.yaml
index 938b6d67..2979c75c 100644
--- a/packages/apps/kubernetes/templates/csi/deploy.yaml
+++ b/packages/apps/kubernetes/templates/csi/deploy.yaml
@@ -24,6 +24,22 @@ spec:
         - key: node-role.kubernetes.io/control-plane
           operator: Exists
           effect: "NoSchedule"
+      initContainers:
+        - name: wait-for-kubeconfig
+          image: "{{ $.Files.Get "images/busybox.tag" | trim }}"
+          command:
+          - sh
+          - -c
+          - |
+            set -eu
+            until [ -s /etc/kubernetes/kubeconfig/super-admin.svc ]; do
+              echo "waiting for admin kubeconfig to be provisioned by Kamaji..."
+              sleep 5
+            done
+          volumeMounts:
+          - name: kubeconfig
+            mountPath: /etc/kubernetes/kubeconfig
+            readOnly: true
       containers:
         - name: csi-driver
           imagePullPolicy: Always
@@ -234,4 +250,5 @@ spec:
           emptyDir: {}
         - secret:
             secretName: {{ .Release.Name }}-admin-kubeconfig
+            optional: true
           name: kubeconfig
diff --git a/packages/apps/kubernetes/templates/kccm/manager.yaml b/packages/apps/kubernetes/templates/kccm/manager.yaml
index 81426d4e..5c7f15a4 100644
--- a/packages/apps/kubernetes/templates/kccm/manager.yaml
+++ b/packages/apps/kubernetes/templates/kccm/manager.yaml
@@ -22,6 +22,22 @@ spec:
         - key: node-role.kubernetes.io/control-plane
           operator: Exists
           effect: "NoSchedule"
+      initContainers:
+      - name: wait-for-kubeconfig
+        image: "{{ $.Files.Get "images/busybox.tag" | trim }}"
+        command:
+        - sh
+        - -c
+        - |
+          set -eu
+          until [ -s /etc/kubernetes/kubeconfig/super-admin.svc ]; do
+            echo "waiting for admin kubeconfig to be provisioned by Kamaji..."
+            sleep 5
+          done
+        volumeMounts:
+        - name: kubeconfig
+          mountPath: /etc/kubernetes/kubeconfig
+          readOnly: true
       containers:
       - name: kubevirt-cloud-controller-manager
         args:
@@ -55,5 +71,6 @@ spec:
         name: cloud-config
       - secret:
           secretName: {{ .Release.Name }}-admin-kubeconfig
+          optional: true
         name: kubeconfig
       serviceAccountName: {{ .Release.Name }}-kccm

From cac514b60fc00717df37780d2183e1702b07e9c6 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Thu, 16 Apr 2026 20:26:09 +0300
Subject: [PATCH 005/151] fix(kubernetes): fail fast when tenant has no etcd
 DataStore

When a Kubernetes tenant is created without a parent tenant that has
etcd enabled, .Values._namespace.etcd is empty and the rendered
KamajiControlPlane spec carries an empty dataStoreName. The Kamaji
admission webhook then rejects every TenantControlPlane create with
"tenant-root DataStore does not exist" and the control plane never
comes up.

Add a helm template-level guard that fails rendering with a
descriptive, actionable error message before the HelmRelease even
reaches the webhook. This also closes the narrow race where a
Kubernetes HelmRelease reconciles before the etcd HelmRelease has
created the DataStore CR - flux retries on its interval and picks up
the DataStore once it appears.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 packages/apps/kubernetes/templates/cluster.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/packages/apps/kubernetes/templates/cluster.yaml b/packages/apps/kubernetes/templates/cluster.yaml
index 10d6fd80..99249c1a 100644
--- a/packages/apps/kubernetes/templates/cluster.yaml
+++ b/packages/apps/kubernetes/templates/cluster.yaml
@@ -1,4 +1,7 @@
 {{- $etcd := .Values._namespace.etcd }}
+{{- if not $etcd }}
+{{- fail "Kubernetes tenant requires a parent tenant with etcd enabled: set .Values.etcd=true on tenant-root (or any ancestor tenant), wait for the etcd HelmRelease to reconcile (DataStore CR appears in the tenant namespace), then retry." }}
+{{- end }}
 {{- $ingress := .Values._namespace.ingress }}
 {{- $host := .Values._namespace.host }}
 {{- $kubevirtmachinetemplateNames := list }}

From a7d994365d067dee5d9546a9b95bb555b206a6af Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Thu, 16 Apr 2026 20:26:47 +0300
Subject: [PATCH 006/151] test(kubernetes): assert parent HelmRelease did not
 remediate in e2e

Before cleanup, inspect the parent HelmRelease installFailures and
upgradeFailures counters. A non-zero value means flux helm-controller
hit its wait timeout, ran install/upgrade remediation (uninstall),
and re-installed - the exact race condition this PR closes. Fail the
bats test in that case so the signal surfaces in CI instead of being
masked by a green retry.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 hack/e2e-apps/run-kubernetes.sh | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/hack/e2e-apps/run-kubernetes.sh b/hack/e2e-apps/run-kubernetes.sh
index be6dcd6f..f05ec370 100644
--- a/hack/e2e-apps/run-kubernetes.sh
+++ b/hack/e2e-apps/run-kubernetes.sh
@@ -320,6 +320,19 @@ EOF
     done
     kubectl wait hr kubernetes-${test_name}-ingress-nginx -n tenant-test --timeout=5m --for=condition=ready
 
+  # Guard: parent HelmRelease must not have entered an install/upgrade remediation cycle.
+  # A non-zero installFailures/upgradeFailures indicates the helm-wait budget expired while
+  # admin-kubeconfig was still being provisioned, which would trigger uninstall remediation
+  # and churn the Cluster CR.
+  install_failures=$(kubectl get hr -n tenant-test "kubernetes-${test_name}" -ojsonpath='{.status.installFailures}')
+  upgrade_failures=$(kubectl get hr -n tenant-test "kubernetes-${test_name}" -ojsonpath='{.status.upgradeFailures}')
+  if [ "${install_failures:-0}" != "0" ] && [ -n "${install_failures}" ] || \
+     [ "${upgrade_failures:-0}" != "0" ] && [ -n "${upgrade_failures}" ]; then
+    echo "Parent HelmRelease entered remediation cycle: installFailures=${install_failures:-0}, upgradeFailures=${upgrade_failures:-0}" >&2
+    kubectl -n tenant-test describe hr "kubernetes-${test_name}" >&2
+    exit 1
+  fi
+
   # Clean up
   pkill -f "port-forward.*${port}:" 2>/dev/null || true
   rm -f "tenantkubeconfig-${test_name}"

From f87834a3edba32945f22e8d419a9592e5303b34b Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Thu, 16 Apr 2026 20:48:47 +0300
Subject: [PATCH 007/151] fix(hack): group e2e remediation guard conditions
 correctly

Shell && and || have equal precedence and left-to-right associativity,
so the previous guard parsed as (((A && B) || C) && D) and silently
passed on the canonical failure mode: install_failures=1 with an empty
upgrade_failures.

Extract the check into helmrelease_has_remediation_cycle() in a
dedicated helper sourced from run-kubernetes.sh, and add unit tests
under hack/remediation-guard.bats that pin the expected behavior for
every combination of empty, zero, and positive counter values.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 hack/e2e-apps/remediation-guard.sh | 24 +++++++++
 hack/e2e-apps/run-kubernetes.sh    |  5 +-
 hack/remediation-guard.bats        | 87 ++++++++++++++++++++++++++++++
 3 files changed, 114 insertions(+), 2 deletions(-)
 create mode 100644 hack/e2e-apps/remediation-guard.sh
 create mode 100644 hack/remediation-guard.bats

diff --git a/hack/e2e-apps/remediation-guard.sh b/hack/e2e-apps/remediation-guard.sh
new file mode 100644
index 00000000..6da69f13
--- /dev/null
+++ b/hack/e2e-apps/remediation-guard.sh
@@ -0,0 +1,24 @@
+# Helpers for asserting that a Flux HelmRelease did not fall into an
+# install/upgrade remediation cycle during an e2e run.
+#
+# A non-zero installFailures/upgradeFailures counter means flux
+# helm-controller hit its wait timeout, ran remediation (uninstall),
+# and re-installed. That is exactly the race this guard is meant to
+# catch, so the function returns success (0) when a cycle is detected
+# and failure (1) otherwise.
+#
+# Both arguments may be empty strings, the literal "0", or a positive
+# integer. Shell's && and || have equal precedence with left-to-right
+# associativity, so each half of the disjunction is grouped explicitly
+# to avoid (A && B) || C && D parsing that masks the common
+# install_failures=1, upgrade_failures="" case.
+
+helmrelease_has_remediation_cycle() {
+    install_failures="$1"
+    upgrade_failures="$2"
+    if { [ -n "${install_failures}" ] && [ "${install_failures}" != "0" ]; } || \
+       { [ -n "${upgrade_failures}" ] && [ "${upgrade_failures}" != "0" ]; }; then
+        return 0
+    fi
+    return 1
+}
diff --git a/hack/e2e-apps/run-kubernetes.sh b/hack/e2e-apps/run-kubernetes.sh
index f05ec370..811f42a7 100644
--- a/hack/e2e-apps/run-kubernetes.sh
+++ b/hack/e2e-apps/run-kubernetes.sh
@@ -1,3 +1,5 @@
+. hack/e2e-apps/remediation-guard.sh
+
 run_kubernetes_test() {
     local version_expr="$1"
     local test_name="$2"
@@ -326,8 +328,7 @@ EOF
   # and churn the Cluster CR.
   install_failures=$(kubectl get hr -n tenant-test "kubernetes-${test_name}" -ojsonpath='{.status.installFailures}')
   upgrade_failures=$(kubectl get hr -n tenant-test "kubernetes-${test_name}" -ojsonpath='{.status.upgradeFailures}')
-  if [ "${install_failures:-0}" != "0" ] && [ -n "${install_failures}" ] || \
-     [ "${upgrade_failures:-0}" != "0" ] && [ -n "${upgrade_failures}" ]; then
+  if helmrelease_has_remediation_cycle "${install_failures}" "${upgrade_failures}"; then
     echo "Parent HelmRelease entered remediation cycle: installFailures=${install_failures:-0}, upgradeFailures=${upgrade_failures:-0}" >&2
     kubectl -n tenant-test describe hr "kubernetes-${test_name}" >&2
     exit 1
diff --git a/hack/remediation-guard.bats b/hack/remediation-guard.bats
new file mode 100644
index 00000000..64aa83af
--- /dev/null
+++ b/hack/remediation-guard.bats
@@ -0,0 +1,87 @@
+#!/usr/bin/env bats
+# -----------------------------------------------------------------------------
+# Unit tests for hack/e2e-apps/remediation-guard.sh
+#
+# helmrelease_has_remediation_cycle is consumed from e2e tests to assert that
+# the parent HelmRelease did not hit flux helm-controller's wait timeout and
+# enter uninstall remediation. The function accepts two arguments (values of
+# .status.installFailures and .status.upgradeFailures) and returns 0 when a
+# remediation cycle is detected, 1 otherwise.
+#
+# Each argument can be empty (controller never populated the field), "0"
+# (populated but never failed), or a positive integer. Shell's && and ||
+# have equal precedence with left-to-right associativity, which used to
+# break this check on the most common failure mode - install_failures=1
+# and upgrade_failures="". These tests pin the correct behavior.
+#
+# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on
+# its own line; there is no bats `run` or `$status`. Assertions are
+# expressed as direct shell tests that exit non-zero on failure.
+#
+# Run with: hack/cozytest.sh hack/remediation-guard.bats
+# -----------------------------------------------------------------------------
+
+@test "no counters set returns not-detected" {
+    . hack/e2e-apps/remediation-guard.sh
+    rc=0
+    helmrelease_has_remediation_cycle "" "" || rc=$?
+    [ "$rc" -eq 1 ]
+}
+
+@test "both counters zero returns not-detected" {
+    . hack/e2e-apps/remediation-guard.sh
+    rc=0
+    helmrelease_has_remediation_cycle "0" "0" || rc=$?
+    [ "$rc" -eq 1 ]
+}
+
+@test "install zero upgrade empty returns not-detected" {
+    . hack/e2e-apps/remediation-guard.sh
+    rc=0
+    helmrelease_has_remediation_cycle "0" "" || rc=$?
+    [ "$rc" -eq 1 ]
+}
+
+@test "install empty upgrade zero returns not-detected" {
+    . hack/e2e-apps/remediation-guard.sh
+    rc=0
+    helmrelease_has_remediation_cycle "" "0" || rc=$?
+    [ "$rc" -eq 1 ]
+}
+
+@test "install one upgrade empty returns detected" {
+    # Canonical race: first install exceeded helm-wait, remediation fired,
+    # no upgrade has happened yet.
+    . hack/e2e-apps/remediation-guard.sh
+    rc=0
+    helmrelease_has_remediation_cycle "1" "" || rc=$?
+    [ "$rc" -eq 0 ]
+}
+
+@test "install empty upgrade one returns detected" {
+    . hack/e2e-apps/remediation-guard.sh
+    rc=0
+    helmrelease_has_remediation_cycle "" "1" || rc=$?
+    [ "$rc" -eq 0 ]
+}
+
+@test "install two upgrade zero returns detected" {
+    . hack/e2e-apps/remediation-guard.sh
+    rc=0
+    helmrelease_has_remediation_cycle "2" "0" || rc=$?
+    [ "$rc" -eq 0 ]
+}
+
+@test "install zero upgrade two returns detected" {
+    . hack/e2e-apps/remediation-guard.sh
+    rc=0
+    helmrelease_has_remediation_cycle "0" "2" || rc=$?
+    [ "$rc" -eq 0 ]
+}
+
+@test "both counters positive returns detected" {
+    . hack/e2e-apps/remediation-guard.sh
+    rc=0
+    helmrelease_has_remediation_cycle "3" "5" || rc=$?
+    [ "$rc" -eq 0 ]
+}

From 73b80bfb940ec24459a826b32540f676a00bca36 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Thu, 16 Apr 2026 20:51:32 +0300
Subject: [PATCH 008/151] fix(api): scope 15m helm wait budget to Kubernetes
 Application kind

The previous change set Install.Timeout and Upgrade.Timeout to 15m
on every Application's parent HelmRelease, but the admin-kubeconfig
race documented in #2412 is specific to the Kubernetes kind: only its
parent chart creates CAPI/Kamaji resources whose admin-kubeconfig
Secret is asynchronously provisioned and mounted by Deployments in
the same chart. Other kinds (Qdrant, MongoDB, Postgres, ...) have no
such race and should not have their failed installs linger three times
longer before flux triggers remediation.

Gate the timeout on r.kindName == "Kubernetes". Rewrite
rest_timeout_test.go to cover both sides: Kubernetes must get a
>= 15m timeout, other kinds must keep the flux defaults. Both tests
also pin Install/Upgrade Remediation.Retries == -1 so a future edit
that removes unbounded remediation would show up as a red test.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 pkg/registry/apps/application/rest.go         | 16 ++++-
 .../apps/application/rest_timeout_test.go     | 58 ++++++++++++++++---
 2 files changed, 64 insertions(+), 10 deletions(-)

diff --git a/pkg/registry/apps/application/rest.go b/pkg/registry/apps/application/rest.go
index 77d1d458..9b92d2c0 100644
--- a/pkg/registry/apps/application/rest.go
+++ b/pkg/registry/apps/application/rest.go
@@ -1509,13 +1509,11 @@ func (r *REST) convertApplicationToHelmRelease(app *appsv1alpha1.Application) (*
 			},
 			Interval: metav1.Duration{Duration: 5 * time.Minute},
 			Install: &helmv2.Install{
-				Timeout: &metav1.Duration{Duration: 15 * time.Minute},
 				Remediation: &helmv2.InstallRemediation{
 					Retries: -1,
 				},
 			},
 			Upgrade: &helmv2.Upgrade{
-				Timeout: &metav1.Duration{Duration: 15 * time.Minute},
 				Remediation: &helmv2.UpgradeRemediation{
 					Retries: -1,
 				},
@@ -1530,6 +1528,20 @@ func (r *REST) convertApplicationToHelmRelease(app *appsv1alpha1.Application) (*
 		},
 	}
 
+	// The Kubernetes Application's parent chart creates CAPI/Kamaji resources
+	// whose admin-kubeconfig Secret is provisioned asynchronously and mounted
+	// by Deployments in the same chart. On a cold node, Kamaji control-plane
+	// bootstrap routinely exceeds flux helm-controller's default wait budget,
+	// so install remediation loops uninstall the Cluster CR and churn. Extend
+	// the wait budget to 15m for Kubernetes only - other Application kinds
+	// without this race keep flux defaults, so their failed installs do not
+	// linger unnecessarily before remediation fires.
+	if r.kindName == "Kubernetes" {
+		timeout := metav1.Duration{Duration: 15 * time.Minute}
+		helmRelease.Spec.Install.Timeout = &timeout
+		helmRelease.Spec.Upgrade.Timeout = &timeout
+	}
+
 	return helmRelease, nil
 }
 
diff --git a/pkg/registry/apps/application/rest_timeout_test.go b/pkg/registry/apps/application/rest_timeout_test.go
index 8ae13183..fa6d5427 100644
--- a/pkg/registry/apps/application/rest_timeout_test.go
+++ b/pkg/registry/apps/application/rest_timeout_test.go
@@ -10,18 +10,22 @@ import (
 	"github.com/cozystack/cozystack/pkg/config"
 )
 
-func TestConvertApplicationToHelmRelease_SetsInstallAndUpgradeTimeout(t *testing.T) {
-	r := &REST{
+func newRESTForKind(kind, prefix string) *REST {
+	return &REST{
+		kindName: kind,
 		releaseConfig: config.ReleaseConfig{
-			Prefix: "kubernetes-",
+			Prefix: prefix,
 			ChartRef: config.ChartRefConfig{
 				Kind:      "HelmChart",
-				Name:      "kubernetes",
+				Name:      "x",
 				Namespace: "cozy-system",
 			},
 		},
 	}
+}
 
+func TestConvertApplicationToHelmRelease_KubernetesKindGetsLongTimeout(t *testing.T) {
+	r := newRESTForKind("Kubernetes", "kubernetes-")
 	app := &appsv1alpha1.Application{
 		ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "tenant-root"},
 	}
@@ -35,19 +39,57 @@ func TestConvertApplicationToHelmRelease_SetsInstallAndUpgradeTimeout(t *testing
 		t.Fatal("Spec.Install must not be nil")
 	}
 	if hr.Spec.Install.Timeout == nil {
-		t.Fatal("Spec.Install.Timeout must be set to cover async admin-kubeconfig provisioning")
+		t.Fatal("Spec.Install.Timeout must be set for Kubernetes kind")
 	}
 	if hr.Spec.Install.Timeout.Duration < 15*time.Minute {
-		t.Errorf("Spec.Install.Timeout must be >= 15m (cold bootstrap budget), got %v", hr.Spec.Install.Timeout.Duration)
+		t.Errorf("Spec.Install.Timeout must be >= 15m for Kubernetes, got %v", hr.Spec.Install.Timeout.Duration)
 	}
 
 	if hr.Spec.Upgrade == nil {
 		t.Fatal("Spec.Upgrade must not be nil")
 	}
 	if hr.Spec.Upgrade.Timeout == nil {
-		t.Fatal("Spec.Upgrade.Timeout must be set to cover async admin-kubeconfig provisioning")
+		t.Fatal("Spec.Upgrade.Timeout must be set for Kubernetes kind")
 	}
 	if hr.Spec.Upgrade.Timeout.Duration < 15*time.Minute {
-		t.Errorf("Spec.Upgrade.Timeout must be >= 15m (cold bootstrap budget), got %v", hr.Spec.Upgrade.Timeout.Duration)
+		t.Errorf("Spec.Upgrade.Timeout must be >= 15m for Kubernetes, got %v", hr.Spec.Upgrade.Timeout.Duration)
+	}
+
+	if hr.Spec.Install.Remediation == nil || hr.Spec.Install.Remediation.Retries != -1 {
+		t.Errorf("Spec.Install.Remediation.Retries must remain -1, got %+v", hr.Spec.Install.Remediation)
+	}
+	if hr.Spec.Upgrade.Remediation == nil || hr.Spec.Upgrade.Remediation.Retries != -1 {
+		t.Errorf("Spec.Upgrade.Remediation.Retries must remain -1, got %+v", hr.Spec.Upgrade.Remediation)
+	}
+}
+
+func TestConvertApplicationToHelmRelease_NonKubernetesKindKeepsFluxDefaults(t *testing.T) {
+	// For Applications whose parent chart has no admin-kubeconfig race
+	// (Qdrant, MongoDB, Postgres, etc.), do NOT extend the helm-wait
+	// budget - otherwise failed installs would block three times as long
+	// before Flux starts remediating.
+	r := newRESTForKind("Qdrant", "qdrant-")
+	app := &appsv1alpha1.Application{
+		ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "tenant-root"},
+	}
+
+	hr, err := r.convertApplicationToHelmRelease(app)
+	if err != nil {
+		t.Fatalf("convertApplicationToHelmRelease returned error: %v", err)
+	}
+
+	if hr.Spec.Install != nil && hr.Spec.Install.Timeout != nil {
+		t.Errorf("Spec.Install.Timeout must be unset for non-Kubernetes kinds, got %v", hr.Spec.Install.Timeout.Duration)
+	}
+	if hr.Spec.Upgrade != nil && hr.Spec.Upgrade.Timeout != nil {
+		t.Errorf("Spec.Upgrade.Timeout must be unset for non-Kubernetes kinds, got %v", hr.Spec.Upgrade.Timeout.Duration)
+	}
+
+	// But remediation must still be -1 across the board.
+	if hr.Spec.Install.Remediation == nil || hr.Spec.Install.Remediation.Retries != -1 {
+		t.Errorf("Spec.Install.Remediation.Retries must remain -1, got %+v", hr.Spec.Install.Remediation)
+	}
+	if hr.Spec.Upgrade.Remediation == nil || hr.Spec.Upgrade.Remediation.Retries != -1 {
+		t.Errorf("Spec.Upgrade.Remediation.Retries must remain -1, got %+v", hr.Spec.Upgrade.Remediation)
 	}
 }

From 03606091df5672b6e857057a349875d534990f3a Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Thu, 16 Apr 2026 20:53:17 +0300
Subject: [PATCH 009/151] chore(kubernetes): align busybox image with project
 convention

Adds images/busybox/Dockerfile and an image-busybox Makefile target
that mirror the same pattern as the rest of this chart's images (the
Dockerfile pins the upstream busybox by digest; the Makefile target
builds and tags for ghcr.io/cozystack/cozystack/busybox the same way
cluster-autoscaler et al. are handled). Also wires it into the
umbrella image target so 'make image' rebuilds everything.

Until the first release build runs image-busybox and rewrites the
.tag to point at ghcr.io, the .tag keeps a fully-qualified
docker.io/library/busybox:1.37.0@sha256:... reference so pods do not
silently resolve the short name via the default registry and pulls
remain immutable by digest. The release workflow overwrites this
file the same way it does for the other images.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 packages/apps/kubernetes/Makefile                  | 14 +++++++++++++-
 packages/apps/kubernetes/images/busybox.tag        |  2 +-
 packages/apps/kubernetes/images/busybox/Dockerfile |  1 +
 3 files changed, 15 insertions(+), 2 deletions(-)
 create mode 100644 packages/apps/kubernetes/images/busybox/Dockerfile

diff --git a/packages/apps/kubernetes/Makefile b/packages/apps/kubernetes/Makefile
index 0f9e6d57..eb23c2ef 100644
--- a/packages/apps/kubernetes/Makefile
+++ b/packages/apps/kubernetes/Makefile
@@ -15,7 +15,7 @@ update:
 	hack/update-versions.sh
 	make generate
 
-image: image-ubuntu-container-disk image-kubevirt-cloud-provider image-kubevirt-csi-driver image-cluster-autoscaler
+image: image-ubuntu-container-disk image-kubevirt-cloud-provider image-kubevirt-csi-driver image-cluster-autoscaler image-busybox
 
 image-ubuntu-container-disk:
 	$(foreach ver,$(KUBERNETES_VERSIONS), \
@@ -70,3 +70,15 @@ image-cluster-autoscaler:
 	echo "$(REGISTRY)/cluster-autoscaler:$(call settag,$(KUBERNETES_PKG_TAG))@$$(yq e '."containerimage.digest"' images/cluster-autoscaler.json -o json -r)" \
 		> images/cluster-autoscaler.tag
 	rm -f images/cluster-autoscaler.json
+
+image-busybox:
+	docker buildx build images/busybox \
+		--tag $(REGISTRY)/busybox:$(call settag,$(KUBERNETES_PKG_TAG)) \
+		--tag $(REGISTRY)/busybox:$(call settag,$(KUBERNETES_PKG_TAG)-$(TAG)) \
+		--cache-from type=registry,ref=$(REGISTRY)/busybox:latest \
+		--cache-to type=inline \
+		--metadata-file images/busybox.json \
+		$(BUILDX_ARGS)
+	echo "$(REGISTRY)/busybox:$(call settag,$(KUBERNETES_PKG_TAG))@$$(yq e '."containerimage.digest"' images/busybox.json -o json -r)" \
+		> images/busybox.tag
+	rm -f images/busybox.json
diff --git a/packages/apps/kubernetes/images/busybox.tag b/packages/apps/kubernetes/images/busybox.tag
index e358c12e..39de220a 100644
--- a/packages/apps/kubernetes/images/busybox.tag
+++ b/packages/apps/kubernetes/images/busybox.tag
@@ -1 +1 @@
-busybox:1.37.0@sha256:1487d0af5f52b4ba31c7e465126ee2123fe3f2305d638e7827681e7cf6c83d5e
+docker.io/library/busybox:1.37.0@sha256:1487d0af5f52b4ba31c7e465126ee2123fe3f2305d638e7827681e7cf6c83d5e
diff --git a/packages/apps/kubernetes/images/busybox/Dockerfile b/packages/apps/kubernetes/images/busybox/Dockerfile
new file mode 100644
index 00000000..0c57fd39
--- /dev/null
+++ b/packages/apps/kubernetes/images/busybox/Dockerfile
@@ -0,0 +1 @@
+FROM docker.io/library/busybox:1.37.0@sha256:1487d0af5f52b4ba31c7e465126ee2123fe3f2305d638e7827681e7cf6c83d5e

From 97696b2b036be79ecafd8e96401dc2af3bc39d32 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Thu, 16 Apr 2026 20:54:10 +0300
Subject: [PATCH 010/151] test(kubernetes): add positive cluster render test
 and pin document kind

Adds an assertion that cluster.yaml renders successfully when
_namespace.etcd is set and produces a KamajiControlPlane whose
dataStoreName equals the tenant's etcd DataStore name. Without this
positive case a future edit that inverts or removes the existing etcd
guard would pass the suite as long as the negative case still fails.

Also adds documentSelector: kind=Deployment to the kccm and csi
controller assertions so the jsonpath filter operates on a single
document, matching the cluster-autoscaler case and removing
reliance on helm-unittest filter-vs-single-value coercion.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 .../tests/admin_kubeconfig_wait_test.yaml     | 34 +++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml b/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml
index e2d9f359..3b2b1d72 100644
--- a/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml
+++ b/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml
@@ -39,6 +39,9 @@ tests:
 
   - it: kccm mounts admin-kubeconfig as optional
     template: templates/kccm/manager.yaml
+    documentSelector:
+      path: kind
+      value: Deployment
     asserts:
       - equal:
           path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.secretName
@@ -49,13 +52,25 @@ tests:
 
   - it: kccm waits for admin-kubeconfig via initContainer
     template: templates/kccm/manager.yaml
+    documentSelector:
+      path: kind
+      value: Deployment
     asserts:
       - equal:
           path: spec.template.spec.initContainers[0].name
           value: wait-for-kubeconfig
+      - contains:
+          path: spec.template.spec.initContainers[0].volumeMounts
+          content:
+            name: kubeconfig
+            mountPath: /etc/kubernetes/kubeconfig
+            readOnly: true
 
   - it: csi controller mounts admin-kubeconfig as optional
     template: templates/csi/deploy.yaml
+    documentSelector:
+      path: kind
+      value: Deployment
     asserts:
       - equal:
           path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.secretName
@@ -66,10 +81,29 @@ tests:
 
   - it: csi controller waits for admin-kubeconfig via initContainer
     template: templates/csi/deploy.yaml
+    documentSelector:
+      path: kind
+      value: Deployment
     asserts:
       - equal:
           path: spec.template.spec.initContainers[0].name
           value: wait-for-kubeconfig
+      - contains:
+          path: spec.template.spec.initContainers[0].volumeMounts
+          content:
+            name: kubeconfig
+            mountPath: /etc/kubernetes/kubeconfig
+            readOnly: true
+
+  - it: cluster.yaml renders and wires dataStoreName when tenant has etcd
+    template: templates/cluster.yaml
+    documentSelector:
+      path: kind
+      value: KamajiControlPlane
+    asserts:
+      - equal:
+          path: spec.dataStoreName
+          value: tenant-root
 
   - it: cluster.yaml fails render when tenant has no etcd DataStore
     template: templates/cluster.yaml

From 6afc0eb370f827c55f478045d1aff2a329b25100 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Thu, 16 Apr 2026 20:55:17 +0300
Subject: [PATCH 011/151] fix(kubernetes): bound init wait and reword fail-fast
 message

Cap the wait-for-kubeconfig init container at 20m. If Kamaji genuinely
fails to produce the admin-kubeconfig Secret (misconfigured tenant,
etcd outage after the guard already passed, Kamaji crash-loop), the
pod now exits non-zero and goes CrashLoopBackOff so the failure is
visible in dashboards, instead of silently sleeping in Init forever
and leaving only the flux helm-wait timeout to surface the problem.

Reword the etcd DataStore guard to reference the parent Tenant
application's etcd flag (not .Values.etcd of the Kubernetes chart,
which is a different chart). Update the helm unittest errorPattern
accordingly.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 .../kubernetes/templates/cluster-autoscaler/deployment.yaml  | 5 +++++
 packages/apps/kubernetes/templates/cluster.yaml              | 2 +-
 packages/apps/kubernetes/templates/csi/deploy.yaml           | 5 +++++
 packages/apps/kubernetes/templates/kccm/manager.yaml         | 5 +++++
 .../apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml    | 2 +-
 5 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml b/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml
index 47c22f76..277b6df8 100644
--- a/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml
+++ b/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml
@@ -31,7 +31,12 @@ spec:
         - -c
         - |
           set -eu
+          deadline=$(( $(date +%s) + 1200 ))
           until [ -s /etc/kubernetes/kubeconfig/super-admin.svc ]; do
+            if [ "$(date +%s)" -ge "$deadline" ]; then
+              echo "admin kubeconfig was not provisioned within 20m; exiting so the pod goes CrashLoopBackOff and surfaces in dashboards" >&2
+              exit 1
+            fi
             echo "waiting for admin kubeconfig to be provisioned by Kamaji..."
             sleep 5
           done
diff --git a/packages/apps/kubernetes/templates/cluster.yaml b/packages/apps/kubernetes/templates/cluster.yaml
index 99249c1a..1747df4e 100644
--- a/packages/apps/kubernetes/templates/cluster.yaml
+++ b/packages/apps/kubernetes/templates/cluster.yaml
@@ -1,6 +1,6 @@
 {{- $etcd := .Values._namespace.etcd }}
 {{- if not $etcd }}
-{{- fail "Kubernetes tenant requires a parent tenant with etcd enabled: set .Values.etcd=true on tenant-root (or any ancestor tenant), wait for the etcd HelmRelease to reconcile (DataStore CR appears in the tenant namespace), then retry." }}
+{{- fail "Kubernetes tenant requires a parent Tenant application with etcd: true so the etcd module deploys a DataStore CR in the tenant namespace. Set spec.etcd: true on the root Tenant (or any ancestor Tenant), wait for the etcd HelmRelease to reconcile and the DataStore CR to appear, then retry." }}
 {{- end }}
 {{- $ingress := .Values._namespace.ingress }}
 {{- $host := .Values._namespace.host }}
diff --git a/packages/apps/kubernetes/templates/csi/deploy.yaml b/packages/apps/kubernetes/templates/csi/deploy.yaml
index 2979c75c..ff170cb8 100644
--- a/packages/apps/kubernetes/templates/csi/deploy.yaml
+++ b/packages/apps/kubernetes/templates/csi/deploy.yaml
@@ -32,7 +32,12 @@ spec:
           - -c
           - |
             set -eu
+            deadline=$(( $(date +%s) + 1200 ))
             until [ -s /etc/kubernetes/kubeconfig/super-admin.svc ]; do
+              if [ "$(date +%s)" -ge "$deadline" ]; then
+                echo "admin kubeconfig was not provisioned within 20m; exiting so the pod goes CrashLoopBackOff and surfaces in dashboards" >&2
+                exit 1
+              fi
               echo "waiting for admin kubeconfig to be provisioned by Kamaji..."
               sleep 5
             done
diff --git a/packages/apps/kubernetes/templates/kccm/manager.yaml b/packages/apps/kubernetes/templates/kccm/manager.yaml
index 5c7f15a4..67321647 100644
--- a/packages/apps/kubernetes/templates/kccm/manager.yaml
+++ b/packages/apps/kubernetes/templates/kccm/manager.yaml
@@ -30,7 +30,12 @@ spec:
         - -c
         - |
           set -eu
+          deadline=$(( $(date +%s) + 1200 ))
           until [ -s /etc/kubernetes/kubeconfig/super-admin.svc ]; do
+            if [ "$(date +%s)" -ge "$deadline" ]; then
+              echo "admin kubeconfig was not provisioned within 20m; exiting so the pod goes CrashLoopBackOff and surfaces in dashboards" >&2
+              exit 1
+            fi
             echo "waiting for admin kubeconfig to be provisioned by Kamaji..."
             sleep 5
           done
diff --git a/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml b/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml
index 3b2b1d72..2a43e13f 100644
--- a/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml
+++ b/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml
@@ -116,4 +116,4 @@ tests:
         host: ""
     asserts:
       - failedTemplate:
-          errorPattern: "requires a parent tenant with etcd enabled"
+          errorPattern: "requires a parent Tenant application with etcd: true"

From b38ae605495ff30a71013037af94839c4944e815 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Thu, 16 Apr 2026 21:05:23 +0300
Subject: [PATCH 012/151] fix(kubernetes): soft-skip cluster resources when
 tenant has no DataStore

A hard helm fail in cluster.yaml made every cold bootstrap racy: if
the parent Tenant chart had not yet populated _namespace.etcd in
cozystack-values when the Kubernetes HelmRelease first reconciled, the
fail fired, install.remediation triggered, installFailures incremented
and the new e2e remediation-guard flagged it as a bug. That directly
contradicts the race the rest of this PR is trying to close.

Replace fail with a graceful skip: render only a status-beacon
ConfigMap (test-awaiting-etcd) when etcd is empty, wrap all
CAPI/Kamaji resources in {{ if $etcd }}. The HelmRelease installs
successfully and goes Ready; flux retries on its 5m interval and picks
up the DataStore as soon as the Tenant chart finishes reconciling.

Update the helm unittest: positive test still asserts dataStoreName on
KamajiControlPlane; the negative test now asserts exactly one
ConfigMap document with status=awaiting-etcd, no Cluster / KCP /
KubevirtCluster / WorkloadMonitor rendered.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 .../apps/kubernetes/templates/cluster.yaml    | 37 +++++++++++++++++--
 .../tests/admin_kubeconfig_wait_test.yaml     | 19 ++++++++--
 2 files changed, 50 insertions(+), 6 deletions(-)

diff --git a/packages/apps/kubernetes/templates/cluster.yaml b/packages/apps/kubernetes/templates/cluster.yaml
index 1747df4e..12da9818 100644
--- a/packages/apps/kubernetes/templates/cluster.yaml
+++ b/packages/apps/kubernetes/templates/cluster.yaml
@@ -1,7 +1,15 @@
 {{- $etcd := .Values._namespace.etcd }}
-{{- if not $etcd }}
-{{- fail "Kubernetes tenant requires a parent Tenant application with etcd: true so the etcd module deploys a DataStore CR in the tenant namespace. Set spec.etcd: true on the root Tenant (or any ancestor Tenant), wait for the etcd HelmRelease to reconcile and the DataStore CR to appear, then retry." }}
-{{- end }}
+{{- /*
+  When $etcd is empty, the parent Tenant application has not populated
+  _namespace.etcd in cozystack-values yet - either the operator forgot to
+  set etcd: true on an ancestor Tenant, or the Tenant HelmRelease is still
+  reconciling. Either way, rendering a KamajiControlPlane with an empty
+  dataStoreName would be rejected by Kamaji's admission webhook and the
+  HelmRelease would fail to install, triggering remediation. Instead, emit
+  a single ConfigMap as a user-visible status beacon and skip the rest so
+  flux marks the HelmRelease Ready and retries its 5m reconcile loop until
+  the Tenant chart catches up.
+*/}}
 {{- $ingress := .Values._namespace.ingress }}
 {{- $host := .Values._namespace.host }}
 {{- $kubevirtmachinetemplateNames := list }}
@@ -87,6 +95,28 @@ spec:
           - name: default
             pod: {}
 {{- end }}
+{{- if not $etcd }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Release.Name }}-awaiting-etcd
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/name: kubernetes
+    app.kubernetes.io/instance: {{ .Release.Name }}
+  annotations:
+    cozystack.io/status-beacon: "true"
+data:
+  status: "awaiting-etcd"
+  message: |
+    No DataStore is available for this tenant Kubernetes cluster. The parent
+    Tenant application has not populated _namespace.etcd. Set spec.etcd: true
+    on an ancestor Tenant (usually tenant-root) and wait for its HelmRelease
+    to reconcile - this HelmRelease will pick up the DataStore on its next
+    5m reconcile loop and provision the cluster.
+{{- else }}
 ---
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: Cluster
@@ -407,3 +437,4 @@ metadata:
 spec:
   {{- .spec | toYaml | nindent 2 }}
 {{- end }}
+{{- end }}
diff --git a/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml b/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml
index 2a43e13f..e7c03bd1 100644
--- a/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml
+++ b/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml
@@ -105,7 +105,12 @@ tests:
           path: spec.dataStoreName
           value: tenant-root
 
-  - it: cluster.yaml fails render when tenant has no etcd DataStore
+  - it: cluster.yaml skips Cluster resources when tenant has no etcd DataStore
+    # Must NOT fail rendering - the parent Tenant chart populates
+    # _namespace.etcd asynchronously, so rendering failures here would cause
+    # flux install remediation on every cold bootstrap. Instead, emit only a
+    # ConfigMap status beacon so the HelmRelease reports Ready while flux
+    # retries on its interval until the DataStore appears.
     template: templates/cluster.yaml
     set:
       _namespace:
@@ -115,5 +120,13 @@ tests:
         seaweedfs: ""
         host: ""
     asserts:
-      - failedTemplate:
-          errorPattern: "requires a parent Tenant application with etcd: true"
+      - hasDocuments:
+          count: 1
+      - isKind:
+          of: ConfigMap
+      - equal:
+          path: metadata.name
+          value: test-awaiting-etcd
+      - equal:
+          path: data.status
+          value: awaiting-etcd

From 1757567218ac313a030da1f82e64541a3f9bfbae Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Thu, 16 Apr 2026 21:06:47 +0300
Subject: [PATCH 013/151] refactor(kubernetes): extract wait-for-kubeconfig
 init into shared helper

The three control-plane-side Deployments (cluster-autoscaler, kccm,
kcsi-controller) carried three copies of the same 20-line init
container. That already drifted: the CSI copy used 4-space nesting
while the other two used 2-space. Any future update to the image,
the deadline, or the poll script had to land in three places or
silently diverge.

Extract the block into a new kubernetes.waitForAdminKubeconfig helper
in templates/_helpers.tpl and include it at each call site. Tighten
the deadline from 20m to 10m so it stays strictly below the 15m
HelmRelease Install.Timeout and the CrashLoopBackOff surfaces in
dashboards before flux remediation can fire. Also clarify the wait
message so operators debugging a stuck init container do not chase
Kamaji for what is actually kubelet's optional-Secret refresh cadence.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 .../apps/kubernetes/templates/_helpers.tpl    | 40 +++++++++++++++++++
 .../cluster-autoscaler/deployment.yaml        | 21 +---------
 .../apps/kubernetes/templates/csi/deploy.yaml | 21 +---------
 .../kubernetes/templates/kccm/manager.yaml    | 21 +---------
 4 files changed, 43 insertions(+), 60 deletions(-)

diff --git a/packages/apps/kubernetes/templates/_helpers.tpl b/packages/apps/kubernetes/templates/_helpers.tpl
index 36c06b64..89f06934 100644
--- a/packages/apps/kubernetes/templates/_helpers.tpl
+++ b/packages/apps/kubernetes/templates/_helpers.tpl
@@ -49,3 +49,43 @@ Selector labels
 app.kubernetes.io/name: {{ include "kubernetes.name" . }}
 app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
+
+{{/*
+wait-for-kubeconfig init container shared by the control-plane-side
+Deployments (cluster-autoscaler, kccm, kcsi-controller) that mount the
+*-admin-kubeconfig Secret provisioned asynchronously by Kamaji. The
+Secret volume is declared optional so kubelet does not FailedMount while
+Kamaji is still bootstrapping; this container polls the mounted path and
+exits only when super-admin.svc appears, which happens after kubelet's
+optional-Secret refresh cycle.
+
+The 10m deadline stays strictly below the 15m HelmRelease
+Install.Timeout scoped to the Kubernetes Application kind so the
+CrashLoopBackOff surfaces before flux remediation fires and uninstalls
+the Cluster CR.
+
+Call site owns the surrounding volumes block; the kubeconfig volume
+must exist on the pod and mount at /etc/kubernetes/kubeconfig.
+*/}}
+{{- define "kubernetes.waitForAdminKubeconfig" -}}
+- name: wait-for-kubeconfig
+  image: "{{ .Files.Get "images/busybox.tag" | trim }}"
+  command:
+  - sh
+  - -c
+  - |
+    set -eu
+    deadline=$(( $(date +%s) + 600 ))
+    until [ -s /etc/kubernetes/kubeconfig/super-admin.svc ]; do
+      if [ "$(date +%s)" -ge "$deadline" ]; then
+        echo "admin kubeconfig was not provisioned within 10m; exiting so the pod goes CrashLoopBackOff and surfaces in dashboards" >&2
+        exit 1
+      fi
+      echo "waiting for admin kubeconfig (provisioned by Kamaji, visible after kubelet Secret refresh)..."
+      sleep 5
+    done
+  volumeMounts:
+  - name: kubeconfig
+    mountPath: /etc/kubernetes/kubeconfig
+    readOnly: true
+{{- end }}
diff --git a/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml b/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml
index 277b6df8..348b017d 100644
--- a/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml
+++ b/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml
@@ -24,26 +24,7 @@ spec:
           operator: Exists
           effect: "NoSchedule"
       initContainers:
-      - name: wait-for-kubeconfig
-        image: "{{ $.Files.Get "images/busybox.tag" | trim }}"
-        command:
-        - sh
-        - -c
-        - |
-          set -eu
-          deadline=$(( $(date +%s) + 1200 ))
-          until [ -s /etc/kubernetes/kubeconfig/super-admin.svc ]; do
-            if [ "$(date +%s)" -ge "$deadline" ]; then
-              echo "admin kubeconfig was not provisioned within 20m; exiting so the pod goes CrashLoopBackOff and surfaces in dashboards" >&2
-              exit 1
-            fi
-            echo "waiting for admin kubeconfig to be provisioned by Kamaji..."
-            sleep 5
-          done
-        volumeMounts:
-        - name: kubeconfig
-          mountPath: /etc/kubernetes/kubeconfig
-          readOnly: true
+      {{- include "kubernetes.waitForAdminKubeconfig" $ | nindent 6 }}
       containers:
       - image: "{{ $.Files.Get "images/cluster-autoscaler.tag" | trim }}"
         name: cluster-autoscaler
diff --git a/packages/apps/kubernetes/templates/csi/deploy.yaml b/packages/apps/kubernetes/templates/csi/deploy.yaml
index ff170cb8..c1af7f13 100644
--- a/packages/apps/kubernetes/templates/csi/deploy.yaml
+++ b/packages/apps/kubernetes/templates/csi/deploy.yaml
@@ -25,26 +25,7 @@ spec:
           operator: Exists
           effect: "NoSchedule"
       initContainers:
-        - name: wait-for-kubeconfig
-          image: "{{ $.Files.Get "images/busybox.tag" | trim }}"
-          command:
-          - sh
-          - -c
-          - |
-            set -eu
-            deadline=$(( $(date +%s) + 1200 ))
-            until [ -s /etc/kubernetes/kubeconfig/super-admin.svc ]; do
-              if [ "$(date +%s)" -ge "$deadline" ]; then
-                echo "admin kubeconfig was not provisioned within 20m; exiting so the pod goes CrashLoopBackOff and surfaces in dashboards" >&2
-                exit 1
-              fi
-              echo "waiting for admin kubeconfig to be provisioned by Kamaji..."
-              sleep 5
-            done
-          volumeMounts:
-          - name: kubeconfig
-            mountPath: /etc/kubernetes/kubeconfig
-            readOnly: true
+      {{- include "kubernetes.waitForAdminKubeconfig" $ | nindent 6 }}
       containers:
         - name: csi-driver
           imagePullPolicy: Always
diff --git a/packages/apps/kubernetes/templates/kccm/manager.yaml b/packages/apps/kubernetes/templates/kccm/manager.yaml
index 67321647..20ac0a1e 100644
--- a/packages/apps/kubernetes/templates/kccm/manager.yaml
+++ b/packages/apps/kubernetes/templates/kccm/manager.yaml
@@ -23,26 +23,7 @@ spec:
           operator: Exists
           effect: "NoSchedule"
       initContainers:
-      - name: wait-for-kubeconfig
-        image: "{{ $.Files.Get "images/busybox.tag" | trim }}"
-        command:
-        - sh
-        - -c
-        - |
-          set -eu
-          deadline=$(( $(date +%s) + 1200 ))
-          until [ -s /etc/kubernetes/kubeconfig/super-admin.svc ]; do
-            if [ "$(date +%s)" -ge "$deadline" ]; then
-              echo "admin kubeconfig was not provisioned within 20m; exiting so the pod goes CrashLoopBackOff and surfaces in dashboards" >&2
-              exit 1
-            fi
-            echo "waiting for admin kubeconfig to be provisioned by Kamaji..."
-            sleep 5
-          done
-        volumeMounts:
-        - name: kubeconfig
-          mountPath: /etc/kubernetes/kubeconfig
-          readOnly: true
+      {{- include "kubernetes.waitForAdminKubeconfig" $ | nindent 6 }}
       containers:
       - name: kubevirt-cloud-controller-manager
         args:

From 7b146cbe56c26511d2228c17cdb7c2201f480395 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Thu, 16 Apr 2026 21:09:35 +0300
Subject: [PATCH 014/151] feat(api): make HelmRelease Install/Upgrade timeout
 per-Application

Replace the hardcoded r.kindName == "Kubernetes" switch in rest.go
with a config-driven path. The ApplicationDefinition CR now accepts a
release.cozystack.io/helm-install-timeout annotation that is parsed
at cozystack-api startup into config.ReleaseConfig.HelmInstallTimeout
and applied to both Install.Timeout and Upgrade.Timeout on the
rendered HelmRelease. Applications that leave the annotation unset
keep flux defaults so their failed installs remediate on the normal
cadence - only the Kubernetes kind carries the override and gets a
15m budget. New kinds with a similar race can opt in by setting the
same annotation; no rest.go patch needed.

Kubernetes-rd sets the annotation to 15m. Table-driven test in
rest_timeout_test.go covers three cases: Kubernetes with 15m, Qdrant
unset, and an arbitrary future kind with 20m - all of which pin the
Remediation.Retries == -1 contract as well.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 .../kubernetes-rd/cozyrds/kubernetes.yaml     |   8 +
 pkg/cmd/server/start.go                       |  35 +++--
 pkg/config/config.go                          |   8 +
 pkg/registry/apps/application/rest.go         |  21 +--
 .../apps/application/rest_timeout_test.go     | 143 ++++++++++--------
 5 files changed, 133 insertions(+), 82 deletions(-)

diff --git a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml
index 5e9e8f94..1128abe8 100644
--- a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml
+++ b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml
@@ -2,6 +2,14 @@ apiVersion: cozystack.io/v1alpha1
 kind: ApplicationDefinition
 metadata:
   name: kubernetes
+  annotations:
+    # Kubernetes tenants boot a Kamaji control plane whose admin-kubeconfig
+    # Secret is provisioned asynchronously. Cold Kamaji start (image pull +
+    # etcd + apiserver Ready) plus admin-kubeconfig generation can exceed
+    # Flux helm-controller's default wait budget, causing remediation loops
+    # that uninstall the Cluster CR. This override applied by cozystack-api
+    # to the HelmRelease Spec.Install.Timeout and Spec.Upgrade.Timeout.
+    release.cozystack.io/helm-install-timeout: "15m"
 spec:
   application:
     kind: Kubernetes
diff --git a/pkg/cmd/server/start.go b/pkg/cmd/server/start.go
index f7791332..3e905735 100644
--- a/pkg/cmd/server/start.go
+++ b/pkg/cmd/server/start.go
@@ -160,6 +160,31 @@ func (o *CozyServerOptions) Complete() error {
 	// Convert to ResourceConfig
 	o.ResourceConfig = &config.ResourceConfig{}
 	for _, crd := range crdList.Items {
+		release := config.ReleaseConfig{
+			Prefix: crd.Spec.Release.Prefix,
+			Labels: crd.Spec.Release.Labels,
+			ChartRef: config.ChartRefConfig{
+				Kind:      crd.Spec.Release.ChartRef.Kind,
+				Name:      crd.Spec.Release.ChartRef.Name,
+				Namespace: crd.Spec.Release.ChartRef.Namespace,
+			},
+		}
+		// Per-Application HelmRelease Install/Upgrade timeout. Applications
+		// whose parent chart contains asynchronously-provisioned resources
+		// the chart itself depends on (for example, the Kamaji-provisioned
+		// admin-kubeconfig Secret for Kubernetes tenants) need a longer
+		// wait budget than the Flux default. Consumed by the REST storage
+		// layer when building the HelmRelease Spec.
+		if raw, ok := crd.Annotations["release.cozystack.io/helm-install-timeout"]; ok && raw != "" {
+			d, err := time.ParseDuration(raw)
+			if err != nil {
+				return fmt.Errorf(
+					"ApplicationDefinition %q has invalid release.cozystack.io/helm-install-timeout %q: %w",
+					crd.Name, raw, err,
+				)
+			}
+			release.HelmInstallTimeout = d
+		}
 		resource := config.Resource{
 			Application: config.ApplicationConfig{
 				Kind:          crd.Spec.Application.Kind,
@@ -168,15 +193,7 @@ func (o *CozyServerOptions) Complete() error {
 				ShortNames:    []string{}, // TODO: implement shortnames
 				OpenAPISchema: crd.Spec.Application.OpenAPISchema,
 			},
-			Release: config.ReleaseConfig{
-				Prefix: crd.Spec.Release.Prefix,
-				Labels: crd.Spec.Release.Labels,
-				ChartRef: config.ChartRefConfig{
-					Kind:      crd.Spec.Release.ChartRef.Kind,
-					Name:      crd.Spec.Release.ChartRef.Name,
-					Namespace: crd.Spec.Release.ChartRef.Namespace,
-				},
-			},
+			Release: release,
 		}
 		o.ResourceConfig.Resources = append(o.ResourceConfig.Resources, resource)
 	}
diff --git a/pkg/config/config.go b/pkg/config/config.go
index 1e123e2c..16cf4f0c 100644
--- a/pkg/config/config.go
+++ b/pkg/config/config.go
@@ -16,6 +16,8 @@ limitations under the License.
 
 package config
 
+import "time"
+
 // ResourceConfig represents the structure of the configuration file.
 type ResourceConfig struct {
 	Resources []Resource `yaml:"resources"`
@@ -41,6 +43,12 @@ type ReleaseConfig struct {
 	Prefix   string            `yaml:"prefix"`
 	Labels   map[string]string `yaml:"labels"`
 	ChartRef ChartRefConfig    `yaml:"chartRef"`
+	// HelmInstallTimeout overrides the Flux HelmRelease Install.Timeout and
+	// Upgrade.Timeout for this Application kind. When zero, flux defaults
+	// apply. Populated from the
+	// release.cozystack.io/helm-install-timeout annotation on the
+	// ApplicationDefinition at start-up.
+	HelmInstallTimeout time.Duration `yaml:"helmInstallTimeout,omitempty"`
 }
 
 // ChartRefConfig references a Flux source artifact for the Helm chart.
diff --git a/pkg/registry/apps/application/rest.go b/pkg/registry/apps/application/rest.go
index 9b92d2c0..406d3738 100644
--- a/pkg/registry/apps/application/rest.go
+++ b/pkg/registry/apps/application/rest.go
@@ -1528,16 +1528,17 @@ func (r *REST) convertApplicationToHelmRelease(app *appsv1alpha1.Application) (*
 		},
 	}
 
-	// The Kubernetes Application's parent chart creates CAPI/Kamaji resources
-	// whose admin-kubeconfig Secret is provisioned asynchronously and mounted
-	// by Deployments in the same chart. On a cold node, Kamaji control-plane
-	// bootstrap routinely exceeds flux helm-controller's default wait budget,
-	// so install remediation loops uninstall the Cluster CR and churn. Extend
-	// the wait budget to 15m for Kubernetes only - other Application kinds
-	// without this race keep flux defaults, so their failed installs do not
-	// linger unnecessarily before remediation fires.
-	if r.kindName == "Kubernetes" {
-		timeout := metav1.Duration{Duration: 15 * time.Minute}
+	// Per-Application HelmRelease wait budget. When an ApplicationDefinition
+	// sets release.cozystack.io/helm-install-timeout, the annotation is
+	// parsed at startup into ReleaseConfig.HelmInstallTimeout and applied
+	// to both Install and Upgrade here. Applications that leave it unset
+	// (the common case) keep flux defaults, so their failed installs
+	// remediate on the normal cadence. Needed for the Kubernetes kind
+	// because its parent chart contains CAPI/Kamaji resources whose
+	// admin-kubeconfig Secret is provisioned asynchronously and Kamaji
+	// cold-start routinely exceeds flux's default wait budget.
+	if r.releaseConfig.HelmInstallTimeout > 0 {
+		timeout := metav1.Duration{Duration: r.releaseConfig.HelmInstallTimeout}
 		helmRelease.Spec.Install.Timeout = &timeout
 		helmRelease.Spec.Upgrade.Timeout = &timeout
 	}
diff --git a/pkg/registry/apps/application/rest_timeout_test.go b/pkg/registry/apps/application/rest_timeout_test.go
index fa6d5427..8474d1cb 100644
--- a/pkg/registry/apps/application/rest_timeout_test.go
+++ b/pkg/registry/apps/application/rest_timeout_test.go
@@ -10,7 +10,7 @@ import (
 	"github.com/cozystack/cozystack/pkg/config"
 )
 
-func newRESTForKind(kind, prefix string) *REST {
+func newRESTForTimeout(kind, prefix string, helmInstallTimeout time.Duration) *REST {
 	return &REST{
 		kindName: kind,
 		releaseConfig: config.ReleaseConfig{
@@ -20,76 +20,93 @@ func newRESTForKind(kind, prefix string) *REST {
 				Name:      "x",
 				Namespace: "cozy-system",
 			},
+			HelmInstallTimeout: helmInstallTimeout,
 		},
 	}
 }
 
-func TestConvertApplicationToHelmRelease_KubernetesKindGetsLongTimeout(t *testing.T) {
-	r := newRESTForKind("Kubernetes", "kubernetes-")
-	app := &appsv1alpha1.Application{
-		ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "tenant-root"},
+// Table-driven: every Application kind carries a per-CRD HelmRelease wait
+// budget. The Kubernetes kind's parent chart contains CAPI/Kamaji resources
+// whose admin-kubeconfig Secret is provisioned asynchronously, so its
+// ApplicationDefinition sets release.cozystack.io/helm-install-timeout=15m
+// (or longer). Other kinds leave the annotation unset and keep flux defaults
+// so their failed installs remediate on the normal cadence. The test must
+// cover both paths: a kind with the timeout set and one without.
+func TestConvertApplicationToHelmRelease_AppliesReleaseConfigTimeout(t *testing.T) {
+	cases := []struct {
+		name       string
+		kind       string
+		prefix     string
+		configured time.Duration
+		wantSet    bool
+	}{
+		{
+			name:       "Kubernetes kind with 15m configured gets Install and Upgrade Timeout",
+			kind:       "Kubernetes",
+			prefix:     "kubernetes-",
+			configured: 15 * time.Minute,
+			wantSet:    true,
+		},
+		{
+			name:       "Qdrant kind without configured timeout keeps flux defaults",
+			kind:       "Qdrant",
+			prefix:     "qdrant-",
+			configured: 0,
+			wantSet:    false,
+		},
+		{
+			name:       "arbitrary future kind with 20m configured gets 20m",
+			kind:       "TalosCluster",
+			prefix:     "talos-",
+			configured: 20 * time.Minute,
+			wantSet:    true,
+		},
 	}
 
-	hr, err := r.convertApplicationToHelmRelease(app)
-	if err != nil {
-		t.Fatalf("convertApplicationToHelmRelease returned error: %v", err)
-	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			r := newRESTForTimeout(tc.kind, tc.prefix, tc.configured)
+			app := &appsv1alpha1.Application{
+				ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "tenant-root"},
+			}
 
-	if hr.Spec.Install == nil {
-		t.Fatal("Spec.Install must not be nil")
-	}
-	if hr.Spec.Install.Timeout == nil {
-		t.Fatal("Spec.Install.Timeout must be set for Kubernetes kind")
-	}
-	if hr.Spec.Install.Timeout.Duration < 15*time.Minute {
-		t.Errorf("Spec.Install.Timeout must be >= 15m for Kubernetes, got %v", hr.Spec.Install.Timeout.Duration)
-	}
+			hr, err := r.convertApplicationToHelmRelease(app)
+			if err != nil {
+				t.Fatalf("convertApplicationToHelmRelease returned error: %v", err)
+			}
 
-	if hr.Spec.Upgrade == nil {
-		t.Fatal("Spec.Upgrade must not be nil")
-	}
-	if hr.Spec.Upgrade.Timeout == nil {
-		t.Fatal("Spec.Upgrade.Timeout must be set for Kubernetes kind")
-	}
-	if hr.Spec.Upgrade.Timeout.Duration < 15*time.Minute {
-		t.Errorf("Spec.Upgrade.Timeout must be >= 15m for Kubernetes, got %v", hr.Spec.Upgrade.Timeout.Duration)
-	}
+			if hr.Spec.Install == nil || hr.Spec.Upgrade == nil {
+				t.Fatalf("Spec.Install/Upgrade must be non-nil")
+			}
 
-	if hr.Spec.Install.Remediation == nil || hr.Spec.Install.Remediation.Retries != -1 {
-		t.Errorf("Spec.Install.Remediation.Retries must remain -1, got %+v", hr.Spec.Install.Remediation)
-	}
-	if hr.Spec.Upgrade.Remediation == nil || hr.Spec.Upgrade.Remediation.Retries != -1 {
-		t.Errorf("Spec.Upgrade.Remediation.Retries must remain -1, got %+v", hr.Spec.Upgrade.Remediation)
-	}
-}
-
-func TestConvertApplicationToHelmRelease_NonKubernetesKindKeepsFluxDefaults(t *testing.T) {
-	// For Applications whose parent chart has no admin-kubeconfig race
-	// (Qdrant, MongoDB, Postgres, etc.), do NOT extend the helm-wait
-	// budget - otherwise failed installs would block three times as long
-	// before Flux starts remediating.
-	r := newRESTForKind("Qdrant", "qdrant-")
-	app := &appsv1alpha1.Application{
-		ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "tenant-root"},
-	}
-
-	hr, err := r.convertApplicationToHelmRelease(app)
-	if err != nil {
-		t.Fatalf("convertApplicationToHelmRelease returned error: %v", err)
-	}
-
-	if hr.Spec.Install != nil && hr.Spec.Install.Timeout != nil {
-		t.Errorf("Spec.Install.Timeout must be unset for non-Kubernetes kinds, got %v", hr.Spec.Install.Timeout.Duration)
-	}
-	if hr.Spec.Upgrade != nil && hr.Spec.Upgrade.Timeout != nil {
-		t.Errorf("Spec.Upgrade.Timeout must be unset for non-Kubernetes kinds, got %v", hr.Spec.Upgrade.Timeout.Duration)
-	}
-
-	// But remediation must still be -1 across the board.
-	if hr.Spec.Install.Remediation == nil || hr.Spec.Install.Remediation.Retries != -1 {
-		t.Errorf("Spec.Install.Remediation.Retries must remain -1, got %+v", hr.Spec.Install.Remediation)
-	}
-	if hr.Spec.Upgrade.Remediation == nil || hr.Spec.Upgrade.Remediation.Retries != -1 {
-		t.Errorf("Spec.Upgrade.Remediation.Retries must remain -1, got %+v", hr.Spec.Upgrade.Remediation)
+			if tc.wantSet {
+				if hr.Spec.Install.Timeout == nil {
+					t.Fatalf("Spec.Install.Timeout must be set when HelmInstallTimeout=%v", tc.configured)
+				}
+				if hr.Spec.Install.Timeout.Duration != tc.configured {
+					t.Errorf("Spec.Install.Timeout = %v, want %v", hr.Spec.Install.Timeout.Duration, tc.configured)
+				}
+				if hr.Spec.Upgrade.Timeout == nil {
+					t.Fatalf("Spec.Upgrade.Timeout must be set when HelmInstallTimeout=%v", tc.configured)
+				}
+				if hr.Spec.Upgrade.Timeout.Duration != tc.configured {
+					t.Errorf("Spec.Upgrade.Timeout = %v, want %v", hr.Spec.Upgrade.Timeout.Duration, tc.configured)
+				}
+			} else {
+				if hr.Spec.Install.Timeout != nil {
+					t.Errorf("Spec.Install.Timeout must be nil when HelmInstallTimeout=0, got %v", hr.Spec.Install.Timeout.Duration)
+				}
+				if hr.Spec.Upgrade.Timeout != nil {
+					t.Errorf("Spec.Upgrade.Timeout must be nil when HelmInstallTimeout=0, got %v", hr.Spec.Upgrade.Timeout.Duration)
+				}
+			}
+
+			if hr.Spec.Install.Remediation == nil || hr.Spec.Install.Remediation.Retries != -1 {
+				t.Errorf("Spec.Install.Remediation.Retries must remain -1, got %+v", hr.Spec.Install.Remediation)
+			}
+			if hr.Spec.Upgrade.Remediation == nil || hr.Spec.Upgrade.Remediation.Retries != -1 {
+				t.Errorf("Spec.Upgrade.Remediation.Retries must remain -1, got %+v", hr.Spec.Upgrade.Remediation)
+			}
+		})
 	}
 }

From bc5473d4fc9bb1b43be79ea5eb8d370d9e6c00b7 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Thu, 16 Apr 2026 21:11:38 +0300
Subject: [PATCH 015/151] test(kubernetes): chart-wide invariant for
 admin-kubeconfig guards

The per-template unittests in packages/apps/kubernetes/tests/ assert
that cluster-autoscaler, kccm, and the csi controller each mount the
admin-kubeconfig Secret optional and carry the wait-for-kubeconfig
init. That locks in today's three Deployments by name - a fourth
Deployment that mounts the same Secret but forgets the guard would
slip past them.

Add a bats-unit test that renders the entire chart, enumerates every
Deployment whose spec mounts a Secret ending in -admin-kubeconfig, and
asserts optional:true plus wait-for-kubeconfig init on all of them.
Verified by temporarily removing optional:true from csi/deploy.yaml:
the test correctly flagged invariant-kcsi-controller as an offender.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 hack/admin-kubeconfig-invariant.bats | 72 ++++++++++++++++++++++++++++
 1 file changed, 72 insertions(+)
 create mode 100644 hack/admin-kubeconfig-invariant.bats

diff --git a/hack/admin-kubeconfig-invariant.bats b/hack/admin-kubeconfig-invariant.bats
new file mode 100644
index 00000000..d699de83
--- /dev/null
+++ b/hack/admin-kubeconfig-invariant.bats
@@ -0,0 +1,72 @@
+#!/usr/bin/env bats
+# -----------------------------------------------------------------------------
+# Chart-wide invariant for packages/apps/kubernetes:
+#
+# Every Deployment in this chart that mounts <release>-admin-kubeconfig as a
+# Secret volume MUST:
+#   - declare that volume optional: true (so kubelet does not FailedMount
+#     while Kamaji is still provisioning the Secret), AND
+#   - include the wait-for-kubeconfig init container (so the pod becomes
+#     Ready only after Kamaji publishes the Secret).
+#
+# The per-template unittests in packages/apps/kubernetes/tests/ lock in
+# today's three Deployments (cluster-autoscaler, kccm, csi controller) by
+# name. This invariant is stricter: any future Deployment added to this
+# chart that mounts the same Secret but forgets the guard will fail here.
+#
+# Requires: helm, yq (mikefarah v4+), jq. All three are available on the
+# project's CI runners and on the maintainer workstation.
+# -----------------------------------------------------------------------------
+
+@test "every Deployment mounting admin-kubeconfig has optional:true and wait-for-kubeconfig init" {
+  values_file="packages/apps/kubernetes/tests/values-ci.yaml"
+  [ -f "$values_file" ]
+
+  tmp=$(mktemp -d)
+  trap 'rm -rf "$tmp"' EXIT
+
+  helm template invariant packages/apps/kubernetes \
+    --namespace tenant-root \
+    --values "$values_file" \
+    2>/dev/null > "$tmp/rendered.yaml"
+
+  # yq streams one JSON object per input document. jq -s slurps the stream
+  # into an array so we can treat all Deployments as a single collection.
+  yq --output-format=json eval-all '.' "$tmp/rendered.yaml" \
+    | jq -s --raw-output '
+        map(select(.kind == "Deployment")) |
+        map({
+          name: .metadata.name,
+          volumes: (.spec.template.spec.volumes // []),
+          initNames: ((.spec.template.spec.initContainers // []) | map(.name)),
+        }) |
+        map(
+          .name as $n |
+          .initNames as $ins |
+          (.volumes[] | select(.secret.secretName | test("-admin-kubeconfig$")?))
+          | {
+              name: $n,
+              optional: (.secret.optional == true),
+              hasInit: ($ins | index("wait-for-kubeconfig") != null),
+            }
+        )
+      ' > "$tmp/summary.json"
+
+  # At least one Deployment must match; if a refactor removes every
+  # admin-kubeconfig volume from this chart, the test must be updated
+  # deliberately rather than silently passing.
+  matched=$(jq 'length' "$tmp/summary.json")
+  [ "$matched" -ge 1 ]
+
+  offenders=$(jq --raw-output '.[] | select(.optional != true or .hasInit != true) | .name' "$tmp/summary.json")
+
+  if [ -n "$offenders" ]; then
+    echo "Deployments mounting *-admin-kubeconfig without optional:true + wait-for-kubeconfig init:" >&2
+    echo "$offenders" >&2
+    echo "Full summary:" >&2
+    cat "$tmp/summary.json" >&2
+    exit 1
+  fi
+
+  echo "Invariant holds for $matched Deployment(s)"
+}

From 03426fbd718029e40d6680f299189e97f915a614 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Thu, 16 Apr 2026 21:13:06 +0300
Subject: [PATCH 016/151] test(hack): pin HelmRelease v2 status shape used by
 remediation guard

run-kubernetes.sh extracts .status.installFailures and
.status.upgradeFailures via kubectl -o jsonpath. If a future flux
release renames the counters, kubectl returns empty, the guard reports
no cycle, and e2e silently misses real remediation loops.

Add a bats unit test that feeds a pinned HelmRelease v2 status snippet
through the same jsonpath and asserts the extraction still yields the
expected values. Also leave a pointer comment in run-kubernetes.sh so a
future flux bump surfaces the version coupling in review.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 hack/e2e-apps/run-kubernetes.sh |  7 ++++++
 hack/remediation-guard.bats     | 44 +++++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+)

diff --git a/hack/e2e-apps/run-kubernetes.sh b/hack/e2e-apps/run-kubernetes.sh
index 811f42a7..e6002b53 100644
--- a/hack/e2e-apps/run-kubernetes.sh
+++ b/hack/e2e-apps/run-kubernetes.sh
@@ -326,6 +326,13 @@ EOF
   # A non-zero installFailures/upgradeFailures indicates the helm-wait budget expired while
   # admin-kubeconfig was still being provisioned, which would trigger uninstall remediation
   # and churn the Cluster CR.
+  # Flux helm-controller v2 status shape: .status.installFailures and
+  # .status.upgradeFailures are counters populated by the controller on
+  # every failed install/upgrade. If a future flux release renames them,
+  # kubectl returns the empty string and the guard silently passes. The
+  # shape is pinned by hack/remediation-guard.bats (see that file for
+  # details), and the vendored API types live under
+  # vendor/github.com/fluxcd/helm-controller/api/v2.
   install_failures=$(kubectl get hr -n tenant-test "kubernetes-${test_name}" -ojsonpath='{.status.installFailures}')
   upgrade_failures=$(kubectl get hr -n tenant-test "kubernetes-${test_name}" -ojsonpath='{.status.upgradeFailures}')
   if helmrelease_has_remediation_cycle "${install_failures}" "${upgrade_failures}"; then
diff --git a/hack/remediation-guard.bats b/hack/remediation-guard.bats
index 64aa83af..9c33452a 100644
--- a/hack/remediation-guard.bats
+++ b/hack/remediation-guard.bats
@@ -85,3 +85,47 @@
     helmrelease_has_remediation_cycle "3" "5" || rc=$?
     [ "$rc" -eq 0 ]
 }
+
+@test "installFailures and upgradeFailures extraction pins HR v2 status shape" {
+    # Pins the Flux HelmRelease v2 status shape that run-kubernetes.sh relies
+    # on. If a future Flux version renames .status.installFailures (or
+    # .status.upgradeFailures), kubectl get -o jsonpath returns an empty
+    # string, the guard quietly says "no cycle", and real remediation loops
+    # slip past the e2e assertion.
+    #
+    # This test uses yq to read the exact path used in the e2e script. yq
+    # evaluates the same json-ish jsonpath against a pinned HR snippet, so
+    # the test fails loudly if the field ever disappears or moves. Cross
+    # reference: vendor/github.com/fluxcd/helm-controller/api/v2/ status
+    # struct field tags.
+    tmp=$(mktemp -d)
+    trap 'rm -rf "$tmp"' EXIT
+
+    cat > "$tmp/hr.yaml" <<'YAML'
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: kubernetes-test
+  namespace: tenant-test
+spec:
+  interval: 5m
+status:
+  installFailures: 2
+  upgradeFailures: 0
+  conditions:
+    - type: Ready
+      status: "False"
+      reason: UninstallSucceeded
+YAML
+
+    install_failures=$(yq '.status.installFailures' "$tmp/hr.yaml")
+    upgrade_failures=$(yq '.status.upgradeFailures' "$tmp/hr.yaml")
+
+    [ "$install_failures" = "2" ]
+    [ "$upgrade_failures" = "0" ]
+
+    . hack/e2e-apps/remediation-guard.sh
+    rc=0
+    helmrelease_has_remediation_cycle "$install_failures" "$upgrade_failures" || rc=$?
+    [ "$rc" -eq 0 ]
+}

From 12632c60c76b77639c8e2edd7ab5bdb248398a7e Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Thu, 16 Apr 2026 21:25:50 +0300
Subject: [PATCH 017/151] fix(kubernetes): gate CP-side Deployments on tenant
 etcd DataStore

The soft-skip wrap in cluster.yaml only silenced Cluster and
KamajiControlPlane rendering when _namespace.etcd is empty. The three
CP-side Deployments (cluster-autoscaler, kccm, kcsi-controller) still
rendered, their wait-for-kubeconfig init containers CrashLoopBackOff'd
forever (no KamajiControlPlane = no admin-kubeconfig Secret),
HelmRelease hit its 15m wait timeout and triggered the very install
remediation cycle the rest of this PR prevents. Self-contradiction.

Wrap each of the three Deployment templates in
{{- if .Values._namespace.etcd }}...{{- end }} so they render only
when there is a DataStore to back them. Add an invariant bats test
that renders the whole chart with etcd empty and asserts zero
Deployments reference *-admin-kubeconfig.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 hack/admin-kubeconfig-invariant.bats          | 45 +++++++++++++++++++
 .../cluster-autoscaler/deployment.yaml        | 12 +++++
 .../apps/kubernetes/templates/csi/deploy.yaml |  2 +
 .../kubernetes/templates/kccm/manager.yaml    |  2 +
 4 files changed, 61 insertions(+)

diff --git a/hack/admin-kubeconfig-invariant.bats b/hack/admin-kubeconfig-invariant.bats
index d699de83..c027e2c4 100644
--- a/hack/admin-kubeconfig-invariant.bats
+++ b/hack/admin-kubeconfig-invariant.bats
@@ -70,3 +70,48 @@
 
   echo "Invariant holds for $matched Deployment(s)"
 }
+
+@test "chart emits zero admin-kubeconfig Deployments when tenant has no etcd DataStore" {
+  # Without a DataStore (parent Tenant has not populated _namespace.etcd yet)
+  # the control-plane-side Deployments must NOT render at all. If they did,
+  # the wait-for-kubeconfig init would CrashLoopBackOff indefinitely - there
+  # would be no KamajiControlPlane to provision the Secret - consuming the
+  # HelmRelease wait budget and triggering exactly the remediation cycle the
+  # rest of this chart tries to avoid. This test renders the whole chart
+  # with etcd empty and asserts no Deployment references the admin-kubeconfig
+  # Secret.
+  tmp=$(mktemp -d)
+  trap 'rm -rf "$tmp"' EXIT
+
+  helm template invariant packages/apps/kubernetes \
+    --namespace tenant-root \
+    --set _namespace.etcd="" \
+    --set _namespace.monitoring="" \
+    --set _namespace.ingress="" \
+    --set _namespace.seaweedfs="" \
+    --set _namespace.host="" \
+    --set _cluster.cluster-domain=cozy.local \
+    --set 'nodeGroups=null' \
+    2>/dev/null > "$tmp/rendered.yaml"
+
+  matched=$(
+    yq --output-format=json eval-all '.' "$tmp/rendered.yaml" \
+      | jq -s '
+          map(select(.kind == "Deployment")) |
+          map(select(
+            (.spec.template.spec.volumes // [])
+              | any(.secret.secretName | test("-admin-kubeconfig$")?)
+          )) |
+          length
+        '
+  )
+
+  if [ "$matched" -ne 0 ]; then
+    echo "Expected zero Deployments mounting *-admin-kubeconfig when etcd is empty, got $matched:" >&2
+    yq --output-format=json eval-all '.' "$tmp/rendered.yaml" \
+      | jq -s 'map(select(.kind == "Deployment") | .metadata.name)' >&2
+    exit 1
+  fi
+
+  echo "No admin-kubeconfig Deployments rendered for empty etcd (as expected)"
+}
diff --git a/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml b/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml
index 348b017d..298d86db 100644
--- a/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml
+++ b/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml
@@ -1,3 +1,14 @@
+{{- /*
+  Gate the control-plane-side workloads on the parent tenant having an etcd
+  DataStore. Without it no KamajiControlPlane is ever created, Kamaji never
+  provisions -admin-kubeconfig, and rendering these Deployments would cause
+  the wait-for-kubeconfig init to CrashLoopBackOff indefinitely, consuming
+  the parent HelmRelease install timeout and triggering the very uninstall
+  remediation cycle this chart is supposed to avoid. Rendering them only
+  when $etcd is set keeps the HelmRelease Ready while flux retries on its
+  interval and picks up the DataStore as soon as the Tenant chart finishes.
+*/}}
+{{- if .Values._namespace.etcd }}
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -108,3 +119,4 @@ rules:
     - list
     - update
     - watch
+{{- end }}
diff --git a/packages/apps/kubernetes/templates/csi/deploy.yaml b/packages/apps/kubernetes/templates/csi/deploy.yaml
index c1af7f13..de62104c 100644
--- a/packages/apps/kubernetes/templates/csi/deploy.yaml
+++ b/packages/apps/kubernetes/templates/csi/deploy.yaml
@@ -1,3 +1,4 @@
+{{- if .Values._namespace.etcd }}
 kind: Deployment
 apiVersion: apps/v1
 metadata:
@@ -238,3 +239,4 @@ spec:
             secretName: {{ .Release.Name }}-admin-kubeconfig
             optional: true
           name: kubeconfig
+{{- end }}
diff --git a/packages/apps/kubernetes/templates/kccm/manager.yaml b/packages/apps/kubernetes/templates/kccm/manager.yaml
index 20ac0a1e..bd9e2798 100644
--- a/packages/apps/kubernetes/templates/kccm/manager.yaml
+++ b/packages/apps/kubernetes/templates/kccm/manager.yaml
@@ -1,3 +1,4 @@
+{{- if .Values._namespace.etcd }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -60,3 +61,4 @@ spec:
           optional: true
         name: kubeconfig
       serviceAccountName: {{ .Release.Name }}-kccm
+{{- end }}

From 1ddd2aaea97024f6803c60d29a946273b55e4185 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Thu, 16 Apr 2026 21:28:11 +0300
Subject: [PATCH 018/151] fix(hack): detect remediation via status.history, not
 transient counters

Flux helm-controller's ClearFailures() zeroes installFailures and
upgradeFailures on every successful reconciliation (see the upstream
HelmReleaseStatus method). The previous guard ran after the HelmRelease
was Ready, at which point the counters were always 0 - the assertion
was vacuous and would have passed against a reverted fix.

Switch to .status.history, which retains per-revision release
Snapshots that survive a subsequent successful reconciliation. A
remediation cycle leaves behind a Snapshot with status=uninstalled
(the install-remediation code path) or status=failed (Helm release
failure that remediation then uninstalled). Either one signals the
race actually fired.

Rewrite the bats unit tests to cover: empty history, deployed-only,
deployed+superseded (happy path - not detected), single failed,
single uninstalled, uninstalled-then-deployed, and deployed-then-failed
(all detected). The pinned-shape test feeds a realistic HR status
snippet through yq the same way run-kubernetes.sh does via kubectl
-o jsonpath.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 hack/e2e-apps/remediation-guard.sh |  45 +++++---
 hack/e2e-apps/run-kubernetes.sh    |  24 +++--
 hack/remediation-guard.bats        | 158 ++++++++++++++---------------
 3 files changed, 120 insertions(+), 107 deletions(-)

diff --git a/hack/e2e-apps/remediation-guard.sh b/hack/e2e-apps/remediation-guard.sh
index 6da69f13..38463310 100644
--- a/hack/e2e-apps/remediation-guard.sh
+++ b/hack/e2e-apps/remediation-guard.sh
@@ -1,24 +1,39 @@
 # Helpers for asserting that a Flux HelmRelease did not fall into an
 # install/upgrade remediation cycle during an e2e run.
 #
-# A non-zero installFailures/upgradeFailures counter means flux
-# helm-controller hit its wait timeout, ran remediation (uninstall),
-# and re-installed. That is exactly the race this guard is meant to
-# catch, so the function returns success (0) when a cycle is detected
-# and failure (1) otherwise.
+# Background: Flux helm-controller's ClearFailures() zeroes
+# .status.installFailures / .status.upgradeFailures on every successful
+# reconciliation (see the upstream ClearFailures method on
+# HelmReleaseStatus). That makes those counters useless for a guard that
+# runs after the HelmRelease has reached Ready - the values are always 0.
 #
-# Both arguments may be empty strings, the literal "0", or a positive
-# integer. Shell's && and || have equal precedence with left-to-right
-# associativity, so each half of the disjunction is grouped explicitly
-# to avoid (A && B) || C && D parsing that masks the common
-# install_failures=1, upgrade_failures="" case.
+# What survives a successful reconciliation is .status.history, a bounded
+# list of release Snapshots. Each Snapshot carries a status field that
+# tracks the Helm release state: deployed, superseded, failed, uninstalled,
+# and so on. A remediation cycle leaves the footprint behind: a snapshot
+# with status "uninstalled" (from install/upgrade remediation) or "failed"
+# (Helm release failure that remediation then uninstalled). Those stay in
+# history even after a subsequent successful reinstall.
+#
+# helmrelease_has_remediation_cycle takes a newline-delimited list of
+# snapshot statuses (whatever the caller extracted via kubectl -o jsonpath
+# or equivalent) and returns 0 (detected) when any entry is "failed" or
+# "uninstalled", 1 otherwise. Empty input is treated as "no history yet,
+# no cycle observed".
 
 helmrelease_has_remediation_cycle() {
-    install_failures="$1"
-    upgrade_failures="$2"
-    if { [ -n "${install_failures}" ] && [ "${install_failures}" != "0" ]; } || \
-       { [ -n "${upgrade_failures}" ] && [ "${upgrade_failures}" != "0" ]; }; then
-        return 0
+    statuses="$1"
+    if [ -z "${statuses}" ]; then
+        return 1
     fi
+    while IFS= read -r status; do
+        case "${status}" in
+            failed|uninstalled)
+                return 0
+                ;;
+        esac
+    done <<EOF
+${statuses}
+EOF
     return 1
 }
diff --git a/hack/e2e-apps/run-kubernetes.sh b/hack/e2e-apps/run-kubernetes.sh
index e6002b53..64b1ae90 100644
--- a/hack/e2e-apps/run-kubernetes.sh
+++ b/hack/e2e-apps/run-kubernetes.sh
@@ -326,17 +326,19 @@ EOF
   # A non-zero installFailures/upgradeFailures indicates the helm-wait budget expired while
   # admin-kubeconfig was still being provisioned, which would trigger uninstall remediation
   # and churn the Cluster CR.
-  # Flux helm-controller v2 status shape: .status.installFailures and
-  # .status.upgradeFailures are counters populated by the controller on
-  # every failed install/upgrade. If a future flux release renames them,
-  # kubectl returns the empty string and the guard silently passes. The
-  # shape is pinned by hack/remediation-guard.bats (see that file for
-  # details), and the vendored API types live under
-  # vendor/github.com/fluxcd/helm-controller/api/v2.
-  install_failures=$(kubectl get hr -n tenant-test "kubernetes-${test_name}" -ojsonpath='{.status.installFailures}')
-  upgrade_failures=$(kubectl get hr -n tenant-test "kubernetes-${test_name}" -ojsonpath='{.status.upgradeFailures}')
-  if helmrelease_has_remediation_cycle "${install_failures}" "${upgrade_failures}"; then
-    echo "Parent HelmRelease entered remediation cycle: installFailures=${install_failures:-0}, upgradeFailures=${upgrade_failures:-0}" >&2
+  # Flux helm-controller v2 retains per-revision release Snapshots in
+  # .status.history; each Snapshot's .status reflects the Helm release
+  # state (deployed/superseded/failed/uninstalled). A remediation cycle
+  # leaves a "failed" or "uninstalled" entry behind that survives a later
+  # successful reinstall, unlike the installFailures/upgradeFailures
+  # counters (which ClearFailures zeroes on every successful reconcile).
+  # The shape is pinned by hack/remediation-guard.bats; the upstream
+  # types are github.com/fluxcd/helm-controller/api v2 Snapshot.
+  history_statuses=$(kubectl get hr -n tenant-test "kubernetes-${test_name}" \
+    -ojsonpath='{range .status.history[*]}{.status}{"\n"}{end}')
+  if helmrelease_has_remediation_cycle "${history_statuses}"; then
+    echo "Parent HelmRelease entered remediation cycle. History statuses:" >&2
+    printf '%s\n' "${history_statuses}" >&2
     kubectl -n tenant-test describe hr "kubernetes-${test_name}" >&2
     exit 1
   fi
diff --git a/hack/remediation-guard.bats b/hack/remediation-guard.bats
index 9c33452a..30e4e04c 100644
--- a/hack/remediation-guard.bats
+++ b/hack/remediation-guard.bats
@@ -2,17 +2,15 @@
 # -----------------------------------------------------------------------------
 # Unit tests for hack/e2e-apps/remediation-guard.sh
 #
-# helmrelease_has_remediation_cycle is consumed from e2e tests to assert that
-# the parent HelmRelease did not hit flux helm-controller's wait timeout and
-# enter uninstall remediation. The function accepts two arguments (values of
-# .status.installFailures and .status.upgradeFailures) and returns 0 when a
-# remediation cycle is detected, 1 otherwise.
+# helmrelease_has_remediation_cycle takes a newline-delimited list of
+# HelmRelease history snapshot status values (deployed/superseded/failed/
+# uninstalled/...) and returns 0 when any entry is "failed" or "uninstalled"
+# (meaning flux helm-controller performed install/upgrade remediation).
 #
-# Each argument can be empty (controller never populated the field), "0"
-# (populated but never failed), or a positive integer. Shell's && and ||
-# have equal precedence with left-to-right associativity, which used to
-# break this check on the most common failure mode - install_failures=1
-# and upgrade_failures="". These tests pin the correct behavior.
+# This is used by the e2e script after the HelmRelease reaches Ready. The
+# failure/upgrade counters (.status.installFailures / .status.upgradeFailures)
+# are useless there because flux's ClearFailures zeroes them on successful
+# reconciliation; .status.history retains the snapshot trail.
 #
 # cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on
 # its own line; there is no bats `run` or `$status`. Assertions are
@@ -21,83 +19,76 @@
 # Run with: hack/cozytest.sh hack/remediation-guard.bats
 # -----------------------------------------------------------------------------
 
-@test "no counters set returns not-detected" {
+@test "empty history returns not-detected" {
     . hack/e2e-apps/remediation-guard.sh
-    rc=0
-    helmrelease_has_remediation_cycle "" "" || rc=$?
-    [ "$rc" -eq 1 ]
+    if helmrelease_has_remediation_cycle ""; then
+        echo "expected not-detected for empty history" >&2
+        exit 1
+    fi
 }
 
-@test "both counters zero returns not-detected" {
+@test "single deployed snapshot returns not-detected" {
     . hack/e2e-apps/remediation-guard.sh
-    rc=0
-    helmrelease_has_remediation_cycle "0" "0" || rc=$?
-    [ "$rc" -eq 1 ]
+    if helmrelease_has_remediation_cycle "deployed"; then
+        echo "expected not-detected for deployed-only history" >&2
+        exit 1
+    fi
 }
 
-@test "install zero upgrade empty returns not-detected" {
+@test "deployed then superseded returns not-detected" {
     . hack/e2e-apps/remediation-guard.sh
-    rc=0
-    helmrelease_has_remediation_cycle "0" "" || rc=$?
-    [ "$rc" -eq 1 ]
+    statuses=$(printf 'deployed\nsuperseded\n')
+    if helmrelease_has_remediation_cycle "${statuses}"; then
+        echo "expected not-detected for deployed+superseded history" >&2
+        exit 1
+    fi
 }
 
-@test "install empty upgrade zero returns not-detected" {
+@test "single failed snapshot returns detected" {
     . hack/e2e-apps/remediation-guard.sh
-    rc=0
-    helmrelease_has_remediation_cycle "" "0" || rc=$?
-    [ "$rc" -eq 1 ]
+    if ! helmrelease_has_remediation_cycle "failed"; then
+        echo "expected detected when history contains failed snapshot" >&2
+        exit 1
+    fi
 }
 
-@test "install one upgrade empty returns detected" {
-    # Canonical race: first install exceeded helm-wait, remediation fired,
-    # no upgrade has happened yet.
+@test "single uninstalled snapshot returns detected" {
+    # The exact signature of the install-remediation race: the first install
+    # exceeded flux's wait budget, remediation uninstalled, the next retry
+    # eventually succeeded. History still carries the uninstalled snapshot.
     . hack/e2e-apps/remediation-guard.sh
-    rc=0
-    helmrelease_has_remediation_cycle "1" "" || rc=$?
-    [ "$rc" -eq 0 ]
+    if ! helmrelease_has_remediation_cycle "uninstalled"; then
+        echo "expected detected when history contains uninstalled snapshot" >&2
+        exit 1
+    fi
 }
 
-@test "install empty upgrade one returns detected" {
+@test "uninstalled then deployed still returns detected" {
     . hack/e2e-apps/remediation-guard.sh
-    rc=0
-    helmrelease_has_remediation_cycle "" "1" || rc=$?
-    [ "$rc" -eq 0 ]
+    statuses=$(printf 'uninstalled\ndeployed\n')
+    if ! helmrelease_has_remediation_cycle "${statuses}"; then
+        echo "expected detected despite later successful deploy" >&2
+        exit 1
+    fi
 }
 
-@test "install two upgrade zero returns detected" {
+@test "deployed then failed still returns detected" {
     . hack/e2e-apps/remediation-guard.sh
-    rc=0
-    helmrelease_has_remediation_cycle "2" "0" || rc=$?
-    [ "$rc" -eq 0 ]
+    statuses=$(printf 'deployed\nfailed\n')
+    if ! helmrelease_has_remediation_cycle "${statuses}"; then
+        echo "expected detected when any entry is failed" >&2
+        exit 1
+    fi
 }
 
-@test "install zero upgrade two returns detected" {
-    . hack/e2e-apps/remediation-guard.sh
-    rc=0
-    helmrelease_has_remediation_cycle "0" "2" || rc=$?
-    [ "$rc" -eq 0 ]
-}
-
-@test "both counters positive returns detected" {
-    . hack/e2e-apps/remediation-guard.sh
-    rc=0
-    helmrelease_has_remediation_cycle "3" "5" || rc=$?
-    [ "$rc" -eq 0 ]
-}
-
-@test "installFailures and upgradeFailures extraction pins HR v2 status shape" {
-    # Pins the Flux HelmRelease v2 status shape that run-kubernetes.sh relies
-    # on. If a future Flux version renames .status.installFailures (or
-    # .status.upgradeFailures), kubectl get -o jsonpath returns an empty
-    # string, the guard quietly says "no cycle", and real remediation loops
-    # slip past the e2e assertion.
-    #
-    # This test uses yq to read the exact path used in the e2e script. yq
-    # evaluates the same json-ish jsonpath against a pinned HR snippet, so
-    # the test fails loudly if the field ever disappears or moves. Cross
-    # reference: vendor/github.com/fluxcd/helm-controller/api/v2/ status
-    # struct field tags.
+@test "installFailures extraction pins HR v2 status.history shape" {
+    # Pins the Flux HelmRelease v2 .status.history[].status shape that
+    # run-kubernetes.sh relies on. If a future flux release renames the
+    # field, the jsonpath returns nothing, the guard reports no cycle,
+    # and real remediation loops slip past the e2e assertion. This test
+    # uses yq to read the exact path used in the e2e script; the upstream
+    # Snapshot type lives at
+    # github.com/fluxcd/helm-controller/api/v2.Snapshot (via go.mod).
     tmp=$(mktemp -d)
     trap 'rm -rf "$tmp"' EXIT
 
@@ -107,25 +98,30 @@ kind: HelmRelease
 metadata:
   name: kubernetes-test
   namespace: tenant-test
-spec:
-  interval: 5m
 status:
-  installFailures: 2
-  upgradeFailures: 0
-  conditions:
-    - type: Ready
-      status: "False"
-      reason: UninstallSucceeded
+  history:
+    - name: kubernetes-test
+      namespace: tenant-test
+      version: 1
+      status: uninstalled
+    - name: kubernetes-test
+      namespace: tenant-test
+      version: 2
+      status: deployed
 YAML
 
-    install_failures=$(yq '.status.installFailures' "$tmp/hr.yaml")
-    upgrade_failures=$(yq '.status.upgradeFailures' "$tmp/hr.yaml")
+    # Default yq output is yaml scalar format, which for string values emits
+    # bare unquoted tokens - matching what kubectl -o jsonpath produces in
+    # e2e. Do not switch to JSON output here; that would quote the values
+    # and break the loop in helmrelease_has_remediation_cycle.
+    statuses=$(yq '.status.history[].status' "$tmp/hr.yaml")
 
-    [ "$install_failures" = "2" ]
-    [ "$upgrade_failures" = "0" ]
+    [ -n "$statuses" ]
+    echo "$statuses" | grep --quiet '^uninstalled$'
 
     . hack/e2e-apps/remediation-guard.sh
-    rc=0
-    helmrelease_has_remediation_cycle "$install_failures" "$upgrade_failures" || rc=$?
-    [ "$rc" -eq 0 ]
+    if ! helmrelease_has_remediation_cycle "$statuses"; then
+        echo "expected detected for pinned HR snippet with uninstalled + deployed history" >&2
+        exit 1
+    fi
 }

From 48312cc3696904235e87fcf2bc33d05e21a77b72 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Thu, 16 Apr 2026 21:29:49 +0300
Subject: [PATCH 019/151] build: wire go-unit-tests into make unit-tests

CI runs make unit-tests on every PR, which already covers helm
unittests and bats (hack/admin-kubeconfig-invariant.bats and
hack/remediation-guard.bats are both picked up by the existing
hack/*.bats glob). What was missing was any go test invocation.

Add a go-unit-tests target scoped to pkg/registry, pkg/config, and
pkg/cmd/server - the cozystack-api surface this repo actually owns
and tests in-tree. Running go test ./... pulls in generated-code
round-trip suites whose behavior is governed by generator tool
versions outside this repo's control; those are better exercised
from their own generator workflows.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 Makefile | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 0e94c5e6..10a55314 100644
--- a/Makefile
+++ b/Makefile
@@ -82,11 +82,19 @@ test:
 	make -C packages/core/testing apply
 	make -C packages/core/testing test
 
-unit-tests: helm-unit-tests bats-unit-tests
+unit-tests: helm-unit-tests bats-unit-tests go-unit-tests
 
 helm-unit-tests:
 	hack/helm-unit-tests.sh
 
+# Scoped go test over the cozystack-api surface that this repo owns. Kept
+# narrow intentionally - running `go test ./...` pulls in generated code
+# round-trip suites whose behavior depends on tool versions outside this
+# repo's control (kubebuilder, openapi-gen, etc.) and is better exercised
+# from their generator workflows.
+go-unit-tests:
+	go test ./pkg/registry/... ./pkg/config/... ./pkg/cmd/server/...
+
 # Discover every hack/*.bats file that is NOT an e2e test and run it
 # through cozytest.sh. Drop a new *.bats file in hack/ and it is picked
 # up automatically on the next `make unit-tests` run.

From d2e8f7e86cc7157aa83a7130b3f9857f10518218 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Thu, 16 Apr 2026 21:31:29 +0300
Subject: [PATCH 020/151] chore(kubernetes): drop busybox mirror Containerfile,
 pin upstream directly

The image-busybox Makefile target + images/busybox/Dockerfile wrapper
just rebuilt an upstream busybox digest as ghcr.io/cozystack/cozystack/
busybox. Payload is a one-shot sh loop run once per pod; the pinned
upstream digest is already immutable, so maintaining a private mirror
adds churn (rebuild on every release) for no real hardening benefit.
Delete the wrapper and reference docker.io/library/busybox:<digest>
directly from images/busybox.tag. Document the choice in _helpers.tpl.

Also drop the false coupling in the go table test: the "unrelated
kind without configured timeout" case used the real kind name Qdrant,
which tied the test to the Qdrant ApplicationDefinition for no reason.
Switch to a clearly fictional kind so a future Qdrant timeout override
does not break this assertion.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 packages/apps/kubernetes/Makefile                  | 13 +------------
 packages/apps/kubernetes/images/busybox/Dockerfile |  1 -
 packages/apps/kubernetes/templates/_helpers.tpl    |  9 ++++++++-
 pkg/registry/apps/application/rest_timeout_test.go | 10 +++++++---
 4 files changed, 16 insertions(+), 17 deletions(-)
 delete mode 100644 packages/apps/kubernetes/images/busybox/Dockerfile

diff --git a/packages/apps/kubernetes/Makefile b/packages/apps/kubernetes/Makefile
index eb23c2ef..4fea42ef 100644
--- a/packages/apps/kubernetes/Makefile
+++ b/packages/apps/kubernetes/Makefile
@@ -15,7 +15,7 @@ update:
 	hack/update-versions.sh
 	make generate
 
-image: image-ubuntu-container-disk image-kubevirt-cloud-provider image-kubevirt-csi-driver image-cluster-autoscaler image-busybox
+image: image-ubuntu-container-disk image-kubevirt-cloud-provider image-kubevirt-csi-driver image-cluster-autoscaler
 
 image-ubuntu-container-disk:
 	$(foreach ver,$(KUBERNETES_VERSIONS), \
@@ -71,14 +71,3 @@ image-cluster-autoscaler:
 		> images/cluster-autoscaler.tag
 	rm -f images/cluster-autoscaler.json
 
-image-busybox:
-	docker buildx build images/busybox \
-		--tag $(REGISTRY)/busybox:$(call settag,$(KUBERNETES_PKG_TAG)) \
-		--tag $(REGISTRY)/busybox:$(call settag,$(KUBERNETES_PKG_TAG)-$(TAG)) \
-		--cache-from type=registry,ref=$(REGISTRY)/busybox:latest \
-		--cache-to type=inline \
-		--metadata-file images/busybox.json \
-		$(BUILDX_ARGS)
-	echo "$(REGISTRY)/busybox:$(call settag,$(KUBERNETES_PKG_TAG))@$$(yq e '."containerimage.digest"' images/busybox.json -o json -r)" \
-		> images/busybox.tag
-	rm -f images/busybox.json
diff --git a/packages/apps/kubernetes/images/busybox/Dockerfile b/packages/apps/kubernetes/images/busybox/Dockerfile
deleted file mode 100644
index 0c57fd39..00000000
--- a/packages/apps/kubernetes/images/busybox/Dockerfile
+++ /dev/null
@@ -1 +0,0 @@
-FROM docker.io/library/busybox:1.37.0@sha256:1487d0af5f52b4ba31c7e465126ee2123fe3f2305d638e7827681e7cf6c83d5e
diff --git a/packages/apps/kubernetes/templates/_helpers.tpl b/packages/apps/kubernetes/templates/_helpers.tpl
index 89f06934..e6ad9dcf 100644
--- a/packages/apps/kubernetes/templates/_helpers.tpl
+++ b/packages/apps/kubernetes/templates/_helpers.tpl
@@ -60,10 +60,17 @@ exits only when super-admin.svc appears, which happens after kubelet's
 optional-Secret refresh cycle.
 
 The 10m deadline stays strictly below the 15m HelmRelease
-Install.Timeout scoped to the Kubernetes Application kind so the
+Install.Timeout set by cozystack-api for the Kubernetes kind (via the
+release.cozystack.io/helm-install-timeout annotation) so the
 CrashLoopBackOff surfaces before flux remediation fires and uninstalls
 the Cluster CR.
 
+The pinned busybox image in images/busybox.tag points directly at
+docker.io by digest (not mirrored to ghcr.io like the other .tag files
+here): the payload is a one-shot sh loop, the digest pin makes the
+pull immutable, and the cost of maintaining a private mirror of a tiny
+upstream image that does not move often is not worth it.
+
 Call site owns the surrounding volumes block; the kubeconfig volume
 must exist on the pod and mount at /etc/kubernetes/kubeconfig.
 */}}
diff --git a/pkg/registry/apps/application/rest_timeout_test.go b/pkg/registry/apps/application/rest_timeout_test.go
index 8474d1cb..85ac0b97 100644
--- a/pkg/registry/apps/application/rest_timeout_test.go
+++ b/pkg/registry/apps/application/rest_timeout_test.go
@@ -48,9 +48,13 @@ func TestConvertApplicationToHelmRelease_AppliesReleaseConfigTimeout(t *testing.
 			wantSet:    true,
 		},
 		{
-			name:       "Qdrant kind without configured timeout keeps flux defaults",
-			kind:       "Qdrant",
-			prefix:     "qdrant-",
+			// Fictional kind on purpose: the test is about the unset path
+			// regardless of which real Application kind ends up needing a
+			// timeout override. Using a real kind name would create false
+			// coupling to that Application's ApplicationDefinition.
+			name:       "unrelated kind without configured timeout keeps flux defaults",
+			kind:       "PlaceholderKindForDefaults",
+			prefix:     "placeholder-",
 			configured: 0,
 			wantSet:    false,
 		},

From 64b216e10a2392394add9eb7fbf7e908f4c8864a Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Thu, 16 Apr 2026 21:46:01 +0300
Subject: [PATCH 021/151] fix(kubernetes): gate child HelmReleases on tenant
 etcd DataStore

17 child HelmReleases (cilium, coredns, csi, cert-manager,
metrics-server, ...) referenced *-admin-kubeconfig via
kubeConfig.secretRef and rendered even when _namespace.etcd was empty.
On an etcd-less tenant each one sat in NotReady forever because the
admin-kubeconfig Secret only exists after a KamajiControlPlane
reconciles, and KamajiControlPlane now only renders when etcd is set.
The outcome contradicted the "beacon only" contract claimed in the
soft-skip commit.

Extend the existing addon guards to also require _namespace.etcd, and
wrap the four unconditional HelmReleases (csi, metrics-server,
prometheus-operator-crds, volumesnapshot-crd) plus the always-on
cilium/coredns HR resources in the same gate. Add an invariant bats
test that renders the whole chart with etcd empty and asserts zero
HelmReleases reference *-admin-kubeconfig.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 hack/admin-kubeconfig-invariant.bats          | 40 +++++++++++++++++++
 .../helmreleases/cert-manager-crds.yaml       |  2 +-
 .../templates/helmreleases/cert-manager.yaml  |  2 +-
 .../templates/helmreleases/cilium.yaml        |  2 +
 .../templates/helmreleases/coredns.yaml       |  2 +
 .../templates/helmreleases/csi.yaml           |  2 +
 .../templates/helmreleases/fluxcd.yaml        |  2 +-
 .../helmreleases/gateway-api-crds.yaml        |  2 +-
 .../templates/helmreleases/gpu-operator.yaml  |  2 +-
 .../templates/helmreleases/ingress-nginx.yaml |  2 +-
 .../helmreleases/metrics-server.yaml          |  2 +
 .../helmreleases/monitoring-agents.yaml       |  2 +-
 .../prometheus-operator-crds.yaml             |  2 +
 .../templates/helmreleases/velero.yaml        |  2 +-
 .../vertical-pod-autoscaler-crds.yaml         |  2 +-
 .../helmreleases/vertical-pod-autoscaler.yaml |  2 +-
 .../victoria-metrics-operator.yaml            |  2 +-
 .../helmreleases/volumesnapshot-crd.yaml      |  2 +
 18 files changed, 63 insertions(+), 11 deletions(-)

diff --git a/hack/admin-kubeconfig-invariant.bats b/hack/admin-kubeconfig-invariant.bats
index c027e2c4..4c809c41 100644
--- a/hack/admin-kubeconfig-invariant.bats
+++ b/hack/admin-kubeconfig-invariant.bats
@@ -115,3 +115,43 @@
 
   echo "No admin-kubeconfig Deployments rendered for empty etcd (as expected)"
 }
+
+@test "chart emits zero admin-kubeconfig HelmReleases when tenant has no etcd DataStore" {
+  # Same principle as the Deployment variant above, extended to every child
+  # HelmRelease (cilium, coredns, csi, cert-manager, ...). They reference
+  # *-admin-kubeconfig via kubeConfig.secretRef and would otherwise sit in
+  # NotReady forever on an etcd-less tenant, polluting the HelmRelease list
+  # the operator sees and contradicting the "awaiting-etcd beacon only"
+  # contract of the soft-skip path.
+  tmp=$(mktemp -d)
+  trap 'rm -rf "$tmp"' EXIT
+
+  helm template invariant packages/apps/kubernetes \
+    --namespace tenant-root \
+    --set _namespace.etcd="" \
+    --set _namespace.monitoring="" \
+    --set _namespace.ingress="" \
+    --set _namespace.seaweedfs="" \
+    --set _namespace.host="" \
+    --set _cluster.cluster-domain=cozy.local \
+    --set 'nodeGroups=null' \
+    2>/dev/null > "$tmp/rendered.yaml"
+
+  matched=$(
+    yq --output-format=json eval-all '.' "$tmp/rendered.yaml" \
+      | jq -s '
+          map(select(.kind == "HelmRelease")) |
+          map(select(.spec.kubeConfig.secretRef.name | test("-admin-kubeconfig$")?)) |
+          length
+        '
+  )
+
+  if [ "$matched" -ne 0 ]; then
+    echo "Expected zero HelmReleases referencing *-admin-kubeconfig when etcd is empty, got $matched:" >&2
+    yq --output-format=json eval-all '.' "$tmp/rendered.yaml" \
+      | jq -s 'map(select(.kind == "HelmRelease") | .metadata.name)' >&2
+    exit 1
+  fi
+
+  echo "No admin-kubeconfig HelmReleases rendered for empty etcd (as expected)"
+}
diff --git a/packages/apps/kubernetes/templates/helmreleases/cert-manager-crds.yaml b/packages/apps/kubernetes/templates/helmreleases/cert-manager-crds.yaml
index be07a8b9..fd9dac7e 100644
--- a/packages/apps/kubernetes/templates/helmreleases/cert-manager-crds.yaml
+++ b/packages/apps/kubernetes/templates/helmreleases/cert-manager-crds.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.addons.certManager.enabled }}
+{{- if and .Values.addons.certManager.enabled .Values._namespace.etcd }}
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
diff --git a/packages/apps/kubernetes/templates/helmreleases/cert-manager.yaml b/packages/apps/kubernetes/templates/helmreleases/cert-manager.yaml
index 6857581a..700b666e 100644
--- a/packages/apps/kubernetes/templates/helmreleases/cert-manager.yaml
+++ b/packages/apps/kubernetes/templates/helmreleases/cert-manager.yaml
@@ -8,7 +8,7 @@ cert-manager:
 {{- end }}
 {{- end }}
 
-{{- if .Values.addons.certManager.enabled }}
+{{- if and .Values.addons.certManager.enabled .Values._namespace.etcd }}
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
diff --git a/packages/apps/kubernetes/templates/helmreleases/cilium.yaml b/packages/apps/kubernetes/templates/helmreleases/cilium.yaml
index d032f5b6..d8c90cbf 100644
--- a/packages/apps/kubernetes/templates/helmreleases/cilium.yaml
+++ b/packages/apps/kubernetes/templates/helmreleases/cilium.yaml
@@ -14,6 +14,7 @@ cilium:
   {{- end }}
 {{- end }}
 
+{{- if .Values._namespace.etcd }}
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
@@ -55,3 +56,4 @@ spec:
   - name: {{ .Release.Name }}-gateway-api-crds
     namespace: {{ .Release.Namespace }}
   {{- end }}
+{{- end }}
diff --git a/packages/apps/kubernetes/templates/helmreleases/coredns.yaml b/packages/apps/kubernetes/templates/helmreleases/coredns.yaml
index bdb6c682..0711a51d 100644
--- a/packages/apps/kubernetes/templates/helmreleases/coredns.yaml
+++ b/packages/apps/kubernetes/templates/helmreleases/coredns.yaml
@@ -4,6 +4,7 @@ coredns:
     clusterIP: "10.95.0.10"
 {{- end }}
 
+{{- if .Values._namespace.etcd }}
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
@@ -42,3 +43,4 @@ spec:
   {{- end }}
   - name: {{ .Release.Name }}-cilium
     namespace: {{ .Release.Namespace }}
+{{- end }}
diff --git a/packages/apps/kubernetes/templates/helmreleases/csi.yaml b/packages/apps/kubernetes/templates/helmreleases/csi.yaml
index dd2c69a6..109d78e1 100644
--- a/packages/apps/kubernetes/templates/helmreleases/csi.yaml
+++ b/packages/apps/kubernetes/templates/helmreleases/csi.yaml
@@ -1,3 +1,4 @@
+{{- if .Values._namespace.etcd }}
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
@@ -39,3 +40,4 @@ spec:
   - name: {{ .Release.Name }}
     namespace: {{ .Release.Namespace }}
   {{- end }}
+{{- end }}
diff --git a/packages/apps/kubernetes/templates/helmreleases/fluxcd.yaml b/packages/apps/kubernetes/templates/helmreleases/fluxcd.yaml
index 76499dfe..25fff01c 100644
--- a/packages/apps/kubernetes/templates/helmreleases/fluxcd.yaml
+++ b/packages/apps/kubernetes/templates/helmreleases/fluxcd.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.addons.fluxcd.enabled }}
+{{- if and .Values.addons.fluxcd.enabled .Values._namespace.etcd }}
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
diff --git a/packages/apps/kubernetes/templates/helmreleases/gateway-api-crds.yaml b/packages/apps/kubernetes/templates/helmreleases/gateway-api-crds.yaml
index 2bcc8d4d..b4172ed1 100644
--- a/packages/apps/kubernetes/templates/helmreleases/gateway-api-crds.yaml
+++ b/packages/apps/kubernetes/templates/helmreleases/gateway-api-crds.yaml
@@ -1,4 +1,4 @@
-{{- if $.Values.addons.gatewayAPI.enabled }}
+{{- if and $.Values.addons.gatewayAPI.enabled $.Values._namespace.etcd }}
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
diff --git a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml
index 5ef48912..e243bfad 100644
--- a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml
+++ b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.addons.gpuOperator.enabled }}
+{{- if and .Values.addons.gpuOperator.enabled .Values._namespace.etcd }}
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
diff --git a/packages/apps/kubernetes/templates/helmreleases/ingress-nginx.yaml b/packages/apps/kubernetes/templates/helmreleases/ingress-nginx.yaml
index 5cafff90..6e2183d3 100644
--- a/packages/apps/kubernetes/templates/helmreleases/ingress-nginx.yaml
+++ b/packages/apps/kubernetes/templates/helmreleases/ingress-nginx.yaml
@@ -20,7 +20,7 @@ ingress-nginx:
     node-role.kubernetes.io/ingress-nginx: ""
 {{- end }}
 
-{{- if .Values.addons.ingressNginx.enabled }}
+{{- if and .Values.addons.ingressNginx.enabled .Values._namespace.etcd }}
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
diff --git a/packages/apps/kubernetes/templates/helmreleases/metrics-server.yaml b/packages/apps/kubernetes/templates/helmreleases/metrics-server.yaml
index 3cc81a14..3e6f9660 100644
--- a/packages/apps/kubernetes/templates/helmreleases/metrics-server.yaml
+++ b/packages/apps/kubernetes/templates/helmreleases/metrics-server.yaml
@@ -1,3 +1,4 @@
+{{- if .Values._namespace.etcd }}
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
@@ -36,3 +37,4 @@ spec:
     namespace: {{ .Release.Namespace }}
   - name: {{ .Release.Name }}-prometheus-operator-crds
     namespace: {{ .Release.Namespace }}
+{{- end }}
diff --git a/packages/apps/kubernetes/templates/helmreleases/monitoring-agents.yaml b/packages/apps/kubernetes/templates/helmreleases/monitoring-agents.yaml
index ea84dec0..a811f7dd 100644
--- a/packages/apps/kubernetes/templates/helmreleases/monitoring-agents.yaml
+++ b/packages/apps/kubernetes/templates/helmreleases/monitoring-agents.yaml
@@ -1,6 +1,6 @@
 {{- $targetTenant := .Values._namespace.monitoring }}
 {{- $clusterDomain := (index .Values._cluster "cluster-domain") | default "cozy.local" }}
-{{- if .Values.addons.monitoringAgents.enabled }}
+{{- if and .Values.addons.monitoringAgents.enabled .Values._namespace.etcd }}
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
diff --git a/packages/apps/kubernetes/templates/helmreleases/prometheus-operator-crds.yaml b/packages/apps/kubernetes/templates/helmreleases/prometheus-operator-crds.yaml
index 600a7994..3038a058 100644
--- a/packages/apps/kubernetes/templates/helmreleases/prometheus-operator-crds.yaml
+++ b/packages/apps/kubernetes/templates/helmreleases/prometheus-operator-crds.yaml
@@ -1,3 +1,4 @@
+{{- if .Values._namespace.etcd }}
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
@@ -31,3 +32,4 @@ spec:
   - name: {{ .Release.Name }}
     namespace: {{ .Release.Namespace }}
   {{- end }}
+{{- end }}
diff --git a/packages/apps/kubernetes/templates/helmreleases/velero.yaml b/packages/apps/kubernetes/templates/helmreleases/velero.yaml
index ad236d53..781b9c49 100644
--- a/packages/apps/kubernetes/templates/helmreleases/velero.yaml
+++ b/packages/apps/kubernetes/templates/helmreleases/velero.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.addons.velero.enabled }}
+{{- if and .Values.addons.velero.enabled .Values._namespace.etcd }}
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
diff --git a/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler-crds.yaml b/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler-crds.yaml
index a3b7a9b4..55a5faac 100644
--- a/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler-crds.yaml
+++ b/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler-crds.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.addons.monitoringAgents.enabled }}
+{{- if and .Values.addons.monitoringAgents.enabled .Values._namespace.etcd }}
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
diff --git a/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler.yaml b/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler.yaml
index 178df3e3..74fb5a39 100644
--- a/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler.yaml
+++ b/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler.yaml
@@ -24,7 +24,7 @@ vertical-pod-autoscaler:
         memory: 1600Mi
 {{- end }}
 
-{{- if .Values.addons.monitoringAgents.enabled }}
+{{- if and .Values.addons.monitoringAgents.enabled .Values._namespace.etcd }}
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
diff --git a/packages/apps/kubernetes/templates/helmreleases/victoria-metrics-operator.yaml b/packages/apps/kubernetes/templates/helmreleases/victoria-metrics-operator.yaml
index 99744277..7302f8f5 100644
--- a/packages/apps/kubernetes/templates/helmreleases/victoria-metrics-operator.yaml
+++ b/packages/apps/kubernetes/templates/helmreleases/victoria-metrics-operator.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.addons.monitoringAgents.enabled }}
+{{- if and .Values.addons.monitoringAgents.enabled .Values._namespace.etcd }}
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
diff --git a/packages/apps/kubernetes/templates/helmreleases/volumesnapshot-crd.yaml b/packages/apps/kubernetes/templates/helmreleases/volumesnapshot-crd.yaml
index 025f01b7..d50fd93c 100644
--- a/packages/apps/kubernetes/templates/helmreleases/volumesnapshot-crd.yaml
+++ b/packages/apps/kubernetes/templates/helmreleases/volumesnapshot-crd.yaml
@@ -1,3 +1,4 @@
+{{- if .Values._namespace.etcd }}
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
@@ -33,3 +34,4 @@ spec:
   - name: {{ .Release.Name }}
     namespace: {{ .Release.Namespace }}
   {{- end }}
+{{- end }}

From 6072723e1e03c0072fe3bd83e2a5723c50d6ebd8 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Thu, 16 Apr 2026 21:47:19 +0300
Subject: [PATCH 022/151] feat(config): extract + test annotation-timeout
 parser with Flux unit gate
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Pulls the release.cozystack.io/helm-install-timeout parsing out of
start.go into ParseHelmInstallTimeoutAnnotation in pkg/config. The
helper rejects units that time.ParseDuration accepts but Flux
helm-controller rejects (ns, us, µs): feeding one of those through
would cause the HelmRelease admission webhook to reject the object at
install time, giving a silent drop to flux defaults that is hard to
debug. Fail loudly at cozystack-api startup instead.

Adds a table-driven unit test covering: unset (empty), accepted units
ms/s/m/h, compound 2h30m, decimal 1.5m, and the rejected shapes
(bare digits, garbage, negative, ns/us/µs). The test lives in
pkg/config so it runs under the existing go-unit-tests make target.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 pkg/cmd/server/start.go   |  23 ++++----
 pkg/config/config.go      |  42 +++++++++++++-
 pkg/config/config_test.go | 119 ++++++++++++++++++++++++++++++++++++++
 3 files changed, 173 insertions(+), 11 deletions(-)
 create mode 100644 pkg/config/config_test.go

diff --git a/pkg/cmd/server/start.go b/pkg/cmd/server/start.go
index 3e905735..7ebbbc89 100644
--- a/pkg/cmd/server/start.go
+++ b/pkg/cmd/server/start.go
@@ -174,17 +174,20 @@ func (o *CozyServerOptions) Complete() error {
 		// the chart itself depends on (for example, the Kamaji-provisioned
 		// admin-kubeconfig Secret for Kubernetes tenants) need a longer
 		// wait budget than the Flux default. Consumed by the REST storage
-		// layer when building the HelmRelease Spec.
-		if raw, ok := crd.Annotations["release.cozystack.io/helm-install-timeout"]; ok && raw != "" {
-			d, err := time.ParseDuration(raw)
-			if err != nil {
-				return fmt.Errorf(
-					"ApplicationDefinition %q has invalid release.cozystack.io/helm-install-timeout %q: %w",
-					crd.Name, raw, err,
-				)
-			}
-			release.HelmInstallTimeout = d
+		// layer when building the HelmRelease Spec. The parser rejects
+		// units Flux would reject at webhook time, so a bad annotation
+		// surfaces as a loud startup failure instead of a silent drop to
+		// defaults.
+		d, err := config.ParseHelmInstallTimeoutAnnotation(
+			crd.Annotations[config.HelmInstallTimeoutAnnotation],
+		)
+		if err != nil {
+			return fmt.Errorf(
+				"ApplicationDefinition %q has invalid %s annotation: %w",
+				crd.Name, config.HelmInstallTimeoutAnnotation, err,
+			)
 		}
+		release.HelmInstallTimeout = d
 		resource := config.Resource{
 			Application: config.ApplicationConfig{
 				Kind:          crd.Spec.Application.Kind,
diff --git a/pkg/config/config.go b/pkg/config/config.go
index 16cf4f0c..21de27b2 100644
--- a/pkg/config/config.go
+++ b/pkg/config/config.go
@@ -16,7 +16,47 @@ limitations under the License.
 
 package config
 
-import "time"
+import (
+	"fmt"
+	"regexp"
+	"time"
+)
+
+// HelmInstallTimeoutAnnotation is the ApplicationDefinition metadata
+// annotation key that overrides the Flux HelmRelease Install.Timeout and
+// Upgrade.Timeout for a given Application kind.
+const HelmInstallTimeoutAnnotation = "release.cozystack.io/helm-install-timeout"
+
+// helmTimeoutPattern mirrors the CRD validation pattern used by Flux
+// helm-controller on HelmReleaseSpec.Install.Timeout (ms/s/m/h units only).
+// time.ParseDuration accepts ns/us/µs, but Flux rejects them - parsing here
+// with the same shape avoids feeding the controller a value it will later
+// reject at webhook time. See
+// github.com/fluxcd/helm-controller/api/v2 HelmReleaseSpec.Install.Timeout
+// in the go module cache.
+var helmTimeoutPattern = regexp.MustCompile(`^([0-9]+(\.[0-9]+)?(ms|s|m|h))+$`)
+
+// ParseHelmInstallTimeoutAnnotation parses the value of the
+// release.cozystack.io/helm-install-timeout annotation. The empty string is
+// treated as "unset" and returns (0, nil) so callers can leave
+// HelmInstallTimeout zeroed and let flux defaults apply. Values accepted by
+// time.ParseDuration but rejected by Flux (ns/us/µs) return a helpful
+// error instead of silently parsing and failing later at HelmRelease
+// admission.
+func ParseHelmInstallTimeoutAnnotation(raw string) (time.Duration, error) {
+	if raw == "" {
+		return 0, nil
+	}
+	if !helmTimeoutPattern.MatchString(raw) {
+		return 0, fmt.Errorf("must match %s (Flux accepts ms/s/m/h units only), got %q",
+			helmTimeoutPattern, raw)
+	}
+	d, err := time.ParseDuration(raw)
+	if err != nil {
+		return 0, fmt.Errorf("time.ParseDuration(%q): %w", raw, err)
+	}
+	return d, nil
+}
 
 // ResourceConfig represents the structure of the configuration file.
 type ResourceConfig struct {
diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go
new file mode 100644
index 00000000..eb6990af
--- /dev/null
+++ b/pkg/config/config_test.go
@@ -0,0 +1,119 @@
+package config
+
+import (
+	"strings"
+	"testing"
+	"time"
+)
+
+// Cover the annotation parser used by cozystack-api at startup. The parser
+// is consumed by pkg/cmd/server/start.go on every ApplicationDefinition; a
+// typo here silently drops back to flux defaults and the Kubernetes tenant
+// race described in cozystack#2412 reappears, so the table must exercise:
+// - the unset path (empty string treated as "no override"),
+// - every unit Flux accepts (ms, s, m, h),
+// - compound forms (the CRD pattern accepts repeats),
+// - units time.ParseDuration accepts but Flux rejects (ns, us, µs),
+// - outright garbage.
+func TestParseHelmInstallTimeoutAnnotation(t *testing.T) {
+	cases := []struct {
+		name     string
+		input    string
+		want     time.Duration
+		wantErr  bool
+		errMatch string
+	}{
+		{
+			name:  "empty string leaves flux defaults in place",
+			input: "",
+			want:  0,
+		},
+		{
+			name:  "minutes",
+			input: "15m",
+			want:  15 * time.Minute,
+		},
+		{
+			name:  "hours",
+			input: "1h",
+			want:  time.Hour,
+		},
+		{
+			name:  "seconds",
+			input: "45s",
+			want:  45 * time.Second,
+		},
+		{
+			name:  "milliseconds",
+			input: "500ms",
+			want:  500 * time.Millisecond,
+		},
+		{
+			name:  "compound hour and minutes",
+			input: "2h30m",
+			want:  2*time.Hour + 30*time.Minute,
+		},
+		{
+			name:  "decimal minutes",
+			input: "1.5m",
+			want:  90 * time.Second,
+		},
+		{
+			name:     "nanoseconds rejected - Flux CRD pattern excludes ns",
+			input:    "500ns",
+			wantErr:  true,
+			errMatch: "Flux accepts ms/s/m/h units only",
+		},
+		{
+			name:     "microseconds rejected - Flux CRD pattern excludes us",
+			input:    "500us",
+			wantErr:  true,
+			errMatch: "Flux accepts ms/s/m/h units only",
+		},
+		{
+			name:     "microseconds unicode rejected",
+			input:    "500µs",
+			wantErr:  true,
+			errMatch: "Flux accepts ms/s/m/h units only",
+		},
+		{
+			name:     "bare digits rejected",
+			input:    "15",
+			wantErr:  true,
+			errMatch: "Flux accepts ms/s/m/h units only",
+		},
+		{
+			name:     "garbage rejected",
+			input:    "abc",
+			wantErr:  true,
+			errMatch: "Flux accepts ms/s/m/h units only",
+		},
+		{
+			name:     "negative rejected",
+			input:    "-15m",
+			wantErr:  true,
+			errMatch: "Flux accepts ms/s/m/h units only",
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := ParseHelmInstallTimeoutAnnotation(tc.input)
+			if tc.wantErr {
+				if err == nil {
+					t.Fatalf("expected error, got duration=%v", got)
+				}
+				if tc.errMatch != "" && !strings.Contains(err.Error(), tc.errMatch) {
+					t.Errorf("error %q does not contain %q", err.Error(), tc.errMatch)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+			if got != tc.want {
+				t.Errorf("got %v, want %v", got, tc.want)
+			}
+		})
+	}
+}

From b8aec9a9731885cae9217271381680f3e274f46f Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Thu, 16 Apr 2026 21:50:10 +0300
Subject: [PATCH 023/151] fix(kubernetes): history guard non-empty check + nits
 from review

- Log .status.history regardless of content so a silently empty result
  (Flux field rename) shows up in CI logs, and treat empty history on
  a Ready HelmRelease as a distinct failure. A Ready HR by definition
  has at least one snapshot; anything else is a shape-drift signal.
- Replace the unquoted heredoc in remediation-guard.sh with a printf |
  grep pipeline. printf %s treats statuses as literal payload (no $
  expansion surprises for future callers), grep --quiet --extended-regexp
  returns exit status the caller can forward directly.
- Share the etcd-absent values file between both invariant tests
  (packages/apps/kubernetes/tests/values-ci-no-etcd.yaml) instead of
  duplicating the --set block.
- Fix typo "override applied" -> "override is applied" in the
  Kubernetes ApplicationDefinition.
- Add a coupling comment in the ApplicationDefinition annotation that
  points at the wait-for-kubeconfig init deadline in _helpers.tpl, so
  a future operator raising the HR timeout updates the init deadline
  too.
- Clarify the per-annotation timeout comment in rest.go so it stops
  implying the feature is Kubernetes-only (it is not - only today's
  one user is).

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 hack/admin-kubeconfig-invariant.bats          | 16 ++------------
 hack/e2e-apps/remediation-guard.sh            | 18 ++++++++--------
 hack/e2e-apps/run-kubernetes.sh               | 12 +++++++++--
 .../kubernetes/tests/values-ci-no-etcd.yaml   |  9 ++++++++
 .../kubernetes-rd/cozyrds/kubernetes.yaml     |  8 ++++++-
 pkg/registry/apps/application/rest.go         | 21 +++++++++++--------
 6 files changed, 49 insertions(+), 35 deletions(-)
 create mode 100644 packages/apps/kubernetes/tests/values-ci-no-etcd.yaml

diff --git a/hack/admin-kubeconfig-invariant.bats b/hack/admin-kubeconfig-invariant.bats
index 4c809c41..a4b98b00 100644
--- a/hack/admin-kubeconfig-invariant.bats
+++ b/hack/admin-kubeconfig-invariant.bats
@@ -85,13 +85,7 @@
 
   helm template invariant packages/apps/kubernetes \
     --namespace tenant-root \
-    --set _namespace.etcd="" \
-    --set _namespace.monitoring="" \
-    --set _namespace.ingress="" \
-    --set _namespace.seaweedfs="" \
-    --set _namespace.host="" \
-    --set _cluster.cluster-domain=cozy.local \
-    --set 'nodeGroups=null' \
+    --values packages/apps/kubernetes/tests/values-ci-no-etcd.yaml \
     2>/dev/null > "$tmp/rendered.yaml"
 
   matched=$(
@@ -128,13 +122,7 @@
 
   helm template invariant packages/apps/kubernetes \
     --namespace tenant-root \
-    --set _namespace.etcd="" \
-    --set _namespace.monitoring="" \
-    --set _namespace.ingress="" \
-    --set _namespace.seaweedfs="" \
-    --set _namespace.host="" \
-    --set _cluster.cluster-domain=cozy.local \
-    --set 'nodeGroups=null' \
+    --values packages/apps/kubernetes/tests/values-ci-no-etcd.yaml \
     2>/dev/null > "$tmp/rendered.yaml"
 
   matched=$(
diff --git a/hack/e2e-apps/remediation-guard.sh b/hack/e2e-apps/remediation-guard.sh
index 38463310..b45561b9 100644
--- a/hack/e2e-apps/remediation-guard.sh
+++ b/hack/e2e-apps/remediation-guard.sh
@@ -26,14 +26,14 @@ helmrelease_has_remediation_cycle() {
     if [ -z "${statuses}" ]; then
         return 1
     fi
-    while IFS= read -r status; do
-        case "${status}" in
-            failed|uninstalled)
-                return 0
-                ;;
-        esac
-    done <<EOF
-${statuses}
-EOF
+    # printf + grep over the pipe, rather than a heredoc plus while read.
+    # printf %s treats the status string as a literal payload, so any stray
+    # $ in a future caller's input does not trigger shell expansion. grep
+    # returns 0 iff at least one line matches the allowlist, which is
+    # exactly the contract the caller wants, so we can return its exit
+    # status directly.
+    if printf '%s\n' "${statuses}" | grep --extended-regexp --quiet '^(failed|uninstalled)$'; then
+        return 0
+    fi
     return 1
 }
diff --git a/hack/e2e-apps/run-kubernetes.sh b/hack/e2e-apps/run-kubernetes.sh
index 64b1ae90..761ee57d 100644
--- a/hack/e2e-apps/run-kubernetes.sh
+++ b/hack/e2e-apps/run-kubernetes.sh
@@ -336,9 +336,17 @@ EOF
   # types are github.com/fluxcd/helm-controller/api v2 Snapshot.
   history_statuses=$(kubectl get hr -n tenant-test "kubernetes-${test_name}" \
     -ojsonpath='{range .status.history[*]}{.status}{"\n"}{end}')
+  # Always emit the raw value so a silent future-Flux field rename shows
+  # up as "empty history on a Ready HR" in CI logs rather than vanishing.
+  echo "Parent HelmRelease history statuses:"
+  printf '%s\n' "${history_statuses:-<empty>}"
+  if [ -z "${history_statuses}" ]; then
+    echo "Unexpected empty .status.history on a Ready HelmRelease - Flux API shape may have changed." >&2
+    kubectl -n tenant-test describe hr "kubernetes-${test_name}" >&2
+    exit 1
+  fi
   if helmrelease_has_remediation_cycle "${history_statuses}"; then
-    echo "Parent HelmRelease entered remediation cycle. History statuses:" >&2
-    printf '%s\n' "${history_statuses}" >&2
+    echo "Parent HelmRelease entered remediation cycle." >&2
     kubectl -n tenant-test describe hr "kubernetes-${test_name}" >&2
     exit 1
   fi
diff --git a/packages/apps/kubernetes/tests/values-ci-no-etcd.yaml b/packages/apps/kubernetes/tests/values-ci-no-etcd.yaml
new file mode 100644
index 00000000..c7c8196f
--- /dev/null
+++ b/packages/apps/kubernetes/tests/values-ci-no-etcd.yaml
@@ -0,0 +1,9 @@
+_namespace:
+  etcd: ""
+  monitoring: ""
+  ingress: ""
+  seaweedfs: ""
+  host: ""
+_cluster:
+  cluster-domain: cozy.local
+nodeGroups: null
diff --git a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml
index 1128abe8..aaa01247 100644
--- a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml
+++ b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml
@@ -7,8 +7,14 @@ metadata:
     # Secret is provisioned asynchronously. Cold Kamaji start (image pull +
     # etcd + apiserver Ready) plus admin-kubeconfig generation can exceed
     # Flux helm-controller's default wait budget, causing remediation loops
-    # that uninstall the Cluster CR. This override applied by cozystack-api
+    # that uninstall the Cluster CR. This override is applied by cozystack-api
     # to the HelmRelease Spec.Install.Timeout and Spec.Upgrade.Timeout.
+    #
+    # Coupling: the wait-for-kubeconfig init container in
+    # packages/apps/kubernetes/templates/_helpers.tpl hard-codes a 10m
+    # deadline chosen to stay strictly below this value so the pod's
+    # CrashLoopBackOff surfaces before flux remediation fires. If this
+    # annotation is raised, update that init deadline correspondingly.
     release.cozystack.io/helm-install-timeout: "15m"
 spec:
   application:
diff --git a/pkg/registry/apps/application/rest.go b/pkg/registry/apps/application/rest.go
index 406d3738..d233ecc3 100644
--- a/pkg/registry/apps/application/rest.go
+++ b/pkg/registry/apps/application/rest.go
@@ -1528,15 +1528,18 @@ func (r *REST) convertApplicationToHelmRelease(app *appsv1alpha1.Application) (*
 		},
 	}
 
-	// Per-Application HelmRelease wait budget. When an ApplicationDefinition
-	// sets release.cozystack.io/helm-install-timeout, the annotation is
-	// parsed at startup into ReleaseConfig.HelmInstallTimeout and applied
-	// to both Install and Upgrade here. Applications that leave it unset
-	// (the common case) keep flux defaults, so their failed installs
-	// remediate on the normal cadence. Needed for the Kubernetes kind
-	// because its parent chart contains CAPI/Kamaji resources whose
-	// admin-kubeconfig Secret is provisioned asynchronously and Kamaji
-	// cold-start routinely exceeds flux's default wait budget.
+	// Per-Application HelmRelease wait budget. The mechanism is generic:
+	// an ApplicationDefinition that sets
+	// release.cozystack.io/helm-install-timeout gets Install.Timeout and
+	// Upgrade.Timeout populated from ReleaseConfig.HelmInstallTimeout
+	// (parsed at startup). Applications that leave it unset keep flux
+	// defaults so their failed installs remediate on the normal cadence.
+	// Today only kubernetes-rd carries the annotation because the
+	// Kubernetes Application's parent chart contains CAPI/Kamaji
+	// resources whose admin-kubeconfig Secret is provisioned
+	// asynchronously and Kamaji cold-start routinely exceeds flux's
+	// default wait budget; any future kind with the same shape can opt
+	// in by setting the same annotation.
 	if r.releaseConfig.HelmInstallTimeout > 0 {
 		timeout := metav1.Duration{Duration: r.releaseConfig.HelmInstallTimeout}
 		helmRelease.Spec.Install.Timeout = &timeout

From 39b8f0252b6fb07d510c83aacf48fc20d7fdd96c Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Thu, 16 Apr 2026 22:32:24 +0300
Subject: [PATCH 024/151] test(hack): rename remediation-guard bats test to
 match what it pins

The test body asserts .status.history[].status extraction, but the
test name still referenced the old installFailures counter (leftover
from when the guard used that field before switching to status.history
to avoid ClearFailures zeroing the counters on successful reconcile).

Address review feedback from coderabbitai on
hack/remediation-guard.bats:84: rename so grep for what the test
actually pins matches the test name.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 hack/remediation-guard.bats | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hack/remediation-guard.bats b/hack/remediation-guard.bats
index 30e4e04c..092fe06d 100644
--- a/hack/remediation-guard.bats
+++ b/hack/remediation-guard.bats
@@ -81,7 +81,7 @@
     fi
 }
 
-@test "installFailures extraction pins HR v2 status.history shape" {
+@test "status.history extraction pins HR v2 status.history shape" {
     # Pins the Flux HelmRelease v2 .status.history[].status shape that
     # run-kubernetes.sh relies on. If a future flux release renames the
     # field, the jsonpath returns nothing, the guard reports no cycle,

From cb3e0adf6438a2520bf7a943c798da7de8104c42 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Fri, 17 Apr 2026 16:55:42 +0300
Subject: [PATCH 025/151] feat(monitoring): add GPU recording rules
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add VMRule with recording rules for DCGM metrics at three levels:

- gpu.recording.30s: per-GPU aggregates over 30s windows
- gpu.recording.cluster.1m: cluster-wide totals for overview panels
- gpu.recording.namespace.1m: per-namespace aggregates for tenant
  reporting and GPU-hour calculations

The rules are safe to ship on clusters without DCGM — they evaluate to
empty series when no matching metrics are scraped.

Used by dashboards/gpu/gpu-performance.json.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 .../alerts/gpu-recording.rules.yaml           | 57 +++++++++++++++++++
 1 file changed, 57 insertions(+)
 create mode 100644 packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml

diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
new file mode 100644
index 00000000..871a46fc
--- /dev/null
+++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
@@ -0,0 +1,57 @@
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMRule
+metadata:
+  name: alerts-gpu-recording.rules
+spec:
+  groups:
+    - name: gpu.recording.30s
+      interval: 30s
+      rules:
+        - record: gpu:util:avg
+          expr: avg by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_DEV_GPU_UTIL)
+        - record: gpu:mem_copy_util:avg
+          expr: avg by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_DEV_MEM_COPY_UTIL)
+        - record: gpu:fb_used_bytes:max
+          expr: max by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_DEV_FB_USED) * 1048576
+        - record: gpu:fb_free_bytes:max
+          expr: max by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_DEV_FB_FREE) * 1048576
+        - record: gpu:power_watts:avg
+          expr: avg by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_DEV_POWER_USAGE)
+        - record: gpu:temp_celsius:max
+          expr: max by (Hostname, gpu, UUID, modelName) (DCGM_FI_DEV_GPU_TEMP)
+        - record: gpu:tensor_active:avg
+          expr: avg by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_PROF_PIPE_TENSOR_ACTIVE)
+        - record: gpu:gr_engine_active:avg
+          expr: avg by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_PROF_GR_ENGINE_ACTIVE)
+
+    - name: gpu.recording.cluster.1m
+      interval: 1m
+      rules:
+        - record: cluster:gpu_count:total
+          expr: count(group by (UUID) (DCGM_FI_DEV_GPU_UTIL))
+        - record: cluster:gpu_count:allocated
+          expr: count(group by (UUID) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}))
+        - record: cluster:gpu_count:free
+          expr: cluster:gpu_count:total - (cluster:gpu_count:allocated or vector(0))
+        - record: cluster:gpu_util:avg
+          expr: avg(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"})
+        - record: cluster:gpu_power_watts:sum
+          expr: sum(DCGM_FI_DEV_POWER_USAGE)
+
+    - name: gpu.recording.namespace.1m
+      interval: 1m
+      rules:
+        - record: namespace:gpu_count:sum
+          expr: count by (namespace) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"})
+        - record: namespace:gpu_util:avg
+          expr: avg by (namespace) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"})
+        - record: namespace:tensor_active:avg
+          expr: avg by (namespace) (DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace!="", namespace!~"cozy-.*|kube-.*"})
+        - record: namespace:fb_used_bytes:sum
+          expr: sum by (namespace) (DCGM_FI_DEV_FB_USED{namespace!="", namespace!~"cozy-.*|kube-.*"}) * 1048576
+        - record: namespace:power_watts:sum
+          expr: sum by (namespace) (DCGM_FI_DEV_POWER_USAGE{namespace!="", namespace!~"cozy-.*|kube-.*"})
+        - record: namespace:energy_joules:sum
+          expr: sum by (namespace) (DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION{namespace!="", namespace!~"cozy-.*|kube-.*"}) / 1000
+        - record: namespace:gpu_allocated_count:gauge
+          expr: count by (namespace) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"})

From d1d19e9978f965b65ed910b3b6486469e2d07f8d Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Fri, 17 Apr 2026 16:55:51 +0300
Subject: [PATCH 026/151] feat(monitoring): add GPU performance Grafana
 dashboard
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add the gpu/gpu-performance dashboard and register it in the infra
dashboard list. The dashboard provides:

- Cluster overview: total/allocated GPUs, average utilization,
  aggregate power draw.
- Utilization: GPU util (NVML), tensor pipe active (realistic load
  for LLM/AI workloads), graphics engine active, memory copy util.
- Memory: VRAM used/free per GPU.
- Power and temperature per GPU.
- Health: XID errors, power and thermal throttling.

The dashboard relies on DCGM_FI_* metrics plus the cluster:gpu_* and
namespace:gpu_* recording rules added to monitoring-agents.

The JSON follows the cozystack convention — Prometheus data source is
selected via the $ds_prometheus template variable.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 dashboards/gpu/gpu-performance.json           | 935 ++++++++++++++++++
 .../system/monitoring/dashboards-infra.list   |   1 +
 2 files changed, 936 insertions(+)
 create mode 100644 dashboards/gpu/gpu-performance.json

diff --git a/dashboards/gpu/gpu-performance.json b/dashboards/gpu/gpu-performance.json
new file mode 100644
index 00000000..0b458d4e
--- /dev/null
+++ b/dashboards/gpu/gpu-performance.json
@@ -0,0 +1,935 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "liveNow": false,
+  "schemaVersion": 39,
+  "tags": [
+    "gpu",
+    "dcgm"
+  ],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false,
+          "text": "default",
+          "value": "default"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "label": "Prometheus",
+        "multi": false,
+        "name": "ds_prometheus",
+        "options": [],
+        "query": "prometheus",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      },
+      {
+        "current": {
+          "selected": true,
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "$ds_prometheus"
+        },
+        "definition": "label_values(DCGM_FI_DEV_GPU_UTIL, Hostname)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "Host",
+        "multi": true,
+        "name": "Hostname",
+        "options": [],
+        "query": {
+          "query": "label_values(DCGM_FI_DEV_GPU_UTIL, Hostname)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "type": "query"
+      },
+      {
+        "current": {
+          "selected": true,
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "$ds_prometheus"
+        },
+        "definition": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "Namespace",
+        "multi": true,
+        "name": "namespace",
+        "options": [],
+        "query": {
+          "query": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "type": "query"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "GPU Performance",
+  "uid": "gpu-performance",
+  "weekStart": "",
+  "panels": [
+    {
+      "type": "row",
+      "title": "Overview",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "collapsed": false,
+      "id": 1,
+      "panels": []
+    },
+    {
+      "type": "stat",
+      "title": "Total GPUs",
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 0,
+        "y": 1
+      },
+      "id": 2,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "targets": [
+        {
+          "expr": "cluster:gpu_count:total",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "color": {
+            "mode": "thresholds"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "blue"
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value"
+      }
+    },
+    {
+      "type": "stat",
+      "title": "Allocated",
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 6,
+        "y": 1
+      },
+      "id": 3,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "targets": [
+        {
+          "expr": "cluster:gpu_count:allocated",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "color": {
+            "mode": "thresholds"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value"
+      }
+    },
+    {
+      "type": "stat",
+      "title": "Average utilization",
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 12,
+        "y": 1
+      },
+      "id": 4,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "targets": [
+        {
+          "expr": "cluster:gpu_util:avg",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "color": {
+            "mode": "thresholds"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "blue",
+                "value": null
+              },
+              {
+                "color": "green",
+                "value": 30
+              },
+              {
+                "color": "orange",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value"
+      }
+    },
+    {
+      "type": "stat",
+      "title": "Power draw",
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 18,
+        "y": 1
+      },
+      "id": 5,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "targets": [
+        {
+          "expr": "cluster:gpu_power_watts:sum",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "watt",
+          "decimals": 0,
+          "color": {
+            "mode": "thresholds"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green"
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value"
+      }
+    },
+    {
+      "type": "row",
+      "title": "Utilization",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 5
+      },
+      "collapsed": false,
+      "id": 10,
+      "panels": []
+    },
+    {
+      "type": "timeseries",
+      "title": "GPU Utilization (NVML)",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 6
+      },
+      "id": 11,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "targets": [
+        {
+          "expr": "DCGM_FI_DEV_GPU_UTIL{Hostname=~\"$Hostname\", namespace=~\"$namespace\"}",
+          "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "linear",
+            "fillOpacity": 10,
+            "showPoints": "never",
+            "spanNulls": false
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "calcs": [
+            "mean",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Tensor Pipe Active (realistic load for LLM/AI)",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 6
+      },
+      "id": 12,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "targets": [
+        {
+          "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 100",
+          "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "linear",
+            "fillOpacity": 10,
+            "showPoints": "never",
+            "spanNulls": false
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "calcs": [
+            "mean",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Graphics Engine Active",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 14
+      },
+      "id": 13,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "targets": [
+        {
+          "expr": "DCGM_FI_PROF_GR_ENGINE_ACTIVE{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 100",
+          "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "linear",
+            "fillOpacity": 10,
+            "showPoints": "never",
+            "spanNulls": false
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "calcs": [
+            "mean",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Memory Copy Utilization",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 14
+      },
+      "id": 14,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "targets": [
+        {
+          "expr": "DCGM_FI_DEV_MEM_COPY_UTIL{Hostname=~\"$Hostname\", namespace=~\"$namespace\"}",
+          "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "linear",
+            "fillOpacity": 10,
+            "showPoints": "never",
+            "spanNulls": false
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "calcs": [
+            "mean",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "type": "row",
+      "title": "Memory",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 22
+      },
+      "collapsed": false,
+      "id": 20,
+      "panels": []
+    },
+    {
+      "type": "timeseries",
+      "title": "VRAM Used",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 23
+      },
+      "id": 21,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "targets": [
+        {
+          "expr": "DCGM_FI_DEV_FB_USED{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 1048576",
+          "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "linear",
+            "fillOpacity": 25,
+            "showPoints": "never",
+            "stacking": {
+              "mode": "none"
+            }
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "calcs": [
+            "mean",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "VRAM Free",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 23
+      },
+      "id": 22,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "targets": [
+        {
+          "expr": "DCGM_FI_DEV_FB_FREE{Hostname=~\"$Hostname\"} * 1048576",
+          "legendFormat": "{{Hostname}}/{{gpu}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "linear",
+            "fillOpacity": 10,
+            "showPoints": "never"
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "calcs": [
+            "mean",
+            "min"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "type": "row",
+      "title": "Power & Temperature",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 31
+      },
+      "collapsed": false,
+      "id": 30,
+      "panels": []
+    },
+    {
+      "type": "timeseries",
+      "title": "Power Usage per GPU",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 32
+      },
+      "id": 31,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "targets": [
+        {
+          "expr": "DCGM_FI_DEV_POWER_USAGE{Hostname=~\"$Hostname\"}",
+          "legendFormat": "{{Hostname}}/{{gpu}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "watt",
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "linear",
+            "fillOpacity": 10,
+            "showPoints": "never"
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "calcs": [
+            "mean",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "GPU Temperature",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 32
+      },
+      "id": 32,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "targets": [
+        {
+          "expr": "DCGM_FI_DEV_GPU_TEMP{Hostname=~\"$Hostname\"}",
+          "legendFormat": "{{Hostname}}/{{gpu}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "celsius",
+          "min": 20,
+          "max": 100,
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "linear",
+            "fillOpacity": 10,
+            "showPoints": "never"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green"
+              },
+              {
+                "color": "yellow",
+                "value": 75
+              },
+              {
+                "color": "red",
+                "value": 85
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "calcs": [
+            "mean",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "type": "row",
+      "title": "Health",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 40
+      },
+      "collapsed": false,
+      "id": 40,
+      "panels": []
+    },
+    {
+      "type": "stat",
+      "title": "XID errors (latest)",
+      "gridPos": {
+        "h": 5,
+        "w": 8,
+        "x": 0,
+        "y": 41
+      },
+      "id": 41,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "targets": [
+        {
+          "expr": "max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS{Hostname=~\"$Hostname\"})",
+          "legendFormat": "{{Hostname}}/{{gpu}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "color": {
+            "mode": "thresholds"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green"
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Power Violation (\u00b5s/s throttled due to power)",
+      "gridPos": {
+        "h": 5,
+        "w": 8,
+        "x": 8,
+        "y": 41
+      },
+      "id": 42,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "targets": [
+        {
+          "expr": "rate(DCGM_FI_DEV_POWER_VIOLATION{Hostname=~\"$Hostname\"}[5m])",
+          "legendFormat": "{{Hostname}}/{{gpu}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "\u00b5s",
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "linear",
+            "fillOpacity": 10,
+            "showPoints": "never"
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Thermal Violation (\u00b5s/s throttled due to thermals)",
+      "gridPos": {
+        "h": 5,
+        "w": 8,
+        "x": 16,
+        "y": 41
+      },
+      "id": 43,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "targets": [
+        {
+          "expr": "rate(DCGM_FI_DEV_THERMAL_VIOLATION{Hostname=~\"$Hostname\"}[5m])",
+          "legendFormat": "{{Hostname}}/{{gpu}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "\u00b5s",
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "linear",
+            "fillOpacity": 10,
+            "showPoints": "never"
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    }
+  ]
+}
\ No newline at end of file
diff --git a/packages/system/monitoring/dashboards-infra.list b/packages/system/monitoring/dashboards-infra.list
index b5b9f52a..581f9a05 100644
--- a/packages/system/monitoring/dashboards-infra.list
+++ b/packages/system/monitoring/dashboards-infra.list
@@ -32,3 +32,4 @@ hubble/overview
 hubble/dns-namespace
 hubble/l7-http-metrics
 hubble/network-overview
+gpu/gpu-performance

From 5d6654c6f4eb63447d91bbf448e6116317228931 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Fri, 17 Apr 2026 16:56:03 +0300
Subject: [PATCH 027/151] docs(gpu-operator): add native-pod Talos reference
 manifests

Add reference manifests (not templates) under
packages/system/gpu-operator/examples/ documenting one working
configuration for running CUDA workloads directly in pods on a Talos
cluster, with DCGM metrics that drive the gpu/gpu-performance
dashboard.

- values-native-talos.yaml: Cozystack Package values that disable the
  sandbox path, enable the device plugin, and wire DCGM to the custom
  metrics ConfigMap.
- dcgm-custom-metrics.yaml: ConfigMap extending the default DCGM CSV
  with profiling, ECC, throttling and energy counters used by the
  dashboard and recording rules.
- nvidia-driver-compat.yaml: DaemonSet that stages libnvidia-ml.so.1
  and nvidia-smi from the Talos glibc tree into a location the
  gpu-operator validator inspects. Workaround for
  NVIDIA/gpu-operator#1687.
- README.md: explains why these are shipped as references rather than
  first-class templates (sandbox vs native is a deployment choice),
  and how the pieces connect.

The out-of-the-box values-talos.yaml still targets the sandbox (VFIO
passthrough) scenario. Operators who want native pod GPU workloads can
start from these references.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 .../system/gpu-operator/examples/README.md    | 69 ++++++++++++++++
 .../examples/dcgm-custom-metrics.yaml         | 82 +++++++++++++++++++
 .../examples/nvidia-driver-compat.yaml        | 72 ++++++++++++++++
 .../examples/values-native-talos.yaml         | 45 ++++++++++
 4 files changed, 268 insertions(+)
 create mode 100644 packages/system/gpu-operator/examples/README.md
 create mode 100644 packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml
 create mode 100644 packages/system/gpu-operator/examples/nvidia-driver-compat.yaml
 create mode 100644 packages/system/gpu-operator/examples/values-native-talos.yaml

diff --git a/packages/system/gpu-operator/examples/README.md b/packages/system/gpu-operator/examples/README.md
new file mode 100644
index 00000000..dac3703e
--- /dev/null
+++ b/packages/system/gpu-operator/examples/README.md
@@ -0,0 +1,69 @@
+# GPU operator — native pod workload on Talos (reference)
+
+The files in this directory are **not** templates. They are reference
+artifacts that document one working configuration for running GPU
+workloads directly in pods on a Talos-based Cozystack cluster, together
+with the DCGM metrics needed by the `gpu/gpu-performance` Grafana
+dashboard.
+
+The out-of-the-box `values-talos.yaml` for this package targets the
+sandbox (VFIO passthrough to KubeVirt VMs) scenario. The files here
+illustrate an alternative — running CUDA workloads in regular pods with
+the NVIDIA device plugin — and the workarounds it currently requires on
+Talos.
+
+## Files
+
+- [`values-native-talos.yaml`](./values-native-talos.yaml) — Cozystack
+  `Package` values that disable sandbox workloads, enable the device
+  plugin, point `hostPaths.driverInstallDir` at the staging location
+  used by the compat DaemonSet, and wire DCGM to the custom metrics
+  ConfigMap.
+- [`dcgm-custom-metrics.yaml`](./dcgm-custom-metrics.yaml) — `ConfigMap`
+  with a DCGM metrics CSV that adds profiling, ECC, throttling and
+  energy counters on top of the upstream defaults. Required by the
+  recording rules in `packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml`
+  and by several panels in the `gpu/gpu-performance` dashboard.
+- [`nvidia-driver-compat.yaml`](./nvidia-driver-compat.yaml) — DaemonSet
+  that stages `libnvidia-ml.so.1` and `nvidia-smi` from the Talos glibc
+  tree into a path where the NVIDIA GPU Operator validator expects
+  them. See the "Why the compat DaemonSet exists" section below.
+
+## Why these are reference, not templates
+
+Shipping these as first-class templates would silently impose
+assumptions that do not hold for every user:
+
+- Whether the NVIDIA Talos system extension is installed on the nodes.
+- Whether GPUs are exposed directly to pods or passed through to VMs.
+- The exact path the installed driver ends up at (depends on the
+  extension version and Talos release).
+
+The sandbox-oriented `values-talos.yaml` remains the default. Operators
+who want native pod GPU workloads can start from this directory and
+adapt as needed.
+
+## Why the compat DaemonSet exists
+
+The NVIDIA GPU Operator validator checks for `libnvidia-ml.so.1` and
+`bin/nvidia-smi` in the path given by `hostPaths.driverInstallDir`.
+Talos installs them under `/usr/local/glibc/usr/lib/` and
+`/usr/local/bin/`, which the validator does not look at. Until upstream
+addresses [NVIDIA/gpu-operator#1687][1], the DaemonSet copies those
+files into a directory the validator does inspect and creates the
+`.driver-ctr-ready` flag file so the validator proceeds.
+
+[1]: https://github.com/NVIDIA/gpu-operator/issues/1687
+
+## How the dashboard and recording rules fit in
+
+- `dashboards/gpu/gpu-performance.json` expects `DCGM_FI_*` metrics,
+  including profiling series (`DCGM_FI_PROF_PIPE_TENSOR_ACTIVE`,
+  `DCGM_FI_PROF_GR_ENGINE_ACTIVE`) and throttling counters
+  (`DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION`).
+  These are only emitted when DCGM Exporter is started with the custom
+  CSV in `dcgm-custom-metrics.yaml`.
+- `packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml`
+  precomputes cluster-wide and per-namespace aggregations used by the
+  overview panels of the dashboard. The rules are safe to ship on any
+  cluster — they evaluate to empty series when DCGM is not scraped.
diff --git a/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml b/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml
new file mode 100644
index 00000000..14b89b5c
--- /dev/null
+++ b/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml
@@ -0,0 +1,82 @@
+# Custom DCGM Exporter metrics CSV. Referenced from
+# examples/values-native-talos.yaml via dcgmExporter.config.name.
+#
+# Extends the upstream default set with profiling counters, ECC, page
+# retirement, row remap, energy and throttling violations — everything
+# the gpu/gpu-performance dashboard and the GPU recording rules in
+# monitoring-agents expect.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: dcgm-custom-metrics
+  namespace: cozy-gpu-operator
+data:
+  dcgm-metrics.csv: |
+    # Format
+    # If line starts with a '#' it is considered a comment
+    # DCGM FIELD, Prometheus metric type, help message
+
+    # Clocks
+    DCGM_FI_DEV_SM_CLOCK,  gauge, SM clock frequency (in MHz).
+    DCGM_FI_DEV_MEM_CLOCK, gauge, Memory clock frequency (in MHz).
+
+    # Temperature
+    DCGM_FI_DEV_MEMORY_TEMP, gauge, Memory temperature (in C).
+    DCGM_FI_DEV_GPU_TEMP,    gauge, GPU temperature (in C).
+
+    # Power
+    DCGM_FI_DEV_POWER_USAGE,              gauge, Power draw (in W).
+    DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION, counter, Total energy consumption since boot (in mJ).
+
+    # PCIE
+    DCGM_FI_DEV_PCIE_REPLAY_COUNTER, counter, Total number of PCIe retries.
+
+    # Utilization (the sample period varies depending on the product)
+    DCGM_FI_DEV_GPU_UTIL,      gauge, GPU utilization (in %).
+    DCGM_FI_DEV_MEM_COPY_UTIL, gauge, Memory utilization (in %).
+    DCGM_FI_DEV_ENC_UTIL,      gauge, Encoder utilization (in %).
+    DCGM_FI_DEV_DEC_UTIL,      gauge, Decoder utilization (in %).
+
+    # Errors and violations
+    DCGM_FI_DEV_XID_ERRORS, gauge, Value of the last XID error encountered.
+
+    # Memory usage
+    DCGM_FI_DEV_FB_FREE,     gauge, Framebuffer memory free (in MiB).
+    DCGM_FI_DEV_FB_USED,     gauge, Framebuffer memory used (in MiB).
+    DCGM_FI_DEV_FB_RESERVED, gauge, Framebuffer memory reserved (in MiB).
+
+    # ECC (supported on datacenter-class GPUs such as A10)
+    DCGM_FI_DEV_ECC_SBE_VOL_TOTAL, counter, Total number of single-bit volatile ECC errors.
+    DCGM_FI_DEV_ECC_DBE_VOL_TOTAL, counter, Total number of double-bit volatile ECC errors.
+    DCGM_FI_DEV_ECC_SBE_AGG_TOTAL, counter, Total number of single-bit persistent ECC errors.
+    DCGM_FI_DEV_ECC_DBE_AGG_TOTAL, counter, Total number of double-bit persistent ECC errors.
+
+    # Retired pages
+    DCGM_FI_DEV_RETIRED_SBE,     counter, Total number of retired pages due to single-bit errors.
+    DCGM_FI_DEV_RETIRED_DBE,     counter, Total number of retired pages due to double-bit errors.
+    DCGM_FI_DEV_RETIRED_PENDING, counter, Total number of pages pending retirement.
+
+    # Row remapping (not applicable to GDDR6 but left for datacenter GPUs)
+    DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for uncorrectable errors.
+    DCGM_FI_DEV_CORRECTABLE_REMAPPED_ROWS,   counter, Number of remapped rows for correctable errors.
+    DCGM_FI_DEV_ROW_REMAP_FAILURE,           gauge,   Whether remapping of rows has failed.
+
+    # Throttle / violation counters (crucial for SLA and tenant monitoring)
+    DCGM_FI_DEV_POWER_VIOLATION,       counter, Throttling duration due to power constraints (in us).
+    DCGM_FI_DEV_THERMAL_VIOLATION,     counter, Throttling duration due to thermal constraints (in us).
+    DCGM_FI_DEV_SYNC_BOOST_VIOLATION,  counter, Throttling duration due to sync-boost constraints (in us).
+    DCGM_FI_DEV_BOARD_LIMIT_VIOLATION, counter, Throttling duration due to board limit constraints (in us).
+    DCGM_FI_DEV_LOW_UTIL_VIOLATION,    counter, Throttling duration due to low utilization (in us).
+    DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (in us).
+
+    # NVLink — enable only for GPUs that actually have NVLink (A10 has none).
+    # DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL, counter, Total number of NVLink bandwidth counters for all lanes.
+
+    # DCP (profiling) metrics — supported from Ampere onwards.
+    DCGM_FI_PROF_GR_ENGINE_ACTIVE,   gauge, Ratio of time the graphics engine is active.
+    DCGM_FI_PROF_SM_ACTIVE,          gauge, The ratio of cycles an SM has at least 1 warp assigned.
+    DCGM_FI_PROF_SM_OCCUPANCY,       gauge, The ratio of number of warps resident on an SM.
+    DCGM_FI_PROF_PIPE_TENSOR_ACTIVE, gauge, Ratio of cycles the tensor (HMMA) pipe is active.
+    DCGM_FI_PROF_DRAM_ACTIVE,        gauge, Ratio of cycles the device memory interface is active sending or receiving data.
+    DCGM_FI_PROF_PCIE_TX_BYTES,      counter, The number of bytes of active PCIe tx (transmit) data including both header and payload.
+    DCGM_FI_PROF_PCIE_RX_BYTES,      counter, The number of bytes of active PCIe rx (read) data including both header and payload.
diff --git a/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml b/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml
new file mode 100644
index 00000000..e4718e0a
--- /dev/null
+++ b/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml
@@ -0,0 +1,72 @@
+# Workaround for https://github.com/NVIDIA/gpu-operator/issues/1687
+#
+# On Talos, the NVIDIA system extension installs driver files under
+# /usr/local/glibc/usr/lib/ and /usr/local/bin/. The GPU Operator
+# validator only looks for libnvidia-ml.so.1 and bin/nvidia-smi under
+# the path configured as hostPaths.driverInstallDir. This DaemonSet
+# stages the required files into /var/nvidia-driver on each node and
+# creates the .driver-ctr-ready flag so the validator proceeds.
+#
+# Paired with examples/values-native-talos.yaml, which sets
+# hostPaths.driverInstallDir to /var/nvidia-driver.
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: nvidia-driver-compat
+  namespace: cozy-gpu-operator
+  labels:
+    app: nvidia-driver-compat
+spec:
+  selector:
+    matchLabels:
+      app: nvidia-driver-compat
+  template:
+    metadata:
+      labels:
+        app: nvidia-driver-compat
+    spec:
+      hostPID: true
+      priorityClassName: system-node-critical
+      tolerations:
+        - operator: Exists
+      initContainers:
+        - name: create-driver-tree
+          image: busybox:1.37
+          command:
+            - sh
+            - -c
+            - |
+              set -e
+              GLIBC_LIB="/host/usr/local/glibc/usr/lib"
+              SRC_BIN="/host/usr/local/bin"
+              DST="/host/var/nvidia-driver"
+
+              mkdir -p "$DST/bin"
+
+              if [ -f "$GLIBC_LIB/libnvidia-ml.so.1" ]; then
+                cp -f "$GLIBC_LIB/libnvidia-ml.so.1" "$DST/libnvidia-ml.so.1"
+                echo "Copied libnvidia-ml.so.1"
+              fi
+
+              if [ -f "$SRC_BIN/nvidia-smi" ]; then
+                cp -f "$SRC_BIN/nvidia-smi" "$DST/bin/nvidia-smi"
+                chmod +x "$DST/bin/nvidia-smi"
+                echo "Copied nvidia-smi"
+              fi
+
+              mkdir -p /host/run/nvidia/validations
+              touch /host/run/nvidia/validations/.driver-ctr-ready
+              echo "Created driver-ctr-ready flag"
+              echo "Done"
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: host-root
+              mountPath: /host
+      containers:
+        - name: pause
+          image: registry.k8s.io/pause:3.10
+      volumes:
+        - name: host-root
+          hostPath:
+            path: /
diff --git a/packages/system/gpu-operator/examples/values-native-talos.yaml b/packages/system/gpu-operator/examples/values-native-talos.yaml
new file mode 100644
index 00000000..86e436c4
--- /dev/null
+++ b/packages/system/gpu-operator/examples/values-native-talos.yaml
@@ -0,0 +1,45 @@
+# Cozystack Package values for running GPU workloads natively in pods
+# on Talos. This is the counterpart to values-talos.yaml, which targets
+# the sandbox (VFIO passthrough) scenario.
+#
+# Prerequisites:
+#   - NVIDIA Talos system extension installed on GPU nodes.
+#   - examples/nvidia-driver-compat.yaml deployed to stage driver files
+#     where the gpu-operator validator looks for them.
+#   - examples/dcgm-custom-metrics.yaml applied so DCGM Exporter exports
+#     the full set of metrics used by the dashboard and recording rules.
+apiVersion: cozystack.io/v1alpha1
+kind: Package
+metadata:
+  name: cozystack.gpu-operator
+spec:
+  variant: default
+  components:
+    gpu-operator:
+      values:
+        gpu-operator:
+          # The compat DaemonSet stages libnvidia-ml.so.1 and nvidia-smi
+          # under /var/nvidia-driver. Point the validator at that path.
+          hostPaths:
+            driverInstallDir: "/var/nvidia-driver"
+          # Disable the sandbox path — workloads run in pods, not VMs.
+          sandboxWorkloads:
+            enabled: false
+          vfioManager:
+            enabled: false
+          # The Talos extension provides the driver and runtime hooks,
+          # so the operator's own toolkit and driver components must be
+          # switched off to avoid conflicting installations.
+          toolkit:
+            enabled: false
+          devicePlugin:
+            enabled: true
+          # Export full set of DCGM metrics using the ConfigMap in
+          # examples/dcgm-custom-metrics.yaml.
+          dcgmExporter:
+            serviceMonitor:
+              enabled: true
+              interval: "15s"
+              honorLabels: true
+            config:
+              name: dcgm-custom-metrics

From c1b9a06a3647c48df5a8e25327c38c05425504c8 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Fri, 17 Apr 2026 17:08:18 +0300
Subject: [PATCH 028/151] =?UTF-8?q?feat(monitoring):=20expand=20GPU=20dash?=
 =?UTF-8?q?boards=20=E2=80=94=20efficiency=20and=20quotas?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Revise gpu-performance and add two new dashboards, registered in
dashboards-infra.list:

- gpu-efficiency (GPU Efficiency Score) — utilization vs. capacity
  and workload efficiency signals.
- gpu-quotas (GPU Quotas & Allocation) — per-namespace requested vs.
  used GPUs for tenant capacity planning.

All three dashboards use the $ds_prometheus template variable, per
the project convention.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 dashboards/gpu/gpu-efficiency.json            | 1131 ++++++++++++
 dashboards/gpu/gpu-performance.json           | 1523 +++++++++++------
 dashboards/gpu/gpu-quotas.json                |  822 +++++++++
 .../system/monitoring/dashboards-infra.list   |    2 +
 4 files changed, 2956 insertions(+), 522 deletions(-)
 create mode 100644 dashboards/gpu/gpu-efficiency.json
 create mode 100644 dashboards/gpu/gpu-quotas.json

diff --git a/dashboards/gpu/gpu-efficiency.json b/dashboards/gpu/gpu-efficiency.json
new file mode 100644
index 00000000..cc2bca6c
--- /dev/null
+++ b/dashboards/gpu/gpu-efficiency.json
@@ -0,0 +1,1131 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_VM-SHORTTERM",
+      "label": "vm-shortterm",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "prometheus",
+      "pluginName": "Prometheus"
+    }
+  ],
+  "__elements": {},
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "11.4.0"
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "1.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "stat",
+      "name": "Stat",
+      "version": ""
+    },
+    {
+      "type": "panel",
+      "id": "table",
+      "name": "Table",
+      "version": ""
+    },
+    {
+      "type": "panel",
+      "id": "timeseries",
+      "name": "Time series",
+      "version": ""
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "description": "Tensor saturation, util/watt and throttling \u2014 reveals inefficient GPU workloads",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "panels": [],
+      "title": "\u041e\u0431\u0449\u0438\u0435 \u043c\u0435\u0442\u0440\u0438\u043a\u0438 \u044d\u0444\u0444\u0435\u043a\u0442\u0438\u0432\u043d\u043e\u0441\u0442\u0438",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${ds_prometheus}"
+      },
+      "description": "\u0421\u0440\u0435\u0434\u043d\u044f\u044f \u0437\u0430\u0433\u0440\u0443\u0437\u043a\u0430 \u0442\u0435\u043d\u0437\u043e\u0440\u043d\u044b\u0445 \u044f\u0434\u0435\u0440. <10% \u2014 GPU \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u044e\u0442\u0441\u044f \u043d\u0435\u044d\u0444\u0444\u0435\u043a\u0442\u0438\u0432\u043d\u043e (\u043a\u043b\u0438\u0435\u043d\u0442\u044b \u043c\u043e\u0433\u043b\u0438 \u0431\u044b \u043f\u0435\u0440\u0435\u0435\u0445\u0430\u0442\u044c \u043d\u0430 CPU \u0438\u043b\u0438 \u043e\u043f\u0442\u0438\u043c\u0438\u0437\u0438\u0440\u043e\u0432\u0430\u0442\u044c \u043a\u043e\u0434).",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 10
+              },
+              {
+                "color": "yellow",
+                "value": 30
+              },
+              {
+                "color": "green",
+                "value": 60
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 8,
+        "x": 0,
+        "y": 1
+      },
+      "id": 2,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "percentChangeColorMode": "standard",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "value",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.4.0",
+      "targets": [
+        {
+          "expr": "avg(pod:tensor_saturation:avg5m)",
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${ds_prometheus}"
+          }
+        }
+      ],
+      "title": "Cluster Tensor Saturation",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_VM-SHORTTERM}"
+      },
+      "description": "NVML utilization % \u043d\u0430 \u043a\u0430\u0436\u0434\u044b\u0439 \u0432\u0430\u0442\u0442. \u0412\u044b\u0441\u043e\u043a\u043e\u0435 \u0437\u043d\u0430\u0447\u0435\u043d\u0438\u0435 = \u044d\u0444\u0444\u0435\u043a\u0442\u0438\u0432\u043d\u0430\u044f \u043d\u0430\u0433\u0440\u0443\u0437\u043a\u0430.",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "decimals": 3,
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 0.5
+              },
+              {
+                "color": "green",
+                "value": 1
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 8,
+        "x": 8,
+        "y": 1
+      },
+      "id": 3,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "percentChangeColorMode": "standard",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "value",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.4.0",
+      "targets": [
+        {
+          "expr": "avg(pod:util_per_watt:avg5m)",
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_VM-SHORTTERM}"
+          }
+        }
+      ],
+      "title": "Avg Utilization per Watt",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${ds_prometheus}"
+      },
+      "description": "\u0414\u043e\u043b\u044f \u0432\u0440\u0435\u043c\u0435\u043d\u0438, \u043a\u043e\u0433\u0434\u0430 GPU \u0443\u043f\u0438\u0440\u0430\u044e\u0442\u0441\u044f \u0432 TDP \u0438 \u0442\u0435\u0440\u044f\u044e\u0442 \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c. >5% \u2014 \u043a\u043b\u0438\u0435\u043d\u0442\u044b \u043d\u0435\u0434\u043e\u043f\u043e\u043b\u0443\u0447\u0430\u044e\u0442 FLOPS.",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "decimals": 2,
+          "mappings": [],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 5
+              },
+              {
+                "color": "red",
+                "value": 20
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 8,
+        "x": 16,
+        "y": 1
+      },
+      "id": 4,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "percentChangeColorMode": "standard",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "value",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.4.0",
+      "targets": [
+        {
+          "expr": "avg(gpu:power_throttle_fraction:rate5m) * 100",
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${ds_prometheus}"
+          }
+        }
+      ],
+      "title": "Avg Power Throttling",
+      "type": "stat"
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 6
+      },
+      "id": 10,
+      "panels": [],
+      "title": "NVML vs Tensor (\u043a\u0442\u043e \u043e\u0431\u043c\u0430\u043d\u044b\u0432\u0430\u0435\u0442\u0441\u044f)",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_VM-SHORTTERM}"
+      },
+      "description": "\u041a\u043b\u0430\u0441\u0441\u0438\u0447\u0435\u0441\u043a\u0430\u044f \u043c\u0435\u0442\u0440\u0438\u043a\u0430 \u0443\u0442\u0438\u043b\u0438\u0437\u0430\u0446\u0438\u0438. \u041f\u043e\u043a\u0430\u0437\u044b\u0432\u0430\u0435\u0442 \u0437\u0430\u043d\u044f\u0442\u043e\u0441\u0442\u044c \u043b\u044e\u0431\u043e\u0433\u043e \u0434\u0432\u0438\u0436\u043a\u0430 (SM, copy, encoder).",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 7
+      },
+      "id": 11,
+      "options": {
+        "legend": {
+          "calcs": [
+            "mean",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.4.0",
+      "targets": [
+        {
+          "expr": "DCGM_FI_DEV_GPU_UTIL{namespace=~\"$namespace\", namespace!=\"\"}",
+          "legendFormat": "{{namespace}}/{{pod}}",
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_VM-SHORTTERM}"
+          }
+        }
+      ],
+      "title": "NVML GPU Utilization",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${ds_prometheus}"
+      },
+      "description": "\u0420\u0435\u0430\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u0433\u0440\u0443\u0437\u043a\u0430 \u0442\u0435\u043d\u0437\u043e\u0440\u043d\u044b\u0445 \u044f\u0434\u0435\u0440 (HMMA). \u0414\u043b\u044f AI/LLM \u0438\u043d\u0444\u0435\u0440\u0435\u043d\u0441\u0430 \u0434\u043e\u043b\u0436\u043d\u0430 \u0431\u044b\u0442\u044c \u226520%, \u0438\u043d\u0430\u0447\u0435 workload \u043d\u0435 \u043e\u043f\u0442\u0438\u043c\u0438\u0437\u0438\u0440\u043e\u0432\u0430\u043d.",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 7
+      },
+      "id": 12,
+      "options": {
+        "legend": {
+          "calcs": [
+            "mean",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.4.0",
+      "targets": [
+        {
+          "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace=~\"$namespace\", namespace!=\"\"} * 100",
+          "legendFormat": "{{namespace}}/{{pod}}",
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${ds_prometheus}"
+          }
+        }
+      ],
+      "title": "Tensor Pipe Active",
+      "type": "timeseries"
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 15
+      },
+      "id": 20,
+      "panels": [],
+      "title": "Per-pod \u0440\u0435\u0439\u0442\u0438\u043d\u0433",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_VM-SHORTTERM}"
+      },
+      "description": "\u041a\u0442\u043e \u0436\u043c\u0451\u0442 \u0442\u0435\u043d\u0437\u043e\u0440\u043d\u044b\u0435 \u044f\u0434\u0440\u0430, \u0430 \u043a\u0442\u043e \u043d\u0435\u0442. \u0421\u043e\u0440\u0442\u0438\u0440\u043e\u0432\u043a\u0430 \u043e\u0442 \u043c\u0430\u043a\u0441\u0438\u043c\u0443\u043c\u0430.",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "align": "auto",
+            "cellOptions": {
+              "type": "auto"
+            },
+            "inspect": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Saturation"
+            },
+            "properties": [
+              {
+                "id": "unit",
+                "value": "percent"
+              },
+              {
+                "id": "min",
+                "value": 0
+              },
+              {
+                "id": "max",
+                "value": 100
+              },
+              {
+                "id": "custom.cellOptions",
+                "value": {
+                  "mode": "gradient",
+                  "type": "gauge"
+                }
+              },
+              {
+                "id": "thresholds",
+                "value": {
+                  "mode": "absolute",
+                  "steps": [
+                    {
+                      "color": "red",
+                      "value": null
+                    },
+                    {
+                      "color": "orange",
+                      "value": 10
+                    },
+                    {
+                      "color": "yellow",
+                      "value": 30
+                    },
+                    {
+                      "color": "green",
+                      "value": 60
+                    }
+                  ]
+                }
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "id": 21,
+      "options": {
+        "cellHeight": "sm",
+        "footer": {
+          "countRows": false,
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
+        "showHeader": true,
+        "sortBy": [
+          {
+            "desc": true,
+            "displayName": "Saturation"
+          }
+        ]
+      },
+      "pluginVersion": "11.4.0",
+      "targets": [
+        {
+          "expr": "topk(20, pod:tensor_saturation:avg5m)",
+          "format": "table",
+          "instant": true,
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_VM-SHORTTERM}"
+          }
+        }
+      ],
+      "title": "Tensor Saturation per pod (5m avg)",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Hostname": true,
+              "Time": true,
+              "UUID": true,
+              "__name__": true,
+              "cluster": true,
+              "container": true,
+              "endpoint": true,
+              "gpu": true,
+              "instance": true,
+              "job": true,
+              "modelName": true,
+              "prometheus": true,
+              "service": true,
+              "tenant": true,
+              "tier": true,
+              "uid": true,
+              "unit": true
+            },
+            "indexByName": {
+              "Value": 2,
+              "namespace": 0,
+              "pod": 1
+            },
+            "renameByName": {
+              "Value": "Saturation",
+              "namespace": "Namespace",
+              "pod": "Pod"
+            }
+          }
+        }
+      ],
+      "type": "table"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${ds_prometheus}"
+      },
+      "description": "\u041d\u0430\u0441\u043a\u043e\u043b\u044c\u043a\u043e \u044d\u0444\u0444\u0435\u043a\u0442\u0438\u0432\u043d\u043e \u043a\u0430\u0436\u0434\u044b\u0439 \u043f\u043e\u0434 \u0442\u0440\u0430\u0442\u0438\u0442 \u0432\u0430\u0442\u0442\u044b. \u041d\u0438\u0437\u043a\u043e\u0435 \u0437\u043d\u0430\u0447\u0435\u043d\u0438\u0435 = \u043f\u043b\u043e\u0445\u0430\u044f \u043e\u043f\u0442\u0438\u043c\u0438\u0437\u0430\u0446\u0438\u044f.",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "align": "auto",
+            "cellOptions": {
+              "type": "auto"
+            },
+            "inspect": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Util/Watt"
+            },
+            "properties": [
+              {
+                "id": "unit",
+                "value": "none"
+              },
+              {
+                "id": "decimals",
+                "value": 3
+              },
+              {
+                "id": "min",
+                "value": 0
+              },
+              {
+                "id": "max",
+                "value": 1.5
+              },
+              {
+                "id": "custom.cellOptions",
+                "value": {
+                  "mode": "gradient",
+                  "type": "gauge"
+                }
+              },
+              {
+                "id": "thresholds",
+                "value": {
+                  "mode": "absolute",
+                  "steps": [
+                    {
+                      "color": "red",
+                      "value": null
+                    },
+                    {
+                      "color": "yellow",
+                      "value": 0.5
+                    },
+                    {
+                      "color": "green",
+                      "value": 1
+                    }
+                  ]
+                }
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "id": 22,
+      "options": {
+        "cellHeight": "sm",
+        "footer": {
+          "countRows": false,
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
+        "showHeader": true,
+        "sortBy": [
+          {
+            "desc": true,
+            "displayName": "Util/Watt"
+          }
+        ]
+      },
+      "pluginVersion": "11.4.0",
+      "targets": [
+        {
+          "expr": "topk(20, pod:util_per_watt:avg5m)",
+          "format": "table",
+          "instant": true,
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${ds_prometheus}"
+          }
+        }
+      ],
+      "title": "Utilization / Watt per pod (5m avg)",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Hostname": true,
+              "Time": true,
+              "UUID": true,
+              "__name__": true,
+              "cluster": true,
+              "container": true,
+              "endpoint": true,
+              "gpu": true,
+              "instance": true,
+              "job": true,
+              "modelName": true,
+              "prometheus": true,
+              "service": true,
+              "tenant": true,
+              "tier": true,
+              "uid": true,
+              "unit": true
+            },
+            "indexByName": {
+              "Value": 2,
+              "namespace": 0,
+              "pod": 1
+            },
+            "renameByName": {
+              "Value": "Util/Watt",
+              "namespace": "Namespace",
+              "pod": "Pod"
+            }
+          }
+        }
+      ],
+      "type": "table"
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 24
+      },
+      "id": 30,
+      "panels": [],
+      "title": "Throttling",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_VM-SHORTTERM}"
+      },
+      "description": "\u0414\u043e\u043b\u044f \u0432\u0440\u0435\u043c\u0435\u043d\u0438, \u043a\u043e\u0433\u0434\u0430 GPU \u0434\u0440\u043e\u0441\u0441\u0435\u043b\u0438\u0440\u043e\u0432\u0430\u043d\u0430 \u043f\u043e \u043f\u0438\u0442\u0430\u043d\u0438\u044e. 1.0 = \u043f\u043e\u0441\u0442\u043e\u044f\u043d\u043d\u043e.",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 25,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "max": 1,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 0.05
+              },
+              {
+                "color": "red",
+                "value": 0.2
+              }
+            ]
+          },
+          "unit": "percentunit"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 25
+      },
+      "id": 31,
+      "options": {
+        "legend": {
+          "calcs": [
+            "mean",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.4.0",
+      "targets": [
+        {
+          "expr": "gpu:power_throttle_fraction:rate5m",
+          "legendFormat": "{{Hostname}}/{{gpu}}",
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_VM-SHORTTERM}"
+          }
+        }
+      ],
+      "title": "Power throttle fraction per GPU",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${ds_prometheus}"
+      },
+      "description": "\u0414\u043e\u043b\u044f \u0432\u0440\u0435\u043c\u0435\u043d\u0438, \u043a\u043e\u0433\u0434\u0430 GPU \u0434\u0440\u043e\u0441\u0441\u0435\u043b\u0438\u0440\u043e\u0432\u0430\u043d\u0430 \u043f\u043e \u0442\u0435\u043c\u043f\u0435\u0440\u0430\u0442\u0443\u0440\u0435.",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 25,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "max": 1,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 0.05
+              },
+              {
+                "color": "red",
+                "value": 0.2
+              }
+            ]
+          },
+          "unit": "percentunit"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 25
+      },
+      "id": 32,
+      "options": {
+        "legend": {
+          "calcs": [
+            "mean",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.4.0",
+      "targets": [
+        {
+          "expr": "gpu:thermal_throttle_fraction:rate5m",
+          "legendFormat": "{{Hostname}}/{{gpu}}",
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${ds_prometheus}"
+          }
+        }
+      ],
+      "title": "Thermal throttle fraction per GPU",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "",
+  "schemaVersion": 40,
+  "tags": [
+    "gpu",
+    "efficiency",
+    "finops"
+  ],
+  "templating": {
+    "list": [
+      {
+        "current": {},
+        "includeAll": false,
+        "label": "Datasource",
+        "name": "ds_prometheus",
+        "options": [],
+        "query": "prometheus",
+        "refresh": 1,
+        "regex": "vm-.*",
+        "type": "datasource"
+      },
+      {
+        "current": {},
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_VM-SHORTTERM}"
+        },
+        "definition": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)",
+        "includeAll": true,
+        "label": "Namespace",
+        "multi": true,
+        "name": "namespace",
+        "options": [],
+        "query": {
+          "query": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 2,
+        "regex": "",
+        "sort": 1,
+        "type": "query"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "GPU Efficiency Score",
+  "uid": "gpu-efficiency",
+  "version": 2,
+  "weekStart": ""
+}
\ No newline at end of file
diff --git a/dashboards/gpu/gpu-performance.json b/dashboards/gpu/gpu-performance.json
index 0b458d4e..906d6a11 100644
--- a/dashboards/gpu/gpu-performance.json
+++ b/dashboards/gpu/gpu-performance.json
@@ -1,120 +1,100 @@
 {
+  "__inputs": [
+    {
+      "name": "DS_VM-SHORTTERM",
+      "label": "vm-shortterm",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "prometheus",
+      "pluginName": "Prometheus"
+    }
+  ],
+  "__elements": {},
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "11.4.0"
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "1.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "stat",
+      "name": "Stat",
+      "version": ""
+    },
+    {
+      "type": "panel",
+      "id": "timeseries",
+      "name": "Time series",
+      "version": ""
+    }
+  ],
   "annotations": {
-    "list": []
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
   },
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 1,
   "id": null,
   "links": [],
-  "liveNow": false,
-  "schemaVersion": 39,
-  "tags": [
-    "gpu",
-    "dcgm"
-  ],
-  "templating": {
-    "list": [
-      {
-        "current": {
-          "selected": false,
-          "text": "default",
-          "value": "default"
-        },
-        "hide": 0,
-        "includeAll": false,
-        "label": "Prometheus",
-        "multi": false,
-        "name": "ds_prometheus",
-        "options": [],
-        "query": "prometheus",
-        "refresh": 1,
-        "regex": "",
-        "skipUrlSync": false,
-        "type": "datasource"
-      },
-      {
-        "current": {
-          "selected": true,
-          "text": "All",
-          "value": "$__all"
-        },
-        "datasource": {
-          "type": "prometheus",
-          "uid": "$ds_prometheus"
-        },
-        "definition": "label_values(DCGM_FI_DEV_GPU_UTIL, Hostname)",
-        "hide": 0,
-        "includeAll": true,
-        "label": "Host",
-        "multi": true,
-        "name": "Hostname",
-        "options": [],
-        "query": {
-          "query": "label_values(DCGM_FI_DEV_GPU_UTIL, Hostname)",
-          "refId": "StandardVariableQuery"
-        },
-        "refresh": 2,
-        "regex": "",
-        "skipUrlSync": false,
-        "sort": 1,
-        "type": "query"
-      },
-      {
-        "current": {
-          "selected": true,
-          "text": "All",
-          "value": "$__all"
-        },
-        "datasource": {
-          "type": "prometheus",
-          "uid": "$ds_prometheus"
-        },
-        "definition": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)",
-        "hide": 0,
-        "includeAll": true,
-        "label": "Namespace",
-        "multi": true,
-        "name": "namespace",
-        "options": [],
-        "query": {
-          "query": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)",
-          "refId": "StandardVariableQuery"
-        },
-        "refresh": 2,
-        "regex": "",
-        "skipUrlSync": false,
-        "sort": 1,
-        "type": "query"
-      }
-    ]
-  },
-  "time": {
-    "from": "now-1h",
-    "to": "now"
-  },
-  "timepicker": {},
-  "timezone": "",
-  "title": "GPU Performance",
-  "uid": "gpu-performance",
-  "weekStart": "",
   "panels": [
     {
-      "type": "row",
-      "title": "Overview",
+      "collapsed": false,
       "gridPos": {
         "h": 1,
         "w": 24,
         "x": 0,
         "y": 0
       },
-      "collapsed": false,
       "id": 1,
-      "panels": []
+      "panels": [],
+      "title": "Overview",
+      "type": "row"
     },
     {
-      "type": "stat",
-      "title": "Total GPUs",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "blue",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
       "gridPos": {
         "h": 4,
         "w": 6,
@@ -122,36 +102,12 @@
         "y": 1
       },
       "id": 2,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "targets": [
-        {
-          "expr": "cluster:gpu_count:total",
-          "refId": "A"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "unit": "short",
-          "color": {
-            "mode": "thresholds"
-          },
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "blue"
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
       "options": {
         "colorMode": "value",
         "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
@@ -159,35 +115,35 @@
           "fields": "",
           "values": false
         },
-        "textMode": "value"
-      }
-    },
-    {
-      "type": "stat",
-      "title": "Allocated",
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 6,
-        "y": 1
-      },
-      "id": 3,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
+        "showPercentChange": false,
+        "textMode": "value",
+        "wideLayout": true
       },
+      "pluginVersion": "11.4.0",
       "targets": [
         {
-          "expr": "cluster:gpu_count:allocated",
-          "refId": "A"
+          "expr": "cluster:gpu_count:total",
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds_prometheus"
+          }
         }
       ],
+      "title": "Total GPUs",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_VM-SHORTTERM}"
+      },
       "fieldConfig": {
         "defaults": {
-          "unit": "short",
           "color": {
             "mode": "thresholds"
           },
+          "mappings": [],
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -200,13 +156,24 @@
                 "value": 1
               }
             ]
-          }
+          },
+          "unit": "short"
         },
         "overrides": []
       },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 6,
+        "y": 1
+      },
+      "id": 3,
       "options": {
         "colorMode": "value",
         "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
@@ -214,37 +181,37 @@
           "fields": "",
           "values": false
         },
-        "textMode": "value"
-      }
+        "showPercentChange": false,
+        "textMode": "value",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.4.0",
+      "targets": [
+        {
+          "expr": "cluster:gpu_count:allocated",
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_VM-SHORTTERM}"
+          }
+        }
+      ],
+      "title": "Allocated",
+      "type": "stat"
     },
     {
-      "type": "stat",
-      "title": "Average utilization",
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 12,
-        "y": 1
-      },
-      "id": 4,
       "datasource": {
         "type": "prometheus",
         "uid": "$ds_prometheus"
       },
-      "targets": [
-        {
-          "expr": "cluster:gpu_util:avg",
-          "refId": "A"
-        }
-      ],
       "fieldConfig": {
         "defaults": {
-          "unit": "percent",
-          "min": 0,
-          "max": 100,
           "color": {
             "mode": "thresholds"
           },
+          "mappings": [],
+          "max": 100,
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -261,13 +228,24 @@
                 "value": 80
               }
             ]
-          }
+          },
+          "unit": "percent"
         },
         "overrides": []
       },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 12,
+        "y": 1
+      },
+      "id": 4,
       "options": {
         "colorMode": "value",
         "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
@@ -275,12 +253,49 @@
           "fields": "",
           "values": false
         },
-        "textMode": "value"
-      }
+        "showPercentChange": false,
+        "textMode": "value",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.4.0",
+      "targets": [
+        {
+          "expr": "cluster:gpu_util:avg",
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds_prometheus"
+          }
+        }
+      ],
+      "title": "Average utilization",
+      "type": "stat"
     },
     {
-      "type": "stat",
-      "title": "Power draw",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_VM-SHORTTERM}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "decimals": 0,
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "watt"
+        },
+        "overrides": []
+      },
       "gridPos": {
         "h": 4,
         "w": 6,
@@ -288,37 +303,12 @@
         "y": 1
       },
       "id": 5,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "targets": [
-        {
-          "expr": "cluster:gpu_power_watts:sum",
-          "refId": "A"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "unit": "watt",
-          "decimals": 0,
-          "color": {
-            "mode": "thresholds"
-          },
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green"
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
       "options": {
         "colorMode": "value",
         "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
@@ -326,25 +316,100 @@
           "fields": "",
           "values": false
         },
-        "textMode": "value"
-      }
+        "showPercentChange": false,
+        "textMode": "value",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.4.0",
+      "targets": [
+        {
+          "expr": "cluster:gpu_power_watts:sum",
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_VM-SHORTTERM}"
+          }
+        }
+      ],
+      "title": "Power draw",
+      "type": "stat"
     },
     {
-      "type": "row",
-      "title": "Utilization",
+      "collapsed": false,
       "gridPos": {
         "h": 1,
         "w": 24,
         "x": 0,
         "y": 5
       },
-      "collapsed": false,
       "id": 10,
-      "panels": []
+      "panels": [],
+      "title": "Utilization",
+      "type": "row"
     },
     {
-      "type": "timeseries",
-      "title": "GPU Utilization (NVML)",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
       "gridPos": {
         "h": 8,
         "w": 12,
@@ -352,49 +417,99 @@
         "y": 6
       },
       "id": 11,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
+      "options": {
+        "legend": {
+          "calcs": [
+            "mean",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
       },
+      "pluginVersion": "11.4.0",
       "targets": [
         {
           "expr": "DCGM_FI_DEV_GPU_UTIL{Hostname=~\"$Hostname\", namespace=~\"$namespace\"}",
           "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}",
-          "refId": "A"
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds_prometheus"
+          }
         }
       ],
+      "title": "GPU Utilization (NVML)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_VM-SHORTTERM}"
+      },
       "fieldConfig": {
         "defaults": {
-          "unit": "percent",
-          "min": 0,
-          "max": 100,
+          "color": {
+            "mode": "palette-classic"
+          },
           "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
-            "lineInterpolation": "linear",
             "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
             "showPoints": "never",
-            "spanNulls": false
-          }
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percent"
         },
         "overrides": []
       },
-      "options": {
-        "legend": {
-          "displayMode": "table",
-          "placement": "bottom",
-          "calcs": [
-            "mean",
-            "max"
-          ]
-        },
-        "tooltip": {
-          "mode": "multi"
-        }
-      }
-    },
-    {
-      "type": "timeseries",
-      "title": "Tensor Pipe Active (realistic load for LLM/AI)",
       "gridPos": {
         "h": 8,
         "w": 12,
@@ -402,49 +517,99 @@
         "y": 6
       },
       "id": 12,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
+      "options": {
+        "legend": {
+          "calcs": [
+            "mean",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
       },
+      "pluginVersion": "11.4.0",
       "targets": [
         {
           "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 100",
           "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}",
-          "refId": "A"
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_VM-SHORTTERM}"
+          }
         }
       ],
+      "title": "Tensor Pipe Active (realistic load for LLM/AI)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
       "fieldConfig": {
         "defaults": {
-          "unit": "percent",
-          "min": 0,
-          "max": 100,
+          "color": {
+            "mode": "palette-classic"
+          },
           "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
-            "lineInterpolation": "linear",
             "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
             "showPoints": "never",
-            "spanNulls": false
-          }
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percent"
         },
         "overrides": []
       },
-      "options": {
-        "legend": {
-          "displayMode": "table",
-          "placement": "bottom",
-          "calcs": [
-            "mean",
-            "max"
-          ]
-        },
-        "tooltip": {
-          "mode": "multi"
-        }
-      }
-    },
-    {
-      "type": "timeseries",
-      "title": "Graphics Engine Active",
       "gridPos": {
         "h": 8,
         "w": 12,
@@ -452,49 +617,99 @@
         "y": 14
       },
       "id": 13,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
+      "options": {
+        "legend": {
+          "calcs": [
+            "mean",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
       },
+      "pluginVersion": "11.4.0",
       "targets": [
         {
           "expr": "DCGM_FI_PROF_GR_ENGINE_ACTIVE{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 100",
           "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}",
-          "refId": "A"
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds_prometheus"
+          }
         }
       ],
+      "title": "Graphics Engine Active",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_VM-SHORTTERM}"
+      },
       "fieldConfig": {
         "defaults": {
-          "unit": "percent",
-          "min": 0,
-          "max": 100,
+          "color": {
+            "mode": "palette-classic"
+          },
           "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
-            "lineInterpolation": "linear",
             "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
             "showPoints": "never",
-            "spanNulls": false
-          }
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percent"
         },
         "overrides": []
       },
-      "options": {
-        "legend": {
-          "displayMode": "table",
-          "placement": "bottom",
-          "calcs": [
-            "mean",
-            "max"
-          ]
-        },
-        "tooltip": {
-          "mode": "multi"
-        }
-      }
-    },
-    {
-      "type": "timeseries",
-      "title": "Memory Copy Utilization",
       "gridPos": {
         "h": 8,
         "w": 12,
@@ -502,62 +717,110 @@
         "y": 14
       },
       "id": 14,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
+      "options": {
+        "legend": {
+          "calcs": [
+            "mean",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
       },
+      "pluginVersion": "11.4.0",
       "targets": [
         {
           "expr": "DCGM_FI_DEV_MEM_COPY_UTIL{Hostname=~\"$Hostname\", namespace=~\"$namespace\"}",
           "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}",
-          "refId": "A"
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_VM-SHORTTERM}"
+          }
         }
       ],
-      "fieldConfig": {
-        "defaults": {
-          "unit": "percent",
-          "min": 0,
-          "max": 100,
-          "custom": {
-            "drawStyle": "line",
-            "lineInterpolation": "linear",
-            "fillOpacity": 10,
-            "showPoints": "never",
-            "spanNulls": false
-          }
-        },
-        "overrides": []
-      },
-      "options": {
-        "legend": {
-          "displayMode": "table",
-          "placement": "bottom",
-          "calcs": [
-            "mean",
-            "max"
-          ]
-        },
-        "tooltip": {
-          "mode": "multi"
-        }
-      }
+      "title": "Memory Copy Utilization",
+      "type": "timeseries"
     },
     {
-      "type": "row",
-      "title": "Memory",
+      "collapsed": false,
       "gridPos": {
         "h": 1,
         "w": 24,
         "x": 0,
         "y": 22
       },
-      "collapsed": false,
       "id": 20,
-      "panels": []
+      "panels": [],
+      "title": "Memory",
+      "type": "row"
     },
     {
-      "type": "timeseries",
-      "title": "VRAM Used",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 25,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "bytes"
+        },
+        "overrides": []
+      },
       "gridPos": {
         "h": 8,
         "w": 12,
@@ -565,49 +828,97 @@
         "y": 23
       },
       "id": 21,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
+      "options": {
+        "legend": {
+          "calcs": [
+            "mean",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
       },
+      "pluginVersion": "11.4.0",
       "targets": [
         {
           "expr": "DCGM_FI_DEV_FB_USED{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 1048576",
           "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}",
-          "refId": "A"
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds_prometheus"
+          }
         }
       ],
+      "title": "VRAM Used",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_VM-SHORTTERM}"
+      },
       "fieldConfig": {
         "defaults": {
-          "unit": "bytes",
+          "color": {
+            "mode": "palette-classic"
+          },
           "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
             "lineInterpolation": "linear",
-            "fillOpacity": 25,
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
             "showPoints": "never",
+            "spanNulls": false,
             "stacking": {
+              "group": "A",
               "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
             }
-          }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "bytes"
         },
         "overrides": []
       },
-      "options": {
-        "legend": {
-          "displayMode": "table",
-          "placement": "bottom",
-          "calcs": [
-            "mean",
-            "max"
-          ]
-        },
-        "tooltip": {
-          "mode": "multi"
-        }
-      }
-    },
-    {
-      "type": "timeseries",
-      "title": "VRAM Free",
       "gridPos": {
         "h": 8,
         "w": 12,
@@ -615,59 +926,110 @@
         "y": 23
       },
       "id": 22,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
+      "options": {
+        "legend": {
+          "calcs": [
+            "mean",
+            "min"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
       },
+      "pluginVersion": "11.4.0",
       "targets": [
         {
           "expr": "DCGM_FI_DEV_FB_FREE{Hostname=~\"$Hostname\"} * 1048576",
           "legendFormat": "{{Hostname}}/{{gpu}}",
-          "refId": "A"
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_VM-SHORTTERM}"
+          }
         }
       ],
-      "fieldConfig": {
-        "defaults": {
-          "unit": "bytes",
-          "custom": {
-            "drawStyle": "line",
-            "lineInterpolation": "linear",
-            "fillOpacity": 10,
-            "showPoints": "never"
-          }
-        },
-        "overrides": []
-      },
-      "options": {
-        "legend": {
-          "displayMode": "table",
-          "placement": "bottom",
-          "calcs": [
-            "mean",
-            "min"
-          ]
-        },
-        "tooltip": {
-          "mode": "multi"
-        }
-      }
+      "title": "VRAM Free",
+      "type": "timeseries"
     },
     {
-      "type": "row",
-      "title": "Power & Temperature",
+      "collapsed": false,
       "gridPos": {
         "h": 1,
         "w": 24,
         "x": 0,
         "y": 31
       },
-      "collapsed": false,
       "id": 30,
-      "panels": []
+      "panels": [],
+      "title": "Power & Temperature",
+      "type": "row"
     },
     {
-      "type": "timeseries",
-      "title": "Power Usage per GPU",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "watt"
+        },
+        "overrides": []
+      },
       "gridPos": {
         "h": 8,
         "w": 12,
@@ -675,80 +1037,88 @@
         "y": 32
       },
       "id": 31,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
+      "options": {
+        "legend": {
+          "calcs": [
+            "mean",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
       },
+      "pluginVersion": "11.4.0",
       "targets": [
         {
           "expr": "DCGM_FI_DEV_POWER_USAGE{Hostname=~\"$Hostname\"}",
           "legendFormat": "{{Hostname}}/{{gpu}}",
-          "refId": "A"
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds_prometheus"
+          }
         }
       ],
-      "fieldConfig": {
-        "defaults": {
-          "unit": "watt",
-          "custom": {
-            "drawStyle": "line",
-            "lineInterpolation": "linear",
-            "fillOpacity": 10,
-            "showPoints": "never"
-          }
-        },
-        "overrides": []
-      },
-      "options": {
-        "legend": {
-          "displayMode": "table",
-          "placement": "bottom",
-          "calcs": [
-            "mean",
-            "max"
-          ]
-        },
-        "tooltip": {
-          "mode": "multi"
-        }
-      }
+      "title": "Power Usage per GPU",
+      "type": "timeseries"
     },
     {
-      "type": "timeseries",
-      "title": "GPU Temperature",
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 32
-      },
-      "id": 32,
       "datasource": {
         "type": "prometheus",
-        "uid": "$ds_prometheus"
+        "uid": "${DS_VM-SHORTTERM}"
       },
-      "targets": [
-        {
-          "expr": "DCGM_FI_DEV_GPU_TEMP{Hostname=~\"$Hostname\"}",
-          "legendFormat": "{{Hostname}}/{{gpu}}",
-          "refId": "A"
-        }
-      ],
       "fieldConfig": {
         "defaults": {
-          "unit": "celsius",
-          "min": 20,
-          "max": 100,
-          "custom": {
-            "drawStyle": "line",
-            "lineInterpolation": "linear",
-            "fillOpacity": 10,
-            "showPoints": "never"
+          "color": {
+            "mode": "palette-classic"
           },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "max": 100,
+          "min": 20,
           "thresholds": {
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": null
               },
               {
                 "color": "yellow",
@@ -759,61 +1129,68 @@
                 "value": 85
               }
             ]
-          }
+          },
+          "unit": "celsius"
         },
         "overrides": []
       },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 32
+      },
+      "id": 32,
       "options": {
         "legend": {
-          "displayMode": "table",
-          "placement": "bottom",
           "calcs": [
             "mean",
             "max"
-          ]
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
         },
         "tooltip": {
-          "mode": "multi"
+          "mode": "multi",
+          "sort": "none"
         }
-      }
+      },
+      "pluginVersion": "11.4.0",
+      "targets": [
+        {
+          "expr": "DCGM_FI_DEV_GPU_TEMP{Hostname=~\"$Hostname\"}",
+          "legendFormat": "{{Hostname}}/{{gpu}}",
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_VM-SHORTTERM}"
+          }
+        }
+      ],
+      "title": "GPU Temperature",
+      "type": "timeseries"
     },
     {
-      "type": "row",
-      "title": "Health",
+      "collapsed": false,
       "gridPos": {
         "h": 1,
         "w": 24,
         "x": 0,
         "y": 40
       },
-      "collapsed": false,
       "id": 40,
-      "panels": []
+      "panels": [],
+      "title": "Health",
+      "type": "row"
     },
     {
-      "type": "stat",
-      "title": "XID errors (latest)",
-      "gridPos": {
-        "h": 5,
-        "w": 8,
-        "x": 0,
-        "y": 41
-      },
-      "id": 41,
       "datasource": {
         "type": "prometheus",
         "uid": "$ds_prometheus"
       },
-      "targets": [
-        {
-          "expr": "max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS{Hostname=~\"$Hostname\"})",
-          "legendFormat": "{{Hostname}}/{{gpu}}",
-          "refId": "A"
-        }
-      ],
       "fieldConfig": {
         "defaults": {
-          "unit": "short",
           "color": {
             "mode": "thresholds"
           },
@@ -828,10 +1205,18 @@
                 "value": 1
               }
             ]
-          }
+          },
+          "unit": "short"
         },
         "overrides": []
       },
+      "gridPos": {
+        "h": 5,
+        "w": 8,
+        "x": 0,
+        "y": 41
+      },
+      "id": 41,
       "options": {
         "colorMode": "background",
         "graphMode": "none",
@@ -843,11 +1228,38 @@
           "values": false
         },
         "textMode": "value_and_name"
-      }
+      },
+      "targets": [
+        {
+          "expr": "max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS{Hostname=~\"$Hostname\"})",
+          "legendFormat": "{{Hostname}}/{{gpu}}",
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds_prometheus"
+          }
+        }
+      ],
+      "title": "XID errors (latest)",
+      "type": "stat"
     },
     {
-      "type": "timeseries",
-      "title": "Power Violation (\u00b5s/s throttled due to power)",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_VM-SHORTTERM}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "lineInterpolation": "linear",
+            "showPoints": "never"
+          },
+          "unit": "µs"
+        },
+        "overrides": []
+      },
       "gridPos": {
         "h": 5,
         "w": 8,
@@ -855,29 +1267,6 @@
         "y": 41
       },
       "id": 42,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "targets": [
-        {
-          "expr": "rate(DCGM_FI_DEV_POWER_VIOLATION{Hostname=~\"$Hostname\"}[5m])",
-          "legendFormat": "{{Hostname}}/{{gpu}}",
-          "refId": "A"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "unit": "\u00b5s",
-          "custom": {
-            "drawStyle": "line",
-            "lineInterpolation": "linear",
-            "fillOpacity": 10,
-            "showPoints": "never"
-          }
-        },
-        "overrides": []
-      },
       "options": {
         "legend": {
           "displayMode": "list",
@@ -886,11 +1275,38 @@
         "tooltip": {
           "mode": "multi"
         }
-      }
+      },
+      "targets": [
+        {
+          "expr": "rate(DCGM_FI_DEV_POWER_VIOLATION{Hostname=~\"$Hostname\"}[5m])",
+          "legendFormat": "{{Hostname}}/{{gpu}}",
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_VM-SHORTTERM}"
+          }
+        }
+      ],
+      "title": "Power Violation (µs/s throttled due to power)",
+      "type": "timeseries"
     },
     {
-      "type": "timeseries",
-      "title": "Thermal Violation (\u00b5s/s throttled due to thermals)",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "lineInterpolation": "linear",
+            "showPoints": "never"
+          },
+          "unit": "µs"
+        },
+        "overrides": []
+      },
       "gridPos": {
         "h": 5,
         "w": 8,
@@ -898,29 +1314,6 @@
         "y": 41
       },
       "id": 43,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "targets": [
-        {
-          "expr": "rate(DCGM_FI_DEV_THERMAL_VIOLATION{Hostname=~\"$Hostname\"}[5m])",
-          "legendFormat": "{{Hostname}}/{{gpu}}",
-          "refId": "A"
-        }
-      ],
-      "fieldConfig": {
-        "defaults": {
-          "unit": "\u00b5s",
-          "custom": {
-            "drawStyle": "line",
-            "lineInterpolation": "linear",
-            "fillOpacity": 10,
-            "showPoints": "never"
-          }
-        },
-        "overrides": []
-      },
       "options": {
         "legend": {
           "displayMode": "list",
@@ -929,7 +1322,93 @@
         "tooltip": {
           "mode": "multi"
         }
-      }
+      },
+      "targets": [
+        {
+          "expr": "rate(DCGM_FI_DEV_THERMAL_VIOLATION{Hostname=~\"$Hostname\"}[5m])",
+          "legendFormat": "{{Hostname}}/{{gpu}}",
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds_prometheus"
+          }
+        }
+      ],
+      "title": "Thermal Violation (µs/s throttled due to thermals)",
+      "type": "timeseries"
     }
-  ]
-}
\ No newline at end of file
+  ],
+  "refresh": "",
+  "schemaVersion": 40,
+  "tags": [
+    "gpu",
+    "dcgm"
+  ],
+  "templating": {
+    "list": [
+      {
+        "current": {},
+        "includeAll": false,
+        "label": "Prometheus",
+        "name": "ds_prometheus",
+        "options": [],
+        "query": "prometheus",
+        "refresh": 1,
+        "regex": "",
+        "type": "datasource"
+      },
+      {
+        "current": {},
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_VM-SHORTTERM}"
+        },
+        "definition": "label_values(DCGM_FI_DEV_GPU_UTIL, Hostname)",
+        "includeAll": true,
+        "label": "Host",
+        "multi": true,
+        "name": "Hostname",
+        "options": [],
+        "query": {
+          "query": "label_values(DCGM_FI_DEV_GPU_UTIL, Hostname)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 2,
+        "regex": "",
+        "sort": 1,
+        "type": "query"
+      },
+      {
+        "current": {},
+        "datasource": {
+          "type": "prometheus",
+          "uid": "$ds_prometheus"
+        },
+        "definition": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)",
+        "includeAll": true,
+        "label": "Namespace",
+        "multi": true,
+        "name": "namespace",
+        "options": [],
+        "query": {
+          "query": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 2,
+        "regex": "",
+        "sort": 1,
+        "type": "query"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "GPU Performance",
+  "uid": "gpu-performance",
+  "version": 2,
+  "weekStart": ""
+}
diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json
new file mode 100644
index 00000000..28f2c892
--- /dev/null
+++ b/dashboards/gpu/gpu-quotas.json
@@ -0,0 +1,822 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_VM-SHORTTERM",
+      "label": "vm-shortterm",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "prometheus",
+      "pluginName": "Prometheus"
+    }
+  ],
+  "__elements": {},
+  "__requires": [
+    {
+      "type": "panel",
+      "id": "bargauge",
+      "name": "Bar gauge",
+      "version": ""
+    },
+    {
+      "type": "panel",
+      "id": "gauge",
+      "name": "Gauge",
+      "version": ""
+    },
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "11.4.0"
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "1.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "stat",
+      "name": "Stat",
+      "version": ""
+    },
+    {
+      "type": "panel",
+      "id": "table",
+      "name": "Table",
+      "version": ""
+    },
+    {
+      "type": "panel",
+      "id": "timeseries",
+      "name": "Time series",
+      "version": ""
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "panels": [],
+      "title": "Allocation overview",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "blue",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 0,
+        "y": 1
+      },
+      "id": 2,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "percentChangeColorMode": "standard",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "value",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.4.0",
+      "targets": [
+        {
+          "expr": "sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"})",
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds_prometheus"
+          }
+        }
+      ],
+      "title": "GPU allocatable",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_VM-SHORTTERM}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 1
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 6,
+        "y": 1
+      },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "percentChangeColorMode": "standard",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "value",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.4.0",
+      "targets": [
+        {
+          "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"})",
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_VM-SHORTTERM}"
+          }
+        }
+      ],
+      "title": "GPU requested",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "blue",
+                "value": null
+              },
+              {
+                "color": "green",
+                "value": 40
+              },
+              {
+                "color": "yellow",
+                "value": 80
+              },
+              {
+                "color": "red",
+                "value": 95
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 12,
+        "y": 1
+      },
+      "id": 4,
+      "options": {
+        "minVizHeight": 75,
+        "minVizWidth": 75,
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true,
+        "sizing": "auto"
+      },
+      "pluginVersion": "11.4.0",
+      "targets": [
+        {
+          "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}) / sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"}) * 100",
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds_prometheus"
+          }
+        }
+      ],
+      "title": "Allocation ratio",
+      "type": "gauge"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_VM-SHORTTERM}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 18,
+        "y": 1
+      },
+      "id": 5,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "percentChangeColorMode": "standard",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "value",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.4.0",
+      "targets": [
+        {
+          "expr": "count((kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"} > 0) * on(namespace, pod) group_left() (kube_pod_status_phase{phase=\"Pending\"} == 1)) or vector(0)",
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_VM-SHORTTERM}"
+          }
+        }
+      ],
+      "title": "Pending pods (GPU)",
+      "type": "stat"
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 5
+      },
+      "id": 10,
+      "panels": [],
+      "title": "Per namespace",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "max": 8,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 6
+      },
+      "id": 11,
+      "options": {
+        "displayMode": "gradient",
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": false
+        },
+        "maxVizHeight": 300,
+        "minVizHeight": 16,
+        "minVizWidth": 8,
+        "namePlacement": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showUnfilled": true,
+        "sizing": "auto",
+        "valueMode": "color"
+      },
+      "pluginVersion": "11.4.0",
+      "targets": [
+        {
+          "expr": "sum by (namespace) (kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"})",
+          "legendFormat": "{{namespace}}",
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds_prometheus"
+          }
+        }
+      ],
+      "title": "GPU requested per namespace",
+      "type": "bargauge"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_VM-SHORTTERM}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "stepAfter",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Allocatable (total)"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "blue",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 6
+      },
+      "id": 12,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.4.0",
+      "targets": [
+        {
+          "expr": "sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"})",
+          "legendFormat": "Allocatable (total)",
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_VM-SHORTTERM}"
+          }
+        },
+        {
+          "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"})",
+          "legendFormat": "Requested",
+          "refId": "B",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_VM-SHORTTERM}"
+          }
+        }
+      ],
+      "title": "GPU allocated over time",
+      "type": "timeseries"
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 14
+      },
+      "id": 20,
+      "panels": [],
+      "title": "Pods with GPU",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "align": "auto",
+            "cellOptions": {
+              "type": "auto"
+            },
+            "inspect": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Status"
+            },
+            "properties": [
+              {
+                "id": "custom.cellOptions",
+                "value": {
+                  "mode": "basic",
+                  "type": "color-background"
+                }
+              },
+              {
+                "id": "mappings",
+                "value": [
+                  {
+                    "options": {
+                      "Failed": {
+                        "color": "red",
+                        "index": 2
+                      },
+                      "Pending": {
+                        "color": "orange",
+                        "index": 1
+                      },
+                      "Running": {
+                        "color": "green",
+                        "index": 0
+                      }
+                    },
+                    "type": "value"
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 10,
+        "w": 24,
+        "x": 0,
+        "y": 15
+      },
+      "id": 21,
+      "options": {
+        "cellHeight": "sm",
+        "footer": {
+          "countRows": false,
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
+        "showHeader": true
+      },
+      "pluginVersion": "11.4.0",
+      "targets": [
+        {
+          "expr": "kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left(phase) (kube_pod_status_phase{phase=~\"Running|Pending|Failed|Succeeded\"} == 1)",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "",
+          "refId": "requested",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds_prometheus"
+          }
+        },
+        {
+          "expr": "kube_pod_container_resource_limits{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"}",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "",
+          "refId": "limits",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds_prometheus"
+          }
+        }
+      ],
+      "title": "Pods requesting GPU",
+      "transformations": [
+        {
+          "id": "joinByField",
+          "options": {
+            "byField": "pod",
+            "mode": "outer"
+          }
+        },
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "Time 1": true,
+              "Time 2": true,
+              "__name__": true,
+              "__name__ 1": true,
+              "__name__ 2": true,
+              "cluster": true,
+              "cluster 1": true,
+              "cluster 2": true,
+              "container 2": true,
+              "endpoint": true,
+              "endpoint 1": true,
+              "endpoint 2": true,
+              "instance": true,
+              "instance 1": true,
+              "instance 2": true,
+              "job": true,
+              "job 1": true,
+              "job 2": true,
+              "namespace 2": true,
+              "node 2": true,
+              "prometheus": true,
+              "prometheus 1": true,
+              "prometheus 2": true,
+              "resource": true,
+              "resource 1": true,
+              "resource 2": true,
+              "service": true,
+              "service 1": true,
+              "service 2": true,
+              "tenant": true,
+              "tenant 1": true,
+              "tenant 2": true,
+              "tier": true,
+              "tier 1": true,
+              "tier 2": true,
+              "uid": true,
+              "uid 1": true,
+              "uid 2": true,
+              "unit": true,
+              "unit 1": true,
+              "unit 2": true
+            },
+            "indexByName": {
+              "Value #limits": 5,
+              "Value #requested": 4,
+              "container 1": 2,
+              "namespace 1": 0,
+              "node 1": 3,
+              "phase": 6,
+              "pod": 1
+            },
+            "renameByName": {
+              "Value #limits": "Limit",
+              "Value #requested": "Req",
+              "container 1": "Container",
+              "namespace 1": "Namespace",
+              "node 1": "Node",
+              "phase": "Status"
+            }
+          }
+        }
+      ],
+      "type": "table"
+    }
+  ],
+  "refresh": "",
+  "schemaVersion": 40,
+  "tags": [
+    "gpu",
+    "quotas"
+  ],
+  "templating": {
+    "list": [
+      {
+        "current": {},
+        "includeAll": false,
+        "label": "Prometheus",
+        "name": "ds_prometheus",
+        "options": [],
+        "query": "prometheus",
+        "refresh": 1,
+        "regex": "",
+        "type": "datasource"
+      },
+      {
+        "current": {},
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_VM-SHORTTERM}"
+        },
+        "definition": "label_values(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}, namespace)",
+        "includeAll": true,
+        "label": "Namespace",
+        "multi": true,
+        "name": "namespace",
+        "options": [],
+        "query": {
+          "query": "label_values(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}, namespace)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 2,
+        "regex": "",
+        "sort": 1,
+        "type": "query"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "GPU Quotas & Allocation",
+  "uid": "gpu-quotas",
+  "version": 4,
+  "weekStart": ""
+}
diff --git a/packages/system/monitoring/dashboards-infra.list b/packages/system/monitoring/dashboards-infra.list
index 581f9a05..b402fa68 100644
--- a/packages/system/monitoring/dashboards-infra.list
+++ b/packages/system/monitoring/dashboards-infra.list
@@ -33,3 +33,5 @@ hubble/dns-namespace
 hubble/l7-http-metrics
 hubble/network-overview
 gpu/gpu-performance
+gpu/gpu-efficiency
+gpu/gpu-quotas

From 7e5f3a7f12ae6ff744032c30eb0e7b05071986a2 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Fri, 17 Apr 2026 17:45:06 +0300
Subject: [PATCH 029/151] refactor(monitoring): clean up GPU dashboards

Strip Grafana export boilerplate (__inputs, __elements, __requires,
default annotations, embedded datasource inputs) and tighten panel
layouts across the three GPU dashboards. All three continue to use
the $ds_prometheus template variable.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 dashboards/gpu/gpu-efficiency.json  | 1186 ++++++++------------
 dashboards/gpu/gpu-performance.json | 1574 ++++++++++-----------------
 dashboards/gpu/gpu-quotas.json      |  876 ++++++---------
 3 files changed, 1358 insertions(+), 2278 deletions(-)

diff --git a/dashboards/gpu/gpu-efficiency.json b/dashboards/gpu/gpu-efficiency.json
index cc2bca6c..1a305b4b 100644
--- a/dashboards/gpu/gpu-efficiency.json
+++ b/dashboards/gpu/gpu-efficiency.json
@@ -1,72 +1,26 @@
 {
-  "__inputs": [
-    {
-      "name": "DS_VM-SHORTTERM",
-      "label": "vm-shortterm",
-      "description": "",
-      "type": "datasource",
-      "pluginId": "prometheus",
-      "pluginName": "Prometheus"
-    }
+  "uid": "gpu-efficiency",
+  "title": "GPU Efficiency Score",
+  "description": "Tensor saturation, util/watt and throttling — reveals inefficient GPU workloads",
+  "tags": [
+    "gpu",
+    "efficiency",
+    "finops"
   ],
-  "__elements": {},
-  "__requires": [
-    {
-      "type": "grafana",
-      "id": "grafana",
-      "name": "Grafana",
-      "version": "11.4.0"
-    },
-    {
-      "type": "datasource",
-      "id": "prometheus",
-      "name": "Prometheus",
-      "version": "1.0.0"
-    },
-    {
-      "type": "panel",
-      "id": "stat",
-      "name": "Stat",
-      "version": ""
-    },
-    {
-      "type": "panel",
-      "id": "table",
-      "name": "Table",
-      "version": ""
-    },
-    {
-      "type": "panel",
-      "id": "timeseries",
-      "name": "Time series",
-      "version": ""
-    }
-  ],
-  "annotations": {
-    "list": [
-      {
-        "builtIn": 1,
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "type": "dashboard"
-      }
-    ]
-  },
-  "description": "Tensor saturation, util/watt and throttling \u2014 reveals inefficient GPU workloads",
+  "timezone": "browser",
   "editable": true,
-  "fiscalYearStartMonth": 0,
   "graphTooltip": 1,
-  "id": null,
-  "links": [],
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "fiscalYearStartMonth": 0,
+  "schemaVersion": 42,
   "panels": [
     {
+      "type": "row",
       "collapsed": false,
+      "title": "Overall efficiency metrics",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -74,48 +28,23 @@
         "y": 0
       },
       "id": 1,
-      "panels": [],
-      "title": "\u041e\u0431\u0449\u0438\u0435 \u043c\u0435\u0442\u0440\u0438\u043a\u0438 \u044d\u0444\u0444\u0435\u043a\u0442\u0438\u0432\u043d\u043e\u0441\u0442\u0438",
-      "type": "row"
+      "panels": []
     },
     {
+      "type": "stat",
+      "id": 2,
+      "targets": [
+        {
+          "expr": "avg(pod:tensor_saturation:avg5m)",
+          "refId": "A"
+        }
+      ],
+      "title": "Cluster Tensor Saturation",
+      "description": "Mean tensor core saturation. \u003c10% means GPUs are used inefficiently (tenants could move to CPU or optimize their code).",
+      "transparent": false,
       "datasource": {
         "type": "prometheus",
-        "uid": "${ds_prometheus}"
-      },
-      "description": "\u0421\u0440\u0435\u0434\u043d\u044f\u044f \u0437\u0430\u0433\u0440\u0443\u0437\u043a\u0430 \u0442\u0435\u043d\u0437\u043e\u0440\u043d\u044b\u0445 \u044f\u0434\u0435\u0440. <10% \u2014 GPU \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u044e\u0442\u0441\u044f \u043d\u0435\u044d\u0444\u0444\u0435\u043a\u0442\u0438\u0432\u043d\u043e (\u043a\u043b\u0438\u0435\u043d\u0442\u044b \u043c\u043e\u0433\u043b\u0438 \u0431\u044b \u043f\u0435\u0440\u0435\u0435\u0445\u0430\u0442\u044c \u043d\u0430 CPU \u0438\u043b\u0438 \u043e\u043f\u0442\u0438\u043c\u0438\u0437\u0438\u0440\u043e\u0432\u0430\u0442\u044c \u043a\u043e\u0434).",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "mappings": [],
-          "max": 100,
-          "min": 0,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "red",
-                "value": null
-              },
-              {
-                "color": "orange",
-                "value": 10
-              },
-              {
-                "color": "yellow",
-                "value": 30
-              },
-              {
-                "color": "green",
-                "value": 60
-              }
-            ]
-          },
-          "unit": "percent"
-        },
-        "overrides": []
+        "uid": "$ds_prometheus"
       },
       "gridPos": {
         "h": 5,
@@ -123,71 +52,67 @@
         "x": 0,
         "y": 1
       },
-      "id": 2,
+      "repeatDirection": "h",
       "options": {
-        "colorMode": "background",
         "graphMode": "area",
+        "colorMode": "background",
         "justifyMode": "auto",
-        "orientation": "auto",
-        "percentChangeColorMode": "standard",
+        "textMode": "value",
+        "wideLayout": true,
+        "showPercentChange": false,
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
+          ]
         },
-        "showPercentChange": false,
-        "textMode": "value",
-        "wideLayout": true
+        "percentChangeColorMode": "standard",
+        "orientation": ""
       },
-      "pluginVersion": "11.4.0",
-      "targets": [
-        {
-          "expr": "avg(pod:tensor_saturation:avg5m)",
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${ds_prometheus}"
-          }
-        }
-      ],
-      "title": "Cluster Tensor Saturation",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${DS_VM-SHORTTERM}"
-      },
-      "description": "NVML utilization % \u043d\u0430 \u043a\u0430\u0436\u0434\u044b\u0439 \u0432\u0430\u0442\u0442. \u0412\u044b\u0441\u043e\u043a\u043e\u0435 \u0437\u043d\u0430\u0447\u0435\u043d\u0438\u0435 = \u044d\u0444\u0444\u0435\u043a\u0442\u0438\u0432\u043d\u0430\u044f \u043d\u0430\u0433\u0440\u0443\u0437\u043a\u0430.",
       "fieldConfig": {
         "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "decimals": 3,
-          "mappings": [],
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
           "thresholds": {
             "mode": "absolute",
             "steps": [
               {
-                "color": "red",
-                "value": null
+                "value": null,
+                "color": "red"
               },
               {
-                "color": "yellow",
-                "value": 0.5
+                "value": 10,
+                "color": "orange"
               },
               {
-                "color": "green",
-                "value": 1
+                "value": 30,
+                "color": "yellow"
+              },
+              {
+                "value": 60,
+                "color": "green"
               }
             ]
-          },
-          "unit": "none"
+          }
         },
         "overrides": []
+      }
+    },
+    {
+      "type": "stat",
+      "id": 3,
+      "targets": [
+        {
+          "expr": "avg(pod:util_per_watt:avg5m)",
+          "refId": "A"
+        }
+      ],
+      "title": "Avg Utilization per Watt",
+      "description": "NVML utilization % per watt. Higher value = more efficient workload.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
       },
       "gridPos": {
         "h": 5,
@@ -195,73 +120,62 @@
         "x": 8,
         "y": 1
       },
-      "id": 3,
+      "repeatDirection": "h",
       "options": {
-        "colorMode": "background",
         "graphMode": "area",
+        "colorMode": "background",
         "justifyMode": "auto",
-        "orientation": "auto",
-        "percentChangeColorMode": "standard",
+        "textMode": "value",
+        "wideLayout": true,
+        "showPercentChange": false,
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
+          ]
         },
-        "showPercentChange": false,
-        "textMode": "value",
-        "wideLayout": true
+        "percentChangeColorMode": "standard",
+        "orientation": ""
       },
-      "pluginVersion": "11.4.0",
-      "targets": [
-        {
-          "expr": "avg(pod:util_per_watt:avg5m)",
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_VM-SHORTTERM}"
-          }
-        }
-      ],
-      "title": "Avg Utilization per Watt",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${ds_prometheus}"
-      },
-      "description": "\u0414\u043e\u043b\u044f \u0432\u0440\u0435\u043c\u0435\u043d\u0438, \u043a\u043e\u0433\u0434\u0430 GPU \u0443\u043f\u0438\u0440\u0430\u044e\u0442\u0441\u044f \u0432 TDP \u0438 \u0442\u0435\u0440\u044f\u044e\u0442 \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c. >5% \u2014 \u043a\u043b\u0438\u0435\u043d\u0442\u044b \u043d\u0435\u0434\u043e\u043f\u043e\u043b\u0443\u0447\u0430\u044e\u0442 FLOPS.",
       "fieldConfig": {
         "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "decimals": 2,
-          "mappings": [],
-          "max": 100,
-          "min": 0,
+          "unit": "none",
+          "decimals": 3,
           "thresholds": {
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
-                "value": null
+                "value": null,
+                "color": "red"
               },
               {
-                "color": "yellow",
-                "value": 5
+                "value": 0.5,
+                "color": "yellow"
               },
               {
-                "color": "red",
-                "value": 20
+                "value": 1,
+                "color": "green"
               }
             ]
-          },
-          "unit": "percent"
+          }
         },
         "overrides": []
+      }
+    },
+    {
+      "type": "stat",
+      "id": 4,
+      "targets": [
+        {
+          "expr": "avg(gpu:power_throttle_fraction:rate5m)",
+          "refId": "A"
+        }
+      ],
+      "title": "Avg Power Throttling",
+      "description": "Fraction of time GPUs hit the TDP cap and lose performance. \u003e5% means tenants underutilize billed FLOPS.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
       },
       "gridPos": {
         "h": 5,
@@ -269,40 +183,51 @@
         "x": 16,
         "y": 1
       },
-      "id": 4,
+      "repeatDirection": "h",
       "options": {
-        "colorMode": "background",
         "graphMode": "area",
+        "colorMode": "background",
         "justifyMode": "auto",
-        "orientation": "auto",
-        "percentChangeColorMode": "standard",
+        "textMode": "value",
+        "wideLayout": true,
+        "showPercentChange": false,
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
+          ]
         },
-        "showPercentChange": false,
-        "textMode": "value",
-        "wideLayout": true
+        "percentChangeColorMode": "standard",
+        "orientation": ""
       },
-      "pluginVersion": "11.4.0",
-      "targets": [
-        {
-          "expr": "avg(gpu:power_throttle_fraction:rate5m) * 100",
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${ds_prometheus}"
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "decimals": 2,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "value": null,
+                "color": "green"
+              },
+              {
+                "value": 0.05,
+                "color": "yellow"
+              },
+              {
+                "value": 0.2,
+                "color": "red"
+              }
+            ]
           }
-        }
-      ],
-      "title": "Avg Power Throttling",
-      "type": "stat"
+        },
+        "overrides": []
+      }
     },
     {
+      "type": "row",
       "collapsed": false,
+      "title": "NVML vs Tensor (mismatch detector)",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -310,73 +235,24 @@
         "y": 6
       },
       "id": 10,
-      "panels": [],
-      "title": "NVML vs Tensor (\u043a\u0442\u043e \u043e\u0431\u043c\u0430\u043d\u044b\u0432\u0430\u0435\u0442\u0441\u044f)",
-      "type": "row"
+      "panels": []
     },
     {
+      "type": "timeseries",
+      "id": 11,
+      "targets": [
+        {
+          "expr": "DCGM_FI_DEV_GPU_UTIL{namespace=~\"$namespace\", namespace!=\"\"}",
+          "legendFormat": "{{namespace}}/{{pod}}",
+          "refId": "A"
+        }
+      ],
+      "title": "NVML GPU Utilization",
+      "description": "Classic utilization metric. Shows activity of any engine (SM, copy, encoder).",
+      "transparent": false,
       "datasource": {
         "type": "prometheus",
-        "uid": "${DS_VM-SHORTTERM}"
-      },
-      "description": "\u041a\u043b\u0430\u0441\u0441\u0438\u0447\u0435\u0441\u043a\u0430\u044f \u043c\u0435\u0442\u0440\u0438\u043a\u0430 \u0443\u0442\u0438\u043b\u0438\u0437\u0430\u0446\u0438\u0438. \u041f\u043e\u043a\u0430\u0437\u044b\u0432\u0430\u0435\u0442 \u0437\u0430\u043d\u044f\u0442\u043e\u0441\u0442\u044c \u043b\u044e\u0431\u043e\u0433\u043e \u0434\u0432\u0438\u0436\u043a\u0430 (SM, copy, encoder).",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "barWidthFactor": 0.6,
-            "drawStyle": "line",
-            "fillOpacity": 10,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "never",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "max": 100,
-          "min": 0,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "percent"
-        },
-        "overrides": []
+        "uid": "$ds_prometheus"
       },
       "gridPos": {
         "h": 8,
@@ -384,100 +260,53 @@
         "x": 0,
         "y": 7
       },
-      "id": 11,
+      "repeatDirection": "h",
       "options": {
         "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": false,
           "calcs": [
             "mean",
             "max"
-          ],
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": true
+          ]
         },
         "tooltip": {
           "mode": "multi",
-          "sort": "none"
+          "sort": ""
         }
       },
-      "pluginVersion": "11.4.0",
-      "targets": [
-        {
-          "expr": "DCGM_FI_DEV_GPU_UTIL{namespace=~\"$namespace\", namespace!=\"\"}",
-          "legendFormat": "{{namespace}}/{{pod}}",
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_VM-SHORTTERM}"
-          }
-        }
-      ],
-      "title": "NVML GPU Utilization",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${ds_prometheus}"
-      },
-      "description": "\u0420\u0435\u0430\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u0433\u0440\u0443\u0437\u043a\u0430 \u0442\u0435\u043d\u0437\u043e\u0440\u043d\u044b\u0445 \u044f\u0434\u0435\u0440 (HMMA). \u0414\u043b\u044f AI/LLM \u0438\u043d\u0444\u0435\u0440\u0435\u043d\u0441\u0430 \u0434\u043e\u043b\u0436\u043d\u0430 \u0431\u044b\u0442\u044c \u226520%, \u0438\u043d\u0430\u0447\u0435 workload \u043d\u0435 \u043e\u043f\u0442\u0438\u043c\u0438\u0437\u0438\u0440\u043e\u0432\u0430\u043d.",
       "fieldConfig": {
         "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "barWidthFactor": 0.6,
-            "drawStyle": "line",
-            "fillOpacity": 10,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "never",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "max": 100,
+          "unit": "percent",
           "min": 0,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "percent"
+          "max": 100,
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "linear",
+            "fillOpacity": 10,
+            "showPoints": "never"
+          }
         },
         "overrides": []
+      }
+    },
+    {
+      "type": "timeseries",
+      "id": 12,
+      "targets": [
+        {
+          "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace=~\"$namespace\", namespace!=\"\"} * 100",
+          "legendFormat": "{{namespace}}/{{pod}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Tensor Pipe Active",
+      "description": "Real tensor core load (HMMA). For AI/LLM inference it should be ≥20%, otherwise the workload is unoptimized.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
       },
       "gridPos": {
         "h": 8,
@@ -485,39 +314,41 @@
         "x": 12,
         "y": 7
       },
-      "id": 12,
+      "repeatDirection": "h",
       "options": {
         "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": false,
           "calcs": [
             "mean",
             "max"
-          ],
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": true
+          ]
         },
         "tooltip": {
           "mode": "multi",
-          "sort": "none"
+          "sort": ""
         }
       },
-      "pluginVersion": "11.4.0",
-      "targets": [
-        {
-          "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace=~\"$namespace\", namespace!=\"\"} * 100",
-          "legendFormat": "{{namespace}}/{{pod}}",
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${ds_prometheus}"
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "linear",
+            "fillOpacity": 10,
+            "showPoints": "never"
           }
-        }
-      ],
-      "title": "Tensor Pipe Active",
-      "type": "timeseries"
+        },
+        "overrides": []
+      }
     },
     {
+      "type": "row",
       "collapsed": false,
+      "title": "Per-pod ranking",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -525,40 +356,92 @@
         "y": 15
       },
       "id": 20,
-      "panels": [],
-      "title": "Per-pod \u0440\u0435\u0439\u0442\u0438\u043d\u0433",
-      "type": "row"
+      "panels": []
     },
     {
+      "type": "table",
+      "id": 21,
+      "targets": [
+        {
+          "expr": "topk(20, pod:tensor_saturation:avg5m)",
+          "instant": true,
+          "range": false,
+          "format": "table",
+          "refId": "A"
+        }
+      ],
+      "title": "Tensor Saturation per pod (5m avg)",
+      "description": "Who is exercising tensor cores and who is not. Sorted descending.",
+      "transparent": false,
       "datasource": {
         "type": "prometheus",
-        "uid": "${DS_VM-SHORTTERM}"
+        "uid": "$ds_prometheus"
       },
-      "description": "\u041a\u0442\u043e \u0436\u043c\u0451\u0442 \u0442\u0435\u043d\u0437\u043e\u0440\u043d\u044b\u0435 \u044f\u0434\u0440\u0430, \u0430 \u043a\u0442\u043e \u043d\u0435\u0442. \u0421\u043e\u0440\u0442\u0438\u0440\u043e\u0432\u043a\u0430 \u043e\u0442 \u043c\u0430\u043a\u0441\u0438\u043c\u0443\u043c\u0430.",
-      "fieldConfig": {
-        "defaults": {
-          "custom": {
-            "align": "auto",
-            "cellOptions": {
-              "type": "auto"
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "repeatDirection": "h",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "DCGM_FI_DRIVER_VERSION": true,
+              "Hostname": true,
+              "Time": true,
+              "UUID": true,
+              "__name__": true,
+              "cluster": true,
+              "container": true,
+              "device": true,
+              "endpoint": true,
+              "gpu": true,
+              "gpu_driver_version": true,
+              "instance": true,
+              "job": true,
+              "modelName": true,
+              "pci_bus_id": true,
+              "prometheus": true,
+              "service": true,
+              "tenant": true,
+              "tier": true,
+              "uid": true,
+              "unit": true
             },
-            "inspect": false
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
+            "indexByName": {
+              "Value": 2,
+              "namespace": 0,
+              "pod": 1
+            },
+            "renameByName": {
+              "Value": "Saturation",
+              "namespace": "Namespace",
+              "pod": "Pod"
+            }
           }
+        }
+      ],
+      "options": {
+        "frameIndex": 0,
+        "showHeader": true,
+        "showTypeIcons": false,
+        "sortBy": [
+          {
+            "displayName": "Saturation",
+            "desc": true
+          }
+        ],
+        "footer": {
+          "show": false,
+          "reducer": []
         },
+        "cellHeight": "sm"
+      },
+      "fieldConfig": {
+        "defaults": {},
         "overrides": [
           {
             "matcher": {
@@ -591,8 +474,7 @@
                   "mode": "absolute",
                   "steps": [
                     {
-                      "color": "red",
-                      "value": null
+                      "color": "red"
                     },
                     {
                       "color": "orange",
@@ -612,62 +494,54 @@
             ]
           }
         ]
+      }
+    },
+    {
+      "type": "table",
+      "id": 22,
+      "targets": [
+        {
+          "expr": "topk(20, pod:util_per_watt:avg5m)",
+          "instant": true,
+          "range": false,
+          "format": "table",
+          "refId": "A"
+        }
+      ],
+      "title": "Utilization / Watt per pod (5m avg)",
+      "description": "How efficiently each pod spends watts. Low value = poor optimization.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
       },
       "gridPos": {
         "h": 8,
         "w": 12,
-        "x": 0,
+        "x": 12,
         "y": 16
       },
-      "id": 21,
-      "options": {
-        "cellHeight": "sm",
-        "footer": {
-          "countRows": false,
-          "fields": "",
-          "reducer": [
-            "sum"
-          ],
-          "show": false
-        },
-        "showHeader": true,
-        "sortBy": [
-          {
-            "desc": true,
-            "displayName": "Saturation"
-          }
-        ]
-      },
-      "pluginVersion": "11.4.0",
-      "targets": [
-        {
-          "expr": "topk(20, pod:tensor_saturation:avg5m)",
-          "format": "table",
-          "instant": true,
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_VM-SHORTTERM}"
-          }
-        }
-      ],
-      "title": "Tensor Saturation per pod (5m avg)",
+      "repeatDirection": "h",
       "transformations": [
         {
           "id": "organize",
           "options": {
             "excludeByName": {
+              "DCGM_FI_DRIVER_VERSION": true,
               "Hostname": true,
               "Time": true,
               "UUID": true,
               "__name__": true,
               "cluster": true,
               "container": true,
+              "device": true,
               "endpoint": true,
               "gpu": true,
+              "gpu_driver_version": true,
               "instance": true,
               "job": true,
               "modelName": true,
+              "pci_bus_id": true,
               "prometheus": true,
               "service": true,
               "tenant": true,
@@ -681,45 +555,31 @@
               "pod": 1
             },
             "renameByName": {
-              "Value": "Saturation",
+              "Value": "Util/Watt",
               "namespace": "Namespace",
               "pod": "Pod"
             }
           }
         }
       ],
-      "type": "table"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${ds_prometheus}"
-      },
-      "description": "\u041d\u0430\u0441\u043a\u043e\u043b\u044c\u043a\u043e \u044d\u0444\u0444\u0435\u043a\u0442\u0438\u0432\u043d\u043e \u043a\u0430\u0436\u0434\u044b\u0439 \u043f\u043e\u0434 \u0442\u0440\u0430\u0442\u0438\u0442 \u0432\u0430\u0442\u0442\u044b. \u041d\u0438\u0437\u043a\u043e\u0435 \u0437\u043d\u0430\u0447\u0435\u043d\u0438\u0435 = \u043f\u043b\u043e\u0445\u0430\u044f \u043e\u043f\u0442\u0438\u043c\u0438\u0437\u0430\u0446\u0438\u044f.",
-      "fieldConfig": {
-        "defaults": {
-          "custom": {
-            "align": "auto",
-            "cellOptions": {
-              "type": "auto"
-            },
-            "inspect": false
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
+      "options": {
+        "frameIndex": 0,
+        "showHeader": true,
+        "showTypeIcons": false,
+        "sortBy": [
+          {
+            "displayName": "Util/Watt",
+            "desc": true
           }
+        ],
+        "footer": {
+          "show": false,
+          "reducer": []
         },
+        "cellHeight": "sm"
+      },
+      "fieldConfig": {
+        "defaults": {},
         "overrides": [
           {
             "matcher": {
@@ -756,8 +616,7 @@
                   "mode": "absolute",
                   "steps": [
                     {
-                      "color": "red",
-                      "value": null
+                      "color": "red"
                     },
                     {
                       "color": "yellow",
@@ -773,86 +632,12 @@
             ]
           }
         ]
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 16
-      },
-      "id": 22,
-      "options": {
-        "cellHeight": "sm",
-        "footer": {
-          "countRows": false,
-          "fields": "",
-          "reducer": [
-            "sum"
-          ],
-          "show": false
-        },
-        "showHeader": true,
-        "sortBy": [
-          {
-            "desc": true,
-            "displayName": "Util/Watt"
-          }
-        ]
-      },
-      "pluginVersion": "11.4.0",
-      "targets": [
-        {
-          "expr": "topk(20, pod:util_per_watt:avg5m)",
-          "format": "table",
-          "instant": true,
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${ds_prometheus}"
-          }
-        }
-      ],
-      "title": "Utilization / Watt per pod (5m avg)",
-      "transformations": [
-        {
-          "id": "organize",
-          "options": {
-            "excludeByName": {
-              "Hostname": true,
-              "Time": true,
-              "UUID": true,
-              "__name__": true,
-              "cluster": true,
-              "container": true,
-              "endpoint": true,
-              "gpu": true,
-              "instance": true,
-              "job": true,
-              "modelName": true,
-              "prometheus": true,
-              "service": true,
-              "tenant": true,
-              "tier": true,
-              "uid": true,
-              "unit": true
-            },
-            "indexByName": {
-              "Value": 2,
-              "namespace": 0,
-              "pod": 1
-            },
-            "renameByName": {
-              "Value": "Util/Watt",
-              "namespace": "Namespace",
-              "pod": "Pod"
-            }
-          }
-        }
-      ],
-      "type": "table"
+      }
     },
     {
+      "type": "row",
       "collapsed": false,
+      "title": "Throttling",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -860,77 +645,24 @@
         "y": 24
       },
       "id": 30,
-      "panels": [],
-      "title": "Throttling",
-      "type": "row"
+      "panels": []
     },
     {
+      "type": "timeseries",
+      "id": 31,
+      "targets": [
+        {
+          "expr": "gpu:power_throttle_fraction:rate5m",
+          "legendFormat": "{{Hostname}}/{{gpu}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Power throttle fraction per GPU",
+      "description": "Fraction of time the GPU was power-throttled. 1.0 = always throttled.",
+      "transparent": false,
       "datasource": {
         "type": "prometheus",
-        "uid": "${DS_VM-SHORTTERM}"
-      },
-      "description": "\u0414\u043e\u043b\u044f \u0432\u0440\u0435\u043c\u0435\u043d\u0438, \u043a\u043e\u0433\u0434\u0430 GPU \u0434\u0440\u043e\u0441\u0441\u0435\u043b\u0438\u0440\u043e\u0432\u0430\u043d\u0430 \u043f\u043e \u043f\u0438\u0442\u0430\u043d\u0438\u044e. 1.0 = \u043f\u043e\u0441\u0442\u043e\u044f\u043d\u043d\u043e.",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "barWidthFactor": 0.6,
-            "drawStyle": "line",
-            "fillOpacity": 25,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "never",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "max": 1,
-          "min": 0,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "yellow",
-                "value": 0.05
-              },
-              {
-                "color": "red",
-                "value": 0.2
-              }
-            ]
-          },
-          "unit": "percentunit"
-        },
-        "overrides": []
+        "uid": "$ds_prometheus"
       },
       "gridPos": {
         "h": 8,
@@ -938,104 +670,70 @@
         "x": 0,
         "y": 25
       },
-      "id": 31,
+      "repeatDirection": "h",
       "options": {
         "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": false,
           "calcs": [
             "mean",
             "max"
-          ],
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": true
+          ]
         },
         "tooltip": {
           "mode": "multi",
-          "sort": "none"
+          "sort": ""
         }
       },
-      "pluginVersion": "11.4.0",
-      "targets": [
-        {
-          "expr": "gpu:power_throttle_fraction:rate5m",
-          "legendFormat": "{{Hostname}}/{{gpu}}",
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_VM-SHORTTERM}"
-          }
-        }
-      ],
-      "title": "Power throttle fraction per GPU",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${ds_prometheus}"
-      },
-      "description": "\u0414\u043e\u043b\u044f \u0432\u0440\u0435\u043c\u0435\u043d\u0438, \u043a\u043e\u0433\u0434\u0430 GPU \u0434\u0440\u043e\u0441\u0441\u0435\u043b\u0438\u0440\u043e\u0432\u0430\u043d\u0430 \u043f\u043e \u0442\u0435\u043c\u043f\u0435\u0440\u0430\u0442\u0443\u0440\u0435.",
       "fieldConfig": {
         "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "barWidthFactor": 0.6,
-            "drawStyle": "line",
-            "fillOpacity": 25,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "never",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "max": 1,
+          "unit": "percentunit",
           "min": 0,
+          "max": 1,
           "thresholds": {
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
-                "value": null
+                "value": null,
+                "color": "green"
               },
               {
-                "color": "yellow",
-                "value": 0.05
+                "value": 0.05,
+                "color": "yellow"
               },
               {
-                "color": "red",
-                "value": 0.2
+                "value": 0.2,
+                "color": "red"
               }
             ]
           },
-          "unit": "percentunit"
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "linear",
+            "fillOpacity": 25,
+            "showPoints": "never"
+          }
         },
         "overrides": []
+      }
+    },
+    {
+      "type": "timeseries",
+      "id": 32,
+      "targets": [
+        {
+          "expr": "gpu:thermal_throttle_fraction:rate5m",
+          "legendFormat": "{{Hostname}}/{{gpu}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Thermal throttle fraction per GPU",
+      "description": "Fraction of time the GPU was thermal-throttled.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
       },
       "gridPos": {
         "h": 8,
@@ -1043,89 +741,101 @@
         "x": 12,
         "y": 25
       },
-      "id": 32,
+      "repeatDirection": "h",
       "options": {
         "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": false,
           "calcs": [
             "mean",
             "max"
-          ],
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": true
+          ]
         },
         "tooltip": {
           "mode": "multi",
-          "sort": "none"
+          "sort": ""
         }
       },
-      "pluginVersion": "11.4.0",
-      "targets": [
-        {
-          "expr": "gpu:thermal_throttle_fraction:rate5m",
-          "legendFormat": "{{Hostname}}/{{gpu}}",
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${ds_prometheus}"
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "min": 0,
+          "max": 1,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "value": null,
+                "color": "green"
+              },
+              {
+                "value": 0.05,
+                "color": "yellow"
+              },
+              {
+                "value": 0.2,
+                "color": "red"
+              }
+            ]
+          },
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "linear",
+            "fillOpacity": 25,
+            "showPoints": "never"
           }
-        }
-      ],
-      "title": "Thermal throttle fraction per GPU",
-      "type": "timeseries"
+        },
+        "overrides": []
+      }
     }
   ],
-  "refresh": "",
-  "schemaVersion": 40,
-  "tags": [
-    "gpu",
-    "efficiency",
-    "finops"
-  ],
   "templating": {
     "list": [
       {
-        "current": {},
-        "includeAll": false,
-        "label": "Datasource",
+        "type": "datasource",
         "name": "ds_prometheus",
-        "options": [],
+        "label": "Prometheus",
+        "skipUrlSync": false,
         "query": "prometheus",
-        "refresh": 1,
-        "regex": "vm-.*",
-        "type": "datasource"
+        "current": {
+          "selected": false,
+          "text": "default",
+          "value": "default"
+        },
+        "multi": false,
+        "allowCustomValue": true,
+        "includeAll": false,
+        "regex": "",
+        "auto": false,
+        "auto_min": "10s",
+        "auto_count": 30
       },
       {
-        "current": {},
+        "type": "query",
+        "name": "namespace",
+        "label": "Namespace",
+        "skipUrlSync": false,
+        "query": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)",
         "datasource": {
           "type": "prometheus",
-          "uid": "${DS_VM-SHORTTERM}"
+          "uid": "$ds_prometheus"
+        },
+        "current": {
+          "selected": true,
+          "text": "All",
+          "value": "$__all"
         },
-        "definition": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)",
-        "includeAll": true,
-        "label": "Namespace",
         "multi": true,
-        "name": "namespace",
-        "options": [],
-        "query": {
-          "query": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)",
-          "refId": "StandardVariableQuery"
-        },
+        "allowCustomValue": true,
         "refresh": 2,
-        "regex": "",
         "sort": 1,
-        "type": "query"
+        "includeAll": true,
+        "auto": false,
+        "auto_min": "10s",
+        "auto_count": 30
       }
     ]
   },
-  "time": {
-    "from": "now-1h",
-    "to": "now"
-  },
-  "timepicker": {},
-  "timezone": "",
-  "title": "GPU Efficiency Score",
-  "uid": "gpu-efficiency",
-  "version": 2,
-  "weekStart": ""
+  "annotations": {}
 }
\ No newline at end of file
diff --git a/dashboards/gpu/gpu-performance.json b/dashboards/gpu/gpu-performance.json
index 906d6a11..b0f26958 100644
--- a/dashboards/gpu/gpu-performance.json
+++ b/dashboards/gpu/gpu-performance.json
@@ -1,65 +1,24 @@
 {
-  "__inputs": [
-    {
-      "name": "DS_VM-SHORTTERM",
-      "label": "vm-shortterm",
-      "description": "",
-      "type": "datasource",
-      "pluginId": "prometheus",
-      "pluginName": "Prometheus"
-    }
+  "uid": "gpu-performance",
+  "title": "GPU Performance",
+  "tags": [
+    "gpu",
+    "dcgm"
   ],
-  "__elements": {},
-  "__requires": [
-    {
-      "type": "grafana",
-      "id": "grafana",
-      "name": "Grafana",
-      "version": "11.4.0"
-    },
-    {
-      "type": "datasource",
-      "id": "prometheus",
-      "name": "Prometheus",
-      "version": "1.0.0"
-    },
-    {
-      "type": "panel",
-      "id": "stat",
-      "name": "Stat",
-      "version": ""
-    },
-    {
-      "type": "panel",
-      "id": "timeseries",
-      "name": "Time series",
-      "version": ""
-    }
-  ],
-  "annotations": {
-    "list": [
-      {
-        "builtIn": 1,
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "type": "dashboard"
-      }
-    ]
-  },
+  "timezone": "browser",
   "editable": true,
-  "fiscalYearStartMonth": 0,
   "graphTooltip": 1,
-  "id": null,
-  "links": [],
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "fiscalYearStartMonth": 0,
+  "schemaVersion": 42,
   "panels": [
     {
+      "type": "row",
       "collapsed": false,
+      "title": "Overview",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -67,99 +26,75 @@
         "y": 0
       },
       "id": 1,
-      "panels": [],
-      "title": "Overview",
-      "type": "row"
+      "panels": []
     },
     {
+      "type": "stat",
+      "id": 2,
+      "targets": [
+        {
+          "expr": "cluster:gpu_count:total",
+          "refId": "A"
+        }
+      ],
+      "title": "Total GPUs",
+      "transparent": false,
       "datasource": {
         "type": "prometheus",
         "uid": "$ds_prometheus"
       },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "blue",
-                "value": null
-              }
-            ]
-          },
-          "unit": "short"
-        },
-        "overrides": []
-      },
       "gridPos": {
         "h": 4,
         "w": 6,
         "x": 0,
         "y": 1
       },
-      "id": 2,
+      "repeatDirection": "h",
       "options": {
-        "colorMode": "value",
         "graphMode": "none",
+        "colorMode": "value",
         "justifyMode": "auto",
-        "orientation": "auto",
-        "percentChangeColorMode": "standard",
+        "textMode": "value",
+        "wideLayout": true,
+        "showPercentChange": false,
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
+          ]
         },
-        "showPercentChange": false,
-        "textMode": "value",
-        "wideLayout": true
-      },
-      "pluginVersion": "11.4.0",
-      "targets": [
-        {
-          "expr": "cluster:gpu_count:total",
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "$ds_prometheus"
-          }
-        }
-      ],
-      "title": "Total GPUs",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${DS_VM-SHORTTERM}"
+        "percentChangeColorMode": "standard",
+        "orientation": ""
       },
       "fieldConfig": {
         "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "mappings": [],
+          "unit": "short",
           "thresholds": {
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "yellow",
-                "value": 1
+                "value": null,
+                "color": "blue"
               }
             ]
-          },
-          "unit": "short"
+          }
         },
         "overrides": []
+      }
+    },
+    {
+      "type": "stat",
+      "id": 3,
+      "targets": [
+        {
+          "expr": "cluster:gpu_count:allocated",
+          "refId": "A"
+        }
+      ],
+      "title": "Allocated",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
       },
       "gridPos": {
         "h": 4,
@@ -167,71 +102,56 @@
         "x": 6,
         "y": 1
       },
-      "id": 3,
+      "repeatDirection": "h",
       "options": {
-        "colorMode": "value",
         "graphMode": "none",
+        "colorMode": "value",
         "justifyMode": "auto",
-        "orientation": "auto",
-        "percentChangeColorMode": "standard",
+        "textMode": "value",
+        "wideLayout": true,
+        "showPercentChange": false,
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
+          ]
         },
-        "showPercentChange": false,
-        "textMode": "value",
-        "wideLayout": true
-      },
-      "pluginVersion": "11.4.0",
-      "targets": [
-        {
-          "expr": "cluster:gpu_count:allocated",
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_VM-SHORTTERM}"
-          }
-        }
-      ],
-      "title": "Allocated",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
+        "percentChangeColorMode": "standard",
+        "orientation": ""
       },
       "fieldConfig": {
         "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "mappings": [],
-          "max": 100,
-          "min": 0,
+          "unit": "short",
           "thresholds": {
             "mode": "absolute",
             "steps": [
               {
-                "color": "blue",
-                "value": null
+                "value": null,
+                "color": "green"
               },
               {
-                "color": "green",
-                "value": 30
-              },
-              {
-                "color": "orange",
-                "value": 80
+                "value": 1,
+                "color": "yellow"
               }
             ]
-          },
-          "unit": "percent"
+          }
         },
         "overrides": []
+      }
+    },
+    {
+      "type": "stat",
+      "id": 4,
+      "targets": [
+        {
+          "expr": "cluster:gpu_util:avg",
+          "refId": "A"
+        }
+      ],
+      "title": "Average utilization",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
       },
       "gridPos": {
         "h": 4,
@@ -239,62 +159,62 @@
         "x": 12,
         "y": 1
       },
-      "id": 4,
+      "repeatDirection": "h",
       "options": {
-        "colorMode": "value",
         "graphMode": "area",
+        "colorMode": "value",
         "justifyMode": "auto",
-        "orientation": "auto",
-        "percentChangeColorMode": "standard",
+        "textMode": "value",
+        "wideLayout": true,
+        "showPercentChange": false,
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
+          ]
         },
-        "showPercentChange": false,
-        "textMode": "value",
-        "wideLayout": true
-      },
-      "pluginVersion": "11.4.0",
-      "targets": [
-        {
-          "expr": "cluster:gpu_util:avg",
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "$ds_prometheus"
-          }
-        }
-      ],
-      "title": "Average utilization",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${DS_VM-SHORTTERM}"
+        "percentChangeColorMode": "standard",
+        "orientation": ""
       },
       "fieldConfig": {
         "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "decimals": 0,
-          "mappings": [],
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
           "thresholds": {
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
-                "value": null
+                "value": null,
+                "color": "blue"
+              },
+              {
+                "value": 30,
+                "color": "green"
+              },
+              {
+                "value": 80,
+                "color": "orange"
               }
             ]
-          },
-          "unit": "watt"
+          }
         },
         "overrides": []
+      }
+    },
+    {
+      "type": "stat",
+      "id": 5,
+      "targets": [
+        {
+          "expr": "cluster:gpu_power_watts:sum",
+          "refId": "A"
+        }
+      ],
+      "title": "Power draw",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
       },
       "gridPos": {
         "h": 4,
@@ -302,40 +222,43 @@
         "x": 18,
         "y": 1
       },
-      "id": 5,
+      "repeatDirection": "h",
       "options": {
-        "colorMode": "value",
         "graphMode": "area",
+        "colorMode": "value",
         "justifyMode": "auto",
-        "orientation": "auto",
-        "percentChangeColorMode": "standard",
+        "textMode": "value",
+        "wideLayout": true,
+        "showPercentChange": false,
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
+          ]
         },
-        "showPercentChange": false,
-        "textMode": "value",
-        "wideLayout": true
+        "percentChangeColorMode": "standard",
+        "orientation": ""
       },
-      "pluginVersion": "11.4.0",
-      "targets": [
-        {
-          "expr": "cluster:gpu_power_watts:sum",
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_VM-SHORTTERM}"
+      "fieldConfig": {
+        "defaults": {
+          "unit": "watt",
+          "decimals": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "value": null,
+                "color": "green"
+              }
+            ]
           }
-        }
-      ],
-      "title": "Power draw",
-      "type": "stat"
+        },
+        "overrides": []
+      }
     },
     {
+      "type": "row",
       "collapsed": false,
+      "title": "Utilization",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -343,172 +266,77 @@
         "y": 5
       },
       "id": 10,
-      "panels": [],
-      "title": "Utilization",
-      "type": "row"
+      "panels": []
     },
     {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "barWidthFactor": 0.6,
-            "drawStyle": "line",
-            "fillOpacity": 10,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "never",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "max": 100,
-          "min": 0,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "percent"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 6
-      },
+      "type": "timeseries",
       "id": 11,
-      "options": {
-        "legend": {
-          "calcs": [
-            "mean",
-            "max"
-          ],
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "multi",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "11.4.0",
       "targets": [
         {
           "expr": "DCGM_FI_DEV_GPU_UTIL{Hostname=~\"$Hostname\", namespace=~\"$namespace\"}",
           "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}",
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "$ds_prometheus"
-          }
+          "refId": "A"
         }
       ],
       "title": "GPU Utilization (NVML)",
-      "type": "timeseries"
-    },
-    {
+      "transparent": false,
       "datasource": {
         "type": "prometheus",
-        "uid": "${DS_VM-SHORTTERM}"
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 6
+      },
+      "repeatDirection": "h",
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": false,
+          "calcs": [
+            "mean",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": ""
+        }
       },
       "fieldConfig": {
         "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "barWidthFactor": 0.6,
-            "drawStyle": "line",
-            "fillOpacity": 10,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "never",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "max": 100,
+          "unit": "percent",
           "min": 0,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "percent"
+          "max": 100,
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "linear",
+            "fillOpacity": 10,
+            "showPoints": "never",
+            "spanNulls": false
+          }
         },
         "overrides": []
+      }
+    },
+    {
+      "type": "timeseries",
+      "id": 12,
+      "targets": [
+        {
+          "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 100",
+          "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Tensor Pipe Active (realistic load for LLM/AI)",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
       },
       "gridPos": {
         "h": 8,
@@ -516,99 +344,53 @@
         "x": 12,
         "y": 6
       },
-      "id": 12,
+      "repeatDirection": "h",
       "options": {
         "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": false,
           "calcs": [
             "mean",
             "max"
-          ],
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": true
+          ]
         },
         "tooltip": {
           "mode": "multi",
-          "sort": "none"
+          "sort": ""
         }
       },
-      "pluginVersion": "11.4.0",
-      "targets": [
-        {
-          "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 100",
-          "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}",
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_VM-SHORTTERM}"
-          }
-        }
-      ],
-      "title": "Tensor Pipe Active (realistic load for LLM/AI)",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
       "fieldConfig": {
         "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "barWidthFactor": 0.6,
-            "drawStyle": "line",
-            "fillOpacity": 10,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "never",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "max": 100,
+          "unit": "percent",
           "min": 0,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "percent"
+          "max": 100,
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "linear",
+            "fillOpacity": 10,
+            "showPoints": "never",
+            "spanNulls": false
+          }
         },
         "overrides": []
+      }
+    },
+    {
+      "type": "timeseries",
+      "id": 13,
+      "targets": [
+        {
+          "expr": "DCGM_FI_PROF_GR_ENGINE_ACTIVE{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 100",
+          "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Graphics Engine Active",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
       },
       "gridPos": {
         "h": 8,
@@ -616,99 +398,53 @@
         "x": 0,
         "y": 14
       },
-      "id": 13,
+      "repeatDirection": "h",
       "options": {
         "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": false,
           "calcs": [
             "mean",
             "max"
-          ],
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": true
+          ]
         },
         "tooltip": {
           "mode": "multi",
-          "sort": "none"
+          "sort": ""
         }
       },
-      "pluginVersion": "11.4.0",
-      "targets": [
-        {
-          "expr": "DCGM_FI_PROF_GR_ENGINE_ACTIVE{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 100",
-          "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}",
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "$ds_prometheus"
-          }
-        }
-      ],
-      "title": "Graphics Engine Active",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${DS_VM-SHORTTERM}"
-      },
       "fieldConfig": {
         "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "barWidthFactor": 0.6,
-            "drawStyle": "line",
-            "fillOpacity": 10,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "never",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "max": 100,
+          "unit": "percent",
           "min": 0,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "percent"
+          "max": 100,
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "linear",
+            "fillOpacity": 10,
+            "showPoints": "never",
+            "spanNulls": false
+          }
         },
         "overrides": []
+      }
+    },
+    {
+      "type": "timeseries",
+      "id": 14,
+      "targets": [
+        {
+          "expr": "DCGM_FI_DEV_MEM_COPY_UTIL{Hostname=~\"$Hostname\", namespace=~\"$namespace\"}",
+          "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Memory Copy Utilization",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
       },
       "gridPos": {
         "h": 8,
@@ -716,39 +452,42 @@
         "x": 12,
         "y": 14
       },
-      "id": 14,
+      "repeatDirection": "h",
       "options": {
         "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": false,
           "calcs": [
             "mean",
             "max"
-          ],
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": true
+          ]
         },
         "tooltip": {
           "mode": "multi",
-          "sort": "none"
+          "sort": ""
         }
       },
-      "pluginVersion": "11.4.0",
-      "targets": [
-        {
-          "expr": "DCGM_FI_DEV_MEM_COPY_UTIL{Hostname=~\"$Hostname\", namespace=~\"$namespace\"}",
-          "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}",
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_VM-SHORTTERM}"
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "linear",
+            "fillOpacity": 10,
+            "showPoints": "never",
+            "spanNulls": false
           }
-        }
-      ],
-      "title": "Memory Copy Utilization",
-      "type": "timeseries"
+        },
+        "overrides": []
+      }
     },
     {
+      "type": "row",
       "collapsed": false,
+      "title": "Memory",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -756,168 +495,74 @@
         "y": 22
       },
       "id": 20,
-      "panels": [],
-      "title": "Memory",
-      "type": "row"
+      "panels": []
     },
     {
+      "type": "timeseries",
+      "id": 21,
+      "targets": [
+        {
+          "expr": "DCGM_FI_DEV_FB_USED{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 1048576",
+          "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}",
+          "refId": "A"
+        }
+      ],
+      "title": "VRAM Used",
+      "transparent": false,
       "datasource": {
         "type": "prometheus",
         "uid": "$ds_prometheus"
       },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "barWidthFactor": 0.6,
-            "drawStyle": "line",
-            "fillOpacity": 25,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "never",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "bytes"
-        },
-        "overrides": []
-      },
       "gridPos": {
         "h": 8,
         "w": 12,
         "x": 0,
         "y": 23
       },
-      "id": 21,
+      "repeatDirection": "h",
       "options": {
         "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": false,
           "calcs": [
             "mean",
             "max"
-          ],
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": true
+          ]
         },
         "tooltip": {
           "mode": "multi",
-          "sort": "none"
+          "sort": ""
         }
       },
-      "pluginVersion": "11.4.0",
-      "targets": [
-        {
-          "expr": "DCGM_FI_DEV_FB_USED{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 1048576",
-          "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}",
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "$ds_prometheus"
-          }
-        }
-      ],
-      "title": "VRAM Used",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${DS_VM-SHORTTERM}"
-      },
       "fieldConfig": {
         "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
+          "unit": "bytes",
           "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "barWidthFactor": 0.6,
             "drawStyle": "line",
-            "fillOpacity": 10,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
             "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "never",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "bytes"
+            "fillOpacity": 25,
+            "showPoints": "never"
+          }
         },
         "overrides": []
+      }
+    },
+    {
+      "type": "timeseries",
+      "id": 22,
+      "targets": [
+        {
+          "expr": "DCGM_FI_DEV_FB_FREE{Hostname=~\"$Hostname\"} * 1048576",
+          "legendFormat": "{{Hostname}}/{{gpu}}",
+          "refId": "A"
+        }
+      ],
+      "title": "VRAM Free",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
       },
       "gridPos": {
         "h": 8,
@@ -925,39 +570,39 @@
         "x": 12,
         "y": 23
       },
-      "id": 22,
+      "repeatDirection": "h",
       "options": {
         "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": false,
           "calcs": [
             "mean",
             "min"
-          ],
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": true
+          ]
         },
         "tooltip": {
           "mode": "multi",
-          "sort": "none"
+          "sort": ""
         }
       },
-      "pluginVersion": "11.4.0",
-      "targets": [
-        {
-          "expr": "DCGM_FI_DEV_FB_FREE{Hostname=~\"$Hostname\"} * 1048576",
-          "legendFormat": "{{Hostname}}/{{gpu}}",
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_VM-SHORTTERM}"
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "linear",
+            "fillOpacity": 10,
+            "showPoints": "never"
           }
-        }
-      ],
-      "title": "VRAM Free",
-      "type": "timeseries"
+        },
+        "overrides": []
+      }
     },
     {
+      "type": "row",
       "collapsed": false,
+      "title": "Power \u0026 Temperature",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -965,174 +610,74 @@
         "y": 31
       },
       "id": 30,
-      "panels": [],
-      "title": "Power & Temperature",
-      "type": "row"
+      "panels": []
     },
     {
+      "type": "timeseries",
+      "id": 31,
+      "targets": [
+        {
+          "expr": "DCGM_FI_DEV_POWER_USAGE{Hostname=~\"$Hostname\"}",
+          "legendFormat": "{{Hostname}}/{{gpu}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Power Usage per GPU",
+      "transparent": false,
       "datasource": {
         "type": "prometheus",
         "uid": "$ds_prometheus"
       },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "barWidthFactor": 0.6,
-            "drawStyle": "line",
-            "fillOpacity": 10,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "never",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "watt"
-        },
-        "overrides": []
-      },
       "gridPos": {
         "h": 8,
         "w": 12,
         "x": 0,
         "y": 32
       },
-      "id": 31,
+      "repeatDirection": "h",
       "options": {
         "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": false,
           "calcs": [
             "mean",
             "max"
-          ],
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": true
+          ]
         },
         "tooltip": {
           "mode": "multi",
-          "sort": "none"
+          "sort": ""
         }
       },
-      "pluginVersion": "11.4.0",
-      "targets": [
-        {
-          "expr": "DCGM_FI_DEV_POWER_USAGE{Hostname=~\"$Hostname\"}",
-          "legendFormat": "{{Hostname}}/{{gpu}}",
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "$ds_prometheus"
-          }
-        }
-      ],
-      "title": "Power Usage per GPU",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${DS_VM-SHORTTERM}"
-      },
       "fieldConfig": {
         "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
+          "unit": "watt",
           "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "barWidthFactor": 0.6,
             "drawStyle": "line",
-            "fillOpacity": 10,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
             "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "never",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "max": 100,
-          "min": 20,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "yellow",
-                "value": 75
-              },
-              {
-                "color": "red",
-                "value": 85
-              }
-            ]
-          },
-          "unit": "celsius"
+            "fillOpacity": 10,
+            "showPoints": "never"
+          }
         },
         "overrides": []
+      }
+    },
+    {
+      "type": "timeseries",
+      "id": 32,
+      "targets": [
+        {
+          "expr": "DCGM_FI_DEV_GPU_TEMP{Hostname=~\"$Hostname\"}",
+          "legendFormat": "{{Hostname}}/{{gpu}}",
+          "refId": "A"
+        }
+      ],
+      "title": "GPU Temperature",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
       },
       "gridPos": {
         "h": 8,
@@ -1140,39 +685,58 @@
         "x": 12,
         "y": 32
       },
-      "id": 32,
+      "repeatDirection": "h",
       "options": {
         "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": false,
           "calcs": [
             "mean",
             "max"
-          ],
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": true
+          ]
         },
         "tooltip": {
           "mode": "multi",
-          "sort": "none"
+          "sort": ""
         }
       },
-      "pluginVersion": "11.4.0",
-      "targets": [
-        {
-          "expr": "DCGM_FI_DEV_GPU_TEMP{Hostname=~\"$Hostname\"}",
-          "legendFormat": "{{Hostname}}/{{gpu}}",
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_VM-SHORTTERM}"
+      "fieldConfig": {
+        "defaults": {
+          "unit": "celsius",
+          "min": 20,
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "value": null,
+                "color": "green"
+              },
+              {
+                "value": 75,
+                "color": "yellow"
+              },
+              {
+                "value": 85,
+                "color": "red"
+              }
+            ]
+          },
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "linear",
+            "fillOpacity": 10,
+            "showPoints": "never"
           }
-        }
-      ],
-      "title": "GPU Temperature",
-      "type": "timeseries"
+        },
+        "overrides": []
+      }
     },
     {
+      "type": "row",
       "collapsed": false,
+      "title": "Health",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -1180,85 +744,81 @@
         "y": 40
       },
       "id": 40,
-      "panels": [],
-      "title": "Health",
-      "type": "row"
+      "panels": []
     },
     {
+      "type": "stat",
+      "id": 41,
+      "targets": [
+        {
+          "expr": "max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS{Hostname=~\"$Hostname\"})",
+          "legendFormat": "{{Hostname}}/{{gpu}}",
+          "refId": "A"
+        }
+      ],
+      "title": "XID errors (latest)",
+      "transparent": false,
       "datasource": {
         "type": "prometheus",
         "uid": "$ds_prometheus"
       },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green"
-              },
-              {
-                "color": "red",
-                "value": 1
-              }
-            ]
-          },
-          "unit": "short"
-        },
-        "overrides": []
-      },
       "gridPos": {
         "h": 5,
         "w": 8,
         "x": 0,
         "y": 41
       },
-      "id": 41,
+      "repeatDirection": "h",
       "options": {
-        "colorMode": "background",
         "graphMode": "none",
+        "colorMode": "background",
+        "justifyMode": "auto",
+        "textMode": "value_and_name",
+        "wideLayout": true,
+        "showPercentChange": false,
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
+          ]
         },
-        "textMode": "value_and_name"
-      },
-      "targets": [
-        {
-          "expr": "max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS{Hostname=~\"$Hostname\"})",
-          "legendFormat": "{{Hostname}}/{{gpu}}",
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "$ds_prometheus"
-          }
-        }
-      ],
-      "title": "XID errors (latest)",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${DS_VM-SHORTTERM}"
+        "percentChangeColorMode": "standard",
+        "orientation": ""
       },
       "fieldConfig": {
         "defaults": {
-          "custom": {
-            "drawStyle": "line",
-            "fillOpacity": 10,
-            "lineInterpolation": "linear",
-            "showPoints": "never"
-          },
-          "unit": "µs"
+          "unit": "short",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "value": null,
+                "color": "green"
+              },
+              {
+                "value": 1,
+                "color": "red"
+              }
+            ]
+          }
         },
         "overrides": []
+      }
+    },
+    {
+      "type": "timeseries",
+      "id": 42,
+      "targets": [
+        {
+          "expr": "rate(DCGM_FI_DEV_POWER_VIOLATION{Hostname=~\"$Hostname\"}[5m])",
+          "legendFormat": "{{Hostname}}/{{gpu}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Power Violation (µs/s throttled due to power)",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
       },
       "gridPos": {
         "h": 5,
@@ -1266,46 +826,47 @@
         "x": 8,
         "y": 41
       },
-      "id": 42,
+      "repeatDirection": "h",
       "options": {
         "legend": {
           "displayMode": "list",
-          "placement": "bottom"
+          "placement": "bottom",
+          "showLegend": false,
+          "calcs": []
         },
         "tooltip": {
-          "mode": "multi"
+          "mode": "multi",
+          "sort": ""
         }
       },
-      "targets": [
-        {
-          "expr": "rate(DCGM_FI_DEV_POWER_VIOLATION{Hostname=~\"$Hostname\"}[5m])",
-          "legendFormat": "{{Hostname}}/{{gpu}}",
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_VM-SHORTTERM}"
-          }
-        }
-      ],
-      "title": "Power Violation (µs/s throttled due to power)",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
       "fieldConfig": {
         "defaults": {
+          "unit": "µs",
           "custom": {
             "drawStyle": "line",
-            "fillOpacity": 10,
             "lineInterpolation": "linear",
+            "fillOpacity": 10,
             "showPoints": "never"
-          },
-          "unit": "µs"
+          }
         },
         "overrides": []
+      }
+    },
+    {
+      "type": "timeseries",
+      "id": 43,
+      "targets": [
+        {
+          "expr": "rate(DCGM_FI_DEV_THERMAL_VIOLATION{Hostname=~\"$Hostname\"}[5m])",
+          "legendFormat": "{{Hostname}}/{{gpu}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Thermal Violation (µs/s throttled due to thermals)",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
       },
       "gridPos": {
         "h": 5,
@@ -1313,102 +874,103 @@
         "x": 16,
         "y": 41
       },
-      "id": 43,
+      "repeatDirection": "h",
       "options": {
         "legend": {
           "displayMode": "list",
-          "placement": "bottom"
+          "placement": "bottom",
+          "showLegend": false,
+          "calcs": []
         },
         "tooltip": {
-          "mode": "multi"
+          "mode": "multi",
+          "sort": ""
         }
       },
-      "targets": [
-        {
-          "expr": "rate(DCGM_FI_DEV_THERMAL_VIOLATION{Hostname=~\"$Hostname\"}[5m])",
-          "legendFormat": "{{Hostname}}/{{gpu}}",
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "$ds_prometheus"
+      "fieldConfig": {
+        "defaults": {
+          "unit": "µs",
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "linear",
+            "fillOpacity": 10,
+            "showPoints": "never"
           }
-        }
-      ],
-      "title": "Thermal Violation (µs/s throttled due to thermals)",
-      "type": "timeseries"
+        },
+        "overrides": []
+      }
     }
   ],
-  "refresh": "",
-  "schemaVersion": 40,
-  "tags": [
-    "gpu",
-    "dcgm"
-  ],
   "templating": {
     "list": [
       {
-        "current": {},
-        "includeAll": false,
-        "label": "Prometheus",
+        "type": "datasource",
         "name": "ds_prometheus",
-        "options": [],
+        "label": "Prometheus",
+        "skipUrlSync": false,
         "query": "prometheus",
-        "refresh": 1,
+        "current": {
+          "selected": false,
+          "text": "default",
+          "value": "default"
+        },
+        "multi": false,
+        "allowCustomValue": true,
+        "includeAll": false,
         "regex": "",
-        "type": "datasource"
+        "auto": false,
+        "auto_min": "10s",
+        "auto_count": 30
       },
       {
-        "current": {},
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_VM-SHORTTERM}"
-        },
-        "definition": "label_values(DCGM_FI_DEV_GPU_UTIL, Hostname)",
-        "includeAll": true,
-        "label": "Host",
-        "multi": true,
+        "type": "query",
         "name": "Hostname",
-        "options": [],
-        "query": {
-          "query": "label_values(DCGM_FI_DEV_GPU_UTIL, Hostname)",
-          "refId": "StandardVariableQuery"
-        },
-        "refresh": 2,
-        "regex": "",
-        "sort": 1,
-        "type": "query"
-      },
-      {
-        "current": {},
+        "label": "Host",
+        "skipUrlSync": false,
+        "query": "label_values(DCGM_FI_DEV_GPU_UTIL, Hostname)",
         "datasource": {
           "type": "prometheus",
           "uid": "$ds_prometheus"
         },
-        "definition": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)",
-        "includeAll": true,
-        "label": "Namespace",
-        "multi": true,
-        "name": "namespace",
-        "options": [],
-        "query": {
-          "query": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)",
-          "refId": "StandardVariableQuery"
+        "current": {
+          "selected": true,
+          "text": "All",
+          "value": "$__all"
         },
+        "multi": true,
+        "allowCustomValue": true,
         "refresh": 2,
-        "regex": "",
         "sort": 1,
-        "type": "query"
+        "includeAll": true,
+        "auto": false,
+        "auto_min": "10s",
+        "auto_count": 30
+      },
+      {
+        "type": "query",
+        "name": "namespace",
+        "label": "Namespace",
+        "skipUrlSync": false,
+        "query": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "$ds_prometheus"
+        },
+        "current": {
+          "selected": true,
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "allowCustomValue": true,
+        "refresh": 2,
+        "sort": 1,
+        "includeAll": true,
+        "auto": false,
+        "auto_min": "10s",
+        "auto_count": 30
       }
     ]
   },
-  "time": {
-    "from": "now-1h",
-    "to": "now"
-  },
-  "timepicker": {},
-  "timezone": "",
-  "title": "GPU Performance",
-  "uid": "gpu-performance",
-  "version": 2,
-  "weekStart": ""
+  "annotations": {}
 }
diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json
index 28f2c892..840065f3 100644
--- a/dashboards/gpu/gpu-quotas.json
+++ b/dashboards/gpu/gpu-quotas.json
@@ -1,83 +1,24 @@
 {
-  "__inputs": [
-    {
-      "name": "DS_VM-SHORTTERM",
-      "label": "vm-shortterm",
-      "description": "",
-      "type": "datasource",
-      "pluginId": "prometheus",
-      "pluginName": "Prometheus"
-    }
+  "uid": "gpu-quotas",
+  "title": "GPU Quotas \u0026 Allocation",
+  "tags": [
+    "gpu",
+    "quotas"
   ],
-  "__elements": {},
-  "__requires": [
-    {
-      "type": "panel",
-      "id": "bargauge",
-      "name": "Bar gauge",
-      "version": ""
-    },
-    {
-      "type": "panel",
-      "id": "gauge",
-      "name": "Gauge",
-      "version": ""
-    },
-    {
-      "type": "grafana",
-      "id": "grafana",
-      "name": "Grafana",
-      "version": "11.4.0"
-    },
-    {
-      "type": "datasource",
-      "id": "prometheus",
-      "name": "Prometheus",
-      "version": "1.0.0"
-    },
-    {
-      "type": "panel",
-      "id": "stat",
-      "name": "Stat",
-      "version": ""
-    },
-    {
-      "type": "panel",
-      "id": "table",
-      "name": "Table",
-      "version": ""
-    },
-    {
-      "type": "panel",
-      "id": "timeseries",
-      "name": "Time series",
-      "version": ""
-    }
-  ],
-  "annotations": {
-    "list": [
-      {
-        "builtIn": 1,
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "type": "dashboard"
-      }
-    ]
-  },
+  "timezone": "browser",
   "editable": true,
-  "fiscalYearStartMonth": 0,
   "graphTooltip": 1,
-  "id": null,
-  "links": [],
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "fiscalYearStartMonth": 0,
+  "schemaVersion": 42,
   "panels": [
     {
+      "type": "row",
       "collapsed": false,
+      "title": "Allocation overview",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -85,99 +26,75 @@
         "y": 0
       },
       "id": 1,
-      "panels": [],
-      "title": "Allocation overview",
-      "type": "row"
+      "panels": []
     },
     {
+      "type": "stat",
+      "id": 2,
+      "targets": [
+        {
+          "expr": "sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"})",
+          "refId": "A"
+        }
+      ],
+      "title": "GPU allocatable",
+      "transparent": false,
       "datasource": {
         "type": "prometheus",
         "uid": "$ds_prometheus"
       },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "blue",
-                "value": null
-              }
-            ]
-          },
-          "unit": "short"
-        },
-        "overrides": []
-      },
       "gridPos": {
         "h": 4,
         "w": 6,
         "x": 0,
         "y": 1
       },
-      "id": 2,
+      "repeatDirection": "h",
       "options": {
-        "colorMode": "value",
         "graphMode": "none",
+        "colorMode": "value",
         "justifyMode": "auto",
-        "orientation": "auto",
-        "percentChangeColorMode": "standard",
+        "textMode": "value",
+        "wideLayout": true,
+        "showPercentChange": false,
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
+          ]
         },
-        "showPercentChange": false,
-        "textMode": "value",
-        "wideLayout": true
-      },
-      "pluginVersion": "11.4.0",
-      "targets": [
-        {
-          "expr": "sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"})",
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "$ds_prometheus"
-          }
-        }
-      ],
-      "title": "GPU allocatable",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${DS_VM-SHORTTERM}"
+        "percentChangeColorMode": "standard",
+        "orientation": ""
       },
       "fieldConfig": {
         "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "mappings": [],
+          "unit": "short",
           "thresholds": {
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "yellow",
-                "value": 1
+                "value": null,
+                "color": "blue"
               }
             ]
-          },
-          "unit": "short"
+          }
         },
         "overrides": []
+      }
+    },
+    {
+      "type": "stat",
+      "id": 3,
+      "targets": [
+        {
+          "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"})",
+          "refId": "A"
+        }
+      ],
+      "title": "GPU requested",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
       },
       "gridPos": {
         "h": 4,
@@ -185,75 +102,56 @@
         "x": 6,
         "y": 1
       },
-      "id": 3,
+      "repeatDirection": "h",
       "options": {
-        "colorMode": "value",
         "graphMode": "none",
+        "colorMode": "value",
         "justifyMode": "auto",
-        "orientation": "auto",
-        "percentChangeColorMode": "standard",
+        "textMode": "value",
+        "wideLayout": true,
+        "showPercentChange": false,
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
+          ]
         },
-        "showPercentChange": false,
-        "textMode": "value",
-        "wideLayout": true
-      },
-      "pluginVersion": "11.4.0",
-      "targets": [
-        {
-          "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"})",
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_VM-SHORTTERM}"
-          }
-        }
-      ],
-      "title": "GPU requested",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
+        "percentChangeColorMode": "standard",
+        "orientation": ""
       },
       "fieldConfig": {
         "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "mappings": [],
-          "max": 100,
-          "min": 0,
+          "unit": "short",
           "thresholds": {
             "mode": "absolute",
             "steps": [
               {
-                "color": "blue",
-                "value": null
+                "value": null,
+                "color": "green"
               },
               {
-                "color": "green",
-                "value": 40
-              },
-              {
-                "color": "yellow",
-                "value": 80
-              },
-              {
-                "color": "red",
-                "value": 95
+                "value": 1,
+                "color": "yellow"
               }
             ]
-          },
-          "unit": "percent"
+          }
         },
         "overrides": []
+      }
+    },
+    {
+      "type": "gauge",
+      "id": 4,
+      "targets": [
+        {
+          "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}) / sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"}) * 100",
+          "refId": "A"
+        }
+      ],
+      "title": "Allocation ratio",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
       },
       "gridPos": {
         "h": 4,
@@ -261,63 +159,64 @@
         "x": 12,
         "y": 1
       },
-      "id": 4,
+      "repeatDirection": "h",
       "options": {
-        "minVizHeight": 75,
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true,
+        "sizing": "auto",
         "minVizWidth": 75,
-        "orientation": "auto",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
+          ]
         },
-        "showThresholdLabels": false,
-        "showThresholdMarkers": true,
-        "sizing": "auto"
-      },
-      "pluginVersion": "11.4.0",
-      "targets": [
-        {
-          "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}) / sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"}) * 100",
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "$ds_prometheus"
-          }
-        }
-      ],
-      "title": "Allocation ratio",
-      "type": "gauge"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${DS_VM-SHORTTERM}"
+        "minVizHeight": 75,
+        "orientation": ""
       },
       "fieldConfig": {
         "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "mappings": [],
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
           "thresholds": {
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
-                "value": null
+                "value": null,
+                "color": "blue"
               },
               {
-                "color": "red",
-                "value": 1
+                "value": 40,
+                "color": "green"
+              },
+              {
+                "value": 80,
+                "color": "yellow"
+              },
+              {
+                "value": 95,
+                "color": "red"
               }
             ]
-          },
-          "unit": "short"
+          }
         },
         "overrides": []
+      }
+    },
+    {
+      "type": "stat",
+      "id": 5,
+      "targets": [
+        {
+          "expr": "count((kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"} \u003e 0) * on(namespace, pod) group_left() (kube_pod_status_phase{phase=\"Pending\"} == 1)) or vector(0)",
+          "refId": "A"
+        }
+      ],
+      "title": "Pending pods (GPU)",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
       },
       "gridPos": {
         "h": 4,
@@ -325,40 +224,46 @@
         "x": 18,
         "y": 1
       },
-      "id": 5,
+      "repeatDirection": "h",
       "options": {
-        "colorMode": "background",
         "graphMode": "none",
+        "colorMode": "background",
         "justifyMode": "auto",
-        "orientation": "auto",
-        "percentChangeColorMode": "standard",
+        "textMode": "value",
+        "wideLayout": true,
+        "showPercentChange": false,
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
+          ]
         },
-        "showPercentChange": false,
-        "textMode": "value",
-        "wideLayout": true
+        "percentChangeColorMode": "standard",
+        "orientation": ""
       },
-      "pluginVersion": "11.4.0",
-      "targets": [
-        {
-          "expr": "count((kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"} > 0) * on(namespace, pod) group_left() (kube_pod_status_phase{phase=\"Pending\"} == 1)) or vector(0)",
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_VM-SHORTTERM}"
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "value": null,
+                "color": "green"
+              },
+              {
+                "value": 1,
+                "color": "red"
+              }
+            ]
           }
-        }
-      ],
-      "title": "Pending pods (GPU)",
-      "type": "stat"
+        },
+        "overrides": []
+      }
     },
     {
+      "type": "row",
       "collapsed": false,
+      "title": "Per namespace",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -366,140 +271,120 @@
         "y": 5
       },
       "id": 10,
-      "panels": [],
-      "title": "Per namespace",
-      "type": "row"
+      "panels": []
     },
     {
+      "type": "bargauge",
+      "id": 11,
+      "targets": [
+        {
+          "expr": "sum by (namespace) (kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"})",
+          "legendFormat": "{{namespace}}",
+          "refId": "A"
+        }
+      ],
+      "title": "GPU requested per namespace",
+      "transparent": false,
       "datasource": {
         "type": "prometheus",
         "uid": "$ds_prometheus"
       },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "mappings": [],
-          "max": 8,
-          "min": 0,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          },
-          "unit": "short"
-        },
-        "overrides": []
-      },
       "gridPos": {
         "h": 8,
         "w": 12,
         "x": 0,
         "y": 6
       },
-      "id": 11,
+      "repeatDirection": "h",
       "options": {
         "displayMode": "gradient",
+        "valueMode": "color",
+        "namePlacement": "auto",
+        "showUnfilled": true,
+        "sizing": "auto",
+        "minVizWidth": 8,
+        "minVizHeight": 16,
         "legend": {
-          "calcs": [],
           "displayMode": "list",
           "placement": "bottom",
-          "showLegend": false
+          "showLegend": false,
+          "calcs": []
         },
-        "maxVizHeight": 300,
-        "minVizHeight": 16,
-        "minVizWidth": 8,
-        "namePlacement": "auto",
-        "orientation": "horizontal",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
+          ]
         },
-        "showUnfilled": true,
-        "sizing": "auto",
-        "valueMode": "color"
-      },
-      "pluginVersion": "11.4.0",
-      "targets": [
-        {
-          "expr": "sum by (namespace) (kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"})",
-          "legendFormat": "{{namespace}}",
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "$ds_prometheus"
-          }
-        }
-      ],
-      "title": "GPU requested per namespace",
-      "type": "bargauge"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${DS_VM-SHORTTERM}"
+        "maxVizHeight": 300,
+        "orientation": "horizontal"
       },
       "fieldConfig": {
         "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "barWidthFactor": 0.6,
-            "drawStyle": "line",
-            "fillOpacity": 10,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "stepAfter",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "never",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
+          "unit": "short",
+          "min": 0,
+          "max": 8,
           "thresholds": {
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
+                "value": null,
+                "color": "green"
               }
             ]
-          },
-          "unit": "short"
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "timeseries",
+      "id": 12,
+      "targets": [
+        {
+          "expr": "sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"})",
+          "legendFormat": "Allocatable (total)",
+          "refId": "A"
+        },
+        {
+          "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"})",
+          "legendFormat": "Requested",
+          "refId": "B"
+        }
+      ],
+      "title": "GPU allocated over time",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 6
+      },
+      "repeatDirection": "h",
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": false,
+          "calcs": []
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": ""
+        }
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "stepAfter",
+            "fillOpacity": 10,
+            "showPoints": "never"
+          }
         },
         "overrides": [
           {
@@ -518,52 +403,12 @@
             ]
           }
         ]
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 6
-      },
-      "id": 12,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "multi",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "11.4.0",
-      "targets": [
-        {
-          "expr": "sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"})",
-          "legendFormat": "Allocatable (total)",
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_VM-SHORTTERM}"
-          }
-        },
-        {
-          "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"})",
-          "legendFormat": "Requested",
-          "refId": "B",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_VM-SHORTTERM}"
-          }
-        }
-      ],
-      "title": "GPU allocated over time",
-      "type": "timeseries"
+      }
     },
     {
+      "type": "row",
       "collapsed": false,
+      "title": "Pods with GPU",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -571,124 +416,40 @@
         "y": 14
       },
       "id": 20,
-      "panels": [],
-      "title": "Pods with GPU",
-      "type": "row"
+      "panels": []
     },
     {
+      "type": "table",
+      "id": 21,
+      "targets": [
+        {
+          "expr": "kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left(phase) (kube_pod_status_phase{phase=~\"Running|Pending|Failed|Succeeded\"} == 1)",
+          "instant": true,
+          "range": false,
+          "format": "table",
+          "refId": "requested"
+        },
+        {
+          "expr": "kube_pod_container_resource_limits{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"}",
+          "instant": true,
+          "range": false,
+          "format": "table",
+          "refId": "limits"
+        }
+      ],
+      "title": "Pods requesting GPU",
+      "transparent": false,
       "datasource": {
         "type": "prometheus",
         "uid": "$ds_prometheus"
       },
-      "fieldConfig": {
-        "defaults": {
-          "custom": {
-            "align": "auto",
-            "cellOptions": {
-              "type": "auto"
-            },
-            "inspect": false
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": [
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Status"
-            },
-            "properties": [
-              {
-                "id": "custom.cellOptions",
-                "value": {
-                  "mode": "basic",
-                  "type": "color-background"
-                }
-              },
-              {
-                "id": "mappings",
-                "value": [
-                  {
-                    "options": {
-                      "Failed": {
-                        "color": "red",
-                        "index": 2
-                      },
-                      "Pending": {
-                        "color": "orange",
-                        "index": 1
-                      },
-                      "Running": {
-                        "color": "green",
-                        "index": 0
-                      }
-                    },
-                    "type": "value"
-                  }
-                ]
-              }
-            ]
-          }
-        ]
-      },
       "gridPos": {
         "h": 10,
         "w": 24,
         "x": 0,
         "y": 15
       },
-      "id": 21,
-      "options": {
-        "cellHeight": "sm",
-        "footer": {
-          "countRows": false,
-          "fields": "",
-          "reducer": [
-            "sum"
-          ],
-          "show": false
-        },
-        "showHeader": true
-      },
-      "pluginVersion": "11.4.0",
-      "targets": [
-        {
-          "expr": "kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left(phase) (kube_pod_status_phase{phase=~\"Running|Pending|Failed|Succeeded\"} == 1)",
-          "format": "table",
-          "instant": true,
-          "legendFormat": "",
-          "refId": "requested",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "$ds_prometheus"
-          }
-        },
-        {
-          "expr": "kube_pod_container_resource_limits{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"}",
-          "format": "table",
-          "instant": true,
-          "legendFormat": "",
-          "refId": "limits",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "$ds_prometheus"
-          }
-        }
-      ],
-      "title": "Pods requesting GPU",
+      "repeatDirection": "h",
       "transformations": [
         {
           "id": "joinByField",
@@ -764,59 +525,106 @@
           }
         }
       ],
-      "type": "table"
+      "options": {
+        "frameIndex": 0,
+        "showHeader": true,
+        "showTypeIcons": false,
+        "footer": {
+          "show": false,
+          "reducer": []
+        },
+        "cellHeight": "sm"
+      },
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Status"
+            },
+            "properties": [
+              {
+                "id": "custom.cellOptions",
+                "value": {
+                  "mode": "basic",
+                  "type": "color-background"
+                }
+              },
+              {
+                "id": "mappings",
+                "value": [
+                  {
+                    "options": {
+                      "Failed": {
+                        "color": "red",
+                        "index": 2
+                      },
+                      "Pending": {
+                        "color": "orange",
+                        "index": 1
+                      },
+                      "Running": {
+                        "color": "green",
+                        "index": 0
+                      }
+                    },
+                    "type": "value"
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      }
     }
   ],
-  "refresh": "",
-  "schemaVersion": 40,
-  "tags": [
-    "gpu",
-    "quotas"
-  ],
   "templating": {
     "list": [
       {
-        "current": {},
-        "includeAll": false,
-        "label": "Prometheus",
+        "type": "datasource",
         "name": "ds_prometheus",
-        "options": [],
+        "label": "Prometheus",
+        "skipUrlSync": false,
         "query": "prometheus",
-        "refresh": 1,
+        "current": {
+          "selected": false,
+          "text": "default",
+          "value": "default"
+        },
+        "multi": false,
+        "allowCustomValue": true,
+        "includeAll": false,
         "regex": "",
-        "type": "datasource"
+        "auto": false,
+        "auto_min": "10s",
+        "auto_count": 30
       },
       {
-        "current": {},
+        "type": "query",
+        "name": "namespace",
+        "label": "Namespace",
+        "skipUrlSync": false,
+        "query": "label_values(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}, namespace)",
         "datasource": {
           "type": "prometheus",
-          "uid": "${DS_VM-SHORTTERM}"
+          "uid": "$ds_prometheus"
+        },
+        "current": {
+          "selected": true,
+          "text": "All",
+          "value": "$__all"
         },
-        "definition": "label_values(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}, namespace)",
-        "includeAll": true,
-        "label": "Namespace",
         "multi": true,
-        "name": "namespace",
-        "options": [],
-        "query": {
-          "query": "label_values(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}, namespace)",
-          "refId": "StandardVariableQuery"
-        },
+        "allowCustomValue": true,
         "refresh": 2,
-        "regex": "",
         "sort": 1,
-        "type": "query"
+        "includeAll": true,
+        "auto": false,
+        "auto_min": "10s",
+        "auto_count": 30
       }
     ]
   },
-  "time": {
-    "from": "now-6h",
-    "to": "now"
-  },
-  "timepicker": {},
-  "timezone": "",
-  "title": "GPU Quotas & Allocation",
-  "uid": "gpu-quotas",
-  "version": 4,
-  "weekStart": ""
-}
+  "annotations": {}
+}
\ No newline at end of file

From 11f7d3589b3284fca96d0df21a02a5e967dadce7 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sat, 18 Apr 2026 05:47:33 +0300
Subject: [PATCH 030/151] chore: ignore CLAUDE.local.md

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 61003de5..64470222 100644
--- a/.gitignore
+++ b/.gitignore
@@ -84,3 +84,4 @@ tmp/
 # build revision marker (generated by make image-packages)
 packages/core/platform/.build-revision
 .claude/
+CLAUDE.local.md

From 4f8cef47bf307d7e23eac6536a1e7153a940d283 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sat, 18 Apr 2026 05:54:21 +0300
Subject: [PATCH 031/151] fix(monitoring): restore trailing newline in GPU
 dashboards

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 dashboards/gpu/gpu-efficiency.json | 2 +-
 dashboards/gpu/gpu-quotas.json     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dashboards/gpu/gpu-efficiency.json b/dashboards/gpu/gpu-efficiency.json
index 1a305b4b..596961e0 100644
--- a/dashboards/gpu/gpu-efficiency.json
+++ b/dashboards/gpu/gpu-efficiency.json
@@ -838,4 +838,4 @@
     ]
   },
   "annotations": {}
-}
\ No newline at end of file
+}
diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json
index 840065f3..60c628f6 100644
--- a/dashboards/gpu/gpu-quotas.json
+++ b/dashboards/gpu/gpu-quotas.json
@@ -627,4 +627,4 @@
     ]
   },
   "annotations": {}
-}
\ No newline at end of file
+}

From 4e8731b58882edab23d79c99b5b6b49dda96c572 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sat, 18 Apr 2026 06:02:17 +0300
Subject: [PATCH 032/151] refactor(monitoring): rework GPU recording rules

- Drop gpu.recording.30s group: per-GPU 30s aggregates had no consumers
  in tracked dashboards, only burned cardinality.
- Drop namespace:gpu_allocated_count:gauge: identical expression to
  namespace:gpu_count:sum under a different name.
- Reground :allocated on kube_pod_container_resource_requests so it
  reflects what tenants requested (Pending+Running) rather than what
  DCGM currently sees. namespace:gpu_count:sum stays DCGM-based and
  represents actually-running pods; the gap between the two is the
  signal admins want.
- Add namespace:gpu_count:allocated as the per-namespace counterpart.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 .../alerts/gpu-recording.rules.yaml           | 82 +++++++++++++------
 1 file changed, 59 insertions(+), 23 deletions(-)

diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
index 871a46fc..a54860b7 100644
--- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
+++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
@@ -4,33 +4,17 @@ metadata:
   name: alerts-gpu-recording.rules
 spec:
   groups:
-    - name: gpu.recording.30s
-      interval: 30s
-      rules:
-        - record: gpu:util:avg
-          expr: avg by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_DEV_GPU_UTIL)
-        - record: gpu:mem_copy_util:avg
-          expr: avg by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_DEV_MEM_COPY_UTIL)
-        - record: gpu:fb_used_bytes:max
-          expr: max by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_DEV_FB_USED) * 1048576
-        - record: gpu:fb_free_bytes:max
-          expr: max by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_DEV_FB_FREE) * 1048576
-        - record: gpu:power_watts:avg
-          expr: avg by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_DEV_POWER_USAGE)
-        - record: gpu:temp_celsius:max
-          expr: max by (Hostname, gpu, UUID, modelName) (DCGM_FI_DEV_GPU_TEMP)
-        - record: gpu:tensor_active:avg
-          expr: avg by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_PROF_PIPE_TENSOR_ACTIVE)
-        - record: gpu:gr_engine_active:avg
-          expr: avg by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_PROF_GR_ENGINE_ACTIVE)
-
     - name: gpu.recording.cluster.1m
       interval: 1m
       rules:
         - record: cluster:gpu_count:total
           expr: count(group by (UUID) (DCGM_FI_DEV_GPU_UTIL))
+        # Kube-allocated GPU count: GPUs requested by *all* pods regardless of
+        # phase (Pending+Running). Source of truth for "what tenants asked for"
+        # — used for capacity planning and billing. Includes system pods so it
+        # stays consistent with cluster:gpu_count:total when computing :free.
         - record: cluster:gpu_count:allocated
-          expr: count(group by (UUID) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}))
+          expr: sum(kube_pod_container_resource_requests{resource="nvidia_com_gpu"})
         - record: cluster:gpu_count:free
           expr: cluster:gpu_count:total - (cluster:gpu_count:allocated or vector(0))
         - record: cluster:gpu_util:avg
@@ -41,8 +25,16 @@ spec:
     - name: gpu.recording.namespace.1m
       interval: 1m
       rules:
+        # DCGM-visible GPU count per namespace — counts GPUs that are actually
+        # running a tenant pod right now (driver loaded, scheduler placed it).
+        # Differs from :allocated when pods are Pending or stuck.
         - record: namespace:gpu_count:sum
           expr: count by (namespace) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"})
+        # Kube-requested GPU count per namespace — billable view, includes
+        # Pending pods. Use this for GPU-hour reporting via
+        # sum_over_time(...[1h:1m])/60.
+        - record: namespace:gpu_count:allocated
+          expr: sum by (namespace) (kube_pod_container_resource_requests{resource="nvidia_com_gpu"})
         - record: namespace:gpu_util:avg
           expr: avg by (namespace) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"})
         - record: namespace:tensor_active:avg
@@ -53,5 +45,49 @@ spec:
           expr: sum by (namespace) (DCGM_FI_DEV_POWER_USAGE{namespace!="", namespace!~"cozy-.*|kube-.*"})
         - record: namespace:energy_joules:sum
           expr: sum by (namespace) (DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION{namespace!="", namespace!~"cozy-.*|kube-.*"}) / 1000
-        - record: namespace:gpu_allocated_count:gauge
-          expr: count by (namespace) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"})
+
+    - name: gpu.recording.efficiency.1m
+      interval: 1m
+      rules:
+        # Tensor hardware saturation — the honest "am I using the GPU" metric
+        # for AI/LLM workloads. Unlike NVML, idle tensor cores are visible.
+        - record: pod:tensor_saturation:avg5m
+          expr: |
+            avg by (Hostname, gpu, UUID, namespace, pod) (
+              avg_over_time(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m])
+            ) * 100
+
+        # NVML vs tensor gap — ratio <10% means NVML lies: user thinks GPU
+        # is busy but specialized hardware is idle (cheap tenant signal).
+        - record: pod:tensor_to_nvml_ratio:avg5m
+          expr: |
+            (
+              avg_over_time(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]) * 100
+            )
+            /
+            clamp_min(
+              avg_over_time(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]),
+              1
+            )
+
+        # Power efficiency — utilization per watt, reveals unoptimized clients.
+        - record: pod:util_per_watt:avg5m
+          expr: |
+            avg_over_time(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m])
+            /
+            clamp_min(
+              avg_over_time(DCGM_FI_DEV_POWER_USAGE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]),
+              1
+            )
+
+        # Fraction of time power-throttled (TDP cap) — 1.0 = fully throttled.
+        # DCGM_FI_DEV_*_VIOLATION is documented as µs but on A10/DCGM 3.x the
+        # counter grows in nanoseconds in practice — divide by 1e9 to get a
+        # 0..1 fraction (verified empirically when /1e6 yielded >100× reality).
+        # clamp_max protects against rate() artefacts at counter resets.
+        - record: gpu:power_throttle_fraction:rate5m
+          expr: clamp_max(rate(DCGM_FI_DEV_POWER_VIOLATION[5m]) / 1e9, 1)
+
+        # Fraction of time thermal-throttled.
+        - record: gpu:thermal_throttle_fraction:rate5m
+          expr: clamp_max(rate(DCGM_FI_DEV_THERMAL_VIOLATION[5m]) / 1e9, 1)

From 0e20159bd999795fd92f9b2099529bbe9b4de040 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sat, 18 Apr 2026 06:09:44 +0300
Subject: [PATCH 033/151] fix(gpu-operator): scope compat DaemonSet to GPU
 nodes
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Restrict the nvidia-driver-compat DaemonSet to nodes labelled
nvidia.com/gpu.present=true (NFD/GPU Operator label). Without the
nodeSelector it was scheduling onto every node — control-plane and
CPU-only workers included — burning a privileged pod slot per host
for no benefit.

Add resource requests and limits to the init and pause containers so
the DaemonSet stays within control-plane budget on small clusters.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 .../examples/nvidia-driver-compat.yaml        | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml b/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml
index e4718e0a..63e07c4f 100644
--- a/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml
+++ b/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml
@@ -27,6 +27,14 @@ spec:
     spec:
       hostPID: true
       priorityClassName: system-node-critical
+      # Restrict to GPU nodes only. Without this the DaemonSet schedules onto
+      # every node (including control-plane and CPU-only workers) and burns
+      # resources on hosts where the compat tree is meaningless.
+      # The label is published by Node Feature Discovery / GPU Operator's NFD
+      # subchart; if NFD is disabled, replace this with whatever label your
+      # cluster uses to mark GPU hosts.
+      nodeSelector:
+        nvidia.com/gpu.present: "true"
       tolerations:
         - operator: Exists
       initContainers:
@@ -60,12 +68,26 @@ spec:
               echo "Done"
           securityContext:
             privileged: true
+          resources:
+            requests:
+              cpu: 10m
+              memory: 16Mi
+            limits:
+              cpu: 100m
+              memory: 64Mi
           volumeMounts:
             - name: host-root
               mountPath: /host
       containers:
         - name: pause
           image: registry.k8s.io/pause:3.10
+          resources:
+            requests:
+              cpu: 10m
+              memory: 8Mi
+            limits:
+              cpu: 50m
+              memory: 16Mi
       volumes:
         - name: host-root
           hostPath:

From 38c8a37cb585ff235c7d90d0824a457e7da93095 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sat, 18 Apr 2026 06:15:58 +0300
Subject: [PATCH 034/151] docs(gpu-operator): clarify minimum required DCGM
 metrics
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The previous wording implied that the entire custom DCGM CSV was
required by the recording rules. In fact only the profiling counters
(DCGM_FI_PROF_*) need to be added on top of the upstream defaults —
everything else the rules consume is already in default-counters.csv.

Add a Verification status block flagging that the minimum-set claim is
derived from the DCGM Exporter version pinned in the currently shipped
gpu-operator package and must be re-checked when that package moves to
a newer release.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 .../system/gpu-operator/examples/README.md    | 30 +++++++++++++++++--
 1 file changed, 27 insertions(+), 3 deletions(-)

diff --git a/packages/system/gpu-operator/examples/README.md b/packages/system/gpu-operator/examples/README.md
index dac3703e..11c2bb09 100644
--- a/packages/system/gpu-operator/examples/README.md
+++ b/packages/system/gpu-operator/examples/README.md
@@ -21,9 +21,16 @@ Talos.
   ConfigMap.
 - [`dcgm-custom-metrics.yaml`](./dcgm-custom-metrics.yaml) — `ConfigMap`
   with a DCGM metrics CSV that adds profiling, ECC, throttling and
-  energy counters on top of the upstream defaults. Required by the
-  recording rules in `packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml`
-  and by several panels in the `gpu/gpu-performance` dashboard.
+  energy counters on top of the upstream defaults. The CSV is the
+  superset needed for full dashboard coverage; the **recording rules
+  themselves** only require the profiling subset
+  (`DCGM_FI_PROF_PIPE_TENSOR_ACTIVE`, `DCGM_FI_PROF_GR_ENGINE_ACTIVE`)
+  on top of the upstream `default-counters.csv` — every other DCGM
+  series the rules consume (utilization, FB used/free, power,
+  temperature, energy) is already in the default set. The
+  `gpu/gpu-performance` dashboard additionally needs the throttle
+  counters (`DCGM_FI_DEV_POWER_VIOLATION`,
+  `DCGM_FI_DEV_THERMAL_VIOLATION`), which are not in the default set.
 - [`nvidia-driver-compat.yaml`](./nvidia-driver-compat.yaml) — DaemonSet
   that stages `libnvidia-ml.so.1` and `nvidia-smi` from the Talos glibc
   tree into a path where the NVIDIA GPU Operator validator expects
@@ -55,6 +62,23 @@ files into a directory the validator does inspect and creates the
 
 [1]: https://github.com/NVIDIA/gpu-operator/issues/1687
 
+## Verification status
+
+> **Pending verification on an updated GPU Operator release.**
+>
+> The minimum-CSV claim above (only `DCGM_FI_PROF_*` is needed beyond
+> the upstream default counters) is derived by cross-referencing
+> `gpu-recording.rules.yaml` against the DCGM Exporter
+> [`default-counters.csv`][default-csv] for the version pinned in the
+> currently shipped `gpu-operator` package. The package in this branch
+> is **not** the latest GPU Operator release; once we move to a newer
+> version, the claim must be re-checked because the upstream default
+> set occasionally adds or removes counters between releases. Until
+> then, treat the CSV in `dcgm-custom-metrics.yaml` as a known-good
+> superset rather than a minimal config.
+
+[default-csv]: https://github.com/NVIDIA/dcgm-exporter/blob/main/etc/default-counters.csv
+
 ## How the dashboard and recording rules fit in
 
 - `dashboards/gpu/gpu-performance.json` expects `DCGM_FI_*` metrics,

From 49c1d7e7abeddebf6047a52b6de7e13f53c60ba5 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sat, 18 Apr 2026 06:22:31 +0300
Subject: [PATCH 035/151] test(monitoring): cross-check GPU dashboards against
 recording rules
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Catch dangling references at PR time: every recording-rule name used
inside a tracked GPU dashboard must exist in
packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml. The
first iteration of gpu-efficiency.json shipped panels keyed on
pod:tensor_saturation:avg5m without the rule defined; the test fails
on exactly that class of bug.

Scoped to dashboards listed under gpu/* in dashboards-infra.list, so
untracked drafts stay out of scope until they are registered. Reverse
direction (rule defined but unused) is intentionally NOT enforced —
some rules exist for ad-hoc PromQL or upcoming dashboards.

Auto-discovered by make bats-unit-tests via hack/cozytest.sh.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 hack/check-gpu-recording-rules.bats | 114 ++++++++++++++++++++++++++++
 1 file changed, 114 insertions(+)
 create mode 100644 hack/check-gpu-recording-rules.bats

diff --git a/hack/check-gpu-recording-rules.bats b/hack/check-gpu-recording-rules.bats
new file mode 100644
index 00000000..87e7c8f1
--- /dev/null
+++ b/hack/check-gpu-recording-rules.bats
@@ -0,0 +1,114 @@
+#!/usr/bin/env bats
+# -----------------------------------------------------------------------------
+# Cross-validation between GPU recording rules and the dashboards that consume
+# them. Catches:
+#
+#   1. dangling references — a dashboard query mentions a recording rule name
+#      that doesn't exist in gpu-recording.rules.yaml. This is the bug the
+#      pre-merge review caught: gpu-efficiency.json shipped panels keyed on
+#      pod:tensor_saturation:avg5m without the rule being defined, so the
+#      panel showed "No data" everywhere.
+#
+#   2. typos in rule names — same bug class, manifested as a single-character
+#      difference between rule and reference.
+#
+# Scope: only dashboards listed in packages/system/monitoring/dashboards-infra.list
+# under the "gpu/" prefix (i.e. shipped to production), not every JSON file in
+# dashboards/gpu/. Untracked drafts stay out of scope on purpose — adding one
+# to dashboards-infra.list is what brings it under the test.
+#
+# Reverse direction (rule defined but never consumed) is intentionally NOT
+# enforced: some rules exist for ad-hoc PromQL or upcoming dashboards. Treat
+# unused rules as an editorial concern, not a regression.
+#
+# Title syntax constraints from cozytest.sh's awk parser:
+#   - Titles delimited by ASCII double quotes; embedded quotes truncate.
+#   - Only [A-Za-z0-9] from the title survives into the function name; titles
+#     differing only in punctuation collapse to the same function.
+#
+# Run with: hack/cozytest.sh hack/check-gpu-recording-rules.bats
+# -----------------------------------------------------------------------------
+
+REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)"
+RULES_FILE="$REPO_ROOT/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml"
+DASHBOARDS_LIST="$REPO_ROOT/packages/system/monitoring/dashboards-infra.list"
+DASHBOARDS_DIR="$REPO_ROOT/dashboards"
+
+# Extract the set of "- record: NAME" entries from the rules YAML.
+# Outputs one rule name per line, sorted and deduplicated.
+extract_rules() {
+  awk '/^[[:space:]]*-[[:space:]]*record:[[:space:]]/ {
+    sub(/^[[:space:]]*-[[:space:]]*record:[[:space:]]*/, "")
+    sub(/[[:space:]]*$/, "")
+    print
+  }' "$RULES_FILE" | sort -u
+}
+
+# Extract the set of recording-rule references from a dashboard JSON.
+# A recording-rule reference is matched by the pattern
+#   <segment>:<segment>(:<segment>)+
+# where each <segment> is [a-z0-9_]. Raw DCGM metrics (DCGM_FI_*),
+# kube-state-metrics (kube_*) and similar uppercase / single-word metric
+# names do not match because the leading segment must be lowercase and the
+# whole expression must contain at least two ':' characters.
+extract_refs() {
+  json_file=$1
+  grep -hoE '[a-z][a-z0-9_]*:[a-z0-9_]+:[a-z0-9_]+' "$json_file" | sort -u
+}
+
+# Resolve "gpu/foo" -> "$DASHBOARDS_DIR/gpu/foo.json"
+list_tracked_gpu_dashboards() {
+  awk '/^gpu\// { print $0 ".json" }' "$DASHBOARDS_LIST"
+}
+
+@test "every recording rule reference in tracked GPU dashboards has a matching record" {
+  TMP=$(mktemp -d)
+  trap 'rm -rf "$TMP"' EXIT
+
+  extract_rules > "$TMP/rules.txt"
+  [ -s "$TMP/rules.txt" ] || { echo "no recording rules extracted from $RULES_FILE" >&2; exit 1; }
+
+  list_tracked_gpu_dashboards > "$TMP/dashboards.txt"
+  [ -s "$TMP/dashboards.txt" ] || { echo "no gpu/* dashboards listed in $DASHBOARDS_LIST" >&2; exit 1; }
+
+  failed=0
+  while IFS= read -r dashboard_rel; do
+    dashboard="$DASHBOARDS_DIR/$dashboard_rel"
+    if [ ! -f "$dashboard" ]; then
+      echo "ERROR: dashboard listed but file missing: $dashboard" >&2
+      failed=1
+      continue
+    fi
+
+    extract_refs "$dashboard" > "$TMP/refs.txt"
+    # comm -23: lines unique to refs.txt (referenced but not defined)
+    # Both inputs must be sorted; extract_* helpers already sort.
+    comm -23 "$TMP/refs.txt" "$TMP/rules.txt" > "$TMP/missing.txt"
+    if [ -s "$TMP/missing.txt" ]; then
+      echo "ERROR: $dashboard_rel references undefined recording rules:" >&2
+      sed 's/^/  - /' "$TMP/missing.txt" >&2
+      failed=1
+    fi
+  done < "$TMP/dashboards.txt"
+
+  [ "$failed" -eq 0 ]
+}
+
+@test "every tracked GPU dashboard listed in dashboards-infra.list exists on disk" {
+  TMP=$(mktemp -d)
+  trap 'rm -rf "$TMP"' EXIT
+
+  list_tracked_gpu_dashboards > "$TMP/dashboards.txt"
+  [ -s "$TMP/dashboards.txt" ] || { echo "no gpu/* dashboards listed in $DASHBOARDS_LIST" >&2; exit 1; }
+
+  failed=0
+  while IFS= read -r dashboard_rel; do
+    dashboard="$DASHBOARDS_DIR/$dashboard_rel"
+    if [ ! -f "$dashboard" ]; then
+      echo "ERROR: $dashboard_rel listed in dashboards-infra.list but $dashboard does not exist" >&2
+      failed=1
+    fi
+  done < "$TMP/dashboards.txt"
+
+  [ "$failed" -eq 0 ]
+}

From dbed4992b08392c28bac8f0b5f917619b88cec65 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sat, 18 Apr 2026 06:28:49 +0300
Subject: [PATCH 036/151] fix(monitoring): exclude system namespaces from
 namespace:gpu_count:allocated
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Align namespace:gpu_count:allocated with every other namespace:* rule
by filtering out cozy-*/kube-*. All other per-namespace rules
(gpu_util, tensor_active, fb_used_bytes, power_watts) already exclude
system namespaces, so the label set produced by :allocated diverged
from them — any dashboard variable or join that reads across these
rules could end up with a different namespace list depending on which
rule supplied the :allocated column.

Trade-off: per-namespace GPU accounting for system workloads is no
longer available through this rule. If it's ever needed, add a
dedicated system:gpu_count:allocated rather than widening this one —
the "billable tenant view" invariant is what the filter is protecting.

Cluster-level cluster:gpu_count:allocated intentionally keeps system
pods so it stays aligned with cluster:gpu_count:total and
cluster:gpu_count:free remains meaningful. As a consequence,
sum(namespace:gpu_count:allocated) no longer equals
cluster:gpu_count:allocated; the delta is system-pod GPU usage, which
is fine for the cluster-admin view.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 .../alerts/gpu-recording.rules.yaml           | 21 ++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
index a54860b7..3a31f5a2 100644
--- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
+++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
@@ -33,8 +33,27 @@ spec:
         # Kube-requested GPU count per namespace — billable view, includes
         # Pending pods. Use this for GPU-hour reporting via
         # sum_over_time(...[1h:1m])/60.
+        #
+        # Filters cozy-*/kube-* so the namespace set here matches every
+        # other namespace:* rule below. This keeps the $namespace variable
+        # in the tenant dashboard consistent with panels that read filtered
+        # rules — otherwise picking a system ns from the dropdown leaves
+        # half the panels empty while others show values.
+        #
+        # Trade-off: per-namespace GPU accounting for system workloads is
+        # no longer available here. If it's ever needed (e.g. operator
+        # pods pinning tenant GPUs), add a separate rule like
+        # system:gpu_count:allocated rather than widening this one — the
+        # "billable tenant view" invariant is relied on by tenants.json
+        # and keeping it narrow protects consumers from the asymmetry
+        # that triggered this filter in the first place.
+        #
+        # Note: sum(namespace:gpu_count:allocated) no longer equals
+        # cluster:gpu_count:allocated — the cluster-level rule intentionally
+        # keeps system pods to stay aligned with cluster:gpu_count:total
+        # when computing :free.
         - record: namespace:gpu_count:allocated
-          expr: sum by (namespace) (kube_pod_container_resource_requests{resource="nvidia_com_gpu"})
+          expr: sum by (namespace) (kube_pod_container_resource_requests{resource="nvidia_com_gpu", namespace!="", namespace!~"cozy-.*|kube-.*"})
         - record: namespace:gpu_util:avg
           expr: avg by (namespace) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"})
         - record: namespace:tensor_active:avg

From 6d9066f0748744d78f823e3137f72b5e912069e9 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sat, 18 Apr 2026 06:35:12 +0300
Subject: [PATCH 037/151] feat(monitoring): add GPU fleet and tenants
 dashboards
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- gpu-fleet: cluster-wide admin view — inventory, capacity (total /
  allocated / free), per-node utilization and power, throttling,
  temperatures, XID errors.
- gpu-tenants: per-namespace view — live allocation, utilization,
  tensor saturation, power, and 24h GPU-hours / kWh integrations for
  billing inputs.

Register both under gpu/* in dashboards-infra.list so they ship as
GrafanaDashboard CRs and fall under the bats cross-check introduced
earlier on this branch.

Update examples/README to spell out which DCGM counters each of the
five gpu/* dashboards actually needs on top of the upstream default
CSV — gpu-performance needs profiling and throttling counters,
gpu-efficiency needs profiling, gpu-tenants needs only
DCGM_FI_PROF_PIPE_TENSOR_ACTIVE for its tensor panel, and gpu-fleet
and gpu-quotas work on the default counter set alone.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 dashboards/gpu/gpu-fleet.json                 | 1157 +++++++++++++++
 dashboards/gpu/gpu-tenants.json               | 1236 +++++++++++++++++
 .../system/gpu-operator/examples/README.md    |   74 +-
 .../system/monitoring/dashboards-infra.list   |    2 +
 4 files changed, 2436 insertions(+), 33 deletions(-)
 create mode 100644 dashboards/gpu/gpu-fleet.json
 create mode 100644 dashboards/gpu/gpu-tenants.json

diff --git a/dashboards/gpu/gpu-fleet.json b/dashboards/gpu/gpu-fleet.json
new file mode 100644
index 00000000..af5ad2fa
--- /dev/null
+++ b/dashboards/gpu/gpu-fleet.json
@@ -0,0 +1,1157 @@
+{
+  "uid": "gpu-fleet",
+  "title": "GPU Fleet Overview",
+  "description": "Cluster-wide GPU inventory, capacity, utilization and health — admin view across the whole fleet.",
+  "tags": [
+    "gpu",
+    "fleet",
+    "admin"
+  ],
+  "timezone": "browser",
+  "editable": true,
+  "graphTooltip": 1,
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "fiscalYearStartMonth": 0,
+  "schemaVersion": 42,
+  "panels": [
+    {
+      "type": "row",
+      "collapsed": false,
+      "title": "Cluster snapshot",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "panels": []
+    },
+    {
+      "type": "stat",
+      "id": 2,
+      "targets": [
+        {
+          "expr": "cluster:gpu_count:total",
+          "refId": "A"
+        }
+      ],
+      "title": "Total GPUs",
+      "description": "Physical GPUs discovered by DCGM across all nodes.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 0,
+        "y": 1
+      },
+      "repeatDirection": "h",
+      "options": {
+        "graphMode": "none",
+        "colorMode": "value",
+        "justifyMode": "auto",
+        "textMode": "value",
+        "wideLayout": true,
+        "showPercentChange": false,
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "percentChangeColorMode": "standard",
+        "orientation": ""
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "value": null,
+                "color": "blue"
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "stat",
+      "id": 3,
+      "targets": [
+        {
+          "expr": "cluster:gpu_count:allocated",
+          "refId": "A"
+        }
+      ],
+      "title": "Allocated",
+      "description": "Sum of kube GPU requests across all pods. Billable view — includes Pending pods.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 4,
+        "y": 1
+      },
+      "repeatDirection": "h",
+      "options": {
+        "graphMode": "none",
+        "colorMode": "background",
+        "justifyMode": "auto",
+        "textMode": "value",
+        "wideLayout": true,
+        "showPercentChange": false,
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "percentChangeColorMode": "standard",
+        "orientation": ""
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "value": null,
+                "color": "blue"
+              },
+              {
+                "value": 1,
+                "color": "green"
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "stat",
+      "id": 4,
+      "targets": [
+        {
+          "expr": "cluster:gpu_count:free",
+          "refId": "A"
+        }
+      ],
+      "title": "Free",
+      "description": "Unallocated capacity available for new tenants.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 8,
+        "y": 1
+      },
+      "repeatDirection": "h",
+      "options": {
+        "graphMode": "none",
+        "colorMode": "background",
+        "justifyMode": "auto",
+        "textMode": "value",
+        "wideLayout": true,
+        "showPercentChange": false,
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "percentChangeColorMode": "standard",
+        "orientation": ""
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "value": null,
+                "color": "red"
+              },
+              {
+                "value": 1,
+                "color": "yellow"
+              },
+              {
+                "value": 2,
+                "color": "green"
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "stat",
+      "id": 5,
+      "targets": [
+        {
+          "expr": "count(kube_node_labels{label_nvidia_com_gpu_present=\"true\"})",
+          "refId": "A"
+        }
+      ],
+      "title": "Nodes with GPU",
+      "description": "Kubernetes nodes advertising nvidia.com/gpu.present=true.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 12,
+        "y": 1
+      },
+      "repeatDirection": "h",
+      "options": {
+        "graphMode": "none",
+        "colorMode": "value",
+        "justifyMode": "auto",
+        "textMode": "value",
+        "wideLayout": true,
+        "showPercentChange": false,
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "percentChangeColorMode": "standard",
+        "orientation": ""
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "value": null,
+                "color": "blue"
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "stat",
+      "id": 6,
+      "targets": [
+        {
+          "expr": "cluster:gpu_util:avg",
+          "refId": "A"
+        }
+      ],
+      "title": "Average utilization",
+      "description": "NVML utilization across all tenant GPUs (engine-active %).",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 16,
+        "y": 1
+      },
+      "repeatDirection": "h",
+      "options": {
+        "graphMode": "area",
+        "colorMode": "value",
+        "justifyMode": "auto",
+        "textMode": "value",
+        "wideLayout": true,
+        "showPercentChange": false,
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "percentChangeColorMode": "standard",
+        "orientation": ""
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "value": null,
+                "color": "blue"
+              },
+              {
+                "value": 30,
+                "color": "green"
+              },
+              {
+                "value": 80,
+                "color": "orange"
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "stat",
+      "id": 7,
+      "targets": [
+        {
+          "expr": "cluster:gpu_power_watts:sum",
+          "refId": "A"
+        }
+      ],
+      "title": "Total power",
+      "description": "Live power draw summed across the fleet.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 20,
+        "y": 1
+      },
+      "repeatDirection": "h",
+      "options": {
+        "graphMode": "area",
+        "colorMode": "value",
+        "justifyMode": "auto",
+        "textMode": "value",
+        "wideLayout": true,
+        "showPercentChange": false,
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "percentChangeColorMode": "standard",
+        "orientation": ""
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "watt",
+          "decimals": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "value": null,
+                "color": "green"
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "row",
+      "collapsed": false,
+      "title": "Per-node GPU activity",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 5
+      },
+      "id": 10,
+      "panels": []
+    },
+    {
+      "type": "bargauge",
+      "id": 11,
+      "targets": [
+        {
+          "expr": "avg by (Hostname) (DCGM_FI_DEV_GPU_UTIL)",
+          "legendFormat": "{{Hostname}}",
+          "refId": "A"
+        }
+      ],
+      "title": "GPU utilization per node",
+      "description": "Average NVML utilization per node — spot idle or saturated hosts at a glance.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 6
+      },
+      "repeatDirection": "h",
+      "options": {
+        "displayMode": "gradient",
+        "valueMode": "color",
+        "namePlacement": "auto",
+        "showUnfilled": true,
+        "sizing": "auto",
+        "minVizWidth": 8,
+        "minVizHeight": 16,
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": false,
+          "calcs": []
+        },
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "maxVizHeight": 300,
+        "orientation": "horizontal"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "value": null,
+                "color": "blue"
+              },
+              {
+                "value": 30,
+                "color": "green"
+              },
+              {
+                "value": 80,
+                "color": "orange"
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "bargauge",
+      "id": 12,
+      "targets": [
+        {
+          "expr": "sum by (Hostname) (DCGM_FI_DEV_POWER_USAGE)",
+          "legendFormat": "{{Hostname}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Power draw per node",
+      "description": "Sum of GPU power usage per node in watts.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 6
+      },
+      "repeatDirection": "h",
+      "options": {
+        "displayMode": "gradient",
+        "valueMode": "color",
+        "namePlacement": "auto",
+        "showUnfilled": true,
+        "sizing": "auto",
+        "minVizWidth": 8,
+        "minVizHeight": 16,
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": false,
+          "calcs": []
+        },
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "maxVizHeight": 300,
+        "orientation": "horizontal"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "watt",
+          "min": 0,
+          "max": 200,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "value": null,
+                "color": "green"
+              },
+              {
+                "value": 120,
+                "color": "yellow"
+              },
+              {
+                "value": 160,
+                "color": "orange"
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "row",
+      "collapsed": false,
+      "title": "Usage trends",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 14
+      },
+      "id": 20,
+      "panels": []
+    },
+    {
+      "type": "timeseries",
+      "id": 21,
+      "targets": [
+        {
+          "expr": "avg by (Hostname) (DCGM_FI_DEV_GPU_UTIL)",
+          "legendFormat": "{{Hostname}}",
+          "refId": "A"
+        }
+      ],
+      "title": "GPU utilization per node (trend)",
+      "description": "Per-node NVML utilization over time — shows shifting workload across hosts.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 15
+      },
+      "repeatDirection": "h",
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": false,
+          "calcs": [
+            "mean",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": ""
+        }
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "linear",
+            "fillOpacity": 20,
+            "showPoints": "never"
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "timeseries",
+      "id": 22,
+      "targets": [
+        {
+          "expr": "sum by (Hostname) (DCGM_FI_DEV_POWER_USAGE)",
+          "legendFormat": "{{Hostname}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Power draw per node (trend)",
+      "description": "Per-node GPU power consumption over time.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 15
+      },
+      "repeatDirection": "h",
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": false,
+          "calcs": [
+            "mean",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": ""
+        }
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "watt",
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "linear",
+            "fillOpacity": 20,
+            "showPoints": "never"
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "row",
+      "collapsed": false,
+      "title": "Health",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 23
+      },
+      "id": 30,
+      "panels": []
+    },
+    {
+      "type": "stat",
+      "id": 31,
+      "targets": [
+        {
+          "expr": "count((max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS)) \u003e 0) or vector(0)",
+          "refId": "A"
+        }
+      ],
+      "title": "GPUs with XID errors",
+      "description": "Count of GPUs reporting a non-zero XID code. XID = NVIDIA hardware/driver error — any non-zero value needs investigation.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 6,
+        "x": 0,
+        "y": 24
+      },
+      "repeatDirection": "h",
+      "options": {
+        "graphMode": "none",
+        "colorMode": "background",
+        "justifyMode": "auto",
+        "textMode": "value",
+        "wideLayout": true,
+        "showPercentChange": false,
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "percentChangeColorMode": "standard",
+        "orientation": ""
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "value": null,
+                "color": "green"
+              },
+              {
+                "value": 1,
+                "color": "red"
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "stat",
+      "id": 32,
+      "targets": [
+        {
+          "expr": "avg(gpu:power_throttle_fraction:rate5m)",
+          "refId": "A"
+        }
+      ],
+      "title": "Power throttling (cluster avg)",
+      "description": "Fraction of time the fleet spent power-throttled. \u003e5% = billed FLOPS are being underdelivered.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 6,
+        "x": 6,
+        "y": 24
+      },
+      "repeatDirection": "h",
+      "options": {
+        "graphMode": "none",
+        "colorMode": "background",
+        "justifyMode": "auto",
+        "textMode": "value",
+        "wideLayout": true,
+        "showPercentChange": false,
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "percentChangeColorMode": "standard",
+        "orientation": ""
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "decimals": 2,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "value": null,
+                "color": "green"
+              },
+              {
+                "value": 0.05,
+                "color": "yellow"
+              },
+              {
+                "value": 0.2,
+                "color": "red"
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "stat",
+      "id": 33,
+      "targets": [
+        {
+          "expr": "avg(gpu:thermal_throttle_fraction:rate5m)",
+          "refId": "A"
+        }
+      ],
+      "title": "Thermal throttling (cluster avg)",
+      "description": "Fraction of time the fleet spent thermal-throttled. Indicates cooling/datacenter issues.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 6,
+        "x": 12,
+        "y": 24
+      },
+      "repeatDirection": "h",
+      "options": {
+        "graphMode": "none",
+        "colorMode": "background",
+        "justifyMode": "auto",
+        "textMode": "value",
+        "wideLayout": true,
+        "showPercentChange": false,
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "percentChangeColorMode": "standard",
+        "orientation": ""
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "decimals": 2,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "value": null,
+                "color": "green"
+              },
+              {
+                "value": 0.05,
+                "color": "yellow"
+              },
+              {
+                "value": 0.2,
+                "color": "red"
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "stat",
+      "id": 34,
+      "targets": [
+        {
+          "expr": "max(DCGM_FI_DEV_GPU_TEMP)",
+          "refId": "A"
+        }
+      ],
+      "title": "Max GPU temperature",
+      "description": "Hottest GPU in the fleet right now. A10 sustained target \u003c83°C.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 6,
+        "x": 18,
+        "y": 24
+      },
+      "repeatDirection": "h",
+      "options": {
+        "graphMode": "none",
+        "colorMode": "background",
+        "justifyMode": "auto",
+        "textMode": "value",
+        "wideLayout": true,
+        "showPercentChange": false,
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "percentChangeColorMode": "standard",
+        "orientation": ""
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "celsius",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "value": null,
+                "color": "green"
+              },
+              {
+                "value": 75,
+                "color": "yellow"
+              },
+              {
+                "value": 85,
+                "color": "red"
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "timeseries",
+      "id": 35,
+      "targets": [
+        {
+          "expr": "DCGM_FI_DEV_GPU_TEMP",
+          "legendFormat": "{{Hostname}}/{{gpu}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Temperature per GPU",
+      "description": "Per-GPU temperature trace. Sustained red zone = cooling issue or aggressive workload.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 29
+      },
+      "repeatDirection": "h",
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": false,
+          "calcs": [
+            "mean",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": ""
+        }
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "celsius",
+          "min": 20,
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "value": null,
+                "color": "green"
+              },
+              {
+                "value": 75,
+                "color": "yellow"
+              },
+              {
+                "value": 85,
+                "color": "red"
+              }
+            ]
+          },
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "linear",
+            "fillOpacity": 10,
+            "showPoints": "never"
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "timeseries",
+      "id": 36,
+      "targets": [
+        {
+          "expr": "gpu:power_throttle_fraction:rate5m",
+          "legendFormat": "power {{Hostname}}/{{gpu}}",
+          "refId": "A"
+        },
+        {
+          "expr": "gpu:thermal_throttle_fraction:rate5m",
+          "legendFormat": "thermal {{Hostname}}/{{gpu}}",
+          "refId": "B"
+        }
+      ],
+      "title": "Throttle fraction per GPU",
+      "description": "Red = power throttle, blue = thermal throttle. Both on same chart to compare causes.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 29
+      },
+      "repeatDirection": "h",
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": false,
+          "calcs": [
+            "mean",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": ""
+        }
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "min": 0,
+          "max": 1,
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "linear",
+            "fillOpacity": 25,
+            "showPoints": "never"
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "row",
+      "collapsed": false,
+      "title": "Hardware inventory",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 37
+      },
+      "id": 40,
+      "panels": []
+    },
+    {
+      "type": "table",
+      "id": 41,
+      "targets": [
+        {
+          "expr": "group by (Hostname, modelName, gpu, UUID) (DCGM_FI_DEV_GPU_UTIL)",
+          "instant": true,
+          "range": false,
+          "format": "table",
+          "refId": "A"
+        }
+      ],
+      "title": "GPU inventory",
+      "description": "One row per physical GPU. Model / index / UUID sourced from DCGM labels.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 24,
+        "x": 0,
+        "y": 38
+      },
+      "repeatDirection": "h",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "Value": true,
+              "__name__": true,
+              "cluster": true,
+              "container": true,
+              "device": true,
+              "endpoint": true,
+              "instance": true,
+              "job": true,
+              "pci_bus_id": true,
+              "prometheus": true,
+              "service": true,
+              "tenant": true,
+              "tier": true,
+              "uid": true,
+              "unit": true
+            },
+            "indexByName": {
+              "Hostname": 0,
+              "UUID": 3,
+              "gpu": 2,
+              "modelName": 1
+            },
+            "renameByName": {
+              "Hostname": "Node",
+              "UUID": "UUID",
+              "gpu": "Index",
+              "modelName": "GPU Model"
+            }
+          }
+        }
+      ],
+      "options": {
+        "frameIndex": 0,
+        "showHeader": true,
+        "showTypeIcons": false,
+        "footer": {
+          "show": false,
+          "reducer": []
+        },
+        "cellHeight": "sm"
+      }
+    }
+  ],
+  "templating": {
+    "list": [
+      {
+        "type": "datasource",
+        "name": "ds_prometheus",
+        "label": "Prometheus",
+        "skipUrlSync": false,
+        "query": "prometheus",
+        "current": {
+          "selected": false,
+          "text": "default",
+          "value": "default"
+        },
+        "multi": false,
+        "allowCustomValue": true,
+        "includeAll": false,
+        "regex": "",
+        "auto": false,
+        "auto_min": "10s",
+        "auto_count": 30
+      }
+    ]
+  },
+  "annotations": {}
+}
diff --git a/dashboards/gpu/gpu-tenants.json b/dashboards/gpu/gpu-tenants.json
new file mode 100644
index 00000000..d0265c0d
--- /dev/null
+++ b/dashboards/gpu/gpu-tenants.json
@@ -0,0 +1,1236 @@
+{
+  "uid": "gpu-tenants",
+  "title": "GPU Tenant Usage",
+  "description": "Per-tenant GPU consumption: live allocation, utilization, tensor saturation, power, and 24h GPU-hours / kWh — admin view across all namespaces.",
+  "tags": [
+    "gpu",
+    "tenants",
+    "admin"
+  ],
+  "timezone": "browser",
+  "editable": true,
+  "graphTooltip": 1,
+  "time": {
+    "from": "now-24h",
+    "to": "now"
+  },
+  "fiscalYearStartMonth": 0,
+  "schemaVersion": 42,
+  "panels": [
+    {
+      "type": "row",
+      "collapsed": false,
+      "title": "Tenant fleet snapshot",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "panels": []
+    },
+    {
+      "type": "stat",
+      "id": 2,
+      "targets": [
+        {
+          "expr": "count(namespace:gpu_count:allocated{namespace=~\"$namespace\"})",
+          "refId": "A"
+        }
+      ],
+      "title": "Active tenants",
+      "description": "Namespaces with at least one GPU request (Pending + Running).",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 0,
+        "y": 1
+      },
+      "repeatDirection": "h",
+      "options": {
+        "graphMode": "none",
+        "colorMode": "value",
+        "justifyMode": "auto",
+        "textMode": "value",
+        "wideLayout": true,
+        "showPercentChange": false,
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "percentChangeColorMode": "standard",
+        "orientation": ""
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "value": null,
+                "color": "blue"
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "stat",
+      "id": 3,
+      "targets": [
+        {
+          "expr": "sum(namespace:gpu_count:allocated{namespace=~\"$namespace\"})",
+          "refId": "A"
+        }
+      ],
+      "title": "Allocated GPUs",
+      "description": "Sum of kube GPU requests across tenants. Billable view — includes Pending pods.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 4,
+        "y": 1
+      },
+      "repeatDirection": "h",
+      "options": {
+        "graphMode": "none",
+        "colorMode": "background",
+        "justifyMode": "auto",
+        "textMode": "value",
+        "wideLayout": true,
+        "showPercentChange": false,
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "percentChangeColorMode": "standard",
+        "orientation": ""
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "value": null,
+                "color": "blue"
+              },
+              {
+                "value": 1,
+                "color": "green"
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "stat",
+      "id": 4,
+      "targets": [
+        {
+          "expr": "avg(namespace:gpu_util:avg{namespace=~\"$namespace\"})",
+          "refId": "A"
+        }
+      ],
+      "title": "Avg tenant util",
+      "description": "Average NVML utilization across all tenant GPUs.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 8,
+        "y": 1
+      },
+      "repeatDirection": "h",
+      "options": {
+        "graphMode": "area",
+        "colorMode": "value",
+        "justifyMode": "auto",
+        "textMode": "value",
+        "wideLayout": true,
+        "showPercentChange": false,
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "percentChangeColorMode": "standard",
+        "orientation": ""
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "value": null,
+                "color": "blue"
+              },
+              {
+                "value": 30,
+                "color": "green"
+              },
+              {
+                "value": 80,
+                "color": "orange"
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "stat",
+      "id": 5,
+      "targets": [
+        {
+          "expr": "sum(namespace:power_watts:sum{namespace=~\"$namespace\"})",
+          "refId": "A"
+        }
+      ],
+      "title": "Total tenant power",
+      "description": "Live power draw summed across tenant workloads.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 12,
+        "y": 1
+      },
+      "repeatDirection": "h",
+      "options": {
+        "graphMode": "area",
+        "colorMode": "value",
+        "justifyMode": "auto",
+        "textMode": "value",
+        "wideLayout": true,
+        "showPercentChange": false,
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "percentChangeColorMode": "standard",
+        "orientation": ""
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "watt",
+          "decimals": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "value": null,
+                "color": "green"
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "stat",
+      "id": 6,
+      "targets": [
+        {
+          "expr": "sum(sum_over_time(namespace:power_watts:sum{namespace=~\"$namespace\"}[24h:1m])) / 60 / 1000",
+          "refId": "A"
+        }
+      ],
+      "title": "Energy (last 24h)",
+      "description": "Integrated power draw over 24h. Feed into billing/chargeback as kWh.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 16,
+        "y": 1
+      },
+      "repeatDirection": "h",
+      "options": {
+        "graphMode": "none",
+        "colorMode": "value",
+        "justifyMode": "auto",
+        "textMode": "value",
+        "wideLayout": true,
+        "showPercentChange": false,
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "percentChangeColorMode": "standard",
+        "orientation": ""
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "kwatth",
+          "decimals": 1,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "value": null,
+                "color": "green"
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "stat",
+      "id": 7,
+      "targets": [
+        {
+          "expr": "sum(sum_over_time(namespace:gpu_count:allocated{namespace=~\"$namespace\"}[24h:1m])) / 60",
+          "refId": "A"
+        }
+      ],
+      "title": "GPU-hours (last 24h)",
+      "description": "Sum of (GPU count × time requested) across all tenants over the last 24h. Integrated from kube requests, so Pending pods count.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 20,
+        "y": 1
+      },
+      "repeatDirection": "h",
+      "options": {
+        "graphMode": "none",
+        "colorMode": "value",
+        "justifyMode": "auto",
+        "textMode": "value",
+        "wideLayout": true,
+        "showPercentChange": false,
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "percentChangeColorMode": "standard",
+        "orientation": ""
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 1,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "value": null,
+                "color": "blue"
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "row",
+      "collapsed": false,
+      "title": "Current allocation",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 5
+      },
+      "id": 10,
+      "panels": []
+    },
+    {
+      "type": "bargauge",
+      "id": 11,
+      "targets": [
+        {
+          "expr": "namespace:gpu_count:allocated{namespace=~\"$namespace\"}",
+          "legendFormat": "{{namespace}}",
+          "refId": "A"
+        }
+      ],
+      "title": "GPUs per namespace",
+      "description": "Kube GPU requests per tenant (billable). Includes Pending pods.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 6
+      },
+      "repeatDirection": "h",
+      "options": {
+        "displayMode": "gradient",
+        "valueMode": "color",
+        "namePlacement": "auto",
+        "showUnfilled": true,
+        "sizing": "auto",
+        "minVizWidth": 8,
+        "minVizHeight": 16,
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": false,
+          "calcs": []
+        },
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "maxVizHeight": 300,
+        "orientation": "horizontal"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "value": null,
+                "color": "green"
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "bargauge",
+      "id": 12,
+      "targets": [
+        {
+          "expr": "namespace:fb_used_bytes:sum{namespace=~\"$namespace\"}",
+          "legendFormat": "{{namespace}}",
+          "refId": "A"
+        }
+      ],
+      "title": "VRAM used per namespace",
+      "description": "FB memory actively used by tenant workloads.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 6
+      },
+      "repeatDirection": "h",
+      "options": {
+        "displayMode": "gradient",
+        "valueMode": "color",
+        "namePlacement": "auto",
+        "showUnfilled": true,
+        "sizing": "auto",
+        "minVizWidth": 8,
+        "minVizHeight": 16,
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": false,
+          "calcs": []
+        },
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "maxVizHeight": 300,
+        "orientation": "horizontal"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "value": null,
+                "color": "green"
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "row",
+      "collapsed": false,
+      "title": "Utilization",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 14
+      },
+      "id": 20,
+      "panels": []
+    },
+    {
+      "type": "timeseries",
+      "id": 21,
+      "targets": [
+        {
+          "expr": "namespace:gpu_util:avg{namespace=~\"$namespace\"}",
+          "legendFormat": "{{namespace}}",
+          "refId": "A"
+        }
+      ],
+      "title": "NVML utilization per namespace",
+      "description": "Per-tenant NVML utilization. Stacked to show fleet-wide pressure over time.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 15
+      },
+      "repeatDirection": "h",
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": false,
+          "calcs": [
+            "mean",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": ""
+        }
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "linear",
+            "fillOpacity": 40,
+            "showPoints": "never",
+            "stacking": {
+              "mode": "normal"
+            }
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "timeseries",
+      "id": 22,
+      "targets": [
+        {
+          "expr": "namespace:tensor_active:avg{namespace=~\"$namespace\"} * 100",
+          "legendFormat": "{{namespace}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Tensor saturation per namespace",
+      "description": "Real tensor core load per tenant. Useful to find AI tenants that claim GPUs but leave tensor cores idle.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 15
+      },
+      "repeatDirection": "h",
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": false,
+          "calcs": [
+            "mean",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": ""
+        }
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "linear",
+            "fillOpacity": 40,
+            "showPoints": "never",
+            "stacking": {
+              "mode": "normal"
+            }
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "row",
+      "collapsed": false,
+      "title": "Power \u0026 allocation trend",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 23
+      },
+      "id": 30,
+      "panels": []
+    },
+    {
+      "type": "timeseries",
+      "id": 31,
+      "targets": [
+        {
+          "expr": "namespace:power_watts:sum{namespace=~\"$namespace\"}",
+          "legendFormat": "{{namespace}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Power draw per namespace",
+      "description": "Per-tenant power draw. Stacked to see cluster-wide energy burn over time.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 24
+      },
+      "repeatDirection": "h",
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": false,
+          "calcs": [
+            "mean",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": ""
+        }
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "watt",
+          "min": 0,
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "linear",
+            "fillOpacity": 40,
+            "showPoints": "never",
+            "stacking": {
+              "mode": "normal"
+            }
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "timeseries",
+      "id": 32,
+      "targets": [
+        {
+          "expr": "namespace:gpu_count:allocated{namespace=~\"$namespace\"}",
+          "legendFormat": "{{namespace}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Allocated GPUs per namespace",
+      "description": "Kube-requested GPU count per tenant over time. Step-after interpolation reflects discrete allocation changes.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 24
+      },
+      "repeatDirection": "h",
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": false,
+          "calcs": [
+            "mean",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": ""
+        }
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "min": 0,
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "stepAfter",
+            "fillOpacity": 40,
+            "showPoints": "never",
+            "stacking": {
+              "mode": "normal"
+            }
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "row",
+      "collapsed": false,
+      "title": "Top consumers (live)",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 32
+      },
+      "id": 40,
+      "panels": []
+    },
+    {
+      "type": "table",
+      "id": 41,
+      "targets": [
+        {
+          "expr": "namespace:gpu_count:allocated{namespace=~\"$namespace\"}",
+          "instant": true,
+          "range": false,
+          "format": "table",
+          "refId": "gpus"
+        },
+        {
+          "expr": "namespace:gpu_util:avg{namespace=~\"$namespace\"}",
+          "instant": true,
+          "range": false,
+          "format": "table",
+          "refId": "util"
+        },
+        {
+          "expr": "namespace:tensor_active:avg{namespace=~\"$namespace\"} * 100",
+          "instant": true,
+          "range": false,
+          "format": "table",
+          "refId": "tensor"
+        },
+        {
+          "expr": "namespace:power_watts:sum{namespace=~\"$namespace\"}",
+          "instant": true,
+          "range": false,
+          "format": "table",
+          "refId": "power"
+        },
+        {
+          "expr": "namespace:fb_used_bytes:sum{namespace=~\"$namespace\"}",
+          "instant": true,
+          "range": false,
+          "format": "table",
+          "refId": "vram"
+        }
+      ],
+      "title": "Top consumers",
+      "description": "Live ranking of tenants by power draw. Columns joined from namespace recording rules.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 10,
+        "w": 24,
+        "x": 0,
+        "y": 33
+      },
+      "repeatDirection": "h",
+      "transformations": [
+        {
+          "id": "joinByField",
+          "options": {
+            "byField": "namespace",
+            "mode": "outer"
+          }
+        },
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "Time 1": true,
+              "Time 2": true,
+              "Time 3": true,
+              "Time 4": true,
+              "Time 5": true,
+              "__name__": true,
+              "__name__ 1": true,
+              "__name__ 2": true,
+              "__name__ 3": true,
+              "__name__ 4": true,
+              "__name__ 5": true,
+              "cluster": true,
+              "cluster 1": true,
+              "cluster 2": true,
+              "cluster 3": true,
+              "cluster 4": true,
+              "cluster 5": true,
+              "endpoint": true,
+              "endpoint 1": true,
+              "endpoint 2": true,
+              "endpoint 3": true,
+              "endpoint 4": true,
+              "endpoint 5": true,
+              "instance": true,
+              "instance 1": true,
+              "instance 2": true,
+              "instance 3": true,
+              "instance 4": true,
+              "instance 5": true,
+              "job": true,
+              "job 1": true,
+              "job 2": true,
+              "job 3": true,
+              "job 4": true,
+              "job 5": true,
+              "prometheus": true,
+              "prometheus 1": true,
+              "prometheus 2": true,
+              "prometheus 3": true,
+              "prometheus 4": true,
+              "prometheus 5": true,
+              "service": true,
+              "service 1": true,
+              "service 2": true,
+              "service 3": true,
+              "service 4": true,
+              "service 5": true,
+              "tenant": true,
+              "tenant 1": true,
+              "tenant 2": true,
+              "tenant 3": true,
+              "tenant 4": true,
+              "tenant 5": true,
+              "tier": true,
+              "tier 1": true,
+              "tier 2": true,
+              "tier 3": true,
+              "tier 4": true,
+              "tier 5": true,
+              "uid": true,
+              "uid 1": true,
+              "uid 2": true,
+              "uid 3": true,
+              "uid 4": true,
+              "uid 5": true,
+              "unit": true,
+              "unit 1": true,
+              "unit 2": true,
+              "unit 3": true,
+              "unit 4": true,
+              "unit 5": true
+            },
+            "indexByName": {
+              "Value #gpus": 1,
+              "Value #power": 4,
+              "Value #tensor": 3,
+              "Value #util": 2,
+              "Value #vram": 5,
+              "namespace": 0
+            },
+            "renameByName": {
+              "Value #gpus": "GPUs",
+              "Value #power": "Power",
+              "Value #tensor": "Tensor %",
+              "Value #util": "Util %",
+              "Value #vram": "VRAM",
+              "namespace": "Namespace"
+            }
+          }
+        }
+      ],
+      "options": {
+        "frameIndex": 0,
+        "showHeader": true,
+        "showTypeIcons": false,
+        "sortBy": [
+          {
+            "displayName": "Power",
+            "desc": true
+          }
+        ],
+        "footer": {
+          "show": false,
+          "reducer": []
+        },
+        "cellHeight": "sm"
+      },
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Util %"
+            },
+            "properties": [
+              {
+                "id": "unit",
+                "value": "percent"
+              },
+              {
+                "id": "min",
+                "value": 0
+              },
+              {
+                "id": "max",
+                "value": 100
+              },
+              {
+                "id": "decimals",
+                "value": 1
+              },
+              {
+                "id": "custom.cellOptions",
+                "value": {
+                  "mode": "gradient",
+                  "type": "gauge"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Tensor %"
+            },
+            "properties": [
+              {
+                "id": "unit",
+                "value": "percent"
+              },
+              {
+                "id": "min",
+                "value": 0
+              },
+              {
+                "id": "max",
+                "value": 100
+              },
+              {
+                "id": "decimals",
+                "value": 1
+              },
+              {
+                "id": "custom.cellOptions",
+                "value": {
+                  "mode": "gradient",
+                  "type": "gauge"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Power"
+            },
+            "properties": [
+              {
+                "id": "unit",
+                "value": "watt"
+              },
+              {
+                "id": "decimals",
+                "value": 0
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "VRAM"
+            },
+            "properties": [
+              {
+                "id": "unit",
+                "value": "bytes"
+              }
+            ]
+          }
+        ]
+      }
+    },
+    {
+      "type": "row",
+      "collapsed": false,
+      "title": "Historical usage (last 24h)",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 43
+      },
+      "id": 50,
+      "panels": []
+    },
+    {
+      "type": "bargauge",
+      "id": 51,
+      "targets": [
+        {
+          "expr": "sum_over_time(namespace:gpu_count:allocated{namespace=~\"$namespace\"}[24h:1m]) / 60",
+          "legendFormat": "{{namespace}}",
+          "refId": "A"
+        }
+      ],
+      "title": "GPU-hours per namespace (last 24h)",
+      "description": "Billable GPU-hours per tenant over the last 24h window.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 44
+      },
+      "repeatDirection": "h",
+      "options": {
+        "displayMode": "gradient",
+        "valueMode": "color",
+        "namePlacement": "auto",
+        "showUnfilled": true,
+        "sizing": "auto",
+        "minVizWidth": 8,
+        "minVizHeight": 16,
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": false,
+          "calcs": []
+        },
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "maxVizHeight": 300,
+        "orientation": "horizontal"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 1,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "value": null,
+                "color": "blue"
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "bargauge",
+      "id": 52,
+      "targets": [
+        {
+          "expr": "sum_over_time(namespace:power_watts:sum{namespace=~\"$namespace\"}[24h:1m]) / 60 / 1000",
+          "legendFormat": "{{namespace}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Energy per namespace (last 24h)",
+      "description": "kWh consumed per tenant over the last 24h. Integrated from 1-minute power samples.",
+      "transparent": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds_prometheus"
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 44
+      },
+      "repeatDirection": "h",
+      "options": {
+        "displayMode": "gradient",
+        "valueMode": "color",
+        "namePlacement": "auto",
+        "showUnfilled": true,
+        "sizing": "auto",
+        "minVizWidth": 8,
+        "minVizHeight": 16,
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": false,
+          "calcs": []
+        },
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "maxVizHeight": 300,
+        "orientation": "horizontal"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "kwatth",
+          "decimals": 2,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "value": null,
+                "color": "green"
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    }
+  ],
+  "templating": {
+    "list": [
+      {
+        "type": "datasource",
+        "name": "ds_prometheus",
+        "label": "Prometheus",
+        "skipUrlSync": false,
+        "query": "prometheus",
+        "current": {
+          "selected": false,
+          "text": "default",
+          "value": "default"
+        },
+        "multi": false,
+        "allowCustomValue": true,
+        "includeAll": false,
+        "regex": "",
+        "auto": false,
+        "auto_min": "10s",
+        "auto_count": 30
+      },
+      {
+        "type": "query",
+        "name": "namespace",
+        "label": "Namespace",
+        "skipUrlSync": false,
+        "query": "label_values(namespace:gpu_count:allocated, namespace)",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "$ds_prometheus"
+        },
+        "current": {
+          "selected": true,
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true,
+        "allowCustomValue": true,
+        "refresh": 2,
+        "sort": 1,
+        "includeAll": true,
+        "auto": false,
+        "auto_min": "10s",
+        "auto_count": 30
+      }
+    ]
+  },
+  "annotations": {}
+}
diff --git a/packages/system/gpu-operator/examples/README.md b/packages/system/gpu-operator/examples/README.md
index 11c2bb09..5374bffc 100644
--- a/packages/system/gpu-operator/examples/README.md
+++ b/packages/system/gpu-operator/examples/README.md
@@ -21,16 +21,10 @@ Talos.
   ConfigMap.
 - [`dcgm-custom-metrics.yaml`](./dcgm-custom-metrics.yaml) — `ConfigMap`
   with a DCGM metrics CSV that adds profiling, ECC, throttling and
-  energy counters on top of the upstream defaults. The CSV is the
-  superset needed for full dashboard coverage; the **recording rules
-  themselves** only require the profiling subset
-  (`DCGM_FI_PROF_PIPE_TENSOR_ACTIVE`, `DCGM_FI_PROF_GR_ENGINE_ACTIVE`)
-  on top of the upstream `default-counters.csv` — every other DCGM
-  series the rules consume (utilization, FB used/free, power,
-  temperature, energy) is already in the default set. The
-  `gpu/gpu-performance` dashboard additionally needs the throttle
-  counters (`DCGM_FI_DEV_POWER_VIOLATION`,
-  `DCGM_FI_DEV_THERMAL_VIOLATION`), which are not in the default set.
+  energy counters on top of the upstream defaults. The CSV is a
+  superset needed for full coverage of the `gpu/gpu-performance`
+  dashboard. Which parts are actually required depends on which
+  dashboards you ship — see the table below.
 - [`nvidia-driver-compat.yaml`](./nvidia-driver-compat.yaml) — DaemonSet
   that stages `libnvidia-ml.so.1` and `nvidia-smi` from the Talos glibc
   tree into a path where the NVIDIA GPU Operator validator expects
@@ -62,32 +56,46 @@ files into a directory the validator does inspect and creates the
 
 [1]: https://github.com/NVIDIA/gpu-operator/issues/1687
 
+## Dashboards and what DCGM metrics they need
+
+Five GPU dashboards live under `gpu/*` in
+`packages/system/monitoring/dashboards-infra.list`. All of them share
+`packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml` as
+their source of aggregated series. The recording rules are safe to
+ship on any cluster — they evaluate to empty series when DCGM is not
+scraped, or when optional counters are missing.
+
+What each dashboard needs on top of the upstream DCGM Exporter
+[`default-counters.csv`][default-csv]:
+
+| Dashboard         | Scope                              | Needs beyond defaults                                                                        |
+| ----------------- | ---------------------------------- | -------------------------------------------------------------------------------------------- |
+| `gpu-performance` | Per-node, per-GPU deep dive        | `DCGM_FI_PROF_PIPE_TENSOR_ACTIVE`, `DCGM_FI_PROF_GR_ENGINE_ACTIVE`, `DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION` |
+| `gpu-efficiency`  | Per-workload util vs tensor active | `DCGM_FI_PROF_PIPE_TENSOR_ACTIVE`                                                            |
+| `gpu-fleet`       | Cluster-wide admin inventory       | nothing (works on default counters)                                                          |
+| `gpu-quotas`      | Kube-quota vs live usage           | nothing (kube-state-metrics + default counters)                                              |
+| `gpu-tenants`     | Per-namespace tenant view          | `DCGM_FI_PROF_PIPE_TENSOR_ACTIVE` for the tensor-saturation panel; other panels work on defaults |
+
+The throttling counters (`DCGM_FI_DEV_POWER_VIOLATION`,
+`DCGM_FI_DEV_THERMAL_VIOLATION`) are only required by `gpu-performance`.
+The profiling counters (`DCGM_FI_PROF_*`) are required by
+`gpu-performance` and `gpu-efficiency`, and unlock the tensor panel in
+`gpu-tenants`. Everything else the recording rules consume —
+utilization, FB used/free, power, temperature, energy — is already in
+the default counter set.
+
 ## Verification status
 
 > **Pending verification on an updated GPU Operator release.**
 >
-> The minimum-CSV claim above (only `DCGM_FI_PROF_*` is needed beyond
-> the upstream default counters) is derived by cross-referencing
-> `gpu-recording.rules.yaml` against the DCGM Exporter
-> [`default-counters.csv`][default-csv] for the version pinned in the
-> currently shipped `gpu-operator` package. The package in this branch
-> is **not** the latest GPU Operator release; once we move to a newer
-> version, the claim must be re-checked because the upstream default
-> set occasionally adds or removes counters between releases. Until
-> then, treat the CSV in `dcgm-custom-metrics.yaml` as a known-good
-> superset rather than a minimal config.
+> The minimum-CSV claims above are derived by cross-referencing
+> `gpu-recording.rules.yaml` and each dashboard against the DCGM
+> Exporter [`default-counters.csv`][default-csv] for the version pinned
+> in the currently shipped `gpu-operator` package. The package in this
+> branch is **not** the latest GPU Operator release; once we move to a
+> newer version, the claims must be re-checked because the upstream
+> default set occasionally adds or removes counters between releases.
+> Until then, treat the CSV in `dcgm-custom-metrics.yaml` as a
+> known-good superset rather than a minimal config.
 
 [default-csv]: https://github.com/NVIDIA/dcgm-exporter/blob/main/etc/default-counters.csv
-
-## How the dashboard and recording rules fit in
-
-- `dashboards/gpu/gpu-performance.json` expects `DCGM_FI_*` metrics,
-  including profiling series (`DCGM_FI_PROF_PIPE_TENSOR_ACTIVE`,
-  `DCGM_FI_PROF_GR_ENGINE_ACTIVE`) and throttling counters
-  (`DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION`).
-  These are only emitted when DCGM Exporter is started with the custom
-  CSV in `dcgm-custom-metrics.yaml`.
-- `packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml`
-  precomputes cluster-wide and per-namespace aggregations used by the
-  overview panels of the dashboard. The rules are safe to ship on any
-  cluster — they evaluate to empty series when DCGM is not scraped.
diff --git a/packages/system/monitoring/dashboards-infra.list b/packages/system/monitoring/dashboards-infra.list
index b402fa68..4c7e494d 100644
--- a/packages/system/monitoring/dashboards-infra.list
+++ b/packages/system/monitoring/dashboards-infra.list
@@ -35,3 +35,5 @@ hubble/network-overview
 gpu/gpu-performance
 gpu/gpu-efficiency
 gpu/gpu-quotas
+gpu/gpu-fleet
+gpu/gpu-tenants

From 7eb9fe8ade53e450962e71548129d776c8fa7e6d Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sat, 18 Apr 2026 06:41:45 +0300
Subject: [PATCH 038/151] refactor(monitoring): drop unused GPU recording rules
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- namespace:gpu_count:sum — never consumed by any tracked dashboard;
  the billable view is already covered by namespace:gpu_count:allocated,
  and the admin view by cluster:gpu_count:allocated.
- namespace:energy_joules:sum — no panel integrates joules; kWh
  readings on the tenant dashboard compute their own integrations
  from namespace:power_watts:sum.
- pod:tensor_to_nvml_ratio:avg5m — interesting tenant signal in
  theory, but not wired into any panel and carrying it just burns
  cardinality on large fleets.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 .../alerts/gpu-recording.rules.yaml           | 20 -------------------
 1 file changed, 20 deletions(-)

diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
index 3a31f5a2..28d6a45e 100644
--- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
+++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
@@ -25,11 +25,6 @@ spec:
     - name: gpu.recording.namespace.1m
       interval: 1m
       rules:
-        # DCGM-visible GPU count per namespace — counts GPUs that are actually
-        # running a tenant pod right now (driver loaded, scheduler placed it).
-        # Differs from :allocated when pods are Pending or stuck.
-        - record: namespace:gpu_count:sum
-          expr: count by (namespace) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"})
         # Kube-requested GPU count per namespace — billable view, includes
         # Pending pods. Use this for GPU-hour reporting via
         # sum_over_time(...[1h:1m])/60.
@@ -62,8 +57,6 @@ spec:
           expr: sum by (namespace) (DCGM_FI_DEV_FB_USED{namespace!="", namespace!~"cozy-.*|kube-.*"}) * 1048576
         - record: namespace:power_watts:sum
           expr: sum by (namespace) (DCGM_FI_DEV_POWER_USAGE{namespace!="", namespace!~"cozy-.*|kube-.*"})
-        - record: namespace:energy_joules:sum
-          expr: sum by (namespace) (DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION{namespace!="", namespace!~"cozy-.*|kube-.*"}) / 1000
 
     - name: gpu.recording.efficiency.1m
       interval: 1m
@@ -76,19 +69,6 @@ spec:
               avg_over_time(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m])
             ) * 100
 
-        # NVML vs tensor gap — ratio <10% means NVML lies: user thinks GPU
-        # is busy but specialized hardware is idle (cheap tenant signal).
-        - record: pod:tensor_to_nvml_ratio:avg5m
-          expr: |
-            (
-              avg_over_time(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]) * 100
-            )
-            /
-            clamp_min(
-              avg_over_time(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]),
-              1
-            )
-
         # Power efficiency — utilization per watt, reveals unoptimized clients.
         - record: pod:util_per_watt:avg5m
           expr: |

From 2fa4b3e31cd9777bdc88743272b064f076429c87 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sat, 18 Apr 2026 06:48:33 +0300
Subject: [PATCH 039/151] refactor(monitoring): tighten GPU dashboard queries
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- gpu-efficiency: scope Tensor Saturation, Util-per-Watt and Power
  Throttle stats to the $namespace selector. Cluster-wide means were
  misleading when a user had narrowed the dashboard to specific
  tenants — the headline numbers lied relative to the panels below.
- gpu-fleet: show per-node power draw as % of combined TDP cap
  (DCGM_FI_DEV_POWER_MGMT_LIMIT) instead of raw watts. Thresholds
  (60 / 80 %) generalize across GPU SKUs without per-model tuning.
- gpu-quotas: read cluster:gpu_count:allocated from the recording
  rules instead of recomputing sum(kube_pod_container_resource_requests)
  inline. Keeps the dashboard aligned with the canonical definition
  in gpu-recording.rules.yaml so the two can't drift.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 dashboards/gpu/gpu-efficiency.json                 | 12 ++++++------
 dashboards/gpu/gpu-fleet.json                      | 14 +++++++-------
 dashboards/gpu/gpu-quotas.json                     |  4 ++--
 .../alerts/gpu-recording.rules.yaml                |  1 +
 4 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/dashboards/gpu/gpu-efficiency.json b/dashboards/gpu/gpu-efficiency.json
index 596961e0..07000f71 100644
--- a/dashboards/gpu/gpu-efficiency.json
+++ b/dashboards/gpu/gpu-efficiency.json
@@ -35,12 +35,12 @@
       "id": 2,
       "targets": [
         {
-          "expr": "avg(pod:tensor_saturation:avg5m)",
+          "expr": "avg(pod:tensor_saturation:avg5m{namespace=~\"$namespace\"})",
           "refId": "A"
         }
       ],
-      "title": "Cluster Tensor Saturation",
-      "description": "Mean tensor core saturation. \u003c10% means GPUs are used inefficiently (tenants could move to CPU or optimize their code).",
+      "title": "Avg Tensor Saturation",
+      "description": "Mean tensor core saturation across selected namespaces. \u003c10% means GPUs are used inefficiently (tenants could move to CPU or optimize their code).",
       "transparent": false,
       "datasource": {
         "type": "prometheus",
@@ -103,12 +103,12 @@
       "id": 3,
       "targets": [
         {
-          "expr": "avg(pod:util_per_watt:avg5m)",
+          "expr": "avg(pod:util_per_watt:avg5m{namespace=~\"$namespace\"})",
           "refId": "A"
         }
       ],
       "title": "Avg Utilization per Watt",
-      "description": "NVML utilization % per watt. Higher value = more efficient workload.",
+      "description": "NVML utilization % per watt across selected namespaces. Higher value = more efficient workload.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",
@@ -166,7 +166,7 @@
       "id": 4,
       "targets": [
         {
-          "expr": "avg(gpu:power_throttle_fraction:rate5m)",
+          "expr": "avg(gpu:power_throttle_fraction:rate5m{namespace=~\"$namespace\"})",
           "refId": "A"
         }
       ],
diff --git a/dashboards/gpu/gpu-fleet.json b/dashboards/gpu/gpu-fleet.json
index af5ad2fa..4cd5b4b5 100644
--- a/dashboards/gpu/gpu-fleet.json
+++ b/dashboards/gpu/gpu-fleet.json
@@ -467,13 +467,13 @@
       "id": 12,
       "targets": [
         {
-          "expr": "sum by (Hostname) (DCGM_FI_DEV_POWER_USAGE)",
+          "expr": "sum by (Hostname) (DCGM_FI_DEV_POWER_USAGE) / sum by (Hostname) (DCGM_FI_DEV_POWER_MGMT_LIMIT) * 100",
           "legendFormat": "{{Hostname}}",
           "refId": "A"
         }
       ],
-      "title": "Power draw per node",
-      "description": "Sum of GPU power usage per node in watts.",
+      "title": "Power draw per node (% of TDP)",
+      "description": "GPU power usage as a fraction of the node's combined TDP cap. Works across GPU SKUs without per-model thresholds.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",
@@ -510,9 +510,9 @@
       },
       "fieldConfig": {
         "defaults": {
-          "unit": "watt",
+          "unit": "percent",
           "min": 0,
-          "max": 200,
+          "max": 100,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -521,11 +521,11 @@
                 "color": "green"
               },
               {
-                "value": 120,
+                "value": 60,
                 "color": "yellow"
               },
               {
-                "value": 160,
+                "value": 80,
                 "color": "orange"
               }
             ]
diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json
index 60c628f6..3bc01a83 100644
--- a/dashboards/gpu/gpu-quotas.json
+++ b/dashboards/gpu/gpu-quotas.json
@@ -86,7 +86,7 @@
       "id": 3,
       "targets": [
         {
-          "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"})",
+          "expr": "cluster:gpu_count:allocated",
           "refId": "A"
         }
       ],
@@ -143,7 +143,7 @@
       "id": 4,
       "targets": [
         {
-          "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}) / sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"}) * 100",
+          "expr": "cluster:gpu_count:allocated / sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"}) * 100",
           "refId": "A"
         }
       ],
diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
index 28d6a45e..a24951f5 100644
--- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
+++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
@@ -2,6 +2,7 @@ apiVersion: operator.victoriametrics.com/v1beta1
 kind: VMRule
 metadata:
   name: alerts-gpu-recording.rules
+  namespace: cozy-monitoring
 spec:
   groups:
     - name: gpu.recording.cluster.1m

From ccfec2ef62684b854508294ab82e379bffcad771 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sat, 18 Apr 2026 06:55:17 +0300
Subject: [PATCH 040/151] fix(monitoring): close DCGM coverage gap for
 gpu-fleet TDP panel

gpu-fleet.json references DCGM_FI_DEV_POWER_MGMT_LIMIT for its
"TDP vs draw" panel, but the custom DCGM Exporter CSV did not declare
it, so the panel silently rendered "No data" on clusters using that
config. Declare the counter, fix the dashboards table in the
gpu-operator examples README, and add a bats test that cross-checks
every DCGM_FI_* reference in tracked dashboards and recording rules
against the union of the upstream default set (snapshotted under
hack/) and the project's custom CSV.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 hack/check-gpu-recording-rules.bats           |  92 +++++++++++++++-
 hack/dcgm-default-counters.csv                | 104 ++++++++++++++++++
 .../system/gpu-operator/examples/README.md    |  50 ++++-----
 .../examples/dcgm-custom-metrics.yaml         |   1 +
 4 files changed, 220 insertions(+), 27 deletions(-)
 create mode 100644 hack/dcgm-default-counters.csv

diff --git a/hack/check-gpu-recording-rules.bats b/hack/check-gpu-recording-rules.bats
index 87e7c8f1..4981959a 100644
--- a/hack/check-gpu-recording-rules.bats
+++ b/hack/check-gpu-recording-rules.bats
@@ -1,7 +1,7 @@
 #!/usr/bin/env bats
 # -----------------------------------------------------------------------------
-# Cross-validation between GPU recording rules and the dashboards that consume
-# them. Catches:
+# Cross-validation between GPU recording rules, the dashboards that consume
+# them, and the DCGM Exporter metric set the cluster actually scrapes. Catches:
 #
 #   1. dangling references — a dashboard query mentions a recording rule name
 #      that doesn't exist in gpu-recording.rules.yaml. This is the bug the
@@ -12,6 +12,13 @@
 #   2. typos in rule names — same bug class, manifested as a single-character
 #      difference between rule and reference.
 #
+#   3. undeclared DCGM metrics — a dashboard query or recording rule mentions
+#      a DCGM_FI_* metric that is neither in the upstream default CSV nor in
+#      the project's custom CSV (dcgm-custom-metrics.yaml), meaning DCGM
+#      Exporter would never emit it and the panel silently shows "No data".
+#      Example regression: gpu-fleet.json shipped a TDP panel referencing
+#      DCGM_FI_DEV_POWER_MGMT_LIMIT before the custom CSV declared it.
+#
 # Scope: only dashboards listed in packages/system/monitoring/dashboards-infra.list
 # under the "gpu/" prefix (i.e. shipped to production), not every JSON file in
 # dashboards/gpu/. Untracked drafts stay out of scope on purpose — adding one
@@ -33,6 +40,8 @@ REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)"
 RULES_FILE="$REPO_ROOT/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml"
 DASHBOARDS_LIST="$REPO_ROOT/packages/system/monitoring/dashboards-infra.list"
 DASHBOARDS_DIR="$REPO_ROOT/dashboards"
+DCGM_DEFAULT_CSV="$REPO_ROOT/hack/dcgm-default-counters.csv"
+DCGM_CUSTOM_CSV="$REPO_ROOT/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml"
 
 # Extract the set of "- record: NAME" entries from the rules YAML.
 # Outputs one rule name per line, sorted and deduplicated.
@@ -61,6 +70,37 @@ list_tracked_gpu_dashboards() {
   awk '/^gpu\// { print $0 ".json" }' "$DASHBOARDS_LIST"
 }
 
+# Extract the set of DCGM_FI_* metric names declared in a CSV file. Handles
+# both the upstream-style default CSV (unindented) and the ConfigMap-style
+# custom CSV (YAML-indented). A declaration line starts — after any leading
+# whitespace — with "DCGM_FI_<NAME>," ; comment lines begin with "#" and are
+# skipped. Uses POSIX awk's match()+RSTART/RLENGTH so no GNU extensions
+# are required.
+extract_csv_metrics() {
+  file=$1
+  awk '
+    {
+      line = $0
+      sub(/^[[:space:]]+/, "", line)
+      if (line ~ /^#/) next
+      if (match(line, /^DCGM_FI_[A-Z0-9_]+/)) {
+        print substr(line, RSTART, RLENGTH)
+      }
+    }
+  ' "$file" | sort -u
+}
+
+# Extract the set of DCGM_FI_* metric references from a text file (dashboard
+# JSON or rules YAML). A DCGM metric name has at least two underscore-delimited
+# segments after the "DCGM_FI_" prefix (e.g. DCGM_FI_DEV_GPU_UTIL, DCGM_FI_PROF_
+# PIPE_TENSOR_ACTIVE, DCGM_FI_DRIVER_VERSION). Requiring two segments keeps
+# the matcher from latching onto glob stubs like "DCGM_FI_DEV_*_VIOLATION" that
+# appear in comments.
+extract_dcgm_refs() {
+  file=$1
+  grep -hoE 'DCGM_FI_[A-Z0-9]+(_[A-Z0-9]+)+' "$file" | sort -u
+}
+
 @test "every recording rule reference in tracked GPU dashboards has a matching record" {
   TMP=$(mktemp -d)
   trap 'rm -rf "$TMP"' EXIT
@@ -94,6 +134,54 @@ list_tracked_gpu_dashboards() {
   [ "$failed" -eq 0 ]
 }
 
+@test "every DCGM metric referenced in tracked dashboards and rules is declared" {
+  TMP=$(mktemp -d)
+  trap 'rm -rf "$TMP"' EXIT
+
+  [ -f "$DCGM_DEFAULT_CSV" ] || { echo "missing $DCGM_DEFAULT_CSV" >&2; exit 1; }
+  [ -f "$DCGM_CUSTOM_CSV" ]  || { echo "missing $DCGM_CUSTOM_CSV"  >&2; exit 1; }
+
+  {
+    extract_csv_metrics "$DCGM_DEFAULT_CSV"
+    extract_csv_metrics "$DCGM_CUSTOM_CSV"
+  } | sort -u > "$TMP/declared.txt"
+  [ -s "$TMP/declared.txt" ] || { echo "no DCGM metrics extracted from CSVs" >&2; exit 1; }
+
+  list_tracked_gpu_dashboards > "$TMP/dashboards.txt"
+  [ -s "$TMP/dashboards.txt" ] || { echo "no gpu/* dashboards listed in $DASHBOARDS_LIST" >&2; exit 1; }
+
+  failed=0
+
+  # Dashboard coverage — every dashboard's DCGM references must resolve.
+  while IFS= read -r dashboard_rel; do
+    dashboard="$DASHBOARDS_DIR/$dashboard_rel"
+    [ -f "$dashboard" ] || continue  # handled by the existence test
+    extract_dcgm_refs "$dashboard" > "$TMP/refs.txt"
+    [ -s "$TMP/refs.txt" ] || continue  # dashboard relies entirely on recording rules
+    comm -23 "$TMP/refs.txt" "$TMP/declared.txt" > "$TMP/missing.txt"
+    if [ -s "$TMP/missing.txt" ]; then
+      echo "ERROR: $dashboard_rel references DCGM metrics not declared in any CSV:" >&2
+      sed 's/^/  - /' "$TMP/missing.txt" >&2
+      failed=1
+    fi
+  done < "$TMP/dashboards.txt"
+
+  # Rules coverage — recording rules consume DCGM directly, so their set
+  # must be declared too, otherwise derived series on every dashboard
+  # collapse to empty.
+  extract_dcgm_refs "$RULES_FILE" > "$TMP/rule-refs.txt"
+  if [ -s "$TMP/rule-refs.txt" ]; then
+    comm -23 "$TMP/rule-refs.txt" "$TMP/declared.txt" > "$TMP/rule-missing.txt"
+    if [ -s "$TMP/rule-missing.txt" ]; then
+      echo "ERROR: gpu-recording.rules.yaml references DCGM metrics not declared in any CSV:" >&2
+      sed 's/^/  - /' "$TMP/rule-missing.txt" >&2
+      failed=1
+    fi
+  fi
+
+  [ "$failed" -eq 0 ]
+}
+
 @test "every tracked GPU dashboard listed in dashboards-infra.list exists on disk" {
   TMP=$(mktemp -d)
   trap 'rm -rf "$TMP"' EXIT
diff --git a/hack/dcgm-default-counters.csv b/hack/dcgm-default-counters.csv
new file mode 100644
index 00000000..b5e94540
--- /dev/null
+++ b/hack/dcgm-default-counters.csv
@@ -0,0 +1,104 @@
+# Snapshot of the upstream DCGM Exporter default-counters.csv used as a
+# fixture by hack/check-gpu-recording-rules.bats. The test verifies that
+# every DCGM_FI_* metric referenced by a tracked GPU dashboard is either
+# declared here (upstream defaults) or in
+# packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml
+# (the project's custom CSV).
+#
+# Source: https://github.com/NVIDIA/dcgm-exporter/blob/4.1.1-4.0.4/etc/default-counters.csv
+# Pinned to the DCGM Exporter image tag shipped by the gpu-operator
+# chart under packages/system/gpu-operator/charts/gpu-operator/values.yaml
+# (dcgmExporter.version = 4.1.1-4.0.4-ubuntu22.04). When that image is
+# bumped, refresh this file from the matching tag in the NVIDIA/dcgm-exporter
+# repo and re-run `./hack/cozytest.sh hack/check-gpu-recording-rules.bats`.
+
+# Format
+# If line starts with a '#' it is considered a comment
+# DCGM FIELD, Prometheus metric type, help message
+
+# Clocks
+DCGM_FI_DEV_SM_CLOCK,  gauge, SM clock frequency (in MHz).
+DCGM_FI_DEV_MEM_CLOCK, gauge, Memory clock frequency (in MHz).
+
+# Temperature
+DCGM_FI_DEV_MEMORY_TEMP, gauge, Memory temperature (in C).
+DCGM_FI_DEV_GPU_TEMP,    gauge, GPU temperature (in C).
+
+# Power
+DCGM_FI_DEV_POWER_USAGE,              gauge, Power draw (in W).
+DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION, counter, Total energy consumption since boot (in mJ).
+
+# PCIE
+# DCGM_FI_PROF_PCIE_TX_BYTES,  counter, Total number of bytes transmitted through PCIe TX via NVML.
+# DCGM_FI_PROF_PCIE_RX_BYTES,  counter, Total number of bytes received through PCIe RX via NVML.
+DCGM_FI_DEV_PCIE_REPLAY_COUNTER, counter, Total number of PCIe retries.
+
+# Utilization (the sample period varies depending on the product)
+DCGM_FI_DEV_GPU_UTIL,      gauge, GPU utilization (in %).
+DCGM_FI_DEV_MEM_COPY_UTIL, gauge, Memory utilization (in %).
+DCGM_FI_DEV_ENC_UTIL,      gauge, Encoder utilization (in %).
+DCGM_FI_DEV_DEC_UTIL ,     gauge, Decoder utilization (in %).
+
+# Errors and violations
+DCGM_FI_DEV_XID_ERRORS,              gauge,   Value of the last XID error encountered.
+# DCGM_FI_DEV_POWER_VIOLATION,       counter, Throttling duration due to power constraints (in us).
+# DCGM_FI_DEV_THERMAL_VIOLATION,     counter, Throttling duration due to thermal constraints (in us).
+# DCGM_FI_DEV_SYNC_BOOST_VIOLATION,  counter, Throttling duration due to sync-boost constraints (in us).
+# DCGM_FI_DEV_BOARD_LIMIT_VIOLATION, counter, Throttling duration due to board limit constraints (in us).
+# DCGM_FI_DEV_LOW_UTIL_VIOLATION,    counter, Throttling duration due to low utilization (in us).
+# DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (in us).
+
+# Memory usage
+DCGM_FI_DEV_FB_FREE, gauge, Framebuffer memory free (in MiB).
+DCGM_FI_DEV_FB_USED, gauge, Framebuffer memory used (in MiB).
+
+# ECC
+# DCGM_FI_DEV_ECC_SBE_VOL_TOTAL, counter, Total number of single-bit volatile ECC errors.
+# DCGM_FI_DEV_ECC_DBE_VOL_TOTAL, counter, Total number of double-bit volatile ECC errors.
+# DCGM_FI_DEV_ECC_SBE_AGG_TOTAL, counter, Total number of single-bit persistent ECC errors.
+# DCGM_FI_DEV_ECC_DBE_AGG_TOTAL, counter, Total number of double-bit persistent ECC errors.
+
+# Retired pages
+# DCGM_FI_DEV_RETIRED_SBE,     counter, Total number of retired pages due to single-bit errors.
+# DCGM_FI_DEV_RETIRED_DBE,     counter, Total number of retired pages due to double-bit errors.
+# DCGM_FI_DEV_RETIRED_PENDING, counter, Total number of pages pending retirement.
+
+# NVLink
+# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_TOTAL, counter, Total number of NVLink flow-control CRC errors.
+# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_TOTAL, counter, Total number of NVLink data CRC errors.
+# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_TOTAL,   counter, Total number of NVLink retries.
+# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_TOTAL, counter, Total number of NVLink recovery errors.
+DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL,            counter, Total number of NVLink bandwidth counters for all lanes.
+# DCGM_FI_DEV_NVLINK_BANDWIDTH_L0,               counter, The number of bytes of active NVLink rx or tx data including both header and payload.
+
+# VGPU License status
+DCGM_FI_DEV_VGPU_LICENSE_STATUS, gauge, vGPU License status
+
+# Remapped rows
+DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for uncorrectable errors
+DCGM_FI_DEV_CORRECTABLE_REMAPPED_ROWS,   counter, Number of remapped rows for correctable errors
+DCGM_FI_DEV_ROW_REMAP_FAILURE,           gauge,   Whether remapping of rows has failed
+
+# Static configuration information. These appear as labels on the other metrics
+DCGM_FI_DRIVER_VERSION,        label, Driver Version
+# DCGM_FI_NVML_VERSION,          label, NVML Version
+# DCGM_FI_DEV_BRAND,             label, Device Brand
+# DCGM_FI_DEV_SERIAL,            label, Device Serial Number
+# DCGM_FI_DEV_OEM_INFOROM_VER,   label, OEM inforom version
+# DCGM_FI_DEV_ECC_INFOROM_VER,   label, ECC inforom version
+# DCGM_FI_DEV_POWER_INFOROM_VER, label, Power management object inforom version
+# DCGM_FI_DEV_INFOROM_IMAGE_VER, label, Inforom image version
+# DCGM_FI_DEV_VBIOS_VERSION,     label, VBIOS version of the device
+
+# Datacenter Profiling (DCP) metrics
+# NOTE: supported on Nvidia datacenter Volta GPUs and newer
+DCGM_FI_PROF_GR_ENGINE_ACTIVE,   gauge, Ratio of time the graphics engine is active.
+# DCGM_FI_PROF_SM_ACTIVE,          gauge, The ratio of cycles an SM has at least 1 warp assigned.
+# DCGM_FI_PROF_SM_OCCUPANCY,       gauge, The ratio of number of warps resident on an SM.
+DCGM_FI_PROF_PIPE_TENSOR_ACTIVE, gauge, Ratio of cycles the tensor (HMMA) pipe is active.
+DCGM_FI_PROF_DRAM_ACTIVE,        gauge, Ratio of cycles the device memory interface is active sending or receiving data.
+# DCGM_FI_PROF_PIPE_FP64_ACTIVE,   gauge, Ratio of cycles the fp64 pipes are active.
+# DCGM_FI_PROF_PIPE_FP32_ACTIVE,   gauge, Ratio of cycles the fp32 pipes are active.
+# DCGM_FI_PROF_PIPE_FP16_ACTIVE,   gauge, Ratio of cycles the fp16 pipes are active.
+DCGM_FI_PROF_PCIE_TX_BYTES,      gauge, The rate of data transmitted over the PCIe bus - including both protocol headers and data payloads - in bytes per second.
+DCGM_FI_PROF_PCIE_RX_BYTES,      gauge, The rate of data received over the PCIe bus - including both protocol headers and data payloads - in bytes per second.
diff --git a/packages/system/gpu-operator/examples/README.md b/packages/system/gpu-operator/examples/README.md
index 5374bffc..9599acb7 100644
--- a/packages/system/gpu-operator/examples/README.md
+++ b/packages/system/gpu-operator/examples/README.md
@@ -68,34 +68,34 @@ scraped, or when optional counters are missing.
 What each dashboard needs on top of the upstream DCGM Exporter
 [`default-counters.csv`][default-csv]:
 
-| Dashboard         | Scope                              | Needs beyond defaults                                                                        |
-| ----------------- | ---------------------------------- | -------------------------------------------------------------------------------------------- |
-| `gpu-performance` | Per-node, per-GPU deep dive        | `DCGM_FI_PROF_PIPE_TENSOR_ACTIVE`, `DCGM_FI_PROF_GR_ENGINE_ACTIVE`, `DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION` |
-| `gpu-efficiency`  | Per-workload util vs tensor active | `DCGM_FI_PROF_PIPE_TENSOR_ACTIVE`                                                            |
-| `gpu-fleet`       | Cluster-wide admin inventory       | nothing (works on default counters)                                                          |
-| `gpu-quotas`      | Kube-quota vs live usage           | nothing (kube-state-metrics + default counters)                                              |
-| `gpu-tenants`     | Per-namespace tenant view          | `DCGM_FI_PROF_PIPE_TENSOR_ACTIVE` for the tensor-saturation panel; other panels work on defaults |
+| Dashboard         | Scope                              | Needs beyond defaults                                                   |
+| ----------------- | ---------------------------------- | ----------------------------------------------------------------------- |
+| `gpu-performance` | Per-node, per-GPU deep dive        | `DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION`          |
+| `gpu-efficiency`  | Per-workload util vs tensor active | `DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION` (via `gpu:*_throttle_fraction:rate5m` recording rules) |
+| `gpu-fleet`       | Cluster-wide admin inventory       | `DCGM_FI_DEV_POWER_MGMT_LIMIT` (for the TDP vs draw panel)              |
+| `gpu-quotas`      | Kube-quota vs live usage           | nothing (kube-state-metrics + default counters)                         |
+| `gpu-tenants`     | Per-namespace tenant view          | nothing (works on default counters)                                     |
 
-The throttling counters (`DCGM_FI_DEV_POWER_VIOLATION`,
-`DCGM_FI_DEV_THERMAL_VIOLATION`) are only required by `gpu-performance`.
-The profiling counters (`DCGM_FI_PROF_*`) are required by
-`gpu-performance` and `gpu-efficiency`, and unlock the tensor panel in
-`gpu-tenants`. Everything else the recording rules consume —
-utilization, FB used/free, power, temperature, energy — is already in
-the default counter set.
+`DCGM_FI_PROF_PIPE_TENSOR_ACTIVE` and `DCGM_FI_PROF_GR_ENGINE_ACTIVE`
+are already in the upstream default set for the pinned DCGM Exporter
+version, so the tensor-saturation and engine-active panels work without
+any CSV override. The three counters listed in the table — throttling
+violations and the power management limit — are the only extras the
+tracked dashboards need. The recording rules in
+`gpu-recording.rules.yaml` consume utilization, FB used, power,
+temperature and the tensor-active profiling counter, all of which are
+in the default set.
 
 ## Verification status
 
-> **Pending verification on an updated GPU Operator release.**
->
-> The minimum-CSV claims above are derived by cross-referencing
-> `gpu-recording.rules.yaml` and each dashboard against the DCGM
-> Exporter [`default-counters.csv`][default-csv] for the version pinned
-> in the currently shipped `gpu-operator` package. The package in this
-> branch is **not** the latest GPU Operator release; once we move to a
-> newer version, the claims must be re-checked because the upstream
-> default set occasionally adds or removes counters between releases.
-> Until then, treat the CSV in `dcgm-custom-metrics.yaml` as a
-> known-good superset rather than a minimal config.
+The minimum-CSV claims above are verified by
+`hack/check-gpu-recording-rules.bats`, which cross-checks every
+`DCGM_FI_*` reference in the tracked GPU dashboards and recording rules
+against the union of the upstream default set (snapshotted at
+`hack/dcgm-default-counters.csv` for the pinned DCGM Exporter version)
+and the custom CSV in `dcgm-custom-metrics.yaml`. When the DCGM
+Exporter image in `packages/system/gpu-operator/charts/gpu-operator/values.yaml`
+is bumped, refresh the snapshot from the matching tag of the
+[`NVIDIA/dcgm-exporter`][default-csv] repository and rerun the test.
 
 [default-csv]: https://github.com/NVIDIA/dcgm-exporter/blob/main/etc/default-counters.csv
diff --git a/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml b/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml
index 14b89b5c..92064701 100644
--- a/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml
+++ b/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml
@@ -26,6 +26,7 @@ data:
 
     # Power
     DCGM_FI_DEV_POWER_USAGE,              gauge, Power draw (in W).
+    DCGM_FI_DEV_POWER_MGMT_LIMIT,         gauge, Current power management limit (in W).
     DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION, counter, Total energy consumption since boot (in mJ).
 
     # PCIE

From 2518e09d6775ae9916c52dba16b52d88a0e7beaa Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sat, 18 Apr 2026 07:03:42 +0300
Subject: [PATCH 041/151] refactor(monitoring): store pod:tensor_saturation as
 unitless ratio

Align pod:tensor_saturation:avg5m with namespace:tensor_active:avg and
DCGM's native 0..1 range by dropping the * 100 from the recording rule
and multiplying at display time in gpu-efficiency.json. Also scope
pod:util_per_watt:avg5m with avg by (Hostname, gpu, UUID, namespace,
pod) so the series mirrors pod:tensor_saturation's grouping and stays
usable in topk queries.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 dashboards/gpu/gpu-efficiency.json               |  4 ++--
 .../alerts/gpu-recording.rules.yaml              | 16 ++++++++++------
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/dashboards/gpu/gpu-efficiency.json b/dashboards/gpu/gpu-efficiency.json
index 07000f71..dddb2001 100644
--- a/dashboards/gpu/gpu-efficiency.json
+++ b/dashboards/gpu/gpu-efficiency.json
@@ -35,7 +35,7 @@
       "id": 2,
       "targets": [
         {
-          "expr": "avg(pod:tensor_saturation:avg5m{namespace=~\"$namespace\"})",
+          "expr": "avg(pod:tensor_saturation:avg5m{namespace=~\"$namespace\"}) * 100",
           "refId": "A"
         }
       ],
@@ -363,7 +363,7 @@
       "id": 21,
       "targets": [
         {
-          "expr": "topk(20, pod:tensor_saturation:avg5m)",
+          "expr": "topk(20, pod:tensor_saturation:avg5m * 100)",
           "instant": true,
           "range": false,
           "format": "table",
diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
index a24951f5..a6ecb20a 100644
--- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
+++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
@@ -64,20 +64,24 @@ spec:
       rules:
         # Tensor hardware saturation — the honest "am I using the GPU" metric
         # for AI/LLM workloads. Unlike NVML, idle tensor cores are visible.
+        # Stored as a 0..1 ratio to stay consistent with namespace:tensor_active:avg
+        # and DCGM's native units. Consumers multiply by 100 at display time.
         - record: pod:tensor_saturation:avg5m
           expr: |
             avg by (Hostname, gpu, UUID, namespace, pod) (
               avg_over_time(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m])
-            ) * 100
+            )
 
         # Power efficiency — utilization per watt, reveals unoptimized clients.
         - record: pod:util_per_watt:avg5m
           expr: |
-            avg_over_time(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m])
-            /
-            clamp_min(
-              avg_over_time(DCGM_FI_DEV_POWER_USAGE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]),
-              1
+            avg by (Hostname, gpu, UUID, namespace, pod) (
+              avg_over_time(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m])
+              /
+              clamp_min(
+                avg_over_time(DCGM_FI_DEV_POWER_USAGE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]),
+                1
+              )
             )
 
         # Fraction of time power-throttled (TDP cap) — 1.0 = fully throttled.

From 5b210ac7fd29e15804fe787f6de8f29455eeeab4 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sat, 18 Apr 2026 07:11:28 +0300
Subject: [PATCH 042/151] docs(monitoring): mark gpu-fleet average utilization
 as legacy NVML

Clarify that the "Average utilization" panel on gpu-fleet reflects the
legacy NVML view (DCGM_FI_DEV_GPU_UTIL) rather than engine-active
profiling. For AI/LLM workloads the NVML number is optimistic; the
gpu-efficiency dashboard carries the profiling-based view.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 dashboards/gpu/gpu-fleet.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dashboards/gpu/gpu-fleet.json b/dashboards/gpu/gpu-fleet.json
index 4cd5b4b5..ea169433 100644
--- a/dashboards/gpu/gpu-fleet.json
+++ b/dashboards/gpu/gpu-fleet.json
@@ -268,7 +268,7 @@
         }
       ],
       "title": "Average utilization",
-      "description": "NVML utilization across all tenant GPUs (engine-active %).",
+      "description": "Legacy NVML utilization across all tenant GPUs.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",

From 4e37f64553728bab5b59ae7d7e8a50ee91c23232 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sat, 18 Apr 2026 07:19:05 +0300
Subject: [PATCH 043/151] docs(gpu-operator): document tolerate-all on compat
 DaemonSet

Explain why tolerations: [{operator: Exists}] is safe on the driver
compat DaemonSet: the nodeSelector already confines scheduling to GPU
nodes, so the blanket toleration only kicks in when those nodes carry
the dedicated=gpu / nvidia.com/gpu taints that the GPU Operator's
default policy and many deployments apply.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 .../system/gpu-operator/examples/nvidia-driver-compat.yaml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml b/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml
index 63e07c4f..bad8609e 100644
--- a/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml
+++ b/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml
@@ -35,6 +35,13 @@ spec:
       # cluster uses to mark GPU hosts.
       nodeSelector:
         nvidia.com/gpu.present: "true"
+      # Tolerate-all is intentionally broad: the nodeSelector above already
+      # confines scheduling to GPU nodes, so the "Exists" toleration only
+      # matters when those nodes carry extra taints (dedicated=gpu:NoSchedule,
+      # nvidia.com/gpu:NoSchedule from the GPU Operator's default policy,
+      # etc.). This mirrors the stance of the upstream nvidia-driver-daemonset
+      # and nvidia-device-plugin DaemonSets — blanket tolerations plus a
+      # narrow nodeSelector.
       tolerations:
         - operator: Exists
       initContainers:

From 5db6ec3e1fc8b2d30d3aa017f0ff7bc4dd943000 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sat, 18 Apr 2026 07:27:33 +0300
Subject: [PATCH 044/151] fix(dashboard): scope GPU panels to selected
 namespace

Pod-level panels on the efficiency dashboard and DCGM-level panels on
the performance dashboard ignored the $namespace template variable, so
changing it left the visualizations unchanged. Add the filter to each
query. Performance-side queries use the `$namespace|` empty-tolerant
form so host-level DCGM series without a namespace label remain
visible when a specific namespace is selected.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 dashboards/gpu/gpu-efficiency.json  |  4 ++--
 dashboards/gpu/gpu-performance.json | 12 ++++++------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/dashboards/gpu/gpu-efficiency.json b/dashboards/gpu/gpu-efficiency.json
index dddb2001..e2e8c105 100644
--- a/dashboards/gpu/gpu-efficiency.json
+++ b/dashboards/gpu/gpu-efficiency.json
@@ -363,7 +363,7 @@
       "id": 21,
       "targets": [
         {
-          "expr": "topk(20, pod:tensor_saturation:avg5m * 100)",
+          "expr": "topk(20, pod:tensor_saturation:avg5m{namespace=~\"$namespace\"} * 100)",
           "instant": true,
           "range": false,
           "format": "table",
@@ -501,7 +501,7 @@
       "id": 22,
       "targets": [
         {
-          "expr": "topk(20, pod:util_per_watt:avg5m)",
+          "expr": "topk(20, pod:util_per_watt:avg5m{namespace=~\"$namespace\"})",
           "instant": true,
           "range": false,
           "format": "table",
diff --git a/dashboards/gpu/gpu-performance.json b/dashboards/gpu/gpu-performance.json
index b0f26958..e16f8c47 100644
--- a/dashboards/gpu/gpu-performance.json
+++ b/dashboards/gpu/gpu-performance.json
@@ -553,7 +553,7 @@
       "id": 22,
       "targets": [
         {
-          "expr": "DCGM_FI_DEV_FB_FREE{Hostname=~\"$Hostname\"} * 1048576",
+          "expr": "DCGM_FI_DEV_FB_FREE{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"} * 1048576",
           "legendFormat": "{{Hostname}}/{{gpu}}",
           "refId": "A"
         }
@@ -617,7 +617,7 @@
       "id": 31,
       "targets": [
         {
-          "expr": "DCGM_FI_DEV_POWER_USAGE{Hostname=~\"$Hostname\"}",
+          "expr": "DCGM_FI_DEV_POWER_USAGE{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"}",
           "legendFormat": "{{Hostname}}/{{gpu}}",
           "refId": "A"
         }
@@ -668,7 +668,7 @@
       "id": 32,
       "targets": [
         {
-          "expr": "DCGM_FI_DEV_GPU_TEMP{Hostname=~\"$Hostname\"}",
+          "expr": "DCGM_FI_DEV_GPU_TEMP{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"}",
           "legendFormat": "{{Hostname}}/{{gpu}}",
           "refId": "A"
         }
@@ -751,7 +751,7 @@
       "id": 41,
       "targets": [
         {
-          "expr": "max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS{Hostname=~\"$Hostname\"})",
+          "expr": "max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"})",
           "legendFormat": "{{Hostname}}/{{gpu}}",
           "refId": "A"
         }
@@ -809,7 +809,7 @@
       "id": 42,
       "targets": [
         {
-          "expr": "rate(DCGM_FI_DEV_POWER_VIOLATION{Hostname=~\"$Hostname\"}[5m])",
+          "expr": "rate(DCGM_FI_DEV_POWER_VIOLATION{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"}[5m])",
           "legendFormat": "{{Hostname}}/{{gpu}}",
           "refId": "A"
         }
@@ -857,7 +857,7 @@
       "id": 43,
       "targets": [
         {
-          "expr": "rate(DCGM_FI_DEV_THERMAL_VIOLATION{Hostname=~\"$Hostname\"}[5m])",
+          "expr": "rate(DCGM_FI_DEV_THERMAL_VIOLATION{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"}[5m])",
           "legendFormat": "{{Hostname}}/{{gpu}}",
           "refId": "A"
         }

From b64bfcc414ec4faddecf49ff9ece8ef169896bc0 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sat, 18 Apr 2026 07:36:14 +0300
Subject: [PATCH 045/151] fix(dashboard): deduplicate pending GPU pods by
 (namespace, pod)

The Pending GPU pods counter on gpu-quotas joined raw
kube_pod_container_resource_requests (per-container series) against
kube_pod_status_phase (per-pod series). Multi-container pods were
counted once per requesting container instead of once per pod, so the
widget over-reported whenever a Pending pod had more than one GPU
container. Collapse the requests to pod level before the join.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 dashboards/gpu/gpu-quotas.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json
index 3bc01a83..624f072b 100644
--- a/dashboards/gpu/gpu-quotas.json
+++ b/dashboards/gpu/gpu-quotas.json
@@ -208,7 +208,7 @@
       "id": 5,
       "targets": [
         {
-          "expr": "count((kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"} \u003e 0) * on(namespace, pod) group_left() (kube_pod_status_phase{phase=\"Pending\"} == 1)) or vector(0)",
+          "expr": "count((sum by (namespace, pod) (kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}) \u003e 0) * on(namespace, pod) group_left() (kube_pod_status_phase{phase=\"Pending\"} == 1)) or vector(0)",
           "refId": "A"
         }
       ],

From 51b0dedd0820c2a23978880baac0b8a6a3cbca94 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sat, 18 Apr 2026 07:45:02 +0300
Subject: [PATCH 046/151] chore(monitoring): tidy GPU VMRule top-level
 structure

Drop the hardcoded metadata.namespace so the rule inherits the chart's
release namespace, and add an explicit empty params field on every
group for schema consistency. No behavior change.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 .../system/monitoring-agents/alerts/gpu-recording.rules.yaml  | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
index a6ecb20a..22b3edf0 100644
--- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
+++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
@@ -2,11 +2,11 @@ apiVersion: operator.victoriametrics.com/v1beta1
 kind: VMRule
 metadata:
   name: alerts-gpu-recording.rules
-  namespace: cozy-monitoring
 spec:
   groups:
     - name: gpu.recording.cluster.1m
       interval: 1m
+      params: {}
       rules:
         - record: cluster:gpu_count:total
           expr: count(group by (UUID) (DCGM_FI_DEV_GPU_UTIL))
@@ -25,6 +25,7 @@ spec:
 
     - name: gpu.recording.namespace.1m
       interval: 1m
+      params: {}
       rules:
         # Kube-requested GPU count per namespace — billable view, includes
         # Pending pods. Use this for GPU-hour reporting via
@@ -61,6 +62,7 @@ spec:
 
     - name: gpu.recording.efficiency.1m
       interval: 1m
+      params: {}
       rules:
         # Tensor hardware saturation — the honest "am I using the GPU" metric
         # for AI/LLM workloads. Unlike NVML, idle tensor cores are visible.

From 0634654d63d0a9b6ef17074161fce84443e3ba95 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sat, 18 Apr 2026 07:53:48 +0300
Subject: [PATCH 047/151] fix(monitoring): exclude terminated pods from GPU
 allocation count
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

kube-state-metrics keeps kube_pod_container_resource_requests series
for Failed/Succeeded pods until the apiserver garbage-collects them,
which could inflate :allocated beyond what tenants actually hold and
drive cluster:gpu_count:free negative.

Join the request metric against kube_pod_status_phase filtered to
Pending|Running — the canonical pattern from Kubernetes' own
container_resource recording rules — on both the cluster and namespace
aggregates. Add clamp_min(..., 0) on cluster:gpu_count:free as a
second line of defence against transient label drift between
kube-state-metrics and DCGM.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 .../alerts/gpu-recording.rules.yaml           | 35 +++++++++++++++----
 1 file changed, 28 insertions(+), 7 deletions(-)

diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
index 22b3edf0..de81b8cc 100644
--- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
+++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
@@ -10,14 +10,29 @@ spec:
       rules:
         - record: cluster:gpu_count:total
           expr: count(group by (UUID) (DCGM_FI_DEV_GPU_UTIL))
-        # Kube-allocated GPU count: GPUs requested by *all* pods regardless of
-        # phase (Pending+Running). Source of truth for "what tenants asked for"
-        # — used for capacity planning and billing. Includes system pods so it
-        # stays consistent with cluster:gpu_count:total when computing :free.
+        # Kube-allocated GPU count: GPUs requested by *active* pods — Pending
+        # and Running only. Source of truth for "what tenants asked for" — used
+        # for capacity planning and billing. Includes system pods so it stays
+        # consistent with cluster:gpu_count:total when computing :free.
+        #
+        # Phase filter via group_left() on kube_pod_status_phase matches the
+        # canonical pattern from k8s.rules.container_resource.yaml. Without it
+        # kube-state-metrics keeps series for Failed/Succeeded pods until the
+        # apiserver garbage-collects them, which inflates billing hours and
+        # can push :free negative.
         - record: cluster:gpu_count:allocated
-          expr: sum(kube_pod_container_resource_requests{resource="nvidia_com_gpu"})
+          expr: |
+            sum(
+              kube_pod_container_resource_requests{resource="nvidia_com_gpu"}
+              * on (namespace, pod) group_left() max by (namespace, pod) (
+                kube_pod_status_phase{phase=~"Pending|Running"} == 1
+              )
+            )
+        # clamp_min guards against transients at DCGM/kube-state-metrics
+        # restart where cluster:gpu_count:total dips before :allocated catches
+        # up, or rare label drift between the two sources.
         - record: cluster:gpu_count:free
-          expr: cluster:gpu_count:total - (cluster:gpu_count:allocated or vector(0))
+          expr: clamp_min(cluster:gpu_count:total - (cluster:gpu_count:allocated or vector(0)), 0)
         - record: cluster:gpu_util:avg
           expr: avg(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"})
         - record: cluster:gpu_power_watts:sum
@@ -50,7 +65,13 @@ spec:
         # keeps system pods to stay aligned with cluster:gpu_count:total
         # when computing :free.
         - record: namespace:gpu_count:allocated
-          expr: sum by (namespace) (kube_pod_container_resource_requests{resource="nvidia_com_gpu", namespace!="", namespace!~"cozy-.*|kube-.*"})
+          expr: |
+            sum by (namespace) (
+              kube_pod_container_resource_requests{resource="nvidia_com_gpu", namespace!="", namespace!~"cozy-.*|kube-.*"}
+              * on (namespace, pod) group_left() max by (namespace, pod) (
+                kube_pod_status_phase{phase=~"Pending|Running"} == 1
+              )
+            )
         - record: namespace:gpu_util:avg
           expr: avg by (namespace) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"})
         - record: namespace:tensor_active:avg

From 605bcd338c744905097dc5c34a9b26f20b999661 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sat, 18 Apr 2026 08:03:21 +0300
Subject: [PATCH 048/151] fix(monitoring): pin label matching on GPU efficiency
 and throttle rules

pod:util_per_watt:avg5m divided two DCGM metrics without an explicit
on(...) clause, so the match used the intersection of their label
sets. If dcgm-exporter relabeling ever diverges between
DCGM_FI_DEV_GPU_UTIL and DCGM_FI_DEV_POWER_USAGE (e.g. a pod-mapping
label appears on one but not the other after a config change), the
entire result drops to empty silently. Pin the match to the labels we
group by so divergence becomes a missing side, not a missing rule.

Throttle fractions had a related shape problem: dcgm-exporter emits
one series per GPU for each pod-mapping combination. On a shared GPU
(restart races, MIG/MPS) the same physical counter appears under
multiple pod labels and downstream avg(...) panels get diluted by the
pod count. Fold duplicates with max by (Hostname, gpu, UUID) before
clamp_max so the fraction is tied to the physical GPU.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 .../alerts/gpu-recording.rules.yaml              | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
index de81b8cc..c56ea851 100644
--- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
+++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
@@ -96,11 +96,14 @@ spec:
             )
 
         # Power efficiency — utilization per watt, reveals unoptimized clients.
+        # Explicit on(...) pins the bigop matching set to the labels we
+        # group by — protects against empty results if dcgm-exporter
+        # relabeling ever diverges between the two metrics.
         - record: pod:util_per_watt:avg5m
           expr: |
             avg by (Hostname, gpu, UUID, namespace, pod) (
               avg_over_time(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m])
-              /
+              / on (Hostname, gpu, UUID, namespace, pod)
               clamp_min(
                 avg_over_time(DCGM_FI_DEV_POWER_USAGE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]),
                 1
@@ -112,9 +115,16 @@ spec:
         # counter grows in nanoseconds in practice — divide by 1e9 to get a
         # 0..1 fraction (verified empirically when /1e6 yielded >100× reality).
         # clamp_max protects against rate() artefacts at counter resets.
+        #
+        # max by (Hostname, gpu, UUID) collapses duplicate series from
+        # dcgm-exporter's pod-mapping labels (one physical GPU may appear
+        # under multiple pod/container labels during restarts or under
+        # MIG/MPS). Throttling is a physical-GPU property; taking max
+        # keeps consumer avg(...) honest — a naive avg() would dilute the
+        # signal by the number of pods sharing the GPU.
         - record: gpu:power_throttle_fraction:rate5m
-          expr: clamp_max(rate(DCGM_FI_DEV_POWER_VIOLATION[5m]) / 1e9, 1)
+          expr: clamp_max(max by (Hostname, gpu, UUID) (rate(DCGM_FI_DEV_POWER_VIOLATION[5m]) / 1e9), 1)
 
         # Fraction of time thermal-throttled.
         - record: gpu:thermal_throttle_fraction:rate5m
-          expr: clamp_max(rate(DCGM_FI_DEV_THERMAL_VIOLATION[5m]) / 1e9, 1)
+          expr: clamp_max(max by (Hostname, gpu, UUID) (rate(DCGM_FI_DEV_THERMAL_VIOLATION[5m]) / 1e9), 1)

From 165e175d70566b6890abea3efd01c18784f9b9ff Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sat, 18 Apr 2026 08:11:57 +0300
Subject: [PATCH 049/151] feat(monitoring): alert on DCGM throttle divisor
 drift
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The /1e9 divisor in gpu:{power,thermal}_throttle_fraction:rate5m was
derived empirically against DCGM 3.x on A10 — the counter documents
itself as microseconds but ticks in nanoseconds in practice. If a
future exporter release honors the documented units, pre-clamp values
would exceed 1.0 while clamp_max(..., 1) silently masks the drift,
plateauing every throttle fraction at 100% and making the panels
lie in unison.

Add a validation group that fires when the raw max/1e9 value exceeds
1.0 for 15m, so we notice and rescale to /1e6 before dashboards
silently mislead operators.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 .../alerts/gpu-recording.rules.yaml           | 30 +++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
index c56ea851..7589f894 100644
--- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
+++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
@@ -128,3 +128,33 @@ spec:
         # Fraction of time thermal-throttled.
         - record: gpu:thermal_throttle_fraction:rate5m
           expr: clamp_max(max by (Hostname, gpu, UUID) (rate(DCGM_FI_DEV_THERMAL_VIOLATION[5m]) / 1e9), 1)
+
+    # Regression watch — the /1e9 divisor on *_VIOLATION was derived
+    # empirically against DCGM 3.x on A10. If a future DCGM version
+    # restores the documented µs units, pre-clamp values would exceed 1.0
+    # while clamp_max(..., 1) silently masks the drift — recorded
+    # throttle fractions would plateau at 1.0 and consumers would think
+    # every GPU is always throttled. This alert fires on that condition
+    # so we can rescale to /1e6 before dashboards start lying.
+    - name: gpu.recording.throttle.validation.5m
+      interval: 5m
+      params: {}
+      rules:
+        - alert: GPUThrottleFractionOverOne
+          expr: |
+            max by (Hostname, gpu, UUID) (rate(DCGM_FI_DEV_POWER_VIOLATION[5m]) / 1e9) > 1
+            or
+            max by (Hostname, gpu, UUID) (rate(DCGM_FI_DEV_THERMAL_VIOLATION[5m]) / 1e9) > 1
+          for: 15m
+          labels:
+            severity: warning
+            component: gpu-monitoring
+          annotations:
+            summary: "DCGM throttle fraction exceeds 1.0 pre-clamp on {{ $labels.Hostname }}/{{ $labels.gpu }}"
+            description: |
+              gpu:{power,thermal}_throttle_fraction:rate5m clamps to 1.0, but the raw
+              rate(DCGM_FI_DEV_*_VIOLATION)/1e9 value is already >1 on this GPU — the
+              empirical nanosecond assumption is broken. Likely the DCGM exporter
+              version changed and the counter now grows in its documented µs units.
+              Re-verify on the exporter Pod and adjust the divisor (/1e9 → /1e6) in
+              gpu.recording.efficiency.1m before dashboards plateau at 100% throttle.

From 549b341675ea09ce2d61de9c931267adb55801ea Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sun, 19 Apr 2026 10:59:42 +0300
Subject: [PATCH 050/151] fix(efficiency): drop namespace filter on
 cluster-level throttle metrics

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 dashboards/gpu/gpu-efficiency.json | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/dashboards/gpu/gpu-efficiency.json b/dashboards/gpu/gpu-efficiency.json
index e2e8c105..3bd59bc0 100644
--- a/dashboards/gpu/gpu-efficiency.json
+++ b/dashboards/gpu/gpu-efficiency.json
@@ -166,12 +166,12 @@
       "id": 4,
       "targets": [
         {
-          "expr": "avg(gpu:power_throttle_fraction:rate5m{namespace=~\"$namespace\"})",
+          "expr": "avg(gpu:power_throttle_fraction:rate5m)",
           "refId": "A"
         }
       ],
       "title": "Avg Power Throttling",
-      "description": "Fraction of time GPUs hit the TDP cap and lose performance. \u003e5% means tenants underutilize billed FLOPS.",
+      "description": "Fraction of time GPUs hit the TDP cap and lose performance. \u003e5% means tenants underutilize billed FLOPS. Cluster-wide — rule aggregates by (Hostname, gpu, UUID) and drops namespace label.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",
@@ -658,7 +658,7 @@
         }
       ],
       "title": "Power throttle fraction per GPU",
-      "description": "Fraction of time the GPU was power-throttled. 1.0 = always throttled.",
+      "description": "Fraction of time the GPU was power-throttled. 1.0 = always throttled. Cluster-wide — rule aggregates by (Hostname, gpu, UUID) and drops namespace label.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",
@@ -729,7 +729,7 @@
         }
       ],
       "title": "Thermal throttle fraction per GPU",
-      "description": "Fraction of time the GPU was thermal-throttled.",
+      "description": "Fraction of time the GPU was thermal-throttled. Cluster-wide — rule aggregates by (Hostname, gpu, UUID) and drops namespace label.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",

From 5e070840d66aa9a5da934db330a8ec16d5448e26 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sun, 19 Apr 2026 11:00:02 +0300
Subject: [PATCH 051/151] fix(fleet): count GPU nodes via DCGM instead of
 kube_node_labels

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 dashboards/gpu/gpu-fleet.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dashboards/gpu/gpu-fleet.json b/dashboards/gpu/gpu-fleet.json
index ea169433..8139549e 100644
--- a/dashboards/gpu/gpu-fleet.json
+++ b/dashboards/gpu/gpu-fleet.json
@@ -209,12 +209,12 @@
       "id": 5,
       "targets": [
         {
-          "expr": "count(kube_node_labels{label_nvidia_com_gpu_present=\"true\"})",
+          "expr": "count(count by (Hostname) (DCGM_FI_DEV_GPU_UTIL))",
           "refId": "A"
         }
       ],
       "title": "Nodes with GPU",
-      "description": "Kubernetes nodes advertising nvidia.com/gpu.present=true.",
+      "description": "Nodes reporting at least one GPU to DCGM. Counted via DCGM_FI_DEV_GPU_UTIL because kube-state-metrics does not expose node labels by default.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",

From eefb3651a0b8c16ce4b6aa812fefed3b58019e15 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sun, 19 Apr 2026 11:00:53 +0300
Subject: [PATCH 052/151] fix(performance): drop namespace filter on per-GPU
 physical metrics

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 dashboards/gpu/gpu-performance.json | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/dashboards/gpu/gpu-performance.json b/dashboards/gpu/gpu-performance.json
index e16f8c47..e581a76b 100644
--- a/dashboards/gpu/gpu-performance.json
+++ b/dashboards/gpu/gpu-performance.json
@@ -553,7 +553,7 @@
       "id": 22,
       "targets": [
         {
-          "expr": "DCGM_FI_DEV_FB_FREE{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"} * 1048576",
+          "expr": "DCGM_FI_DEV_FB_FREE{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 1048576",
           "legendFormat": "{{Hostname}}/{{gpu}}",
           "refId": "A"
         }
@@ -617,12 +617,13 @@
       "id": 31,
       "targets": [
         {
-          "expr": "DCGM_FI_DEV_POWER_USAGE{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"}",
+          "expr": "DCGM_FI_DEV_POWER_USAGE{Hostname=~\"$Hostname\"}",
           "legendFormat": "{{Hostname}}/{{gpu}}",
           "refId": "A"
         }
       ],
       "title": "Power Usage per GPU",
+      "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",
@@ -668,12 +669,13 @@
       "id": 32,
       "targets": [
         {
-          "expr": "DCGM_FI_DEV_GPU_TEMP{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"}",
+          "expr": "DCGM_FI_DEV_GPU_TEMP{Hostname=~\"$Hostname\"}",
           "legendFormat": "{{Hostname}}/{{gpu}}",
           "refId": "A"
         }
       ],
       "title": "GPU Temperature",
+      "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",
@@ -751,12 +753,13 @@
       "id": 41,
       "targets": [
         {
-          "expr": "max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"})",
+          "expr": "max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS{Hostname=~\"$Hostname\"})",
           "legendFormat": "{{Hostname}}/{{gpu}}",
           "refId": "A"
         }
       ],
       "title": "XID errors (latest)",
+      "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",
@@ -809,12 +812,13 @@
       "id": 42,
       "targets": [
         {
-          "expr": "rate(DCGM_FI_DEV_POWER_VIOLATION{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"}[5m])",
+          "expr": "rate(DCGM_FI_DEV_POWER_VIOLATION{Hostname=~\"$Hostname\"}[5m])",
           "legendFormat": "{{Hostname}}/{{gpu}}",
           "refId": "A"
         }
       ],
       "title": "Power Violation (µs/s throttled due to power)",
+      "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",
@@ -857,12 +861,13 @@
       "id": 43,
       "targets": [
         {
-          "expr": "rate(DCGM_FI_DEV_THERMAL_VIOLATION{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"}[5m])",
+          "expr": "rate(DCGM_FI_DEV_THERMAL_VIOLATION{Hostname=~\"$Hostname\"}[5m])",
           "legendFormat": "{{Hostname}}/{{gpu}}",
           "refId": "A"
         }
       ],
       "title": "Thermal Violation (µs/s throttled due to thermals)",
+      "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",

From 14d9188fcda1a9780912f43d3b3ad3fd11f4c6f4 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sun, 19 Apr 2026 11:01:22 +0300
Subject: [PATCH 053/151] fix(quotas): exclude terminated pods from GPU request
 panel

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 dashboards/gpu/gpu-quotas.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json
index 624f072b..29e727ce 100644
--- a/dashboards/gpu/gpu-quotas.json
+++ b/dashboards/gpu/gpu-quotas.json
@@ -423,7 +423,7 @@
       "id": 21,
       "targets": [
         {
-          "expr": "kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left(phase) (kube_pod_status_phase{phase=~\"Running|Pending|Failed|Succeeded\"} == 1)",
+          "expr": "kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left(phase) (kube_pod_status_phase{phase=~\"Running|Pending\"} == 1)",
           "instant": true,
           "range": false,
           "format": "table",

From 950c5dd6695872850e0ec87f2c3dbb28f8dbef52 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sun, 19 Apr 2026 11:01:40 +0300
Subject: [PATCH 054/151] fix(fleet): guard TDP division and document DCGM
 dependency

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 dashboards/gpu/gpu-fleet.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dashboards/gpu/gpu-fleet.json b/dashboards/gpu/gpu-fleet.json
index 8139549e..a7d3f085 100644
--- a/dashboards/gpu/gpu-fleet.json
+++ b/dashboards/gpu/gpu-fleet.json
@@ -467,13 +467,13 @@
       "id": 12,
       "targets": [
         {
-          "expr": "sum by (Hostname) (DCGM_FI_DEV_POWER_USAGE) / sum by (Hostname) (DCGM_FI_DEV_POWER_MGMT_LIMIT) * 100",
+          "expr": "sum by (Hostname) (DCGM_FI_DEV_POWER_USAGE) / clamp_min(sum by (Hostname) (DCGM_FI_DEV_POWER_MGMT_LIMIT), 1) * 100",
           "legendFormat": "{{Hostname}}",
           "refId": "A"
         }
       ],
       "title": "Power draw per node (% of TDP)",
-      "description": "GPU power usage as a fraction of the node's combined TDP cap. Works across GPU SKUs without per-model thresholds.",
+      "description": "GPU power usage as a fraction of the node's combined TDP cap. Works across GPU SKUs without per-model thresholds. Requires DCGM_FI_DEV_POWER_MGMT_LIMIT from custom DCGM CSV (see packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml).",
       "transparent": false,
       "datasource": {
         "type": "prometheus",

From 43fe172d2f08c9bdf669473913cb485b4e6a6b86 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sun, 19 Apr 2026 11:02:05 +0300
Subject: [PATCH 055/151] docs(gpu-operator): document POWER/THERMAL_VIOLATION
 and PSS requirements

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 .../system/gpu-operator/examples/README.md    | 24 +++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/packages/system/gpu-operator/examples/README.md b/packages/system/gpu-operator/examples/README.md
index 9599acb7..7922d85a 100644
--- a/packages/system/gpu-operator/examples/README.md
+++ b/packages/system/gpu-operator/examples/README.md
@@ -56,6 +56,14 @@ files into a directory the validator does inspect and creates the
 
 [1]: https://github.com/NVIDIA/gpu-operator/issues/1687
 
+The compat DaemonSet runs privileged and bind-mounts host paths, so
+the target namespace must allow privileged pods. On clusters that
+enforce the Kubernetes Pod Security Standards at `baseline` or
+`restricted`, label the namespace with
+`pod-security.kubernetes.io/enforce: privileged` (and the matching
+`audit`/`warn` labels if the admission webhook is configured to
+surface violations) before applying the manifest.
+
 ## Dashboards and what DCGM metrics they need
 
 Five GPU dashboards live under `gpu/*` in
@@ -83,8 +91,20 @@ any CSV override. The three counters listed in the table — throttling
 violations and the power management limit — are the only extras the
 tracked dashboards need. The recording rules in
 `gpu-recording.rules.yaml` consume utilization, FB used, power,
-temperature and the tensor-active profiling counter, all of which are
-in the default set.
+temperature and the tensor-active profiling counter from the default
+set, plus `DCGM_FI_DEV_POWER_VIOLATION` and
+`DCGM_FI_DEV_THERMAL_VIOLATION` — used by the
+`gpu.recording.efficiency.1m` group to derive the
+`gpu:power_throttle_fraction:rate5m` and
+`gpu:thermal_throttle_fraction:rate5m` series consumed by the
+throttling panels on the efficiency and fleet dashboards.
+
+The `gpu.recording.throttle.validation.5m` group additionally ships the
+`GPUThrottleFractionOverOne` alert (severity `warning`) as a regression
+detector: it fires when either throttle-fraction series exceeds 1.0,
+which would indicate that DCGM changed the scale/divisor of the
+underlying violation counters and the recording rules need to be
+re-derived.
 
 ## Verification status
 

From f8b99008737b7f1067d89220a517edbed77ef777 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sun, 19 Apr 2026 11:08:25 +0300
Subject: [PATCH 056/151] fix(quotas): use allocated recording rules to exclude
 terminated pods

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 dashboards/gpu/gpu-quotas.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json
index 29e727ce..f8ce4ba8 100644
--- a/dashboards/gpu/gpu-quotas.json
+++ b/dashboards/gpu/gpu-quotas.json
@@ -278,7 +278,7 @@
       "id": 11,
       "targets": [
         {
-          "expr": "sum by (namespace) (kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"})",
+          "expr": "sum by (namespace) (namespace:gpu_count:allocated{namespace=~\"$namespace\"})",
           "legendFormat": "{{namespace}}",
           "refId": "A"
         }
@@ -346,7 +346,7 @@
           "refId": "A"
         },
         {
-          "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"})",
+          "expr": "sum(namespace:gpu_count:allocated{namespace=~\"$namespace\"})",
           "legendFormat": "Requested",
           "refId": "B"
         }

From a3241bf51bf255e24b77ebc23de89e33064370eb Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sun, 19 Apr 2026 11:08:34 +0300
Subject: [PATCH 057/151] fix(quotas): apply phase join to GPU limits column

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 dashboards/gpu/gpu-quotas.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json
index f8ce4ba8..ce1e387e 100644
--- a/dashboards/gpu/gpu-quotas.json
+++ b/dashboards/gpu/gpu-quotas.json
@@ -430,7 +430,7 @@
           "refId": "requested"
         },
         {
-          "expr": "kube_pod_container_resource_limits{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"}",
+          "expr": "kube_pod_container_resource_limits{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left() (kube_pod_status_phase{phase=~\"Running|Pending\"} == 1)",
           "instant": true,
           "range": false,
           "format": "table",

From 2cc60f170c3c370a6939e2dfffc20185fb29d0c1 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sun, 19 Apr 2026 11:08:47 +0300
Subject: [PATCH 058/151] docs(gpu-operator): reflect recording-rule dependency
 for gpu-quotas

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 packages/system/gpu-operator/examples/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/system/gpu-operator/examples/README.md b/packages/system/gpu-operator/examples/README.md
index 7922d85a..f9094549 100644
--- a/packages/system/gpu-operator/examples/README.md
+++ b/packages/system/gpu-operator/examples/README.md
@@ -81,7 +81,7 @@ What each dashboard needs on top of the upstream DCGM Exporter
 | `gpu-performance` | Per-node, per-GPU deep dive        | `DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION`          |
 | `gpu-efficiency`  | Per-workload util vs tensor active | `DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION` (via `gpu:*_throttle_fraction:rate5m` recording rules) |
 | `gpu-fleet`       | Cluster-wide admin inventory       | `DCGM_FI_DEV_POWER_MGMT_LIMIT` (for the TDP vs draw panel)              |
-| `gpu-quotas`      | Kube-quota vs live usage           | nothing (kube-state-metrics + default counters)                         |
+| `gpu-quotas`      | Kube-quota vs live usage           | `kube_pod_container_resource_requests`, `kube_pod_status_phase`, `DCGM_FI_DEV_GPU_UTIL` (via `namespace:gpu_count:allocated` / `cluster:gpu_count:*` recording rules) |
 | `gpu-tenants`     | Per-namespace tenant view          | nothing (works on default counters)                                     |
 
 `DCGM_FI_PROF_PIPE_TENSOR_ACTIVE` and `DCGM_FI_PROF_GR_ENGINE_ACTIVE`

From 95ea20119e7fddc601addc9f79777d0e24aadd30 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sun, 19 Apr 2026 11:09:01 +0300
Subject: [PATCH 059/151] fix(gpu-operator): drop unused hostPID on
 driver-compat DaemonSet

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 .../system/gpu-operator/examples/nvidia-driver-compat.yaml   | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml b/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml
index bad8609e..7706b221 100644
--- a/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml
+++ b/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml
@@ -24,8 +24,11 @@ spec:
     metadata:
       labels:
         app: nvidia-driver-compat
+    # DaemonSet rather than a one-shot Job: staging is idempotent per-node,
+    # must re-run after every Talos reboot (hostPath contents are wiped on
+    # reboot when the system extension re-installs), and the pause container
+    # keeps the pod visible for operator observability (kubectl get pods).
     spec:
-      hostPID: true
       priorityClassName: system-node-critical
       # Restrict to GPU nodes only. Without this the DaemonSet schedules onto
       # every node (including control-plane and CPU-only workers) and burns

From 5e8194c850a889d03ba60e900c7e91d3f25b8e53 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sun, 19 Apr 2026 11:09:50 +0300
Subject: [PATCH 060/151] docs(monitoring): explain cluster-layer filter
 asymmetry in GPU rules

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 .../alerts/gpu-recording.rules.yaml            | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
index 7589f894..681ac3e8 100644
--- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
+++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
@@ -33,6 +33,24 @@ spec:
         # up, or rare label drift between the two sources.
         - record: cluster:gpu_count:free
           expr: clamp_min(cluster:gpu_count:total - (cluster:gpu_count:allocated or vector(0)), 0)
+        # Cluster-layer filter asymmetry — intentional, not a bug:
+        #
+        #   * cluster:gpu_count:total and cluster:gpu_power_watts:sum do NOT
+        #     filter cozy-*/kube-* namespaces because they describe the
+        #     physical hardware fleet. A GPU draws power and occupies a
+        #     slot regardless of which namespace's pod happens to hold it;
+        #     excluding infra pods would under-report raw capacity and
+        #     break capacity-planning math.
+        #
+        #   * cluster:gpu_util:avg DOES filter infra namespaces because it
+        #     is a tenant-oriented KPI. Admins want to see the mean
+        #     utilization of tenant workloads, not have it diluted by
+        #     GPU Operator DaemonSet pods whose containers hold a GPU
+        #     handle but do no useful compute.
+        #
+        # Do not "normalize" by adding the filter to the hardware rules —
+        # downstream consumers (capacity planning, billing) rely on the
+        # raw totals.
         - record: cluster:gpu_util:avg
           expr: avg(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"})
         - record: cluster:gpu_power_watts:sum

From f5f083e84102d8c7e743766774cd54b226fc6ef1 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sun, 19 Apr 2026 11:09:51 +0300
Subject: [PATCH 061/151] feat(monitoring): annotate GPUThrottleFractionOverOne
 with verified hardware

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 .../alerts/gpu-recording.rules.yaml                  | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
index 681ac3e8..093a2939 100644
--- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
+++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
@@ -99,6 +99,16 @@ spec:
         - record: namespace:power_watts:sum
           expr: sum by (namespace) (DCGM_FI_DEV_POWER_USAGE{namespace!="", namespace!~"cozy-.*|kube-.*"})
 
+    # Known gap: there is no lower-bound sanity alert for under-reporting.
+    # If DCGM ever switches POWER/THERMAL_VIOLATION counter units the other
+    # direction — e.g. from the current nanoseconds back to microseconds,
+    # making rate()/1e9 produce values around 0.001 instead of real
+    # fractions — the throttle signal would silently collapse to near-zero
+    # instead of plateauing at 1.0. GPUThrottleFractionOverOne below catches
+    # the >1 drift; the <<1 drift would require correlation with independent
+    # signals (power draw vs TDP, clock dips) that we do not yet wire up.
+    # Documented here so future maintainers know this is a known gap, not
+    # an oversight.
     - name: gpu.recording.efficiency.1m
       interval: 1m
       params: {}
@@ -176,3 +186,5 @@ spec:
               version changed and the counter now grows in its documented µs units.
               Re-verify on the exporter Pod and adjust the divisor (/1e9 → /1e6) in
               gpu.recording.efficiency.1m before dashboards plateau at 100% throttle.
+            verified_on: "NVIDIA A10, DCGM 3.x — other GPU families (H100, L40S) may require a different divisor"
+            runbook: "If this alert fires, DCGM may have changed POWER_VIOLATION/THERMAL_VIOLATION counter units. Recalibrate by comparing raw counter rate against known throttle events; see gpu-recording.rules.yaml comments."

From 16b2fd008ba7bc4db6dae122202884791a22887c Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sun, 19 Apr 2026 11:10:43 +0300
Subject: [PATCH 062/151] docs(monitoring): comment bats regex rule-name
 convention

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 hack/check-gpu-recording-rules.bats | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/hack/check-gpu-recording-rules.bats b/hack/check-gpu-recording-rules.bats
index 4981959a..bae6785e 100644
--- a/hack/check-gpu-recording-rules.bats
+++ b/hack/check-gpu-recording-rules.bats
@@ -62,6 +62,10 @@ extract_rules() {
 # whole expression must contain at least two ':' characters.
 extract_refs() {
   json_file=$1
+  # Prometheus convention allows 2-segment rule names (level:metric); this
+  # regex is tuned to the 3+ segment convention used in this repo
+  # (level:metric:op — e.g. cluster:gpu_count:total). Update if future
+  # rules use 2 segments, otherwise they will be silently skipped.
   grep -hoE '[a-z][a-z0-9_]*:[a-z0-9_]+:[a-z0-9_]+' "$json_file" | sort -u
 }
 

From b5232bd15c55fb4a1faf25fa3f51e201a3002a60 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sun, 19 Apr 2026 11:10:46 +0300
Subject: [PATCH 063/151] feat(gpu-operator): enable NVLINK bandwidth in
 default DCGM CSV

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 .../system/gpu-operator/examples/dcgm-custom-metrics.yaml   | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml b/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml
index 92064701..8e57788b 100644
--- a/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml
+++ b/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml
@@ -70,8 +70,10 @@ data:
     DCGM_FI_DEV_LOW_UTIL_VIOLATION,    counter, Throttling duration due to low utilization (in us).
     DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (in us).
 
-    # NVLink — enable only for GPUs that actually have NVLink (A10 has none).
-    # DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL, counter, Total number of NVLink bandwidth counters for all lanes.
+    # NVLink — DCGM silently drops this metric on GPUs without NVLink, so
+    # enabling it unconditionally is safe and keeps this file reusable
+    # across GPU families.
+    DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL, counter, Total number of NVLink bandwidth counters for all lanes.
 
     # DCP (profiling) metrics — supported from Ampere onwards.
     DCGM_FI_PROF_GR_ENGINE_ACTIVE,   gauge, Ratio of time the graphics engine is active.

From 4d9a61a0ec49538c70d7b0e3c7d3cd37949781fa Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Sun, 19 Apr 2026 11:10:47 +0300
Subject: [PATCH 064/151] docs(gpu-operator): document native-talos
 service-monitor interval

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 packages/system/gpu-operator/examples/values-native-talos.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/packages/system/gpu-operator/examples/values-native-talos.yaml b/packages/system/gpu-operator/examples/values-native-talos.yaml
index 86e436c4..5a3d7786 100644
--- a/packages/system/gpu-operator/examples/values-native-talos.yaml
+++ b/packages/system/gpu-operator/examples/values-native-talos.yaml
@@ -39,6 +39,9 @@ spec:
           dcgmExporter:
             serviceMonitor:
               enabled: true
+              # Matches the 1m evaluation cadence of GPU recording rules with
+              # 4x oversampling headroom. DCGM_FI_PROF_* counters have
+              # documented overhead concerns below 1s; 15s is safe.
               interval: "15s"
               honorLabels: true
             config:

From e148343fd9bf9440d726bf01ebb07374119927d0 Mon Sep 17 00:00:00 2001
From: IvanHunters <xorokhotnikov@gmail.com>
Date: Sun, 19 Apr 2026 19:01:53 +0300
Subject: [PATCH 065/151] fix(kamaji): increase memory limits and add startup
 probe
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Increase memory limit from 500Mi to 512Mi
- Increase memory request from 100Mi to 256Mi
- Add startup probe with 60s timeout (12 attempts × 5s)
- Increase readiness/liveness initialDelaySeconds from 5/15 to 30s

This fixes OOMKilled crashes observed in production where kamaji
controller was being killed due to insufficient memory during startup.

Signed-off-by: IvanHunters <xorokhotnikov@gmail.com>
---
 packages/system/kamaji/values.yaml | 27 +++++++++++++++++++++++++--
 1 file changed, 25 insertions(+), 2 deletions(-)

diff --git a/packages/system/kamaji/values.yaml b/packages/system/kamaji/values.yaml
index b1ca3cb7..e6dbe99b 100644
--- a/packages/system/kamaji/values.yaml
+++ b/packages/system/kamaji/values.yaml
@@ -8,9 +8,32 @@ kamaji:
   resources:
     limits:
       cpu: 200m
-      memory: 500Mi
+      memory: 512Mi
     requests:
       cpu: 100m
-      memory: 100Mi
+      memory: 256Mi
+  startupProbe:
+    httpGet:
+      path: /healthz
+      port: healthcheck
+    initialDelaySeconds: 5
+    periodSeconds: 5
+    timeoutSeconds: 3
+    successThreshold: 1
+    failureThreshold: 12
+  livenessProbe:
+    httpGet:
+      path: /healthz
+      port: healthcheck
+    initialDelaySeconds: 30
+    periodSeconds: 20
+    timeoutSeconds: 1
+  readinessProbe:
+    httpGet:
+      path: /readyz
+      port: healthcheck
+    initialDelaySeconds: 30
+    periodSeconds: 10
+    timeoutSeconds: 1
   extraArgs:
     - --migrate-image=ghcr.io/cozystack/cozystack/kamaji:v1.3.0-rc.1@sha256:8c9af083b60600c0800eb56a2cda75f26007b7272a1cf019140de003bfce1a4d

From 531bc00524be0789ba99e2082234a1104e010b51 Mon Sep 17 00:00:00 2001
From: IvanHunters <xorokhotnikov@gmail.com>
Date: Wed, 8 Apr 2026 22:17:34 +0300
Subject: [PATCH 066/151] feat(postgres): add serverName parameter for backup
 recovery

Add serverName field to bootstrap configuration to explicitly specify
Barman server name from backup.info. This fixes "no target backup found"
errors when server_name in backup.info differs from Kubernetes cluster name.

Signed-off-by: IvanHunters <xorokhotnikov@gmail.com>
---
 api/apps/v1alpha1/postgresql/types.go             |  5 ++++-
 packages/apps/postgres/README.md                  | 13 +++++++------
 packages/apps/postgres/templates/db.yaml          |  3 +++
 packages/apps/postgres/values.schema.json         |  7 ++++++-
 packages/apps/postgres/values.yaml                |  4 +++-
 packages/system/postgres-rd/cozyrds/postgres.yaml |  4 ++--
 6 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/api/apps/v1alpha1/postgresql/types.go b/api/apps/v1alpha1/postgresql/types.go
index fa85d51f..56580018 100644
--- a/api/apps/v1alpha1/postgresql/types.go
+++ b/api/apps/v1alpha1/postgresql/types.go
@@ -86,12 +86,15 @@ type Bootstrap struct {
 	// Whether to restore from a backup.
 	// +kubebuilder:default:=false
 	Enabled bool `json:"enabled"`
-	// Previous cluster name before deletion.
+	// Previous cluster name before deletion (matches serverName in backup.info).
 	// +kubebuilder:default:=""
 	OldName string `json:"oldName"`
 	// Timestamp (RFC3339) for point-in-time recovery; empty means latest.
 	// +kubebuilder:default:=""
 	RecoveryTime string `json:"recoveryTime,omitempty"`
+	// Barman server name from the old cluster's backup.info. Use when the original cluster used a different serverName than its Kubernetes cluster name.
+	// +kubebuilder:default:=""
+	ServerName string `json:"serverName,omitempty"`
 }
 
 type Database struct {
diff --git a/packages/apps/postgres/README.md b/packages/apps/postgres/README.md
index 5a550f7a..648b3d71 100644
--- a/packages/apps/postgres/README.md
+++ b/packages/apps/postgres/README.md
@@ -133,12 +133,13 @@ See:
 
 ### Bootstrap (recovery) parameters
 
-| Name                     | Description                                                         | Type     | Value   |
-| ------------------------ | ------------------------------------------------------------------- | -------- | ------- |
-| `bootstrap`              | Bootstrap configuration.                                            | `object` | `{}`    |
-| `bootstrap.enabled`      | Whether to restore from a backup.                                   | `bool`   | `false` |
-| `bootstrap.recoveryTime` | Timestamp (RFC3339) for point-in-time recovery; empty means latest. | `string` | `""`    |
-| `bootstrap.oldName`      | Previous cluster name before deletion.                              | `string` | `""`    |
+| Name                     | Description                                                                                                                                        | Type     | Value   |
+| ------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------- |
+| `bootstrap`              | Bootstrap configuration.                                                                                                                           | `object` | `{}`    |
+| `bootstrap.enabled`      | Whether to restore from a backup.                                                                                                                  | `bool`   | `false` |
+| `bootstrap.recoveryTime` | Timestamp (RFC3339) for point-in-time recovery; empty means latest.                                                                                | `string` | `""`    |
+| `bootstrap.oldName`      | Previous cluster name before deletion (matches serverName in backup.info).                                                                         | `string` | `""`    |
+| `bootstrap.serverName`   | Barman server name from the old cluster's backup.info. Use when the original cluster used a different serverName than its Kubernetes cluster name. | `string` | `""`    |
 
 
 ## Parameter examples and reference
diff --git a/packages/apps/postgres/templates/db.yaml b/packages/apps/postgres/templates/db.yaml
index 5c40b2c7..2cdaec1d 100644
--- a/packages/apps/postgres/templates/db.yaml
+++ b/packages/apps/postgres/templates/db.yaml
@@ -32,6 +32,9 @@ spec:
     - name: {{ .Values.bootstrap.oldName }}
       barmanObjectStore:
         destinationPath: {{ .Values.backup.destinationPath }}
+        {{- if .Values.bootstrap.serverName }}
+        serverName: {{ .Values.bootstrap.serverName }}
+        {{- end }}
         endpointURL: {{ .Values.backup.endpointURL }}
         s3Credentials:
           accessKeyId:
diff --git a/packages/apps/postgres/values.schema.json b/packages/apps/postgres/values.schema.json
index b2a4aeba..28acc5a3 100644
--- a/packages/apps/postgres/values.schema.json
+++ b/packages/apps/postgres/values.schema.json
@@ -246,7 +246,7 @@
           "default": false
         },
         "oldName": {
-          "description": "Previous cluster name before deletion.",
+          "description": "Previous cluster name before deletion (matches serverName in backup.info).",
           "type": "string",
           "default": ""
         },
@@ -254,6 +254,11 @@
           "description": "Timestamp (RFC3339) for point-in-time recovery; empty means latest.",
           "type": "string",
           "default": ""
+        },
+        "serverName": {
+          "description": "Barman server name from the old cluster's backup.info. Use when the original cluster used a different serverName than its Kubernetes cluster name.",
+          "type": "string",
+          "default": ""
         }
       }
     }
diff --git a/packages/apps/postgres/values.yaml b/packages/apps/postgres/values.yaml
index b8f07f63..a4a4bf86 100644
--- a/packages/apps/postgres/values.yaml
+++ b/packages/apps/postgres/values.yaml
@@ -153,7 +153,8 @@ backup:
 ## @typedef {struct} Bootstrap - Bootstrap configuration for restoring a database cluster from a backup.
 ## @field {bool} enabled - Whether to restore from a backup.
 ## @field {string} [recoveryTime] - Timestamp (RFC3339) for point-in-time recovery; empty means latest.
-## @field {string} oldName - Previous cluster name before deletion.
+## @field {string} oldName - Previous cluster name before deletion (matches serverName in backup.info).
+## @field {string} [serverName] - Barman server name from the old cluster's backup.info. Use when the original cluster used a different serverName than its Kubernetes cluster name.
 
 ## @param {Bootstrap} bootstrap - Bootstrap configuration.
 bootstrap:
@@ -161,3 +162,4 @@ bootstrap:
   # example: 2020-11-26 15:22:00.00000+00
   recoveryTime: ""
   oldName: ""
+  serverName: ""
diff --git a/packages/system/postgres-rd/cozyrds/postgres.yaml b/packages/system/postgres-rd/cozyrds/postgres.yaml
index c74c5783..41457db4 100644
--- a/packages/system/postgres-rd/cozyrds/postgres.yaml
+++ b/packages/system/postgres-rd/cozyrds/postgres.yaml
@@ -8,7 +8,7 @@ spec:
     singular: postgres
     plural: postgreses
     openAPISchema: |-
-      {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Postgres replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each PostgreSQL replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"version":{"description":"PostgreSQL major version to deploy","type":"string","default":"v18","enum":["v18","v17","v16","v15","v14","v13"]},"postgresql":{"description":"PostgreSQL server configuration.","type":"object","default":{},"properties":{"parameters":{"description":"PostgreSQL server parameters.","type":"object","default":{},"properties":{"max_connections":{"description":"Maximum number of concurrent connections to the database server.","type":"integer","default":100}}}}},"quorum":{"description":"Quorum configuration for synchronous replication.","type":"object","default":{},"required":["maxSyncReplicas","minSyncReplicas"],"properties":{"maxSyncReplicas":{"description":"Maximum number of synchronous replicas allowed (must be less than total replicas).","type":"integer","default":0},"minSyncReplicas":{"description":"Minimum number of synchronous replicas required for commit.","type":"integer","default":0}}},"users":{"description":"Users configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"password":{"description":"Password for the user.","type":"string"},"replication":{"description":"Whether the user has replication privileges.","type":"boolean"}}}},"databases":{"description":"Databases configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"extensions":{"description":"List of enabled PostgreSQL extensions.","type":"array","items":{"type":"string"}},"roles":{"description":"Roles assigned to users.","type":"object","properties":{"admin":{"description":"List of users with admin privileges.","type":"array","items":{"type":"string"}},"readonly":{"description":"List of users with read-only privileges.","type":"array","items":{"type":"string"}}}}}}},"backup":{"description":"Backup configuration.","type":"object","default":{},"required":["enabled"],"properties":{"destinationPath":{"description":"Destination path for backups (e.g. s3://bucket/path/).","type":"string","default":"s3://bucket/path/to/folder/"},"enabled":{"description":"Enable regular backups.","type":"boolean","default":false},"endpointURL":{"description":"S3 endpoint URL for uploads.","type":"string","default":"http://minio-gateway-service:9000"},"retentionPolicy":{"description":"Retention policy (e.g. \"30d\").","type":"string","default":"30d"},"s3AccessKey":{"description":"Access key for S3 authentication.","type":"string","default":"<your-access-key>"},"s3SecretKey":{"description":"Secret key for S3 authentication.","type":"string","default":"<your-secret-key>"},"schedule":{"description":"Cron schedule for automated backups.","type":"string","default":"0 2 * * * *"}}},"bootstrap":{"description":"Bootstrap configuration.","type":"object","default":{},"required":["enabled","oldName"],"properties":{"enabled":{"description":"Whether to restore from a backup.","type":"boolean","default":false},"oldName":{"description":"Previous cluster name before deletion.","type":"string","default":""},"recoveryTime":{"description":"Timestamp (RFC3339) for point-in-time recovery; empty means latest.","type":"string","default":""}}}}}
+      {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Postgres replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each PostgreSQL replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"version":{"description":"PostgreSQL major version to deploy","type":"string","default":"v18","enum":["v18","v17","v16","v15","v14","v13"]},"postgresql":{"description":"PostgreSQL server configuration.","type":"object","default":{},"properties":{"parameters":{"description":"PostgreSQL server parameters.","type":"object","default":{},"properties":{"max_connections":{"description":"Maximum number of concurrent connections to the database server.","type":"integer","default":100}}}}},"quorum":{"description":"Quorum configuration for synchronous replication.","type":"object","default":{},"required":["maxSyncReplicas","minSyncReplicas"],"properties":{"maxSyncReplicas":{"description":"Maximum number of synchronous replicas allowed (must be less than total replicas).","type":"integer","default":0},"minSyncReplicas":{"description":"Minimum number of synchronous replicas required for commit.","type":"integer","default":0}}},"users":{"description":"Users configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"password":{"description":"Password for the user.","type":"string"},"replication":{"description":"Whether the user has replication privileges.","type":"boolean"}}}},"databases":{"description":"Databases configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"extensions":{"description":"List of enabled PostgreSQL extensions.","type":"array","items":{"type":"string"}},"roles":{"description":"Roles assigned to users.","type":"object","properties":{"admin":{"description":"List of users with admin privileges.","type":"array","items":{"type":"string"}},"readonly":{"description":"List of users with read-only privileges.","type":"array","items":{"type":"string"}}}}}}},"backup":{"description":"Backup configuration.","type":"object","default":{},"required":["enabled"],"properties":{"destinationPath":{"description":"Destination path for backups (e.g. s3://bucket/path/).","type":"string","default":"s3://bucket/path/to/folder/"},"enabled":{"description":"Enable regular backups.","type":"boolean","default":false},"endpointURL":{"description":"S3 endpoint URL for uploads.","type":"string","default":"http://minio-gateway-service:9000"},"retentionPolicy":{"description":"Retention policy (e.g. \"30d\").","type":"string","default":"30d"},"s3AccessKey":{"description":"Access key for S3 authentication.","type":"string","default":"<your-access-key>"},"s3SecretKey":{"description":"Secret key for S3 authentication.","type":"string","default":"<your-secret-key>"},"schedule":{"description":"Cron schedule for automated backups.","type":"string","default":"0 2 * * * *"}}},"bootstrap":{"description":"Bootstrap configuration.","type":"object","default":{},"required":["enabled","oldName"],"properties":{"enabled":{"description":"Whether to restore from a backup.","type":"boolean","default":false},"oldName":{"description":"Previous cluster name before deletion (matches serverName in backup.info).","type":"string","default":""},"recoveryTime":{"description":"Timestamp (RFC3339) for point-in-time recovery; empty means latest.","type":"string","default":""},"serverName":{"description":"Barman server name from the old cluster's backup.info. Use when the original cluster used a different serverName than its Kubernetes cluster name.","type":"string","default":""}}}}}
   release:
     prefix: postgres-
     labels:
@@ -33,7 +33,7 @@ spec:
     #    labelSelector:
     #      helm.toolkit.fluxcd.io/name: "{reqs[0]['metadata','name']}"
 
-    keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "replicas"], ["spec", "resources"], ["spec", "resourcesPreset"], ["spec", "size"], ["spec", "storageClass"], ["spec", "external"], ["spec", "version"], ["spec", "postgresql"], ["spec", "postgresql", "parameters"], ["spec", "postgresql", "parameters", "max_connections"], ["spec", "quorum"], ["spec", "quorum", "minSyncReplicas"], ["spec", "quorum", "maxSyncReplicas"], ["spec", "users"], ["spec", "databases"], ["spec", "backup"], ["spec", "backup", "enabled"], ["spec", "backup", "retentionPolicy"], ["spec", "backup", "destinationPath"], ["spec", "backup", "endpointURL"], ["spec", "backup", "schedule"], ["spec", "backup", "s3AccessKey"], ["spec", "backup", "s3SecretKey"], ["spec", "bootstrap"], ["spec", "bootstrap", "enabled"], ["spec", "bootstrap", "recoveryTime"], ["spec", "bootstrap", "oldName"]]
+    keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "replicas"], ["spec", "resources"], ["spec", "resourcesPreset"], ["spec", "size"], ["spec", "storageClass"], ["spec", "external"], ["spec", "version"], ["spec", "postgresql"], ["spec", "postgresql", "parameters"], ["spec", "postgresql", "parameters", "max_connections"], ["spec", "quorum"], ["spec", "quorum", "minSyncReplicas"], ["spec", "quorum", "maxSyncReplicas"], ["spec", "users"], ["spec", "databases"], ["spec", "backup"], ["spec", "backup", "enabled"], ["spec", "backup", "retentionPolicy"], ["spec", "backup", "destinationPath"], ["spec", "backup", "endpointURL"], ["spec", "backup", "schedule"], ["spec", "backup", "s3AccessKey"], ["spec", "backup", "s3SecretKey"], ["spec", "bootstrap"], ["spec", "bootstrap", "enabled"], ["spec", "bootstrap", "recoveryTime"], ["spec", "bootstrap", "oldName"], ["spec", "bootstrap", "serverName"]]
   secrets:
     exclude: []
     include:

From 9f41dc3228b18308f9cd6819a8b2470778f27f0c Mon Sep 17 00:00:00 2001
From: IvanHunters <xorokhotnikov@gmail.com>
Date: Tue, 21 Apr 2026 13:17:32 +0300
Subject: [PATCH 067/151] docs(postgres): clarify bootstrap field descriptions

Update oldName and serverName field descriptions based on code review
feedback to avoid confusion about their actual roles:

- oldName: Remove misleading "(matches serverName in backup.info)"
  text. This field represents the Kubernetes cluster resource name,
  not the Barman server name.

- serverName: Provide clearer explanation that it's the S3 path prefix
  (barmanObjectStore.serverName) used by the original cluster, and should
  only be set when it differs from the Kubernetes resource name.

Updated in:
- values.yaml (source of truth for field documentation)
- types.go (Go API type comments)
- values.schema.json (JSON schema for validation)
- postgres.yaml (CRD with embedded OpenAPI schema)

Signed-off-by: IvanHunters <xorokhotnikov@gmail.com>
---
 api/apps/v1alpha1/postgresql/types.go             |  4 ++--
 api/backups/v1alpha1/zz_generated.deepcopy.go     |  5 +++++
 packages/apps/postgres/README.md                  | 14 +++++++-------
 packages/apps/postgres/values.schema.json         |  4 ++--
 packages/apps/postgres/values.yaml                |  4 ++--
 packages/system/postgres-rd/cozyrds/postgres.yaml |  2 +-
 6 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/api/apps/v1alpha1/postgresql/types.go b/api/apps/v1alpha1/postgresql/types.go
index 56580018..a2ff77a8 100644
--- a/api/apps/v1alpha1/postgresql/types.go
+++ b/api/apps/v1alpha1/postgresql/types.go
@@ -86,13 +86,13 @@ type Bootstrap struct {
 	// Whether to restore from a backup.
 	// +kubebuilder:default:=false
 	Enabled bool `json:"enabled"`
-	// Previous cluster name before deletion (matches serverName in backup.info).
+	// Previous cluster name before deletion.
 	// +kubebuilder:default:=""
 	OldName string `json:"oldName"`
 	// Timestamp (RFC3339) for point-in-time recovery; empty means latest.
 	// +kubebuilder:default:=""
 	RecoveryTime string `json:"recoveryTime,omitempty"`
-	// Barman server name from the old cluster's backup.info. Use when the original cluster used a different serverName than its Kubernetes cluster name.
+	// Barman server name (S3 path prefix) used by the original cluster when writing backups. Set this only when the original cluster had an explicit barmanObjectStore.serverName that differed from its Kubernetes resource name.
 	// +kubebuilder:default:=""
 	ServerName string `json:"serverName,omitempty"`
 }
diff --git a/api/backups/v1alpha1/zz_generated.deepcopy.go b/api/backups/v1alpha1/zz_generated.deepcopy.go
index 89f6171f..61b8f839 100644
--- a/api/backups/v1alpha1/zz_generated.deepcopy.go
+++ b/api/backups/v1alpha1/zz_generated.deepcopy.go
@@ -620,6 +620,11 @@ func (in *RestoreJobSpec) DeepCopyInto(out *RestoreJobSpec) {
 		*out = new(v1.TypedLocalObjectReference)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.Options != nil {
+		in, out := &in.Options, &out.Options
+		*out = new(runtime.RawExtension)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestoreJobSpec.
diff --git a/packages/apps/postgres/README.md b/packages/apps/postgres/README.md
index 648b3d71..4cda7284 100644
--- a/packages/apps/postgres/README.md
+++ b/packages/apps/postgres/README.md
@@ -133,13 +133,13 @@ See:
 
 ### Bootstrap (recovery) parameters
 
-| Name                     | Description                                                                                                                                        | Type     | Value   |
-| ------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------- |
-| `bootstrap`              | Bootstrap configuration.                                                                                                                           | `object` | `{}`    |
-| `bootstrap.enabled`      | Whether to restore from a backup.                                                                                                                  | `bool`   | `false` |
-| `bootstrap.recoveryTime` | Timestamp (RFC3339) for point-in-time recovery; empty means latest.                                                                                | `string` | `""`    |
-| `bootstrap.oldName`      | Previous cluster name before deletion (matches serverName in backup.info).                                                                         | `string` | `""`    |
-| `bootstrap.serverName`   | Barman server name from the old cluster's backup.info. Use when the original cluster used a different serverName than its Kubernetes cluster name. | `string` | `""`    |
+| Name                     | Description                                                                                                                                                                                                                  | Type     | Value   |
+| ------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------- |
+| `bootstrap`              | Bootstrap configuration.                                                                                                                                                                                                     | `object` | `{}`    |
+| `bootstrap.enabled`      | Whether to restore from a backup.                                                                                                                                                                                            | `bool`   | `false` |
+| `bootstrap.recoveryTime` | Timestamp (RFC3339) for point-in-time recovery; empty means latest.                                                                                                                                                          | `string` | `""`    |
+| `bootstrap.oldName`      | Previous cluster name before deletion.                                                                                                                                                                                       | `string` | `""`    |
+| `bootstrap.serverName`   | Barman server name (S3 path prefix) used by the original cluster when writing backups. Set this only when the original cluster had an explicit barmanObjectStore.serverName that differed from its Kubernetes resource name. | `string` | `""`    |
 
 
 ## Parameter examples and reference
diff --git a/packages/apps/postgres/values.schema.json b/packages/apps/postgres/values.schema.json
index 28acc5a3..98e29822 100644
--- a/packages/apps/postgres/values.schema.json
+++ b/packages/apps/postgres/values.schema.json
@@ -246,7 +246,7 @@
           "default": false
         },
         "oldName": {
-          "description": "Previous cluster name before deletion (matches serverName in backup.info).",
+          "description": "Previous cluster name before deletion.",
           "type": "string",
           "default": ""
         },
@@ -256,7 +256,7 @@
           "default": ""
         },
         "serverName": {
-          "description": "Barman server name from the old cluster's backup.info. Use when the original cluster used a different serverName than its Kubernetes cluster name.",
+          "description": "Barman server name (S3 path prefix) used by the original cluster when writing backups. Set this only when the original cluster had an explicit barmanObjectStore.serverName that differed from its Kubernetes resource name.",
           "type": "string",
           "default": ""
         }
diff --git a/packages/apps/postgres/values.yaml b/packages/apps/postgres/values.yaml
index a4a4bf86..2ceaa9cf 100644
--- a/packages/apps/postgres/values.yaml
+++ b/packages/apps/postgres/values.yaml
@@ -153,8 +153,8 @@ backup:
 ## @typedef {struct} Bootstrap - Bootstrap configuration for restoring a database cluster from a backup.
 ## @field {bool} enabled - Whether to restore from a backup.
 ## @field {string} [recoveryTime] - Timestamp (RFC3339) for point-in-time recovery; empty means latest.
-## @field {string} oldName - Previous cluster name before deletion (matches serverName in backup.info).
-## @field {string} [serverName] - Barman server name from the old cluster's backup.info. Use when the original cluster used a different serverName than its Kubernetes cluster name.
+## @field {string} oldName - Previous cluster name before deletion.
+## @field {string} [serverName] - Barman server name (S3 path prefix) used by the original cluster when writing backups. Set this only when the original cluster had an explicit barmanObjectStore.serverName that differed from its Kubernetes resource name.
 
 ## @param {Bootstrap} bootstrap - Bootstrap configuration.
 bootstrap:
diff --git a/packages/system/postgres-rd/cozyrds/postgres.yaml b/packages/system/postgres-rd/cozyrds/postgres.yaml
index 41457db4..72d013f1 100644
--- a/packages/system/postgres-rd/cozyrds/postgres.yaml
+++ b/packages/system/postgres-rd/cozyrds/postgres.yaml
@@ -8,7 +8,7 @@ spec:
     singular: postgres
     plural: postgreses
     openAPISchema: |-
-      {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Postgres replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each PostgreSQL replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"version":{"description":"PostgreSQL major version to deploy","type":"string","default":"v18","enum":["v18","v17","v16","v15","v14","v13"]},"postgresql":{"description":"PostgreSQL server configuration.","type":"object","default":{},"properties":{"parameters":{"description":"PostgreSQL server parameters.","type":"object","default":{},"properties":{"max_connections":{"description":"Maximum number of concurrent connections to the database server.","type":"integer","default":100}}}}},"quorum":{"description":"Quorum configuration for synchronous replication.","type":"object","default":{},"required":["maxSyncReplicas","minSyncReplicas"],"properties":{"maxSyncReplicas":{"description":"Maximum number of synchronous replicas allowed (must be less than total replicas).","type":"integer","default":0},"minSyncReplicas":{"description":"Minimum number of synchronous replicas required for commit.","type":"integer","default":0}}},"users":{"description":"Users configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"password":{"description":"Password for the user.","type":"string"},"replication":{"description":"Whether the user has replication privileges.","type":"boolean"}}}},"databases":{"description":"Databases configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"extensions":{"description":"List of enabled PostgreSQL extensions.","type":"array","items":{"type":"string"}},"roles":{"description":"Roles assigned to users.","type":"object","properties":{"admin":{"description":"List of users with admin privileges.","type":"array","items":{"type":"string"}},"readonly":{"description":"List of users with read-only privileges.","type":"array","items":{"type":"string"}}}}}}},"backup":{"description":"Backup configuration.","type":"object","default":{},"required":["enabled"],"properties":{"destinationPath":{"description":"Destination path for backups (e.g. s3://bucket/path/).","type":"string","default":"s3://bucket/path/to/folder/"},"enabled":{"description":"Enable regular backups.","type":"boolean","default":false},"endpointURL":{"description":"S3 endpoint URL for uploads.","type":"string","default":"http://minio-gateway-service:9000"},"retentionPolicy":{"description":"Retention policy (e.g. \"30d\").","type":"string","default":"30d"},"s3AccessKey":{"description":"Access key for S3 authentication.","type":"string","default":"<your-access-key>"},"s3SecretKey":{"description":"Secret key for S3 authentication.","type":"string","default":"<your-secret-key>"},"schedule":{"description":"Cron schedule for automated backups.","type":"string","default":"0 2 * * * *"}}},"bootstrap":{"description":"Bootstrap configuration.","type":"object","default":{},"required":["enabled","oldName"],"properties":{"enabled":{"description":"Whether to restore from a backup.","type":"boolean","default":false},"oldName":{"description":"Previous cluster name before deletion (matches serverName in backup.info).","type":"string","default":""},"recoveryTime":{"description":"Timestamp (RFC3339) for point-in-time recovery; empty means latest.","type":"string","default":""},"serverName":{"description":"Barman server name from the old cluster's backup.info. Use when the original cluster used a different serverName than its Kubernetes cluster name.","type":"string","default":""}}}}}
+      {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Postgres replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each PostgreSQL replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"version":{"description":"PostgreSQL major version to deploy","type":"string","default":"v18","enum":["v18","v17","v16","v15","v14","v13"]},"postgresql":{"description":"PostgreSQL server configuration.","type":"object","default":{},"properties":{"parameters":{"description":"PostgreSQL server parameters.","type":"object","default":{},"properties":{"max_connections":{"description":"Maximum number of concurrent connections to the database server.","type":"integer","default":100}}}}},"quorum":{"description":"Quorum configuration for synchronous replication.","type":"object","default":{},"required":["maxSyncReplicas","minSyncReplicas"],"properties":{"maxSyncReplicas":{"description":"Maximum number of synchronous replicas allowed (must be less than total replicas).","type":"integer","default":0},"minSyncReplicas":{"description":"Minimum number of synchronous replicas required for commit.","type":"integer","default":0}}},"users":{"description":"Users configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"password":{"description":"Password for the user.","type":"string"},"replication":{"description":"Whether the user has replication privileges.","type":"boolean"}}}},"databases":{"description":"Databases configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"extensions":{"description":"List of enabled PostgreSQL extensions.","type":"array","items":{"type":"string"}},"roles":{"description":"Roles assigned to users.","type":"object","properties":{"admin":{"description":"List of users with admin privileges.","type":"array","items":{"type":"string"}},"readonly":{"description":"List of users with read-only privileges.","type":"array","items":{"type":"string"}}}}}}},"backup":{"description":"Backup configuration.","type":"object","default":{},"required":["enabled"],"properties":{"destinationPath":{"description":"Destination path for backups (e.g. s3://bucket/path/).","type":"string","default":"s3://bucket/path/to/folder/"},"enabled":{"description":"Enable regular backups.","type":"boolean","default":false},"endpointURL":{"description":"S3 endpoint URL for uploads.","type":"string","default":"http://minio-gateway-service:9000"},"retentionPolicy":{"description":"Retention policy (e.g. \"30d\").","type":"string","default":"30d"},"s3AccessKey":{"description":"Access key for S3 authentication.","type":"string","default":"<your-access-key>"},"s3SecretKey":{"description":"Secret key for S3 authentication.","type":"string","default":"<your-secret-key>"},"schedule":{"description":"Cron schedule for automated backups.","type":"string","default":"0 2 * * * *"}}},"bootstrap":{"description":"Bootstrap configuration.","type":"object","default":{},"required":["enabled","oldName"],"properties":{"enabled":{"description":"Whether to restore from a backup.","type":"boolean","default":false},"oldName":{"description":"Previous cluster name before deletion.","type":"string","default":""},"recoveryTime":{"description":"Timestamp (RFC3339) for point-in-time recovery; empty means latest.","type":"string","default":""},"serverName":{"description":"Barman server name (S3 path prefix) used by the original cluster when writing backups. Set this only when the original cluster had an explicit barmanObjectStore.serverName that differed from its Kubernetes resource name.","type":"string","default":""}}}}}
   release:
     prefix: postgres-
     labels:

From 0fdb25df724ba5d44c794271cc0661d97b1d9b13 Mon Sep 17 00:00:00 2001
From: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
Date: Wed, 22 Apr 2026 11:59:50 +0500
Subject: [PATCH 068/151] ci(api): add codegen drift check

Run root 'make generate' as a pre-commit hook and as a dedicated
CI workflow so missed codegen updates (CRDs, deepcopy, clients, RBAC)
are caught instead of merging stale generated files.

Pre-commit hook is scoped to files that actually affect codegen
(api/, pkg/apis/, hack/update-codegen.sh, hack/boilerplate.go.txt)
so unrelated commits are not slowed down. CI job sets up Go from
go.mod, runs make generate, and fails on drift with a pointer to
the local fix.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
---
 .github/workflows/codegen-drift.yml | 42 +++++++++++++++++++++++++++++
 .pre-commit-config.yaml             | 11 ++++++++
 2 files changed, 53 insertions(+)
 create mode 100644 .github/workflows/codegen-drift.yml

diff --git a/.github/workflows/codegen-drift.yml b/.github/workflows/codegen-drift.yml
new file mode 100644
index 00000000..bd7927f7
--- /dev/null
+++ b/.github/workflows/codegen-drift.yml
@@ -0,0 +1,42 @@
+name: Codegen Drift Check
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths:
+      - 'api/**'
+      - 'pkg/apis/**'
+      - 'hack/update-codegen.sh'
+      - 'hack/boilerplate.go.txt'
+      - 'go.mod'
+      - 'go.sum'
+      - '.github/workflows/codegen-drift.yml'
+
+concurrency:
+  group: codegen-drift-${{ github.workflow }}-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  codegen-drift:
+    name: Verify generated code is up to date
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+
+      - name: Run make generate
+        run: make generate
+
+      - name: Fail on drift
+        run: |
+          if ! git diff --exit-code; then
+            echo "::error::'make generate' produced changes. Run 'make generate' locally and commit the result."
+            git status --short
+            exit 1
+          fi
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index ac0f7e30..836f79c5 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,6 +1,17 @@
 repos:
 - repo: local
   hooks:
+    - id: run-make-generate-root
+      name: Run 'make generate' at repo root
+      entry: |
+        flock -x .git/pre-commit.lock sh -c '
+          echo "Running make generate at repo root"
+          make generate || exit $?
+          git diff --color=always | cat
+        '
+      language: system
+      files: ^(api/|pkg/apis/|hack/update-codegen\.sh$|hack/boilerplate\.go\.txt$)
+      pass_filenames: false
     - id: run-make-generate
       name: Run 'make generate' in all app directories
       entry: |

From 860f431187c3f577bfd2b7ab60201583bdef9384 Mon Sep 17 00:00:00 2001
From: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
Date: Wed, 22 Apr 2026 12:00:22 +0500
Subject: [PATCH 069/151] chore(api): regenerate deepcopy for
 RestoreJobSpec.Options

The Options field was added to RestoreJobSpec without re-running
'make generate', leaving zz_generated.deepcopy.go out of sync.
Regenerated to include the missing DeepCopyInto handling for the
runtime.RawExtension pointer.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
---
 api/backups/v1alpha1/zz_generated.deepcopy.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/api/backups/v1alpha1/zz_generated.deepcopy.go b/api/backups/v1alpha1/zz_generated.deepcopy.go
index 89f6171f..61b8f839 100644
--- a/api/backups/v1alpha1/zz_generated.deepcopy.go
+++ b/api/backups/v1alpha1/zz_generated.deepcopy.go
@@ -620,6 +620,11 @@ func (in *RestoreJobSpec) DeepCopyInto(out *RestoreJobSpec) {
 		*out = new(v1.TypedLocalObjectReference)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.Options != nil {
+		in, out := &in.Options, &out.Options
+		*out = new(runtime.RawExtension)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestoreJobSpec.

From 51c2ec0ad41722f34d6cb8a5d111ceac021c4464 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Wed, 22 Apr 2026 13:25:16 +0300
Subject: [PATCH 070/151] feat(hami): add HAMi GPU virtualization system chart

Add HAMi v2.8.1 (CNCF Sandbox) as a new system package for fractional
GPU sharing in tenant Kubernetes clusters. The chart enables workloads
to request specific amounts of GPU memory and compute cores instead of
claiming entire GPUs.

Vendored upstream chart includes device plugin, scheduler extender,
mutating webhook, and DRA subchart. Wrapper values configure nvidia
RuntimeClass and clear hardcoded node config from upstream defaults.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 packages/system/hami/Chart.yaml               |   3 +
 packages/system/hami/Makefile                 |  11 +
 packages/system/hami/charts/hami/Chart.lock   |   6 +
 packages/system/hami/charts/hami/Chart.yaml   |  22 +
 packages/system/hami/charts/hami/README.md    | 237 +++++++++
 .../charts/hami/charts/hami-dra/.helmignore   |  24 +
 .../charts/hami/charts/hami-dra/Chart.yaml    |  18 +
 .../charts/hami-dra/templates/_commons.tpl    |  84 ++++
 .../charts/hami-dra/templates/_helpers.tpl    |  73 +++
 .../hami-dra-driver/deviceclass.yaml          |  11 +
 .../nvidia-dra-driver-daemonset.yaml          | 145 ++++++
 .../templates/hami-dra-driver/rbac.yaml       | 110 ++++
 .../monitor/monitor-clusterrole.yaml          |  11 +
 .../monitor/monitor-clusterrolebinding.yaml   |  15 +
 .../templates/monitor/monitor-deployment.yaml |  62 +++
 .../templates/monitor/monitor-service.yaml    |  26 +
 .../monitor/monitor-serviceaccount.yaml       |   8 +
 .../templates/webhook/cert-manager.yaml       |  29 ++
 .../templates/webhook/cert-secret.yaml        |  13 +
 .../templates/webhook/device-config.yaml      | 394 +++++++++++++++
 .../webhook/mutatingwebhookconfiguration.yaml |  28 ++
 .../validatingwebhookconfiguration.yaml       |  29 ++
 .../webhook/webhook-clusterrole.yaml          |  14 +
 .../webhook/webhook-clusterrolebinding.yaml   |  12 +
 .../templates/webhook/webhook-deployment.yaml |  59 +++
 .../templates/webhook/webhook-service.yaml    |  15 +
 .../webhook/webhook-serviceaccount.yaml       |   5 +
 .../charts/hami/charts/hami-dra/values.yaml   | 166 ++++++
 .../hami/charts/hami/templates/NOTES.txt      |   3 +
 .../hami/charts/hami/templates/_commons.tpl   |  49 ++
 .../hami/charts/hami/templates/_helpers.tpl   | 163 ++++++
 .../templates/device-plugin/configmap.yaml    |  13 +
 .../device-plugin/daemonsetmock.yaml          |  55 ++
 .../device-plugin/daemonsetnvidia.yaml        | 262 ++++++++++
 .../templates/device-plugin/monitorrole.yaml  |  29 ++
 .../device-plugin/monitorrolebinding.yaml     |  17 +
 .../device-plugin/monitorservice.yaml         |  29 ++
 .../device-plugin/monitorserviceaccount.yaml  |  10 +
 .../device-plugin/runtime-class.yaml          |  11 +
 .../hami/templates/scheduler/certmanager.yaml |  31 ++
 .../hami/templates/scheduler/clusterrole.yaml |  36 ++
 .../scheduler/clusterrolebinding.yaml         |  49 ++
 .../hami/templates/scheduler/configmap.yaml   | 142 ++++++
 .../templates/scheduler/configmapnew.yaml     | 102 ++++
 .../hami/templates/scheduler/deployment.yaml  | 227 +++++++++
 .../templates/scheduler/device-configmap.yaml | 410 +++++++++++++++
 .../scheduler/job-patch/clusterrole.yaml      |  30 ++
 .../job-patch/clusterrolebinding.yaml         |  22 +
 .../scheduler/job-patch/job-createSecret.yaml |  70 +++
 .../scheduler/job-patch/job-patchWebhook.yaml |  65 +++
 .../templates/scheduler/job-patch/psp.yaml    |  40 ++
 .../templates/scheduler/job-patch/role.yaml   |  23 +
 .../scheduler/job-patch/rolebinding.yaml      |  23 +
 .../scheduler/job-patch/serviceaccount.yaml   |  15 +
 .../charts/hami/templates/scheduler/role.yaml |  14 +
 .../hami/templates/scheduler/rolebinding.yaml |  33 ++
 .../hami/templates/scheduler/service.yaml     |  36 ++
 .../templates/scheduler/serviceaccount.yaml   |  18 +
 .../hami/templates/scheduler/webhook.yaml     |  57 +++
 packages/system/hami/charts/hami/values.yaml  | 476 ++++++++++++++++++
 packages/system/hami/values.yaml              |   8 +
 61 files changed, 4198 insertions(+)
 create mode 100644 packages/system/hami/Chart.yaml
 create mode 100644 packages/system/hami/Makefile
 create mode 100644 packages/system/hami/charts/hami/Chart.lock
 create mode 100644 packages/system/hami/charts/hami/Chart.yaml
 create mode 100644 packages/system/hami/charts/hami/README.md
 create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/.helmignore
 create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml
 create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/_commons.tpl
 create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/_helpers.tpl
 create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/deviceclass.yaml
 create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/nvidia-dra-driver-daemonset.yaml
 create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/rbac.yaml
 create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrole.yaml
 create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrolebinding.yaml
 create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-deployment.yaml
 create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-service.yaml
 create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-serviceaccount.yaml
 create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-manager.yaml
 create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-secret.yaml
 create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/device-config.yaml
 create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/mutatingwebhookconfiguration.yaml
 create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/validatingwebhookconfiguration.yaml
 create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrole.yaml
 create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrolebinding.yaml
 create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-deployment.yaml
 create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-service.yaml
 create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-serviceaccount.yaml
 create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/values.yaml
 create mode 100644 packages/system/hami/charts/hami/templates/NOTES.txt
 create mode 100644 packages/system/hami/charts/hami/templates/_commons.tpl
 create mode 100644 packages/system/hami/charts/hami/templates/_helpers.tpl
 create mode 100644 packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml
 create mode 100644 packages/system/hami/charts/hami/templates/device-plugin/daemonsetmock.yaml
 create mode 100644 packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml
 create mode 100644 packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml
 create mode 100644 packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml
 create mode 100644 packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml
 create mode 100644 packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml
 create mode 100644 packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml
 create mode 100644 packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml
 create mode 100644 packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml
 create mode 100644 packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml
 create mode 100644 packages/system/hami/charts/hami/templates/scheduler/configmap.yaml
 create mode 100644 packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml
 create mode 100644 packages/system/hami/charts/hami/templates/scheduler/deployment.yaml
 create mode 100644 packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml
 create mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml
 create mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml
 create mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml
 create mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml
 create mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml
 create mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml
 create mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml
 create mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml
 create mode 100644 packages/system/hami/charts/hami/templates/scheduler/role.yaml
 create mode 100644 packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml
 create mode 100644 packages/system/hami/charts/hami/templates/scheduler/service.yaml
 create mode 100644 packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml
 create mode 100644 packages/system/hami/charts/hami/templates/scheduler/webhook.yaml
 create mode 100644 packages/system/hami/charts/hami/values.yaml
 create mode 100644 packages/system/hami/values.yaml

diff --git a/packages/system/hami/Chart.yaml b/packages/system/hami/Chart.yaml
new file mode 100644
index 00000000..3fcf4c5d
--- /dev/null
+++ b/packages/system/hami/Chart.yaml
@@ -0,0 +1,3 @@
+apiVersion: v2
+name: cozy-hami
+version: 0.0.0 # Placeholder, the actual version will be automatically set during the build process
diff --git a/packages/system/hami/Makefile b/packages/system/hami/Makefile
new file mode 100644
index 00000000..b7a169c2
--- /dev/null
+++ b/packages/system/hami/Makefile
@@ -0,0 +1,11 @@
+export NAME=hami
+export NAMESPACE=cozy-$(NAME)
+
+include ../../../hack/common-envs.mk
+include ../../../hack/package.mk
+
+update:
+	rm -rf charts
+	helm repo add hami-charts https://project-hami.github.io/HAMi/
+	helm repo update hami-charts
+	helm pull hami-charts/hami --untar --untardir charts
diff --git a/packages/system/hami/charts/hami/Chart.lock b/packages/system/hami/charts/hami/Chart.lock
new file mode 100644
index 00000000..25856c07
--- /dev/null
+++ b/packages/system/hami/charts/hami/Chart.lock
@@ -0,0 +1,6 @@
+dependencies:
+- name: hami-dra
+  repository: https://project-hami.github.io/HAMi-DRA/
+  version: 0.1.0
+digest: sha256:374551539570bcee82c1c4dea93eac59e6f410b6d5967d188efd630e203d43bb
+generated: "2026-04-17T04:06:47.689117678Z"
diff --git a/packages/system/hami/charts/hami/Chart.yaml b/packages/system/hami/charts/hami/Chart.yaml
new file mode 100644
index 00000000..aba7e95c
--- /dev/null
+++ b/packages/system/hami/charts/hami/Chart.yaml
@@ -0,0 +1,22 @@
+apiVersion: v2
+appVersion: 2.8.1
+dependencies:
+- condition: dra.enabled
+  name: hami-dra
+  repository: https://project-hami.github.io/HAMi-DRA/
+  version: 0.1.0
+description: Heterogeneous AI Computing Virtualization Middleware
+keywords:
+- vgpu
+- gpu
+kubeVersion: '>= 1.18.0-0'
+maintainers:
+- email: archlitchi@gmail.com
+  name: limengxuan
+- email: xiaozhang0210@hotmail.com
+  name: zhangxiao
+name: hami
+sources:
+- https://github.com/Project-HAMi/HAMi
+type: application
+version: 2.8.1
diff --git a/packages/system/hami/charts/hami/README.md b/packages/system/hami/charts/hami/README.md
new file mode 100644
index 00000000..e8217290
--- /dev/null
+++ b/packages/system/hami/charts/hami/README.md
@@ -0,0 +1,237 @@
+# HAMi Helm Chart Values Documentation
+
+This document provides detailed descriptions of all configurable values parameters for the HAMi Helm Chart.
+
+## Global Configuration
+
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `global.imageRegistry` | Global Docker image registry | `""` |
+| `global.imagePullSecrets` | Global Docker image pull secrets | `[]` |
+| `global.imageTag` | Image tag | `"v2.8.1"` |
+| `global.gpuHookPath` | GPU Hook path | `/usr/local` |
+| `global.labels` | Global labels | `{}` |
+| `global.annotations` | Global annotations | `{}` |
+| `global.managedNodeSelectorEnable` | Whether to enable managed node selector | `false` |
+| `global.managedNodeSelector.usage` | Managed node selector usage | `"gpu"` |
+| `nameOverride` | Name override | `""` |
+| `fullnameOverride` | Full name override | `""` |
+| `namespaceOverride` | Namespace override | `""` |
+
+## Resource Name Configuration
+
+### NVIDIA GPU Resources
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `resourceName` | GPU resource name | `"nvidia.com/gpu"` |
+| `resourceMem` | GPU memory resource name | `"nvidia.com/gpumem"` |
+| `resourceMemPercentage` | GPU memory percentage resource name | `"nvidia.com/gpumem-percentage"` |
+| `resourceCores` | GPU core resource name | `"nvidia.com/gpucores"` |
+| `resourcePriority` | GPU priority resource name | `"nvidia.com/priority"` |
+
+### Cambricon MLU Resources
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `mluResourceName` | MLU resource name | `"cambricon.com/vmlu"` |
+| `mluResourceMem` | MLU memory resource name | `"cambricon.com/mlu.smlu.vmemory"` |
+| `mluResourceCores` | MLU core resource name | `"cambricon.com/mlu.smlu.vcore"` |
+
+### Hygon DCU Resources
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `dcuResourceName` | DCU resource name | `"hygon.com/dcunum"` |
+| `dcuResourceMem` | DCU memory resource name | `"hygon.com/dcumem"` |
+| `dcuResourceCores` | DCU core resource name | `"hygon.com/dcucores"` |
+
+### Metax GPU Resources
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `metaxResourceName` | GPU resource name | `"metax-tech.com/sgpu"` |
+| `metaxResourceCore` | GPU core resource name | `"metax-tech.com/vcore"` |
+| `metaxResourceMem` | GPU memory resource name | `"metax-tech.com/vmemory"` |
+| `metaxsGPUTopologyAware` | GPU topology awareness | `"false"` |
+
+### Enflame GCU Resources
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `enflameResourceNameVGCU` | vGCU resource name | `"enflame.com/vgcu"` |
+| `enflameResourceNameVGCUPercentage` | vGCU percentage resource name | `"enflame.com/vgcu-percentage"` |
+
+### Kunlunxin XPU Resources
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `kunlunResourceName` | XPU resource name | `"kunlunxin.com/xpu"` |
+
+## Scheduler Configuration
+
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `schedulerName` | Scheduler name | `"hami-scheduler"` |
+| `scheduler.nodeName` | Define node name, scheduler will schedule to this node | `""` |
+| `scheduler.overwriteEnv` | Whether to overwrite environment variables | `"false"` |
+| `scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy` | Node scheduler policy | `binpack` |
+| `scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy` | GPU scheduler policy | `spread` |
+| `scheduler.metricsBindAddress` | Metrics bind address | `":9395"` |
+| `scheduler.forceOverwriteDefaultScheduler` | Whether to force overwrite default scheduler | `true` |
+| `scheduler.livenessProbe` | Whether to enable liveness probe | `false` |
+| `scheduler.leaderElect` | Whether to enable leader election | `true` |
+| `scheduler.replicas` | Number of replicas | `1` |
+
+### Kube Scheduler Configuration
+
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `scheduler.kubeScheduler.enabled` | Whether to run kube-scheduler container in scheduler pod | `true` |
+| `scheduler.kubeScheduler.image.registry` | Kube scheduler image registry | `"registry.cn-hangzhou.aliyuncs.com"` |
+| `scheduler.kubeScheduler.image.repository` | Kube scheduler image repository | `"google_containers/kube-scheduler"` |
+| `scheduler.kubeScheduler.image.tag` | Kube scheduler image tag | `""` |
+| `scheduler.kubeScheduler.image.pullPolicy` | Kube scheduler image pull policy | `IfNotPresent` |
+| `scheduler.kubeScheduler.image.pullSecrets` | Kube scheduler image pull secrets | `[]` |
+| `scheduler.kubeScheduler.extraNewArgs` | Extra new arguments | `["--config=/config/config.yaml", "-v=4"]` |
+| `scheduler.kubeScheduler.extraArgs` | Extra arguments | `["--policy-config-file=/config/config.json", "-v=4"]` |
+
+### Extender Configuration
+
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `scheduler.extender.image.registry` | Scheduler extender image registry | `"docker.io"` |
+| `scheduler.extender.image.repository` | Scheduler extender image repository | `"projecthami/hami"` |
+| `scheduler.extender.image.tag` | Scheduler extender image tag | `""` |
+| `scheduler.extender.image.pullPolicy` | Scheduler extender image pull policy | `IfNotPresent` |
+| `scheduler.extender.image.pullSecrets` | Scheduler extender image pull secrets | `[]` |
+| `scheduler.extender.extraArgs` | Scheduler extender extra arguments | `["--debug", "-v=4"]` |
+
+### Admission Webhook Configuration
+
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `scheduler.admissionWebhook.enabled` | Whether to enable admission webhook | `true` |
+| `scheduler.admissionWebhook.customURL.enabled` | Whether to enable custom URL | `false` |
+| `scheduler.admissionWebhook.customURL.host` | Custom URL host | `127.0.0.1` |
+| `scheduler.admissionWebhook.customURL.port` | Custom URL port | `31998` |
+| `scheduler.admissionWebhook.customURL.path` | Custom URL path | `/webhook` |
+| `scheduler.admissionWebhook.reinvocationPolicy` | Reinvocation policy | `Never` |
+| `scheduler.admissionWebhook.failurePolicy` | Failure policy | `Ignore` |
+
+### TLS Certificate Configuration
+
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `scheduler.certManager.enabled` | Whether to use cert-manager to generate self-signed certificates | `false` |
+| `scheduler.patch.enabled` | Whether to use kube-webhook-certgen to generate self-signed certificates | `true` |
+| `scheduler.patch.image.registry` | Certgen image registry | `"docker.io"` |
+| `scheduler.patch.image.repository` | Certgen image repository | `"jettech/kube-webhook-certgen"` |
+| `scheduler.patch.image.tag` | Certgen image tag | `"v1.5.2"` |
+| `scheduler.patch.image.pullPolicy` | Certgen image pull policy | `IfNotPresent` |
+| `scheduler.patch.imageNew.registry` | New certgen image registry | `"docker.io"` |
+| `scheduler.patch.imageNew.repository` | New certgen image repository | `"liangjw/kube-webhook-certgen"` |
+| `scheduler.patch.imageNew.tag` | New certgen image tag | `"v1.1.1"` |
+
+### Scheduler Service Configuration
+
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `scheduler.service.type` | Service type | `NodePort` |
+| `scheduler.service.httpPort` | HTTP port | `443` |
+| `scheduler.service.schedulerPort` | Scheduler NodePort | `31998` |
+| `scheduler.service.monitorPort` | Monitor port | `31993` |
+| `scheduler.service.monitorTargetPort` | Monitor target port | `9395` |
+
+## Device Plugin Configuration
+
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `devicePlugin.image.registry` | Device plugin image registry | `"docker.io"` |
+| `devicePlugin.image.repository` | Device plugin image repository | `"projecthami/hami"` |
+| `devicePlugin.image.tag` | Device plugin image tag | `""` |
+| `devicePlugin.image.pullPolicy` | Device plugin image pull policy | `IfNotPresent` |
+| `devicePlugin.image.pullSecrets` | Device plugin image pull secrets | `[]` |
+
+### Monitor Configuration
+
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `devicePlugin.monitor.image.registry` | Monitor image registry | `"docker.io"` |
+| `devicePlugin.monitor.image.repository` | Monitor image repository | `"projecthami/hami"` |
+| `devicePlugin.monitor.image.tag` | Monitor image tag | `""` |
+| `devicePlugin.monitor.image.pullPolicy` | Monitor image pull policy | `IfNotPresent` |
+| `devicePlugin.monitor.image.pullSecrets` | Monitor image pull secrets | `[]` |
+| `devicePlugin.monitor.ctrPath` | Container path | `/usr/local/vgpu/containers` |
+| `devicePlugin.monitor.extraArgs` | Monitor extra arguments | `["-v=4"]` |
+| `devicePlugin.monitor.extraEnvs` | Monitor extra environments | `{}` |
+
+### Device Plugin Other Configuration
+
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `devicePlugin.deviceSplitCount` | Integer type, default value: 10. Maximum number of tasks assigned to a single GPU device | `10` |
+| `devicePlugin.deviceMemoryScaling` | Device memory scaling ratio | `1` |
+| `devicePlugin.deviceCoreScaling` | Device core scaling ratio | `1` |
+| `devicePlugin.runtimeClassName` | Runtime class name | `""` |
+| `devicePlugin.createRuntimeClass` | Whether to create runtime class | `false` |
+| `devicePlugin.migStrategy` | String type, "none" means ignore MIG functionality, "mixed" means allocate MIG devices through independent resources | `"none"` |
+| `devicePlugin.disablecorelimit` | String type, "true" means disable core limit, "false" means enable core limit | `"false"` |
+| `devicePlugin.passDeviceSpecsEnabled` | Whether to enable passing device specs | `false` |
+| `devicePlugin.extraArgs` | Device plugin extra arguments | `["-v=4"]` |
+| `devicePlugin.nodeConfiguration.config` | Node configuration for device plugin by json | An example of default configuration. |
+| `devicePlugin.nodeConfiguration.externalConfigName` | Node configuration for device plugin by external congimap | `""` |
+| `devicePlugin.extraEnvs` | Device plugin extra environments | `{}` |
+
+### Device Plugin Service Configuration
+
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `devicePlugin.service.type` | Service type | `NodePort` |
+| `devicePlugin.service.httpPort` | HTTP port | `31992` |
+
+### Device Plugin Deployment Configuration
+
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `devicePlugin.pluginPath` | Plugin path | `/var/lib/kubelet/device-plugins` |
+| `devicePlugin.libPath` | Library path | `/usr/local/vgpu` |
+| `devicePlugin.nvidiaNodeSelector` | NVIDIA node selector | `{"gpu": "on"}` |
+| `devicePlugin.updateStrategy.type` | Update strategy type | `RollingUpdate` |
+| `devicePlugin.updateStrategy.rollingUpdate.maxUnavailable` | Maximum unavailable count | `1` |
+
+## Device Configuration
+
+### AWS Neuron
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `devices.awsneuron.customresources` | Custom resources | `["aws.amazon.com/neuron", "aws.amazon.com/neuroncore"]` |
+
+### Kunlunxin
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `devices.kunlun.enabled` | Whether to enable | `true` |
+| `devices.kunlun.customresources` | Custom resources | `["kunlunxin.com/xpu"]` |
+
+### Mthreads
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `devices.mthreads.enabled` | Whether to enable | `true` |
+| `devices.mthreads.customresources` | Custom resources | `["mthreads.com/vgpu"]` |
+
+### NVIDIA
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `devices.nvidia.gpuCorePolicy` | GPU core policy | `default` |
+| `devices.nvidia.libCudaLogLevel` | CUDA library log level | `1` |
+
+### Huawei Ascend
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `devices.ascend.enabled` | Whether to enable | `false` |
+| `devices.ascend.image` | Image | `""` |
+| `devices.ascend.imagePullPolicy` | Image pull policy | `IfNotPresent` |
+| `devices.ascend.extraArgs` | Extra arguments | `[]` |
+| `devices.ascend.nodeSelector` | Node selector | `{"ascend": "on"}` |
+| `devices.ascend.tolerations` | Tolerations | `[]` |
+| `devices.ascend.customresources` | Custom resources | `["huawei.com/Ascend910A", "huawei.com/Ascend910A-memory", ...]` |
+
+### Iluvatar
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `devices.iluvatar.enabled` | Whether to enable | `false` |
+| `devices.iluvatar.customresources` | Custom resources | `["iluvatar.ai/BI-V150-vgpu", "iluvatar.ai/BI-V150.vMem","iluvatar.ai/BI-V150.vCore", ...]` |
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/.helmignore b/packages/system/hami/charts/hami/charts/hami-dra/.helmignore
new file mode 100644
index 00000000..898df488
--- /dev/null
+++ b/packages/system/hami/charts/hami/charts/hami-dra/.helmignore
@@ -0,0 +1,24 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
+
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml b/packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml
new file mode 100644
index 00000000..d01f678f
--- /dev/null
+++ b/packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml
@@ -0,0 +1,18 @@
+apiVersion: v2
+appVersion: 0.1.0
+description: A Helm chart for HAMi DRA
+home: https://github.com/Project-HAMi/HAMi-DRA
+keywords:
+- webhook
+- kubernetes
+- admission-controller
+- hami
+- dra
+maintainers:
+- name: hami-dra
+  url: https://github.com/Project-HAMi/HAMi-DRA
+name: hami-dra
+sources:
+- https://github.com/Project-HAMi/HAMi-DRA
+type: application
+version: 0.1.0
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/_commons.tpl b/packages/system/hami/charts/hami/charts/hami-dra/templates/_commons.tpl
new file mode 100644
index 00000000..02fbed20
--- /dev/null
+++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/_commons.tpl
@@ -0,0 +1,84 @@
+{{/* vim: set filetype=mustache: */}}
+
+{{/*
+Return the proper image name
+{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" $) }}
+*/}}
+{{- define "common.images.image" -}}
+{{- $registryName := .imageRoot.registry -}}
+{{- $repositoryName := .imageRoot.repository -}}
+{{- $tag := .imageRoot.tag | toString -}}
+{{- if .global }}
+    {{- if .global.imageRegistry }}
+     {{- $registryName = .global.imageRegistry -}}
+    {{- end -}}
+{{- end -}}
+{{- if .tag }}
+    {{- if .tag.imageTag }}
+     {{- $tag = .tag.imageTag -}}
+    {{- end -}}
+{{- end -}}
+{{- if $registryName }}
+{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}}
+{{- else -}}
+{{- printf "%s:%s" $repositoryName $tag -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead)
+{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }}
+*/}}
+{{- define "common.images.pullSecrets" -}}
+  {{- $pullSecrets := list }}
+
+  {{- if .global }}
+    {{- range .global.imagePullSecrets -}}
+      {{- $pullSecrets = append $pullSecrets . -}}
+    {{- end -}}
+  {{- end -}}
+
+  {{- range .images -}}
+    {{- range .pullSecrets -}}
+      {{- $pullSecrets = append $pullSecrets . -}}
+    {{- end -}}
+  {{- end -}}
+
+  {{- if (not (empty $pullSecrets)) }}
+imagePullSecrets:
+    {{- range $pullSecrets }}
+  - name: {{ . }}
+    {{- end }}
+  {{- end }}
+{{- end -}}
+
+{{/*
+Renders a value that contains template.
+Usage:
+{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $) }}
+*/}}
+{{- define "common.tplvalues.render" -}}
+    {{- if typeIs "string" .value }}
+        {{- tpl .value .context }}
+    {{- else }}
+        {{- tpl (.value | toYaml) .context }}
+    {{- end }}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "common.names.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
\ No newline at end of file
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/_helpers.tpl b/packages/system/hami/charts/hami/charts/hami-dra/templates/_helpers.tpl
new file mode 100644
index 00000000..99beec77
--- /dev/null
+++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/_helpers.tpl
@@ -0,0 +1,73 @@
+{{- define "hami.dra.webhook.fullname" -}}
+{{- printf "%s-%s" (include "common.names.fullname" .) "webhook" | trunc 63 | trimSuffix "-" -}}
+{{- end }}
+
+{{- define "hami.dra.webhook.image" -}}
+{{ include "common.images.image" (dict "imageRoot" .Values.webhook.image "global" .Values.global) }}
+{{- end -}}
+
+{{- define "hami.dra.webhook.imagePullSecrets" -}}
+{{ include "common.images.pullSecrets" (dict "images" (list .Values.webhook.image) "global" .Values.global) }}
+{{- end -}}
+
+{{- define "hami.dra.driver.nvidia.image" -}}
+{{ include "common.images.image" (dict "imageRoot" .Values.drivers.nvidia.image "global" .Values.global) }}
+{{- end -}}
+
+{{- define "hami.dra.driver.nvidia.imagePullSecrets" -}}
+{{ include "common.images.pullSecrets" (dict "images" (list .Values.drivers.nvidia.image) "global" .Values.global) }}
+{{- end -}}
+
+{{/*
+Common labels
+*/}}
+{{- define "hami-dra-webhook.labels" -}}
+helm.sh/chart: {{ .Chart.Name }}
+{{ include "hami-dra-webhook.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "hami-dra-webhook.selectorLabels" -}}
+app.kubernetes.io/name: {{ .Release.Name }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/component: webhook
+{{- end }}
+
+{{- define "hami.dra.monitor.fullname" -}}
+{{- printf "%s-%s" (include "common.names.fullname" .) "monitor" | trunc 63 | trimSuffix "-" -}}
+{{- end }}
+
+{{- define "hami.dra.monitor.image" -}}
+{{ include "common.images.image" (dict "imageRoot" .Values.monitor.image "global" .Values.global) }}
+{{- end -}}
+
+{{- define "hami.dra.monitor.imagePullSecrets" -}}
+{{ include "common.images.pullSecrets" (dict "images" (list .Values.monitor.image) "global" .Values.global) }}
+{{- end -}}
+
+{{/*
+Common labels for monitor
+*/}}
+{{- define "hami-dra-monitor.labels" -}}
+helm.sh/chart: {{ .Chart.Name }}
+{{ include "hami-dra-monitor.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels for monitor
+*/}}
+{{- define "hami-dra-monitor.selectorLabels" -}}
+app.kubernetes.io/name: {{ .Release.Name }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/component: monitor
+{{- end }}
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/deviceclass.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/deviceclass.yaml
new file mode 100644
index 00000000..bcb6d2c5
--- /dev/null
+++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/deviceclass.yaml
@@ -0,0 +1,11 @@
+{{- if .Values.drivers.nvidia.enabled }}
+apiVersion: resource.k8s.io/v1
+kind: DeviceClass
+metadata:
+  name: hami-core-gpu.project-hami.io
+spec:
+  selectors:
+  - cel:
+      expression: |-
+        device.driver == "hami-core-gpu.project-hami.io" && device.attributes["hami-core-gpu.project-hami.io"].type == "hami-gpu"
+{{- end }}
\ No newline at end of file
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/nvidia-dra-driver-daemonset.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/nvidia-dra-driver-daemonset.yaml
new file mode 100644
index 00000000..a3fb0d01
--- /dev/null
+++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/nvidia-dra-driver-daemonset.yaml
@@ -0,0 +1,145 @@
+{{- if .Values.drivers.nvidia.enabled }}
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: hami-dra-driver-kubelet-plugin
+  namespace: {{ .Release.Namespace }}
+spec:
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      hami-dra-driver-component: kubelet-plugin
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/instance: hami-dra-driver
+        app.kubernetes.io/name: hami-dra-driver
+        hami-dra-driver-component: kubelet-plugin
+    spec:
+      {{- include "hami.dra.driver.nvidia.imagePullSecrets" . | nindent 6 }}
+      initContainers:
+      - name: init-container
+        image: {{ include "hami.dra.driver.nvidia.image" . }}
+        securityContext:
+          privileged: true
+        command: [bash, /usr/bin/kubelet-plugin-prestart.sh]
+        env:
+        - name: NVIDIA_DRIVER_ROOT
+          value: {{ if .Values.drivers.nvidia.containerDriver }}/run/nvidia/driver{{ else }}/{{ end }}
+        - name: NVIDIA_VISIBLE_DEVICES
+          value: void
+        - name: KUBELET_REGISTRAR_DIRECTORY_PATH
+          value: /var/lib/kubelet/plugins_registry
+        - name: KUBELET_PLUGINS_DIRECTORY_PATH
+          value: /var/lib/kubelet/plugins
+        volumeMounts:
+        - name: driver-root-parent
+          mountPath: /driver-root-parent
+          readOnly: true
+      containers:
+      - args:
+        - |-
+          # Conditionally mask the params file to prevent this container from
+          # recreating any missing GPU device nodes. This is necessary, for
+          # example, when running under nvkind to limit the set GPUs governed
+          # by the plugin even though it has cgroup access to all of them.
+          if [ "${MASK_NVIDIA_DRIVER_PARAMS}" = "true" ]; then
+            cp /proc/driver/nvidia/params root/gpu-params
+            sed -i 's/^ModifyDeviceFiles: 1$/ModifyDeviceFiles: 0/' root/gpu-params
+            mount --bind root/gpu-params /proc/driver/nvidia/params
+          fi
+          hami-kubelet-plugin -v 6
+        command:
+        - bash
+        - -c
+        env:
+        - name: MASK_NVIDIA_DRIVER_PARAMS
+          value: ""
+        - name: NVIDIA_DRIVER_ROOT
+          value: {{ if .Values.drivers.nvidia.containerDriver }}/run/nvidia/driver{{ else }}/{{ end }}
+        - name: NVIDIA_VISIBLE_DEVICES
+          value: void
+        - name: CDI_ROOT
+          value: /var/run/cdi
+        - name: NVIDIA_MIG_CONFIG_DEVICES
+          value: all
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.namespace
+        - name: KUBELET_REGISTRAR_DIRECTORY_PATH
+          value: /var/lib/kubelet/plugins_registry
+        - name: KUBELET_PLUGINS_DIRECTORY_PATH
+          value: /var/lib/kubelet/plugins
+        - name: IMAGE_NAME
+          value: {{ include "hami.dra.driver.nvidia.image" . }}
+        lifecycle:
+          postStart:
+            exec:
+              command: ["/bin/bash", "-c", "/usr/bin/vgpu-init.sh /usr/local/vgpu/"]
+        image: {{ include "hami.dra.driver.nvidia.image" . }}
+        imagePullPolicy: {{ .Values.drivers.nvidia.image.pullPolicy | default "IfNotPresent" }}
+        name: gpus
+        securityContext:
+          privileged: true
+        terminationMessagePath: /dev/termination-log
+        terminationMessagePolicy: File
+        volumeMounts:
+        - mountPath: /var/lib/kubelet/plugins_registry
+          name: plugins-registry
+        - mountPath: /var/lib/kubelet/plugins
+          mountPropagation: Bidirectional
+          name: plugins
+        - mountPath: /var/run/cdi
+          name: cdi
+        - mountPath: /driver-root
+          mountPropagation: HostToContainer
+          name: driver-root
+          readOnly: true
+        - mountPath: /usr/local/vgpu
+          name: host-vgpu
+        - mountPath: /tmp
+          name: host-tmp
+      dnsPolicy: ClusterFirst
+      serviceAccount: hami-dra-driver-service-account
+      serviceAccountName: hami-dra-driver-service-account
+      volumes:
+      - hostPath:
+          path: /var/lib/kubelet/plugins_registry
+          type: ""
+        name: plugins-registry
+      - hostPath:
+          path: /var/lib/kubelet/plugins
+          type: ""
+        name: plugins
+      - hostPath:
+          path: /var/run/cdi
+          type: ""
+        name: cdi
+      - hostPath:
+          path: {{if .Values.drivers.nvidia.containerDriver }}/run/nvidia{{ else }}/{{ end }}
+          type: DirectoryOrCreate
+        name: driver-root-parent
+      - hostPath:
+          path: {{if .Values.drivers.nvidia.containerDriver }}/run/nvidia/driver{{ else }}/{{ end }}
+          type: DirectoryOrCreate
+        name: driver-root
+      - hostPath:
+          path: /dev
+          type: ""
+        name: host-dev
+      - hostPath:
+          path: /usr/local/vgpu
+          type: DirectoryOrCreate
+        name: host-vgpu
+      - hostPath:
+          path: /tmp
+          type: ""
+        name: host-tmp
+{{- end }}
\ No newline at end of file
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/rbac.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/rbac.yaml
new file mode 100644
index 00000000..435c37cc
--- /dev/null
+++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/rbac.yaml
@@ -0,0 +1,110 @@
+{{- if .Values.drivers.nvidia.enabled }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: hami-dra-driver-service-account
+  namespace: {{ .Release.Namespace }}
+---
+apiVersion: v1
+items:
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: Role
+  metadata:
+    name: hami-dra-driver-role
+    namespace: {{ .Release.Namespace }}
+  rules:
+  - apiGroups:
+    - apps
+    resources:
+    - daemonsets
+    - deployments
+    verbs:
+    - get
+    - list
+    - watch
+    - create
+    - update
+    - patch
+    - delete
+kind: List
+metadata:
+  resourceVersion: ""
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: hami-dra-driver-role-binding
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: hami-dra-driver-role
+subjects:
+- kind: ServiceAccount
+  name: hami-dra-driver-service-account
+  namespace: {{ .Release.Namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: hami-dra-driver-clusterrole
+rules:
+- apiGroups:
+  - resource.k8s.io
+  resources:
+  - resourceclaims
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - resource.k8s.io
+  resources:
+  - resourceclaimtemplates
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - delete
+- apiGroups:
+  - resource.k8s.io
+  resources:
+  - resourceslices
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - delete
+- apiGroups:
+  - resource.k8s.io
+  resources:
+  - resourceclaims/status
+  verbs:
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: hami-dra-driver-clusterrole-binding-hami-dra-driver
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: hami-dra-driver-clusterrole
+subjects:
+- kind: ServiceAccount
+  name: hami-dra-driver-service-account
+  namespace: {{ .Release.Namespace }}
+{{- end }}
\ No newline at end of file
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrole.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrole.yaml
new file mode 100644
index 00000000..45c24863
--- /dev/null
+++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrole.yaml
@@ -0,0 +1,11 @@
+{{- if .Values.monitor.enabled }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "hami.dra.monitor.fullname" . }}
+rules:
+- apiGroups: ["resource.k8s.io"]
+  resources: ["resourceslices", "resourceclaims"]
+  verbs: ["get", "list", "watch"]
+{{- end }}
+
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrolebinding.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrolebinding.yaml
new file mode 100644
index 00000000..766d3098
--- /dev/null
+++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrolebinding.yaml
@@ -0,0 +1,15 @@
+{{- if .Values.monitor.enabled }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "hami.dra.monitor.fullname" . }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "hami.dra.monitor.fullname" . }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "hami.dra.monitor.fullname" . }}
+    namespace: {{ .Release.Namespace }}
+{{- end }}
+
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-deployment.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-deployment.yaml
new file mode 100644
index 00000000..b544b376
--- /dev/null
+++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-deployment.yaml
@@ -0,0 +1,62 @@
+{{- if .Values.monitor.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "hami.dra.monitor.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ include "hami.dra.monitor.fullname" . }}
+    {{- include "hami-dra-monitor.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.monitor.replicas | default 1 }}
+  selector:
+    matchLabels:
+      {{- include "hami-dra-monitor.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "hami-dra-monitor.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- include "hami.dra.monitor.imagePullSecrets" . | nindent 6 }}
+      serviceAccountName: {{ include "hami.dra.monitor.fullname" . }}
+      containers:
+        - name: monitor
+          image: {{ include "hami.dra.monitor.image" . }}
+          imagePullPolicy: {{ .Values.monitor.image.pullPolicy | default "IfNotPresent" }}
+          command:
+            - /bin/monitor
+            - --v={{ .Values.monitor.logLevel | default 2 }}
+            - --metrics-bind-address={{ .Values.monitor.metricsBindAddress | default ":8080" }}
+            - --health-probe-bind-address={{ .Values.monitor.healthProbeBindAddress | default ":8000" }}
+            - --kube-api-qps={{ .Values.monitor.kubeAPIQPS | default 40.0 }}
+            - --kube-api-burst={{ .Values.monitor.kubeAPIBurst | default 60 }}
+            - --collect-interval={{ .Values.monitor.collectInterval | default "30s" }}
+          ports:
+            - containerPort: 8080
+              name: metrics
+              protocol: TCP
+            - containerPort: 8000
+              name: health
+              protocol: TCP
+          {{- if .Values.monitor.resources }}
+          resources:
+            {{- toYaml .Values.monitor.resources | nindent 12 }}
+          {{- end }}
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: 8000
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /readyz
+              port: 8000
+            initialDelaySeconds: 5
+            periodSeconds: 5
+            timeoutSeconds: 3
+            failureThreshold: 3
+{{- end }}
+
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-service.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-service.yaml
new file mode 100644
index 00000000..d740cc38
--- /dev/null
+++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-service.yaml
@@ -0,0 +1,26 @@
+{{- if .Values.monitor.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "hami.dra.monitor.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "hami-dra-monitor.selectorLabels" . | nindent 4 }}
+spec:
+  type: {{ .Values.monitor.service.type | default "ClusterIP" }}
+  selector:
+    {{- include "hami-dra-monitor.selectorLabels" . | nindent 4 }}
+  ports:
+    - port: 8080
+      targetPort: 8080
+      name: metrics
+      protocol: TCP
+      {{- if and (eq (.Values.monitor.service.type | default "ClusterIP") "NodePort") .Values.monitor.service.nodePort.metrics }}
+      nodePort: {{ .Values.monitor.service.nodePort.metrics }}
+      {{- end }}
+    - port: 8000
+      targetPort: 8000
+      name: health
+      protocol: TCP
+{{- end }}
+
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-serviceaccount.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-serviceaccount.yaml
new file mode 100644
index 00000000..c61ad454
--- /dev/null
+++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-serviceaccount.yaml
@@ -0,0 +1,8 @@
+{{- if .Values.monitor.enabled }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "hami.dra.monitor.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+{{- end }}
+
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-manager.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-manager.yaml
new file mode 100644
index 00000000..dd31c71d
--- /dev/null
+++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-manager.yaml
@@ -0,0 +1,29 @@
+{{- if .Values.certs.certManager.enabled }}
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: {{ .Release.Name }}-dra-webhook-serving-cert
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "hami-dra-webhook.labels" . | nindent 4 }}
+spec:
+  dnsNames:
+    - {{ .Release.Name }}-dra-webhook.{{ .Release.Namespace }}.svc
+    - {{ .Release.Name }}-dra-webhook.{{ .Release.Namespace }}.svc.cluster.local
+  issuerRef:
+    kind: Issuer
+    name: {{ .Release.Name }}-dra-webhook-selfsigned-issuer
+  secretName: {{ .Release.Name }}-dra-webhook-tls
+  privateKey:
+    rotationPolicy: {{ .Values.certs.certManager.privateKeyRotationPolicy | default "Never" }}
+---
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: {{ .Release.Name }}-dra-webhook-selfsigned-issuer
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/component: hami-dra-webhook
+spec:
+  selfSigned: {}
+{{- end }}
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-secret.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-secret.yaml
new file mode 100644
index 00000000..eadbb517
--- /dev/null
+++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-secret.yaml
@@ -0,0 +1,13 @@
+{{- if eq .Values.certs.mode "custom" }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "hami-dra-webhook.tlsSecretName" . }}
+  namespace: {{ .Release.Namespace }}
+type: kubernetes.io/tls
+data:
+  tls.crt: |
+    {{- .Values.certs.custom.crt | b64enc | indent 4 }}
+  tls.key: |
+    {{- .Values.certs.custom.key | b64enc | indent 4 }}
+{{- end }}
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/device-config.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/device-config.yaml
new file mode 100644
index 00000000..97283f3e
--- /dev/null
+++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/device-config.yaml
@@ -0,0 +1,394 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "hami.dra.webhook.fullname" . }}-device-config
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/component: hami-scheduler
+    {{- include "hami-dra-webhook.selectorLabels" . | nindent 4 }}
+data:
+  device-config.yaml: |-
+  {{- if .Files.Glob "files/device-config.yaml" }}
+  {{- .Files.Get "files/device-config.yaml" | nindent 4}}
+  {{- else }}
+    nvidia:
+      resourceCountName: {{ .Values.resourceName }}
+      resourceMemoryName: {{ .Values.resourceMem }}
+      resourceMemoryPercentageName: {{ .Values.resourceMemPercentage }}
+      resourceCoreName: {{ .Values.resourceCores }}
+      resourcePriorityName: {{ .Values.resourcePriority }}
+      overwriteEnv: false
+      defaultMemory: 0
+      defaultCores: 0
+      defaultGPUNum: 1
+      knownMigGeometries:
+      - models: [ "A30" ]
+        allowedGeometries:
+          - 
+            - name: 1g.6gb
+              core: 25
+              memory: 6144
+              count: 4
+          - 
+            - name: 2g.12gb
+              core: 50
+              memory: 12288
+              count: 2
+          - 
+            - name: 4g.24gb
+              core: 100
+              memory: 24576
+              count: 1
+      - models: [ "A100-SXM4-40GB", "A100-40GB-PCIe", "A100-PCIE-40GB"]
+        allowedGeometries:
+          - 
+            - name: 1g.5gb
+              core: 14
+              memory: 5120
+              count: 7
+          - 
+            - name: 1g.5gb
+              core: 14
+              memory: 5120
+              count: 1
+            - name: 2g.10gb
+              core: 28
+              memory: 10240
+              count: 3
+          - 
+            - name: 3g.20gb
+              core: 42
+              memory: 20480
+              count: 2
+          - 
+            - name: 7g.40gb
+              core: 100
+              memory: 40960
+              count: 1
+      - models: [ "A100-SXM4-80GB", "A100-80GB-PCIe", "A100-PCIE-80GB"]
+        allowedGeometries:
+          - 
+            - name: 1g.10gb
+              core: 14
+              memory: 10240
+              count: 7
+          - 
+            - name: 1g.10gb
+              core: 14
+              memory: 10240
+              count: 1
+            - name: 2g.20gb
+              core: 28
+              memory: 20480
+              count: 3
+          - 
+            - name: 3g.40gb
+              core: 42
+              memory: 40960
+              count: 2
+          - 
+            - name: 7g.79gb
+              core: 100
+              memory: 80896
+              count: 1
+      - models: [ "H100-PCIE-80GB", "H100-SXM5-80GB"]
+        allowedGeometries:
+          - 
+            - name: 1g.10gb
+              core: 14
+              memory: 10240
+              count: 7
+          - 
+            - name: 1g.10gb
+              core: 14
+              memory: 10240
+              count: 1
+            - name: 2g.20gb
+              core: 28
+              memory: 20480
+              count: 3
+          - 
+            - name: 3g.40gb
+              core: 42
+              memory: 40960
+              count: 2
+          - 
+            - name: 7g.80gb
+              core: 100
+              memory: 81920
+              count: 1
+      - models: [ "H100-PCIE-94GB", "H100-SXM5-94GB"]
+        allowedGeometries:
+          - 
+            - name: 1g.12gb
+              core: 14
+              memory: 12288
+              count: 7
+          - 
+            - name: 1g.12gb
+              core: 14
+              memory: 12288
+              count: 1
+            - name: 2g.24gb
+              core: 28
+              memory: 24576
+              count: 3
+          - 
+            - name: 3g.47gb
+              core: 42
+              memory: 48128
+              count: 2
+          - 
+            - name: 7g.94gb
+              core: 100
+              memory: 96256
+              count: 1
+      - models: [ "H20", "H100 on GH200"]
+        allowedGeometries:
+          - 
+            - name: 1g.12gb
+              core: 14
+              memory: 12288
+              count: 7
+          - 
+            - name: 1g.12gb
+              core: 14
+              memory: 12288
+              count: 1
+            - name: 2g.24gb
+              core: 28
+              memory: 24576
+              count: 3 
+          - 
+            - name: 3g.48gb
+              core: 42
+              memory: 49152
+              count: 2
+          - 
+            - name: 7g.96gb
+              core: 100
+              memory: 98304
+              count: 1
+      - models: [ "H200 NVL", "H200-SXM5"]
+        allowedGeometries:
+          - 
+            - name: 1g.18gb
+              core: 14
+              memory: 18432
+              count: 7
+          - 
+            - name: 1g.18gb
+              core: 14
+              memory: 18432
+              count: 1
+            - name: 2g.35gb
+              core: 28
+              memory: 35840
+              count: 3
+          - 
+            - name: 3g.71gb
+              core: 42
+              memory: 72704
+              count: 2
+          - 
+            - name: 7g.141gb
+              core: 100
+              memory: 144384
+              count: 1
+      - models: [ "B200" ]
+        allowedGeometries:
+          - 
+            - name: 1g.23gb
+              core: 14
+              memory: 23552
+              count: 7
+          - 
+            - name: 1g.23gb
+              core: 14
+              memory: 23552
+              count: 1
+            - name: 2g.45gb
+              core: 28
+              memory: 46080
+              count: 3
+          - 
+            - name: 3g.90gb
+              core: 42
+              memory: 92160
+              count: 2
+          - 
+            - name: 7g.180gb
+              core: 100
+              memory: 184320
+              count: 1
+    cambricon:
+      resourceCountName: {{ .Values.mluResourceName }}
+      resourceMemoryName: {{ .Values.mluResourceMem }}
+      resourceCoreName: {{ .Values.mluResourceCores }}
+    hygon:
+      resourceCountName: {{ .Values.dcuResourceName }}
+      resourceMemoryName: {{ .Values.dcuResourceMem }}
+      resourceCoreName: {{ .Values.dcuResourceCores }}
+    metax:
+      resourceCountName: "metax-tech.com/gpu"
+      resourceVCountName: {{ .Values.metaxResourceName }}
+      resourceVMemoryName: {{ .Values.metaxResourceMem }}
+      resourceVCoreName: {{ .Values.metaxResourceCore }}
+      sgpuTopologyAware: {{ .Values.metaxsGPUTopologyAware }}
+    enflame:
+      resourceNameGCU: "enflame.com/gcu"
+      resourceNameVGCU: {{ .Values.enflameResourceNameVGCU }}
+      resourceNameVGCUPercentage: {{ .Values.enflameResourceNameVGCUPercentage }}
+    mthreads:
+      resourceCountName: "mthreads.com/vgpu"
+      resourceMemoryName: "mthreads.com/sgpu-memory"
+      resourceCoreName: "mthreads.com/sgpu-core"
+    iluvatars:
+    - chipName: MR-V100
+      commonWord: MR-V100
+      resourceCountName: iluvatar.ai/MR-V100-vgpu
+      resourceMemoryName: iluvatar.ai/MR-V100.vMem
+      resourceCoreName: iluvatar.ai/MR-V100.vCore
+    - chipName: MR-V50
+      commonWord: MR-V50
+      resourceCountName: iluvatar.ai/MR-V50-vgpu
+      resourceMemoryName: iluvatar.ai/MR-V50.vMem
+      resourceCoreName: iluvatar.ai/MR-V50.vCore
+    - chipName: BI-V150
+      commonWord: BI-V150
+      resourceCountName: iluvatar.ai/BI-V150-vgpu
+      resourceMemoryName: iluvatar.ai/BI-V150.vMem
+      resourceCoreName: iluvatar.ai/BI-V150.vCore
+    - chipName: BI-V100
+      commonWord: BI-V100
+      resourceCountName: iluvatar.ai/BI-V100-vgpu
+      resourceMemoryName: iluvatar.ai/BI-V100.vMem
+      resourceCoreName: iluvatar.ai/BI-V100.vCore
+    kunlun:
+      resourceCountName: {{ .Values.kunlunResourceName }}
+      resourceVCountName: {{ .Values.kunlunResourceVCountName }}
+      resourceVMemoryName: {{ .Values.kunlunResourceVMemoryName }}
+    awsneuron:
+      resourceCountName: "aws.amazon.com/neuron"
+      resourceCoreName: "aws.amazon.com/neuroncore"
+    amd:
+      resourceCountName: "amd.com/gpu"
+    vnpus:
+    - chipName: 910A
+      commonWord: Ascend910A
+      resourceName: huawei.com/Ascend910A
+      resourceMemoryName: huawei.com/Ascend910A-memory
+      memoryAllocatable: 32768
+      memoryCapacity: 32768
+      aiCore: 30
+      templates:
+        - name: vir02
+          memory: 2184
+          aiCore: 2
+        - name: vir04
+          memory: 4369
+          aiCore: 4
+        - name: vir08
+          memory: 8738
+          aiCore: 8
+        - name: vir16
+          memory: 17476
+          aiCore: 16
+    - chipName: 910B2
+      commonWord: Ascend910B2
+      resourceName: huawei.com/Ascend910B2
+      resourceMemoryName: huawei.com/Ascend910B2-memory
+      memoryAllocatable: 65536
+      memoryCapacity: 65536
+      aiCore: 24
+      aiCPU: 6
+      templates:
+        - name: vir03_1c_8g
+          memory: 8192
+          aiCore: 3
+          aiCPU: 1
+        - name: vir06_1c_16g
+          memory: 16384
+          aiCore: 6
+          aiCPU: 1
+        - name: vir12_3c_32g
+          memory: 32768
+          aiCore: 12
+          aiCPU: 3
+    - chipName: 910B3
+      commonWord: Ascend910B3
+      resourceName: huawei.com/Ascend910B3
+      resourceMemoryName: huawei.com/Ascend910B3-memory
+      memoryAllocatable: 65536
+      memoryCapacity: 65536
+      aiCore: 20
+      aiCPU: 7
+      templates:
+        - name: vir05_1c_16g
+          memory: 16384
+          aiCore: 5
+          aiCPU: 1
+        - name: vir10_3c_32g
+          memory: 32768
+          aiCore: 10
+          aiCPU: 3
+    - chipName: 910B4-1
+      commonWord: Ascend910B4-1
+      resourceName: huawei.com/Ascend910B4-1
+      resourceMemoryName: huawei.com/Ascend910B4-1-memory
+      memoryAllocatable: 65536
+      memoryCapacity: 65536
+      aiCore: 20
+      aiCPU: 7
+      templates:
+        # NOTE: Names of vnpu template for 910B4-1 are fixed by Ascend runtime and must not be changed.
+        # The memory is used for scheduling so the correct values must be set.
+        # Template vir05_1c_8g actually provides 16GB memory,
+        - name: vir05_1c_8g
+          memory: 16384
+          aiCore: 5
+          aiCPU: 1
+        # Template vir10_3c_16g actually provides 32GB memory
+        - name: vir10_3c_16g
+          memory: 32768
+          aiCore: 10
+          aiCPU: 3
+    - chipName: 910B4
+      commonWord: Ascend910B4
+      resourceName: huawei.com/Ascend910B4
+      resourceMemoryName: huawei.com/Ascend910B4-memory
+      memoryAllocatable: 32768
+      memoryCapacity: 32768
+      aiCore: 20
+      aiCPU: 7
+      templates:
+        - name: vir05_1c_8g
+          memory: 8192
+          aiCore: 5
+          aiCPU: 1
+        - name: vir10_3c_16g
+          memory: 16384
+          aiCore: 10
+          aiCPU: 3
+    - chipName: 310P3
+      commonWord: Ascend310P
+      resourceName: huawei.com/Ascend310P
+      resourceMemoryName: huawei.com/Ascend310P-memory
+      memoryAllocatable: 21527
+      memoryCapacity: 24576
+      aiCore: 8
+      aiCPU: 7
+      templates:
+        - name: vir01
+          memory: 3072
+          aiCore: 1
+          aiCPU: 1
+        - name: vir02
+          memory: 6144
+          aiCore: 2
+          aiCPU: 2
+        - name: vir04
+          memory: 12288
+          aiCore: 4
+          aiCPU: 4
+  {{ end }}
\ No newline at end of file
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/mutatingwebhookconfiguration.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/mutatingwebhookconfiguration.yaml
new file mode 100644
index 00000000..c763b6aa
--- /dev/null
+++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/mutatingwebhookconfiguration.yaml
@@ -0,0 +1,28 @@
+{{- if .Values.webhook.config.mutating.enabled }}
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: {{ .Release.Name }}-mutatingwebhookconfiguration
+  annotations:
+    cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ .Release.Name }}-dra-webhook-serving-cert
+webhooks:
+  - name: mutate.hami.io
+    admissionReviewVersions: ["v1"]
+    clientConfig:
+      service:
+        name: {{ .Release.Name }}-dra-webhook
+        namespace: {{ .Release.Namespace }}
+        path: /mutate
+    rules:
+      - apiGroups:
+          - ""
+        apiVersions:
+          - v1
+        operations:
+          - CREATE
+        resources:
+          - pods
+        scope: '*'
+    sideEffects: None
+    timeoutSeconds: 10
+{{- end }}
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/validatingwebhookconfiguration.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/validatingwebhookconfiguration.yaml
new file mode 100644
index 00000000..acdcbb96
--- /dev/null
+++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/validatingwebhookconfiguration.yaml
@@ -0,0 +1,29 @@
+{{- if .Values.webhook.config.validating.enabled }}
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: {{ .Release.Name }}-validatingwebhookconfiguration
+  annotations:
+    cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ .Release.Name }}-dra-webhook-serving-cert
+webhooks:
+  - name: validate.hami.io
+    admissionReviewVersions: ["v1"]
+    clientConfig:
+      service:
+        name: {{ .Release.Name }}-dra-webhook
+        namespace: {{ .Release.Namespace }}
+        path: /validate
+    rules:
+      - apiGroups:
+          - ""
+        apiVersions:
+          - v1
+        operations:
+          - DELETE
+        resources:
+          - pods
+        scope: '*'
+    sideEffects: None
+    timeoutSeconds: 10
+    failurePolicy: Ignore
+{{- end }}
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrole.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrole.yaml
new file mode 100644
index 00000000..28916a36
--- /dev/null
+++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrole.yaml
@@ -0,0 +1,14 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "hami.dra.webhook.fullname" . }}
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["admissionregistration.k8s.io"]
+  resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["resource.k8s.io"]
+  resources: ["resourceclaims"]
+  verbs: ["*"]
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrolebinding.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrolebinding.yaml
new file mode 100644
index 00000000..daad52f2
--- /dev/null
+++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrolebinding.yaml
@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "hami.dra.webhook.fullname" . }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "hami.dra.webhook.fullname" . }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "hami.dra.webhook.fullname" . }}
+    namespace: {{ .Release.Namespace }}
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-deployment.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-deployment.yaml
new file mode 100644
index 00000000..8e3cf90c
--- /dev/null
+++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-deployment.yaml
@@ -0,0 +1,59 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "hami.dra.webhook.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ include "hami.dra.webhook.fullname" . }}
+    {{- include "hami-dra-webhook.labels" . | nindent 4 }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      {{- include "hami-dra-webhook.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "hami-dra-webhook.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- include "hami.dra.webhook.imagePullSecrets" . | nindent 6 }}
+      serviceAccountName: {{ include "hami.dra.webhook.fullname" . }}
+      containers:
+        - name: webhook
+          image: {{ include "hami.dra.webhook.image" . }}
+          imagePullPolicy: {{ .Values.webhook.image.pullPolicy | default "IfNotPresent" }}
+          command:
+            - /bin/webhook
+            - --v=5
+            - --bind-address=0.0.0.0
+            - --secure-port=8443
+            - --cert-dir=/tls
+            - --tls-cert-file-name=tls.crt
+            - --tls-private-key-file-name=tls.key
+            - --metrics-bind-address=:8080
+            - --health-probe-bind-address=:8000
+            - --device-config-file=/device-config.yaml
+          ports:
+            - containerPort: 8443
+              name: webhook
+              protocol: TCP
+            - containerPort: 8080
+              name: metrics
+              protocol: TCP
+            - containerPort: 8000
+              name: health
+              protocol: TCP
+          volumeMounts:
+            - name: device-config
+              mountPath: /device-config.yaml
+              subPath: device-config.yaml
+            - name: tls-config
+              mountPath: /tls
+              readOnly: true
+      volumes:
+        - name: device-config
+          configMap:
+            name: {{ include "hami.dra.webhook.fullname" . }}-device-config
+        - name: tls-config
+          secret:
+            secretName: {{ if .Values.certs.certManager.enabled }}{{ .Release.Name }}-dra-webhook-tls{{ else }}{{ .Release.Name }}-tls{{ end }}
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-service.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-service.yaml
new file mode 100644
index 00000000..290174ec
--- /dev/null
+++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-service.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Release.Name }}-dra-webhook
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "hami-dra-webhook.selectorLabels" . | nindent 4 }}
+spec:
+  type: "ClusterIP"
+  selector:
+    {{- include "hami-dra-webhook.selectorLabels" . | nindent 4 }}
+  ports:
+    - port: 443
+      targetPort: 8443
+      name: webhook
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-serviceaccount.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-serviceaccount.yaml
new file mode 100644
index 00000000..b0dc2fb2
--- /dev/null
+++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-serviceaccount.yaml
@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "hami.dra.webhook.fullname" . }}
+  namespace: {{ .Release.Namespace }}
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/values.yaml b/packages/system/hami/charts/hami/charts/hami-dra/values.yaml
new file mode 100644
index 00000000..ebbbfed7
--- /dev/null
+++ b/packages/system/hami/charts/hami/charts/hami-dra/values.yaml
@@ -0,0 +1,166 @@
+## @section Global configuration
+## @param global.imageRegistry Global Docker image registry
+## @param global.imagePullSecrets Global Docker image pull secrets
+global:
+  ## @param global.imageRegistry Global Docker image registry
+  imageRegistry: ""
+  ## E.g.
+  ## imagePullSecrets:
+  ##   - myRegistryKeySecretName
+  ## @param global.imagePullSecrets Global Docker image pull secrets
+  imagePullSecrets: []
+
+#Nvidia GPU Parameters
+resourceName: "nvidia.com/gpu"
+resourceMem: "nvidia.com/gpumem"
+resourceMemPercentage: "nvidia.com/gpumem-percentage"
+resourceCores: "nvidia.com/gpucores"
+resourcePriority: "nvidia.com/priority"
+
+#MLU Parameters
+mluResourceName: "cambricon.com/vmlu"
+mluResourceMem: "cambricon.com/mlu.smlu.vmemory"
+mluResourceCores: "cambricon.com/mlu.smlu.vcore"
+
+#Hygon DCU Parameters
+dcuResourceName: "hygon.com/dcunum"
+dcuResourceMem: "hygon.com/dcumem"
+dcuResourceCores: "hygon.com/dcucores"
+
+#Metax sGPU Parameters
+metaxResourceName: "metax-tech.com/sgpu"
+metaxResourceCore: "metax-tech.com/vcore"
+metaxResourceMem: "metax-tech.com/vmemory"
+metaxsGPUTopologyAware: "false"
+
+#Enflame VGCU Parameters
+enflameResourceNameVGCU: "enflame.com/vgcu"
+enflameResourceNameVGCUPercentage: "enflame.com/vgcu-percentage"
+
+#Kunlun XPU Parameters
+kunlunResourceName: "kunlunxin.com/xpu"
+kunlunResourceVCountName: "kunlunxin.com/vxpu"
+kunlunResourceVMemoryName: "kunlunxin.com/vxpu-memory"
+
+# Webhook deployment configuration
+webhook:
+  image:
+    registry: ghcr.io
+    repository: project-hami/hami-dra-webhook
+    tag: "v0.1.0"
+    ## Specify a imagePullPolicy
+    ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
+    ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
+    ##
+    pullPolicy: Always
+    ## Optionally specify an array of imagePullSecrets.
+    ## Secrets must be manually created in the namespace.
+    ## Example:
+    ## pullSecrets:
+    ##   - myRegistryKeySecretName
+    ##
+    pullSecrets: []
+  ## @param resources controllerManager resource requests and limits
+  resources:
+  # If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+    limits:
+      cpu: 500m
+      memory: 512Mi
+    requests:
+      cpu: 100m
+      memory: 128Mi
+  ## @param nodeSelector controllerManager node labels for pod assignment
+  args:
+    webhookName: ""  # Default: {{ .Release.Name }}-hami-dra-webhook
+    secretName: ""  # Default: {{ .Release.Name }}-hami-dra-webhook-tls
+  # Webhook configuration
+  config:
+    mutating:
+      enabled: true
+    validating:
+      enabled: true
+
+# Monitor deployment configuration
+monitor:
+  enabled: true
+  replicas: 1
+  image:
+    registry: ghcr.io
+    repository: project-hami/hami-dra-monitor
+    tag: "v0.1.0"
+    ## Specify a imagePullPolicy
+    ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
+    ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
+    ##
+    pullPolicy: Always
+    ## Optionally specify an array of imagePullSecrets.
+    ## Secrets must be manually created in the namespace.
+    ## Example:
+    ## pullSecrets:
+    ##   - myRegistryKeySecretName
+    ##
+    pullSecrets: []
+  ## @param resources monitor resource requests and limits
+  resources:
+    limits:
+      cpu: 500m
+      memory: 512Mi
+    requests:
+      cpu: 100m
+      memory: 128Mi
+  ## Monitor configuration
+  logLevel: 2
+  metricsBindAddress: ":8080"
+  healthProbeBindAddress: ":8000"
+  kubeAPIQPS: 40.0
+  kubeAPIBurst: 60
+  collectInterval: "30s"
+  ## Monitor service configuration
+  service:
+    ## Service type: ClusterIP, NodePort, or LoadBalancer
+    ## Defaults to ClusterIP
+    type: ClusterIP
+    ## NodePort configuration (only used when type is NodePort)
+    nodePort:
+      ## Metrics port (NodePort). If not specified, Kubernetes will assign a random port
+      metrics: ""
+
+# Certificate configuration
+certs:
+  # Custom certificate (used when mode is "custom")
+  custom:
+    crt: ""
+    key: ""
+  # Cert-manager configuration (used when mode is "cert-manager")
+  certManager:
+    enabled: true
+    # Private key rotation policy: "Never" or "Always"
+    # In cert-manager >= v1.18.0, the default changed from "Never" to "Always"
+    # Setting to "Never" avoids frequent certificate rotation for webhooks
+    privateKeyRotationPolicy: "Never"
+    issuer:
+      clusterIssuerName: ""  # Required when type is "clusterIssuer"
+
+# DRA Drivers configuration
+drivers:
+  nvidia:
+    enabled: true
+    # If you are using gpu driver on host, you need to set this to false
+    containerDriver: true
+    image:
+      registry: ghcr.io
+      repository: projecthami/k8s-dra-driver
+      tag: "v0.0.1-dev"
+      ## Specify a imagePullPolicy
+      ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
+      ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
+      ##
+      pullPolicy: IfNotPresent
+      ## Optionally specify an array of imagePullSecrets.
+      ## Secrets must be manually created in the namespace.
+      ## Example:
+      ## pullSecrets:
+      ##   - myRegistryKeySecretName
+      ##
+      pullSecrets: []
diff --git a/packages/system/hami/charts/hami/templates/NOTES.txt b/packages/system/hami/charts/hami/templates/NOTES.txt
new file mode 100644
index 00000000..15bb1218
--- /dev/null
+++ b/packages/system/hami/charts/hami/templates/NOTES.txt
@@ -0,0 +1,3 @@
+** Please be patient while the chart is being deployed **
+Resource name: {{ .Values.resourceName }}
+
diff --git a/packages/system/hami/charts/hami/templates/_commons.tpl b/packages/system/hami/charts/hami/templates/_commons.tpl
new file mode 100644
index 00000000..78a262c9
--- /dev/null
+++ b/packages/system/hami/charts/hami/templates/_commons.tpl
@@ -0,0 +1,49 @@
+{{/*
+Return the proper image name
+{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" $) }}
+*/}}
+{{- define "common.images.image" -}}
+{{- $registryName := .imageRoot.registry -}}
+{{- $repositoryName := .imageRoot.repository -}}
+{{- $tag := .imageRoot.tag | toString -}}
+{{- if .global }}
+    {{- if .global.imageRegistry }}
+     {{- $registryName = .global.imageRegistry -}}
+    {{- end -}}
+{{- end -}}
+{{- if .tag }}
+     {{- $tag = .tag | toString -}}
+{{- end -}}
+{{- if $registryName }}
+{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}}
+{{- else -}}
+{{- printf "%s:%s" $repositoryName $tag -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead)
+{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }}
+*/}}
+{{- define "common.images.pullSecrets" -}}
+  {{- $pullSecrets := list }}
+
+  {{- if .global }}
+    {{- range .global.imagePullSecrets -}}
+      {{- $pullSecrets = append $pullSecrets . -}}
+    {{- end -}}
+  {{- end -}}
+
+  {{- range .images -}}
+    {{- range .pullSecrets -}}
+      {{- $pullSecrets = append $pullSecrets . -}}
+    {{- end -}}
+  {{- end -}}
+
+  {{- if (not (empty $pullSecrets)) }}
+imagePullSecrets:
+    {{- range $pullSecrets }}
+  - name: {{ . }}
+    {{- end }}
+  {{- end }}
+{{- end -}}
\ No newline at end of file
diff --git a/packages/system/hami/charts/hami/templates/_helpers.tpl b/packages/system/hami/charts/hami/templates/_helpers.tpl
new file mode 100644
index 00000000..ffcc61da
--- /dev/null
+++ b/packages/system/hami/charts/hami/templates/_helpers.tpl
@@ -0,0 +1,163 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "hami-vgpu.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "hami-vgpu.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Allow the release namespace to be overridden for multi-namespace deployments in combined charts
+*/}}
+{{- define "hami-vgpu.namespace" -}}
+  {{- if .Values.namespaceOverride -}}
+    {{- .Values.namespaceOverride -}}
+  {{- else -}}
+    {{- .Release.Namespace -}}
+  {{- end -}}
+{{- end -}}
+
+{{/*
+The app name for Scheduler
+*/}}
+{{- define "hami-vgpu.scheduler" -}}
+{{- printf "%s-scheduler" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+The app name for DevicePlugin
+*/}}
+{{- define "hami-vgpu.device-plugin" -}}
+{{- printf "%s-device-plugin" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+  The app name for MockDevicePlugin
+  */}}
+{{- define "hami-vgpu.mock-device-plugin" -}}
+{{- printf "%s-mock-device-plugin" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+The tls secret name for Scheduler
+*/}}
+{{- define "hami-vgpu.scheduler.tls" -}}
+{{- printf "%s-scheduler-tls" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+The webhook name
+*/}}
+{{- define "hami-vgpu.scheduler.webhook" -}}
+{{- printf "%s-webhook" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "hami-vgpu.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "hami-vgpu.labels" -}}
+helm.sh/chart: {{ include "hami-vgpu.chart" . }}
+{{ include "hami-vgpu.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "hami-vgpu.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "hami-vgpu.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+
+{{/*
+    Resolve the tag for kubeScheduler.
+*/}}
+{{- define "resolvedKubeSchedulerTag" -}}
+{{- if .Values.scheduler.kubeScheduler.image.tag }}
+{{- .Values.scheduler.kubeScheduler.image.tag | trim -}}
+{{- else }}
+{{- include "strippedKubeVersion" . | trim -}}
+{{- end }}
+{{- end }}
+
+{{/*
+    Return the stripped Kubernetes version string by removing extra parts after semantic version number.
+    v1.31.1+k3s1 -> v1.31.1
+    v1.30.8-eks-2d5f260 -> v1.30.8
+    v1.31.1 -> v1.31.1
+*/}}
+{{- define "strippedKubeVersion" -}}
+{{ regexReplaceAll "^(v[0-9]+\\.[0-9]+\\.[0-9]+)(.*)$" .Capabilities.KubeVersion.Version "$1" }}
+{{- end -}}
+
+{{- define "hami.scheduler.kubeScheduler.image" -}}
+{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.kubeScheduler.image "global" .Values.global "tag" (include "resolvedKubeSchedulerTag" .)) }}
+{{- end -}}
+
+{{- define "hami.scheduler.extender.image" -}}
+{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.extender.image "global" .Values.global "tag" .Values.global.imageTag) }}
+{{- end -}}
+
+{{- define "hami.devicePlugin.image" -}}
+{{ include "common.images.image" (dict "imageRoot" .Values.devicePlugin.image "global" .Values.global "tag" .Values.global.imageTag) }}
+{{- end -}}
+
+{{- define "hami.mockDevicePlugin.image" -}}
+{{ include "common.images.image" (dict "imageRoot" .Values.mockDevicePlugin.image "global" .Values.global "tag" .Values.mockDevicePlugin.tag) }}
+{{- end -}}
+
+{{- define "hami.devicePlugin.monitor.image" -}}
+{{ include "common.images.image" (dict "imageRoot" .Values.devicePlugin.monitor.image "global" .Values.global "tag" .Values.global.imageTag) }}
+{{- end -}}
+
+{{- define "hami.scheduler.patch.image" -}}
+{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.patch.image "global" .Values.global) }}
+{{- end -}}
+
+{{- define "hami.scheduler.patch.new.image" -}}
+{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.patch.imageNew "global" .Values.global) }}
+{{- end -}}
+
+{{- define "hami.scheduler.extender.imagePullSecrets" -}}
+{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.extender.image) "global" .Values.global) }}
+{{- end -}}
+
+{{- define "hami.devicePlugin.imagePullSecrets" -}}
+{{ include "common.images.pullSecrets" (dict "images" (list .Values.devicePlugin.image) "global" .Values.global) }}
+{{- end -}}
+
+{{- define "hami.scheduler.patch.imagePullSecrets" -}}
+{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.patch.image) "global" .Values.global) }}
+{{- end -}}
+
+{{- define "hami.scheduler.patch.new.imagePullSecrets" -}}
+{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.patch.imageNew) "global" .Values.global) }}
+{{- end -}}
diff --git a/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml b/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml
new file mode 100644
index 00000000..b36347f2
--- /dev/null
+++ b/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml
@@ -0,0 +1,13 @@
+{{- if and .Values.devicePlugin.enabled (not .Values.devicePlugin.nodeConfiguration.externalConfigName) (not .Values.dra.enabled) -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "hami-vgpu.device-plugin" . }}
+  namespace: {{ include "hami-vgpu.namespace" . }}
+  labels:
+    app.kubernetes.io/component: hami-device-plugin
+    {{- include "hami-vgpu.labels" . | nindent 4 }}
+data:
+  config.json: |
+{{- .Values.devicePlugin.nodeConfiguration.config | nindent 4 }}
+{{- end }}
\ No newline at end of file
diff --git a/packages/system/hami/charts/hami/templates/device-plugin/daemonsetmock.yaml b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetmock.yaml
new file mode 100644
index 00000000..a0dc4ff3
--- /dev/null
+++ b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetmock.yaml
@@ -0,0 +1,55 @@
+{{- if .Values.mockDevicePlugin.enabled }}
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: {{ include "hami-vgpu.mock-device-plugin" . }}
+  namespace: {{ include "hami-vgpu.namespace" . }}
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: hami-mock-device-plugin
+      {{- include "hami-vgpu.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ""
+      labels:
+        app.kubernetes.io/component: hami-mock-device-plugin
+        {{- include "hami-vgpu.selectorLabels" . | nindent 8 }}
+    spec:
+      serviceAccountName: {{ include "hami-vgpu.mock-device-plugin" . }}
+      tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+      containers:
+      - image: {{ include "hami.mockDevicePlugin.image" . }}
+        imagePullPolicy: {{ .Values.mockDevicePlugin.image.pullPolicy }}
+        name: hami-mock-dp-cntr
+        env:
+          - name: NODE_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
+        command:
+          - ./k8s-device-plugin
+          - -v=5
+          - --device-config-file=/device-config.yaml
+        volumeMounts:
+          - name: dp
+            mountPath: /var/lib/kubelet/device-plugins
+          - name: sys
+            mountPath: /sys
+          - name: device-config
+            mountPath: /device-config.yaml
+            subPath: device-config.yaml
+      volumes:
+        - name: dp
+          hostPath:
+            path: /var/lib/kubelet/device-plugins
+        - name: sys
+          hostPath:
+            path: /sys
+        - name: device-config
+          configMap:
+            name: {{ include "hami-vgpu.scheduler" . }}-device
+{{- end -}}
diff --git a/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml
new file mode 100644
index 00000000..577dc0f9
--- /dev/null
+++ b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml
@@ -0,0 +1,262 @@
+{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) }}
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: {{ include "hami-vgpu.device-plugin" . }}
+  namespace: {{ include "hami-vgpu.namespace" . }}
+  labels:
+    app.kubernetes.io/component: hami-device-plugin
+    {{- include "hami-vgpu.labels" . | nindent 4 }}
+    {{- with .Values.global.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  {{- if .Values.global.annotations }}
+  annotations: {{ toYaml .Values.global.annotations | nindent 4}}
+  {{- end }}
+spec:
+  updateStrategy:
+    {{- with .Values.devicePlugin.updateStrategy }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: hami-device-plugin
+      {{- include "hami-vgpu.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: hami-device-plugin
+        hami.io/webhook: ignore
+        {{- include "hami-vgpu.selectorLabels" . | nindent 8 }}
+      annotations:
+        {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
+        checksum/hami-scheduler-newversion-config: {{ include (print $.Template.BasePath "/scheduler/configmapnew.yaml") . | sha256sum }}
+        {{- else }}
+        checksum/hami-scheduler-config: {{ include (print $.Template.BasePath "/scheduler/configmap.yaml") . | sha256sum }}
+        {{- end }}
+        checksum/hami-scheduler-device-config: {{ include (print $.Template.BasePath "/scheduler/device-configmap.yaml") . | sha256sum }}
+      {{- if .Values.devicePlugin.podAnnotations }}
+        {{- toYaml .Values.devicePlugin.podAnnotations | nindent 8 }}
+      {{- end }}
+    spec:
+      {{- if .Values.devicePlugin.runtimeClassName }}
+      runtimeClassName: {{ .Values.devicePlugin.runtimeClassName }}
+      {{- end }}
+      serviceAccountName: {{ include "hami-vgpu.device-plugin" . }}
+      priorityClassName: system-node-critical
+      hostPID: true
+      hostNetwork: true
+      {{- include "hami.devicePlugin.imagePullSecrets" . | nindent 6 }}
+      {{- if .Values.devicePlugin.gpuOperatorToolkitReady.enabled }}
+      initContainers:
+        - name: toolkit-validation
+          image: {{ include "hami.devicePlugin.image" . }}
+          imagePullPolicy: {{ .Values.devicePlugin.image.pullPolicy }}
+          securityContext:
+            privileged: true
+            runAsUser: 0
+          command: ["sh", "-c"]
+          args:
+            - |
+              echo "Waiting for NVIDIA Toolkit to be ready..."
+              until [ -f {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }}/toolkit-ready ]; do 
+                echo "Waiting for {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }}/toolkit-ready..."
+                sleep 5
+              done
+              echo "NVIDIA Toolkit is ready!"
+          volumeMounts:
+            - name: nvidia-validations
+              mountPath: {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath | quote }}
+              mountPropagation: HostToContainer
+              readOnly: true
+      {{- end }}
+      containers:
+        - name: device-plugin
+          image: {{ include "hami.devicePlugin.image" . }}
+          imagePullPolicy: {{ .Values.devicePlugin.image.pullPolicy }}
+          lifecycle:
+            postStart:
+              exec:
+                command: ["/bin/sh","-c", {{ printf "/k8s-vgpu/bin/vgpu-init.sh %s/vgpu/" .Values.global.gpuHookPath | quote }}]
+          command:
+            - nvidia-device-plugin
+            - --config-file=/device-config.yaml
+            - --mig-strategy={{ .Values.devicePlugin.migStrategy }}
+            - --disable-core-limit={{ .Values.devicePlugin.disablecorelimit }}
+            {{- range .Values.devicePlugin.extraArgs }}
+            - {{ . }}
+            {{- end }}
+          env:
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: NVIDIA_MIG_MONITOR_DEVICES
+              value: all
+            - name: DEVICE_LIST_STRATEGY
+              value: {{ .Values.devicePlugin.deviceListStrategy }}
+            - name: HOOK_PATH
+              value: {{ .Values.global.gpuHookPath }}
+            {{- if typeIs "bool" .Values.devicePlugin.passDeviceSpecsEnabled }}
+            - name: PASS_DEVICE_SPECS
+              value: {{ .Values.devicePlugin.passDeviceSpecsEnabled | quote }}
+            {{- end }}
+            {{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }}
+            - name: NVIDIA_DRIVER_ROOT
+              value: {{ .Values.devicePlugin.nvidiaDriverRoot }}
+            {{- end }}
+            {{- if typeIs "string" .Values.devicePlugin.nvidiaHookPath }}
+            - name: NVIDIA_CDI_HOOK_PATH
+              value: {{ .Values.devicePlugin.nvidiaHookPath }}
+            {{- end }}
+            {{- if typeIs "bool" .Values.devicePlugin.gdrcopyEnabled }}
+            - name: GDRCOPY_ENABLED
+              value: {{ .Values.devicePlugin.gdrcopyEnabled | quote }}
+            {{- end }}
+            {{- if typeIs "bool" .Values.devicePlugin.gdsEnabled }}
+            - name: GDS_ENABLED
+              value: {{ .Values.devicePlugin.gdsEnabled | quote }}
+            {{- end }}
+            {{- if typeIs "bool" .Values.devicePlugin.mofedEnabled }}
+            - name: MOFED_ENABLED
+              value: {{ .Values.devicePlugin.mofedEnabled | quote }}
+            {{- end }}
+            {{- if eq (.Values.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy | default "spread")  "topology-aware" }}
+            - name: ENABLE_TOPOLOGY_SCORE
+              value: "true"
+            {{- end }}
+            {{- with .Values.devicePlugin.extraEnvs }}
+            {{- . | toYaml | nindent 12 }}
+            {{- end }}
+          securityContext:
+            privileged: true
+            allowPrivilegeEscalation: true
+            capabilities:
+              drop: ["ALL"]
+              add: ["SYS_ADMIN"]
+          resources:
+          {{- toYaml .Values.devicePlugin.resources | nindent 12 }}
+          volumeMounts:
+            - name: device-plugin
+              mountPath: /var/lib/kubelet/device-plugins
+            - name: lib
+              mountPath: {{ printf "%s%s" .Values.global.gpuHookPath "/vgpu" }}
+            - name: usrbin
+              mountPath: /usrbin
+            - name: deviceconfig
+              mountPath: /config
+            - name: hosttmp
+              mountPath: /tmp
+            - name: device-config
+              mountPath: /device-config.yaml
+              subPath: device-config.yaml
+            - name: cdi-root
+              mountPath: /var/run/cdi
+            {{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }}
+            # We always mount the driver root at /driver-root in the container.
+            # This is required for CDI detection to work correctly.
+            - name: driver-root
+              mountPath: /driver-root
+              readOnly: true
+            {{- end }}
+        - name: vgpu-monitor
+          image: {{ include "hami.devicePlugin.monitor.image" . }}
+          imagePullPolicy: {{ .Values.devicePlugin.monitor.image.pullPolicy }}
+          command:
+            - "vGPUmonitor"
+            {{- range .Values.devicePlugin.monitor.extraArgs }}
+            - {{ . }}
+            {{- end }}
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+              add: ["SYS_ADMIN"]
+          env:
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: NVIDIA_VISIBLE_DEVICES
+              value: "all"
+            - name: NVIDIA_MIG_MONITOR_DEVICES
+              value: "all"
+            - name: HOOK_PATH
+              value: "{{ .Values.global.gpuHookPath }}/vgpu"
+            - name: HAMI_RESYNC_INTERVAL
+              value: {{ .Values.devicePlugin.monitor.resyncInterval | default "5m" | quote }}
+            {{- with .Values.devicePlugin.monitor.extraEnvs }}
+            {{- . | toYaml | nindent 12 }}
+            {{- end }}
+          resources:
+          {{- toYaml .Values.devicePlugin.monitor.resources | nindent 12 }}
+          volumeMounts:
+            - name: ctrs
+              mountPath: {{ .Values.devicePlugin.monitor.ctrPath }}
+            - name: dockers
+              mountPath: /run/docker
+            - name: containerds
+              mountPath: /run/containerd
+            - name: sysinfo
+              mountPath: /sysinfo
+            - name: hostvar
+              mountPath: /hostvar
+            - name: hosttmp
+              mountPath: /tmp
+      volumes:
+        - name: ctrs
+          hostPath:
+            path: {{ .Values.devicePlugin.monitor.ctrPath }}
+        - name: hosttmp
+          hostPath:
+            path: /tmp
+        - name: dockers
+          hostPath:
+            path: /run/docker
+        - name: containerds
+          hostPath:
+            path: /run/containerd
+        - name: device-plugin
+          hostPath:
+            path: {{ .Values.devicePlugin.pluginPath }}
+        - name: lib
+          hostPath:
+            path: {{ .Values.devicePlugin.libPath }}
+        {{- if .Values.devicePlugin.gpuOperatorToolkitReady.enabled }}
+        - name: nvidia-validations
+          hostPath:
+            path: {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }}
+            type: DirectoryOrCreate
+        {{- end }}
+        {{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }}
+        - name: driver-root
+          hostPath:
+            path: {{ .Values.devicePlugin.nvidiaDriverRoot }}
+            type: Directory
+        {{- end }}
+        - name: cdi-root
+          hostPath:
+            path: /var/run/cdi
+            type: DirectoryOrCreate
+        - name: usrbin
+          hostPath:
+            path: /usr/bin
+        - name: sysinfo
+          hostPath:
+            path: /sys
+        - name: hostvar
+          hostPath:
+            path: /var
+        - name: deviceconfig
+          configMap:
+            name: {{ .Values.devicePlugin.nodeConfiguration.externalConfigName | default (include "hami-vgpu.device-plugin" .) }}
+        - name: device-config
+          configMap:
+            name: {{ include "hami-vgpu.scheduler" . }}-device
+      {{- if .Values.devicePlugin.nvidiaNodeSelector }}
+      nodeSelector: {{ toYaml .Values.devicePlugin.nvidiaNodeSelector | nindent 8 }}
+      {{- end }}
+      {{- if .Values.devicePlugin.tolerations }}
+      tolerations: {{ toYaml .Values.devicePlugin.tolerations | nindent 8 }}
+      {{- end }}
+{{- end }}
diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml
new file mode 100644
index 00000000..c51ff6c6
--- /dev/null
+++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml
@@ -0,0 +1,29 @@
+{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name:  {{ include "hami-vgpu.device-plugin" . }}-monitor
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+      - create
+      - watch
+      - list
+      - update
+      - patch
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - get
+      - update
+      - list
+      - patch
+{{- end -}}
+    
+    
diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml
new file mode 100644
index 00000000..93a118dd
--- /dev/null
+++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml
@@ -0,0 +1,17 @@
+{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "hami-vgpu.device-plugin" . }}
+  labels:
+    app.kubernetes.io/component: "hami-device-plugin"
+    {{- include "hami-vgpu.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "hami-vgpu.device-plugin" . }}-monitor
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "hami-vgpu.device-plugin" . }}
+    namespace: {{ include "hami-vgpu.namespace" . }}
+{{- end -}}
diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml
new file mode 100644
index 00000000..24daca9b
--- /dev/null
+++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml
@@ -0,0 +1,29 @@
+{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "hami-vgpu.device-plugin" . }}-monitor
+  namespace: {{ include "hami-vgpu.namespace" . }}
+  labels:
+    app.kubernetes.io/component: hami-device-plugin
+    {{- include "hami-vgpu.labels" . | nindent 4 }}
+    {{- if .Values.devicePlugin.service.labels }}  # Use devicePlugin instead of scheduler
+    {{ toYaml .Values.devicePlugin.service.labels | indent 4 }}
+    {{- end }}
+  {{- if .Values.devicePlugin.service.annotations }}  # Use devicePlugin instead of scheduler
+  annotations: {{ toYaml .Values.devicePlugin.service.annotations | nindent 4 }}
+  {{- end }}
+spec:
+  type: {{ .Values.devicePlugin.service.type | default "NodePort" }}  # Default type is NodePort
+  ports:
+    - name: monitorport
+      port: {{ .Values.devicePlugin.service.httpPort | default 31992 }}  # Default HTTP port is 31992
+      targetPort: 9394
+      {{- if eq (.Values.devicePlugin.service.type | default "NodePort") "NodePort" }}  # If type is NodePort, set nodePort
+      nodePort: {{ .Values.devicePlugin.service.httpPort | default 31992 }}
+      {{- end }}
+      protocol: TCP
+  selector:
+    app.kubernetes.io/component: hami-device-plugin
+    {{- include "hami-vgpu.selectorLabels" . | nindent 4 }}
+{{- end -}}
\ No newline at end of file
diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml
new file mode 100644
index 00000000..b10d2cb9
--- /dev/null
+++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml
@@ -0,0 +1,10 @@
+{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "hami-vgpu.device-plugin" . }}
+  namespace: {{ include "hami-vgpu.namespace" . }}
+  labels:
+    app.kubernetes.io/component: "hami-device-plugin"
+    {{- include "hami-vgpu.labels" . | nindent 4 }}
+{{- end -}}
diff --git a/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml b/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml
new file mode 100644
index 00000000..cd3d14cc
--- /dev/null
+++ b/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml
@@ -0,0 +1,11 @@
+{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}}
+{{- if and .Values.devicePlugin.createRuntimeClass .Values.devicePlugin.runtimeClassName }}
+apiVersion: node.k8s.io/v1
+kind: RuntimeClass
+metadata:
+  name: {{ .Values.devicePlugin.runtimeClassName }}
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade
+handler: nvidia
+{{- end }}
+{{- end -}}
diff --git a/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml b/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml
new file mode 100644
index 00000000..b6edb605
--- /dev/null
+++ b/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml
@@ -0,0 +1,31 @@
+{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}}
+{{- if .Values.scheduler.certManager.enabled }}
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: {{ include "hami-vgpu.scheduler" . }}-serving-cert
+  namespace: {{ include "hami-vgpu.namespace" . }}
+  labels:
+    app.kubernetes.io/component: hami-scheduler
+    {{- include "hami-vgpu.labels" . | nindent 4 }}
+spec:
+  dnsNames:
+    - {{ include "hami-vgpu.scheduler" . }}.{{ include "hami-vgpu.namespace" . }}.svc
+    - {{ include "hami-vgpu.scheduler" . }}.{{ include "hami-vgpu.namespace" . }}.svc.cluster.local
+  issuerRef:
+    kind: Issuer
+    name: {{ include "hami-vgpu.scheduler" . }}-selfsigned-issuer
+  secretName: {{ include "hami-vgpu.scheduler.tls" . }}
+---
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: {{ include "hami-vgpu.scheduler" . }}-selfsigned-issuer
+  namespace: {{ include "hami-vgpu.namespace" . }}
+  labels:
+    app.kubernetes.io/component: hami-scheduler
+    {{- include "hami-vgpu.labels" . | nindent 4 }}
+spec:
+  selfSigned: {}
+{{- end }}
+{{- end }}
diff --git a/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml b/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml
new file mode 100644
index 00000000..9c510c00
--- /dev/null
+++ b/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml
@@ -0,0 +1,36 @@
+{{- if not .Values.dra.enabled -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "hami-vgpu.scheduler" . }}
+  labels:
+    app.kubernetes.io/component: "hami-scheduler"
+    {{- include "hami-vgpu.labels" . | nindent 4 }}
+rules:
+  - apiGroups: [""]
+    resources: ["pods", "configmaps"]
+    verbs: ["get", "list", "watch", "patch"]
+  - apiGroups: [""]
+    resources: ["pods/binding"]
+    verbs: ["create"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "patch", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["create", "get", "list"]
+  - apiGroups: [""]
+    resources: ["resourcequotas"]
+    verbs: ["get", "list", "watch"]
+---
+{{- if .Values.mockDevicePlugin.enabled }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "hami-vgpu.mock-device-plugin" . }}
+rules:
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "update", "list", "patch"]
+{{- end -}}
+{{- end }}
diff --git a/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml
new file mode 100644
index 00000000..69d75986
--- /dev/null
+++ b/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml
@@ -0,0 +1,49 @@
+{{- if not .Values.dra.enabled -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "hami-vgpu.scheduler" . }}-kube
+  labels:
+    app.kubernetes.io/component: "hami-scheduler"
+    {{- include "hami-vgpu.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:kube-scheduler
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "hami-vgpu.scheduler" . }}
+    namespace: {{ include "hami-vgpu.namespace" . }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "hami-vgpu.scheduler" . }}-volume
+  labels:
+    app.kubernetes.io/component: "hami-scheduler"
+    {{- include "hami-vgpu.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:volume-scheduler
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "hami-vgpu.scheduler" . }}
+    namespace: {{ include "hami-vgpu.namespace" . }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "hami-vgpu.scheduler" . }}
+  labels:
+    app.kubernetes.io/component: "hami-scheduler"
+    {{- include "hami-vgpu.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "hami-vgpu.scheduler" . }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "hami-vgpu.scheduler" . }}
+    namespace: {{ include "hami-vgpu.namespace" . }}
+{{- end }}
\ No newline at end of file
diff --git a/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml b/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml
new file mode 100644
index 00000000..75889146
--- /dev/null
+++ b/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml
@@ -0,0 +1,142 @@
+{{- if and .Values.scheduler.kubeScheduler.enabled (not .Values.dra.enabled) -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "hami-vgpu.scheduler" . }}
+  namespace: {{ include "hami-vgpu.namespace" . }}
+  labels:
+    app.kubernetes.io/component: hami-scheduler
+    {{- include "hami-vgpu.labels" . | nindent 4 }}
+data:
+  config.json: |
+    {
+        "kind": "Policy",
+        "apiVersion": "v1",
+        "extenders": [
+            {
+                {{- if .Values.scheduler.admissionWebhook.enabled }}
+                "urlPrefix": "https://127.0.0.1:443",
+                "enableHttps": true,
+                "tlsConfig": {
+                    "insecure": true
+                },
+                {{- else }}
+                "urlPrefix": "http://127.0.0.1:80",
+                "enableHttps": false,
+                {{- end }}
+                "filterVerb": "filter",
+                "bindVerb": "bind",
+                "weight": 1,
+                "nodeCacheCapable": true,
+                "httpTimeout": 30000000000,
+                "managedResources": [
+                    {{- range .Values.devices.amd.customresources }}
+                    {
+                      "name": "{{ . }}",
+                      "ignoredByScheduler": true
+                    },
+                    {{- end }}
+                    {{- if .Values.devices.ascend.enabled }}
+                    {{- range .Values.devices.ascend.customresources }}
+                    {
+                      "name": "{{ . }}",
+                      "ignoredByScheduler": true
+                    },
+                    {{- end }}
+                    {{- end }}
+                    {{- if .Values.devices.mthreads.enabled }}
+                    {{- range .Values.devices.mthreads.customresources }}
+                    {
+                      "name": "{{ . }}",
+                      "ignoredByScheduler": true
+                    },
+                    {{- end }}
+                    {{- end }}
+                    {{- if .Values.devices.enflame.enabled }}
+                    {{- range .Values.devices.enflame.customresources }}
+                    {
+                      "name": "{{ . }}",
+                      "ignoredByScheduler": true
+                    },
+                    {{- end }}
+                    {{- end }}
+                    {{- if .Values.devices.kunlun.enabled }}
+                    {{- range .Values.devices.kunlun.customresources }}
+                    {
+                      "name": "{{ . }}",
+                      "ignoredByScheduler": true
+                    },
+                    {{- end }}
+                    {{- end }}
+                    {{- range .Values.devices.awsneuron.customresources }}
+                    {
+                      "name": "{{ . }}",
+                      "ignoredByScheduler": true
+                    },
+                    {{- end }}
+                    {{- if .Values.devices.iluvatar.enabled }}
+                    {{- range .Values.devices.iluvatar.customresources }}
+                    {
+                      "name": "{{ . }}",
+                      "ignoredByScheduler": true
+                    },
+                    {{- end }}
+                    {{- end }}
+                    {
+                        "name": "{{ .Values.resourceName }}",
+                        "ignoredByScheduler": true
+                    },
+                    {
+                        "name": "{{ .Values.resourceMem }}",
+                        "ignoredByScheduler": true
+                    },
+                    {
+                        "name": "{{ .Values.resourceCores }}",
+                        "ignoredByScheduler": true
+                    },
+                    {
+                        "name": "{{ .Values.resourceMemPercentage }}",
+                        "ignoredByScheduler": true
+                    },
+                    {
+                        "name": "{{ .Values.resourcePriority }}",
+                        "ignoredByScheduler": true
+                    },
+                    {
+                        "name": "{{ .Values.mluResourceName }}",
+                        "ignoredByScheduler": true
+                    },
+                    {
+                        "name": "{{ .Values.dcuResourceName }}",
+                        "ignoredByScheduler": true
+                    },
+                    {
+                        "name": "{{ .Values.dcuResourceMem }}",
+                        "ignoredByScheduler": true 
+                    },
+                    {
+                        "name": "{{ .Values.dcuResourceCores }}",
+                        "ignoredByScheduler": true
+                    },
+                    {
+                        "name": "metax-tech.com/gpu",
+                        "ignoredByScheduler": true
+                    },
+                    {
+                        "name": "{{ .Values.metaxResourceName }}",
+                        "ignoredByScheduler": true
+                    },
+                    {
+                        "name": "{{ .Values.metaxResourceCore }}",
+                        "ignoredByScheduler": true
+                    },
+                    {
+                        "name": "{{ .Values.metaxResourceMem }}",
+                        "ignoredByScheduler": true
+                    }
+                ],
+                "ignoreable": false
+            }
+        ]
+    }
+{{- end }}
diff --git a/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml b/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml
new file mode 100644
index 00000000..e2a91d8b
--- /dev/null
+++ b/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml
@@ -0,0 +1,102 @@
+{{- if and .Values.scheduler.kubeScheduler.enabled (not .Values.dra.enabled) -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "hami-vgpu.scheduler" . }}-newversion
+  namespace: {{ include "hami-vgpu.namespace" . }}
+  labels:
+    app.kubernetes.io/component: hami-scheduler
+    {{- include "hami-vgpu.labels" . | nindent 4 }}
+data:
+  config.yaml: |
+    {{- if gt (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 25}}
+    apiVersion: kubescheduler.config.k8s.io/v1
+    {{- else }}
+    apiVersion: kubescheduler.config.k8s.io/v1beta2
+    {{- end }}
+    kind: KubeSchedulerConfiguration
+    leaderElection:
+      leaderElect: false
+    profiles:
+    - schedulerName: {{ .Values.schedulerName }}
+    extenders:
+    {{- if .Values.scheduler.admissionWebhook.enabled }}
+    - urlPrefix: "https://127.0.0.1:443"
+      enableHTTPS: true
+      tlsConfig:
+        insecure: true
+    {{- else }}
+    - urlPrefix: "http://127.0.0.1:80"
+      enableHTTPS: false
+    {{- end }}
+      filterVerb: filter
+      bindVerb: bind
+      nodeCacheCapable: true
+      weight: 1
+      httpTimeout: 30s
+      managedResources:
+      - name: {{ .Values.resourceName }}
+        ignoredByScheduler: true
+      - name: {{ .Values.resourceMem }}
+        ignoredByScheduler: true
+      - name: {{ .Values.resourceCores }}
+        ignoredByScheduler: true
+      - name: {{ .Values.resourceMemPercentage }}
+        ignoredByScheduler: true
+      - name: {{ .Values.resourcePriority }}
+        ignoredByScheduler: true
+      - name: {{ .Values.mluResourceName }}
+        ignoredByScheduler: true
+      - name: {{ .Values.dcuResourceName }}
+        ignoredByScheduler: true
+      - name: {{ .Values.dcuResourceMem }}
+        ignoredByScheduler: true
+      - name: {{ .Values.dcuResourceCores }}
+        ignoredByScheduler: true
+      - name: "metax-tech.com/gpu"
+        ignoredByScheduler: true
+      - name: {{ .Values.metaxResourceName }}
+        ignoredByScheduler: true
+      - name: {{ .Values.metaxResourceCore }}
+        ignoredByScheduler: true
+      - name: {{ .Values.metaxResourceMem }}
+        ignoredByScheduler: true
+      {{- if .Values.devices.ascend.enabled }}
+      {{- range .Values.devices.ascend.customresources }}
+      - name: {{ . }}
+        ignoredByScheduler: true
+      {{- end }}
+      {{- end }}
+      {{- if .Values.devices.mthreads.enabled }}
+      {{- range .Values.devices.mthreads.customresources }}
+      - name: {{ . }}
+        ignoredByScheduler: true
+      {{- end }}
+      {{- end }}
+      {{- if .Values.devices.enflame.enabled }}
+      {{- range .Values.devices.enflame.customresources }}
+      - name: {{ . }}
+        ignoredByScheduler: true
+      {{- end }}
+      {{- end }}
+      {{- if .Values.devices.kunlun.enabled }}
+      {{- range .Values.devices.kunlun.customresources }}
+      - name: {{ . }}
+        ignoredByScheduler: true
+      {{- end }}
+      {{- end }}
+      {{- range .Values.devices.awsneuron.customresources }}
+      - name: {{ . }}
+        ignoredByScheduler: true
+      {{- end }}
+      {{- if .Values.devices.iluvatar.enabled }}
+      {{- range .Values.devices.iluvatar.customresources }}
+      - name: {{ . }}
+        ignoredByScheduler: true
+      {{- end }}
+      {{- end }}
+      {{- range .Values.devices.amd.customresources }}
+      - name: {{ . }}
+        ignoredByScheduler: true
+      {{- end }}
+{{- end }}
diff --git a/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml b/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml
new file mode 100644
index 00000000..6364cae3
--- /dev/null
+++ b/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml
@@ -0,0 +1,227 @@
+{{- if not .Values.dra.enabled -}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "hami-vgpu.scheduler" . }}
+  namespace: {{ include "hami-vgpu.namespace" . }}
+  labels:
+    app.kubernetes.io/component: hami-scheduler
+    {{- include "hami-vgpu.labels" . | nindent 4 }}
+    {{- with .Values.global.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  {{- if .Values.global.annotations }}
+  annotations: {{ toYaml .Values.global.annotations | nindent 4}}
+  {{- end }}
+spec:
+  {{- if .Values.scheduler.leaderElect }}
+  replicas: {{ .Values.scheduler.replicas }}
+  {{- else }}
+  replicas: 1
+  {{- end }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: hami-scheduler
+      {{- include "hami-vgpu.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: hami-scheduler
+        {{- include "hami-vgpu.selectorLabels" . | nindent 8 }}
+        hami.io/webhook: ignore
+      annotations:
+        {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
+        checksum/hami-scheduler-newversion-config: {{ include (print $.Template.BasePath "/scheduler/configmapnew.yaml") . | sha256sum }}
+        {{- else }}
+        checksum/hami-scheduler-config: {{ include (print $.Template.BasePath "/scheduler/configmap.yaml") . | sha256sum }}
+        {{- end }}
+        checksum/hami-scheduler-device-config: {{ include (print $.Template.BasePath "/scheduler/device-configmap.yaml") . | sha256sum }}
+      {{- if .Values.scheduler.podAnnotations }}
+        {{- toYaml .Values.scheduler.podAnnotations | nindent 8 }}
+      {{- end }}
+    spec:
+      serviceAccountName: {{ include "hami-vgpu.scheduler" . }}
+      priorityClassName: system-node-critical
+      {{- include "hami.scheduler.extender.imagePullSecrets" . | nindent 6 }}
+      containers:
+        {{- if .Values.scheduler.kubeScheduler.enabled }}
+        - name: kube-scheduler
+          image: {{ include "hami.scheduler.kubeScheduler.image" . }}
+          imagePullPolicy: {{ .Values.scheduler.kubeScheduler.image.pullPolicy }}
+          command:
+            - kube-scheduler
+            {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
+            {{- range .Values.scheduler.kubeScheduler.extraNewArgs }}
+            - {{ . }}
+            {{- end }}
+            {{- else }}
+            - --scheduler-name={{ .Values.schedulerName }}
+            {{- range .Values.scheduler.kubeScheduler.extraArgs }}
+            - {{ . }}
+            {{- end }}
+            {{- end }}
+            - --leader-elect={{ .Values.scheduler.leaderElect }}
+            - --leader-elect-resource-name={{ .Values.schedulerName }}
+            - --leader-elect-resource-namespace={{ include "hami-vgpu.namespace" . }}
+          resources:
+          {{- toYaml .Values.scheduler.kubeScheduler.resources | nindent 12 }}
+          volumeMounts:
+            - name: scheduler-config
+              mountPath: /config
+        {{- end }}
+          {{- if .Values.scheduler.livenessProbe }}
+          livenessProbe:
+            failureThreshold: 8
+            httpGet:
+              path: /healthz
+              port: 10259
+              scheme: HTTPS
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 15
+          {{- end }}
+        - name: vgpu-scheduler-extender
+          image: {{ include "hami.scheduler.extender.image" . }}
+          imagePullPolicy: {{ .Values.scheduler.extender.image.pullPolicy }}
+          env:
+          {{- if .Values.scheduler.nodeLockExpire }}
+            - name: HAMI_NODELOCK_EXPIRE
+              value: "{{ .Values.scheduler.nodeLockExpire }}"
+          {{- end }}
+          {{- if .Values.global.managedNodeSelectorEnable }}
+          {{- range $key, $value := .Values.global.managedNodeSelector }}
+            - name: NODE_SELECTOR_{{ $key | upper | replace "-" "_" }}
+              value: "{{ $value }}"
+          {{- end }}
+          {{- end }}
+          command:
+            - scheduler
+            {{- if .Values.scheduler.admissionWebhook.enabled }}
+            - --http_bind=0.0.0.0:443
+            - --cert_file=/tls/tls.crt
+            - --key_file=/tls/tls.key
+            {{- else }}
+            - --http_bind=0.0.0.0:80
+            {{- end }}
+            - --scheduler-name={{ .Values.schedulerName }}
+            - --metrics-bind-address={{ .Values.scheduler.metricsBindAddress }}
+            - --node-scheduler-policy={{ .Values.scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy }}
+            - --gpu-scheduler-policy={{ .Values.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy }}
+            - --force-overwrite-default-scheduler={{ .Values.scheduler.forceOverwriteDefaultScheduler}}
+            - --device-config-file=/device-config.yaml
+            - --leader-elect={{ .Values.scheduler.leaderElect }}
+            - --leader-elect-resource-name={{ .Values.schedulerName }}
+            - --leader-elect-resource-namespace={{ include "hami-vgpu.namespace" . }}
+            {{- if .Values.devices.ascend.enabled }}
+            - --enable-ascend=true
+            {{- end }}
+            {{- if .Values.devices.iluvatar.enabled }}
+            - --enable-iluvatar=true
+            {{- end }}
+            {{- if .Values.scheduler.nodeLabelSelector }}
+            - --node-label-selector={{- $first := true -}}
+              {{- range $key, $value := .Values.scheduler.nodeLabelSelector -}}
+              {{- if not $first }},{{ end -}}
+              {{- $key }}={{ $value -}}
+              {{- $first = false -}}
+              {{- end -}}
+            {{- end }}
+            {{- range .Values.scheduler.extender.extraArgs }}
+            - {{ . }}
+            {{- end }}
+          ports:
+            {{- if .Values.scheduler.admissionWebhook.enabled }}
+            - name: https
+              containerPort: 443
+              protocol: TCP
+            {{- else }}
+            - name: http
+              containerPort: 80
+              protocol: TCP
+            {{- end }}
+            - name: metrics
+              containerPort: {{ last (splitList ":" .Values.scheduler.metricsBindAddress) | int }}
+              protocol: TCP
+          resources:
+          {{- toYaml .Values.scheduler.extender.resources | nindent 12 }}
+          volumeMounts:
+            - name: device-config
+              mountPath: /device-config.yaml
+              subPath: device-config.yaml
+          {{- if .Values.scheduler.admissionWebhook.enabled }}
+            - name: tls-config
+              mountPath: /tls
+          {{- end }}
+          {{- if .Values.scheduler.livenessProbe }}
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              {{- if .Values.scheduler.admissionWebhook.enabled }}
+              port: https
+              scheme: HTTPS
+              {{- else }}
+              port: http
+              scheme: HTTP
+              {{- end }}
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            failureThreshold: 3
+            timeoutSeconds: 5
+          {{- end }}
+          {{- if .Values.scheduler.leaderElect }}
+          readinessProbe:
+            httpGet:
+              path: /readyz
+              {{- if .Values.scheduler.admissionWebhook.enabled }}
+              port: https
+              scheme: HTTPS
+              {{- else }}
+              port: http
+              scheme: HTTP
+              {{- end }}
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            failureThreshold: 3
+            timeoutSeconds: 5
+          {{- end }}
+      {{- if .Values.scheduler.leaderElect }}
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+              - key: app.kubernetes.io/component
+                operator: In
+                values:
+                - hami-scheduler
+            topologyKey: "kubernetes.io/hostname"
+      {{- end }}
+      volumes:
+        {{- if .Values.scheduler.admissionWebhook.enabled }}
+        - name: tls-config
+          secret:
+            secretName: {{ template "hami-vgpu.scheduler.tls" . }}
+        {{- end }}
+        {{- if .Values.scheduler.kubeScheduler.enabled }}
+        - name: scheduler-config
+          configMap:
+            {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
+            name: {{ template "hami-vgpu.scheduler" . }}-newversion
+            {{- else }}
+            name: {{ template "hami-vgpu.scheduler" . }}
+            {{- end }}
+        {{- end }}
+        - name: device-config
+          configMap:
+            name: {{ include "hami-vgpu.scheduler" . }}-device
+      {{- if .Values.scheduler.nodeSelector }}
+      nodeSelector: {{ toYaml .Values.scheduler.nodeSelector | nindent 8 }}
+      {{- end }}
+      {{- if .Values.scheduler.tolerations }}
+      tolerations: {{ toYaml .Values.scheduler.tolerations | nindent 8 }}
+      {{- end }}
+      {{- if .Values.scheduler.nodeName }}
+      nodeName: {{ .Values.scheduler.nodeName }}
+      {{- end }}
+{{- end }}
\ No newline at end of file
diff --git a/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml b/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml
new file mode 100644
index 00000000..ccb945ac
--- /dev/null
+++ b/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml
@@ -0,0 +1,410 @@
+{{- if not .Values.dra.enabled -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "hami-vgpu.scheduler" . }}-device
+  namespace: {{ include "hami-vgpu.namespace" . }}
+  labels:
+    app.kubernetes.io/component: hami-scheduler
+    {{- include "hami-vgpu.labels" . | nindent 4 }}
+data:
+  device-config.yaml: |-
+  {{- if .Files.Glob "files/device-config.yaml" }}
+  {{- .Files.Get "files/device-config.yaml" | nindent 4}}
+  {{- else }}
+    nvidia:
+      resourceCountName: {{ .Values.resourceName }}
+      resourceMemoryName: {{ .Values.resourceMem }}
+      resourceMemoryPercentageName: {{ .Values.resourceMemPercentage }}
+      resourceCoreName: {{ .Values.resourceCores }}
+      resourcePriorityName: {{ .Values.resourcePriority }}
+      overwriteEnv: false
+      defaultMemory: 0
+      defaultCores: 0
+      defaultGPUNum: 1
+      memoryFactor: 1
+      deviceSplitCount: {{ .Values.devicePlugin.deviceSplitCount }}
+      deviceMemoryScaling: {{ .Values.devicePlugin.deviceMemoryScaling }}
+      deviceCoreScaling: {{ .Values.devicePlugin.deviceCoreScaling }}
+      gpuCorePolicy: {{ .Values.devices.nvidia.gpuCorePolicy }}
+      libCudaLogLevel: {{ .Values.devices.nvidia.libCudaLogLevel }}
+      runtimeClassName: "{{ .Values.devicePlugin.runtimeClassName }}"
+      knownMigGeometries:
+      - models: [ "A30" ]
+        allowedGeometries:
+          - 
+            - name: 1g.6gb
+              core: 25
+              memory: 6144
+              count: 4
+          - 
+            - name: 2g.12gb
+              core: 50
+              memory: 12288
+              count: 2
+          - 
+            - name: 4g.24gb
+              core: 100
+              memory: 24576
+              count: 1
+      - models: [ "A100-SXM4-40GB", "A100-40GB-PCIe", "A100-PCIE-40GB"]
+        allowedGeometries:
+          - 
+            - name: 1g.5gb
+              core: 14
+              memory: 5120
+              count: 7
+          - 
+            - name: 1g.5gb
+              core: 14
+              memory: 5120
+              count: 1
+            - name: 2g.10gb
+              core: 28
+              memory: 10240
+              count: 3
+          - 
+            - name: 3g.20gb
+              core: 42
+              memory: 20480
+              count: 2
+          - 
+            - name: 7g.40gb
+              core: 100
+              memory: 40960
+              count: 1
+      - models: [ "A100-SXM4-80GB", "A100-80GB-PCIe", "A100-PCIE-80GB"]
+        allowedGeometries:
+          - 
+            - name: 1g.10gb
+              core: 14
+              memory: 10240
+              count: 7
+          - 
+            - name: 1g.10gb
+              core: 14
+              memory: 10240
+              count: 1
+            - name: 2g.20gb
+              core: 28
+              memory: 20480
+              count: 3
+          - 
+            - name: 3g.40gb
+              core: 42
+              memory: 40960
+              count: 2
+          - 
+            - name: 7g.79gb
+              core: 100
+              memory: 80896
+              count: 1
+      - models: [ "H100-PCIE-80GB", "H100-SXM5-80GB"]
+        allowedGeometries:
+          - 
+            - name: 1g.10gb
+              core: 14
+              memory: 10240
+              count: 7
+          - 
+            - name: 1g.10gb
+              core: 14
+              memory: 10240
+              count: 1
+            - name: 2g.20gb
+              core: 28
+              memory: 20480
+              count: 3
+          - 
+            - name: 3g.40gb
+              core: 42
+              memory: 40960
+              count: 2
+          - 
+            - name: 7g.80gb
+              core: 100
+              memory: 81920
+              count: 1
+      - models: [ "H100-PCIE-94GB", "H100-SXM5-94GB"]
+        allowedGeometries:
+          - 
+            - name: 1g.12gb
+              core: 14
+              memory: 12288
+              count: 7
+          - 
+            - name: 1g.12gb
+              core: 14
+              memory: 12288
+              count: 1
+            - name: 2g.24gb
+              core: 28
+              memory: 24576
+              count: 3
+          - 
+            - name: 3g.47gb
+              core: 42
+              memory: 48128
+              count: 2
+          - 
+            - name: 7g.94gb
+              core: 100
+              memory: 96256
+              count: 1
+      - models: [ "H20", "H100 on GH200"]
+        allowedGeometries:
+          - 
+            - name: 1g.12gb
+              core: 14
+              memory: 12288
+              count: 7
+          - 
+            - name: 1g.12gb
+              core: 14
+              memory: 12288
+              count: 1
+            - name: 2g.24gb
+              core: 28
+              memory: 24576
+              count: 3 
+          - 
+            - name: 3g.48gb
+              core: 42
+              memory: 49152
+              count: 2
+          - 
+            - name: 7g.96gb
+              core: 100
+              memory: 98304
+              count: 1
+      - models: [ "H200 NVL", "H200-SXM5"]
+        allowedGeometries:
+          - 
+            - name: 1g.18gb
+              core: 14
+              memory: 18432
+              count: 7
+          - 
+            - name: 1g.18gb
+              core: 14
+              memory: 18432
+              count: 1
+            - name: 2g.35gb
+              core: 28
+              memory: 35840
+              count: 3
+          - 
+            - name: 3g.71gb
+              core: 42
+              memory: 72704
+              count: 2
+          - 
+            - name: 7g.141gb
+              core: 100
+              memory: 144384
+              count: 1
+      - models: [ "B200" ]
+        allowedGeometries:
+          - 
+            - name: 1g.23gb
+              core: 14
+              memory: 23552
+              count: 7
+          - 
+            - name: 1g.23gb
+              core: 14
+              memory: 23552
+              count: 1
+            - name: 2g.45gb
+              core: 28
+              memory: 46080
+              count: 3
+          - 
+            - name: 3g.90gb
+              core: 42
+              memory: 92160
+              count: 2
+          - 
+            - name: 7g.180gb
+              core: 100
+              memory: 184320
+              count: 1
+    cambricon:
+      resourceCountName: {{ .Values.mluResourceName }}
+      resourceMemoryName: {{ .Values.mluResourceMem }}
+      resourceCoreName: {{ .Values.mluResourceCores }}
+    hygon:
+      resourceCountName: {{ .Values.dcuResourceName }}
+      resourceMemoryName: {{ .Values.dcuResourceMem }}
+      resourceCoreName: {{ .Values.dcuResourceCores }}
+      memoryFactor: 1
+    metax:
+      resourceCountName: "metax-tech.com/gpu"
+      resourceVCountName: {{ .Values.metaxResourceName }}
+      resourceVMemoryName: {{ .Values.metaxResourceMem }}
+      resourceVCoreName: {{ .Values.metaxResourceCore }}
+      sgpuTopologyAware: {{ .Values.metaxsGPUTopologyAware }}
+    enflame:
+      resourceNameGCU: "enflame.com/gcu"
+      resourceNameVGCU: {{ .Values.enflameResourceNameVGCU }}
+      resourceNameVGCUPercentage: {{ .Values.enflameResourceNameVGCUPercentage }}
+    mthreads:
+      resourceCountName: "mthreads.com/vgpu"
+      resourceMemoryName: "mthreads.com/sgpu-memory"
+      resourceCoreName: "mthreads.com/sgpu-core"
+    iluvatars:
+    - chipName: MR-V100
+      commonWord: MR-V100
+      resourceCountName: iluvatar.ai/MR-V100-vgpu
+      resourceMemoryName: iluvatar.ai/MR-V100.vMem
+      resourceCoreName: iluvatar.ai/MR-V100.vCore
+    - chipName: MR-V50
+      commonWord: MR-V50
+      resourceCountName: iluvatar.ai/MR-V50-vgpu
+      resourceMemoryName: iluvatar.ai/MR-V50.vMem
+      resourceCoreName: iluvatar.ai/MR-V50.vCore
+    - chipName: BI-V150
+      commonWord: BI-V150
+      resourceCountName: iluvatar.ai/BI-V150-vgpu
+      resourceMemoryName: iluvatar.ai/BI-V150.vMem
+      resourceCoreName: iluvatar.ai/BI-V150.vCore
+    - chipName: BI-V100
+      commonWord: BI-V100
+      resourceCountName: iluvatar.ai/BI-V100-vgpu
+      resourceMemoryName: iluvatar.ai/BI-V100.vMem
+      resourceCoreName: iluvatar.ai/BI-V100.vCore
+    kunlun:
+      resourceCountName: {{ .Values.kunlunResourceName }}
+      resourceVCountName: {{ .Values.kunlunResourceVCountName }}
+      resourceVMemoryName: {{ .Values.kunlunResourceVMemoryName }}
+    awsneuron:
+      resourceCountName: "aws.amazon.com/neuron"
+      resourceCoreName: "aws.amazon.com/neuroncore"
+    amd:
+      resourceCountName: "amd.com/gpu"
+    vnpus:
+    - chipName: 910A
+      commonWord: Ascend910A
+      resourceName: huawei.com/Ascend910A
+      resourceMemoryName: huawei.com/Ascend910A-memory
+      memoryAllocatable: 32768
+      memoryCapacity: 32768
+      memoryFactor: 1
+      aiCore: 30
+      templates:
+        - name: vir02
+          memory: 2184
+          aiCore: 2
+        - name: vir04
+          memory: 4369
+          aiCore: 4
+        - name: vir08
+          memory: 8738
+          aiCore: 8
+        - name: vir16
+          memory: 17476
+          aiCore: 16
+    - chipName: 910B2
+      commonWord: Ascend910B2
+      resourceName: huawei.com/Ascend910B2
+      resourceMemoryName: huawei.com/Ascend910B2-memory
+      memoryAllocatable: 65536
+      memoryCapacity: 65536
+      memoryFactor: 1
+      aiCore: 24
+      aiCPU: 6
+      templates:
+        - name: vir03_1c_8g
+          memory: 8192
+          aiCore: 3
+          aiCPU: 1
+        - name: vir06_1c_16g
+          memory: 16384
+          aiCore: 6
+          aiCPU: 1
+        - name: vir12_3c_32g
+          memory: 32768
+          aiCore: 12
+          aiCPU: 3
+    - chipName: 910B3
+      commonWord: Ascend910B3
+      resourceName: huawei.com/Ascend910B3
+      resourceMemoryName: huawei.com/Ascend910B3-memory
+      memoryAllocatable: 65536
+      memoryCapacity: 65536
+      memoryFactor: 1
+      aiCore: 20
+      aiCPU: 7
+      templates:
+        - name: vir05_1c_16g
+          memory: 16384
+          aiCore: 5
+          aiCPU: 1
+        - name: vir10_3c_32g
+          memory: 32768
+          aiCore: 10
+          aiCPU: 3
+    - chipName: 910B4-1
+      commonWord: Ascend910B4-1
+      resourceName: huawei.com/Ascend910B4-1
+      resourceMemoryName: huawei.com/Ascend910B4-1-memory
+      memoryAllocatable: 65536
+      memoryCapacity: 65536
+      memoryFactor: 1
+      aiCore: 20
+      aiCPU: 7
+      templates:
+        # NOTE: Names of vnpu template for 910B4-1 are fixed by Ascend runtime and must not be changed.
+        # The memory is used for scheduling so the correct values must be set.
+        # Template vir05_1c_8g actually provides 16GB memory,
+        - name: vir05_1c_8g
+          memory: 16384
+          aiCore: 5
+          aiCPU: 1
+        # Template vir10_3c_16g actually provides 32GB memory
+        - name: vir10_3c_16g
+          memory: 32768
+          aiCore: 10
+          aiCPU: 3
+    - chipName: 910B4
+      commonWord: Ascend910B4
+      resourceName: huawei.com/Ascend910B4
+      resourceMemoryName: huawei.com/Ascend910B4-memory
+      memoryAllocatable: 32768
+      memoryCapacity: 32768
+      memoryFactor: 1
+      aiCore: 20
+      aiCPU: 7
+      templates:
+        - name: vir05_1c_8g
+          memory: 8192
+          aiCore: 5
+          aiCPU: 1
+        - name: vir10_3c_16g
+          memory: 16384
+          aiCore: 10
+          aiCPU: 3
+    - chipName: 310P3
+      commonWord: Ascend310P
+      resourceName: huawei.com/Ascend310P
+      resourceMemoryName: huawei.com/Ascend310P-memory
+      memoryAllocatable: 21527
+      memoryCapacity: 24576
+      memoryFactor: 1
+      aiCore: 8
+      aiCPU: 7
+      templates:
+        - name: vir01
+          memory: 3072
+          aiCore: 1
+          aiCPU: 1
+        - name: vir02
+          memory: 6144
+          aiCore: 2
+          aiCPU: 2
+        - name: vir04
+          memory: 12288
+          aiCore: 4
+          aiCPU: 4
+  {{ end }}
+{{- end }}
\ No newline at end of file
diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml
new file mode 100644
index 00000000..b089ae82
--- /dev/null
+++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml
@@ -0,0 +1,30 @@
+{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}}
+{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "hami-vgpu.fullname" . }}-admission
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+  labels:
+    {{- include "hami-vgpu.labels" . | nindent 4 }}
+    app.kubernetes.io/component: admission-webhook
+rules:
+  - apiGroups:
+      - admissionregistration.k8s.io
+    resources:
+      #- validatingwebhookconfigurations
+      - mutatingwebhookconfigurations
+    verbs:
+      - get
+      - update
+{{- if .Values.podSecurityPolicy.enabled }}
+  - apiGroups: ['extensions']
+    resources: ['podsecuritypolicies']
+    verbs:     ['use']
+    resourceNames:
+    - {{ include "hami-vgpu.fullname" . }}-admission
+{{- end }}
+{{- end }}
+{{- end }}
diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml
new file mode 100644
index 00000000..64f01ecf
--- /dev/null
+++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml
@@ -0,0 +1,22 @@
+{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}}
+{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name:  {{ include "hami-vgpu.fullname" . }}-admission
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+  labels:
+    {{- include "hami-vgpu.labels" . | nindent 4 }}
+    app.kubernetes.io/component: admission-webhook
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "hami-vgpu.fullname" . }}-admission
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "hami-vgpu.fullname" . }}-admission
+    namespace: {{ include "hami-vgpu.namespace" . }}
+{{- end }}
+{{- end }}
diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml
new file mode 100644
index 00000000..400168a7
--- /dev/null
+++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml
@@ -0,0 +1,70 @@
+{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}}
+{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "hami-vgpu.fullname" . }}-admission-create
+  namespace: {{ include "hami-vgpu.namespace" . }}
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+  labels:
+    {{- include "hami-vgpu.labels" . | nindent 4 }}
+    app.kubernetes.io/component: admission-webhook
+spec:
+  {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }}
+  # Alpha feature since k8s 1.12
+  ttlSecondsAfterFinished: 0
+  {{- end }}
+  template:
+    metadata:
+      name: {{ include "hami-vgpu.fullname" . }}-admission-create
+      {{- if .Values.scheduler.patch.podAnnotations }}
+      annotations: {{ toYaml .Values.scheduler.patch.podAnnotations | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "hami-vgpu.labels" . | nindent 8 }}
+        app.kubernetes.io/component: admission-webhook
+        hami.io/webhook: ignore
+    spec:
+      {{- if .Values.scheduler.patch.priorityClassName }}
+      priorityClassName: {{ .Values.scheduler.patch.priorityClassName }}
+      {{- end }}
+      {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
+        {{- include "hami.scheduler.patch.new.imagePullSecrets" . | nindent 6 }}
+      {{- else }}
+        {{- include "hami.scheduler.patch.imagePullSecrets" . | nindent 6 }}
+      {{- end }}
+      containers:
+        - name: create
+          {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
+          image: {{ include "hami.scheduler.patch.new.image" . }}
+          imagePullPolicy: {{ .Values.scheduler.patch.imageNew.pullPolicy }}
+          {{- else }}
+          image: {{ include "hami.scheduler.patch.image" . }}
+          imagePullPolicy: {{ .Values.scheduler.patch.image.pullPolicy }}
+          {{- end }}
+          args:
+            - create
+            - --cert-name=tls.crt
+            - --key-name=tls.key
+            {{- if .Values.scheduler.admissionWebhook.customURL.enabled }}
+            - --host={{ printf "%s.%s.svc,127.0.0.1,%s" (include "hami-vgpu.scheduler" .) (include "hami-vgpu.namespace" .) .Values.scheduler.admissionWebhook.customURL.host}}
+            {{- else }}
+            - --host={{ printf "%s.%s.svc,127.0.0.1" (include "hami-vgpu.scheduler" .) (include "hami-vgpu.namespace" .) }}
+            {{- end }}
+            - --namespace={{ include "hami-vgpu.namespace" . }}
+            - --secret-name={{ include "hami-vgpu.scheduler.tls" . }}
+      restartPolicy: OnFailure
+      serviceAccountName: {{ include "hami-vgpu.fullname" . }}-admission
+      {{- if .Values.scheduler.patch.nodeSelector }}
+      nodeSelector: {{ toYaml .Values.scheduler.patch.nodeSelector | nindent 8 }}
+      {{- end }}
+      {{- if .Values.scheduler.patch.tolerations }}
+      tolerations: {{ toYaml .Values.scheduler.patch.tolerations | nindent 8 }}
+      {{- end }}
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: {{ .Values.scheduler.patch.runAsUser }}
+{{- end }}
+{{- end }}
diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml
new file mode 100644
index 00000000..71e97f6b
--- /dev/null
+++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml
@@ -0,0 +1,65 @@
+{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}}
+{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "hami-vgpu.fullname" . }}-admission-patch
+  namespace: {{ include "hami-vgpu.namespace" . }}
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+  labels:
+    {{- include "hami-vgpu.labels" . | nindent 4 }}
+    app.kubernetes.io/component: admission-webhook
+spec:
+  {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }}
+  # Alpha feature since k8s 1.12
+  ttlSecondsAfterFinished: 0
+  {{- end }}
+  template:
+    metadata:
+      name: {{ include "hami-vgpu.fullname" . }}-admission-patch
+      {{- if .Values.scheduler.patch.podAnnotations }}
+      annotations: {{ toYaml .Values.scheduler.patch.podAnnotations | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "hami-vgpu.labels" . | nindent 8 }}
+        app.kubernetes.io/component: admission-webhook
+        hami.io/webhook: ignore
+    spec:
+      {{- if .Values.scheduler.patch.priorityClassName }}
+      priorityClassName: {{ .Values.scheduler.patch.priorityClassName }}
+      {{- end }}
+      {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
+        {{- include "hami.scheduler.patch.new.imagePullSecrets" . | nindent 6 }}
+      {{- else }}
+        {{- include "hami.scheduler.patch.imagePullSecrets" . | nindent 6 }}
+      {{- end }}
+      containers:
+        - name: patch
+          {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
+          image: {{ include "hami.scheduler.patch.new.image" . }}
+          imagePullPolicy: {{ .Values.scheduler.patch.imageNew.pullPolicy }}
+          {{- else }}
+          image: {{ include "hami.scheduler.patch.image" . }}
+          imagePullPolicy: {{ .Values.scheduler.patch.image.pullPolicy }}
+          {{- end }}
+          args:
+            - patch
+            - --webhook-name={{ include "hami-vgpu.scheduler.webhook" . }}
+            - --namespace={{ include "hami-vgpu.namespace" . }}
+            - --patch-validating=false
+            - --secret-name={{ include "hami-vgpu.scheduler.tls" . }}
+      restartPolicy: OnFailure
+      serviceAccountName: {{ include "hami-vgpu.fullname" . }}-admission
+      {{- if .Values.scheduler.patch.nodeSelector }}
+      nodeSelector: {{ toYaml .Values.scheduler.patch.nodeSelector | nindent 8 }}
+      {{- end }}
+      {{- if .Values.scheduler.patch.tolerations }}
+      tolerations: {{ toYaml .Values.scheduler.patch.tolerations | nindent 8 }}
+      {{- end }}
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: {{ .Values.scheduler.patch.runAsUser }}
+{{- end }}
+{{- end }}
diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml
new file mode 100644
index 00000000..a48270e7
--- /dev/null
+++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml
@@ -0,0 +1,40 @@
+{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}}
+{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
+{{- if .Values.podSecurityPolicy.enabled }}
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: {{ include "hami-vgpu.fullname" . }}-admission
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+  labels:
+    {{- include "hami-vgpu.labels" . | nindent 4 }}
+    app.kubernetes.io/component: admission-webhook
+spec:
+  allowPrivilegeEscalation: false
+  fsGroup:
+    ranges:
+    - max: 65535
+      min: 1
+    rule: MustRunAs
+  requiredDropCapabilities:
+  - ALL
+  runAsUser:
+    rule: MustRunAsNonRoot
+  seLinux:
+    rule: RunAsAny
+  supplementalGroups:
+    ranges:
+    - max: 65535
+      min: 1
+    rule: MustRunAs
+  volumes:
+  - configMap
+  - emptyDir
+  - projected
+  - secret
+  - downwardAPI
+{{- end }}
+{{- end }}
+{{- end }}
diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml
new file mode 100644
index 00000000..74e45681
--- /dev/null
+++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml
@@ -0,0 +1,23 @@
+{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}}
+{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name:  {{ include "hami-vgpu.fullname" . }}-admission
+  namespace: {{ include "hami-vgpu.namespace" . }}
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+  labels:
+    {{- include "hami-vgpu.labels" . | nindent 4 }}
+    app.kubernetes.io/component: admission-webhook
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - get
+      - create
+{{- end }}
+{{- end }}
diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml
new file mode 100644
index 00000000..32da5ba4
--- /dev/null
+++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml
@@ -0,0 +1,23 @@
+{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}}
+{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ include "hami-vgpu.fullname" . }}-admission
+  namespace: {{ include "hami-vgpu.namespace" . }}
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+  labels:
+    {{- include "hami-vgpu.labels" . | nindent 4 }}
+    app.kubernetes.io/component: admission-webhook
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ include "hami-vgpu.fullname" . }}-admission
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "hami-vgpu.fullname" . }}-admission
+    namespace: {{ include "hami-vgpu.namespace" . }}
+{{- end }}
+{{- end }}
diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml
new file mode 100644
index 00000000..d2eb41de
--- /dev/null
+++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml
@@ -0,0 +1,15 @@
+{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) }}
+{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "hami-vgpu.fullname" . }}-admission
+  namespace: {{ include "hami-vgpu.namespace" . }}
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+  labels:
+    {{- include "hami-vgpu.labels" . | nindent 4 }}
+    app.kubernetes.io/component: admission-webhook
+{{- end }}
+{{- end }}
diff --git a/packages/system/hami/charts/hami/templates/scheduler/role.yaml b/packages/system/hami/charts/hami/templates/scheduler/role.yaml
new file mode 100644
index 00000000..6e681967
--- /dev/null
+++ b/packages/system/hami/charts/hami/templates/scheduler/role.yaml
@@ -0,0 +1,14 @@
+{{- if not .Values.dra.enabled -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ include "hami-vgpu.scheduler" . }}
+  namespace: {{ include "hami-vgpu.namespace" . }}
+  labels:
+    app.kubernetes.io/component: "hami-scheduler"
+    {{- include "hami-vgpu.labels" . | nindent 4 }}
+rules:
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["create", "list", "watch", "get", "update", "patch"]
+{{- end }}
\ No newline at end of file
diff --git a/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml
new file mode 100644
index 00000000..934f8223
--- /dev/null
+++ b/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml
@@ -0,0 +1,33 @@
+{{- if not .Values.dra.enabled -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ include "hami-vgpu.scheduler" . }}
+  namespace: {{ include "hami-vgpu.namespace" . }}
+  labels:
+    app.kubernetes.io/component: "hami-scheduler"
+    {{- include "hami-vgpu.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ include "hami-vgpu.scheduler" . }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "hami-vgpu.scheduler" . }}
+    namespace: {{ include "hami-vgpu.namespace" . }}
+---
+{{- if .Values.mockDevicePlugin.enabled }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "hami-vgpu.mock-device-plugin" . }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "hami-vgpu.mock-device-plugin" . }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "hami-vgpu.mock-device-plugin" . }}
+    namespace: {{ include "hami-vgpu.namespace" . }}
+{{- end -}}
+{{- end }}
diff --git a/packages/system/hami/charts/hami/templates/scheduler/service.yaml b/packages/system/hami/charts/hami/templates/scheduler/service.yaml
new file mode 100644
index 00000000..3e18bc98
--- /dev/null
+++ b/packages/system/hami/charts/hami/templates/scheduler/service.yaml
@@ -0,0 +1,36 @@
+{{- if not .Values.dra.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "hami-vgpu.scheduler" . }}
+  namespace: {{ include "hami-vgpu.namespace" . }}
+  labels:
+    app.kubernetes.io/component: hami-scheduler
+    {{- include "hami-vgpu.labels" . | nindent 4 }}
+    {{- if .Values.scheduler.service.labels }}
+    {{ toYaml .Values.scheduler.service.labels | indent 4 }}
+    {{- end }}
+  {{- if .Values.scheduler.service.annotations }}
+  annotations: {{ toYaml .Values.scheduler.service.annotations | nindent 4 }}
+  {{- end }}
+spec:
+  type: {{ .Values.scheduler.service.type | default "NodePort" }}  # Default type is NodePort
+  ports:
+    - name: http
+      port: {{ .Values.scheduler.service.httpPort | default 443 }}  # Default HTTP port is 443
+      targetPort: {{ .Values.scheduler.service.httpTargetPort | default 443 }}
+      {{- if eq (.Values.scheduler.service.type | default "NodePort") "NodePort" }}  # If type is NodePort, set nodePort
+      nodePort: {{ .Values.scheduler.service.schedulerPort | default 31998 }}
+      {{- end }}
+      protocol: TCP
+    - name: monitor
+      port: {{ .Values.scheduler.service.monitorPort | default 31993 }}  # Default monitoring port is 31993
+      targetPort: {{ .Values.scheduler.service.monitorTargetPort | default 9395 }}
+      {{- if eq (.Values.scheduler.service.type | default "NodePort") "NodePort" }}  # If type is NodePort, set nodePort
+      nodePort: {{ .Values.scheduler.service.monitorPort | default 31993 }}
+      {{- end }}
+      protocol: TCP
+  selector:
+    app.kubernetes.io/component: hami-scheduler
+    {{- include "hami-vgpu.selectorLabels" . | nindent 4 }}
+{{- end }}
\ No newline at end of file
diff --git a/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml b/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml
new file mode 100644
index 00000000..929bf59a
--- /dev/null
+++ b/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml
@@ -0,0 +1,18 @@
+{{- if not .Values.dra.enabled -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "hami-vgpu.scheduler" . }}
+  namespace: {{ include "hami-vgpu.namespace" . }}
+  labels:
+    app.kubernetes.io/component: "hami-scheduler"
+    {{- include "hami-vgpu.labels" . | nindent 4 }}
+---
+{{- if .Values.mockDevicePlugin.enabled }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "hami-vgpu.mock-device-plugin" . }}
+  namespace: {{ include "hami-vgpu.namespace" . }}
+{{- end -}}
+{{- end }}
diff --git a/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml b/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml
new file mode 100644
index 00000000..148feb5c
--- /dev/null
+++ b/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml
@@ -0,0 +1,57 @@
+{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}}
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  {{- if .Values.scheduler.certManager.enabled }}
+  annotations:
+    cert-manager.io/inject-ca-from: {{ include "hami-vgpu.namespace" . }}/{{ include "hami-vgpu.scheduler" . }}-serving-cert
+  {{- end }}
+  name: {{ include "hami-vgpu.scheduler.webhook" . }}
+webhooks:
+  - admissionReviewVersions:
+    - v1beta1
+    clientConfig:
+      {{- if .Values.scheduler.admissionWebhook.customURL.enabled }}
+      url: https://{{ .Values.scheduler.admissionWebhook.customURL.host}}:{{.Values.scheduler.admissionWebhook.customURL.port}}{{.Values.scheduler.admissionWebhook.customURL.path}}
+      {{- else }}
+      service:
+        name: {{ include "hami-vgpu.scheduler" . }}
+        namespace: {{ include "hami-vgpu.namespace" . }}
+        path: /webhook
+        port: {{ .Values.scheduler.service.httpPort }}
+      {{- end }}
+    failurePolicy: {{ .Values.scheduler.admissionWebhook.failurePolicy }}
+    matchPolicy: Equivalent
+    name: vgpu.hami.io
+    namespaceSelector:
+      matchExpressions:
+      - key: hami.io/webhook
+        operator: NotIn
+        values:
+        - ignore
+      {{- if .Values.scheduler.admissionWebhook.whitelistNamespaces }}
+      - key: kubernetes.io/metadata.name
+        operator: NotIn
+        values:
+        {{- toYaml .Values.scheduler.admissionWebhook.whitelistNamespaces | nindent 10 }}
+      {{- end }}
+    objectSelector:
+      matchExpressions:
+      - key: hami.io/webhook
+        operator: NotIn
+        values:
+        - ignore
+    reinvocationPolicy: {{ .Values.scheduler.admissionWebhook.reinvocationPolicy }}
+    rules:
+      - apiGroups:
+          - ""
+        apiVersions:
+          - v1
+        operations:
+          - CREATE
+        resources:
+          - pods
+        scope: '*'
+    sideEffects: None
+    timeoutSeconds: 10
+{{- end }}
\ No newline at end of file
diff --git a/packages/system/hami/charts/hami/values.yaml b/packages/system/hami/charts/hami/values.yaml
new file mode 100644
index 00000000..9fbeca70
--- /dev/null
+++ b/packages/system/hami/charts/hami/values.yaml
@@ -0,0 +1,476 @@
+## @section Global configuration
+## @param global.imageRegistry Global Docker image registry
+## @param global.imagePullSecrets Global Docker image pull secrets
+global:
+  ## @param global.imageRegistry Global Docker image registry
+  imageRegistry: ""
+  ## E.g.
+  ## imagePullSecrets:
+  ##   - myRegistryKeySecretName
+  ## @param global.imagePullSecrets Global Docker image pull secrets
+  imagePullSecrets: []
+  imageTag: "v2.8.1"
+  gpuHookPath: /usr/local
+  labels: {}
+  annotations: {}
+  managedNodeSelectorEnable: false
+  managedNodeSelector:
+    usage: "gpu"
+
+nameOverride: ""
+fullnameOverride: ""
+namespaceOverride: ""
+
+#Nvidia GPU Parameters
+resourceName: "nvidia.com/gpu"
+resourceMem: "nvidia.com/gpumem"
+resourceMemPercentage: "nvidia.com/gpumem-percentage"
+resourceCores: "nvidia.com/gpucores"
+resourcePriority: "nvidia.com/priority"
+
+#MLU Parameters
+mluResourceName: "cambricon.com/vmlu"
+mluResourceMem: "cambricon.com/mlu.smlu.vmemory"
+mluResourceCores: "cambricon.com/mlu.smlu.vcore"
+
+#Hygon DCU Parameters
+dcuResourceName: "hygon.com/dcunum"
+dcuResourceMem: "hygon.com/dcumem"
+dcuResourceCores: "hygon.com/dcucores"
+
+#Metax sGPU Parameters
+metaxResourceName: "metax-tech.com/sgpu"
+metaxResourceCore: "metax-tech.com/vcore"
+metaxResourceMem: "metax-tech.com/vmemory"
+metaxsGPUTopologyAware: "false"
+
+#Enflame VGCU Parameters
+enflameResourceNameVGCU: "enflame.com/vgcu"
+enflameResourceNameVGCUPercentage: "enflame.com/vgcu-percentage"
+
+#Kunlun XPU Parameters
+kunlunResourceName: "kunlunxin.com/xpu"
+kunlunResourceVCountName: "kunlunxin.com/vxpu"
+kunlunResourceVMemoryName: "kunlunxin.com/vxpu-memory"
+
+schedulerName: "hami-scheduler"
+
+podSecurityPolicy:
+  enabled: false
+
+scheduler:
+  # @param nodeName defines the node name and the nvidia-vgpu-scheduler-scheduler will schedule to the node.
+  # if we install the nvidia-vgpu-scheduler-scheduler as default scheduler, we need to remove the k8s default
+  # scheduler pod from the cluster first, we must specify node name to skip the schedule workflow.
+  nodeName: ""
+  #nodeLabelSelector:
+  #  "gpu": "on"
+  overwriteEnv: "false"
+  defaultSchedulerPolicy:
+    nodeSchedulerPolicy: binpack
+    gpuSchedulerPolicy: spread
+  metricsBindAddress: ":9395"
+  # If set to false, When Pod.Spec.SchedulerName equals to the const DefaultSchedulerName in k8s.io/api/core/v1 package, webhook will not overwrite it
+  forceOverwriteDefaultScheduler: true
+  livenessProbe: false
+  leaderElect: true
+  # when leaderElect is true, replicas is available, otherwise replicas is 1.
+  replicas: 1
+  kubeScheduler:
+    # @param enabled indicate whether to run kube-scheduler container in the scheduler pod, it's true by default.
+    enabled: true
+    ## @param image.registry kube scheduler image registry
+    ## @param image.repository kube scheduler image repository
+    ## @param image.tag kube scheduler image tag (immutable tags are recommended)
+    ## @param image.pullPolicy kube scheduler image pull policy
+    ## @param image.pullSecrets Specify docker-registry secret names as an array
+    image:
+      registry: "registry.cn-hangzhou.aliyuncs.com"
+      repository: "google_containers/kube-scheduler"
+      tag: ""
+      ## Specify a imagePullPolicy
+      ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
+      ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
+      ##
+      pullPolicy: IfNotPresent
+      ## Optionally specify an array of imagePullSecrets.
+      ## Secrets must be manually created in the namespace.
+      ## Example:
+      ## pullSecrets:
+      ##   - myRegistryKeySecretName
+      ##
+      pullSecrets: []
+    resources: {}
+      # If you do want to specify resources, uncomment the following lines, adjust them as necessary.
+      # and remove the curly braces after 'resources:'.
+#      limits:
+#        cpu: 1000m
+#        memory: 1000Mi
+#      requests:
+#        cpu: 100m
+#        memory: 100Mi
+    extraNewArgs:
+      - --config=/config/config.yaml
+      - -v=4
+    extraArgs:
+      - --policy-config-file=/config/config.json
+      - -v=4
+  extender:
+    ## @param image.registry scheduler extender image registry
+    ## @param image.repository scheduler extender image repository
+    ## @param image.tag scheduler extender image tag (immutable tags are recommended)
+    ## @param image.pullPolicy scheduler extender image pull policy
+    ## @param image.pullSecrets Specify docker-registry secret names as an array
+    image:
+      registry: "docker.io"
+      repository: "projecthami/hami"
+      tag: ""
+      ## Specify a imagePullPolicy
+      ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
+      ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
+      ##
+      pullPolicy: IfNotPresent
+      ## Optionally specify an array of imagePullSecrets.
+      ## Secrets must be manually created in the namespace.
+      ## Example:
+      ## pullSecrets:
+      ##   - myRegistryKeySecretName
+      ##
+      pullSecrets: []
+    resources: {}
+      # If you do want to specify resources, uncomment the following lines, adjust them as necessary,
+      # and remove the curly braces after 'resources:'.
+#      limits:
+#        cpu: 1000m
+#        memory: 1000Mi
+#      requests:
+#        cpu: 100m
+#        memory: 100Mi
+    extraArgs:
+      - --debug
+      - -v=4
+  nodeLockExpire: "5m"
+  podAnnotations: {}
+  tolerations: []
+  #serviceAccountName: "hami-vgpu-scheduler-sa"
+  admissionWebhook:
+    # If set to false, the admission webhook is not installed and any pods that should use HAMi must be
+    # configured to use the right 'schedulerName' and other device-specific configurations.
+    enabled: true
+    customURL:
+      enabled: false
+      # must be an endpoint using https.
+      # should generate host certs here
+      host: 127.0.0.1 # hostname or ip, can be your node'IP if you want to use https://<nodeIP>:<schedulerPort>/<path>
+      port: 31998
+      path: /webhook
+    whitelistNamespaces:
+    # Specify the namespaces that the webhook will not be applied to.
+      # - default
+      # - kube-system
+      # - istio-system
+    reinvocationPolicy: Never
+    failurePolicy: Ignore
+  ## TLS Certificate Option 1: Use cert-manager to generate self-signed certificate.
+  ## If enabled, always takes precedence over options 2.
+  certManager:
+    enabled: false
+  ## TLS Certificate Option 2: Use kube-webhook-certgen to generate self-signed certificate.
+  ## If true and certManager.enabled is false, Helm will automatically create a self-signed cert and secret for you.
+  patch:
+    enabled: true
+    ## @param image.registry scheduler extender image registry
+    ## @param image.repository scheduler extender image repository
+    ## @param image.tag scheduler extender image tag (immutable tags are recommended)
+    ## @param image.pullPolicy scheduler extender image pull policy
+    ## @param image.pullSecrets Specify docker-registry secret names as an array
+    image:
+      registry: "docker.io"
+      repository: "jettech/kube-webhook-certgen"
+      tag: "v1.5.2"
+      ## Specify a imagePullPolicy
+      ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
+      ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
+      ##
+      pullPolicy: IfNotPresent
+      ## Optionally specify an array of imagePullSecrets.
+      ## Secrets must be manually created in the namespace.
+      ## Example:
+      ## pullSecrets:
+      ##   - myRegistryKeySecretName
+      ##
+      pullSecrets: []
+    ## @param image.registry scheduler extender image registry
+    ## @param image.repository scheduler extender image repository
+    ## @param image.tag scheduler extender image tag (immutable tags are recommended)
+    ## @param image.pullPolicy scheduler extender image pull policy
+    ## @param image.pullSecrets Specify docker-registry secret names as an array
+    imageNew: 
+      registry: "docker.io"
+      repository: "liangjw/kube-webhook-certgen"
+      tag: "v1.1.1"
+      ## Specify a imagePullPolicy
+      ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
+      ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
+      ##
+      pullPolicy: IfNotPresent
+      ## Optionally specify an array of imagePullSecrets.
+      ## Secrets must be manually created in the namespace.
+      ## Example:
+      ## pullSecrets:
+      ##   - myRegistryKeySecretName
+      ##
+      pullSecrets: []
+    priorityClassName: ""
+    podAnnotations: {}
+    nodeSelector: {}
+    tolerations: []
+    runAsUser: 2000
+  service:
+    type: NodePort  # Default type is NodePort, can be changed to ClusterIP
+    httpPort: 443   # HTTP port
+    schedulerPort: 31998  # NodePort for HTTP
+    monitorPort: 31993    # Monitoring port
+    monitorTargetPort: 9395
+    httpTargetPort: 443
+    labels: {}
+    annotations: {}
+
+devicePlugin:
+  enabled: true
+  gpuOperatorToolkitReady:
+    enabled: false
+    hostPath: "/run/nvidia/validations"
+  ## @param image.registry devicePlugin image registry
+  ## @param image.repository devicePlugin image repository
+  ## @param image.tag devicePlugin image tag (immutable tags are recommended)
+  ## @param image.pullPolicy devicePlugin image pull policy
+  ## @param image.pullSecrets Specify docker-registry secret names as an array
+  image:
+    registry: "docker.io"
+    repository: "projecthami/hami"
+    tag: ""
+    ## Specify a imagePullPolicy
+    ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
+    ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
+    ##
+    pullPolicy: IfNotPresent
+    ## Optionally specify an array of imagePullSecrets.
+    ## Secrets must be manually created in the namespace.
+    ## Example:
+    ## pullSecrets:
+    ##   - myRegistryKeySecretName
+    ##
+    pullSecrets: []
+  monitor:
+    ## @param image.registry monitor image registry
+    ## @param image.repository monitor image repository
+    ## @param image.tag monitor image tag (immutable tags are recommended)
+    ## @param image.pullPolicy monitor image pull policy
+    ## @param image.pullSecrets Specify docker-registry secret names as an array
+    image:
+      registry: "docker.io"
+      repository: "projecthami/hami"
+      tag: ""
+      ## Specify a imagePullPolicy
+      ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
+      ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
+      ##
+      pullPolicy: IfNotPresent
+      ## Optionally specify an array of imagePullSecrets.
+      ## Secrets must be manually created in the namespace.
+      ## Example:
+      ## pullSecrets:
+      ##   - myRegistryKeySecretName
+      ##
+      pullSecrets: []
+    ctrPath: /usr/local/vgpu/containers
+    resyncInterval: "5m"
+    extraArgs:
+      - -v=4
+    extraEnvs: {}
+    resources: {}
+      # If you do want to specify resources, uncomment the following lines, adjust them as necessary.
+      # and remove the curly braces after 'resources:'.
+#      limits:
+#        cpu: 1000m
+#        memory: 1000Mi
+#      requests:
+#        cpu: 100m
+#        memory: 100Mi
+
+  deviceSplitCount: 10
+  deviceMemoryScaling: 1
+  deviceCoreScaling: 1
+  # Node configuration for device plugin, Priority: externalConfigName > config > default config
+  nodeConfiguration:
+    # If you want to use a custom config.json, you can set the content here.
+    # If this is set, it will override the default config.json(An example is as follows).
+    config: |
+      {
+        "nodeconfig": [
+          {
+            "name": "your-node-name",
+            "operatingmode": "hami-core",
+            "devicememoryscaling": 1,
+            "devicesplitcount": 10,
+            "migstrategy": "none",
+            "filterdevices": {
+              "uuid": [],
+              "index": []
+            }
+          }
+        ]
+      }
+    # If you want to use an existing ConfigMap, you can set the name here.
+    # If this is set, the chart will not create the ConfigMap and will use the existing one.
+    externalConfigName: ""
+  # The runtime class name to be used by the device plugin, and added to the pod.spec.runtimeClassName of applications utilizing NVIDIA GPUs
+  runtimeClassName: ""
+  # Whether to create runtime class, name comes from runtimeClassName when it is set
+  createRuntimeClass: false
+  migStrategy: "none"
+  disablecorelimit: "false"
+  passDeviceSpecsEnabled: false
+  deviceListStrategy: "envvar"
+  nvidiaHookPath: null
+  nvidiaDriverRoot: null
+  gdrcopyEnabled: null
+  gdsEnabled: null
+  mofedEnabled: null
+
+  extraArgs:
+    - -v=4
+  extraEnvs: {}
+  service:
+    type: NodePort  # Default type is NodePort, can be changed to ClusterIP
+    httpPort: 31992
+    labels: {}
+    annotations: {}
+
+  pluginPath: /var/lib/kubelet/device-plugins
+  libPath: /usr/local/vgpu
+
+  podAnnotations: {}
+  nvidiaNodeSelector:
+    gpu: "on"
+  tolerations: []
+  # The updateStrategy for DevicePlugin DaemonSet.
+  # If you want to update the DaemonSet by manual, set type as "OnDelete".
+  # We recommend use OnDelete update strategy because DevicePlugin pod restart will cause business pod restart, this behavior is destructive.
+  # Otherwise, you can use RollingUpdate update strategy to rolling update DevicePlugin pod.
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+
+  resources: {}
+    # If you do want to specify resources, uncomment the following lines, adjust them as necessary.
+    # and remove the curly braces after 'resources:'.
+#    limits:
+#       cpu: 1000m
+#       memory: 1000Mi
+#    requests:
+#      cpu: 100m
+#      memory: 100Mi
+
+mockDevicePlugin:
+  enabled: false
+  image:
+    registry: "docker.io"
+    repository: "projecthami/mock-device-plugin"
+    tag: "1.0.1"
+    ## Specify a imagePullPolicy
+    ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
+    ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
+    ##
+    pullPolicy: IfNotPresent
+    ## Optionally specify an array of imagePullSecrets.
+    ## Secrets must be manually created in the namespace.
+    ## Example:
+    ## pullSecrets:
+    ##   - myRegistryKeySecretName
+    ##
+    pullSecrets: []
+# If this option is enabled, the DRA will be installed and the scheduler extender will not be installed.
+dra:
+  enabled: false
+
+# Configuration for the hami-dra subchart, which includes DRA drivers.
+# This is part of config for DRA, please refer to the DRA documentation for more details.
+hami-dra:
+  monitor:
+    enabled: true
+  drivers:
+    nvidia:
+      enabled: true
+      # If you are using gpu driver on host, you need to set this to false
+      containerDriver: true
+      image:
+        repository: projecthami/k8s-dra-driver
+        tag: "v0.0.1-dev"
+
+devices:
+  amd:
+    customresources:
+      - amd.com/gpu
+      - amd.com/gpu-memory
+  awsneuron:
+    customresources:
+      - aws.amazon.com/neuron
+      - aws.amazon.com/neuroncore
+  kunlun:
+    enabled: true
+    customresources:
+      - kunlunxin.com/xpu
+      - kunlunxin.com/vxpu
+      - kunlunxin.com/vxpu-memory
+  enflame:
+    enabled: true
+    customresources:
+      - enflame.com/vgcu
+      - enflame.com/vgcu-percentage
+      - enflame.com/gcu
+  mthreads:
+    enabled: true
+    customresources:
+      - mthreads.com/vgpu
+  nvidia:
+    gpuCorePolicy: default
+    libCudaLogLevel: 1
+  ascend:
+    enabled: false
+    image: ""
+    imagePullPolicy: IfNotPresent
+    extraArgs: []
+    nodeSelector:
+      ascend: "on"
+    tolerations: []
+    customresources:
+      - huawei.com/Ascend910A
+      - huawei.com/Ascend910A-memory
+      - huawei.com/Ascend910B2
+      - huawei.com/Ascend910B2-memory
+      - huawei.com/Ascend910B3
+      - huawei.com/Ascend910B3-memory
+      - huawei.com/Ascend910B4
+      - huawei.com/Ascend910B4-memory
+      - huawei.com/Ascend910B4-1
+      - huawei.com/Ascend910B4-1-memory
+      - huawei.com/Ascend310P
+      - huawei.com/Ascend310P-memory
+  iluvatar:
+    enabled: false
+    customresources:
+      - iluvatar.ai/BI-V100-vgpu
+      - iluvatar.ai/BI-V100.vCore
+      - iluvatar.ai/BI-V100.vMem
+      - iluvatar.ai/BI-V150-vgpu
+      - iluvatar.ai/BI-V150.vCore
+      - iluvatar.ai/BI-V150.vMem
+      - iluvatar.ai/MR-V100-vgpu
+      - iluvatar.ai/MR-V100.vCore
+      - iluvatar.ai/MR-V100.vMem
+      - iluvatar.ai/MR-V50-vgpu
+      - iluvatar.ai/MR-V50.vCore
+      - iluvatar.ai/MR-V50.vMem
diff --git a/packages/system/hami/values.yaml b/packages/system/hami/values.yaml
new file mode 100644
index 00000000..1155e2cb
--- /dev/null
+++ b/packages/system/hami/values.yaml
@@ -0,0 +1,8 @@
+hami:
+  devicePlugin:
+    runtimeClassName: nvidia
+    nodeConfiguration:
+      config: |
+        {
+          "nodeconfig": []
+        }

From a3e2fbd742d0c9f8bf6a255836913f080f42b826 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Wed, 22 Apr 2026 13:25:23 +0300
Subject: [PATCH 071/151] feat(kubernetes): integrate HAMi as optional addon

Add HAMi HelmRelease to the kubernetes app as a toggleable addon.
When enabled, GPU Operator's built-in device plugin is automatically
disabled via a values merge to avoid conflicts with HAMi's own
device plugin.

The HelmRelease fails fast if GPU Operator is not enabled, since HAMi
depends on it for driver management and container toolkit.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 .../templates/helmreleases/gpu-operator.yaml  | 13 ++++-
 .../templates/helmreleases/hami.yaml          | 49 +++++++++++++++++++
 packages/apps/kubernetes/values.schema.json   | 23 +++++++++
 packages/apps/kubernetes/values.yaml          |  8 +++
 4 files changed, 92 insertions(+), 1 deletion(-)
 create mode 100644 packages/apps/kubernetes/templates/helmreleases/hami.yaml

diff --git a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml
index 5ef48912..995aa430 100644
--- a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml
+++ b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml
@@ -1,3 +1,11 @@
+{{- define "cozystack.defaultGpuOperatorValues" -}}
+{{- if and (hasKey .Values.addons "hami") .Values.addons.hami.enabled }}
+gpu-operator:
+  devicePlugin:
+    enabled: false
+{{- end }}
+{{- end }}
+
 {{- if .Values.addons.gpuOperator.enabled }}
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
@@ -29,7 +37,10 @@ spec:
     force: true
     remediation:
       retries: -1
-  {{- with .Values.addons.gpuOperator.valuesOverride }}
+  {{- $defaults := fromYaml (include "cozystack.defaultGpuOperatorValues" .) }}
+  {{- $overrides := deepCopy (default (dict) .Values.addons.gpuOperator.valuesOverride) }}
+  {{- $merged := mergeOverwrite (default (dict) $defaults) $overrides }}
+  {{- with $merged }}
   values:
     {{- toYaml . | nindent 4 }}
   {{- end }}
diff --git a/packages/apps/kubernetes/templates/helmreleases/hami.yaml b/packages/apps/kubernetes/templates/helmreleases/hami.yaml
new file mode 100644
index 00000000..1f7a5358
--- /dev/null
+++ b/packages/apps/kubernetes/templates/helmreleases/hami.yaml
@@ -0,0 +1,49 @@
+{{- if .Values.addons.hami.enabled }}
+{{- if not .Values.addons.gpuOperator.enabled }}
+{{- fail "addons.hami requires addons.gpuOperator to be enabled" }}
+{{- end }}
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: {{ .Release.Name }}-hami
+  labels:
+    cozystack.io/repository: system
+    cozystack.io/target-cluster-name: {{ .Release.Name }}
+    sharding.fluxcd.io/key: tenants
+spec:
+  releaseName: hami
+  chartRef:
+    kind: ExternalArtifact
+    name: cozystack-kubernetes-application-kubevirt-kubernetes-hami
+    namespace: cozy-system
+  kubeConfig:
+    secretRef:
+      name: {{ .Release.Name }}-admin-kubeconfig
+      key: super-admin.svc
+  targetNamespace: cozy-hami
+  storageNamespace: cozy-hami
+  interval: 5m
+  timeout: 10m
+  install:
+    createNamespace: true
+    remediation:
+      retries: -1
+  upgrade:
+    force: true
+    remediation:
+      retries: -1
+  {{- with .Values.addons.hami.valuesOverride }}
+  values:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+
+  dependsOn:
+  {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }}
+  - name: {{ .Release.Name }}
+    namespace: {{ .Release.Namespace }}
+  {{- end }}
+  - name: {{ .Release.Name }}-cilium
+    namespace: {{ .Release.Namespace }}
+  - name: {{ .Release.Name }}-gpu-operator
+    namespace: {{ .Release.Namespace }}
+{{- end }}
diff --git a/packages/apps/kubernetes/values.schema.json b/packages/apps/kubernetes/values.schema.json
index 1beeff9c..3d7a17dd 100644
--- a/packages/apps/kubernetes/values.schema.json
+++ b/packages/apps/kubernetes/values.schema.json
@@ -149,6 +149,7 @@
         "fluxcd",
         "gatewayAPI",
         "gpuOperator",
+        "hami",
         "ingressNginx",
         "monitoringAgents",
         "velero",
@@ -268,6 +269,28 @@
             }
           }
         },
+        "hami": {
+          "description": "HAMi GPU virtualization middleware. Enables fractional GPU sharing (memory and compute isolation) for workloads. Requires GPU Operator. Note: workload containers must use glibc < 2.34 (not musl/Alpine) for full compute isolation.",
+          "type": "object",
+          "default": {},
+          "required": [
+            "enabled",
+            "valuesOverride"
+          ],
+          "properties": {
+            "enabled": {
+              "description": "Enable HAMi.",
+              "type": "boolean",
+              "default": false
+            },
+            "valuesOverride": {
+              "description": "Custom Helm values overrides.",
+              "type": "object",
+              "default": {},
+              "x-kubernetes-preserve-unknown-fields": true
+            }
+          }
+        },
         "ingressNginx": {
           "description": "Ingress-NGINX controller.",
           "type": "object",
diff --git a/packages/apps/kubernetes/values.yaml b/packages/apps/kubernetes/values.yaml
index a67b5d69..f601b979 100644
--- a/packages/apps/kubernetes/values.yaml
+++ b/packages/apps/kubernetes/values.yaml
@@ -94,6 +94,10 @@ host: ""
 ## @field {bool} enabled - Enable FluxCD.
 ## @field {object} valuesOverride - Custom Helm values overrides.
 
+## @typedef {struct} HAMiAddon - HAMi GPU virtualization middleware.
+## @field {bool} enabled - Enable HAMi (requires GPU Operator).
+## @field {object} valuesOverride - Custom Helm values overrides.
+
 ## @typedef {struct} MonitoringAgentsAddon - Monitoring agents (Fluent Bit, VMAgents).
 ## @field {bool} enabled - Enable monitoring agents.
 ## @field {object} valuesOverride - Custom Helm values overrides.
@@ -114,6 +118,7 @@ host: ""
 ## @field {GatewayAPIAddon} gatewayAPI - Gateway API addon.
 ## @field {IngressNginxAddon} ingressNginx - Ingress-NGINX controller.
 ## @field {GPUOperatorAddon} gpuOperator - NVIDIA GPU Operator.
+## @field {HAMiAddon} hami - HAMi GPU virtualization middleware.
 ## @field {FluxCDAddon} fluxcd - FluxCD GitOps operator.
 ## @field {MonitoringAgentsAddon} monitoringAgents - Monitoring agents.
 ## @field {VerticalPodAutoscalerAddon} verticalPodAutoscaler - Vertical Pod Autoscaler.
@@ -137,6 +142,9 @@ addons:
   gpuOperator:
     enabled: false
     valuesOverride: {}
+  hami:
+    enabled: false
+    valuesOverride: {}
   fluxcd:
     enabled: false
     valuesOverride: {}

From 273eb7811a36fb69141d80360705f081a3276194 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Wed, 22 Apr 2026 13:25:29 +0300
Subject: [PATCH 072/151] docs(hami): document glibc < 2.34 limitation and
 upstream issues

HAMi-core relies on _dl_sym (private glibc symbol removed in 2.34)
for CUDA interception. This breaks compute isolation on modern
container images using Ubuntu 22.04+ and makes Alpine/musl completely
incompatible.

Include upstream issue references and a compatibility matrix so users
can make informed decisions about base image selection.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 packages/system/hami/README.md | 81 ++++++++++++++++++++++++++++++++++
 1 file changed, 81 insertions(+)
 create mode 100644 packages/system/hami/README.md

diff --git a/packages/system/hami/README.md b/packages/system/hami/README.md
new file mode 100644
index 00000000..283f9e80
--- /dev/null
+++ b/packages/system/hami/README.md
@@ -0,0 +1,81 @@
+# HAMi — GPU Virtualization Middleware
+
+[HAMi](https://github.com/Project-HAMi/HAMi) (Heterogeneous AI Computing Virtualization Middleware) is a CNCF Sandbox project that enables fractional GPU sharing in Kubernetes. It allows workloads to request specific amounts of GPU memory and compute cores instead of claiming entire GPUs.
+
+## Architecture
+
+HAMi consists of four components:
+
+- **MutatingWebhook** — intercepts pod creation, injects `schedulerName: hami-scheduler`
+- **Scheduler Extender** — extends kube-scheduler with GPU-aware Filter and Bind logic
+- **Device Plugin** (DaemonSet) — registers vGPU resources via the Kubernetes Device Plugin API
+- **HAMi-core** (`libvgpu.so`) — `LD_PRELOAD` library injected into workload containers, intercepts CUDA API calls to enforce memory and compute isolation
+
+## Prerequisites
+
+- GPU Operator must be enabled (`addons.gpuOperator.enabled: true`)
+- NVIDIA driver >= 440 on host nodes
+- nvidia-container-toolkit configured as the default container runtime
+- GPU nodes labeled with `gpu=on`
+
+## Known Limitations
+
+### glibc < 2.34 requirement for workload containers
+
+HAMi-core uses `LD_PRELOAD` to intercept `dlsym()` for CUDA symbol resolution. The fallback code path relies on `_dl_sym`, a private glibc internal symbol that was removed in glibc 2.34 when libdl and libpthread were merged into libc.so.
+
+**This limitation affects workload containers only**, not the host OS or HAMi's own components.
+
+| Distribution    | glibc | Result                                       |
+| --------------- | ----- | -------------------------------------------- |
+| Ubuntu 18.04    | 2.27  | Full isolation (memory + compute)            |
+| Ubuntu 20.04    | 2.31  | Full isolation (memory + compute)            |
+| Ubuntu 22.04    | 2.35  | Memory isolation works, compute breaks       |
+| Ubuntu 24.04    | 2.39  | Both memory and compute isolation break      |
+| Alpine (musl)   | N/A   | Completely incompatible (`dlvsym` absent)    |
+
+Most modern ML/AI base images (CUDA 12.x, PyTorch 2.x, TensorFlow 2.x) use Ubuntu 22.04+ with glibc >= 2.35, which means compute isolation will not work with these images until the upstream fix is merged.
+
+**Upstream tracking issues:**
+
+- [HAMi-core#174](https://github.com/Project-HAMi/HAMi-core/issues/174) — `_dl_sym` removal breaks HAMi-core on glibc >= 2.34
+- [HAMi#1190](https://github.com/Project-HAMi/HAMi/issues/1190) — degraded isolation across glibc versions
+- [HAMi#173](https://github.com/Project-HAMi/HAMi/issues/173) — documentation incorrectly states glibc < 2.30 (actual boundary is 2.34)
+
+### musl libc (Alpine) incompatibility
+
+HAMi-core is completely incompatible with musl libc. The `dlvsym()` function used by HAMi-core is a glibc extension not available in musl. Only glibc-based container images (Debian, Ubuntu, RHEL, etc.) can use HAMi GPU isolation.
+
+## Usage
+
+Enable HAMi in your tenant Kubernetes cluster values:
+
+```yaml
+addons:
+  gpuOperator:
+    enabled: true
+  hami:
+    enabled: true
+```
+
+When HAMi is enabled, GPU Operator's built-in device plugin is automatically disabled to avoid conflicts.
+
+### Requesting fractional GPU resources
+
+```yaml
+resources:
+  limits:
+    nvidia.com/gpu: 1
+    nvidia.com/gpumem: 3000     # 3000 MB of GPU memory
+    nvidia.com/gpucores: 30     # 30% of GPU compute cores
+```
+
+## Parameters
+
+| Name | Description | Default |
+| --- | --- | --- |
+| `hami.devicePlugin.runtimeClassName` | RuntimeClass for device plugin pods | `nvidia` |
+| `hami.devicePlugin.deviceSplitCount` | Max virtual GPUs per physical GPU | `10` |
+| `hami.devicePlugin.deviceMemoryScaling` | Memory overcommit factor (> 1.0 enables overcommit) | `1` |
+| `hami.scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy` | Node packing strategy (`binpack` or `spread`) | `binpack` |
+| `hami.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy` | GPU packing strategy (`binpack` or `spread`) | `spread` |

From 43035562efff34f18c3c439026fe2a233dc8d17b Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Wed, 22 Apr 2026 13:25:35 +0300
Subject: [PATCH 073/151] test(kubernetes): add helm-unittest tests for HAMi
 integration

Cover HelmRelease rendering, gpuOperator dependency validation,
ExternalArtifact chartRef, namespace targeting, dependency chain,
valuesOverride passthrough, and automatic devicePlugin disable in
GPU Operator when HAMi is active.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 .../tests/gpu_operator_hami_test.yaml         |  80 +++++++++++
 packages/apps/kubernetes/tests/hami_test.yaml | 136 ++++++++++++++++++
 2 files changed, 216 insertions(+)
 create mode 100644 packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml
 create mode 100644 packages/apps/kubernetes/tests/hami_test.yaml

diff --git a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml
new file mode 100644
index 00000000..33256ae6
--- /dev/null
+++ b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml
@@ -0,0 +1,80 @@
+suite: GPU Operator HelmRelease HAMi integration tests
+templates:
+  - templates/helmreleases/gpu-operator.yaml
+tests:
+  - it: should disable devicePlugin when hami is enabled
+    set:
+      addons:
+        gpuOperator:
+          enabled: true
+          valuesOverride: {}
+        hami:
+          enabled: true
+          valuesOverride: {}
+    asserts:
+      - equal:
+          path: spec.values.gpu-operator.devicePlugin.enabled
+          value: false
+
+  - it: should not have values when hami is disabled and no overrides
+    set:
+      addons:
+        gpuOperator:
+          enabled: true
+          valuesOverride: {}
+        hami:
+          enabled: false
+          valuesOverride: {}
+    asserts:
+      - notExists:
+          path: spec.values
+
+  - it: should allow user overrides to merge with hami defaults
+    set:
+      addons:
+        gpuOperator:
+          enabled: true
+          valuesOverride:
+            gpu-operator:
+              driver:
+                enabled: false
+        hami:
+          enabled: true
+          valuesOverride: {}
+    asserts:
+      - equal:
+          path: spec.values.gpu-operator.devicePlugin.enabled
+          value: false
+      - equal:
+          path: spec.values.gpu-operator.driver.enabled
+          value: false
+
+  - it: should let user override devicePlugin back to true if needed
+    set:
+      addons:
+        gpuOperator:
+          enabled: true
+          valuesOverride:
+            gpu-operator:
+              devicePlugin:
+                enabled: true
+        hami:
+          enabled: true
+          valuesOverride: {}
+    asserts:
+      - equal:
+          path: spec.values.gpu-operator.devicePlugin.enabled
+          value: true
+
+  - it: should not render when gpuOperator is disabled
+    set:
+      addons:
+        gpuOperator:
+          enabled: false
+          valuesOverride: {}
+        hami:
+          enabled: false
+          valuesOverride: {}
+    asserts:
+      - hasDocuments:
+          count: 0
diff --git a/packages/apps/kubernetes/tests/hami_test.yaml b/packages/apps/kubernetes/tests/hami_test.yaml
new file mode 100644
index 00000000..e9d32236
--- /dev/null
+++ b/packages/apps/kubernetes/tests/hami_test.yaml
@@ -0,0 +1,136 @@
+suite: HAMi HelmRelease tests
+templates:
+  - templates/helmreleases/hami.yaml
+tests:
+  - it: should not render when hami is disabled
+    set:
+      addons:
+        hami:
+          enabled: false
+          valuesOverride: {}
+        gpuOperator:
+          enabled: true
+          valuesOverride: {}
+    asserts:
+      - hasDocuments:
+          count: 0
+
+  - it: should render HelmRelease when hami is enabled
+    set:
+      addons:
+        hami:
+          enabled: true
+          valuesOverride: {}
+        gpuOperator:
+          enabled: true
+          valuesOverride: {}
+    asserts:
+      - hasDocuments:
+          count: 1
+      - isKind:
+          of: HelmRelease
+
+  - it: should fail when gpuOperator is not enabled
+    set:
+      addons:
+        hami:
+          enabled: true
+          valuesOverride: {}
+        gpuOperator:
+          enabled: false
+          valuesOverride: {}
+    asserts:
+      - failedTemplate:
+          errorMessage: "addons.hami requires addons.gpuOperator to be enabled"
+
+  - it: should have correct metadata labels
+    set:
+      addons:
+        hami:
+          enabled: true
+          valuesOverride: {}
+        gpuOperator:
+          enabled: true
+          valuesOverride: {}
+    asserts:
+      - equal:
+          path: metadata.labels["cozystack.io/repository"]
+          value: system
+      - equal:
+          path: metadata.labels["sharding.fluxcd.io/key"]
+          value: tenants
+
+  - it: should use ExternalArtifact chartRef
+    set:
+      addons:
+        hami:
+          enabled: true
+          valuesOverride: {}
+        gpuOperator:
+          enabled: true
+          valuesOverride: {}
+    asserts:
+      - equal:
+          path: spec.chartRef.kind
+          value: ExternalArtifact
+      - equal:
+          path: spec.chartRef.namespace
+          value: cozy-system
+
+  - it: should target cozy-hami namespace
+    set:
+      addons:
+        hami:
+          enabled: true
+          valuesOverride: {}
+        gpuOperator:
+          enabled: true
+          valuesOverride: {}
+    asserts:
+      - equal:
+          path: spec.targetNamespace
+          value: cozy-hami
+      - equal:
+          path: spec.storageNamespace
+          value: cozy-hami
+
+  - it: should depend on gpu-operator and cilium
+    release:
+      name: test
+      namespace: test-ns
+    set:
+      addons:
+        hami:
+          enabled: true
+          valuesOverride: {}
+        gpuOperator:
+          enabled: true
+          valuesOverride: {}
+    asserts:
+      - contains:
+          path: spec.dependsOn
+          content:
+            name: test-cilium
+            namespace: test-ns
+      - contains:
+          path: spec.dependsOn
+          content:
+            name: test-gpu-operator
+            namespace: test-ns
+
+  - it: should pass through valuesOverride
+    set:
+      addons:
+        hami:
+          enabled: true
+          valuesOverride:
+            hami:
+              devicePlugin:
+                deviceSplitCount: 5
+        gpuOperator:
+          enabled: true
+          valuesOverride: {}
+    asserts:
+      - equal:
+          path: spec.values.hami.devicePlugin.deviceSplitCount
+          value: 5

From c1508940bda27b2979639981fd25461c9d4824b3 Mon Sep 17 00:00:00 2001
From: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
Date: Thu, 23 Apr 2026 13:34:21 +0500
Subject: [PATCH 074/151] fix(etcd): remove destructive post-upgrade
 cert-regeneration hook

The etcd chart shipped a `post-upgrade` Helm hook that `kubectl delete`d the
etcd TLS chain (`etcd-{ca,peer-ca,client,peer,server}-tls`) and then deleted
etcd pods on every chart upgrade, gated by a semver compare of an
`etcd-deployed-version` ConfigMap against `2.6.1`.

The hook was added as a one-shot migration for the chart `2.6.0 -> 2.6.1`
transition. Since commit f871fbdb ("Remove versions_map logic") all chart
versions are stamped as `0.0.0+<git-hash>`, which per semver is always
`< 2.6.1`. The gate therefore always resolves to "update certs", firing the
destructive hook on every etcd upgrade. On clusters running Kamaji-managed
tenant control planes this wipes the etcd CA, cert-manager re-issues it, and
tenant kube-apiservers hit `x509: certificate signed by unknown authority`
against `etcd.<ns>.svc:2379` until each tenant DataStore is manually
re-reconciled.

Commit 47d81f70 ("Disabled private key rotation in CA certs") already fixed
the underlying `rotationPolicy: Always` issue the migration was papering
over, so the hook has no remaining use. Remove the hook Job, its RBAC, the
version ConfigMap it read, and add a helm-unittest suite under
`packages/extra/etcd/tests/` that guards against re-introducing the hook
or the version ConfigMap.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
---
 packages/extra/etcd/Makefile                  |  3 ++
 packages/extra/etcd/templates/hook/job.yaml   | 39 -------------------
 packages/extra/etcd/templates/hook/role.yaml  | 26 -------------
 .../etcd/templates/hook/rolebinding.yaml      | 15 -------
 .../etcd/templates/hook/serviceaccount.yaml   |  7 ----
 packages/extra/etcd/templates/version.yaml    |  6 ---
 .../etcd/tests/no-post-upgrade-hook_test.yaml | 34 ++++++++++++++++
 7 files changed, 37 insertions(+), 93 deletions(-)
 delete mode 100644 packages/extra/etcd/templates/hook/job.yaml
 delete mode 100644 packages/extra/etcd/templates/hook/role.yaml
 delete mode 100644 packages/extra/etcd/templates/hook/rolebinding.yaml
 delete mode 100644 packages/extra/etcd/templates/hook/serviceaccount.yaml
 delete mode 100644 packages/extra/etcd/templates/version.yaml
 create mode 100644 packages/extra/etcd/tests/no-post-upgrade-hook_test.yaml

diff --git a/packages/extra/etcd/Makefile b/packages/extra/etcd/Makefile
index f37d6e1e..2b3ed61e 100644
--- a/packages/extra/etcd/Makefile
+++ b/packages/extra/etcd/Makefile
@@ -5,3 +5,6 @@ include ../../../hack/package.mk
 generate:
 	cozyvalues-gen -v values.yaml -s values.schema.json -r README.md
 	../../../hack/update-crd.sh
+
+test:
+	helm unittest .
diff --git a/packages/extra/etcd/templates/hook/job.yaml b/packages/extra/etcd/templates/hook/job.yaml
deleted file mode 100644
index 3bf2f84b..00000000
--- a/packages/extra/etcd/templates/hook/job.yaml
+++ /dev/null
@@ -1,39 +0,0 @@
-{{- $shouldUpdateCerts := true }}
-{{- $configMap := lookup "v1" "ConfigMap" .Release.Namespace "etcd-deployed-version" }}
-{{- if $configMap }}
-  {{- $deployedVersion := index $configMap "data" "version" }}
-  {{- if $deployedVersion | semverCompare ">= 2.6.1" }}
-    {{- $shouldUpdateCerts = false }}
-  {{- end }}
-{{- end }}
-
-{{- if $shouldUpdateCerts }}
----
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: etcd-hook
-  annotations:
-    helm.sh/hook: post-upgrade
-    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
-spec:
-  template:
-    metadata:
-      labels:
-        policy.cozystack.io/allow-to-apiserver: "true"
-    spec:
-      serviceAccountName: etcd-hook
-      containers:
-        - name: kubectl
-          image: docker.io/alpine/k8s:1.33.4
-          command:
-          - sh
-          args:
-            - -exc
-            - |-
-              kubectl --namespace={{ .Release.Namespace }} delete secrets etcd-ca-tls etcd-peer-ca-tls 
-              sleep 10
-              kubectl --namespace={{ .Release.Namespace }} delete secrets etcd-client-tls etcd-peer-tls etcd-server-tls
-              kubectl --namespace={{ .Release.Namespace }} delete pods --selector=app.kubernetes.io/instance=etcd,app.kubernetes.io/managed-by=etcd-operator,app.kubernetes.io/name=etcd,cozystack.io/service=etcd
-      restartPolicy: Never
-{{- end }}
diff --git a/packages/extra/etcd/templates/hook/role.yaml b/packages/extra/etcd/templates/hook/role.yaml
deleted file mode 100644
index 327eeadb..00000000
--- a/packages/extra/etcd/templates/hook/role.yaml
+++ /dev/null
@@ -1,26 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  annotations:
-    helm.sh/hook: post-upgrade
-    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
-  name: etcd-hook
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - secrets
-  - pods
-  verbs:
-  - get
-  - list
-  - watch
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - configmaps
-  verbs:
-  - get
-  - list
-  - watch
diff --git a/packages/extra/etcd/templates/hook/rolebinding.yaml b/packages/extra/etcd/templates/hook/rolebinding.yaml
deleted file mode 100644
index 0ee0ffd1..00000000
--- a/packages/extra/etcd/templates/hook/rolebinding.yaml
+++ /dev/null
@@ -1,15 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: etcd-hook
-  annotations:
-    helm.sh/hook: post-upgrade
-    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: etcd-hook
-subjects:
-  - kind: ServiceAccount
-    name: etcd-hook
-    namespace: {{ .Release.Namespace | quote }}
diff --git a/packages/extra/etcd/templates/hook/serviceaccount.yaml b/packages/extra/etcd/templates/hook/serviceaccount.yaml
deleted file mode 100644
index 552fb5fc..00000000
--- a/packages/extra/etcd/templates/hook/serviceaccount.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: etcd-hook
-  annotations:
-    helm.sh/hook: post-upgrade
-    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
diff --git a/packages/extra/etcd/templates/version.yaml b/packages/extra/etcd/templates/version.yaml
deleted file mode 100644
index cc9375bb..00000000
--- a/packages/extra/etcd/templates/version.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: etcd-deployed-version
-data:
-  version: {{ .Chart.Version }}
diff --git a/packages/extra/etcd/tests/no-post-upgrade-hook_test.yaml b/packages/extra/etcd/tests/no-post-upgrade-hook_test.yaml
new file mode 100644
index 00000000..0e4f5aa3
--- /dev/null
+++ b/packages/extra/etcd/tests/no-post-upgrade-hook_test.yaml
@@ -0,0 +1,34 @@
+suite: etcd chart does not ship a destructive post-upgrade cert-regeneration hook
+
+release:
+  name: etcd
+  namespace: tenant-root
+
+templates:
+  - templates/check-release-name.yaml
+  - templates/dashboard-resourcemap.yaml
+  - templates/datastore.yaml
+  - templates/etcd-defrag.yaml
+  - templates/hook/job.yaml
+  - templates/podscrape.yaml
+  - templates/prometheus-rules.yaml
+  - templates/version.yaml
+
+tests:
+  - it: renders no Job named etcd-hook
+    documentSelector:
+      path: metadata.name
+      value: etcd-hook
+      skipEmptyTemplates: true
+    asserts:
+      - hasDocuments:
+          count: 0
+
+  - it: renders no ConfigMap named etcd-deployed-version
+    documentSelector:
+      path: metadata.name
+      value: etcd-deployed-version
+      skipEmptyTemplates: true
+    asserts:
+      - hasDocuments:
+          count: 0

From 9222b6feda4d97eda7481569d5d9a466b69fec94 Mon Sep 17 00:00:00 2001
From: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
Date: Thu, 23 Apr 2026 14:32:20 +0500
Subject: [PATCH 075/151] ci(api): pre-fetch k8s.io/code-generator in codegen
 drift job

hack/update-codegen.sh sources kube_codegen.sh from the Go module
cache at ~/go/pkg/mod/k8s.io/code-generator@vX.Y.Z/, but the module
is not declared in go.mod so a fresh runner has nothing to source
from. Add a workflow step that parses the pinned version out of the
script and pulls the module into the cache before running
make generate.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
---
 .github/workflows/codegen-drift.yml | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/.github/workflows/codegen-drift.yml b/.github/workflows/codegen-drift.yml
index bd7927f7..b08f5d7e 100644
--- a/.github/workflows/codegen-drift.yml
+++ b/.github/workflows/codegen-drift.yml
@@ -30,6 +30,17 @@ jobs:
           go-version-file: go.mod
           cache: true
 
+      - name: Pre-fetch k8s.io/code-generator module
+        # hack/update-codegen.sh sources kube_codegen.sh from the Go module cache.
+        # The module is not declared in go.mod, so fetch it explicitly at the
+        # version pinned in the script.
+        run: |
+          version=$(grep -oP 'code-generator@\Kv[0-9.]+' hack/update-codegen.sh)
+          tmpdir=$(mktemp -d)
+          cd "$tmpdir"
+          go mod init codegen-fetch
+          go get "k8s.io/code-generator@${version}"
+
       - name: Run make generate
         run: make generate
 

From adc7abe5c16fc81ce1c063d153d9a920c0aac097 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Thu, 23 Apr 2026 14:55:21 +0300
Subject: [PATCH 076/151] feat(ingress): add loadBalancer exposure mode via
 CiliumLoadBalancerIPPool

Service.spec.externalIPs is deprecated upstream in Kubernetes v1.36
(KEP-5707, kubernetes#137293). The AllowServiceExternalIPs feature gate
is expected to default to off around v1.40 and the implementation to
be removed around v1.43. For bare-metal installs that rely on
externalIPs today, cozystack needs a migration path.

This change adds an opt-in 'loadBalancer' exposure mode for the
ingress-nginx Service:

- New platform value 'publishing.exposure' (enum: externalIPs |
  loadBalancer, default externalIPs). Plumbed through cozystack-values
  into each tenant's ingress HelmRelease via the new 'expose-mode' key.
- Unknown values and loadBalancer with an empty externalIPs list fail
  the chart render with explicit error messages, rather than silently
  producing a broken Service.
- When exposure=loadBalancer and the current namespace matches
  publishing.ingressName, the Service becomes type: LoadBalancer with
  externalTrafficPolicy: Local.
- A new template renders a CiliumLoadBalancerIPPool whose blocks come
  from publishing.externalIPs (IPv4 addresses get /32, IPv6 addresses
  get /128) and whose serviceSelector uses Cilium's synthetic
  io.kubernetes.service.namespace key combined with the standard
  app.kubernetes.io/name: ingress-nginx label. No custom label is
  written to the Service itself, avoiding cross-tenant collisions from
  user-defined labels.

Default behaviour is unchanged: without opting in, the Service is
still ClusterIP + spec.externalIPs as today.

Scope: only ingress-nginx is migrated by this setting. Other cozystack
components that still write Service.spec.externalIPs directly (notably
the vpn app) must be migrated separately before the v1.40 feature gate
flip.

Tests: packages/extra/ingress/tests/exposure_test.yaml adds 13
helm-unittest cases covering both modes, IPv4/IPv6, empty-token
filtering, unknown-mode rejection, and the non-matching-namespace
fallback.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 packages/core/platform/templates/apps.yaml    |   1 +
 packages/core/platform/values.yaml            |  35 +++
 packages/extra/ingress/Makefile               |   3 +
 packages/extra/ingress/README.md              |  14 +
 .../ingress/templates/cilium-lb-pool.yaml     |  25 ++
 .../ingress/templates/nginx-ingress.yaml      |  21 +-
 .../extra/ingress/tests/exposure_test.yaml    | 251 ++++++++++++++++++
 7 files changed, 348 insertions(+), 2 deletions(-)
 create mode 100644 packages/extra/ingress/templates/cilium-lb-pool.yaml
 create mode 100644 packages/extra/ingress/tests/exposure_test.yaml

diff --git a/packages/core/platform/templates/apps.yaml b/packages/core/platform/templates/apps.yaml
index 7ee0de52..6e1ae25a 100644
--- a/packages/core/platform/templates/apps.yaml
+++ b/packages/core/platform/templates/apps.yaml
@@ -30,6 +30,7 @@ stringData:
       expose-services: {{ .Values.publishing.exposedServices | join "," | quote }}
       expose-ingress: {{ .Values.publishing.ingressName | quote }}
       expose-external-ips: {{ .Values.publishing.externalIPs | join "," | quote }}
+      expose-mode: {{ .Values.publishing.exposure | default "externalIPs" | quote }}
       cluster-domain: {{ .Values.networking.clusterDomain | quote }}
       api-server-endpoint: {{ .Values.publishing.apiServerEndpoint | quote }}
       {{- with .Values.branding }}
diff --git a/packages/core/platform/values.yaml b/packages/core/platform/values.yaml
index f33926db..605e23bc 100644
--- a/packages/core/platform/values.yaml
+++ b/packages/core/platform/values.yaml
@@ -45,6 +45,41 @@ publishing:
     - cdi-uploadproxy
   apiServerEndpoint: "" # example: "https://api.example.org"
   externalIPs: []
+  # Exposure mode for the ingress-nginx Service. When "externalIPs" (current
+  # default) is selected, the Service is created as ClusterIP with
+  # Service.spec.externalIPs set from publishing.externalIPs. When
+  # "loadBalancer" is selected, the Service is type: LoadBalancer and a
+  # CiliumLoadBalancerIPPool makes those same addresses allocatable via LB IPAM.
+  #
+  # Service.spec.externalIPs is deprecated upstream in Kubernetes v1.36
+  # (KEP-5707). The AllowServiceExternalIPs feature gate is expected to default
+  # to false around v1.40 and the implementation removed around v1.43 — switch
+  # to "loadBalancer" before upgrading past v1.40.
+  #
+  # Caveats for the "loadBalancer" mode:
+  #   - publishing.externalIPs must contain at least one non-empty address,
+  #     otherwise the chart render fails with an explicit error (a LoadBalancer
+  #     Service without a pool would sit in <pending> forever).
+  #   - The ingress-nginx Service is created with externalTrafficPolicy: Local
+  #     to preserve the client source IP. Traffic arriving on a node that does
+  #     not host an ingress-nginx pod is dropped, so the external IP must be
+  #     routed to a node that runs the ingress pod (floating IP / keepalived /
+  #     upstream router / podAntiAffinity).
+  #   - Cilium does NOT announce the IP on its own unless L2 announcements or
+  #     BGP are enabled in the Cilium values (disabled by default in Cozystack).
+  #     This mode assumes the operator already routes the externalIPs to a
+  #     cluster node; enabling announcements is out of scope for this setting.
+  #   - Switching this value on a running cluster causes the ingress-nginx
+  #     Service to be recreated (the HelmRelease has upgrade.force: true and
+  #     the Service kind changes between ClusterIP and LoadBalancer). Expect a
+  #     brief interruption of ingress traffic during the flip.
+  #
+  # Scope: this setting only controls the ingress-nginx Service. Other
+  # cozystack components that currently write Service.spec.externalIPs directly
+  # (e.g. the vpn app at packages/apps/vpn/templates/service.yaml) are NOT
+  # migrated by flipping this value and must be addressed separately before
+  # the AllowServiceExternalIPs feature gate flips to off in ~v1.40.
+  exposure: externalIPs # "externalIPs" or "loadBalancer"
   certificates:
     solver: http01 # "http01" or "dns01"
     issuerName: letsencrypt-prod
diff --git a/packages/extra/ingress/Makefile b/packages/extra/ingress/Makefile
index 958ce484..65b53c03 100644
--- a/packages/extra/ingress/Makefile
+++ b/packages/extra/ingress/Makefile
@@ -10,3 +10,6 @@ get-cloudflare-ips:
 generate:
 	cozyvalues-gen -v values.yaml -s values.schema.json -r README.md
 	../../../hack/update-crd.sh
+
+test:
+	helm unittest .
diff --git a/packages/extra/ingress/README.md b/packages/extra/ingress/README.md
index 0e786dfb..c541d8e9 100644
--- a/packages/extra/ingress/README.md
+++ b/packages/extra/ingress/README.md
@@ -14,3 +14,17 @@
 | `resources.memory` | Memory (RAM) available to each replica.                                                                                                 | `quantity` | `""`    |
 | `resourcesPreset`  | Default sizing preset used when `resources` is omitted.                                                                                 | `string`   | `micro` |
 
+
+## Exposure mode
+
+The ingress Service type is driven by the cluster-wide `publishing.exposure` value in the platform chart, not by any key in this package. Two modes exist:
+
+- `externalIPs` (default) has three rendered shapes:
+  - Release namespace matches `publishing.ingressName` AND `publishing.externalIPs` is non-empty → Service is `ClusterIP` with `Service.spec.externalIPs` set from that list and `externalTrafficPolicy: Cluster`.
+  - Release namespace matches `publishing.ingressName` but `publishing.externalIPs` is empty → Service falls back to `type: LoadBalancer` with `externalTrafficPolicy: Local`.
+  - Release namespace does not match `publishing.ingressName` (non-root tenants) → Service is `type: LoadBalancer` with `externalTrafficPolicy: Local`.
+  `Service.spec.externalIPs` is deprecated upstream in Kubernetes v1.36 (KEP-5707); plan migration before v1.40.
+- `loadBalancer` — Service is `type: LoadBalancer` with `externalTrafficPolicy: Local`, and a `CiliumLoadBalancerIPPool` makes the addresses in `publishing.externalIPs` allocatable via Cilium LB IPAM. Requires `publishing.externalIPs` to contain at least one non-empty address (render fails otherwise) and assumes the addresses are already routed to a cluster node (floating IP / upstream router). See the inline comment on `publishing.exposure` in the platform chart for full caveats, including the note that switching the value on a running cluster causes the ingress Service to be recreated.
+
+This setting only migrates ingress-nginx away from `Service.spec.externalIPs`. Other cozystack components that use the same deprecated field (e.g. the `vpn` app) must be migrated separately before Kubernetes v1.40 flips the `AllowServiceExternalIPs` feature gate off.
+
diff --git a/packages/extra/ingress/templates/cilium-lb-pool.yaml b/packages/extra/ingress/templates/cilium-lb-pool.yaml
new file mode 100644
index 00000000..a383922b
--- /dev/null
+++ b/packages/extra/ingress/templates/cilium-lb-pool.yaml
@@ -0,0 +1,25 @@
+{{- $exposeIngress := (index .Values._cluster "expose-ingress") | default "tenant-root" }}
+{{- $exposeMode := (index .Values._cluster "expose-mode") | default "externalIPs" }}
+{{- $exposeExternalIPs := (index .Values._cluster "expose-external-ips") | default "" | nospace }}
+{{- $exposeIPsList := list }}
+{{- range splitList "," $exposeExternalIPs }}
+  {{- $ip := . | trim }}
+  {{- if $ip }}
+    {{- $exposeIPsList = append $exposeIPsList $ip }}
+  {{- end }}
+{{- end }}
+{{- if and (eq $exposeMode "loadBalancer") (eq $exposeIngress .Release.Namespace) $exposeIPsList }}
+apiVersion: cilium.io/v2
+kind: CiliumLoadBalancerIPPool
+metadata:
+  name: {{ trimPrefix "tenant-" .Release.Namespace }}-ingress
+spec:
+  blocks:
+  {{- range $exposeIPsList }}
+  - cidr: {{ . }}/{{ if contains ":" . }}128{{ else }}32{{ end }}
+  {{- end }}
+  serviceSelector:
+    matchLabels:
+      "io.kubernetes.service.namespace": {{ .Release.Namespace | quote }}
+      "app.kubernetes.io/name": ingress-nginx
+{{- end }}
diff --git a/packages/extra/ingress/templates/nginx-ingress.yaml b/packages/extra/ingress/templates/nginx-ingress.yaml
index ca50d276..8e1f00fb 100644
--- a/packages/extra/ingress/templates/nginx-ingress.yaml
+++ b/packages/extra/ingress/templates/nginx-ingress.yaml
@@ -1,5 +1,19 @@
 {{- $exposeIngress := (index .Values._cluster "expose-ingress") | default "tenant-root" }}
 {{- $exposeExternalIPs := (index .Values._cluster "expose-external-ips") | default "" | nospace }}
+{{- $exposeMode := (index .Values._cluster "expose-mode") | default "externalIPs" }}
+{{- $exposeIPsList := list }}
+{{- range splitList "," $exposeExternalIPs }}
+  {{- $ip := . | trim }}
+  {{- if $ip }}
+    {{- $exposeIPsList = append $exposeIPsList $ip }}
+  {{- end }}
+{{- end }}
+{{- if not (has $exposeMode (list "externalIPs" "loadBalancer")) }}
+{{- fail (printf "unknown publishing.exposure mode %q: must be \"externalIPs\" or \"loadBalancer\"" $exposeMode) }}
+{{- end }}
+{{- if and (eq $exposeMode "loadBalancer") (eq $exposeIngress .Release.Namespace) (not $exposeIPsList) }}
+{{- fail "publishing.exposure=loadBalancer requires publishing.externalIPs to contain at least one non-empty address: the CiliumLoadBalancerIPPool has nothing to advertise and the Service would stay in <pending>." }}
+{{- end }}
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
@@ -41,9 +55,12 @@ spec:
           enabled: false
         {{- end }}
         service:
-          {{- if and (eq $exposeIngress .Release.Namespace) $exposeExternalIPs }}
+          {{- if and (eq $exposeIngress .Release.Namespace) (eq $exposeMode "loadBalancer") }}
+          type: LoadBalancer
+          externalTrafficPolicy: Local
+          {{- else if and (eq $exposeIngress .Release.Namespace) $exposeIPsList }}
           externalIPs:
-            {{- toYaml (splitList "," $exposeExternalIPs) | nindent 12 }}
+            {{- toYaml $exposeIPsList | nindent 12 }}
           type: ClusterIP
           externalTrafficPolicy: Cluster
           {{- else }}
diff --git a/packages/extra/ingress/tests/exposure_test.yaml b/packages/extra/ingress/tests/exposure_test.yaml
new file mode 100644
index 00000000..780e0715
--- /dev/null
+++ b/packages/extra/ingress/tests/exposure_test.yaml
@@ -0,0 +1,251 @@
+suite: ingress exposure modes
+templates:
+  - templates/nginx-ingress.yaml
+  - templates/cilium-lb-pool.yaml
+
+release:
+  name: ingress
+  namespace: tenant-root
+
+tests:
+  - it: default exposure (externalIPs) renders ClusterIP Service with spec.externalIPs and no CiliumLoadBalancerIPPool
+    set:
+      _cluster:
+        expose-ingress: tenant-root
+        expose-external-ips: "192.0.2.10,192.0.2.11"
+    asserts:
+      - template: templates/nginx-ingress.yaml
+        equal:
+          path: spec.values.ingress-nginx.controller.service.type
+          value: ClusterIP
+      - template: templates/nginx-ingress.yaml
+        equal:
+          path: spec.values.ingress-nginx.controller.service.externalTrafficPolicy
+          value: Cluster
+      - template: templates/nginx-ingress.yaml
+        equal:
+          path: spec.values.ingress-nginx.controller.service.externalIPs
+          value:
+            - 192.0.2.10
+            - 192.0.2.11
+      - template: templates/nginx-ingress.yaml
+        notExists:
+          path: spec.values.ingress-nginx.controller.service.labels
+      - template: templates/cilium-lb-pool.yaml
+        hasDocuments:
+          count: 0
+
+  - it: legacy config without expose-mode falls back to externalIPs behavior
+    set:
+      _cluster:
+        expose-ingress: tenant-root
+        expose-external-ips: "192.0.2.10"
+    asserts:
+      - template: templates/nginx-ingress.yaml
+        equal:
+          path: spec.values.ingress-nginx.controller.service.type
+          value: ClusterIP
+      - template: templates/cilium-lb-pool.yaml
+        hasDocuments:
+          count: 0
+
+  - it: externalIPs mode in a namespace other than publishing.ingressName renders LoadBalancer fallback without externalIPs
+    set:
+      _cluster:
+        expose-ingress: tenant-root
+        expose-external-ips: "192.0.2.10"
+    release:
+      namespace: tenant-other
+    asserts:
+      - template: templates/nginx-ingress.yaml
+        equal:
+          path: spec.values.ingress-nginx.controller.service.type
+          value: LoadBalancer
+      - template: templates/nginx-ingress.yaml
+        equal:
+          path: spec.values.ingress-nginx.controller.service.externalTrafficPolicy
+          value: Local
+      - template: templates/nginx-ingress.yaml
+        notExists:
+          path: spec.values.ingress-nginx.controller.service.externalIPs
+      - template: templates/cilium-lb-pool.yaml
+        hasDocuments:
+          count: 0
+
+  - it: loadBalancer mode renders LoadBalancer Service with lb-pool label and a v2 CiliumLoadBalancerIPPool
+    set:
+      _cluster:
+        expose-ingress: tenant-root
+        expose-external-ips: "192.0.2.10,192.0.2.11"
+        expose-mode: loadBalancer
+    asserts:
+      - template: templates/nginx-ingress.yaml
+        equal:
+          path: spec.values.ingress-nginx.controller.service.type
+          value: LoadBalancer
+      - template: templates/nginx-ingress.yaml
+        equal:
+          path: spec.values.ingress-nginx.controller.service.externalTrafficPolicy
+          value: Local
+      - template: templates/nginx-ingress.yaml
+        notExists:
+          path: spec.values.ingress-nginx.controller.service.labels
+      - template: templates/nginx-ingress.yaml
+        notExists:
+          path: spec.values.ingress-nginx.controller.service.externalIPs
+      - template: templates/cilium-lb-pool.yaml
+        hasDocuments:
+          count: 1
+      - template: templates/cilium-lb-pool.yaml
+        equal:
+          path: apiVersion
+          value: cilium.io/v2
+      - template: templates/cilium-lb-pool.yaml
+        equal:
+          path: kind
+          value: CiliumLoadBalancerIPPool
+      - template: templates/cilium-lb-pool.yaml
+        equal:
+          path: metadata.name
+          value: root-ingress
+      - template: templates/cilium-lb-pool.yaml
+        equal:
+          path: spec.blocks
+          value:
+            - cidr: 192.0.2.10/32
+            - cidr: 192.0.2.11/32
+      - template: templates/cilium-lb-pool.yaml
+        equal:
+          path: spec.serviceSelector.matchLabels["io.kubernetes.service.namespace"]
+          value: tenant-root
+      - template: templates/cilium-lb-pool.yaml
+        equal:
+          path: spec.serviceSelector.matchLabels["app.kubernetes.io/name"]
+          value: ingress-nginx
+
+  - it: loadBalancer mode with IPv6 address emits /128 CIDR
+    set:
+      _cluster:
+        expose-ingress: tenant-root
+        expose-external-ips: "2001:db8::1"
+        expose-mode: loadBalancer
+    asserts:
+      - template: templates/cilium-lb-pool.yaml
+        equal:
+          path: spec.blocks
+          value:
+            - cidr: 2001:db8::1/128
+
+  - it: loadBalancer mode with mixed IPv4 and IPv6 emits correct CIDR per family
+    set:
+      _cluster:
+        expose-ingress: tenant-root
+        expose-external-ips: "192.0.2.10,2001:db8::1"
+        expose-mode: loadBalancer
+    asserts:
+      - template: templates/cilium-lb-pool.yaml
+        equal:
+          path: spec.blocks
+          value:
+            - cidr: 192.0.2.10/32
+            - cidr: 2001:db8::1/128
+
+  - it: loadBalancer mode without externalIPs fails chart render with explicit message
+    set:
+      _cluster:
+        expose-ingress: tenant-root
+        expose-external-ips: ""
+        expose-mode: loadBalancer
+    asserts:
+      - template: templates/nginx-ingress.yaml
+        failedTemplate:
+          errorMessage: "publishing.exposure=loadBalancer requires publishing.externalIPs to contain at least one non-empty address: the CiliumLoadBalancerIPPool has nothing to advertise and the Service would stay in <pending>."
+
+  - it: unknown exposure mode is rejected with a clear error (case-sensitive enum)
+    set:
+      _cluster:
+        expose-ingress: tenant-root
+        expose-external-ips: "192.0.2.10"
+        expose-mode: LoadBalancer
+    asserts:
+      - template: templates/nginx-ingress.yaml
+        failedTemplate:
+          errorMessage: "unknown publishing.exposure mode \"LoadBalancer\": must be \"externalIPs\" or \"loadBalancer\""
+
+  - it: another typo in exposure mode also fails
+    set:
+      _cluster:
+        expose-ingress: tenant-root
+        expose-external-ips: "192.0.2.10"
+        expose-mode: loadbalancer
+    asserts:
+      - template: templates/nginx-ingress.yaml
+        failedTemplate:
+          errorMessage: "unknown publishing.exposure mode \"loadbalancer\": must be \"externalIPs\" or \"loadBalancer\""
+
+  - it: loadBalancer mode in a namespace other than publishing.ingressName falls back to LoadBalancer Service without pool
+    set:
+      _cluster:
+        expose-ingress: tenant-root
+        expose-external-ips: "192.0.2.10"
+        expose-mode: loadBalancer
+    release:
+      namespace: tenant-other
+    asserts:
+      - template: templates/nginx-ingress.yaml
+        equal:
+          path: spec.values.ingress-nginx.controller.service.type
+          value: LoadBalancer
+      - template: templates/nginx-ingress.yaml
+        equal:
+          path: spec.values.ingress-nginx.controller.service.externalTrafficPolicy
+          value: Local
+      - template: templates/nginx-ingress.yaml
+        notExists:
+          path: spec.values.ingress-nginx.controller.service.labels
+      - template: templates/nginx-ingress.yaml
+        notExists:
+          path: spec.values.ingress-nginx.controller.service.externalIPs
+      - template: templates/cilium-lb-pool.yaml
+        hasDocuments:
+          count: 0
+
+  - it: loadBalancer mode filters out empty entries from externalIPs (trailing comma, leading comma)
+    set:
+      _cluster:
+        expose-ingress: tenant-root
+        expose-external-ips: "192.0.2.10,,192.0.2.11,"
+        expose-mode: loadBalancer
+    asserts:
+      - template: templates/cilium-lb-pool.yaml
+        hasDocuments:
+          count: 1
+      - template: templates/cilium-lb-pool.yaml
+        equal:
+          path: spec.blocks
+          value:
+            - cidr: 192.0.2.10/32
+            - cidr: 192.0.2.11/32
+
+  - it: loadBalancer mode with only-empty externalIPs fails chart render (comma-only input)
+    set:
+      _cluster:
+        expose-ingress: tenant-root
+        expose-external-ips: ",,"
+        expose-mode: loadBalancer
+    asserts:
+      - template: templates/nginx-ingress.yaml
+        failedTemplate:
+          errorMessage: "publishing.exposure=loadBalancer requires publishing.externalIPs to contain at least one non-empty address: the CiliumLoadBalancerIPPool has nothing to advertise and the Service would stay in <pending>."
+
+  - it: externalIPs mode also filters out empty entries (trailing comma)
+    set:
+      _cluster:
+        expose-ingress: tenant-root
+        expose-external-ips: "192.0.2.10,"
+    asserts:
+      - template: templates/nginx-ingress.yaml
+        equal:
+          path: spec.values.ingress-nginx.controller.service.externalIPs
+          value:
+            - 192.0.2.10

From 64a3edff01533a0483db31686a6385dffa81ee55 Mon Sep 17 00:00:00 2001
From: IvanHunters <xorokhotnikov@gmail.com>
Date: Thu, 23 Apr 2026 18:13:10 +0300
Subject: [PATCH 077/151] fix(api): prevent IDOR in TenantNamespace Get and
 Watch handlers

Fixed two IDOR vulnerabilities allowing authenticated users to access
metadata of any tenant namespace without proper authorization.

Changes:
- Added hasAccessToNamespace() for efficient single-namespace access checks
- Get() now verifies access before returning namespace metadata
- Watch() filters events per-namespace with proper authorization
- Returns NotFound (not Forbidden) to prevent tenant enumeration

Performance optimization:
- hasAccessToNamespace() lists RoleBindings only in target namespace
  instead of listing all cluster RoleBindings (order of magnitude faster)
- Watch handler logs authorization errors for security audit

Additional fixes:
- Handle ServiceAccount subjects with empty namespace correctly
- Add klog error logging for failed authorization checks

Signed-off-by: IvanHunters <xorokhotnikov@gmail.com>
---
 pkg/registry/core/tenantnamespace/rest.go | 89 ++++++++++++++++++++++-
 1 file changed, 87 insertions(+), 2 deletions(-)

diff --git a/pkg/registry/core/tenantnamespace/rest.go b/pkg/registry/core/tenantnamespace/rest.go
index f1ed3fab..7724c0e1 100644
--- a/pkg/registry/core/tenantnamespace/rest.go
+++ b/pkg/registry/core/tenantnamespace/rest.go
@@ -24,6 +24,7 @@ import (
 	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/registry/rest"
+	"k8s.io/klog/v2"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	corev1alpha1 "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1"
@@ -123,8 +124,18 @@ func (r *REST) Get(
 		return nil, apierrors.NewNotFound(r.gvr.GroupResource(), name)
 	}
 
+	// Check if user has access to this namespace
+	hasAccess, err := r.hasAccessToNamespace(ctx, name)
+	if err != nil {
+		return nil, err
+	}
+	if !hasAccess {
+		// Return NotFound instead of Forbidden to prevent enumeration
+		return nil, apierrors.NewNotFound(r.gvr.GroupResource(), name)
+	}
+
 	ns := &corev1.Namespace{}
-	err := r.c.Get(ctx, types.NamespacedName{Namespace: "", Name: name}, ns, &client.GetOptions{Raw: opts})
+	err = r.c.Get(ctx, types.NamespacedName{Namespace: "", Name: name}, ns, &client.GetOptions{Raw: opts})
 	if err != nil {
 		return nil, err
 	}
@@ -189,6 +200,17 @@ func (r *REST) Watch(ctx context.Context, opts *metainternal.ListOptions) (watch
 				continue
 			}
 
+			// Check if user has access to this namespace
+			hasAccess, err := r.hasAccessToNamespace(ctx, ns.Name)
+			if err != nil {
+				klog.Errorf("Failed to check access for namespace %s in watch: %v", ns.Name, err)
+				continue
+			}
+			if !hasAccess {
+				// User doesn't have access, skip this event
+				continue
+			}
+
 			out := &corev1alpha1.TenantNamespace{
 				TypeMeta: metav1.TypeMeta{
 					APIVersion: corev1alpha1.SchemeGroupVersion.String(),
@@ -359,7 +381,11 @@ func (r *REST) filterAccessible(
 					break subjectLoop
 				}
 			case "ServiceAccount":
-				if u.GetName() == fmt.Sprintf("system:serviceaccount:%s:%s", subj.Namespace, subj.Name) {
+				saNamespace := subj.Namespace
+				if saNamespace == "" {
+					saNamespace = rbs.Items[i].Namespace
+				}
+				if u.GetName() == fmt.Sprintf("system:serviceaccount:%s:%s", saNamespace, subj.Name) {
 					allowedNameSet[rbs.Items[i].Namespace] = struct{}{}
 					break subjectLoop
 				}
@@ -373,6 +399,65 @@ func (r *REST) filterAccessible(
 	return allowed, nil
 }
 
+// hasAccessToNamespace checks if the user has access to a single namespace.
+// This is optimized for Get/Watch operations where we check one namespace at a time.
+// It lists RoleBindings only in the target namespace instead of all cluster RoleBindings.
+func (r *REST) hasAccessToNamespace(
+	ctx context.Context,
+	namespace string,
+) (bool, error) {
+	u, ok := request.UserFrom(ctx)
+	if !ok {
+		return false, fmt.Errorf("user missing in context")
+	}
+
+	// Check privileged groups
+	groups := make(map[string]struct{})
+	for _, group := range u.GetGroups() {
+		groups[group] = struct{}{}
+	}
+	if _, ok := groups["system:masters"]; ok {
+		return true, nil
+	}
+	if _, ok := groups["cozystack-cluster-admin"]; ok {
+		return true, nil
+	}
+
+	// List RoleBindings only in the target namespace
+	rbs := &rbacv1.RoleBindingList{}
+	err := r.c.List(ctx, rbs, client.InNamespace(namespace))
+	if err != nil {
+		return false, fmt.Errorf("failed to list rolebindings in %s: %w", namespace, err)
+	}
+
+	// Check if user is in any RoleBinding subjects
+	for i := range rbs.Items {
+		for j := range rbs.Items[i].Subjects {
+			subj := rbs.Items[i].Subjects[j]
+			switch subj.Kind {
+			case "Group":
+				if _, ok := groups[subj.Name]; ok {
+					return true, nil
+				}
+			case "User":
+				if subj.Name == u.GetName() {
+					return true, nil
+				}
+			case "ServiceAccount":
+				saNamespace := subj.Namespace
+				if saNamespace == "" {
+					saNamespace = rbs.Items[i].Namespace
+				}
+				if u.GetName() == fmt.Sprintf("system:serviceaccount:%s:%s", saNamespace, subj.Name) {
+					return true, nil
+				}
+			}
+		}
+	}
+
+	return false, nil
+}
+
 // -----------------------------------------------------------------------------
 // Boiler-plate
 // -----------------------------------------------------------------------------

From 5b2501db91c82d8f9ce1d7187325927f7504847c Mon Sep 17 00:00:00 2001
From: IvanHunters <xorokhotnikov@gmail.com>
Date: Thu, 23 Apr 2026 18:17:47 +0300
Subject: [PATCH 078/151] test(api): add security tests for TenantNamespace
 IDOR fix

Added comprehensive unit tests for authorization logic:

- hasAccessToNamespace() tests:
  - User subject access (positive and negative cases)
  - Group subject access
  - ServiceAccount subject access
  - ServiceAccount with empty namespace (defaults to RoleBinding ns)
  - Privileged groups (system:masters, cozystack-cluster-admin)

- Get() handler tests:
  - Returns namespace when user has access
  - Returns NotFound when user lacks access (not Forbidden)
  - Returns NotFound for non-tenant namespaces

All tests verify that authorization correctly enforces RoleBinding-based
access control and prevents IDOR vulnerabilities.

Signed-off-by: IvanHunters <xorokhotnikov@gmail.com>
---
 .../core/tenantnamespace/rest_test.go         | 461 ++++++++++++++++++
 1 file changed, 461 insertions(+)

diff --git a/pkg/registry/core/tenantnamespace/rest_test.go b/pkg/registry/core/tenantnamespace/rest_test.go
index 7f2979bc..eb678949 100644
--- a/pkg/registry/core/tenantnamespace/rest_test.go
+++ b/pkg/registry/core/tenantnamespace/rest_test.go
@@ -3,10 +3,18 @@
 package tenantnamespace
 
 import (
+	"context"
 	"testing"
 
 	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apiserver/pkg/authentication/user"
+	"k8s.io/apiserver/pkg/endpoints/request"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 )
 
 func TestMakeListSortsAlphabetically(t *testing.T) {
@@ -38,3 +46,456 @@ func TestMakeListSortsAlphabetically(t *testing.T) {
 		}
 	}
 }
+
+// Security tests for IDOR fix
+
+func TestHasAccessToNamespace_WithUserAccess(t *testing.T) {
+	scheme := runtime.NewScheme()
+	_ = corev1.AddToScheme(scheme)
+	_ = rbacv1.AddToScheme(scheme)
+
+	ns := &corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"},
+	}
+
+	rb := &rbacv1.RoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-binding",
+			Namespace: "tenant-test",
+		},
+		Subjects: []rbacv1.Subject{
+			{Kind: "User", Name: "test-user", APIGroup: "rbac.authorization.k8s.io"},
+		},
+		RoleRef: rbacv1.RoleRef{
+			APIGroup: "rbac.authorization.k8s.io",
+			Kind:     "Role",
+			Name:     "test-role",
+		},
+	}
+
+	client := fake.NewClientBuilder().
+		WithScheme(scheme).
+		WithObjects(ns, rb).
+		Build()
+
+	r := &REST{
+		c:   client,
+		gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"},
+	}
+
+	u := &user.DefaultInfo{
+		Name:   "test-user",
+		Groups: []string{"system:authenticated"},
+	}
+	ctx := request.WithUser(context.Background(), u)
+
+	hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test")
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if !hasAccess {
+		t.Error("expected user to have access, but got false")
+	}
+}
+
+func TestHasAccessToNamespace_WithoutAccess(t *testing.T) {
+	scheme := runtime.NewScheme()
+	_ = corev1.AddToScheme(scheme)
+	_ = rbacv1.AddToScheme(scheme)
+
+	ns := &corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"},
+	}
+
+	// RoleBinding for different user
+	rb := &rbacv1.RoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-binding",
+			Namespace: "tenant-test",
+		},
+		Subjects: []rbacv1.Subject{
+			{Kind: "User", Name: "other-user", APIGroup: "rbac.authorization.k8s.io"},
+		},
+		RoleRef: rbacv1.RoleRef{
+			APIGroup: "rbac.authorization.k8s.io",
+			Kind:     "Role",
+			Name:     "test-role",
+		},
+	}
+
+	client := fake.NewClientBuilder().
+		WithScheme(scheme).
+		WithObjects(ns, rb).
+		Build()
+
+	r := &REST{
+		c:   client,
+		gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"},
+	}
+
+	u := &user.DefaultInfo{
+		Name:   "test-user",
+		Groups: []string{"system:authenticated"},
+	}
+	ctx := request.WithUser(context.Background(), u)
+
+	hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test")
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if hasAccess {
+		t.Error("expected user to NOT have access, but got true")
+	}
+}
+
+func TestHasAccessToNamespace_WithGroupAccess(t *testing.T) {
+	scheme := runtime.NewScheme()
+	_ = corev1.AddToScheme(scheme)
+	_ = rbacv1.AddToScheme(scheme)
+
+	ns := &corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"},
+	}
+
+	rb := &rbacv1.RoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-binding",
+			Namespace: "tenant-test",
+		},
+		Subjects: []rbacv1.Subject{
+			{Kind: "Group", Name: "test-group", APIGroup: "rbac.authorization.k8s.io"},
+		},
+		RoleRef: rbacv1.RoleRef{
+			APIGroup: "rbac.authorization.k8s.io",
+			Kind:     "Role",
+			Name:     "test-role",
+		},
+	}
+
+	client := fake.NewClientBuilder().
+		WithScheme(scheme).
+		WithObjects(ns, rb).
+		Build()
+
+	r := &REST{
+		c:   client,
+		gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"},
+	}
+
+	u := &user.DefaultInfo{
+		Name:   "test-user",
+		Groups: []string{"system:authenticated", "test-group"},
+	}
+	ctx := request.WithUser(context.Background(), u)
+
+	hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test")
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if !hasAccess {
+		t.Error("expected user to have access via group, but got false")
+	}
+}
+
+func TestHasAccessToNamespace_SystemMasters(t *testing.T) {
+	scheme := runtime.NewScheme()
+	_ = corev1.AddToScheme(scheme)
+	_ = rbacv1.AddToScheme(scheme)
+
+	ns := &corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"},
+	}
+
+	client := fake.NewClientBuilder().
+		WithScheme(scheme).
+		WithObjects(ns).
+		Build()
+
+	r := &REST{
+		c:   client,
+		gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"},
+	}
+
+	u := &user.DefaultInfo{
+		Name:   "admin",
+		Groups: []string{"system:masters"},
+	}
+	ctx := request.WithUser(context.Background(), u)
+
+	hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test")
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if !hasAccess {
+		t.Error("expected system:masters to have access, but got false")
+	}
+}
+
+func TestHasAccessToNamespace_CozyAdminGroup(t *testing.T) {
+	scheme := runtime.NewScheme()
+	_ = corev1.AddToScheme(scheme)
+	_ = rbacv1.AddToScheme(scheme)
+
+	ns := &corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"},
+	}
+
+	client := fake.NewClientBuilder().
+		WithScheme(scheme).
+		WithObjects(ns).
+		Build()
+
+	r := &REST{
+		c:   client,
+		gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"},
+	}
+
+	u := &user.DefaultInfo{
+		Name:   "cozy-admin",
+		Groups: []string{"cozystack-cluster-admin"},
+	}
+	ctx := request.WithUser(context.Background(), u)
+
+	hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test")
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if !hasAccess {
+		t.Error("expected cozystack-cluster-admin to have access, but got false")
+	}
+}
+
+func TestHasAccessToNamespace_ServiceAccount(t *testing.T) {
+	scheme := runtime.NewScheme()
+	_ = corev1.AddToScheme(scheme)
+	_ = rbacv1.AddToScheme(scheme)
+
+	ns := &corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"},
+	}
+
+	rb := &rbacv1.RoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-binding",
+			Namespace: "tenant-test",
+		},
+		Subjects: []rbacv1.Subject{
+			{Kind: "ServiceAccount", Name: "test-sa", Namespace: "tenant-test"},
+		},
+		RoleRef: rbacv1.RoleRef{
+			APIGroup: "rbac.authorization.k8s.io",
+			Kind:     "Role",
+			Name:     "test-role",
+		},
+	}
+
+	client := fake.NewClientBuilder().
+		WithScheme(scheme).
+		WithObjects(ns, rb).
+		Build()
+
+	r := &REST{
+		c:   client,
+		gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"},
+	}
+
+	u := &user.DefaultInfo{
+		Name:   "system:serviceaccount:tenant-test:test-sa",
+		Groups: []string{"system:authenticated"},
+	}
+	ctx := request.WithUser(context.Background(), u)
+
+	hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test")
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if !hasAccess {
+		t.Error("expected service account to have access, but got false")
+	}
+}
+
+func TestHasAccessToNamespace_ServiceAccountEmptyNamespace(t *testing.T) {
+	scheme := runtime.NewScheme()
+	_ = corev1.AddToScheme(scheme)
+	_ = rbacv1.AddToScheme(scheme)
+
+	ns := &corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"},
+	}
+
+	// ServiceAccount subject with empty namespace should default to RoleBinding namespace
+	rb := &rbacv1.RoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-binding",
+			Namespace: "tenant-test",
+		},
+		Subjects: []rbacv1.Subject{
+			{Kind: "ServiceAccount", Name: "test-sa", Namespace: ""}, // Empty namespace
+		},
+		RoleRef: rbacv1.RoleRef{
+			APIGroup: "rbac.authorization.k8s.io",
+			Kind:     "Role",
+			Name:     "test-role",
+		},
+	}
+
+	client := fake.NewClientBuilder().
+		WithScheme(scheme).
+		WithObjects(ns, rb).
+		Build()
+
+	r := &REST{
+		c:   client,
+		gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"},
+	}
+
+	u := &user.DefaultInfo{
+		Name:   "system:serviceaccount:tenant-test:test-sa",
+		Groups: []string{"system:authenticated"},
+	}
+	ctx := request.WithUser(context.Background(), u)
+
+	hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test")
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if !hasAccess {
+		t.Error("expected service account with empty namespace to have access, but got false")
+	}
+}
+
+func TestGet_WithAccess(t *testing.T) {
+	scheme := runtime.NewScheme()
+	_ = corev1.AddToScheme(scheme)
+	_ = rbacv1.AddToScheme(scheme)
+
+	ns := &corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"},
+	}
+
+	rb := &rbacv1.RoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-binding",
+			Namespace: "tenant-test",
+		},
+		Subjects: []rbacv1.Subject{
+			{Kind: "User", Name: "test-user", APIGroup: "rbac.authorization.k8s.io"},
+		},
+		RoleRef: rbacv1.RoleRef{
+			APIGroup: "rbac.authorization.k8s.io",
+			Kind:     "Role",
+			Name:     "test-role",
+		},
+	}
+
+	client := fake.NewClientBuilder().
+		WithScheme(scheme).
+		WithObjects(ns, rb).
+		Build()
+
+	r := &REST{
+		c:   client,
+		gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"},
+	}
+
+	u := &user.DefaultInfo{
+		Name:   "test-user",
+		Groups: []string{"system:authenticated"},
+	}
+	ctx := request.WithUser(context.Background(), u)
+
+	obj, err := r.Get(ctx, "tenant-test", &metav1.GetOptions{})
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if obj == nil {
+		t.Fatal("expected object, got nil")
+	}
+}
+
+func TestGet_WithoutAccess(t *testing.T) {
+	scheme := runtime.NewScheme()
+	_ = corev1.AddToScheme(scheme)
+	_ = rbacv1.AddToScheme(scheme)
+
+	ns := &corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"},
+	}
+
+	// RoleBinding for different user
+	rb := &rbacv1.RoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-binding",
+			Namespace: "tenant-test",
+		},
+		Subjects: []rbacv1.Subject{
+			{Kind: "User", Name: "other-user", APIGroup: "rbac.authorization.k8s.io"},
+		},
+		RoleRef: rbacv1.RoleRef{
+			APIGroup: "rbac.authorization.k8s.io",
+			Kind:     "Role",
+			Name:     "test-role",
+		},
+	}
+
+	client := fake.NewClientBuilder().
+		WithScheme(scheme).
+		WithObjects(ns, rb).
+		Build()
+
+	r := &REST{
+		c:   client,
+		gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"},
+	}
+
+	u := &user.DefaultInfo{
+		Name:   "test-user",
+		Groups: []string{"system:authenticated"},
+	}
+	ctx := request.WithUser(context.Background(), u)
+
+	obj, err := r.Get(ctx, "tenant-test", &metav1.GetOptions{})
+	if err == nil {
+		t.Fatal("expected error, got nil")
+	}
+	if obj != nil {
+		t.Errorf("expected nil object, got %v", obj)
+	}
+
+	// Verify it returns NotFound (not Forbidden) to prevent enumeration
+	if !apierrors.IsNotFound(err) {
+		t.Errorf("expected NotFound error, got %v", err)
+	}
+}
+
+func TestGet_NonTenantNamespace(t *testing.T) {
+	scheme := runtime.NewScheme()
+	_ = corev1.AddToScheme(scheme)
+	_ = rbacv1.AddToScheme(scheme)
+
+	client := fake.NewClientBuilder().
+		WithScheme(scheme).
+		Build()
+
+	r := &REST{
+		c:   client,
+		gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"},
+	}
+
+	u := &user.DefaultInfo{
+		Name:   "test-user",
+		Groups: []string{"system:masters"},
+	}
+	ctx := request.WithUser(context.Background(), u)
+
+	obj, err := r.Get(ctx, "default", &metav1.GetOptions{})
+	if err == nil {
+		t.Fatal("expected error for non-tenant namespace, got nil")
+	}
+	if obj != nil {
+		t.Errorf("expected nil object, got %v", obj)
+	}
+	if !apierrors.IsNotFound(err) {
+		t.Errorf("expected NotFound error, got %v", err)
+	}
+}

From 9d552d4086726d2f5c6de5adaa2783082893a8c5 Mon Sep 17 00:00:00 2001
From: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
Date: Thu, 23 Apr 2026 22:19:35 +0500
Subject: [PATCH 079/151] ci(api): broaden codegen drift trigger paths and
 detect untracked files

- Add generated-output dirs (pkg/generated, internal/crdinstall/manifests,
  packages/system/*/definitions) and Makefile to the workflow paths: filter
  so PRs that modify only generated artifacts still trigger the drift check.
- Mirror the same paths in the root pre-commit hook's files: regex so
  manual edits to generated files or changes to the root generate target
  re-run make generate through pre-commit.
- Switch drift detection in the workflow from `git diff --exit-code` to
  `git status --porcelain` so new untracked files produced by make generate
  (e.g. generated YAML/Go for a new API type) also fail the job; dump
  `git diff --color=always` on failure for easier debugging.

Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
---
 .github/workflows/codegen-drift.yml | 10 +++++++++-
 .pre-commit-config.yaml             |  2 +-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/codegen-drift.yml b/.github/workflows/codegen-drift.yml
index b08f5d7e..a26caf5e 100644
--- a/.github/workflows/codegen-drift.yml
+++ b/.github/workflows/codegen-drift.yml
@@ -6,8 +6,15 @@ on:
     paths:
       - 'api/**'
       - 'pkg/apis/**'
+      - 'pkg/generated/**'
+      - 'internal/crdinstall/manifests/**'
+      - 'packages/system/cozystack-controller/definitions/**'
+      - 'packages/system/application-definition-crd/definition/**'
+      - 'packages/system/backup-controller/definitions/**'
+      - 'packages/system/backupstrategy-controller/definitions/**'
       - 'hack/update-codegen.sh'
       - 'hack/boilerplate.go.txt'
+      - 'Makefile'
       - 'go.mod'
       - 'go.sum'
       - '.github/workflows/codegen-drift.yml'
@@ -46,8 +53,9 @@ jobs:
 
       - name: Fail on drift
         run: |
-          if ! git diff --exit-code; then
+          if [ -n "$(git status --porcelain)" ]; then
             echo "::error::'make generate' produced changes. Run 'make generate' locally and commit the result."
             git status --short
+            git diff --color=always
             exit 1
           fi
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 836f79c5..6492b92d 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -10,7 +10,7 @@ repos:
           git diff --color=always | cat
         '
       language: system
-      files: ^(api/|pkg/apis/|hack/update-codegen\.sh$|hack/boilerplate\.go\.txt$)
+      files: ^(api/|pkg/apis/|pkg/generated/|internal/crdinstall/manifests/|packages/system/cozystack-controller/definitions/|packages/system/application-definition-crd/definition/|packages/system/backup-controller/definitions/|packages/system/backupstrategy-controller/definitions/|hack/update-codegen\.sh$|hack/boilerplate\.go\.txt$|Makefile$)
       pass_filenames: false
     - id: run-make-generate
       name: Run 'make generate' in all app directories

From 3c5521ee992b2dce74ff8956a6d8c581763bf640 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Thu, 23 Apr 2026 22:16:30 +0300
Subject: [PATCH 080/151] fix(hami): remove broken hami-dra subchart

The hami-dra subchart renders resources even when dra.enabled=false
because Helm dependency conditions don't work for vendored subcharts.
The subchart also contains duplicate YAML label keys
(app.kubernetes.io/component, app.kubernetes.io/name) and references
unpublished container images (v0.0.1-dev). DRA support can be
re-added when the upstream project stabilizes it.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 .../charts/hami/charts/hami-dra/.helmignore   |  24 --
 .../charts/hami/charts/hami-dra/Chart.yaml    |  18 -
 .../charts/hami-dra/templates/_commons.tpl    |  84 ----
 .../charts/hami-dra/templates/_helpers.tpl    |  73 ----
 .../hami-dra-driver/deviceclass.yaml          |  11 -
 .../nvidia-dra-driver-daemonset.yaml          | 145 -------
 .../templates/hami-dra-driver/rbac.yaml       | 110 -----
 .../monitor/monitor-clusterrole.yaml          |  11 -
 .../monitor/monitor-clusterrolebinding.yaml   |  15 -
 .../templates/monitor/monitor-deployment.yaml |  62 ---
 .../templates/monitor/monitor-service.yaml    |  26 --
 .../monitor/monitor-serviceaccount.yaml       |   8 -
 .../templates/webhook/cert-manager.yaml       |  29 --
 .../templates/webhook/cert-secret.yaml        |  13 -
 .../templates/webhook/device-config.yaml      | 394 ------------------
 .../webhook/mutatingwebhookconfiguration.yaml |  28 --
 .../validatingwebhookconfiguration.yaml       |  29 --
 .../webhook/webhook-clusterrole.yaml          |  14 -
 .../webhook/webhook-clusterrolebinding.yaml   |  12 -
 .../templates/webhook/webhook-deployment.yaml |  59 ---
 .../templates/webhook/webhook-service.yaml    |  15 -
 .../webhook/webhook-serviceaccount.yaml       |   5 -
 .../charts/hami/charts/hami-dra/values.yaml   | 166 --------
 23 files changed, 1351 deletions(-)
 delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/.helmignore
 delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml
 delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/_commons.tpl
 delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/_helpers.tpl
 delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/deviceclass.yaml
 delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/nvidia-dra-driver-daemonset.yaml
 delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/rbac.yaml
 delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrole.yaml
 delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrolebinding.yaml
 delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-deployment.yaml
 delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-service.yaml
 delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-serviceaccount.yaml
 delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-manager.yaml
 delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-secret.yaml
 delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/device-config.yaml
 delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/mutatingwebhookconfiguration.yaml
 delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/validatingwebhookconfiguration.yaml
 delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrole.yaml
 delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrolebinding.yaml
 delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-deployment.yaml
 delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-service.yaml
 delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-serviceaccount.yaml
 delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/values.yaml

diff --git a/packages/system/hami/charts/hami/charts/hami-dra/.helmignore b/packages/system/hami/charts/hami/charts/hami-dra/.helmignore
deleted file mode 100644
index 898df488..00000000
--- a/packages/system/hami/charts/hami/charts/hami-dra/.helmignore
+++ /dev/null
@@ -1,24 +0,0 @@
-# Patterns to ignore when building packages.
-# This supports shell glob matching, relative path matching, and
-# negation (prefixed with !). Only one pattern per line.
-.DS_Store
-# Common VCS dirs
-.git/
-.gitignore
-.bzr/
-.bzrignore
-.hg/
-.hgignore
-.svn/
-# Common backup files
-*.swp
-*.bak
-*.tmp
-*.orig
-*~
-# Various IDEs
-.project
-.idea/
-*.tmproj
-.vscode/
-
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml b/packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml
deleted file mode 100644
index d01f678f..00000000
--- a/packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml
+++ /dev/null
@@ -1,18 +0,0 @@
-apiVersion: v2
-appVersion: 0.1.0
-description: A Helm chart for HAMi DRA
-home: https://github.com/Project-HAMi/HAMi-DRA
-keywords:
-- webhook
-- kubernetes
-- admission-controller
-- hami
-- dra
-maintainers:
-- name: hami-dra
-  url: https://github.com/Project-HAMi/HAMi-DRA
-name: hami-dra
-sources:
-- https://github.com/Project-HAMi/HAMi-DRA
-type: application
-version: 0.1.0
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/_commons.tpl b/packages/system/hami/charts/hami/charts/hami-dra/templates/_commons.tpl
deleted file mode 100644
index 02fbed20..00000000
--- a/packages/system/hami/charts/hami/charts/hami-dra/templates/_commons.tpl
+++ /dev/null
@@ -1,84 +0,0 @@
-{{/* vim: set filetype=mustache: */}}
-
-{{/*
-Return the proper image name
-{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" $) }}
-*/}}
-{{- define "common.images.image" -}}
-{{- $registryName := .imageRoot.registry -}}
-{{- $repositoryName := .imageRoot.repository -}}
-{{- $tag := .imageRoot.tag | toString -}}
-{{- if .global }}
-    {{- if .global.imageRegistry }}
-     {{- $registryName = .global.imageRegistry -}}
-    {{- end -}}
-{{- end -}}
-{{- if .tag }}
-    {{- if .tag.imageTag }}
-     {{- $tag = .tag.imageTag -}}
-    {{- end -}}
-{{- end -}}
-{{- if $registryName }}
-{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}}
-{{- else -}}
-{{- printf "%s:%s" $repositoryName $tag -}}
-{{- end -}}
-{{- end -}}
-
-{{/*
-Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead)
-{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }}
-*/}}
-{{- define "common.images.pullSecrets" -}}
-  {{- $pullSecrets := list }}
-
-  {{- if .global }}
-    {{- range .global.imagePullSecrets -}}
-      {{- $pullSecrets = append $pullSecrets . -}}
-    {{- end -}}
-  {{- end -}}
-
-  {{- range .images -}}
-    {{- range .pullSecrets -}}
-      {{- $pullSecrets = append $pullSecrets . -}}
-    {{- end -}}
-  {{- end -}}
-
-  {{- if (not (empty $pullSecrets)) }}
-imagePullSecrets:
-    {{- range $pullSecrets }}
-  - name: {{ . }}
-    {{- end }}
-  {{- end }}
-{{- end -}}
-
-{{/*
-Renders a value that contains template.
-Usage:
-{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $) }}
-*/}}
-{{- define "common.tplvalues.render" -}}
-    {{- if typeIs "string" .value }}
-        {{- tpl .value .context }}
-    {{- else }}
-        {{- tpl (.value | toYaml) .context }}
-    {{- end }}
-{{- end -}}
-
-{{/*
-Create a default fully qualified app name.
-We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
-If release name contains chart name it will be used as a full name.
-*/}}
-{{- define "common.names.fullname" -}}
-{{- if .Values.fullnameOverride -}}
-{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
-{{- else -}}
-{{- $name := default .Chart.Name .Values.nameOverride -}}
-{{- if contains $name .Release.Name -}}
-{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
-{{- else -}}
-{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-{{- end -}}
-{{- end -}}
\ No newline at end of file
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/_helpers.tpl b/packages/system/hami/charts/hami/charts/hami-dra/templates/_helpers.tpl
deleted file mode 100644
index 99beec77..00000000
--- a/packages/system/hami/charts/hami/charts/hami-dra/templates/_helpers.tpl
+++ /dev/null
@@ -1,73 +0,0 @@
-{{- define "hami.dra.webhook.fullname" -}}
-{{- printf "%s-%s" (include "common.names.fullname" .) "webhook" | trunc 63 | trimSuffix "-" -}}
-{{- end }}
-
-{{- define "hami.dra.webhook.image" -}}
-{{ include "common.images.image" (dict "imageRoot" .Values.webhook.image "global" .Values.global) }}
-{{- end -}}
-
-{{- define "hami.dra.webhook.imagePullSecrets" -}}
-{{ include "common.images.pullSecrets" (dict "images" (list .Values.webhook.image) "global" .Values.global) }}
-{{- end -}}
-
-{{- define "hami.dra.driver.nvidia.image" -}}
-{{ include "common.images.image" (dict "imageRoot" .Values.drivers.nvidia.image "global" .Values.global) }}
-{{- end -}}
-
-{{- define "hami.dra.driver.nvidia.imagePullSecrets" -}}
-{{ include "common.images.pullSecrets" (dict "images" (list .Values.drivers.nvidia.image) "global" .Values.global) }}
-{{- end -}}
-
-{{/*
-Common labels
-*/}}
-{{- define "hami-dra-webhook.labels" -}}
-helm.sh/chart: {{ .Chart.Name }}
-{{ include "hami-dra-webhook.selectorLabels" . }}
-{{- if .Chart.AppVersion }}
-app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
-{{- end }}
-app.kubernetes.io/managed-by: {{ .Release.Service }}
-{{- end }}
-
-{{/*
-Selector labels
-*/}}
-{{- define "hami-dra-webhook.selectorLabels" -}}
-app.kubernetes.io/name: {{ .Release.Name }}
-app.kubernetes.io/instance: {{ .Release.Name }}
-app.kubernetes.io/component: webhook
-{{- end }}
-
-{{- define "hami.dra.monitor.fullname" -}}
-{{- printf "%s-%s" (include "common.names.fullname" .) "monitor" | trunc 63 | trimSuffix "-" -}}
-{{- end }}
-
-{{- define "hami.dra.monitor.image" -}}
-{{ include "common.images.image" (dict "imageRoot" .Values.monitor.image "global" .Values.global) }}
-{{- end -}}
-
-{{- define "hami.dra.monitor.imagePullSecrets" -}}
-{{ include "common.images.pullSecrets" (dict "images" (list .Values.monitor.image) "global" .Values.global) }}
-{{- end -}}
-
-{{/*
-Common labels for monitor
-*/}}
-{{- define "hami-dra-monitor.labels" -}}
-helm.sh/chart: {{ .Chart.Name }}
-{{ include "hami-dra-monitor.selectorLabels" . }}
-{{- if .Chart.AppVersion }}
-app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
-{{- end }}
-app.kubernetes.io/managed-by: {{ .Release.Service }}
-{{- end }}
-
-{{/*
-Selector labels for monitor
-*/}}
-{{- define "hami-dra-monitor.selectorLabels" -}}
-app.kubernetes.io/name: {{ .Release.Name }}
-app.kubernetes.io/instance: {{ .Release.Name }}
-app.kubernetes.io/component: monitor
-{{- end }}
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/deviceclass.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/deviceclass.yaml
deleted file mode 100644
index bcb6d2c5..00000000
--- a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/deviceclass.yaml
+++ /dev/null
@@ -1,11 +0,0 @@
-{{- if .Values.drivers.nvidia.enabled }}
-apiVersion: resource.k8s.io/v1
-kind: DeviceClass
-metadata:
-  name: hami-core-gpu.project-hami.io
-spec:
-  selectors:
-  - cel:
-      expression: |-
-        device.driver == "hami-core-gpu.project-hami.io" && device.attributes["hami-core-gpu.project-hami.io"].type == "hami-gpu"
-{{- end }}
\ No newline at end of file
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/nvidia-dra-driver-daemonset.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/nvidia-dra-driver-daemonset.yaml
deleted file mode 100644
index a3fb0d01..00000000
--- a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/nvidia-dra-driver-daemonset.yaml
+++ /dev/null
@@ -1,145 +0,0 @@
-{{- if .Values.drivers.nvidia.enabled }}
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: hami-dra-driver-kubelet-plugin
-  namespace: {{ .Release.Namespace }}
-spec:
-  revisionHistoryLimit: 10
-  selector:
-    matchLabels:
-      hami-dra-driver-component: kubelet-plugin
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/instance: hami-dra-driver
-        app.kubernetes.io/name: hami-dra-driver
-        hami-dra-driver-component: kubelet-plugin
-    spec:
-      {{- include "hami.dra.driver.nvidia.imagePullSecrets" . | nindent 6 }}
-      initContainers:
-      - name: init-container
-        image: {{ include "hami.dra.driver.nvidia.image" . }}
-        securityContext:
-          privileged: true
-        command: [bash, /usr/bin/kubelet-plugin-prestart.sh]
-        env:
-        - name: NVIDIA_DRIVER_ROOT
-          value: {{ if .Values.drivers.nvidia.containerDriver }}/run/nvidia/driver{{ else }}/{{ end }}
-        - name: NVIDIA_VISIBLE_DEVICES
-          value: void
-        - name: KUBELET_REGISTRAR_DIRECTORY_PATH
-          value: /var/lib/kubelet/plugins_registry
-        - name: KUBELET_PLUGINS_DIRECTORY_PATH
-          value: /var/lib/kubelet/plugins
-        volumeMounts:
-        - name: driver-root-parent
-          mountPath: /driver-root-parent
-          readOnly: true
-      containers:
-      - args:
-        - |-
-          # Conditionally mask the params file to prevent this container from
-          # recreating any missing GPU device nodes. This is necessary, for
-          # example, when running under nvkind to limit the set GPUs governed
-          # by the plugin even though it has cgroup access to all of them.
-          if [ "${MASK_NVIDIA_DRIVER_PARAMS}" = "true" ]; then
-            cp /proc/driver/nvidia/params root/gpu-params
-            sed -i 's/^ModifyDeviceFiles: 1$/ModifyDeviceFiles: 0/' root/gpu-params
-            mount --bind root/gpu-params /proc/driver/nvidia/params
-          fi
-          hami-kubelet-plugin -v 6
-        command:
-        - bash
-        - -c
-        env:
-        - name: MASK_NVIDIA_DRIVER_PARAMS
-          value: ""
-        - name: NVIDIA_DRIVER_ROOT
-          value: {{ if .Values.drivers.nvidia.containerDriver }}/run/nvidia/driver{{ else }}/{{ end }}
-        - name: NVIDIA_VISIBLE_DEVICES
-          value: void
-        - name: CDI_ROOT
-          value: /var/run/cdi
-        - name: NVIDIA_MIG_CONFIG_DEVICES
-          value: all
-        - name: NODE_NAME
-          valueFrom:
-            fieldRef:
-              apiVersion: v1
-              fieldPath: spec.nodeName
-        - name: NAMESPACE
-          valueFrom:
-            fieldRef:
-              apiVersion: v1
-              fieldPath: metadata.namespace
-        - name: KUBELET_REGISTRAR_DIRECTORY_PATH
-          value: /var/lib/kubelet/plugins_registry
-        - name: KUBELET_PLUGINS_DIRECTORY_PATH
-          value: /var/lib/kubelet/plugins
-        - name: IMAGE_NAME
-          value: {{ include "hami.dra.driver.nvidia.image" . }}
-        lifecycle:
-          postStart:
-            exec:
-              command: ["/bin/bash", "-c", "/usr/bin/vgpu-init.sh /usr/local/vgpu/"]
-        image: {{ include "hami.dra.driver.nvidia.image" . }}
-        imagePullPolicy: {{ .Values.drivers.nvidia.image.pullPolicy | default "IfNotPresent" }}
-        name: gpus
-        securityContext:
-          privileged: true
-        terminationMessagePath: /dev/termination-log
-        terminationMessagePolicy: File
-        volumeMounts:
-        - mountPath: /var/lib/kubelet/plugins_registry
-          name: plugins-registry
-        - mountPath: /var/lib/kubelet/plugins
-          mountPropagation: Bidirectional
-          name: plugins
-        - mountPath: /var/run/cdi
-          name: cdi
-        - mountPath: /driver-root
-          mountPropagation: HostToContainer
-          name: driver-root
-          readOnly: true
-        - mountPath: /usr/local/vgpu
-          name: host-vgpu
-        - mountPath: /tmp
-          name: host-tmp
-      dnsPolicy: ClusterFirst
-      serviceAccount: hami-dra-driver-service-account
-      serviceAccountName: hami-dra-driver-service-account
-      volumes:
-      - hostPath:
-          path: /var/lib/kubelet/plugins_registry
-          type: ""
-        name: plugins-registry
-      - hostPath:
-          path: /var/lib/kubelet/plugins
-          type: ""
-        name: plugins
-      - hostPath:
-          path: /var/run/cdi
-          type: ""
-        name: cdi
-      - hostPath:
-          path: {{if .Values.drivers.nvidia.containerDriver }}/run/nvidia{{ else }}/{{ end }}
-          type: DirectoryOrCreate
-        name: driver-root-parent
-      - hostPath:
-          path: {{if .Values.drivers.nvidia.containerDriver }}/run/nvidia/driver{{ else }}/{{ end }}
-          type: DirectoryOrCreate
-        name: driver-root
-      - hostPath:
-          path: /dev
-          type: ""
-        name: host-dev
-      - hostPath:
-          path: /usr/local/vgpu
-          type: DirectoryOrCreate
-        name: host-vgpu
-      - hostPath:
-          path: /tmp
-          type: ""
-        name: host-tmp
-{{- end }}
\ No newline at end of file
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/rbac.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/rbac.yaml
deleted file mode 100644
index 435c37cc..00000000
--- a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/rbac.yaml
+++ /dev/null
@@ -1,110 +0,0 @@
-{{- if .Values.drivers.nvidia.enabled }}
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: hami-dra-driver-service-account
-  namespace: {{ .Release.Namespace }}
----
-apiVersion: v1
-items:
-- apiVersion: rbac.authorization.k8s.io/v1
-  kind: Role
-  metadata:
-    name: hami-dra-driver-role
-    namespace: {{ .Release.Namespace }}
-  rules:
-  - apiGroups:
-    - apps
-    resources:
-    - daemonsets
-    - deployments
-    verbs:
-    - get
-    - list
-    - watch
-    - create
-    - update
-    - patch
-    - delete
-kind: List
-metadata:
-  resourceVersion: ""
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: hami-dra-driver-role-binding
-  namespace: {{ .Release.Namespace }}
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: hami-dra-driver-role
-subjects:
-- kind: ServiceAccount
-  name: hami-dra-driver-service-account
-  namespace: {{ .Release.Namespace }}
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: hami-dra-driver-clusterrole
-rules:
-- apiGroups:
-  - resource.k8s.io
-  resources:
-  - resourceclaims
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - resource.k8s.io
-  resources:
-  - resourceclaimtemplates
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - update
-  - delete
-- apiGroups:
-  - resource.k8s.io
-  resources:
-  - resourceslices
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - update
-  - delete
-- apiGroups:
-  - resource.k8s.io
-  resources:
-  - resourceclaims/status
-  verbs:
-  - update
-- apiGroups:
-  - ""
-  resources:
-  - nodes
-  verbs:
-  - get
-  - list
-  - watch
-  - update
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: hami-dra-driver-clusterrole-binding-hami-dra-driver
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: hami-dra-driver-clusterrole
-subjects:
-- kind: ServiceAccount
-  name: hami-dra-driver-service-account
-  namespace: {{ .Release.Namespace }}
-{{- end }}
\ No newline at end of file
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrole.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrole.yaml
deleted file mode 100644
index 45c24863..00000000
--- a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrole.yaml
+++ /dev/null
@@ -1,11 +0,0 @@
-{{- if .Values.monitor.enabled }}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: {{ include "hami.dra.monitor.fullname" . }}
-rules:
-- apiGroups: ["resource.k8s.io"]
-  resources: ["resourceslices", "resourceclaims"]
-  verbs: ["get", "list", "watch"]
-{{- end }}
-
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrolebinding.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrolebinding.yaml
deleted file mode 100644
index 766d3098..00000000
--- a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrolebinding.yaml
+++ /dev/null
@@ -1,15 +0,0 @@
-{{- if .Values.monitor.enabled }}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: {{ include "hami.dra.monitor.fullname" . }}
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: {{ include "hami.dra.monitor.fullname" . }}
-subjects:
-  - kind: ServiceAccount
-    name: {{ include "hami.dra.monitor.fullname" . }}
-    namespace: {{ .Release.Namespace }}
-{{- end }}
-
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-deployment.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-deployment.yaml
deleted file mode 100644
index b544b376..00000000
--- a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-deployment.yaml
+++ /dev/null
@@ -1,62 +0,0 @@
-{{- if .Values.monitor.enabled }}
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: {{ include "hami.dra.monitor.fullname" . }}
-  namespace: {{ .Release.Namespace }}
-  labels:
-    app.kubernetes.io/name: {{ include "hami.dra.monitor.fullname" . }}
-    {{- include "hami-dra-monitor.labels" . | nindent 4 }}
-spec:
-  replicas: {{ .Values.monitor.replicas | default 1 }}
-  selector:
-    matchLabels:
-      {{- include "hami-dra-monitor.selectorLabels" . | nindent 6 }}
-  template:
-    metadata:
-      labels:
-        {{- include "hami-dra-monitor.selectorLabels" . | nindent 8 }}
-    spec:
-      {{- include "hami.dra.monitor.imagePullSecrets" . | nindent 6 }}
-      serviceAccountName: {{ include "hami.dra.monitor.fullname" . }}
-      containers:
-        - name: monitor
-          image: {{ include "hami.dra.monitor.image" . }}
-          imagePullPolicy: {{ .Values.monitor.image.pullPolicy | default "IfNotPresent" }}
-          command:
-            - /bin/monitor
-            - --v={{ .Values.monitor.logLevel | default 2 }}
-            - --metrics-bind-address={{ .Values.monitor.metricsBindAddress | default ":8080" }}
-            - --health-probe-bind-address={{ .Values.monitor.healthProbeBindAddress | default ":8000" }}
-            - --kube-api-qps={{ .Values.monitor.kubeAPIQPS | default 40.0 }}
-            - --kube-api-burst={{ .Values.monitor.kubeAPIBurst | default 60 }}
-            - --collect-interval={{ .Values.monitor.collectInterval | default "30s" }}
-          ports:
-            - containerPort: 8080
-              name: metrics
-              protocol: TCP
-            - containerPort: 8000
-              name: health
-              protocol: TCP
-          {{- if .Values.monitor.resources }}
-          resources:
-            {{- toYaml .Values.monitor.resources | nindent 12 }}
-          {{- end }}
-          livenessProbe:
-            httpGet:
-              path: /healthz
-              port: 8000
-            initialDelaySeconds: 10
-            periodSeconds: 10
-            timeoutSeconds: 5
-            failureThreshold: 3
-          readinessProbe:
-            httpGet:
-              path: /readyz
-              port: 8000
-            initialDelaySeconds: 5
-            periodSeconds: 5
-            timeoutSeconds: 3
-            failureThreshold: 3
-{{- end }}
-
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-service.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-service.yaml
deleted file mode 100644
index d740cc38..00000000
--- a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-service.yaml
+++ /dev/null
@@ -1,26 +0,0 @@
-{{- if .Values.monitor.enabled }}
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ include "hami.dra.monitor.fullname" . }}
-  namespace: {{ .Release.Namespace }}
-  labels:
-    {{- include "hami-dra-monitor.selectorLabels" . | nindent 4 }}
-spec:
-  type: {{ .Values.monitor.service.type | default "ClusterIP" }}
-  selector:
-    {{- include "hami-dra-monitor.selectorLabels" . | nindent 4 }}
-  ports:
-    - port: 8080
-      targetPort: 8080
-      name: metrics
-      protocol: TCP
-      {{- if and (eq (.Values.monitor.service.type | default "ClusterIP") "NodePort") .Values.monitor.service.nodePort.metrics }}
-      nodePort: {{ .Values.monitor.service.nodePort.metrics }}
-      {{- end }}
-    - port: 8000
-      targetPort: 8000
-      name: health
-      protocol: TCP
-{{- end }}
-
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-serviceaccount.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-serviceaccount.yaml
deleted file mode 100644
index c61ad454..00000000
--- a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-serviceaccount.yaml
+++ /dev/null
@@ -1,8 +0,0 @@
-{{- if .Values.monitor.enabled }}
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: {{ include "hami.dra.monitor.fullname" . }}
-  namespace: {{ .Release.Namespace }}
-{{- end }}
-
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-manager.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-manager.yaml
deleted file mode 100644
index dd31c71d..00000000
--- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-manager.yaml
+++ /dev/null
@@ -1,29 +0,0 @@
-{{- if .Values.certs.certManager.enabled }}
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: {{ .Release.Name }}-dra-webhook-serving-cert
-  namespace: {{ .Release.Namespace }}
-  labels:
-    {{- include "hami-dra-webhook.labels" . | nindent 4 }}
-spec:
-  dnsNames:
-    - {{ .Release.Name }}-dra-webhook.{{ .Release.Namespace }}.svc
-    - {{ .Release.Name }}-dra-webhook.{{ .Release.Namespace }}.svc.cluster.local
-  issuerRef:
-    kind: Issuer
-    name: {{ .Release.Name }}-dra-webhook-selfsigned-issuer
-  secretName: {{ .Release.Name }}-dra-webhook-tls
-  privateKey:
-    rotationPolicy: {{ .Values.certs.certManager.privateKeyRotationPolicy | default "Never" }}
----
-apiVersion: cert-manager.io/v1
-kind: Issuer
-metadata:
-  name: {{ .Release.Name }}-dra-webhook-selfsigned-issuer
-  namespace: {{ .Release.Namespace }}
-  labels:
-    app.kubernetes.io/component: hami-dra-webhook
-spec:
-  selfSigned: {}
-{{- end }}
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-secret.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-secret.yaml
deleted file mode 100644
index eadbb517..00000000
--- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-secret.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
-{{- if eq .Values.certs.mode "custom" }}
-apiVersion: v1
-kind: Secret
-metadata:
-  name: {{ include "hami-dra-webhook.tlsSecretName" . }}
-  namespace: {{ .Release.Namespace }}
-type: kubernetes.io/tls
-data:
-  tls.crt: |
-    {{- .Values.certs.custom.crt | b64enc | indent 4 }}
-  tls.key: |
-    {{- .Values.certs.custom.key | b64enc | indent 4 }}
-{{- end }}
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/device-config.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/device-config.yaml
deleted file mode 100644
index 97283f3e..00000000
--- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/device-config.yaml
+++ /dev/null
@@ -1,394 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: {{ include "hami.dra.webhook.fullname" . }}-device-config
-  namespace: {{ .Release.Namespace }}
-  labels:
-    app.kubernetes.io/component: hami-scheduler
-    {{- include "hami-dra-webhook.selectorLabels" . | nindent 4 }}
-data:
-  device-config.yaml: |-
-  {{- if .Files.Glob "files/device-config.yaml" }}
-  {{- .Files.Get "files/device-config.yaml" | nindent 4}}
-  {{- else }}
-    nvidia:
-      resourceCountName: {{ .Values.resourceName }}
-      resourceMemoryName: {{ .Values.resourceMem }}
-      resourceMemoryPercentageName: {{ .Values.resourceMemPercentage }}
-      resourceCoreName: {{ .Values.resourceCores }}
-      resourcePriorityName: {{ .Values.resourcePriority }}
-      overwriteEnv: false
-      defaultMemory: 0
-      defaultCores: 0
-      defaultGPUNum: 1
-      knownMigGeometries:
-      - models: [ "A30" ]
-        allowedGeometries:
-          - 
-            - name: 1g.6gb
-              core: 25
-              memory: 6144
-              count: 4
-          - 
-            - name: 2g.12gb
-              core: 50
-              memory: 12288
-              count: 2
-          - 
-            - name: 4g.24gb
-              core: 100
-              memory: 24576
-              count: 1
-      - models: [ "A100-SXM4-40GB", "A100-40GB-PCIe", "A100-PCIE-40GB"]
-        allowedGeometries:
-          - 
-            - name: 1g.5gb
-              core: 14
-              memory: 5120
-              count: 7
-          - 
-            - name: 1g.5gb
-              core: 14
-              memory: 5120
-              count: 1
-            - name: 2g.10gb
-              core: 28
-              memory: 10240
-              count: 3
-          - 
-            - name: 3g.20gb
-              core: 42
-              memory: 20480
-              count: 2
-          - 
-            - name: 7g.40gb
-              core: 100
-              memory: 40960
-              count: 1
-      - models: [ "A100-SXM4-80GB", "A100-80GB-PCIe", "A100-PCIE-80GB"]
-        allowedGeometries:
-          - 
-            - name: 1g.10gb
-              core: 14
-              memory: 10240
-              count: 7
-          - 
-            - name: 1g.10gb
-              core: 14
-              memory: 10240
-              count: 1
-            - name: 2g.20gb
-              core: 28
-              memory: 20480
-              count: 3
-          - 
-            - name: 3g.40gb
-              core: 42
-              memory: 40960
-              count: 2
-          - 
-            - name: 7g.79gb
-              core: 100
-              memory: 80896
-              count: 1
-      - models: [ "H100-PCIE-80GB", "H100-SXM5-80GB"]
-        allowedGeometries:
-          - 
-            - name: 1g.10gb
-              core: 14
-              memory: 10240
-              count: 7
-          - 
-            - name: 1g.10gb
-              core: 14
-              memory: 10240
-              count: 1
-            - name: 2g.20gb
-              core: 28
-              memory: 20480
-              count: 3
-          - 
-            - name: 3g.40gb
-              core: 42
-              memory: 40960
-              count: 2
-          - 
-            - name: 7g.80gb
-              core: 100
-              memory: 81920
-              count: 1
-      - models: [ "H100-PCIE-94GB", "H100-SXM5-94GB"]
-        allowedGeometries:
-          - 
-            - name: 1g.12gb
-              core: 14
-              memory: 12288
-              count: 7
-          - 
-            - name: 1g.12gb
-              core: 14
-              memory: 12288
-              count: 1
-            - name: 2g.24gb
-              core: 28
-              memory: 24576
-              count: 3
-          - 
-            - name: 3g.47gb
-              core: 42
-              memory: 48128
-              count: 2
-          - 
-            - name: 7g.94gb
-              core: 100
-              memory: 96256
-              count: 1
-      - models: [ "H20", "H100 on GH200"]
-        allowedGeometries:
-          - 
-            - name: 1g.12gb
-              core: 14
-              memory: 12288
-              count: 7
-          - 
-            - name: 1g.12gb
-              core: 14
-              memory: 12288
-              count: 1
-            - name: 2g.24gb
-              core: 28
-              memory: 24576
-              count: 3 
-          - 
-            - name: 3g.48gb
-              core: 42
-              memory: 49152
-              count: 2
-          - 
-            - name: 7g.96gb
-              core: 100
-              memory: 98304
-              count: 1
-      - models: [ "H200 NVL", "H200-SXM5"]
-        allowedGeometries:
-          - 
-            - name: 1g.18gb
-              core: 14
-              memory: 18432
-              count: 7
-          - 
-            - name: 1g.18gb
-              core: 14
-              memory: 18432
-              count: 1
-            - name: 2g.35gb
-              core: 28
-              memory: 35840
-              count: 3
-          - 
-            - name: 3g.71gb
-              core: 42
-              memory: 72704
-              count: 2
-          - 
-            - name: 7g.141gb
-              core: 100
-              memory: 144384
-              count: 1
-      - models: [ "B200" ]
-        allowedGeometries:
-          - 
-            - name: 1g.23gb
-              core: 14
-              memory: 23552
-              count: 7
-          - 
-            - name: 1g.23gb
-              core: 14
-              memory: 23552
-              count: 1
-            - name: 2g.45gb
-              core: 28
-              memory: 46080
-              count: 3
-          - 
-            - name: 3g.90gb
-              core: 42
-              memory: 92160
-              count: 2
-          - 
-            - name: 7g.180gb
-              core: 100
-              memory: 184320
-              count: 1
-    cambricon:
-      resourceCountName: {{ .Values.mluResourceName }}
-      resourceMemoryName: {{ .Values.mluResourceMem }}
-      resourceCoreName: {{ .Values.mluResourceCores }}
-    hygon:
-      resourceCountName: {{ .Values.dcuResourceName }}
-      resourceMemoryName: {{ .Values.dcuResourceMem }}
-      resourceCoreName: {{ .Values.dcuResourceCores }}
-    metax:
-      resourceCountName: "metax-tech.com/gpu"
-      resourceVCountName: {{ .Values.metaxResourceName }}
-      resourceVMemoryName: {{ .Values.metaxResourceMem }}
-      resourceVCoreName: {{ .Values.metaxResourceCore }}
-      sgpuTopologyAware: {{ .Values.metaxsGPUTopologyAware }}
-    enflame:
-      resourceNameGCU: "enflame.com/gcu"
-      resourceNameVGCU: {{ .Values.enflameResourceNameVGCU }}
-      resourceNameVGCUPercentage: {{ .Values.enflameResourceNameVGCUPercentage }}
-    mthreads:
-      resourceCountName: "mthreads.com/vgpu"
-      resourceMemoryName: "mthreads.com/sgpu-memory"
-      resourceCoreName: "mthreads.com/sgpu-core"
-    iluvatars:
-    - chipName: MR-V100
-      commonWord: MR-V100
-      resourceCountName: iluvatar.ai/MR-V100-vgpu
-      resourceMemoryName: iluvatar.ai/MR-V100.vMem
-      resourceCoreName: iluvatar.ai/MR-V100.vCore
-    - chipName: MR-V50
-      commonWord: MR-V50
-      resourceCountName: iluvatar.ai/MR-V50-vgpu
-      resourceMemoryName: iluvatar.ai/MR-V50.vMem
-      resourceCoreName: iluvatar.ai/MR-V50.vCore
-    - chipName: BI-V150
-      commonWord: BI-V150
-      resourceCountName: iluvatar.ai/BI-V150-vgpu
-      resourceMemoryName: iluvatar.ai/BI-V150.vMem
-      resourceCoreName: iluvatar.ai/BI-V150.vCore
-    - chipName: BI-V100
-      commonWord: BI-V100
-      resourceCountName: iluvatar.ai/BI-V100-vgpu
-      resourceMemoryName: iluvatar.ai/BI-V100.vMem
-      resourceCoreName: iluvatar.ai/BI-V100.vCore
-    kunlun:
-      resourceCountName: {{ .Values.kunlunResourceName }}
-      resourceVCountName: {{ .Values.kunlunResourceVCountName }}
-      resourceVMemoryName: {{ .Values.kunlunResourceVMemoryName }}
-    awsneuron:
-      resourceCountName: "aws.amazon.com/neuron"
-      resourceCoreName: "aws.amazon.com/neuroncore"
-    amd:
-      resourceCountName: "amd.com/gpu"
-    vnpus:
-    - chipName: 910A
-      commonWord: Ascend910A
-      resourceName: huawei.com/Ascend910A
-      resourceMemoryName: huawei.com/Ascend910A-memory
-      memoryAllocatable: 32768
-      memoryCapacity: 32768
-      aiCore: 30
-      templates:
-        - name: vir02
-          memory: 2184
-          aiCore: 2
-        - name: vir04
-          memory: 4369
-          aiCore: 4
-        - name: vir08
-          memory: 8738
-          aiCore: 8
-        - name: vir16
-          memory: 17476
-          aiCore: 16
-    - chipName: 910B2
-      commonWord: Ascend910B2
-      resourceName: huawei.com/Ascend910B2
-      resourceMemoryName: huawei.com/Ascend910B2-memory
-      memoryAllocatable: 65536
-      memoryCapacity: 65536
-      aiCore: 24
-      aiCPU: 6
-      templates:
-        - name: vir03_1c_8g
-          memory: 8192
-          aiCore: 3
-          aiCPU: 1
-        - name: vir06_1c_16g
-          memory: 16384
-          aiCore: 6
-          aiCPU: 1
-        - name: vir12_3c_32g
-          memory: 32768
-          aiCore: 12
-          aiCPU: 3
-    - chipName: 910B3
-      commonWord: Ascend910B3
-      resourceName: huawei.com/Ascend910B3
-      resourceMemoryName: huawei.com/Ascend910B3-memory
-      memoryAllocatable: 65536
-      memoryCapacity: 65536
-      aiCore: 20
-      aiCPU: 7
-      templates:
-        - name: vir05_1c_16g
-          memory: 16384
-          aiCore: 5
-          aiCPU: 1
-        - name: vir10_3c_32g
-          memory: 32768
-          aiCore: 10
-          aiCPU: 3
-    - chipName: 910B4-1
-      commonWord: Ascend910B4-1
-      resourceName: huawei.com/Ascend910B4-1
-      resourceMemoryName: huawei.com/Ascend910B4-1-memory
-      memoryAllocatable: 65536
-      memoryCapacity: 65536
-      aiCore: 20
-      aiCPU: 7
-      templates:
-        # NOTE: Names of vnpu template for 910B4-1 are fixed by Ascend runtime and must not be changed.
-        # The memory is used for scheduling so the correct values must be set.
-        # Template vir05_1c_8g actually provides 16GB memory,
-        - name: vir05_1c_8g
-          memory: 16384
-          aiCore: 5
-          aiCPU: 1
-        # Template vir10_3c_16g actually provides 32GB memory
-        - name: vir10_3c_16g
-          memory: 32768
-          aiCore: 10
-          aiCPU: 3
-    - chipName: 910B4
-      commonWord: Ascend910B4
-      resourceName: huawei.com/Ascend910B4
-      resourceMemoryName: huawei.com/Ascend910B4-memory
-      memoryAllocatable: 32768
-      memoryCapacity: 32768
-      aiCore: 20
-      aiCPU: 7
-      templates:
-        - name: vir05_1c_8g
-          memory: 8192
-          aiCore: 5
-          aiCPU: 1
-        - name: vir10_3c_16g
-          memory: 16384
-          aiCore: 10
-          aiCPU: 3
-    - chipName: 310P3
-      commonWord: Ascend310P
-      resourceName: huawei.com/Ascend310P
-      resourceMemoryName: huawei.com/Ascend310P-memory
-      memoryAllocatable: 21527
-      memoryCapacity: 24576
-      aiCore: 8
-      aiCPU: 7
-      templates:
-        - name: vir01
-          memory: 3072
-          aiCore: 1
-          aiCPU: 1
-        - name: vir02
-          memory: 6144
-          aiCore: 2
-          aiCPU: 2
-        - name: vir04
-          memory: 12288
-          aiCore: 4
-          aiCPU: 4
-  {{ end }}
\ No newline at end of file
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/mutatingwebhookconfiguration.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/mutatingwebhookconfiguration.yaml
deleted file mode 100644
index c763b6aa..00000000
--- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/mutatingwebhookconfiguration.yaml
+++ /dev/null
@@ -1,28 +0,0 @@
-{{- if .Values.webhook.config.mutating.enabled }}
-apiVersion: admissionregistration.k8s.io/v1
-kind: MutatingWebhookConfiguration
-metadata:
-  name: {{ .Release.Name }}-mutatingwebhookconfiguration
-  annotations:
-    cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ .Release.Name }}-dra-webhook-serving-cert
-webhooks:
-  - name: mutate.hami.io
-    admissionReviewVersions: ["v1"]
-    clientConfig:
-      service:
-        name: {{ .Release.Name }}-dra-webhook
-        namespace: {{ .Release.Namespace }}
-        path: /mutate
-    rules:
-      - apiGroups:
-          - ""
-        apiVersions:
-          - v1
-        operations:
-          - CREATE
-        resources:
-          - pods
-        scope: '*'
-    sideEffects: None
-    timeoutSeconds: 10
-{{- end }}
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/validatingwebhookconfiguration.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/validatingwebhookconfiguration.yaml
deleted file mode 100644
index acdcbb96..00000000
--- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/validatingwebhookconfiguration.yaml
+++ /dev/null
@@ -1,29 +0,0 @@
-{{- if .Values.webhook.config.validating.enabled }}
-apiVersion: admissionregistration.k8s.io/v1
-kind: ValidatingWebhookConfiguration
-metadata:
-  name: {{ .Release.Name }}-validatingwebhookconfiguration
-  annotations:
-    cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ .Release.Name }}-dra-webhook-serving-cert
-webhooks:
-  - name: validate.hami.io
-    admissionReviewVersions: ["v1"]
-    clientConfig:
-      service:
-        name: {{ .Release.Name }}-dra-webhook
-        namespace: {{ .Release.Namespace }}
-        path: /validate
-    rules:
-      - apiGroups:
-          - ""
-        apiVersions:
-          - v1
-        operations:
-          - DELETE
-        resources:
-          - pods
-        scope: '*'
-    sideEffects: None
-    timeoutSeconds: 10
-    failurePolicy: Ignore
-{{- end }}
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrole.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrole.yaml
deleted file mode 100644
index 28916a36..00000000
--- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrole.yaml
+++ /dev/null
@@ -1,14 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: {{ include "hami.dra.webhook.fullname" . }}
-rules:
-- apiGroups: [""]
-  resources: ["pods"]
-  verbs: ["get", "list", "watch"]
-- apiGroups: ["admissionregistration.k8s.io"]
-  resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"]
-  verbs: ["get", "list", "watch"]
-- apiGroups: ["resource.k8s.io"]
-  resources: ["resourceclaims"]
-  verbs: ["*"]
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrolebinding.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrolebinding.yaml
deleted file mode 100644
index daad52f2..00000000
--- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrolebinding.yaml
+++ /dev/null
@@ -1,12 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: {{ include "hami.dra.webhook.fullname" . }}
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: {{ include "hami.dra.webhook.fullname" . }}
-subjects:
-  - kind: ServiceAccount
-    name: {{ include "hami.dra.webhook.fullname" . }}
-    namespace: {{ .Release.Namespace }}
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-deployment.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-deployment.yaml
deleted file mode 100644
index 8e3cf90c..00000000
--- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-deployment.yaml
+++ /dev/null
@@ -1,59 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: {{ include "hami.dra.webhook.fullname" . }}
-  namespace: {{ .Release.Namespace }}
-  labels:
-    app.kubernetes.io/name: {{ include "hami.dra.webhook.fullname" . }}
-    {{- include "hami-dra-webhook.labels" . | nindent 4 }}
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      {{- include "hami-dra-webhook.selectorLabels" . | nindent 6 }}
-  template:
-    metadata:
-      labels:
-        {{- include "hami-dra-webhook.selectorLabels" . | nindent 8 }}
-    spec:
-      {{- include "hami.dra.webhook.imagePullSecrets" . | nindent 6 }}
-      serviceAccountName: {{ include "hami.dra.webhook.fullname" . }}
-      containers:
-        - name: webhook
-          image: {{ include "hami.dra.webhook.image" . }}
-          imagePullPolicy: {{ .Values.webhook.image.pullPolicy | default "IfNotPresent" }}
-          command:
-            - /bin/webhook
-            - --v=5
-            - --bind-address=0.0.0.0
-            - --secure-port=8443
-            - --cert-dir=/tls
-            - --tls-cert-file-name=tls.crt
-            - --tls-private-key-file-name=tls.key
-            - --metrics-bind-address=:8080
-            - --health-probe-bind-address=:8000
-            - --device-config-file=/device-config.yaml
-          ports:
-            - containerPort: 8443
-              name: webhook
-              protocol: TCP
-            - containerPort: 8080
-              name: metrics
-              protocol: TCP
-            - containerPort: 8000
-              name: health
-              protocol: TCP
-          volumeMounts:
-            - name: device-config
-              mountPath: /device-config.yaml
-              subPath: device-config.yaml
-            - name: tls-config
-              mountPath: /tls
-              readOnly: true
-      volumes:
-        - name: device-config
-          configMap:
-            name: {{ include "hami.dra.webhook.fullname" . }}-device-config
-        - name: tls-config
-          secret:
-            secretName: {{ if .Values.certs.certManager.enabled }}{{ .Release.Name }}-dra-webhook-tls{{ else }}{{ .Release.Name }}-tls{{ end }}
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-service.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-service.yaml
deleted file mode 100644
index 290174ec..00000000
--- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-service.yaml
+++ /dev/null
@@ -1,15 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ .Release.Name }}-dra-webhook
-  namespace: {{ .Release.Namespace }}
-  labels:
-    {{- include "hami-dra-webhook.selectorLabels" . | nindent 4 }}
-spec:
-  type: "ClusterIP"
-  selector:
-    {{- include "hami-dra-webhook.selectorLabels" . | nindent 4 }}
-  ports:
-    - port: 443
-      targetPort: 8443
-      name: webhook
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-serviceaccount.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-serviceaccount.yaml
deleted file mode 100644
index b0dc2fb2..00000000
--- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-serviceaccount.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: {{ include "hami.dra.webhook.fullname" . }}
-  namespace: {{ .Release.Namespace }}
diff --git a/packages/system/hami/charts/hami/charts/hami-dra/values.yaml b/packages/system/hami/charts/hami/charts/hami-dra/values.yaml
deleted file mode 100644
index ebbbfed7..00000000
--- a/packages/system/hami/charts/hami/charts/hami-dra/values.yaml
+++ /dev/null
@@ -1,166 +0,0 @@
-## @section Global configuration
-## @param global.imageRegistry Global Docker image registry
-## @param global.imagePullSecrets Global Docker image pull secrets
-global:
-  ## @param global.imageRegistry Global Docker image registry
-  imageRegistry: ""
-  ## E.g.
-  ## imagePullSecrets:
-  ##   - myRegistryKeySecretName
-  ## @param global.imagePullSecrets Global Docker image pull secrets
-  imagePullSecrets: []
-
-#Nvidia GPU Parameters
-resourceName: "nvidia.com/gpu"
-resourceMem: "nvidia.com/gpumem"
-resourceMemPercentage: "nvidia.com/gpumem-percentage"
-resourceCores: "nvidia.com/gpucores"
-resourcePriority: "nvidia.com/priority"
-
-#MLU Parameters
-mluResourceName: "cambricon.com/vmlu"
-mluResourceMem: "cambricon.com/mlu.smlu.vmemory"
-mluResourceCores: "cambricon.com/mlu.smlu.vcore"
-
-#Hygon DCU Parameters
-dcuResourceName: "hygon.com/dcunum"
-dcuResourceMem: "hygon.com/dcumem"
-dcuResourceCores: "hygon.com/dcucores"
-
-#Metax sGPU Parameters
-metaxResourceName: "metax-tech.com/sgpu"
-metaxResourceCore: "metax-tech.com/vcore"
-metaxResourceMem: "metax-tech.com/vmemory"
-metaxsGPUTopologyAware: "false"
-
-#Enflame VGCU Parameters
-enflameResourceNameVGCU: "enflame.com/vgcu"
-enflameResourceNameVGCUPercentage: "enflame.com/vgcu-percentage"
-
-#Kunlun XPU Parameters
-kunlunResourceName: "kunlunxin.com/xpu"
-kunlunResourceVCountName: "kunlunxin.com/vxpu"
-kunlunResourceVMemoryName: "kunlunxin.com/vxpu-memory"
-
-# Webhook deployment configuration
-webhook:
-  image:
-    registry: ghcr.io
-    repository: project-hami/hami-dra-webhook
-    tag: "v0.1.0"
-    ## Specify a imagePullPolicy
-    ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
-    ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
-    ##
-    pullPolicy: Always
-    ## Optionally specify an array of imagePullSecrets.
-    ## Secrets must be manually created in the namespace.
-    ## Example:
-    ## pullSecrets:
-    ##   - myRegistryKeySecretName
-    ##
-    pullSecrets: []
-  ## @param resources controllerManager resource requests and limits
-  resources:
-  # If you do want to specify resources, uncomment the following
-  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
-    limits:
-      cpu: 500m
-      memory: 512Mi
-    requests:
-      cpu: 100m
-      memory: 128Mi
-  ## @param nodeSelector controllerManager node labels for pod assignment
-  args:
-    webhookName: ""  # Default: {{ .Release.Name }}-hami-dra-webhook
-    secretName: ""  # Default: {{ .Release.Name }}-hami-dra-webhook-tls
-  # Webhook configuration
-  config:
-    mutating:
-      enabled: true
-    validating:
-      enabled: true
-
-# Monitor deployment configuration
-monitor:
-  enabled: true
-  replicas: 1
-  image:
-    registry: ghcr.io
-    repository: project-hami/hami-dra-monitor
-    tag: "v0.1.0"
-    ## Specify a imagePullPolicy
-    ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
-    ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
-    ##
-    pullPolicy: Always
-    ## Optionally specify an array of imagePullSecrets.
-    ## Secrets must be manually created in the namespace.
-    ## Example:
-    ## pullSecrets:
-    ##   - myRegistryKeySecretName
-    ##
-    pullSecrets: []
-  ## @param resources monitor resource requests and limits
-  resources:
-    limits:
-      cpu: 500m
-      memory: 512Mi
-    requests:
-      cpu: 100m
-      memory: 128Mi
-  ## Monitor configuration
-  logLevel: 2
-  metricsBindAddress: ":8080"
-  healthProbeBindAddress: ":8000"
-  kubeAPIQPS: 40.0
-  kubeAPIBurst: 60
-  collectInterval: "30s"
-  ## Monitor service configuration
-  service:
-    ## Service type: ClusterIP, NodePort, or LoadBalancer
-    ## Defaults to ClusterIP
-    type: ClusterIP
-    ## NodePort configuration (only used when type is NodePort)
-    nodePort:
-      ## Metrics port (NodePort). If not specified, Kubernetes will assign a random port
-      metrics: ""
-
-# Certificate configuration
-certs:
-  # Custom certificate (used when mode is "custom")
-  custom:
-    crt: ""
-    key: ""
-  # Cert-manager configuration (used when mode is "cert-manager")
-  certManager:
-    enabled: true
-    # Private key rotation policy: "Never" or "Always"
-    # In cert-manager >= v1.18.0, the default changed from "Never" to "Always"
-    # Setting to "Never" avoids frequent certificate rotation for webhooks
-    privateKeyRotationPolicy: "Never"
-    issuer:
-      clusterIssuerName: ""  # Required when type is "clusterIssuer"
-
-# DRA Drivers configuration
-drivers:
-  nvidia:
-    enabled: true
-    # If you are using gpu driver on host, you need to set this to false
-    containerDriver: true
-    image:
-      registry: ghcr.io
-      repository: projecthami/k8s-dra-driver
-      tag: "v0.0.1-dev"
-      ## Specify a imagePullPolicy
-      ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
-      ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
-      ##
-      pullPolicy: IfNotPresent
-      ## Optionally specify an array of imagePullSecrets.
-      ## Secrets must be manually created in the namespace.
-      ## Example:
-      ## pullSecrets:
-      ##   - myRegistryKeySecretName
-      ##
-      pullSecrets: []

From 385ea7a17f8cfc4526c50db036c105c63a5817fc Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Thu, 23 Apr 2026 22:16:34 +0300
Subject: [PATCH 081/151] feat(platform): register HAMi as optional system
 package

Add PackageSource for HAMi with dependency on gpu-operator.
Include HAMi in the iaas bundle as an optional package, allowing
host-level deployment alongside the existing kubernetes addon path.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 packages/core/platform/sources/hami.yaml      | 24 +++++++++++++++++++
 .../core/platform/templates/bundles/iaas.yaml |  1 +
 2 files changed, 25 insertions(+)
 create mode 100644 packages/core/platform/sources/hami.yaml

diff --git a/packages/core/platform/sources/hami.yaml b/packages/core/platform/sources/hami.yaml
new file mode 100644
index 00000000..0184e405
--- /dev/null
+++ b/packages/core/platform/sources/hami.yaml
@@ -0,0 +1,24 @@
+---
+apiVersion: cozystack.io/v1alpha1
+kind: PackageSource
+metadata:
+  name: cozystack.hami
+spec:
+  sourceRef:
+    kind: OCIRepository
+    name: cozystack-packages
+    namespace: cozy-system
+    path: /
+  variants:
+  - name: default
+    dependsOn:
+    - cozystack.gpu-operator
+    components:
+    - name: hami
+      path: system/hami
+      valuesFiles:
+      - values.yaml
+      install:
+        privileged: true
+        namespace: cozy-hami
+        releaseName: hami
diff --git a/packages/core/platform/templates/bundles/iaas.yaml b/packages/core/platform/templates/bundles/iaas.yaml
index ced0a322..66ec98b2 100644
--- a/packages/core/platform/templates/bundles/iaas.yaml
+++ b/packages/core/platform/templates/bundles/iaas.yaml
@@ -10,6 +10,7 @@
 {{include "cozystack.platform.package.default" (list "cozystack.kubevirt-cdi" $) }}
 {{include "cozystack.platform.package.optional.default" (list "cozystack.vm-default-images" $) }}
 {{include "cozystack.platform.package.optional.default" (list "cozystack.gpu-operator" $) }}
+{{include "cozystack.platform.package.optional.default" (list "cozystack.hami" $) }}
 {{include "cozystack.platform.package.default" (list "cozystack.kamaji" $) }}
 {{include "cozystack.platform.package.default" (list "cozystack.capi-operator" $) }}
 {{include "cozystack.platform.package.default" (list "cozystack.capi-provider-bootstrap-kubeadm" $) }}

From 6af74cec0ea436cbd5aefce5e5c8bedf86e5b81a Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Thu, 23 Apr 2026 23:24:38 +0300
Subject: [PATCH 082/151] fix(ingress): accept pre-CIDR externalIPs in
 CiliumLoadBalancerIPPool

Address review feedback from gemini-code-assist on
packages/extra/ingress/templates/cilium-lb-pool.yaml:19: if the operator
passes an externalIP already in CIDR form (192.0.2.10/32 or 2001:db8::1/128),
the template appended a second /32 or /128 suffix producing an invalid
CiliumLoadBalancerIPPool block. Guard the suffix append on the absence of
"/" in the input.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 .../extra/ingress/templates/cilium-lb-pool.yaml    |  2 +-
 packages/extra/ingress/tests/exposure_test.yaml    | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/packages/extra/ingress/templates/cilium-lb-pool.yaml b/packages/extra/ingress/templates/cilium-lb-pool.yaml
index a383922b..048eb58b 100644
--- a/packages/extra/ingress/templates/cilium-lb-pool.yaml
+++ b/packages/extra/ingress/templates/cilium-lb-pool.yaml
@@ -16,7 +16,7 @@ metadata:
 spec:
   blocks:
   {{- range $exposeIPsList }}
-  - cidr: {{ . }}/{{ if contains ":" . }}128{{ else }}32{{ end }}
+  - cidr: {{ . }}{{ if not (contains "/" .) }}/{{ if contains ":" . }}128{{ else }}32{{ end }}{{ end }}
   {{- end }}
   serviceSelector:
     matchLabels:
diff --git a/packages/extra/ingress/tests/exposure_test.yaml b/packages/extra/ingress/tests/exposure_test.yaml
index 780e0715..323c8e68 100644
--- a/packages/extra/ingress/tests/exposure_test.yaml
+++ b/packages/extra/ingress/tests/exposure_test.yaml
@@ -249,3 +249,17 @@ tests:
           path: spec.values.ingress-nginx.controller.service.externalIPs
           value:
             - 192.0.2.10
+
+  - it: loadBalancer mode accepts pre-CIDR input without double-suffixing
+    set:
+      _cluster:
+        expose-ingress: tenant-root
+        expose-external-ips: "192.0.2.10/32,2001:db8::1/128"
+        expose-mode: loadBalancer
+    asserts:
+      - template: templates/cilium-lb-pool.yaml
+        equal:
+          path: spec.blocks
+          value:
+            - cidr: 192.0.2.10/32
+            - cidr: 2001:db8::1/128

From 9b87bda06af20d7c1ca8d96b8f338f3bc937d0c5 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Fri, 24 Apr 2026 03:14:41 +0300
Subject: [PATCH 083/151] fix(postgres-operator): block HelmRelease Ready until
 the cnpg webhook actually serves
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The cnpg admission webhook's controller pod passes readinessProbe as soon
as the local HTTPS server binds to :9443. The HelmRelease marks itself
Ready right after that via helm install --wait. But the Service
cnpg-webhook-service needs its EndpointSlice populated and the data plane
(kube-proxy / cilium) programmed before kube-apiserver can reach the
webhook through the Service ClusterIP. That gap is short but not zero,
and any HelmRelease that depends on postgres-operator (cozy-keycloak,
tenant Postgres apps) can fire its own install inside the window and hit

  Internal error occurred: failed calling webhook "mcluster.cnpg.io":
  failed to call webhook: Post "https://cnpg-webhook-service.cozy-postgres-operator.svc:443/...":
  dial tcp <svc-ip>:443: connect: connection refused

which fails the install of the downstream release. Seen on cozystack/cozystack#2470
E2E run 24862782568.

Add a post-install,post-upgrade Helm hook Job that blocks the release
from reporting Ready until the webhook answers /readyz through the
apiserver service proxy. Apiserver proxy routes the call over the same
Service IP → EndpointSlice → pod path the admission webhook uses, so
once it responds, the webhook admission path is also working.

RBAC is minimal: a dedicated ServiceAccount with a ClusterRole that only
grants get on services/proxy scoped to https:cnpg-webhook-service:webhook-server.
The Job times out after 120s with 60 attempts at 2s intervals — longer
than any data-plane programming delay seen on E2E, but bounded.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 packages/system/postgres-operator/Makefile    |   3 +
 .../templates/webhook-ready-hook.yaml         | 139 ++++++++
 .../tests/webhook-ready-hook_test.yaml        | 308 ++++++++++++++++++
 packages/system/postgres-operator/values.yaml |  25 ++
 4 files changed, 475 insertions(+)
 create mode 100644 packages/system/postgres-operator/templates/webhook-ready-hook.yaml
 create mode 100644 packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml

diff --git a/packages/system/postgres-operator/Makefile b/packages/system/postgres-operator/Makefile
index f6d7ddaa..5279f8a7 100644
--- a/packages/system/postgres-operator/Makefile
+++ b/packages/system/postgres-operator/Makefile
@@ -3,6 +3,9 @@ export NAMESPACE=cozy-$(NAME)
 
 include ../../../hack/package.mk
 
+test:
+	helm unittest .
+
 update:
 	rm -rf charts
 	helm repo add cnpg https://cloudnative-pg.github.io/charts
diff --git a/packages/system/postgres-operator/templates/webhook-ready-hook.yaml b/packages/system/postgres-operator/templates/webhook-ready-hook.yaml
new file mode 100644
index 00000000..2ad4e7c9
--- /dev/null
+++ b/packages/system/postgres-operator/templates/webhook-ready-hook.yaml
@@ -0,0 +1,139 @@
+{{- /*
+  Post-install gate: block the HelmRelease from reporting Ready until the
+  cnpg admission webhook actually serves through the cluster Service. Helm
+  --wait on the controller pod passes once its readinessProbe passes, but
+  EndpointSlice propagation and kube-proxy/cilium data-plane programming
+  can lag by a second or two — long enough for any HelmRelease that
+  depends on postgres-operator (e.g. cozy-keycloak, tenant Postgres apps)
+  to fire its own install and have kube-apiserver hit the mcluster.cnpg.io
+  mutating webhook with "dial tcp <svc-ip>:443: connect: connection refused".
+
+  The Job uses the apiserver service proxy, which exercises the same
+  endpoint-resolution and apiserver-initiated pod dial that the admission
+  webhook path uses. Once /readyz answers through the proxy the data-plane
+  race is resolved. It does not verify the webhook's TLS CA bundle, so
+  this gate is scoped to reachability regressions, not cert rotation.
+
+  The service name and port name are hardcoded literals. Upstream cnpg
+  pins the service name in charts/cloudnative-pg/values.yaml with a
+  comment "DO NOT CHANGE THE SERVICE NAME as it is currently used to
+  generate the certificate and can not be configured". The port name is
+  fixed in charts/cloudnative-pg/templates/service.yaml (ports[0].name:
+  webhook-server). If a future `make update` ever changes either literal
+  upstream, the sync-check helm-unittest test
+  (tests/webhook-ready-hook_test.yaml) renders the subchart Service and
+  fails if the literal drifts — forcing this template to be updated in
+  the same change.
+*/}}
+{{- $_ := required "webhookReady.image.repository must be set to the container image providing kubectl for the post-install readiness Job" .Values.webhookReady.image.repository -}}
+{{- $_ := required "webhookReady.image.tag must be set for the post-install readiness Job" .Values.webhookReady.image.tag -}}
+{{- /* $svcName and $portName are hardcoded literals; see header comment. */ -}}
+{{- $svcName := "cnpg-webhook-service" -}}
+{{- /* $portName is the service port NAME, not number — matches ports[0].name in the vendored subchart's Service. */ -}}
+{{- $portName := "webhook-server" -}}
+{{- $resourceName := printf "https:%s:%s" $svcName $portName -}}
+{{- $maxAttempts := .Values.webhookReady.maxAttempts | default 60 -}}
+{{- $sleepSeconds := .Values.webhookReady.sleepSeconds | default 2 -}}
+{{- /* Derive activeDeadlineSeconds from retries + 60s slack so a values override that raises maxAttempts doesn't get silently cut. */ -}}
+{{- $deadline := add (mul (int $maxAttempts) (int $sleepSeconds)) 60 -}}
+{{- $image := printf "%s:%s" .Values.webhookReady.image.repository .Values.webhookReady.image.tag -}}
+{{- if .Values.webhookReady.image.digest }}
+{{- $image = printf "%s@%s" $image .Values.webhookReady.image.digest -}}
+{{- end }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Release.Name }}-webhook-ready
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    helm.sh/hook: post-install,post-upgrade
+    helm.sh/hook-weight: "0"
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ .Release.Name }}-webhook-ready
+  annotations:
+    helm.sh/hook: post-install,post-upgrade
+    helm.sh/hook-weight: "0"
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+rules:
+  - apiGroups: [""]
+    resources: ["services/proxy"]
+    resourceNames: [{{ $resourceName | quote }}]
+    verbs: ["get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ .Release.Name }}-webhook-ready
+  annotations:
+    helm.sh/hook: post-install,post-upgrade
+    helm.sh/hook-weight: "0"
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ .Release.Name }}-webhook-ready
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Release.Name }}-webhook-ready
+    namespace: {{ .Release.Namespace }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ .Release.Name }}-webhook-ready
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    helm.sh/hook: post-install,post-upgrade
+    helm.sh/hook-weight: "10"
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+spec:
+  backoffLimit: {{ .Values.webhookReady.backoffLimit | default 2 }}
+  activeDeadlineSeconds: {{ $deadline }}
+  template:
+    spec:
+      restartPolicy: Never
+      serviceAccountName: {{ .Release.Name }}-webhook-ready
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: wait
+          image: {{ $image }}
+          imagePullPolicy: {{ if .Values.webhookReady.image.digest }}IfNotPresent{{ else }}Always{{ end }}
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          command:
+            - sh
+            - -c
+            - |
+              set -e
+              ns={{ .Release.Namespace }}
+              proxy="/api/v1/namespaces/${ns}/services/{{ $resourceName }}/proxy/readyz"
+              max_attempts={{ $maxAttempts }}
+              sleep_seconds={{ $sleepSeconds }}
+              i=0
+              last_err=""
+              until last_err=$(kubectl get --raw "$proxy" 2>&1 >/dev/null); do
+                i=$((i + 1))
+                if [ $i -gt $max_attempts ]; then
+                  echo "timeout: cnpg webhook did not respond through the apiserver proxy after ${max_attempts} attempts (${sleep_seconds}s each)"
+                  echo "last error: ${last_err}"
+                  exit 1
+                fi
+                if [ $((i % 10)) -eq 1 ]; then
+                  echo "attempt $i/${max_attempts}: ${last_err}"
+                fi
+                sleep "$sleep_seconds"
+              done
+              echo "cnpg webhook is ready"
diff --git a/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml b/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml
new file mode 100644
index 00000000..69f5a6b5
--- /dev/null
+++ b/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml
@@ -0,0 +1,308 @@
+suite: cnpg webhook post-install readiness gate
+
+templates:
+  - templates/webhook-ready-hook.yaml
+  - charts/cloudnative-pg/templates/service.yaml
+
+release:
+  name: postgres-operator
+  namespace: cozy-postgres-operator
+
+tests:
+  - it: renders four hook objects (SA + ClusterRole + ClusterRoleBinding + Job)
+    template: templates/webhook-ready-hook.yaml
+    asserts:
+      - hasDocuments:
+          count: 4
+
+  - it: every rendered object carries post-install and post-upgrade hook annotations
+    template: templates/webhook-ready-hook.yaml
+    asserts:
+      - documentIndex: 0
+        equal:
+          path: metadata.annotations["helm.sh/hook"]
+          value: post-install,post-upgrade
+      - documentIndex: 1
+        equal:
+          path: metadata.annotations["helm.sh/hook"]
+          value: post-install,post-upgrade
+      - documentIndex: 2
+        equal:
+          path: metadata.annotations["helm.sh/hook"]
+          value: post-install,post-upgrade
+      - documentIndex: 3
+        equal:
+          path: metadata.annotations["helm.sh/hook"]
+          value: post-install,post-upgrade
+
+  - it: RBAC is created before the Job (hook-weight ordering)
+    template: templates/webhook-ready-hook.yaml
+    asserts:
+      - documentIndex: 0
+        equal:
+          path: kind
+          value: ServiceAccount
+      - documentIndex: 0
+        equal:
+          path: metadata.annotations["helm.sh/hook-weight"]
+          value: "0"
+      - documentIndex: 1
+        equal:
+          path: kind
+          value: ClusterRole
+      - documentIndex: 1
+        equal:
+          path: metadata.annotations["helm.sh/hook-weight"]
+          value: "0"
+      - documentIndex: 2
+        equal:
+          path: kind
+          value: ClusterRoleBinding
+      - documentIndex: 2
+        equal:
+          path: metadata.annotations["helm.sh/hook-weight"]
+          value: "0"
+      - documentIndex: 3
+        equal:
+          path: kind
+          value: Job
+      - documentIndex: 3
+        equal:
+          path: metadata.annotations["helm.sh/hook-weight"]
+          value: "10"
+
+  - it: RBAC resourceName matches the exact proxy URL segment the Job probes
+    template: templates/webhook-ready-hook.yaml
+    asserts:
+      - documentIndex: 1
+        equal:
+          path: rules[0].resources[0]
+          value: services/proxy
+      - documentIndex: 1
+        equal:
+          path: rules[0].resourceNames[0]
+          value: https:cnpg-webhook-service:webhook-server
+      - documentIndex: 3
+        matchRegex:
+          path: spec.template.spec.containers[0].command[2]
+          pattern: "/api/v1/namespaces/\\$\\{ns\\}/services/https:cnpg-webhook-service:webhook-server/proxy/readyz"
+
+  - it: hardcoded service name in the hook matches the vendored cnpg subchart Service (drift guard for make update)
+    template: charts/cloudnative-pg/templates/service.yaml
+    asserts:
+      - equal:
+          path: metadata.name
+          value: cnpg-webhook-service
+      - equal:
+          path: spec.ports[0].name
+          value: webhook-server
+
+  - it: Job calls kubectl get --raw on the proxy path
+    template: templates/webhook-ready-hook.yaml
+    asserts:
+      - documentIndex: 3
+        matchRegex:
+          path: spec.template.spec.containers[0].command[2]
+          pattern: "kubectl get --raw"
+
+  - it: Job image is digest-pinned when webhookReady.image.digest is set
+    template: templates/webhook-ready-hook.yaml
+    set:
+      webhookReady:
+        image:
+          repository: docker.io/clastix/kubectl
+          tag: v1.32
+          digest: sha256:b9ef7d8dbe65bcc81a46c09b8dc7543103055021c4f43287bf59e92a8f4fe05c
+    asserts:
+      - documentIndex: 3
+        equal:
+          path: spec.template.spec.containers[0].image
+          value: docker.io/clastix/kubectl:v1.32@sha256:b9ef7d8dbe65bcc81a46c09b8dc7543103055021c4f43287bf59e92a8f4fe05c
+
+  - it: Job image falls back to tag-only when digest is not configured
+    template: templates/webhook-ready-hook.yaml
+    set:
+      webhookReady:
+        image:
+          repository: docker.io/clastix/kubectl
+          tag: v1.32
+          digest: ""
+    asserts:
+      - documentIndex: 3
+        equal:
+          path: spec.template.spec.containers[0].image
+          value: docker.io/clastix/kubectl:v1.32
+
+  - it: backoffLimit defaults to 2 so transient pod-level failures retry instead of killing the HelmRelease
+    template: templates/webhook-ready-hook.yaml
+    asserts:
+      - documentIndex: 3
+        equal:
+          path: spec.backoffLimit
+          value: 2
+
+  - it: backoffLimit is overridable
+    template: templates/webhook-ready-hook.yaml
+    set:
+      webhookReady:
+        backoffLimit: 5
+    asserts:
+      - documentIndex: 3
+        equal:
+          path: spec.backoffLimit
+          value: 5
+
+  - it: chart render fails when webhookReady.image.repository is empty
+    template: templates/webhook-ready-hook.yaml
+    set:
+      webhookReady:
+        image:
+          repository: ""
+          tag: v1.32
+    asserts:
+      - failedTemplate:
+          errorMessage: "webhookReady.image.repository must be set to the container image providing kubectl for the post-install readiness Job"
+
+  - it: chart render fails when webhookReady.image.tag is empty
+    template: templates/webhook-ready-hook.yaml
+    set:
+      webhookReady:
+        image:
+          repository: docker.io/clastix/kubectl
+          tag: ""
+    asserts:
+      - failedTemplate:
+          errorMessage: "webhookReady.image.tag must be set for the post-install readiness Job"
+
+  - it: retry loop bounds default when blanked so a wiped override still produces a working Job
+    template: templates/webhook-ready-hook.yaml
+    set:
+      webhookReady:
+        image:
+          repository: docker.io/clastix/kubectl
+          tag: v1.32
+        maxAttempts: null
+        sleepSeconds: null
+    asserts:
+      - documentIndex: 3
+        matchRegex:
+          path: spec.template.spec.containers[0].command[2]
+          pattern: "max_attempts=60"
+      - documentIndex: 3
+        matchRegex:
+          path: spec.template.spec.containers[0].command[2]
+          pattern: "sleep_seconds=2"
+
+  - it: Job pod runs non-root with seccomp RuntimeDefault for restricted-PSA clusters
+    template: templates/webhook-ready-hook.yaml
+    asserts:
+      - documentIndex: 3
+        equal:
+          path: spec.template.spec.securityContext.runAsNonRoot
+          value: true
+      - documentIndex: 3
+        equal:
+          path: spec.template.spec.securityContext.seccompProfile.type
+          value: RuntimeDefault
+
+  - it: Job container drops all capabilities and runs read-only rootfs
+    template: templates/webhook-ready-hook.yaml
+    asserts:
+      - documentIndex: 3
+        equal:
+          path: spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation
+          value: false
+      - documentIndex: 3
+        equal:
+          path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem
+          value: true
+      - documentIndex: 3
+        equal:
+          path: spec.template.spec.containers[0].securityContext.capabilities.drop[0]
+          value: ALL
+
+  - it: imagePullPolicy is IfNotPresent when digest-pinned, Always when tag-only
+    template: templates/webhook-ready-hook.yaml
+    set:
+      webhookReady:
+        image:
+          repository: docker.io/clastix/kubectl
+          tag: v1.32
+          digest: sha256:b9ef7d8dbe65bcc81a46c09b8dc7543103055021c4f43287bf59e92a8f4fe05c
+    asserts:
+      - documentIndex: 3
+        equal:
+          path: spec.template.spec.containers[0].imagePullPolicy
+          value: IfNotPresent
+
+  - it: imagePullPolicy is Always when no digest is configured
+    template: templates/webhook-ready-hook.yaml
+    set:
+      webhookReady:
+        image:
+          repository: docker.io/clastix/kubectl
+          tag: v1.32
+          digest: ""
+    asserts:
+      - documentIndex: 3
+        equal:
+          path: spec.template.spec.containers[0].imagePullPolicy
+          value: Always
+
+  - it: retry loop captures and surfaces the last kubectl error message on timeout
+    template: templates/webhook-ready-hook.yaml
+    asserts:
+      - documentIndex: 3
+        matchRegex:
+          path: spec.template.spec.containers[0].command[2]
+          pattern: 'last_err=\$\(kubectl get --raw'
+      - documentIndex: 3
+        matchRegex:
+          path: spec.template.spec.containers[0].command[2]
+          pattern: 'last error: \$\{last_err\}'
+
+  - it: retry loop error message stays in sync when maxAttempts is bumped
+    template: templates/webhook-ready-hook.yaml
+    set:
+      webhookReady:
+        image:
+          repository: docker.io/clastix/kubectl
+          tag: v1.32
+        maxAttempts: 90
+        sleepSeconds: 3
+    asserts:
+      - documentIndex: 3
+        matchRegex:
+          path: spec.template.spec.containers[0].command[2]
+          pattern: "max_attempts=90"
+      - documentIndex: 3
+        matchRegex:
+          path: spec.template.spec.containers[0].command[2]
+          pattern: "sleep_seconds=3"
+      - documentIndex: 3
+        matchRegex:
+          path: spec.template.spec.containers[0].command[2]
+          pattern: 'after \$\{max_attempts\} attempts'
+
+  - it: activeDeadlineSeconds scales with retry bounds so an override raise does not silently cut
+    template: templates/webhook-ready-hook.yaml
+    set:
+      webhookReady:
+        image:
+          repository: docker.io/clastix/kubectl
+          tag: v1.32
+        maxAttempts: 180
+        sleepSeconds: 3
+    asserts:
+      - documentIndex: 3
+        equal:
+          path: spec.activeDeadlineSeconds
+          value: 600
+
+  - it: activeDeadlineSeconds defaults include 60s slack over the default retry window
+    template: templates/webhook-ready-hook.yaml
+    asserts:
+      - documentIndex: 3
+        equal:
+          path: spec.activeDeadlineSeconds
+          value: 180
diff --git a/packages/system/postgres-operator/values.yaml b/packages/system/postgres-operator/values.yaml
index cae3dc53..854a3ac6 100644
--- a/packages/system/postgres-operator/values.yaml
+++ b/packages/system/postgres-operator/values.yaml
@@ -3,3 +3,28 @@ cloudnative-pg:
     create: true
   image:
     tag: "1.27.3"
+# Image used by the post-install webhook-readiness Job (see templates/webhook-ready-hook.yaml).
+# Any image with kubectl on PATH works; the Job calls the apiserver service proxy with the
+# hook ServiceAccount's token to confirm the mcluster.cnpg.io webhook is reachable end-to-end
+# before the HelmRelease reports Ready, closing the "connection refused" bootstrap race.
+#
+# The tag is digest-pinned so an upstream retag does not change what runs on every install
+# and upgrade across the fleet. Refresh by resolving the current manifest-list digest
+# (`docker manifest inspect docker.io/clastix/kubectl:v1.32`) and updating `digest` below.
+# renovate: datasource=docker depName=docker.io/clastix/kubectl
+webhookReady:
+  image:
+    repository: docker.io/clastix/kubectl
+    tag: v1.32
+    digest: sha256:b9ef7d8dbe65bcc81a46c09b8dc7543103055021c4f43287bf59e92a8f4fe05c
+  # Retry loop bounds for the readiness probe. Defaults total ~120s wall clock.
+  # Both are `default`-coerced in the template so an override that blanks them still
+  # produces a working Job.
+  maxAttempts: 60
+  sleepSeconds: 2
+  # Pod-level retry budget for transient node/registry failures (image pull rate limit,
+  # OOM, CNI hiccup). The shell loop inside the container only covers reachability retries.
+  # Pod retries and activeDeadlineSeconds (wall-clock bound on the whole Job across all
+  # pod retries) are ANDed: a 5-minute ImagePullBackOff on attempt 0 leaves only ~60s for
+  # subsequent retries before the Job is cut.
+  backoffLimit: 2

From 3b3540448975391ad7582db58d83ce5b18b15129 Mon Sep 17 00:00:00 2001
From: Denis Chernosov <denis0.ru@gmail.com>
Date: Fri, 24 Apr 2026 11:32:20 +0400
Subject: [PATCH 084/151] fix(cozystack-engine): add managed Kubernetes without
 visible control-plane nodes support to lineage-controller-webhook (#2417)

Signed-off-by: Denis Chernosov <denis0.ru@gmail.com>
---
 .../templates/daemonset.yaml                  |  2 +
 .../templates/deployment.yaml                 | 49 +++++++++++++++++++
 .../templates/pdb.yaml                        | 11 +++++
 .../lineage-controller-webhook/values.yaml    |  5 ++
 4 files changed, 67 insertions(+)
 create mode 100644 packages/system/lineage-controller-webhook/templates/deployment.yaml
 create mode 100644 packages/system/lineage-controller-webhook/templates/pdb.yaml

diff --git a/packages/system/lineage-controller-webhook/templates/daemonset.yaml b/packages/system/lineage-controller-webhook/templates/daemonset.yaml
index 860aee6e..536ea24a 100644
--- a/packages/system/lineage-controller-webhook/templates/daemonset.yaml
+++ b/packages/system/lineage-controller-webhook/templates/daemonset.yaml
@@ -1,3 +1,4 @@
+{{- if not .Values.lineageControllerWebhook.deployment.enabled }}
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
@@ -51,3 +52,4 @@ spec:
           secret:
             secretName: lineage-controller-webhook-cert
             defaultMode: 0400
+{{- end }}
\ No newline at end of file
diff --git a/packages/system/lineage-controller-webhook/templates/deployment.yaml b/packages/system/lineage-controller-webhook/templates/deployment.yaml
new file mode 100644
index 00000000..87717fd3
--- /dev/null
+++ b/packages/system/lineage-controller-webhook/templates/deployment.yaml
@@ -0,0 +1,49 @@
+{{- if .Values.lineageControllerWebhook.deployment.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: lineage-controller-webhook
+  labels:
+    app: lineage-controller-webhook
+spec:
+  replicas: {{ .Values.lineageControllerWebhook.deployment.replicas }}
+  selector:
+    matchLabels:
+      app: lineage-controller-webhook
+  template:
+    metadata:
+      labels:
+        app: lineage-controller-webhook
+    spec:
+      serviceAccountName: lineage-controller-webhook
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                topologyKey: kubernetes.io/hostname
+                labelSelector:
+                  matchLabels:
+                    app: lineage-controller-webhook
+      containers:
+      - name: lineage-controller-webhook
+        image: "{{ .Values.lineageControllerWebhook.image }}"
+        args:
+        {{- if .Values.lineageControllerWebhook.debug }}
+        - --zap-log-level=debug
+        {{- else }}
+        - --zap-log-level=info
+        {{- end }}
+        ports:
+        - name: webhook
+          containerPort: 9443
+        volumeMounts:
+          - name: webhook-certs
+            mountPath: /tmp/k8s-webhook-server/serving-certs
+            readOnly: true
+      volumes:
+        - name: webhook-certs
+          secret:
+            secretName: lineage-controller-webhook-cert
+            defaultMode: 0400
+{{- end }}
\ No newline at end of file
diff --git a/packages/system/lineage-controller-webhook/templates/pdb.yaml b/packages/system/lineage-controller-webhook/templates/pdb.yaml
new file mode 100644
index 00000000..b0a9d4e4
--- /dev/null
+++ b/packages/system/lineage-controller-webhook/templates/pdb.yaml
@@ -0,0 +1,11 @@
+{{- if .Values.lineageControllerWebhook.deployment.enabled }}
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: lineage-controller-webhook
+spec:
+  minAvailable: {{ if gt (int .Values.lineageControllerWebhook.deployment.replicas) 1 }}1{{ else }}0{{ end }}
+  selector:
+    matchLabels:
+      app: lineage-controller-webhook
+{{- end }}
\ No newline at end of file
diff --git a/packages/system/lineage-controller-webhook/values.yaml b/packages/system/lineage-controller-webhook/values.yaml
index c5671548..2b84f4e1 100644
--- a/packages/system/lineage-controller-webhook/values.yaml
+++ b/packages/system/lineage-controller-webhook/values.yaml
@@ -2,9 +2,14 @@ lineageControllerWebhook:
   image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.3.0@sha256:e8984709686a5eaf19b89da378d7b8c688ea5607e0783a88d9c9e4ccfca96fb0
   debug: false
   localK8sAPIEndpoint:
+    # incompatable with Deployment mode
     enabled: true
   # nodeSelector for the DaemonSet
   # Talos uses empty value: "node-role.kubernetes.io/control-plane": ""
   # Generic k8s (k3s, kubeadm) uses: "node-role.kubernetes.io/control-plane": "true"
   nodeSelector:
     node-role.kubernetes.io/control-plane: ""
+  # if enabled, replace DaemonSet by Deployment
+  deployment:
+    enabled: false
+    replicas: 1

From ab7deb2b055c2a677c4c84c1bb22271306ebf372 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Fri, 24 Apr 2026 13:03:49 +0300
Subject: [PATCH 085/151] chore(kubernetes): regenerate after HAMi addon
 integration

Run make generate to update generated types, schema, README,
and CRD definitions for the new HAMi addon.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 api/apps/v1alpha1/kubernetes/types.go                | 12 ++++++++++++
 packages/apps/kubernetes/README.md                   |  3 +++
 packages/apps/kubernetes/values.schema.json          |  4 ++--
 .../system/kubernetes-rd/cozyrds/kubernetes.yaml     |  4 ++--
 4 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/api/apps/v1alpha1/kubernetes/types.go b/api/apps/v1alpha1/kubernetes/types.go
index 774fcdc5..39a05567 100644
--- a/api/apps/v1alpha1/kubernetes/types.go
+++ b/api/apps/v1alpha1/kubernetes/types.go
@@ -66,6 +66,9 @@ type Addons struct {
 	// NVIDIA GPU Operator.
 	// +kubebuilder:default:={}
 	GpuOperator GPUOperatorAddon `json:"gpuOperator"`
+	// HAMi GPU virtualization middleware.
+	// +kubebuilder:default:={}
+	Hami HAMiAddon `json:"hami"`
 	// Ingress-NGINX controller.
 	// +kubebuilder:default:={}
 	IngressNginx IngressNginxAddon `json:"ingressNginx"`
@@ -157,6 +160,15 @@ type GatewayAPIAddon struct {
 	Enabled bool `json:"enabled"`
 }
 
+type HAMiAddon struct {
+	// Enable HAMi (requires GPU Operator).
+	// +kubebuilder:default:=false
+	Enabled bool `json:"enabled"`
+	// Custom Helm values overrides.
+	// +kubebuilder:default:={}
+	ValuesOverride k8sRuntime.RawExtension `json:"valuesOverride"`
+}
+
 type IngressNginxAddon struct {
 	// Enable the controller (requires nodes labeled `ingress-nginx`).
 	// +kubebuilder:default:=false
diff --git a/packages/apps/kubernetes/README.md b/packages/apps/kubernetes/README.md
index d62d3a39..d705af64 100644
--- a/packages/apps/kubernetes/README.md
+++ b/packages/apps/kubernetes/README.md
@@ -128,6 +128,9 @@ See the reference for components utilized in this service:
 | `addons.gpuOperator`                          | NVIDIA GPU Operator.                                                        | `object`   | `{}`      |
 | `addons.gpuOperator.enabled`                  | Enable GPU Operator.                                                        | `bool`     | `false`   |
 | `addons.gpuOperator.valuesOverride`           | Custom Helm values overrides.                                               | `object`   | `{}`      |
+| `addons.hami`                                 | HAMi GPU virtualization middleware.                                         | `object`   | `{}`      |
+| `addons.hami.enabled`                         | Enable HAMi (requires GPU Operator).                                        | `bool`     | `false`   |
+| `addons.hami.valuesOverride`                  | Custom Helm values overrides.                                               | `object`   | `{}`      |
 | `addons.fluxcd`                               | FluxCD GitOps operator.                                                     | `object`   | `{}`      |
 | `addons.fluxcd.enabled`                       | Enable FluxCD.                                                              | `bool`     | `false`   |
 | `addons.fluxcd.valuesOverride`                | Custom Helm values overrides.                                               | `object`   | `{}`      |
diff --git a/packages/apps/kubernetes/values.schema.json b/packages/apps/kubernetes/values.schema.json
index 3d7a17dd..1e457f1f 100644
--- a/packages/apps/kubernetes/values.schema.json
+++ b/packages/apps/kubernetes/values.schema.json
@@ -270,7 +270,7 @@
           }
         },
         "hami": {
-          "description": "HAMi GPU virtualization middleware. Enables fractional GPU sharing (memory and compute isolation) for workloads. Requires GPU Operator. Note: workload containers must use glibc < 2.34 (not musl/Alpine) for full compute isolation.",
+          "description": "HAMi GPU virtualization middleware.",
           "type": "object",
           "default": {},
           "required": [
@@ -279,7 +279,7 @@
           ],
           "properties": {
             "enabled": {
-              "description": "Enable HAMi.",
+              "description": "Enable HAMi (requires GPU Operator).",
               "type": "boolean",
               "default": false
             },
diff --git a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml
index 5e9e8f94..cdf900e1 100644
--- a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml
+++ b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml
@@ -8,7 +8,7 @@ spec:
     singular: kubernetes
     plural: kuberneteses
     openAPISchema: |-
-      {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"ephemeralStorage":"20Gi","gpus":[],"instanceType":"u1.medium","maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"]}},"additionalProperties":{"type":"object","required":["ephemeralStorage","instanceType","maxReplicas","minReplicas","resources"],"properties":{"ephemeralStorage":{"description":"Ephemeral storage size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string"}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium"},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `<cluster-name>.<tenant-host>` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","ingressNginx","monitoringAgents","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"large","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}}}}
+      {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"ephemeralStorage":"20Gi","gpus":[],"instanceType":"u1.medium","maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"]}},"additionalProperties":{"type":"object","required":["ephemeralStorage","instanceType","maxReplicas","minReplicas","resources"],"properties":{"ephemeralStorage":{"description":"Ephemeral storage size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string"}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium"},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `<cluster-name>.<tenant-host>` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","hami","ingressNginx","monitoringAgents","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"hami":{"description":"HAMi GPU virtualization middleware.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable HAMi (requires GPU Operator).","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"large","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}}}}
   release:
     prefix: kubernetes-
     labels:
@@ -26,7 +26,7 @@ spec:
     tags:
       - compute
     icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODFfMjg0NSkiLz4KPHBhdGggZD0iTTcxLjk5NjggMTlDNzAuMzAzOSAxOS4wMDAyIDY4LjkzMTIgMjAuNTMyMiA2OC45MzE0IDIyLjQyMjFDNjguOTMxNCAyMi40NTExIDY4LjkzNzMgMjIuNDc4OCA2OC45Mzc5IDIyLjUwNzZDNjguOTM1NCAyMi43NjQ0IDY4LjkyMzEgMjMuMDczNyA2OC45MzE0IDIzLjI5NzNDNjguOTcxNyAyNC4zODczIDY5LjIwODIgMjUuMjIxNiA2OS4zNTA2IDI2LjIyNThDNjkuNjA4NCAyOC4zNzUyIDY5LjgyNDUgMzAuMTU2OSA2OS42OTEyIDMxLjgxM0M2OS41NjE1IDMyLjQzNzUgNjkuMTAzNyAzMy4wMDg2IDY4LjY5NTYgMzMuNDA1Nkw2OC42MjM1IDM0LjcwODZDNjYuNzgzOSAzNC44NjE3IDY0LjkzMTkgMzUuMTQyMSA2My4wODIxIDM1LjU2NDFDNTUuMTIyNiAzNy4zNzk4IDQ4LjI2OTUgNDEuNDk5MSA0My4wNTIgNDcuMDYwOUM0Mi43MTM0IDQ2LjgyODggNDIuMTIxMSA0Ni40MDE5IDQxLjk0NSA0Ni4yNzEyQzQxLjM5NzcgNDYuMzQ1NCA0MC44NDQ1IDQ2LjUxNTEgNDAuMTI0MSA0Ni4wOTM1QzM4Ljc1MjIgNDUuMTY1NyAzNy41MDI4IDQzLjg4NTEgMzUuOTkxIDQyLjM0MjRDMzUuMjk4MiA0MS42MDQ0IDM0Ljc5NjYgNDAuOTAxOCAzMy45NzM1IDQwLjE5MDRDMzMuNzg2NiA0MC4wMjg5IDMzLjUwMTQgMzkuODEwNCAzMy4yOTIzIDM5LjY0NDJDMzIuNjQ4OSAzOS4xMjg4IDMxLjg5IDM4Ljg2IDMxLjE1NyAzOC44MzQ4QzMwLjIxNDcgMzguODAyNCAyOS4zMDc1IDM5LjE3MjUgMjguNzEzOCAzOS45MjA2QzI3LjY1ODQgNDEuMjUwNiAyNy45OTYzIDQzLjI4MzMgMjkuNDY3MSA0NC40NjE0QzI5LjQ4MiA0NC40NzM0IDI5LjQ5NzkgNDQuNDgyNyAyOS41MTI5IDQ0LjQ5NDNDMjkuNzE1IDQ0LjY1ODkgMjkuOTYyNSA0NC44Njk4IDMwLjE0ODMgNDUuMDA3NkMzMS4wMjE3IDQ1LjY1NTUgMzEuODE5NSA0NS45ODcyIDMyLjY4OTcgNDYuNTAxNUMzNC41MjMxIDQ3LjYzOTEgMzYuMDQzIDQ4LjU4MjMgMzcuMjQ4NiA0OS43MTk2QzM3LjcxOTQgNTAuMjIzNyAzNy44MDE2IDUxLjExMjIgMzcuODY0MyA1MS40OTY0TDM4Ljg0NjggNTIuMzc4MkMzMy41ODcyIDYwLjMzMDggMzEuMTUzIDcwLjE1MzkgMzIuNTkxNSA4MC4xNjI3TDMxLjMwNzcgODAuNTM3OEMzMC45NjkzIDgwLjk3NjggMzAuNDkxMiA4MS42Njc2IDI5Ljk5MTEgODEuODczOEMyOC40MTM4IDgyLjM3MjkgMjYuNjM4NyA4Mi41NTYyIDI0LjQ5NTYgODIuNzgxOUMyMy40ODk0IDgyLjg2NiAyMi42MjEzIDgyLjgxNTggMjEuNTU0NiA4My4wMTg4QzIxLjMxOTggODMuMDYzNSAyMC45OTI3IDgzLjE0OTEgMjAuNzM1OCA4My4yMDk3QzIwLjcyNjkgODMuMjExNiAyMC43MTg2IDgzLjIxNDIgMjAuNzA5NiA4My4yMTYyQzIwLjY5NTYgODMuMjE5NSAyMC42NzcyIDgzLjIyNjMgMjAuNjYzOCA4My4yMjk0QzE4Ljg1NyA4My42NjggMTcuNjk2MyA4NS4zMzY1IDE4LjA2OTkgODYuOTgwNUMxOC40NDM3IDg4LjYyNDggMjAuMjA4NiA4OS42MjQ4IDIyLjAyNjIgODkuMjMxMkMyMi4wMzkzIDg5LjIyODIgMjIuMDU4NCA4OS4yMjc3IDIyLjA3MiA4OS4yMjQ2QzIyLjA5MjYgODkuMjE5OSAyMi4xMTA2IDg5LjIwOTkgMjIuMTMxIDg5LjIwNDlDMjIuMzg0NCA4OS4xNDkgMjIuNzAxOSA4OS4wODY4IDIyLjkyMzYgODkuMDI3MkMyMy45NzIzIDg4Ljc0NTEgMjQuNzMxOCA4OC4zMzA2IDI1LjY3NDYgODcuOTY3N0MyNy43MDI5IDg3LjIzNjggMjkuMzgyOCA4Ni42MjYyIDMxLjAxOTUgODYuMzg4M0MzMS43MDMgODYuMzM0NSAzMi40MjMyIDg2LjgxMiAzMi43ODE0IDg3LjAxMzRMMzQuMTE3NyA4Ni43ODMxQzM3LjE5MjYgOTYuMzYxMyA0My42MzY2IDEwNC4xMDMgNTEuNzk2MyAxMDguOTYxTDUxLjIzOTYgMTEwLjMwM0M1MS40NDAzIDExMC44MjQgNTEuNjYxNiAxMTEuNTMgNTEuNTEyMSAxMTIuMDQ1QzUwLjkxNzEgMTEzLjU5NSA0OS44OTggMTE1LjIzMSA0OC43Mzc0IDExNy4wNTVDNDguMTc1NSAxMTcuODk4IDQ3LjYwMDQgMTE4LjU1MiA0Ny4wOTM0IDExOS41MTZDNDYuOTcyIDExOS43NDcgNDYuODE3NSAxMjAuMTAyIDQ2LjcwMDQgMTIwLjM0NkM0NS45MTI1IDEyMi4wMzkgNDYuNDkwNCAxMjMuOTkgNDguMDAzOCAxMjQuNzIyQzQ5LjUyNjggMTI1LjQ1OSA1MS40MTcxIDEyNC42ODIgNTIuMjM1MiAxMjIuOTg1QzUyLjIzNjQgMTIyLjk4MiA1Mi4yNDA2IDEyMi45OCA1Mi4yNDE3IDEyMi45NzhDNTIuMjQyNiAxMjIuOTc2IDUyLjI0MDkgMTIyLjk3MyA1Mi4yNDE3IDEyMi45NzFDNTIuMzU4MiAxMjIuNzMxIDUyLjUyMzMgMTIyLjQxNSA1Mi42MjE2IDEyMi4xODhDNTMuMDU2IDEyMS4xODkgNTMuMjAwNSAxMjAuMzMyIDUzLjUwNTkgMTE5LjM2NUM1NC4zMTcgMTE3LjMxOCA1NC43NjI2IDExNS4xNyA1NS44NzkxIDExMy44MzJDNTYuMTg0OSAxMTMuNDY2IDU2LjY4MzMgMTEzLjMyNSA1Ny4yMDAxIDExMy4xODZMNTcuODk0NCAxMTEuOTIyQzY1LjAwOCAxMTQuNjY1IDcyLjk3MDUgMTE1LjQwMiA4MC45MjQ1IDExMy41ODdDODIuNzM5MSAxMTMuMTczIDg0LjQ5MDggMTEyLjYzNyA4Ni4xODQzIDExMS45OTRDODYuMzc5NCAxMTIuMzQyIDg2Ljc0MiAxMTMuMDExIDg2LjgzOTMgMTEzLjE3OUM4Ny4zNjQ0IDExMy4zNTEgODcuOTM3NyAxMTMuNDM5IDg4LjQwNDcgMTE0LjEzM0M4OS4yNDAxIDExNS41NjcgODkuODExNCAxMTcuMjYzIDkwLjUwNzMgMTE5LjMxMkM5MC44MTI4IDEyMC4yNzkgOTAuOTYzOCAxMjEuMTM2IDkxLjM5ODEgMTIyLjEzNkM5MS40OTcxIDEyMi4zNjMgOTEuNjYxNCAxMjIuNjg0IDkxLjc3OCAxMjIuOTI1QzkyLjU5NDQgMTI0LjYyOCA5NC40OTA3IDEyNS40MDcgOTYuMDE1OSAxMjQuNjY5Qzk3LjUyOTIgMTIzLjkzNyA5OC4xMDc3IDEyMS45ODYgOTcuMzE5NCAxMjAuMjkzQzk3LjIwMjMgMTIwLjA0OSA5Ny4wNDEyIDExOS42OTUgOTYuOTE5OCAxMTkuNDY0Qzk2LjQxMjcgMTE4LjQ5OSA5NS44Mzc3IDExNy44NTIgOTUuMjc1OCAxMTcuMDA5Qzk0LjExNTIgMTE1LjE4NSA5My4xNTI2IDExMy42NyA5Mi41NTc1IDExMi4xMkM5Mi4zMDg3IDExMS4zMiA5Mi41OTk1IDExMC44MjMgOTIuNzkzMyAxMTAuMzAzQzkyLjY3NzIgMTEwLjE3IDkyLjQyODggMTA5LjQxNCA5Mi4yODI0IDEwOS4wNTlDMTAwLjc2MiAxMDQuMDI5IDEwNy4wMTcgOTUuOTk4NSAxMDkuOTU1IDg2LjcyMzlDMTEwLjM1MSA4Ni43ODY1IDExMS4wNDEgODYuOTA5MSAxMTEuMjY1IDg2Ljk1NDJDMTExLjcyNiA4Ni42NDg3IDExMi4xNDkgODYuMjUwMSAxMTIuOTgxIDg2LjMxNTlDMTE0LjYxNyA4Ni41NTM3IDExNi4yOTcgODcuMTY0NSAxMTguMzI2IDg3Ljg5NTNDMTE5LjI2OCA4OC4yNTgxIDEyMC4wMjggODguNjc5MyAxMjEuMDc3IDg4Ljk2MTRDMTIxLjI5OCA4OS4wMjEgMTIxLjYxNiA4OS4wNzY2IDEyMS44NjkgODkuMTMyNUMxMjEuODg5IDg5LjEzNzUgMTIxLjkwOCA4OS4xNDc1IDEyMS45MjggODkuMTUyMkMxMjEuOTQyIDg5LjE1NTMgMTIxLjk2MSA4OS4xNTU4IDEyMS45NzQgODkuMTU4OEMxMjMuNzkyIDg5LjU1MiAxMjUuNTU3IDg4LjU1MjYgMTI1LjkzIDg2LjkwODFDMTI2LjMwMyA4NS4yNjQxIDEyNS4xNDMgODMuNTk1MiAxMjMuMzM2IDgzLjE1N0MxMjMuMDc0IDgzLjA5NyAxMjIuNzAxIDgyLjk5NSAxMjIuNDQ2IDgyLjk0NjVDMTIxLjM3OSA4Mi43NDM1IDEyMC41MTEgODIuNzkzNSAxMTkuNTA1IDgyLjcwOTVDMTE3LjM2MSA4Mi40ODM5IDExNS41ODYgODIuMzAwNCAxMTQuMDA5IDgxLjgwMTRDMTEzLjM2NiA4MS41NTA3IDExMi45MDggODAuNzgxOSAxMTIuNjg2IDgwLjQ2NTVMMTExLjQ0OCA4MC4xMDM1QzExMi4wOSA3NS40MzggMTExLjkxNyA3MC41ODI1IDExMC44MDYgNjUuNzI0M0MxMDkuNjg1IDYwLjgyMDggMTA3LjcwNCA1Ni4zMzYxIDEwNS4wNjIgNTIuMzg0OEMxMDUuMzc5IDUyLjA5NDggMTA1Ljk3OSA1MS41NjEyIDEwNi4xNDkgNTEuNDA0M0MxMDYuMTk5IDUwLjg1MTcgMTA2LjE1NiA1MC4yNzIyIDEwNi43MjUgNDkuNjYwM0MxMDcuOTMxIDQ4LjUyMyAxMDkuNDUxIDQ3LjU3OTkgMTExLjI4NCA0Ni40NDIzQzExMi4xNTQgNDUuOTI3OSAxMTIuOTU5IDQ1LjU5NjQgMTEzLjgzMiA0NC45NDg0QzExNC4wMyA0NC44MDE5IDExNC4yOTkgNDQuNTY5OSAxMTQuNTA3IDQ0LjQwMjJDMTE1Ljk3NyA0My4yMjM3IDExNi4zMTYgNDEuMTkxMSAxMTUuMjYgMzkuODYxNEMxMTQuMjA0IDM4LjUzMTcgMTEyLjE1OSAzOC40MDY1IDExMC42ODggMzkuNTg1QzExMC40NzkgMzkuNzUxNiAxMTAuMTk1IDM5Ljk2ODggMTEwLjAwNyA0MC4xMzEyQzEwOS4xODQgNDAuODQyNiAxMDguNjc2IDQxLjU0NTIgMTA3Ljk4MyA0Mi4yODMyQzEwNi40NzEgNDMuODI1OSAxMDUuMjIyIDQ1LjExMyAxMDMuODUgNDYuMDQwOUMxMDMuMjU1IDQ2LjM4ODUgMTAyLjM4NSA0Ni4yNjgyIDEwMS45OSA0Ni4yNDQ5TDEwMC44MjQgNDcuMDgwNkM5NC4xNzUzIDQwLjA3NjMgODUuMTIzNSAzNS41OTgyIDc1LjM3NjYgMzQuNzI4M0M3NS4zNDk0IDM0LjMxNzkgNzUuMzEzNyAzMy41NzYxIDc1LjMwNDYgMzMuMzUyOUM3NC45MDU2IDMyLjk2OTMgNzQuNDIzNSAzMi42NDE4IDc0LjMwMjQgMzEuODEzQzc0LjE2OTEgMzAuMTU2OSA3NC4zOTE3IDI4LjM3NTIgNzQuNjQ5NiAyNi4yMjU4Qzc0Ljc5MTkgMjUuMjIxNiA3NS4wMjg0IDI0LjM4NzMgNzUuMDY4OCAyMy4yOTczQzc1LjA3OCAyMy4wNDk1IDc1LjA2MzIgMjIuNjkgNzUuMDYyMiAyMi40MjIxQzc1LjA2MiAyMC41MzIyIDczLjY4OTggMTguOTk5OCA3MS45OTY4IDE5Wk02OC4xNTg1IDQyLjg4ODZMNjcuMjQ4IDU5LjA0NDdMNjcuMTgyNSA1OS4wNzc2QzY3LjEyMTQgNjAuNTIyOSA2NS45Mzc1IDYxLjY3NyA2NC40ODM5IDYxLjY3N0M2My44ODg0IDYxLjY3NyA2My4zMzg4IDYxLjQ4NDkgNjIuODkyMiA2MS4xNTcxTDYyLjg2NiA2MS4xNzAzTDQ5LjY4MDcgNTEuNzc5NEM1My43MzMxIDQ3Ljc3NTkgNTguOTE2NCA0NC44MTcyIDY0Ljg5IDQzLjQ1NDZDNjUuOTgxMiA0My4yMDU2IDY3LjA3MTkgNDMuMDIwOSA2OC4xNTg1IDQyLjg4ODZaTTc1Ljg0MTcgNDIuODg4NkM4Mi44MTU5IDQzLjc1MDQgODkuMjY1NyA0Ni45MjMyIDk0LjIwODEgNTEuNzg2TDgxLjEwOCA2MS4xMTc2TDgxLjA2MjEgNjEuMDk3OUM3OS44OTk0IDYxLjk1MTIgNzguMjYxMSA2MS43Mzk0IDc3LjM1NDggNjAuNTk3OEM3Ni45ODM1IDYwLjEzMDEgNzYuNzg4NyA1OS41ODAxIDc2Ljc2NTMgNTkuMDI0OUw3Ni43NTIyIDU5LjAxODRMNzUuODQxNyA0Mi44ODg2Wk00NC44OTkxIDU3LjgxNEw1Ni45MzgyIDY4LjYzM0w1Ni45MjUxIDY4LjY5ODhDNTguMDExNyA2OS42NDc5IDU4LjE3MiA3MS4yOTQ5IDU3LjI2NTcgNzIuNDM2OEM1Ni44OTQ0IDcyLjkwNDUgNTYuMzk3NSA3My4yMTgyIDU1Ljg2MzkgNzMuMzY0N0w1NS44NTA4IDczLjQxNzNMNDAuNDE4OCA3Ny44OTIzQzM5LjYzMzQgNzAuNjc2NSA0MS4zMjYxIDYzLjY2MjEgNDQuODk5MSA1Ny44MTRaTTk5LjAwOTQgNTcuODIwNkMxMDAuNzk4IDYwLjczMzYgMTAyLjE1MyA2My45ODcxIDEwMi45NTkgNjcuNTE0M0MxMDMuNzU2IDcwLjk5OTEgMTAzLjk1NiA3NC40Nzc4IDEwMy42MjcgNzcuODM5N0w4OC4xMTY2IDczLjM1MTVMODguMTAzNSA3My4yODU3Qzg2LjcxNDUgNzIuOTA0MyA4NS44NjA5IDcxLjQ4NDggODYuMTg0MyA3MC4wNjExQzg2LjMxNjggNjkuNDc3OCA4Ni42MjQ5IDY4Ljk4NDQgODcuMDQyMyA2OC42MTk4TDg3LjAzNTggNjguNTg2OUw5OS4wMDk0IDU3LjgyMDZaTTY5LjUyNzQgNjkuNDY4OEg3NC40NTk2TDc3LjUyNTEgNzMuMzE4Nkw3Ni40MjQ3IDc4LjEyMjZMNzEuOTk2OCA4MC4yNjE0TDY3LjU1NTggNzguMTE2MUw2Ni40NTU0IDczLjMxMkw2OS41Mjc0IDY5LjQ2ODhaTTg1LjMzOTMgODIuNjQzN0M4NS41NDg5IDgyLjYzMzEgODUuNzU3NiA4Mi42NTIgODUuOTYxNiA4Mi42ODk4TDg1Ljk4NzggODIuNjU2OUwxMDEuOTUgODUuMzY4MkM5OS42MTQyIDkxLjk2MjQgOTUuMTQ0IDk3LjY3NSA4OS4xNzExIDEwMS40OThMODIuOTc0NyA4Ni40NjA2TDgyLjk5NDQgODYuNDM0M0M4Mi40MjUyIDg1LjEwNTUgODIuOTk0OCA4My41NDcyIDg0LjMwNDQgODIuOTEzNUM4NC42Mzk3IDgyLjc1MTMgODQuOTkgODIuNjYxNCA4NS4zMzkzIDgyLjY0MzdaTTU4LjUyOTggODIuNzA5NUM1OS43NDggODIuNzI2NyA2MC44NDA2IDgzLjU3NjEgNjEuMTIzNyA4NC44MjJDNjEuMjU2MiA4NS40MDUyIDYxLjE5MTcgODUuOTgzMSA2MC45NzMgODYuNDkzNUw2MS4wMTg5IDg2LjU1MjdMNTQuODg4IDEwMS40MzlDNDkuMTU1OSA5Ny43NDMyIDQ0LjU5MDQgOTIuMjA5OSA0Mi4xNDgxIDg1LjQyMDhMNTcuOTczMSA4Mi43MjI3TDU3Ljk5OTMgODIuNzU1NkM1OC4xNzYzIDgyLjcyMjkgNTguMzU1OCA4Mi43MDcxIDU4LjUyOTggODIuNzA5NVpNNzEuODk4NiA4OS4yMzEyQzcyLjMyMjkgODkuMjE1NSA3Mi43NTM0IDg5LjMwMyA3My4xNjI3IDg5LjUwMUM3My42OTkyIDg5Ljc2MDYgNzQuMTEzNiA5MC4xNjkyIDc0LjM3NDUgOTAuNjU5Mkg3NC40MzM0TDgyLjIzNDYgMTA0LjgyMUM4MS4yMjIxIDEwNS4xNjIgODAuMTgxMyAxMDUuNDU0IDc5LjExNjcgMTA1LjY5N0M3My4xNTA1IDEwNy4wNTggNjcuMjAzMiAxMDYuNjQ1IDYxLjgxOCAxMDQuODAyTDY5LjU5OTUgOTAuNjY1OEg2OS42MTI2QzcwLjA3OTUgODkuNzg4OCA3MC45NjUgODkuMjY1NiA3MS44OTg2IDg5LjIzMTJaIiBmaWxsPSJ3aGl0ZSIgc3Ryb2tlPSJ3aGl0ZSIgc3Ryb2tlLXdpZHRoPSIwLjI1Ii8+CjxkZWZzPgo8bGluZWFyR3JhZGllbnQgaWQ9InBhaW50MF9saW5lYXJfNjgxXzI4NDUiIHgxPSIxMCIgeTE9IjE1LjUiIHgyPSIxNDQiIHkyPSIxMzEuNSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiPgo8c3RvcCBzdG9wLWNvbG9yPSIjNEQ4N0ZGIi8+CjxzdG9wIG9mZnNldD0iMSIgc3RvcC1jb2xvcj0iIzA1NDdEMCIvPgo8L2xpbmVhckdyYWRpZW50Pgo8L2RlZnM+Cjwvc3ZnPgo=
-    keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "nodeGroups"], ["spec", "nodeGroups", "md0"], ["spec", "nodeGroups", "md0", "minReplicas"], ["spec", "nodeGroups", "md0", "maxReplicas"], ["spec", "nodeGroups", "md0", "instanceType"], ["spec", "nodeGroups", "md0", "ephemeralStorage"], ["spec", "nodeGroups", "md0", "roles"], ["spec", "nodeGroups", "md0", "resources"], ["spec", "nodeGroups", "md0", "gpus"], ["spec", "version"], ["spec", "host"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "fluxcd"], ["spec", "addons", "fluxcd", "enabled"], ["spec", "addons", "fluxcd", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"]]
+    keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "nodeGroups"], ["spec", "nodeGroups", "md0"], ["spec", "nodeGroups", "md0", "minReplicas"], ["spec", "nodeGroups", "md0", "maxReplicas"], ["spec", "nodeGroups", "md0", "instanceType"], ["spec", "nodeGroups", "md0", "ephemeralStorage"], ["spec", "nodeGroups", "md0", "roles"], ["spec", "nodeGroups", "md0", "resources"], ["spec", "nodeGroups", "md0", "gpus"], ["spec", "version"], ["spec", "host"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "hami"], ["spec", "addons", "hami", "enabled"], ["spec", "addons", "hami", "valuesOverride"], ["spec", "addons", "fluxcd"], ["spec", "addons", "fluxcd", "enabled"], ["spec", "addons", "fluxcd", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"]]
   secrets:
     exclude: []
     include:

From 648ad82dd3283076cf85f6b4b8953ceda8199476 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Fri, 24 Apr 2026 13:31:57 +0300
Subject: [PATCH 086/151] refactor(postgres-operator): scope webhook-ready RBAC
 to release namespace

Address review feedback from gemini-code-assist on
packages/system/postgres-operator/templates/webhook-ready-hook.yaml:83.

The Job targets the cnpg-webhook-service/services/proxy subresource, which
is namespaced and lives in the release namespace. A namespaced Role and
RoleBinding grant the exact permission needed without creating global
RBAC for a namespaced probe, which is the principle of least privilege.

Also update the kind assertions in tests/webhook-ready-hook_test.yaml so
the unittest suite tracks the new Role/RoleBinding objects.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 .../postgres-operator/templates/webhook-ready-hook.yaml   | 8 +++++---
 .../postgres-operator/tests/webhook-ready-hook_test.yaml  | 6 +++---
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/packages/system/postgres-operator/templates/webhook-ready-hook.yaml b/packages/system/postgres-operator/templates/webhook-ready-hook.yaml
index 2ad4e7c9..70e51621 100644
--- a/packages/system/postgres-operator/templates/webhook-ready-hook.yaml
+++ b/packages/system/postgres-operator/templates/webhook-ready-hook.yaml
@@ -52,9 +52,10 @@ metadata:
     helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
 ---
 apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
+kind: Role
 metadata:
   name: {{ .Release.Name }}-webhook-ready
+  namespace: {{ .Release.Namespace }}
   annotations:
     helm.sh/hook: post-install,post-upgrade
     helm.sh/hook-weight: "0"
@@ -66,16 +67,17 @@ rules:
     verbs: ["get"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
+kind: RoleBinding
 metadata:
   name: {{ .Release.Name }}-webhook-ready
+  namespace: {{ .Release.Namespace }}
   annotations:
     helm.sh/hook: post-install,post-upgrade
     helm.sh/hook-weight: "0"
     helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
 roleRef:
   apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
+  kind: Role
   name: {{ .Release.Name }}-webhook-ready
 subjects:
   - kind: ServiceAccount
diff --git a/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml b/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml
index 69f5a6b5..eb7863da 100644
--- a/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml
+++ b/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml
@@ -9,7 +9,7 @@ release:
   namespace: cozy-postgres-operator
 
 tests:
-  - it: renders four hook objects (SA + ClusterRole + ClusterRoleBinding + Job)
+  - it: renders four hook objects (SA + Role + RoleBinding + Job)
     template: templates/webhook-ready-hook.yaml
     asserts:
       - hasDocuments:
@@ -49,7 +49,7 @@ tests:
       - documentIndex: 1
         equal:
           path: kind
-          value: ClusterRole
+          value: Role
       - documentIndex: 1
         equal:
           path: metadata.annotations["helm.sh/hook-weight"]
@@ -57,7 +57,7 @@ tests:
       - documentIndex: 2
         equal:
           path: kind
-          value: ClusterRoleBinding
+          value: RoleBinding
       - documentIndex: 2
         equal:
           path: metadata.annotations["helm.sh/hook-weight"]

From fc057c039385e7b40cca3f2b6ca77a8f80bab49a Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Fri, 24 Apr 2026 13:32:33 +0300
Subject: [PATCH 087/151] fix(postgres-operator): set resources on
 webhook-ready wait container

Address review feedback from gemini-code-assist on
packages/system/postgres-operator/templates/webhook-ready-hook.yaml:115.

The wait container had no resource requests or limits, so schedulers
treated it as BestEffort and downstream quota enforcement had no signal.
Set small requests (10m CPU, 32Mi memory) and conservative limits
(100m CPU, 64Mi memory) matching the actual footprint of the kubectl
polling loop.

Add a matching unittest assertion so the values stay in sync if anyone
touches the template.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 .../templates/webhook-ready-hook.yaml         |  7 +++++++
 .../tests/webhook-ready-hook_test.yaml        | 20 +++++++++++++++++++
 2 files changed, 27 insertions(+)

diff --git a/packages/system/postgres-operator/templates/webhook-ready-hook.yaml b/packages/system/postgres-operator/templates/webhook-ready-hook.yaml
index 70e51621..51a8ba81 100644
--- a/packages/system/postgres-operator/templates/webhook-ready-hook.yaml
+++ b/packages/system/postgres-operator/templates/webhook-ready-hook.yaml
@@ -110,6 +110,13 @@ spec:
         - name: wait
           image: {{ $image }}
           imagePullPolicy: {{ if .Values.webhookReady.image.digest }}IfNotPresent{{ else }}Always{{ end }}
+          resources:
+            requests:
+              cpu: 10m
+              memory: 32Mi
+            limits:
+              cpu: 100m
+              memory: 64Mi
           securityContext:
             allowPrivilegeEscalation: false
             readOnlyRootFilesystem: true
diff --git a/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml b/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml
index eb7863da..3253c663 100644
--- a/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml
+++ b/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml
@@ -205,6 +205,26 @@ tests:
           path: spec.template.spec.securityContext.seccompProfile.type
           value: RuntimeDefault
 
+  - it: wait container declares resource requests and limits for predictable scheduling
+    template: templates/webhook-ready-hook.yaml
+    asserts:
+      - documentIndex: 3
+        equal:
+          path: spec.template.spec.containers[0].resources.requests.cpu
+          value: 10m
+      - documentIndex: 3
+        equal:
+          path: spec.template.spec.containers[0].resources.requests.memory
+          value: 32Mi
+      - documentIndex: 3
+        equal:
+          path: spec.template.spec.containers[0].resources.limits.cpu
+          value: 100m
+      - documentIndex: 3
+        equal:
+          path: spec.template.spec.containers[0].resources.limits.memory
+          value: 64Mi
+
   - it: Job container drops all capabilities and runs read-only rootfs
     template: templates/webhook-ready-hook.yaml
     asserts:

From 7da29afe6e1f33b69d6040fd1051924d7937a13b Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Fri, 24 Apr 2026 13:33:07 +0300
Subject: [PATCH 088/151] docs(postgres-operator): fix
 backoffLimit/activeDeadlineSeconds comment math

Address review feedback from coderabbitai on
packages/system/postgres-operator/values.yaml:30.

The old comment's 5-minute ImagePullBackOff scenario conflicted with
the 180s activeDeadlineSeconds that the default maxAttempts/sleepSeconds
resolve to, so the numbers could not both be taken at face value.

Rewrite the comment to state the actual deadline math and frame the two
gates as an AND with activeDeadlineSeconds being the shorter one under
defaults, so readers understand why backoffLimit has little headroom
without an accompanying maxAttempts/sleepSeconds bump.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 packages/system/postgres-operator/values.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/packages/system/postgres-operator/values.yaml b/packages/system/postgres-operator/values.yaml
index 854a3ac6..c408220e 100644
--- a/packages/system/postgres-operator/values.yaml
+++ b/packages/system/postgres-operator/values.yaml
@@ -25,6 +25,8 @@ webhookReady:
   # Pod-level retry budget for transient node/registry failures (image pull rate limit,
   # OOM, CNI hiccup). The shell loop inside the container only covers reachability retries.
   # Pod retries and activeDeadlineSeconds (wall-clock bound on the whole Job across all
-  # pod retries) are ANDed: a 5-minute ImagePullBackOff on attempt 0 leaves only ~60s for
-  # subsequent retries before the Job is cut.
+  # pod retries) are ANDed, so activeDeadlineSeconds is the shorter of the two gates with
+  # the defaults above: the 60*2+60 = 180s deadline cuts the Job before backoffLimit=2
+  # ever matters if pod-level failures eat more than ~60s of the budget. Raise
+  # maxAttempts/sleepSeconds alongside backoffLimit when tuning for slow image pulls.
   backoffLimit: 2

From 2734dc0bcb96aaa087cbff6b2ee261e1bde8bffa Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Fri, 24 Apr 2026 14:00:48 +0300
Subject: [PATCH 089/151] fix(hami): clean up leftover hami-dra references and
 harden defaults

Remove broken hami-dra subchart dependency from vendored chart
(Chart.yaml, Chart.lock, values.yaml) and strip DRA condition guards
from all templates since the subchart was already deleted. Override
devicePlugin updateStrategy to OnDelete to prevent destructive rolling
updates of GPU workloads. Align gpu-operator template with project
pattern (unconditional values emission). Add nodeConfiguration format
documentation and test for conflicting valuesOverride scenario.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 .../templates/helmreleases/gpu-operator.yaml  |  6 ++---
 .../tests/gpu_operator_hami_test.yaml         | 25 ++++++++++++++++++-
 packages/system/hami/charts/hami/Chart.lock   |  6 -----
 packages/system/hami/charts/hami/Chart.yaml   |  6 +----
 .../templates/device-plugin/configmap.yaml    |  2 +-
 .../device-plugin/daemonsetnvidia.yaml        |  2 +-
 .../templates/device-plugin/monitorrole.yaml  |  2 +-
 .../device-plugin/monitorrolebinding.yaml     |  2 +-
 .../device-plugin/monitorservice.yaml         |  2 +-
 .../device-plugin/monitorserviceaccount.yaml  |  2 +-
 .../device-plugin/runtime-class.yaml          |  4 +--
 .../hami/templates/scheduler/certmanager.yaml |  2 +-
 .../hami/templates/scheduler/clusterrole.yaml |  2 --
 .../scheduler/clusterrolebinding.yaml         |  4 +--
 .../hami/templates/scheduler/configmap.yaml   |  2 +-
 .../templates/scheduler/configmapnew.yaml     |  2 +-
 .../hami/templates/scheduler/deployment.yaml  |  4 +--
 .../templates/scheduler/device-configmap.yaml |  4 +--
 .../scheduler/job-patch/clusterrole.yaml      |  4 +--
 .../job-patch/clusterrolebinding.yaml         |  4 +--
 .../scheduler/job-patch/job-createSecret.yaml |  4 +--
 .../scheduler/job-patch/job-patchWebhook.yaml |  4 +--
 .../templates/scheduler/job-patch/psp.yaml    |  4 +--
 .../templates/scheduler/job-patch/role.yaml   |  4 +--
 .../scheduler/job-patch/rolebinding.yaml      |  4 +--
 .../scheduler/job-patch/serviceaccount.yaml   |  4 +--
 .../charts/hami/templates/scheduler/role.yaml |  4 +--
 .../hami/templates/scheduler/rolebinding.yaml |  2 --
 .../hami/templates/scheduler/service.yaml     |  4 +--
 .../templates/scheduler/serviceaccount.yaml   |  2 --
 .../hami/templates/scheduler/webhook.yaml     |  2 +-
 packages/system/hami/charts/hami/values.yaml  | 18 -------------
 packages/system/hami/values.yaml              |  4 +++
 33 files changed, 55 insertions(+), 92 deletions(-)
 delete mode 100644 packages/system/hami/charts/hami/Chart.lock

diff --git a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml
index 995aa430..7a5ecaf1 100644
--- a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml
+++ b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml
@@ -1,5 +1,5 @@
 {{- define "cozystack.defaultGpuOperatorValues" -}}
-{{- if and (hasKey .Values.addons "hami") .Values.addons.hami.enabled }}
+{{- if .Values.addons.hami.enabled }}
 gpu-operator:
   devicePlugin:
     enabled: false
@@ -40,10 +40,8 @@ spec:
   {{- $defaults := fromYaml (include "cozystack.defaultGpuOperatorValues" .) }}
   {{- $overrides := deepCopy (default (dict) .Values.addons.gpuOperator.valuesOverride) }}
   {{- $merged := mergeOverwrite (default (dict) $defaults) $overrides }}
-  {{- with $merged }}
   values:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
+    {{- if $merged }}{{ toYaml $merged | nindent 4 }}{{ end }}
 
   dependsOn:
   {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }}
diff --git a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml
index 33256ae6..11de743a 100644
--- a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml
+++ b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml
@@ -26,8 +26,9 @@ tests:
           enabled: false
           valuesOverride: {}
     asserts:
-      - notExists:
+      - equal:
           path: spec.values
+          value: null
 
   - it: should allow user overrides to merge with hami defaults
     set:
@@ -66,6 +67,28 @@ tests:
           path: spec.values.gpu-operator.devicePlugin.enabled
           value: true
 
+  - it: should let user explicitly override devicePlugin.enabled to true with hami enabled
+    set:
+      addons:
+        gpuOperator:
+          enabled: true
+          valuesOverride:
+            gpu-operator:
+              devicePlugin:
+                enabled: true
+              driver:
+                enabled: false
+        hami:
+          enabled: true
+          valuesOverride: {}
+    asserts:
+      - equal:
+          path: spec.values.gpu-operator.devicePlugin.enabled
+          value: true
+      - equal:
+          path: spec.values.gpu-operator.driver.enabled
+          value: false
+
   - it: should not render when gpuOperator is disabled
     set:
       addons:
diff --git a/packages/system/hami/charts/hami/Chart.lock b/packages/system/hami/charts/hami/Chart.lock
deleted file mode 100644
index 25856c07..00000000
--- a/packages/system/hami/charts/hami/Chart.lock
+++ /dev/null
@@ -1,6 +0,0 @@
-dependencies:
-- name: hami-dra
-  repository: https://project-hami.github.io/HAMi-DRA/
-  version: 0.1.0
-digest: sha256:374551539570bcee82c1c4dea93eac59e6f410b6d5967d188efd630e203d43bb
-generated: "2026-04-17T04:06:47.689117678Z"
diff --git a/packages/system/hami/charts/hami/Chart.yaml b/packages/system/hami/charts/hami/Chart.yaml
index aba7e95c..55f32ab6 100644
--- a/packages/system/hami/charts/hami/Chart.yaml
+++ b/packages/system/hami/charts/hami/Chart.yaml
@@ -1,10 +1,6 @@
 apiVersion: v2
 appVersion: 2.8.1
-dependencies:
-- condition: dra.enabled
-  name: hami-dra
-  repository: https://project-hami.github.io/HAMi-DRA/
-  version: 0.1.0
+dependencies: []
 description: Heterogeneous AI Computing Virtualization Middleware
 keywords:
 - vgpu
diff --git a/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml b/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml
index b36347f2..db2ddacb 100644
--- a/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml
+++ b/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml
@@ -1,4 +1,4 @@
-{{- if and .Values.devicePlugin.enabled (not .Values.devicePlugin.nodeConfiguration.externalConfigName) (not .Values.dra.enabled) -}}
+{{- if and .Values.devicePlugin.enabled (not .Values.devicePlugin.nodeConfiguration.externalConfigName) -}}
 apiVersion: v1
 kind: ConfigMap
 metadata:
diff --git a/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml
index 577dc0f9..1f4c24f3 100644
--- a/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml
+++ b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml
@@ -1,4 +1,4 @@
-{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) }}
+{{- if .Values.devicePlugin.enabled }}
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml
index c51ff6c6..377f6a86 100644
--- a/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml
+++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml
@@ -1,4 +1,4 @@
-{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}}
+{{- if .Values.devicePlugin.enabled -}}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml
index 93a118dd..2f0a14ba 100644
--- a/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml
+++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml
@@ -1,4 +1,4 @@
-{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}}
+{{- if .Values.devicePlugin.enabled -}}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml
index 24daca9b..0fb156cc 100644
--- a/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml
+++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml
@@ -1,4 +1,4 @@
-{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}}
+{{- if .Values.devicePlugin.enabled -}}
 apiVersion: v1
 kind: Service
 metadata:
diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml
index b10d2cb9..6e3c2a44 100644
--- a/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml
+++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml
@@ -1,4 +1,4 @@
-{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}}
+{{- if .Values.devicePlugin.enabled -}}
 apiVersion: v1
 kind: ServiceAccount
 metadata:
diff --git a/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml b/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml
index cd3d14cc..ebcc2c9c 100644
--- a/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml
+++ b/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml
@@ -1,5 +1,4 @@
-{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}}
-{{- if and .Values.devicePlugin.createRuntimeClass .Values.devicePlugin.runtimeClassName }}
+{{- if and .Values.devicePlugin.enabled .Values.devicePlugin.createRuntimeClass .Values.devicePlugin.runtimeClassName -}}
 apiVersion: node.k8s.io/v1
 kind: RuntimeClass
 metadata:
@@ -7,5 +6,4 @@ metadata:
   annotations:
     helm.sh/hook: pre-install,pre-upgrade
 handler: nvidia
-{{- end }}
 {{- end -}}
diff --git a/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml b/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml
index b6edb605..e6d28721 100644
--- a/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml
+++ b/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml
@@ -1,4 +1,4 @@
-{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}}
+{{- if .Values.scheduler.admissionWebhook.enabled -}}
 {{- if .Values.scheduler.certManager.enabled }}
 apiVersion: cert-manager.io/v1
 kind: Certificate
diff --git a/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml b/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml
index 9c510c00..81c4fddf 100644
--- a/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml
+++ b/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml
@@ -1,4 +1,3 @@
-{{- if not .Values.dra.enabled -}}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
@@ -33,4 +32,3 @@ rules:
     resources: ["nodes"]
     verbs: ["get", "update", "list", "patch"]
 {{- end -}}
-{{- end }}
diff --git a/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml
index 69d75986..aa0f3c66 100644
--- a/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml
+++ b/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml
@@ -1,4 +1,3 @@
-{{- if not .Values.dra.enabled -}}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
@@ -45,5 +44,4 @@ roleRef:
 subjects:
   - kind: ServiceAccount
     name: {{ include "hami-vgpu.scheduler" . }}
-    namespace: {{ include "hami-vgpu.namespace" . }}
-{{- end }}
\ No newline at end of file
+    namespace: {{ include "hami-vgpu.namespace" . }}
\ No newline at end of file
diff --git a/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml b/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml
index 75889146..109ecbce 100644
--- a/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml
+++ b/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml
@@ -1,4 +1,4 @@
-{{- if and .Values.scheduler.kubeScheduler.enabled (not .Values.dra.enabled) -}}
+{{- if .Values.scheduler.kubeScheduler.enabled -}}
 apiVersion: v1
 kind: ConfigMap
 metadata:
diff --git a/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml b/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml
index e2a91d8b..6f6db097 100644
--- a/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml
+++ b/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml
@@ -1,4 +1,4 @@
-{{- if and .Values.scheduler.kubeScheduler.enabled (not .Values.dra.enabled) -}}
+{{- if .Values.scheduler.kubeScheduler.enabled -}}
 apiVersion: v1
 kind: ConfigMap
 metadata:
diff --git a/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml b/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml
index 6364cae3..12911a42 100644
--- a/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml
+++ b/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml
@@ -1,4 +1,3 @@
-{{- if not .Values.dra.enabled -}}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -223,5 +222,4 @@ spec:
       {{- end }}
       {{- if .Values.scheduler.nodeName }}
       nodeName: {{ .Values.scheduler.nodeName }}
-      {{- end }}
-{{- end }}
\ No newline at end of file
+      {{- end }}
\ No newline at end of file
diff --git a/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml b/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml
index ccb945ac..8738a06f 100644
--- a/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml
+++ b/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml
@@ -1,4 +1,3 @@
-{{- if not .Values.dra.enabled -}}
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -406,5 +405,4 @@ data:
           memory: 12288
           aiCore: 4
           aiCPU: 4
-  {{ end }}
-{{- end }}
\ No newline at end of file
+  {{ end }}
\ No newline at end of file
diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml
index b089ae82..412b1094 100644
--- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml
+++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml
@@ -1,5 +1,4 @@
-{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}}
-{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
+{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
@@ -27,4 +26,3 @@ rules:
     - {{ include "hami-vgpu.fullname" . }}-admission
 {{- end }}
 {{- end }}
-{{- end }}
diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml
index 64f01ecf..2b82f926 100644
--- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml
+++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml
@@ -1,5 +1,4 @@
-{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}}
-{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
+{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
@@ -19,4 +18,3 @@ subjects:
     name: {{ include "hami-vgpu.fullname" . }}-admission
     namespace: {{ include "hami-vgpu.namespace" . }}
 {{- end }}
-{{- end }}
diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml
index 400168a7..0e36d95f 100644
--- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml
+++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml
@@ -1,5 +1,4 @@
-{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}}
-{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
+{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}}
 apiVersion: batch/v1
 kind: Job
 metadata:
@@ -67,4 +66,3 @@ spec:
         runAsNonRoot: true
         runAsUser: {{ .Values.scheduler.patch.runAsUser }}
 {{- end }}
-{{- end }}
diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml
index 71e97f6b..ce52042c 100644
--- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml
+++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml
@@ -1,5 +1,4 @@
-{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}}
-{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
+{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}}
 apiVersion: batch/v1
 kind: Job
 metadata:
@@ -62,4 +61,3 @@ spec:
         runAsNonRoot: true
         runAsUser: {{ .Values.scheduler.patch.runAsUser }}
 {{- end }}
-{{- end }}
diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml
index a48270e7..1e9cd7da 100644
--- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml
+++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml
@@ -1,5 +1,4 @@
-{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}}
-{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
+{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}}
 {{- if .Values.podSecurityPolicy.enabled }}
 apiVersion: policy/v1beta1
 kind: PodSecurityPolicy
@@ -37,4 +36,3 @@ spec:
   - downwardAPI
 {{- end }}
 {{- end }}
-{{- end }}
diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml
index 74e45681..56682f8e 100644
--- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml
+++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml
@@ -1,5 +1,4 @@
-{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}}
-{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
+{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
@@ -20,4 +19,3 @@ rules:
       - get
       - create
 {{- end }}
-{{- end }}
diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml
index 32da5ba4..7239b128 100644
--- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml
+++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml
@@ -1,5 +1,4 @@
-{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}}
-{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
+{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
@@ -20,4 +19,3 @@ subjects:
     name: {{ include "hami-vgpu.fullname" . }}-admission
     namespace: {{ include "hami-vgpu.namespace" . }}
 {{- end }}
-{{- end }}
diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml
index d2eb41de..857c6a8d 100644
--- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml
+++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml
@@ -1,5 +1,4 @@
-{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) }}
-{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
+{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
 apiVersion: v1
 kind: ServiceAccount
 metadata:
@@ -12,4 +11,3 @@ metadata:
     {{- include "hami-vgpu.labels" . | nindent 4 }}
     app.kubernetes.io/component: admission-webhook
 {{- end }}
-{{- end }}
diff --git a/packages/system/hami/charts/hami/templates/scheduler/role.yaml b/packages/system/hami/charts/hami/templates/scheduler/role.yaml
index 6e681967..90987a4f 100644
--- a/packages/system/hami/charts/hami/templates/scheduler/role.yaml
+++ b/packages/system/hami/charts/hami/templates/scheduler/role.yaml
@@ -1,4 +1,3 @@
-{{- if not .Values.dra.enabled -}}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
@@ -10,5 +9,4 @@ metadata:
 rules:
   - apiGroups: ["coordination.k8s.io"]
     resources: ["leases"]
-    verbs: ["create", "list", "watch", "get", "update", "patch"]
-{{- end }}
\ No newline at end of file
+    verbs: ["create", "list", "watch", "get", "update", "patch"]
\ No newline at end of file
diff --git a/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml
index 934f8223..96e175a1 100644
--- a/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml
+++ b/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml
@@ -1,4 +1,3 @@
-{{- if not .Values.dra.enabled -}}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
@@ -30,4 +29,3 @@ subjects:
     name: {{ include "hami-vgpu.mock-device-plugin" . }}
     namespace: {{ include "hami-vgpu.namespace" . }}
 {{- end -}}
-{{- end }}
diff --git a/packages/system/hami/charts/hami/templates/scheduler/service.yaml b/packages/system/hami/charts/hami/templates/scheduler/service.yaml
index 3e18bc98..70378c5c 100644
--- a/packages/system/hami/charts/hami/templates/scheduler/service.yaml
+++ b/packages/system/hami/charts/hami/templates/scheduler/service.yaml
@@ -1,4 +1,3 @@
-{{- if not .Values.dra.enabled }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -32,5 +31,4 @@ spec:
       protocol: TCP
   selector:
     app.kubernetes.io/component: hami-scheduler
-    {{- include "hami-vgpu.selectorLabels" . | nindent 4 }}
-{{- end }}
\ No newline at end of file
+    {{- include "hami-vgpu.selectorLabels" . | nindent 4 }}
\ No newline at end of file
diff --git a/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml b/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml
index 929bf59a..0435c003 100644
--- a/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml
+++ b/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml
@@ -1,4 +1,3 @@
-{{- if not .Values.dra.enabled -}}
 apiVersion: v1
 kind: ServiceAccount
 metadata:
@@ -15,4 +14,3 @@ metadata:
   name: {{ include "hami-vgpu.mock-device-plugin" . }}
   namespace: {{ include "hami-vgpu.namespace" . }}
 {{- end -}}
-{{- end }}
diff --git a/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml b/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml
index 148feb5c..7296eb4e 100644
--- a/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml
+++ b/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml
@@ -1,4 +1,4 @@
-{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}}
+{{- if .Values.scheduler.admissionWebhook.enabled -}}
 apiVersion: admissionregistration.k8s.io/v1
 kind: MutatingWebhookConfiguration
 metadata:
diff --git a/packages/system/hami/charts/hami/values.yaml b/packages/system/hami/charts/hami/values.yaml
index 9fbeca70..02feeada 100644
--- a/packages/system/hami/charts/hami/values.yaml
+++ b/packages/system/hami/charts/hami/values.yaml
@@ -392,24 +392,6 @@ mockDevicePlugin:
     ##   - myRegistryKeySecretName
     ##
     pullSecrets: []
-# If this option is enabled, the DRA will be installed and the scheduler extender will not be installed.
-dra:
-  enabled: false
-
-# Configuration for the hami-dra subchart, which includes DRA drivers.
-# This is part of config for DRA, please refer to the DRA documentation for more details.
-hami-dra:
-  monitor:
-    enabled: true
-  drivers:
-    nvidia:
-      enabled: true
-      # If you are using gpu driver on host, you need to set this to false
-      containerDriver: true
-      image:
-        repository: projecthami/k8s-dra-driver
-        tag: "v0.0.1-dev"
-
 devices:
   amd:
     customresources:
diff --git a/packages/system/hami/values.yaml b/packages/system/hami/values.yaml
index 1155e2cb..1d1407b7 100644
--- a/packages/system/hami/values.yaml
+++ b/packages/system/hami/values.yaml
@@ -1,6 +1,10 @@
 hami:
   devicePlugin:
     runtimeClassName: nvidia
+    updateStrategy:
+      type: OnDelete
+    # See upstream charts/hami/values.yaml for nodeConfiguration format.
+    # Each entry: {"name": "node-name", "operatingmode": "hami-core", "devicesplitcount": N, ...}
     nodeConfiguration:
       config: |
         {

From 37d5ff0c6f0566ca18495e871c5ad626f7976ca1 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Fri, 24 Apr 2026 14:07:17 +0300
Subject: [PATCH 090/151] fix(hami): align templates with project patterns and
 clean dead code

Unconditionally emit values in hami.yaml matching the project pattern.
Remove duplicate test case and add coverage for omitted valuesOverride key.
Delete dead PSP template and RBAC rules (policy/v1beta1 removed in K8s
1.25). Override kube-scheduler image registry to registry.k8s.io to avoid
Chinese registry for international users.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 .../templates/helmreleases/hami.yaml          |  4 +-
 .../tests/gpu_operator_hami_test.yaml         | 29 ++++++--------
 .../scheduler/job-patch/clusterrole.yaml      |  7 ----
 .../templates/scheduler/job-patch/psp.yaml    | 38 -------------------
 packages/system/hami/charts/hami/values.yaml  |  3 --
 packages/system/hami/values.yaml              |  5 +++
 6 files changed, 18 insertions(+), 68 deletions(-)
 delete mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml

diff --git a/packages/apps/kubernetes/templates/helmreleases/hami.yaml b/packages/apps/kubernetes/templates/helmreleases/hami.yaml
index 1f7a5358..ac4714da 100644
--- a/packages/apps/kubernetes/templates/helmreleases/hami.yaml
+++ b/packages/apps/kubernetes/templates/helmreleases/hami.yaml
@@ -32,10 +32,8 @@ spec:
     force: true
     remediation:
       retries: -1
-  {{- with .Values.addons.hami.valuesOverride }}
   values:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
+    {{- if .Values.addons.hami.valuesOverride }}{{ toYaml .Values.addons.hami.valuesOverride | nindent 4 }}{{ end }}
 
   dependsOn:
   {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }}
diff --git a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml
index 11de743a..c3ec30f2 100644
--- a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml
+++ b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml
@@ -30,6 +30,18 @@ tests:
           path: spec.values
           value: null
 
+  - it: should apply hami defaults when valuesOverride key is omitted
+    set:
+      addons:
+        gpuOperator:
+          enabled: true
+        hami:
+          enabled: true
+    asserts:
+      - equal:
+          path: spec.values.gpu-operator.devicePlugin.enabled
+          value: false
+
   - it: should allow user overrides to merge with hami defaults
     set:
       addons:
@@ -50,23 +62,6 @@ tests:
           path: spec.values.gpu-operator.driver.enabled
           value: false
 
-  - it: should let user override devicePlugin back to true if needed
-    set:
-      addons:
-        gpuOperator:
-          enabled: true
-          valuesOverride:
-            gpu-operator:
-              devicePlugin:
-                enabled: true
-        hami:
-          enabled: true
-          valuesOverride: {}
-    asserts:
-      - equal:
-          path: spec.values.gpu-operator.devicePlugin.enabled
-          value: true
-
   - it: should let user explicitly override devicePlugin.enabled to true with hami enabled
     set:
       addons:
diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml
index 412b1094..77e891cc 100644
--- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml
+++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml
@@ -18,11 +18,4 @@ rules:
     verbs:
       - get
       - update
-{{- if .Values.podSecurityPolicy.enabled }}
-  - apiGroups: ['extensions']
-    resources: ['podsecuritypolicies']
-    verbs:     ['use']
-    resourceNames:
-    - {{ include "hami-vgpu.fullname" . }}-admission
-{{- end }}
 {{- end }}
diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml
deleted file mode 100644
index 1e9cd7da..00000000
--- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml
+++ /dev/null
@@ -1,38 +0,0 @@
-{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}}
-{{- if .Values.podSecurityPolicy.enabled }}
-apiVersion: policy/v1beta1
-kind: PodSecurityPolicy
-metadata:
-  name: {{ include "hami-vgpu.fullname" . }}-admission
-  annotations:
-    "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
-    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
-  labels:
-    {{- include "hami-vgpu.labels" . | nindent 4 }}
-    app.kubernetes.io/component: admission-webhook
-spec:
-  allowPrivilegeEscalation: false
-  fsGroup:
-    ranges:
-    - max: 65535
-      min: 1
-    rule: MustRunAs
-  requiredDropCapabilities:
-  - ALL
-  runAsUser:
-    rule: MustRunAsNonRoot
-  seLinux:
-    rule: RunAsAny
-  supplementalGroups:
-    ranges:
-    - max: 65535
-      min: 1
-    rule: MustRunAs
-  volumes:
-  - configMap
-  - emptyDir
-  - projected
-  - secret
-  - downwardAPI
-{{- end }}
-{{- end }}
diff --git a/packages/system/hami/charts/hami/values.yaml b/packages/system/hami/charts/hami/values.yaml
index 02feeada..01caca0b 100644
--- a/packages/system/hami/charts/hami/values.yaml
+++ b/packages/system/hami/charts/hami/values.yaml
@@ -55,9 +55,6 @@ kunlunResourceVMemoryName: "kunlunxin.com/vxpu-memory"
 
 schedulerName: "hami-scheduler"
 
-podSecurityPolicy:
-  enabled: false
-
 scheduler:
   # @param nodeName defines the node name and the nvidia-vgpu-scheduler-scheduler will schedule to the node.
   # if we install the nvidia-vgpu-scheduler-scheduler as default scheduler, we need to remove the k8s default
diff --git a/packages/system/hami/values.yaml b/packages/system/hami/values.yaml
index 1d1407b7..f45d2998 100644
--- a/packages/system/hami/values.yaml
+++ b/packages/system/hami/values.yaml
@@ -1,4 +1,9 @@
 hami:
+  scheduler:
+    kubeScheduler:
+      image:
+        registry: registry.k8s.io
+        repository: kube-scheduler
   devicePlugin:
     runtimeClassName: nvidia
     updateStrategy:

From 6a9c310a4bfdb8bab3cd4f52724205adfbd2219e Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Fri, 24 Apr 2026 15:31:24 +0300
Subject: [PATCH 091/151] fix(hami): conditional values emission and cosmetic
 template cleanup

Only emit values key in hami.yaml when valuesOverride has content,
matching gpu-operator pattern. Add test verifying empty valuesOverride
does not produce spec.values. Fix trailing whitespace and missing
newlines in vendored chart templates.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 .../kubernetes/templates/helmreleases/hami.yaml   |  4 +++-
 packages/apps/kubernetes/tests/hami_test.yaml     | 15 +++++++++++++++
 .../hami/charts/hami/templates/_commons.tpl       |  2 +-
 .../hami/templates/device-plugin/configmap.yaml   |  2 +-
 .../hami/templates/device-plugin/monitorrole.yaml |  3 +--
 .../templates/device-plugin/monitorservice.yaml   |  2 +-
 .../templates/scheduler/clusterrolebinding.yaml   |  2 +-
 .../hami/templates/scheduler/deployment.yaml      |  2 +-
 .../templates/scheduler/device-configmap.yaml     |  2 +-
 .../charts/hami/templates/scheduler/role.yaml     |  2 +-
 .../charts/hami/templates/scheduler/service.yaml  |  2 +-
 .../charts/hami/templates/scheduler/webhook.yaml  |  2 +-
 12 files changed, 28 insertions(+), 12 deletions(-)

diff --git a/packages/apps/kubernetes/templates/helmreleases/hami.yaml b/packages/apps/kubernetes/templates/helmreleases/hami.yaml
index ac4714da..8733e675 100644
--- a/packages/apps/kubernetes/templates/helmreleases/hami.yaml
+++ b/packages/apps/kubernetes/templates/helmreleases/hami.yaml
@@ -32,8 +32,10 @@ spec:
     force: true
     remediation:
       retries: -1
+  {{- if .Values.addons.hami.valuesOverride }}
   values:
-    {{- if .Values.addons.hami.valuesOverride }}{{ toYaml .Values.addons.hami.valuesOverride | nindent 4 }}{{ end }}
+    {{- toYaml .Values.addons.hami.valuesOverride | nindent 4 }}
+  {{- end }}
 
   dependsOn:
   {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }}
diff --git a/packages/apps/kubernetes/tests/hami_test.yaml b/packages/apps/kubernetes/tests/hami_test.yaml
index e9d32236..bdbe990f 100644
--- a/packages/apps/kubernetes/tests/hami_test.yaml
+++ b/packages/apps/kubernetes/tests/hami_test.yaml
@@ -118,6 +118,21 @@ tests:
             name: test-gpu-operator
             namespace: test-ns
 
+  - it: should not render spec.values when valuesOverride is empty
+    set:
+      addons:
+        hami:
+          enabled: true
+          valuesOverride: {}
+        gpuOperator:
+          enabled: true
+          valuesOverride: {}
+    asserts:
+      - hasDocuments:
+          count: 1
+      - notExists:
+          path: spec.values
+
   - it: should pass through valuesOverride
     set:
       addons:
diff --git a/packages/system/hami/charts/hami/templates/_commons.tpl b/packages/system/hami/charts/hami/templates/_commons.tpl
index 78a262c9..b68018e4 100644
--- a/packages/system/hami/charts/hami/templates/_commons.tpl
+++ b/packages/system/hami/charts/hami/templates/_commons.tpl
@@ -46,4 +46,4 @@ imagePullSecrets:
   - name: {{ . }}
     {{- end }}
   {{- end }}
-{{- end -}}
\ No newline at end of file
+{{- end -}}
diff --git a/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml b/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml
index db2ddacb..74631e24 100644
--- a/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml
+++ b/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml
@@ -10,4 +10,4 @@ metadata:
 data:
   config.json: |
 {{- .Values.devicePlugin.nodeConfiguration.config | nindent 4 }}
-{{- end }}
\ No newline at end of file
+{{- end }}
diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml
index 377f6a86..6ac757f2 100644
--- a/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml
+++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml
@@ -25,5 +25,4 @@ rules:
       - list
       - patch
 {{- end -}}
-    
-    
+
diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml
index 0fb156cc..104664ea 100644
--- a/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml
+++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml
@@ -26,4 +26,4 @@ spec:
   selector:
     app.kubernetes.io/component: hami-device-plugin
     {{- include "hami-vgpu.selectorLabels" . | nindent 4 }}
-{{- end -}}
\ No newline at end of file
+{{- end -}}
diff --git a/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml
index aa0f3c66..a81d425c 100644
--- a/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml
+++ b/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml
@@ -44,4 +44,4 @@ roleRef:
 subjects:
   - kind: ServiceAccount
     name: {{ include "hami-vgpu.scheduler" . }}
-    namespace: {{ include "hami-vgpu.namespace" . }}
\ No newline at end of file
+    namespace: {{ include "hami-vgpu.namespace" . }}
diff --git a/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml b/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml
index 12911a42..1d89e189 100644
--- a/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml
+++ b/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml
@@ -222,4 +222,4 @@ spec:
       {{- end }}
       {{- if .Values.scheduler.nodeName }}
       nodeName: {{ .Values.scheduler.nodeName }}
-      {{- end }}
\ No newline at end of file
+      {{- end }}
diff --git a/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml b/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml
index 8738a06f..873b813b 100644
--- a/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml
+++ b/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml
@@ -405,4 +405,4 @@ data:
           memory: 12288
           aiCore: 4
           aiCPU: 4
-  {{ end }}
\ No newline at end of file
+  {{ end }}
diff --git a/packages/system/hami/charts/hami/templates/scheduler/role.yaml b/packages/system/hami/charts/hami/templates/scheduler/role.yaml
index 90987a4f..5f7d7e44 100644
--- a/packages/system/hami/charts/hami/templates/scheduler/role.yaml
+++ b/packages/system/hami/charts/hami/templates/scheduler/role.yaml
@@ -9,4 +9,4 @@ metadata:
 rules:
   - apiGroups: ["coordination.k8s.io"]
     resources: ["leases"]
-    verbs: ["create", "list", "watch", "get", "update", "patch"]
\ No newline at end of file
+    verbs: ["create", "list", "watch", "get", "update", "patch"]
diff --git a/packages/system/hami/charts/hami/templates/scheduler/service.yaml b/packages/system/hami/charts/hami/templates/scheduler/service.yaml
index 70378c5c..d7538fed 100644
--- a/packages/system/hami/charts/hami/templates/scheduler/service.yaml
+++ b/packages/system/hami/charts/hami/templates/scheduler/service.yaml
@@ -31,4 +31,4 @@ spec:
       protocol: TCP
   selector:
     app.kubernetes.io/component: hami-scheduler
-    {{- include "hami-vgpu.selectorLabels" . | nindent 4 }}
\ No newline at end of file
+    {{- include "hami-vgpu.selectorLabels" . | nindent 4 }}
diff --git a/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml b/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml
index 7296eb4e..db9f8029 100644
--- a/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml
+++ b/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml
@@ -54,4 +54,4 @@ webhooks:
         scope: '*'
     sideEffects: None
     timeoutSeconds: 10
-{{- end }}
\ No newline at end of file
+{{- end }}

From 1131e2f113a8fda8402bf9c92014bff9aaf29f25 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Fri, 24 Apr 2026 15:33:07 +0300
Subject: [PATCH 092/151] docs(hami): reference upstream repo for
 nodeConfiguration format

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 packages/system/hami/values.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/system/hami/values.yaml b/packages/system/hami/values.yaml
index f45d2998..64b372a6 100644
--- a/packages/system/hami/values.yaml
+++ b/packages/system/hami/values.yaml
@@ -8,7 +8,7 @@ hami:
     runtimeClassName: nvidia
     updateStrategy:
       type: OnDelete
-    # See upstream charts/hami/values.yaml for nodeConfiguration format.
+    # See https://github.com/Project-HAMi/HAMi/blob/v2.8.1/deployments/charts/hami/values.yaml
     # Each entry: {"name": "node-name", "operatingmode": "hami-core", "devicesplitcount": N, ...}
     nodeConfiguration:
       config: |

From 3eeda2ba3596851393fd62d3d352cdd1017be267 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Fri, 24 Apr 2026 16:12:34 +0300
Subject: [PATCH 093/151] chore(kubernetes): regenerate code after hami addon
 addition

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 .../kubernetes/zz_generated.deepcopy.go         | 17 +++++++++++++++++
 api/backups/v1alpha1/zz_generated.deepcopy.go   |  5 +++++
 2 files changed, 22 insertions(+)

diff --git a/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go b/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go
index 2cbe3ba1..3f1cb5ce 100644
--- a/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go
+++ b/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go
@@ -49,6 +49,7 @@ func (in *Addons) DeepCopyInto(out *Addons) {
 	in.Fluxcd.DeepCopyInto(&out.Fluxcd)
 	out.GatewayAPI = in.GatewayAPI
 	in.GpuOperator.DeepCopyInto(&out.GpuOperator)
+	in.Hami.DeepCopyInto(&out.Hami)
 	in.IngressNginx.DeepCopyInto(&out.IngressNginx)
 	in.MonitoringAgents.DeepCopyInto(&out.MonitoringAgents)
 	in.Velero.DeepCopyInto(&out.Velero)
@@ -260,6 +261,22 @@ func (in *GatewayAPIAddon) DeepCopy() *GatewayAPIAddon {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *HAMiAddon) DeepCopyInto(out *HAMiAddon) {
+	*out = *in
+	in.ValuesOverride.DeepCopyInto(&out.ValuesOverride)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HAMiAddon.
+func (in *HAMiAddon) DeepCopy() *HAMiAddon {
+	if in == nil {
+		return nil
+	}
+	out := new(HAMiAddon)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *IngressNginxAddon) DeepCopyInto(out *IngressNginxAddon) {
 	*out = *in
diff --git a/api/backups/v1alpha1/zz_generated.deepcopy.go b/api/backups/v1alpha1/zz_generated.deepcopy.go
index 89f6171f..61b8f839 100644
--- a/api/backups/v1alpha1/zz_generated.deepcopy.go
+++ b/api/backups/v1alpha1/zz_generated.deepcopy.go
@@ -620,6 +620,11 @@ func (in *RestoreJobSpec) DeepCopyInto(out *RestoreJobSpec) {
 		*out = new(v1.TypedLocalObjectReference)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.Options != nil {
+		in, out := &in.Options, &out.Options
+		*out = new(runtime.RawExtension)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestoreJobSpec.

From 500816b71b1ad3f916e11ff3ddb1fd6dee9fcc76 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Fri, 24 Apr 2026 16:44:46 +0300
Subject: [PATCH 094/151] refactor(ingress): move CiliumLoadBalancerIPPool to
 tenant chart
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The pool is now rendered from packages/apps/tenant/templates/cilium-lb-pool.yaml instead of packages/extra/ingress/templates/cilium-lb-pool.yaml. Cilium LB IPAM forbids overlapping CIDRs across pools regardless of serviceSelector, so both the ingress-loadBalancer path and the upcoming per-tenant Gateway in #2470 cannot each own their own pool on the same publishing.externalIPs range. The tenant chart is the natural per-tenant owner — it already creates the Namespace, the cozystack-values Secret, and the HelmReleases for both ingress and gateway.

The new pool uses a namespace-only serviceSelector (io.kubernetes.service.namespace: <ns>), which matches any LoadBalancer Service in the tenant namespace. The metadata.name changed from <trim>-ingress to <trim>-exposure to reflect that the pool is not ingress-specific.

Only the ingress-loadBalancer signal is wired in this commit (_cluster.expose-mode=loadBalancer plus .Values.ingress=true on the publishing tenant). The gateway branch is added in #2470 on top of this commit — it rebases, drops its own packages/extra/gateway/templates/cilium-lb-pool.yaml, and adds an OR branch for .Values.gateway in the tenant template.

Pool-rendering unit tests moved from packages/extra/ingress/tests/ to packages/apps/tenant/tests/. The ingress chart tests keep the Service-level asserts. packages/apps/tenant/Makefile gains a test target so hack/helm-unit-tests.sh picks up the new suite.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 packages/apps/tenant/Makefile                 |   3 +
 .../tenant}/templates/cilium-lb-pool.yaml     |  21 +--
 packages/apps/tenant/tests/exposure_test.yaml | 141 ++++++++++++++++++
 .../extra/ingress/tests/exposure_test.yaml    | 104 +------------
 4 files changed, 158 insertions(+), 111 deletions(-)
 rename packages/{extra/ingress => apps/tenant}/templates/cilium-lb-pool.yaml (54%)
 create mode 100644 packages/apps/tenant/tests/exposure_test.yaml

diff --git a/packages/apps/tenant/Makefile b/packages/apps/tenant/Makefile
index 3fe3810d..2f51d836 100644
--- a/packages/apps/tenant/Makefile
+++ b/packages/apps/tenant/Makefile
@@ -3,3 +3,6 @@ include ../../../hack/package.mk
 generate:
 	cozyvalues-gen -m 'tenant' -v values.yaml -s values.schema.json -r README.md -g ../../../api/apps/v1alpha1/tenant/types.go
 	../../../hack/update-crd.sh
+
+test:
+	helm unittest .
diff --git a/packages/extra/ingress/templates/cilium-lb-pool.yaml b/packages/apps/tenant/templates/cilium-lb-pool.yaml
similarity index 54%
rename from packages/extra/ingress/templates/cilium-lb-pool.yaml
rename to packages/apps/tenant/templates/cilium-lb-pool.yaml
index 048eb58b..63dbcaca 100644
--- a/packages/extra/ingress/templates/cilium-lb-pool.yaml
+++ b/packages/apps/tenant/templates/cilium-lb-pool.yaml
@@ -1,25 +1,28 @@
-{{- $exposeIngress := (index .Values._cluster "expose-ingress") | default "tenant-root" }}
 {{- $exposeMode := (index .Values._cluster "expose-mode") | default "externalIPs" }}
-{{- $exposeExternalIPs := (index .Values._cluster "expose-external-ips") | default "" | nospace }}
-{{- $exposeIPsList := list }}
-{{- range splitList "," $exposeExternalIPs }}
+{{- $exposeIngress := (index .Values._cluster "expose-ingress") | default "tenant-root" }}
+{{- $exposeIPs := (index .Values._cluster "expose-external-ips") | default "" | nospace }}
+{{- $ipsList := list }}
+{{- range splitList "," $exposeIPs }}
   {{- $ip := . | trim }}
   {{- if $ip }}
-    {{- $exposeIPsList = append $exposeIPsList $ip }}
+    {{- $ipsList = append $ipsList $ip }}
   {{- end }}
 {{- end }}
-{{- if and (eq $exposeMode "loadBalancer") (eq $exposeIngress .Release.Namespace) $exposeIPsList }}
+{{- $isPublishingIngressLB := and
+      (eq $exposeMode "loadBalancer")
+      (eq $exposeIngress .Release.Namespace)
+      .Values.ingress }}
+{{- if and $isPublishingIngressLB $ipsList }}
 apiVersion: cilium.io/v2
 kind: CiliumLoadBalancerIPPool
 metadata:
-  name: {{ trimPrefix "tenant-" .Release.Namespace }}-ingress
+  name: {{ trimPrefix "tenant-" .Release.Namespace }}-exposure
 spec:
   blocks:
-  {{- range $exposeIPsList }}
+  {{- range $ipsList }}
   - cidr: {{ . }}{{ if not (contains "/" .) }}/{{ if contains ":" . }}128{{ else }}32{{ end }}{{ end }}
   {{- end }}
   serviceSelector:
     matchLabels:
       "io.kubernetes.service.namespace": {{ .Release.Namespace | quote }}
-      "app.kubernetes.io/name": ingress-nginx
 {{- end }}
diff --git a/packages/apps/tenant/tests/exposure_test.yaml b/packages/apps/tenant/tests/exposure_test.yaml
new file mode 100644
index 00000000..e86fab5b
--- /dev/null
+++ b/packages/apps/tenant/tests/exposure_test.yaml
@@ -0,0 +1,141 @@
+suite: tenant CiliumLoadBalancerIPPool rendering for publishing.exposure=loadBalancer
+templates:
+  - templates/cilium-lb-pool.yaml
+
+release:
+  name: tenant-root
+  namespace: tenant-root
+
+tests:
+  - it: default exposure (externalIPs) renders no pool
+    set:
+      ingress: true
+      _cluster:
+        expose-ingress: tenant-root
+        expose-external-ips: "192.0.2.10,192.0.2.11"
+    asserts:
+      - hasDocuments:
+          count: 0
+
+  - it: loadBalancer mode in publishing tenant with ingress=true renders v2 pool with namespace-only selector
+    set:
+      ingress: true
+      _cluster:
+        expose-ingress: tenant-root
+        expose-external-ips: "192.0.2.10,192.0.2.11"
+        expose-mode: loadBalancer
+    asserts:
+      - hasDocuments:
+          count: 1
+      - equal:
+          path: apiVersion
+          value: cilium.io/v2
+      - equal:
+          path: kind
+          value: CiliumLoadBalancerIPPool
+      - equal:
+          path: metadata.name
+          value: root-exposure
+      - equal:
+          path: spec.blocks
+          value:
+            - cidr: 192.0.2.10/32
+            - cidr: 192.0.2.11/32
+      - equal:
+          path: spec.serviceSelector.matchLabels["io.kubernetes.service.namespace"]
+          value: tenant-root
+      - notExists:
+          path: spec.serviceSelector.matchLabels["app.kubernetes.io/name"]
+
+  - it: loadBalancer mode with IPv6 emits /128 CIDR
+    set:
+      ingress: true
+      _cluster:
+        expose-ingress: tenant-root
+        expose-external-ips: "2001:db8::1"
+        expose-mode: loadBalancer
+    asserts:
+      - equal:
+          path: spec.blocks
+          value:
+            - cidr: 2001:db8::1/128
+
+  - it: loadBalancer mode with mixed IPv4 and IPv6 emits correct CIDR per family
+    set:
+      ingress: true
+      _cluster:
+        expose-ingress: tenant-root
+        expose-external-ips: "192.0.2.10,2001:db8::1"
+        expose-mode: loadBalancer
+    asserts:
+      - equal:
+          path: spec.blocks
+          value:
+            - cidr: 192.0.2.10/32
+            - cidr: 2001:db8::1/128
+
+  - it: loadBalancer mode accepts pre-CIDR input without double-suffixing
+    set:
+      ingress: true
+      _cluster:
+        expose-ingress: tenant-root
+        expose-external-ips: "192.0.2.10/32,2001:db8::1/128"
+        expose-mode: loadBalancer
+    asserts:
+      - equal:
+          path: spec.blocks
+          value:
+            - cidr: 192.0.2.10/32
+            - cidr: 2001:db8::1/128
+
+  - it: loadBalancer mode filters out empty entries from externalIPs (trailing, leading, repeated commas)
+    set:
+      ingress: true
+      _cluster:
+        expose-ingress: tenant-root
+        expose-external-ips: "192.0.2.10,,192.0.2.11,"
+        expose-mode: loadBalancer
+    asserts:
+      - hasDocuments:
+          count: 1
+      - equal:
+          path: spec.blocks
+          value:
+            - cidr: 192.0.2.10/32
+            - cidr: 192.0.2.11/32
+
+  - it: loadBalancer mode with ingress=false in publishing tenant renders no pool
+    set:
+      ingress: false
+      _cluster:
+        expose-ingress: tenant-root
+        expose-external-ips: "192.0.2.10"
+        expose-mode: loadBalancer
+    asserts:
+      - hasDocuments:
+          count: 0
+
+  - it: loadBalancer mode in a non-publishing tenant renders no pool
+    set:
+      ingress: true
+      _cluster:
+        expose-ingress: tenant-root
+        expose-external-ips: "192.0.2.10"
+        expose-mode: loadBalancer
+    release:
+      name: tenant-u1
+      namespace: tenant-u1
+    asserts:
+      - hasDocuments:
+          count: 0
+
+  - it: loadBalancer mode in publishing tenant with empty externalIPs renders no pool
+    set:
+      ingress: true
+      _cluster:
+        expose-ingress: tenant-root
+        expose-external-ips: ""
+        expose-mode: loadBalancer
+    asserts:
+      - hasDocuments:
+          count: 0
diff --git a/packages/extra/ingress/tests/exposure_test.yaml b/packages/extra/ingress/tests/exposure_test.yaml
index 323c8e68..1c76b3da 100644
--- a/packages/extra/ingress/tests/exposure_test.yaml
+++ b/packages/extra/ingress/tests/exposure_test.yaml
@@ -1,14 +1,13 @@
 suite: ingress exposure modes
 templates:
   - templates/nginx-ingress.yaml
-  - templates/cilium-lb-pool.yaml
 
 release:
   name: ingress
   namespace: tenant-root
 
 tests:
-  - it: default exposure (externalIPs) renders ClusterIP Service with spec.externalIPs and no CiliumLoadBalancerIPPool
+  - it: default exposure (externalIPs) renders ClusterIP Service with spec.externalIPs
     set:
       _cluster:
         expose-ingress: tenant-root
@@ -31,9 +30,6 @@ tests:
       - template: templates/nginx-ingress.yaml
         notExists:
           path: spec.values.ingress-nginx.controller.service.labels
-      - template: templates/cilium-lb-pool.yaml
-        hasDocuments:
-          count: 0
 
   - it: legacy config without expose-mode falls back to externalIPs behavior
     set:
@@ -45,9 +41,6 @@ tests:
         equal:
           path: spec.values.ingress-nginx.controller.service.type
           value: ClusterIP
-      - template: templates/cilium-lb-pool.yaml
-        hasDocuments:
-          count: 0
 
   - it: externalIPs mode in a namespace other than publishing.ingressName renders LoadBalancer fallback without externalIPs
     set:
@@ -68,11 +61,8 @@ tests:
       - template: templates/nginx-ingress.yaml
         notExists:
           path: spec.values.ingress-nginx.controller.service.externalIPs
-      - template: templates/cilium-lb-pool.yaml
-        hasDocuments:
-          count: 0
 
-  - it: loadBalancer mode renders LoadBalancer Service with lb-pool label and a v2 CiliumLoadBalancerIPPool
+  - it: loadBalancer mode renders LoadBalancer Service without externalIPs on the Service
     set:
       _cluster:
         expose-ingress: tenant-root
@@ -93,62 +83,6 @@ tests:
       - template: templates/nginx-ingress.yaml
         notExists:
           path: spec.values.ingress-nginx.controller.service.externalIPs
-      - template: templates/cilium-lb-pool.yaml
-        hasDocuments:
-          count: 1
-      - template: templates/cilium-lb-pool.yaml
-        equal:
-          path: apiVersion
-          value: cilium.io/v2
-      - template: templates/cilium-lb-pool.yaml
-        equal:
-          path: kind
-          value: CiliumLoadBalancerIPPool
-      - template: templates/cilium-lb-pool.yaml
-        equal:
-          path: metadata.name
-          value: root-ingress
-      - template: templates/cilium-lb-pool.yaml
-        equal:
-          path: spec.blocks
-          value:
-            - cidr: 192.0.2.10/32
-            - cidr: 192.0.2.11/32
-      - template: templates/cilium-lb-pool.yaml
-        equal:
-          path: spec.serviceSelector.matchLabels["io.kubernetes.service.namespace"]
-          value: tenant-root
-      - template: templates/cilium-lb-pool.yaml
-        equal:
-          path: spec.serviceSelector.matchLabels["app.kubernetes.io/name"]
-          value: ingress-nginx
-
-  - it: loadBalancer mode with IPv6 address emits /128 CIDR
-    set:
-      _cluster:
-        expose-ingress: tenant-root
-        expose-external-ips: "2001:db8::1"
-        expose-mode: loadBalancer
-    asserts:
-      - template: templates/cilium-lb-pool.yaml
-        equal:
-          path: spec.blocks
-          value:
-            - cidr: 2001:db8::1/128
-
-  - it: loadBalancer mode with mixed IPv4 and IPv6 emits correct CIDR per family
-    set:
-      _cluster:
-        expose-ingress: tenant-root
-        expose-external-ips: "192.0.2.10,2001:db8::1"
-        expose-mode: loadBalancer
-    asserts:
-      - template: templates/cilium-lb-pool.yaml
-        equal:
-          path: spec.blocks
-          value:
-            - cidr: 192.0.2.10/32
-            - cidr: 2001:db8::1/128
 
   - it: loadBalancer mode without externalIPs fails chart render with explicit message
     set:
@@ -206,26 +140,6 @@ tests:
       - template: templates/nginx-ingress.yaml
         notExists:
           path: spec.values.ingress-nginx.controller.service.externalIPs
-      - template: templates/cilium-lb-pool.yaml
-        hasDocuments:
-          count: 0
-
-  - it: loadBalancer mode filters out empty entries from externalIPs (trailing comma, leading comma)
-    set:
-      _cluster:
-        expose-ingress: tenant-root
-        expose-external-ips: "192.0.2.10,,192.0.2.11,"
-        expose-mode: loadBalancer
-    asserts:
-      - template: templates/cilium-lb-pool.yaml
-        hasDocuments:
-          count: 1
-      - template: templates/cilium-lb-pool.yaml
-        equal:
-          path: spec.blocks
-          value:
-            - cidr: 192.0.2.10/32
-            - cidr: 192.0.2.11/32
 
   - it: loadBalancer mode with only-empty externalIPs fails chart render (comma-only input)
     set:
@@ -249,17 +163,3 @@ tests:
           path: spec.values.ingress-nginx.controller.service.externalIPs
           value:
             - 192.0.2.10
-
-  - it: loadBalancer mode accepts pre-CIDR input without double-suffixing
-    set:
-      _cluster:
-        expose-ingress: tenant-root
-        expose-external-ips: "192.0.2.10/32,2001:db8::1/128"
-        expose-mode: loadBalancer
-    asserts:
-      - template: templates/cilium-lb-pool.yaml
-        equal:
-          path: spec.blocks
-          value:
-            - cidr: 192.0.2.10/32
-            - cidr: 2001:db8::1/128

From c8ed1c652cbd3d442036251358c68e7b25ec64a1 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Mon, 27 Apr 2026 02:35:52 +0300
Subject: [PATCH 097/151] chore(ci): adopt CNCF/k8s label conventions
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add .github/labels.yml as the canonical label set, synced into the
repository by .github/workflows/labels.yaml using EndBug/label-sync.

Conventions follow the Kubernetes scheme:
https://github.com/kubernetes/test-infra/blob/master/label_sync/labels.md

Six namespaced groups: kind/, priority/, triage/, lifecycle/, area/,
do-not-merge/. Cozystack-specific labels preserved (epic, community,
security/*, size:*).

Migration via aliases keeps references on existing issues and PRs:
- bug           -> kind/bug
- enhancement   -> kind/feature
- documentation -> kind/documentation
- question      -> kind/support
- frozen        -> lifecycle/frozen
- stale         -> lifecycle/stale
- do-not-merge  -> do-not-merge/work-in-progress

delete-other-labels is false on the initial rollout; redundant labels
("do not merge", duplicate, invalid, wontfix) stay until a follow-up
PR removes them after stabilisation.

The labels workflow has a validate job (python3 schema check) that
runs on PR. Sync runs only on push to main, weekly cron, and manual
dispatch. Schema invariants:
- description <= 100 chars (GitHub REST API limit)
- color is 6-char hex without leading #
- unique top-level names
- aliases do not collide with top-level names

PR auto-labeling (.github/workflows/pr-labeler.yaml):
- Parses PR title as Conventional Commits header (type, scope, !).
- type -> kind/* (feat -> kind/feature, fix -> kind/bug, docs ->
  kind/documentation, chore/refactor -> kind/cleanup; style, perf,
  test, build, ci, revert -> no kind label).
- scope -> area/* via embedded mapping; composite scopes split on
  comma. Bracket-style fallback ([scope] description) maps area/*
  but cannot infer kind/*.
- '[Backport release-1.x]' prefix is stripped; area/release and
  backport labels are added.
- '!' after type or 'BREAKING CHANGE:' footer in body adds
  kind/breaking-change.
- Unmapped scope or non-conventional title adds area/uncategorized
  to flag for human review.
- Additive only — never removes existing labels.

Hardcoded label references updated:
- .github/ISSUE_TEMPLATE/bug_report.md (bug -> kind/bug)
- .github/workflows/tags.yaml (documentation -> kind/documentation)

AGENTS.md gains an Activation entry pointing agents to labels.yml
as the source of truth and to contributing.md for the title
auto-labeling table.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 .github/ISSUE_TEMPLATE/bug_report.md |   2 +-
 .github/labels.yml                   | 371 +++++++++++++++++++++++++++
 .github/workflows/labels.yaml        |  84 ++++++
 .github/workflows/pr-labeler.yaml    | 199 ++++++++++++++
 .github/workflows/tags.yaml          |   6 +-
 AGENTS.md                            |   6 +
 docs/agents/contributing.md          |  43 ++++
 7 files changed, 707 insertions(+), 4 deletions(-)
 create mode 100644 .github/labels.yml
 create mode 100644 .github/workflows/labels.yaml
 create mode 100644 .github/workflows/pr-labeler.yaml

diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
index 1a119259..fefdab82 100644
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -1,7 +1,7 @@
 ---
 name: Bug report
 about: Create a report to help us improve
-labels: 'bug'
+labels: 'kind/bug'
 assignees: ''
 
 ---
diff --git a/.github/labels.yml b/.github/labels.yml
new file mode 100644
index 00000000..1369aa48
--- /dev/null
+++ b/.github/labels.yml
@@ -0,0 +1,371 @@
+# Cozystack repository labels
+#
+# Label conventions follow the Kubernetes scheme:
+# https://github.com/kubernetes/test-infra/blob/master/label_sync/labels.md
+#
+# Synced into the repository by .github/workflows/labels.yaml
+# (EndBug/label-sync@v2). Edit this file via pull request — UI changes
+# will be overwritten on the next sync.
+#
+# Constraints (enforced by the validate job in labels.yaml):
+#   - description ≤ 100 characters (GitHub REST API limit)
+#   - color is a 6-character hex string (no leading #)
+#   - label names are unique
+#   - aliases do not collide with top-level names
+#
+# Categories:
+#   kind/         issue or PR type
+#   priority/     urgency
+#   triage/       review state
+#   lifecycle/    issue or PR lifecycle
+#   area/         subsystem; extensible — add when 3+ open issues exist
+#   do-not-merge/ PR merge blockers
+#   security/     security-finding severity and status (Cozystack-specific)
+#   size:         PR size (auto-applied)
+#
+# `aliases:` lets EndBug/label-sync rename existing labels without losing
+# references on already-tagged issues and PRs.
+#
+# GitHub-default labels not migrated here (`wontfix`, `invalid`) currently
+# carry zero issues/PRs in this repo and will be removed in a follow-up
+# cleanup PR rather than aliased to a different namespace.
+
+# ──────────────────────────────────────────────
+# kind/ — issue or PR type
+# ──────────────────────────────────────────────
+
+- name: kind/bug
+  color: 'd73a4a'
+  description: Categorizes issue or PR as related to a bug
+  aliases: ['bug']
+
+- name: kind/feature
+  color: 'a2eeef'
+  description: Categorizes issue or PR as related to a new feature
+  aliases: ['enhancement']
+
+- name: kind/documentation
+  color: '0075ca'
+  description: Categorizes issue or PR as related to documentation
+  aliases: ['documentation']
+
+- name: kind/support
+  color: 'd876e3'
+  description: Categorizes issue as a support question
+  aliases: ['question']
+
+- name: kind/cleanup
+  color: 'c7def8'
+  description: Categorizes issue or PR as related to cleanup of code, process, or technical debt
+
+- name: kind/regression
+  color: 'e11d21'
+  description: Categorizes issue or PR as related to a regression from a prior release
+
+- name: kind/flake
+  color: 'f7c6c7'
+  description: Categorizes issue or PR as related to a flaky test
+
+- name: kind/failing-test
+  color: 'e11d21'
+  description: Categorizes issue or PR as related to a consistently or frequently failing test
+
+- name: kind/api-change
+  color: 'c7def8'
+  description: Categorizes issue or PR as related to adding, removing, or otherwise changing an API
+
+- name: kind/breaking-change
+  color: 'e11d21'
+  description: Indicates the change introduces a breaking API or behaviour change
+
+# ──────────────────────────────────────────────
+# priority/ — urgency
+# ──────────────────────────────────────────────
+
+- name: priority/critical-urgent
+  color: 'e11d21'
+  description: Highest priority. Must be actively worked on as someone's top priority right now
+
+- name: priority/important-soon
+  color: 'eb6420'
+  description: Must be staffed and worked on either currently, or very soon, ideally in time for the next release
+
+- name: priority/important-longterm
+  color: 'fbca04'
+  description: Important over the long term, but may not be staffed and/or may need multiple releases to complete
+
+- name: priority/backlog
+  color: 'fef2c0'
+  description: General backlog priority. Lower than priority/important-longterm
+
+# ──────────────────────────────────────────────
+# triage/ — review state
+# ──────────────────────────────────────────────
+
+- name: triage/needs-triage
+  color: 'ededed'
+  description: Indicates an issue needs triage by a maintainer
+
+- name: triage/accepted
+  color: '0e8a16'
+  description: Indicates an issue is ready to be actively worked on
+
+- name: triage/needs-information
+  color: 'fbca04'
+  description: Indicates an issue needs more information in order to work on it
+
+- name: triage/not-reproducible
+  color: 'fbca04'
+  description: Indicates an issue can not be reproduced as described
+
+- name: triage/duplicate
+  color: 'cfd3d7'
+  description: Indicates an issue is a duplicate of another issue
+  aliases: ['duplicate']
+
+- name: triage/unresolved
+  color: 'cfd3d7'
+  description: Indicates an issue that can not or will not be resolved
+
+# ──────────────────────────────────────────────
+# lifecycle/ — issue or PR lifecycle
+# ──────────────────────────────────────────────
+
+- name: lifecycle/active
+  color: '1d76db'
+  description: Indicates that an issue or PR is actively being worked on by a contributor
+
+- name: lifecycle/frozen
+  color: 'db5dd6'
+  description: Indicates that an issue or PR should not be auto-closed due to staleness
+  aliases: ['frozen']
+
+- name: lifecycle/stale
+  color: 'dadada'
+  description: Denotes an issue or PR has remained open with no activity and has become stale
+  aliases: ['stale']
+
+- name: lifecycle/rotten
+  color: '795548'
+  description: Denotes an issue or PR that has aged beyond stale and will be auto-closed
+
+# ──────────────────────────────────────────────
+# area/ — subsystem (extensible)
+# Add a new area/* when there are 3+ open issues on the topic.
+# ──────────────────────────────────────────────
+
+- name: area/api
+  color: 'bfd4f2'
+  description: Issues or PRs related to the cozystack-api aggregated API server
+
+- name: area/ai
+  color: 'bfd4f2'
+  description: Issues or PRs related to AI agent guides, AGENTS.md, docs/agents/
+
+- name: area/build
+  color: 'bfd4f2'
+  description: Issues or PRs related to image build infrastructure, multi-arch support
+
+- name: area/ci
+  color: 'bfd4f2'
+  description: Issues or PRs related to CI workflows, GitHub Actions, automation
+
+- name: area/dashboard
+  color: 'bfd4f2'
+  description: Issues or PRs related to the dashboard / UI
+
+- name: area/extra
+  color: 'bfd4f2'
+  description: Issues or PRs related to tenant-specific modules (packages/extra/)
+
+- name: area/database
+  color: 'bfd4f2'
+  description: Issues or PRs related to managed databases (postgres, mariadb, redis, etcd, kafka, clickhouse)
+
+- name: area/kubernetes
+  color: 'bfd4f2'
+  description: Issues or PRs related to the tenant Kubernetes app
+
+- name: area/monitoring
+  color: 'bfd4f2'
+  description: Issues or PRs related to the monitoring stack (vlogs, vmstack, grafana, workloadmonitor)
+
+- name: area/networking
+  color: 'bfd4f2'
+  description: Issues or PRs related to networking (ingress, gateway, vpn, metallb, cilium, kube-ovn)
+
+- name: area/platform
+  color: 'bfd4f2'
+  description: Issues or PRs related to platform infrastructure (bundle, flux, talos, installer)
+
+- name: area/release
+  color: 'bfd4f2'
+  description: Issues or PRs related to release tooling (changelog, backport, release pipeline)
+
+- name: area/storage
+  color: 'bfd4f2'
+  description: Issues or PRs related to storage (linstor, seaweedfs, bucket, velero, harbor)
+
+- name: area/testing
+  color: 'bfd4f2'
+  description: Issues or PRs related to testing (e2e, bats, unit tests)
+
+- name: area/virtualization
+  color: 'bfd4f2'
+  description: Issues or PRs related to virtualization (kubevirt, cdi, vmi, vm-import)
+
+- name: area/uncategorized
+  color: 'fbca04'
+  description: PR auto-labeler could not map title scope to a known area/*; please review
+
+# ──────────────────────────────────────────────
+# do-not-merge/ — PR merge blockers (Prow convention)
+# ──────────────────────────────────────────────
+
+- name: do-not-merge/work-in-progress
+  color: 'e11d21'
+  description: Indicates that a PR should not merge because it is a work in progress
+  # Both legacy spellings collapse here. EndBug processes aliases sequentially;
+  # the second rename hits a name collision and logs a warning — the legacy
+  # label survives and gets cleaned up in the follow-up dedup PR.
+  aliases: ['do-not-merge', 'do not merge']
+
+- name: do-not-merge/hold
+  color: 'e11d21'
+  description: Indicates that a PR should not merge because someone has issued /hold
+
+# ──────────────────────────────────────────────
+# Cozystack-specific (preserved)
+# ──────────────────────────────────────────────
+
+- name: epic
+  color: 'A335EE'
+  description: A large development increment that brings definite value to Cozystack users
+
+- name: community
+  color: '97458A'
+  description: Community contributions are welcome in this issue
+
+- name: help wanted
+  color: '008672'
+  description: Extra attention is needed
+
+- name: good first issue
+  color: '7057ff'
+  description: Good for newcomers
+
+- name: quality-of-life
+  color: 'aaaaaa'
+  description: QoL improvements
+
+- name: upstream-issue
+  color: 'aaaaaa'
+  description: Requires resolving an issue in an upstream project
+
+- name: backport
+  color: 'FBCA04'
+  description: Should change be backported on previous release
+
+- name: backport-previous
+  color: 'fbd876'
+  description: Backport target — previous release line
+
+- name: release
+  color: 'aaaaaa'
+  description: Releasing a new Cozystack version
+
+- name: automated
+  color: 'ededed'
+  description: Created by automation
+
+- name: debug
+  color: '704479'
+  description: Debugging in progress
+
+- name: sponsored
+  color: '00FF00'
+  description: Sponsored work
+
+- name: lgtm
+  color: '238636'
+  description: This PR has been approved by a maintainer
+
+- name: ok-to-test
+  color: '00FF00'
+  description: Indicates a non-member PR is safe to run CI on
+
+# ──────────────────────────────────────────────
+# size: — PR size (auto-applied by sizing bot)
+# ──────────────────────────────────────────────
+
+- name: 'size:XS'
+  color: '00ff00'
+  description: This PR changes 0-9 lines, ignoring generated files
+
+- name: 'size:S'
+  color: '77b800'
+  description: This PR changes 10-29 lines, ignoring generated files
+
+- name: 'size:M'
+  color: 'ebb800'
+  description: This PR changes 30-99 lines, ignoring generated files
+
+- name: 'size:L'
+  color: 'eb9500'
+  description: This PR changes 100-499 lines, ignoring generated files
+
+- name: 'size:XL'
+  color: 'ff823f'
+  description: This PR changes 500-999 lines, ignoring generated files
+
+- name: 'size:XXL'
+  color: 'ffb8b8'
+  description: This PR changes 1000+ lines, ignoring generated files
+
+# ──────────────────────────────────────────────
+# security/ — security-finding severity and status
+# ──────────────────────────────────────────────
+
+- name: security
+  color: 'aaaaaa'
+  description: Security-related issues and features
+
+- name: security/critical
+  color: 'd73a4a'
+  description: Critical security vulnerability
+
+- name: security/high
+  color: 'e99695'
+  description: High severity security finding
+
+- name: security/medium
+  color: 'f9c513'
+  description: Medium severity security finding
+
+- name: security/low
+  color: '0e8a16'
+  description: Low severity security finding
+
+- name: security/triage-needed
+  color: 'fbca04'
+  description: Needs security triage
+
+- name: security/confirmed
+  color: '1d76db'
+  description: Confirmed vulnerability
+
+- name: security/false-positive
+  color: 'c5def5'
+  description: Triaged as false positive
+
+- name: security/accepted-risk
+  color: 'bfd4f2'
+  description: Risk accepted with justification
+
+- name: security/in-progress
+  color: '0075ca'
+  description: Fix in progress
+
+- name: security/fixed
+  color: '0e8a16'
+  description: Fix released
diff --git a/.github/workflows/labels.yaml b/.github/workflows/labels.yaml
new file mode 100644
index 00000000..69c18329
--- /dev/null
+++ b/.github/workflows/labels.yaml
@@ -0,0 +1,84 @@
+name: Labels
+
+on:
+  pull_request:
+    paths:
+      - .github/labels.yml
+      - .github/workflows/labels.yaml
+  push:
+    branches: [main]
+    paths:
+      - .github/labels.yml
+      - .github/workflows/labels.yaml
+  workflow_dispatch:
+  schedule:
+    - cron: '17 4 * * 1'  # Mondays at 04:17 UTC
+
+permissions:
+  contents: read
+
+concurrency:
+  group: labels-sync
+  cancel-in-progress: false
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Validate labels.yml schema
+        run: |
+          python3 - <<'PY'
+          import re, sys, yaml
+
+          path = '.github/labels.yml'
+          data = yaml.safe_load(open(path))
+          errors = []
+
+          # 1. description ≤ 100 chars (GitHub REST API limit)
+          for label in data:
+              desc = label.get('description', '') or ''
+              if len(desc) > 100:
+                  errors.append(f"{label['name']}: description {len(desc)} chars (max 100)")
+
+          # 2. color is 6-char hex without leading #
+          for label in data:
+              color = label.get('color', '') or ''
+              if not re.match(r'^[0-9A-Fa-f]{6}$', color):
+                  errors.append(f"{label['name']}: bad color {color!r} (must be 6-char hex without #)")
+
+          # 3. unique top-level names
+          names = [label['name'] for label in data]
+          dups = sorted({n for n in names if names.count(n) > 1})
+          for n in dups:
+              errors.append(f"duplicate name: {n}")
+
+          # 4. aliases do not collide with any top-level name
+          name_set = set(names)
+          for label in data:
+              for alias in (label.get('aliases') or []):
+                  if alias in name_set:
+                      errors.append(f"alias {alias!r} (under {label['name']!r}) collides with a top-level name")
+
+          if errors:
+              for err in errors:
+                  print(f"::error::{err}")
+              sys.exit(1)
+
+          print(f"labels.yml schema OK ({len(data)} labels)")
+          PY
+
+  sync:
+    needs: validate
+    if: github.event_name != 'pull_request'
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: EndBug/label-sync@v2
+        with:
+          config-file: .github/labels.yml
+          delete-other-labels: false
diff --git a/.github/workflows/pr-labeler.yaml b/.github/workflows/pr-labeler.yaml
new file mode 100644
index 00000000..d882bc50
--- /dev/null
+++ b/.github/workflows/pr-labeler.yaml
@@ -0,0 +1,199 @@
+name: PR Auto-Label
+
+on:
+  pull_request_target:
+    types: [opened, edited, reopened, synchronize]
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  label:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Apply labels from PR title
+        uses: actions/github-script@v7
+        with:
+          script: |
+            // Conventional Commits types accepted by Cozystack (per docs/agents/contributing.md):
+            //   feat, fix, docs, style, refactor, perf, test, build, ci, chore, revert
+            // Mapping below maps a subset to kind/* — types not listed do not produce a kind/*.
+            const typeToKind = {
+              feat:     'kind/feature',
+              fix:      'kind/bug',
+              docs:     'kind/documentation',
+              chore:    'kind/cleanup',
+              refactor: 'kind/cleanup',
+              // style, perf, test, build, ci, revert — no kind mapping
+            };
+
+            // scope -> area/* mapping. Keys are the scopes observed in cozystack issues
+            // and PRs. Add new entries when a scope recurs (3+ times).
+            const scopeToArea = {
+              // area/api
+              'api':                 'area/api',
+              'cozystack-api':       'area/api',
+
+              // area/ai
+              'agents':              'area/ai',
+              'ai':                  'area/ai',
+
+              // area/build
+              'build':               'area/build',
+
+              // area/ci
+              'ci':                  'area/ci',
+
+              // area/dashboard
+              'dashboard':           'area/dashboard',
+
+              // area/database
+              'postgres':            'area/database',
+              'postgres-operator':   'area/database',
+              'mariadb':             'area/database',
+              'mariadb-operator':    'area/database',
+              'redis':               'area/database',
+              'etcd':                'area/database',
+              'kafka':               'area/database',
+              'clickhouse':          'area/database',
+
+              // area/extra
+              'extra':               'area/extra',
+
+              // area/kubernetes
+              'kubernetes':          'area/kubernetes',
+              'apps/kubernetes':     'area/kubernetes',
+
+              // area/monitoring
+              'monitoring':          'area/monitoring',
+              'vlogs':               'area/monitoring',
+              'vmstack':             'area/monitoring',
+              'grafana':             'area/monitoring',
+              'workloadmonitor':     'area/monitoring',
+
+              // area/networking
+              'ingress':             'area/networking',
+              'ingress-nginx':       'area/networking',
+              'gateway':             'area/networking',
+              'vpn':                 'area/networking',
+              'metallb':             'area/networking',
+              'cilium':              'area/networking',
+              'kube-ovn':            'area/networking',
+              'tcp-balancer':        'area/networking',
+              'securitygroups':      'area/networking',
+              'cozy-proxy':          'area/networking',
+
+              // area/platform
+              'platform':            'area/platform',
+              'bundle':              'area/platform',
+              'flux':                'area/platform',
+              'fluxcd':              'area/platform',
+              'cluster-api':         'area/platform',
+              'talos':               'area/platform',
+              'installer':           'area/platform',
+              'cozyctl':             'area/platform',
+              'cozystack-engine':    'area/platform',
+              'cozy-lib':            'area/platform',
+
+              // area/release
+              'backport':            'area/release',
+              'release':             'area/release',
+
+              // area/storage
+              'seaweedfs':           'area/storage',
+              'seaweedfs-cosi-driver': 'area/storage',
+              'bucket':              'area/storage',
+              'linstor':             'area/storage',
+              'velero':              'area/storage',
+              'harbor':              'area/storage',
+              'backups':             'area/storage',
+
+              // area/testing
+              'tests':               'area/testing',
+              'e2e':                 'area/testing',
+
+              // area/virtualization
+              'kubevirt':            'area/virtualization',
+              'cdi':                 'area/virtualization',
+              'vmi':                 'area/virtualization',
+              'vm-import':           'area/virtualization',
+              'virtual-machine':     'area/virtualization',
+              'hami':                'area/virtualization',
+              'gpu-operator':        'area/virtualization',
+            };
+
+            const pr = context.payload.pull_request;
+            const title = pr.title || '';
+            const body  = pr.body  || '';
+            const existing = new Set(pr.labels.map(l => l.name));
+            const toAdd = new Set();
+
+            // 1. Strip "[Backport release-1.x]" prefix if present.
+            const backportMatch = title.match(/^\[Backport ([^\]]+)\]\s+(.+)$/);
+            const cleanTitle = backportMatch ? backportMatch[2] : title;
+            if (backportMatch) {
+              toAdd.add('area/release');
+              toAdd.add('backport');
+            }
+
+            // 2. Try Conventional Commits form: type(scope)?(!)?: description
+            const conv = cleanTitle.match(/^([a-z]+)(?:\(([^)]+)\))?(!)?:\s*.+$/);
+            // 3. Fall back to bracket form: [scope] description
+            const bracket = !conv && cleanTitle.match(/^\[([^\]]+)\]\s+.+$/);
+
+            let type = null, scopeStr = null, breaking = false;
+            if (conv) {
+              type = conv[1];
+              scopeStr = conv[2] || null;
+              breaking = !!conv[3];
+            } else if (bracket) {
+              scopeStr = bracket[1];
+            }
+
+            // 4. Detect BREAKING CHANGE: footer in body.
+            if (/^BREAKING CHANGE:/m.test(body)) {
+              breaking = true;
+            }
+
+            // 5. Apply kind/* from type.
+            if (type && typeToKind[type]) {
+              toAdd.add(typeToKind[type]);
+            }
+
+            // 6. Apply area/* from scope. Composite scopes split on comma.
+            const scopes = (scopeStr || '')
+              .split(/,\s*/)
+              .map(s => s.trim())
+              .filter(Boolean);
+            for (const s of scopes) {
+              if (scopeToArea[s]) {
+                toAdd.add(scopeToArea[s]);
+              }
+            }
+
+            // 7. kind/breaking-change.
+            if (breaking) {
+              toAdd.add('kind/breaking-change');
+            }
+
+            // 8. Fallback: no area/* applied -> area/uncategorized.
+            const hasArea = [...toAdd].some(l => l.startsWith('area/'));
+            if (!hasArea) {
+              toAdd.add('area/uncategorized');
+            }
+
+            // 9. Additive only — never remove existing labels.
+            const newLabels = [...toAdd].filter(l => !existing.has(l));
+            if (newLabels.length === 0) {
+              core.info('No new labels to apply');
+              return;
+            }
+
+            await github.rest.issues.addLabels({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: pr.number,
+              labels: newLabels,
+            });
+            core.info(`Applied labels: ${newLabels.join(', ')}`);
diff --git a/.github/workflows/tags.yaml b/.github/workflows/tags.yaml
index 90f04829..1b68ee32 100644
--- a/.github/workflows/tags.yaml
+++ b/.github/workflows/tags.yaml
@@ -223,7 +223,7 @@ jobs:
                 repo:  context.repo.repo,
                 head,
                 base,
-                title: `Release v${version}`,
+                title: `chore(release): cut v${version}`,
                 body:  `This PR prepares the release \`v${version}\`.`,
                 draft: false
               });
@@ -411,7 +411,7 @@ jobs:
                 repo:  context.repo.repo,
                 head:  changelogBranch,
                 base:  baseBranch,
-                title: `docs: add changelog for v${version}`,
+                title: `docs(release): add changelog for v${version}`,
                 body:  `This PR adds the changelog for release \`v${version}\`.\n\n✅ Changelog has been automatically generated in \`docs/changelogs/v${version}.md\`.`,
                 draft: false
               });
@@ -421,7 +421,7 @@ jobs:
                 owner: context.repo.owner,
                 repo:  context.repo.repo,
                 issue_number: pr.data.number,
-                labels: ['documentation', 'automated']
+                labels: ['kind/documentation', 'automated']
               });
               
               console.log(`Created PR #${pr.data.number} for changelog`);
diff --git a/AGENTS.md b/AGENTS.md
index fe9aa8e1..5298f6fd 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -27,6 +27,12 @@ working with the **Cozystack** project.
   - Read: [`contributing.md`](./docs/agents/contributing.md)
   - Action: Read the file to understand git workflow, commit format, PR process
 
+- **Issue and PR labeling, triage** (e.g., "label this issue", "what label should I use", "triage this", "categorize")
+  - Read: [`.github/labels.yml`](./.github/labels.yml)
+  - Action: Use labels defined there. Conventions follow the Kubernetes scheme — `kind/*` (type), `area/*` (subsystem), `priority/*` (urgency), `triage/*` (review state), `lifecycle/*` (auto-close), `do-not-merge/*` (PR blockers), `security/*` (severity)
+  - For `area/*`: accuracy outweighs reuse. If no existing `area/*` truly fits the change, propose a new one via PR (extend `labels.yml` and the scope mapping in `pr-labeler.yaml`) — do not shoehorn the change into a wrong area. `area/uncategorized` is the auto-labeler fallback; treat it as a signal to pick a fit, create a new area, or correct the PR title
+  - PR titles: a Conventional Commits header (`type(scope): description`, types from [`contributing.md`](./docs/agents/contributing.md)) auto-applies `kind/*` and `area/*` via `.github/workflows/pr-labeler.yaml`. Append `!` (or add a `BREAKING CHANGE:` footer) to apply `kind/breaking-change`
+
 **Important rules:**
 - ✅ **ONLY read the file if the task matches the documented process scope** - do not read files for tasks that don't match their purpose
 - ✅ **ALWAYS read the file FIRST** before starting the task (when applicable)
diff --git a/docs/agents/contributing.md b/docs/agents/contributing.md
index 658dab74..6ff6eef1 100644
--- a/docs/agents/contributing.md
+++ b/docs/agents/contributing.md
@@ -57,6 +57,49 @@ git commit --signoff -m "fix(postgres): update operator to version 1.2.3"
 git commit --signoff -m "docs(contributing): add installation guide"
 ```
 
+## PR Title Auto-Labeling
+
+`.github/workflows/pr-labeler.yaml` parses the PR title on `opened`, `edited`, `reopened`, and `synchronize` events and applies labels additively (never removes). The title is expected to follow Conventional Commits — same format as commit messages above.
+
+**Type → `kind/*`:**
+
+| type      | label              |
+| --------- | ------------------ |
+| feat      | kind/feature       |
+| fix       | kind/bug           |
+| docs      | kind/documentation |
+| chore     | kind/cleanup       |
+| refactor  | kind/cleanup       |
+| style, perf, test, build, ci, revert | (no kind label)|
+
+**Scope → `area/*`** (full mapping in `.github/workflows/pr-labeler.yaml`):
+
+| scope (examples) | label |
+| --- | --- |
+| agents, ai | area/ai |
+| api, cozystack-api | area/api |
+| build | area/build |
+| ci | area/ci |
+| dashboard | area/dashboard |
+| postgres, mariadb, redis, etcd, kafka, clickhouse, postgres-operator, mariadb-operator | area/database |
+| extra | area/extra |
+| kubernetes | area/kubernetes |
+| monitoring, vlogs, vmstack, grafana, workloadmonitor | area/monitoring |
+| ingress, gateway, vpn, metallb, cilium, kube-ovn, cozy-proxy, … | area/networking |
+| platform, bundle, flux, fluxcd, cluster-api, talos, installer, cozyctl, cozystack-engine, cozy-lib | area/platform |
+| backport, release | area/release |
+| seaweedfs, bucket, linstor, velero, harbor, backups | area/storage |
+| tests, e2e | area/testing |
+| kubevirt, cdi, vmi, vm-import, virtual-machine, hami, gpu-operator | area/virtualization |
+
+**Special handling:**
+
+- `[Backport release-1.x]` prefix is stripped before parsing; `area/release` and `backport` labels are added.
+- Composite scope (`feat(platform, system, apps): …`) — each comma-separated part is mapped independently.
+- `!` after type or `BREAKING CHANGE:` footer in the body → `kind/breaking-change`.
+- Unmapped scope or non-conventional title → `area/uncategorized` (signals the PR needs manual area selection).
+- Bracket-style fallback (`[scope] description`) maps `scope` → `area/*` but cannot infer `kind/*`.
+
 ### AI Agent Attribution
 
 When an AI agent authors or materially assists with a commit, add an `Assisted-By:` trailer naming the model:

From 91188702a678c445bf42dafcbc3b697b1377b086 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Mon, 27 Apr 2026 03:30:09 +0300
Subject: [PATCH 098/151] chore(ci): normalize hex color case in labels.yml

Address review feedback from gemini-code-assist on .github/labels.yml:242:
all hex color values use lowercase characters for consistency.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 .github/labels.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/labels.yml b/.github/labels.yml
index 1369aa48..2ce04130 100644
--- a/.github/labels.yml
+++ b/.github/labels.yml
@@ -239,11 +239,11 @@
 # ──────────────────────────────────────────────
 
 - name: epic
-  color: 'A335EE'
+  color: 'a335ee'
   description: A large development increment that brings definite value to Cozystack users
 
 - name: community
-  color: '97458A'
+  color: '97458a'
   description: Community contributions are welcome in this issue
 
 - name: help wanted
@@ -263,7 +263,7 @@
   description: Requires resolving an issue in an upstream project
 
 - name: backport
-  color: 'FBCA04'
+  color: 'fbca04'
   description: Should change be backported on previous release
 
 - name: backport-previous
@@ -283,7 +283,7 @@
   description: Debugging in progress
 
 - name: sponsored
-  color: '00FF00'
+  color: '00ff00'
   description: Sponsored work
 
 - name: lgtm
@@ -291,7 +291,7 @@
   description: This PR has been approved by a maintainer
 
 - name: ok-to-test
-  color: '00FF00'
+  color: '00ff00'
   description: Indicates a non-member PR is safe to run CI on
 
 # ──────────────────────────────────────────────

From 31f4435eb0d02f905b6d0ba2f06c22034fc437d9 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Mon, 27 Apr 2026 03:30:29 +0300
Subject: [PATCH 099/151] docs(agents): replace Unicode ellipsis with ASCII in
 contributing.md
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Address review feedback from gemini-code-assist on docs/agents/contributing.md:88:
… replaced with ... for compatibility across editors and tools.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 docs/agents/contributing.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/agents/contributing.md b/docs/agents/contributing.md
index 6ff6eef1..ac6aaf90 100644
--- a/docs/agents/contributing.md
+++ b/docs/agents/contributing.md
@@ -85,7 +85,7 @@ git commit --signoff -m "docs(contributing): add installation guide"
 | extra | area/extra |
 | kubernetes | area/kubernetes |
 | monitoring, vlogs, vmstack, grafana, workloadmonitor | area/monitoring |
-| ingress, gateway, vpn, metallb, cilium, kube-ovn, cozy-proxy, … | area/networking |
+| ingress, gateway, vpn, metallb, cilium, kube-ovn, cozy-proxy, ... | area/networking |
 | platform, bundle, flux, fluxcd, cluster-api, talos, installer, cozyctl, cozystack-engine, cozy-lib | area/platform |
 | backport, release | area/release |
 | seaweedfs, bucket, linstor, velero, harbor, backups | area/storage |
@@ -95,7 +95,7 @@ git commit --signoff -m "docs(contributing): add installation guide"
 **Special handling:**
 
 - `[Backport release-1.x]` prefix is stripped before parsing; `area/release` and `backport` labels are added.
-- Composite scope (`feat(platform, system, apps): …`) — each comma-separated part is mapped independently.
+- Composite scope (`feat(platform, system, apps): ...`) — each comma-separated part is mapped independently.
 - `!` after type or `BREAKING CHANGE:` footer in the body → `kind/breaking-change`.
 - Unmapped scope or non-conventional title → `area/uncategorized` (signals the PR needs manual area selection).
 - Bracket-style fallback (`[scope] description`) maps `scope` → `area/*` but cannot infer `kind/*`.

From 10b98ceb62308808aefbaa1ebdd4b1f07d713779 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Mon, 27 Apr 2026 03:30:43 +0300
Subject: [PATCH 100/151] docs(agents): use full path .github/labels.yml in
 AGENTS.md

Address review feedback from gemini-code-assist on AGENTS.md:33:
expand bare labels.yml and pr-labeler.yaml to .github/labels.yml and
.github/workflows/pr-labeler.yaml for consistency with surrounding refs.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 AGENTS.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/AGENTS.md b/AGENTS.md
index 5298f6fd..1e9c4d15 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -30,7 +30,7 @@ working with the **Cozystack** project.
 - **Issue and PR labeling, triage** (e.g., "label this issue", "what label should I use", "triage this", "categorize")
   - Read: [`.github/labels.yml`](./.github/labels.yml)
   - Action: Use labels defined there. Conventions follow the Kubernetes scheme — `kind/*` (type), `area/*` (subsystem), `priority/*` (urgency), `triage/*` (review state), `lifecycle/*` (auto-close), `do-not-merge/*` (PR blockers), `security/*` (severity)
-  - For `area/*`: accuracy outweighs reuse. If no existing `area/*` truly fits the change, propose a new one via PR (extend `labels.yml` and the scope mapping in `pr-labeler.yaml`) — do not shoehorn the change into a wrong area. `area/uncategorized` is the auto-labeler fallback; treat it as a signal to pick a fit, create a new area, or correct the PR title
+  - For `area/*`: accuracy outweighs reuse. If no existing `area/*` truly fits the change, propose a new one via PR (extend `.github/labels.yml` and the scope mapping in `.github/workflows/pr-labeler.yaml`) — do not shoehorn the change into a wrong area. `area/uncategorized` is the auto-labeler fallback; treat it as a signal to pick a fit, create a new area, or correct the PR title
   - PR titles: a Conventional Commits header (`type(scope): description`, types from [`contributing.md`](./docs/agents/contributing.md)) auto-applies `kind/*` and `area/*` via `.github/workflows/pr-labeler.yaml`. Append `!` (or add a `BREAKING CHANGE:` footer) to apply `kind/breaking-change`
 
 **Important rules:**

From 738762994e10f878cab27a685a5dbf116ccc43fd Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Mon, 27 Apr 2026 03:30:52 +0300
Subject: [PATCH 101/151] docs(agents): fix markdown table cell spacing in
 contributing.md

Address review feedback from gemini-code-assist on docs/agents/contributing.md:73:
add space before closing pipe in the type to kind mapping table for
consistency with other rows.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 docs/agents/contributing.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/agents/contributing.md b/docs/agents/contributing.md
index ac6aaf90..d2ff94e2 100644
--- a/docs/agents/contributing.md
+++ b/docs/agents/contributing.md
@@ -70,7 +70,7 @@ git commit --signoff -m "docs(contributing): add installation guide"
 | docs      | kind/documentation |
 | chore     | kind/cleanup       |
 | refactor  | kind/cleanup       |
-| style, perf, test, build, ci, revert | (no kind label)|
+| style, perf, test, build, ci, revert | (no kind label) |
 
 **Scope → `area/*`** (full mapping in `.github/workflows/pr-labeler.yaml`):
 

From 104b3b3d2b1c25cc06a86669238cad3b744c86d3 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Mon, 20 Apr 2026 18:43:02 +0300
Subject: [PATCH 102/151] feat(operator): add per-package upgradeCRDs policy
 for HelmRelease

Add an opt-in UpgradeCRDs field to ComponentInstall that maps to
HelmRelease.Spec.Upgrade.CRDs, allowing a PackageSource component to
declare how Flux should handle CRDs from the chart's crds/ directory
on upgrade.

The helm-controller default on upgrade is Skip, which means new CRDs
added between chart versions never reach existing clusters and must be
applied manually. Setting upgradeCRDs: CreateReplace makes Flux apply
new CRDs declaratively with the chart.

Allowed values are restricted to Skip, Create, CreateReplace via a
kubebuilder enum marker. Empty / unset preserves the existing Flux
default, so all existing PackageSource resources keep working.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 api/v1alpha1/packagesource_types.go           |  10 ++
 .../cozystack.io_packagesources.yaml          |  13 ++
 internal/operator/package_reconciler.go       |  11 ++
 internal/operator/package_reconciler_test.go  | 138 ++++++++++++++++++
 4 files changed, 172 insertions(+)
 create mode 100644 internal/operator/package_reconciler_test.go

diff --git a/api/v1alpha1/packagesource_types.go b/api/v1alpha1/packagesource_types.go
index c0b943e8..75bf217b 100644
--- a/api/v1alpha1/packagesource_types.go
+++ b/api/v1alpha1/packagesource_types.go
@@ -132,6 +132,16 @@ type ComponentInstall struct {
 	// DependsOn is a list of component names that must be installed before this component
 	// +optional
 	DependsOn []string `json:"dependsOn,omitempty"`
+
+	// UpgradeCRDs controls how CRDs from the chart's crds/ directory are
+	// handled on HelmRelease upgrades. Maps to HelmRelease.Spec.Upgrade.CRDs.
+	// Empty string (default) preserves the helm-controller default (Skip).
+	// Use "CreateReplace" for operators that evolve their CRD set between
+	// versions. Warning: CreateReplace overwrites CRDs and may cause data
+	// loss if upstream drops fields from a CRD with live objects.
+	// +optional
+	// +kubebuilder:validation:Enum=Skip;Create;CreateReplace
+	UpgradeCRDs string `json:"upgradeCRDs,omitempty"`
 }
 
 // Component defines a single Helm release component within a package source
diff --git a/internal/crdinstall/manifests/cozystack.io_packagesources.yaml b/internal/crdinstall/manifests/cozystack.io_packagesources.yaml
index 0acfcdd2..1d0037df 100644
--- a/internal/crdinstall/manifests/cozystack.io_packagesources.yaml
+++ b/internal/crdinstall/manifests/cozystack.io_packagesources.yaml
@@ -118,6 +118,19 @@ spec:
                                   ReleaseName is the name of the HelmRelease resource that will be created
                                   If not specified, defaults to the component Name field
                                 type: string
+                              upgradeCRDs:
+                                description: |-
+                                  UpgradeCRDs controls how CRDs from the chart's crds/ directory are
+                                  handled on HelmRelease upgrades. Maps to HelmRelease.Spec.Upgrade.CRDs.
+                                  Empty string (default) preserves the helm-controller default (Skip).
+                                  Use "CreateReplace" for operators that evolve their CRD set between
+                                  versions. Warning: CreateReplace overwrites CRDs and may cause data
+                                  loss if upstream drops fields from a CRD with live objects.
+                                enum:
+                                - Skip
+                                - Create
+                                - CreateReplace
+                                type: string
                             type: object
                           libraries:
                             description: |-
diff --git a/internal/operator/package_reconciler.go b/internal/operator/package_reconciler.go
index 32e667e4..0e724e49 100644
--- a/internal/operator/package_reconciler.go
+++ b/internal/operator/package_reconciler.go
@@ -45,6 +45,16 @@ const (
 	SecretCozystackValues = "cozystack-values"
 )
 
+// parseCRDPolicy maps ComponentInstall.UpgradeCRDs to a helmv2.CRDsPolicy.
+// Empty / nil preserves the helm-controller default (Skip on upgrade);
+// the CRD enum marker restricts the string to Skip/Create/CreateReplace.
+func parseCRDPolicy(install *cozyv1alpha1.ComponentInstall) helmv2.CRDsPolicy {
+	if install == nil || install.UpgradeCRDs == "" {
+		return ""
+	}
+	return helmv2.CRDsPolicy(install.UpgradeCRDs)
+}
+
 // PackageReconciler reconciles Package resources
 type PackageReconciler struct {
 	client.Client
@@ -221,6 +231,7 @@ func (r *PackageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 					Remediation: &helmv2.UpgradeRemediation{
 						Retries: -1,
 					},
+					CRDs: parseCRDPolicy(component.Install),
 				},
 			},
 		}
diff --git a/internal/operator/package_reconciler_test.go b/internal/operator/package_reconciler_test.go
new file mode 100644
index 00000000..f0ee2c19
--- /dev/null
+++ b/internal/operator/package_reconciler_test.go
@@ -0,0 +1,138 @@
+/*
+Copyright 2025 The Cozystack Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package operator
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+
+	cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
+	helmv2 "github.com/fluxcd/helm-controller/api/v2"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"sigs.k8s.io/yaml"
+)
+
+func TestParseCRDPolicy(t *testing.T) {
+	tests := []struct {
+		name    string
+		install *cozyv1alpha1.ComponentInstall
+		want    helmv2.CRDsPolicy
+	}{
+		{
+			name:    "nil install leaves flux default",
+			install: nil,
+			want:    "",
+		},
+		{
+			name:    "empty upgradeCRDs leaves flux default",
+			install: &cozyv1alpha1.ComponentInstall{},
+			want:    "",
+		},
+		{
+			name:    "Skip is passed through",
+			install: &cozyv1alpha1.ComponentInstall{UpgradeCRDs: "Skip"},
+			want:    helmv2.Skip,
+		},
+		{
+			name:    "Create is passed through",
+			install: &cozyv1alpha1.ComponentInstall{UpgradeCRDs: "Create"},
+			want:    helmv2.Create,
+		},
+		{
+			name:    "CreateReplace is passed through",
+			install: &cozyv1alpha1.ComponentInstall{UpgradeCRDs: "CreateReplace"},
+			want:    helmv2.CreateReplace,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			got := parseCRDPolicy(tc.install)
+			if got != tc.want {
+				t.Errorf("parseCRDPolicy() = %q, want %q", got, tc.want)
+			}
+		})
+	}
+}
+
+// TestPackageSourceCRDHasUpgradeCRDsEnum guards the generated CRD schema: the
+// invalid-value case from the spec is enforced at the API server via a
+// kubebuilder enum marker, not in the reconciler. If someone drops the marker
+// and forgets to regenerate, this test catches it.
+func TestPackageSourceCRDHasUpgradeCRDsEnum(t *testing.T) {
+	path := filepath.Join("..", "crdinstall", "manifests", "cozystack.io_packagesources.yaml")
+	data, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("read %s: %v", path, err)
+	}
+
+	var crd apiextensionsv1.CustomResourceDefinition
+	if err := yaml.Unmarshal(data, &crd); err != nil {
+		t.Fatalf("unmarshal CRD: %v", err)
+	}
+
+	var field *apiextensionsv1.JSONSchemaProps
+	for i := range crd.Spec.Versions {
+		v := &crd.Spec.Versions[i]
+		if v.Schema == nil || v.Schema.OpenAPIV3Schema == nil {
+			continue
+		}
+		spec, ok := v.Schema.OpenAPIV3Schema.Properties["spec"]
+		if !ok {
+			continue
+		}
+		variants, ok := spec.Properties["variants"]
+		if !ok || variants.Items == nil || variants.Items.Schema == nil {
+			continue
+		}
+		components, ok := variants.Items.Schema.Properties["components"]
+		if !ok || components.Items == nil || components.Items.Schema == nil {
+			continue
+		}
+		install, ok := components.Items.Schema.Properties["install"]
+		if !ok {
+			continue
+		}
+		f, ok := install.Properties["upgradeCRDs"]
+		if !ok {
+			continue
+		}
+		field = &f
+		break
+	}
+
+	if field == nil {
+		t.Fatal("upgradeCRDs field not found in PackageSource CRD schema")
+	}
+
+	got := map[string]bool{}
+	for _, e := range field.Enum {
+		var s string
+		if err := json.Unmarshal(e.Raw, &s); err != nil {
+			t.Fatalf("unmarshal enum value %q: %v", e.Raw, err)
+		}
+		got[s] = true
+	}
+
+	for _, want := range []string{"Skip", "Create", "CreateReplace"} {
+		if !got[want] {
+			t.Errorf("enum value %q missing from upgradeCRDs; got %v", want, got)
+		}
+	}
+}

From d86bc7760a77c93c6fd837ef5e23c3df3b9b5898 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Mon, 20 Apr 2026 18:43:18 +0300
Subject: [PATCH 103/151] docs(agents): document PackageSource upgradeCRDs
 field

Describe when to set upgradeCRDs: CreateReplace (operators that evolve
their CRD set additively between versions) and the data-loss risk of
enabling it on operators that drop fields.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 docs/agents/overview.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/docs/agents/overview.md b/docs/agents/overview.md
index 35798961..ae0fd1c4 100644
--- a/docs/agents/overview.md
+++ b/docs/agents/overview.md
@@ -83,6 +83,14 @@ packages/<category>/<package-name>/
 - Reference PR numbers when available
 - Keep commits atomic and focused
 
+### PackageSource CRD upgrade policy
+
+Each component in a `PackageSource` may set `install.upgradeCRDs` to control how CRDs from the chart's `crds/` directory are handled on `HelmRelease` upgrades. Allowed values: `Skip` (default — helm-controller does not touch CRDs on upgrade), `Create` (create new CRDs only), `CreateReplace` (create new and overwrite existing).
+
+Set `upgradeCRDs: CreateReplace` for operators whose upstream regularly adds new CRDs between versions (etcd-operator, cnpg, kubevirt, kamaji). Without it, new CRDs from a chart bump do not land on existing clusters — only fresh installs get them.
+
+Do **not** set `CreateReplace` blindly: it overwrites every CRD in `crds/` and can cause silent data loss if upstream drops a field from a CRD that has live objects. Only enable it for operators whose schema evolution is additive-only. When in doubt, leave it unset and apply new CRDs manually.
+
 ### Documentation
 
 Documentation is organized as follows:

From f527ce683b28401c62cdc420188392b32367401f Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Mon, 20 Apr 2026 19:08:11 +0300
Subject: [PATCH 104/151] docs(agents): clarify that Scopes list is
 illustrative

The Scopes section was read as an exhaustive enumeration, which led to
review feedback flagging any scope outside the list as invalid. The
intent has always been that contributors pick the most specific scope
for the change and extend the list when a genuinely new area appears.
Reword the section accordingly and add operator as an example scope.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 docs/agents/contributing.md | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/docs/agents/contributing.md b/docs/agents/contributing.md
index 658dab74..9f3ff779 100644
--- a/docs/agents/contributing.md
+++ b/docs/agents/contributing.md
@@ -43,10 +43,11 @@ git commit --signoff -m "type(scope): brief description"
 
 **Types:** `feat`, `fix`, `docs`, `style`, `refactor`, `perf`, `test`, `build`, `ci`, `chore`
 
-**Scopes** (e.g., not exhaustive — use any scope that names the component you are touching):
-- System: `dashboard`, `platform`, `cilium`, `kube-ovn`, `linstor`, `fluxcd`, `cluster-api`
-- Apps: `postgres`, `mariadb`, `redis`, `kafka`, `clickhouse`, `virtual-machine`, `kubernetes`
-- Other: `api`, `hack`, `tests`, `ci`, `docs`, `agents`, `maintenance`
+**Scopes** (examples — not an exhaustive list; pick the most specific scope that describes the change, and introduce a new one if a genuinely new area needs its own):
+
+- System, e.g.: `dashboard`, `platform`, `operator`, `cilium`, `kube-ovn`, `linstor`, `fluxcd`, `cluster-api`
+- Apps, e.g.: `postgres`, `mariadb`, `redis`, `kafka`, `clickhouse`, `virtual-machine`, `kubernetes`
+- Other, e.g.: `api`, `hack`, `tests`, `ci`, `docs`, `agents`, `maintenance`
 
 Breaking changes: append `!` after type/scope (`feat(api)!: ...`) or add a `BREAKING CHANGE:` footer.
 

From 32ae993d3ebb51a34ece2b6bbe8e6626c72ac300 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Mon, 20 Apr 2026 19:13:06 +0300
Subject: [PATCH 105/151] docs(maintenance): mirror illustrative-scopes wording
 in PR template

Match the wording adopted in docs/agents/contributing.md so that human
contributors and AI agents see the same guidance in both places.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 .github/PULL_REQUEST_TEMPLATE.md | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index b9475846..044dd6a0 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -1,9 +1,10 @@
 <!-- Thank you for making a contribution! Here are some tips for you:
 - Use Conventional Commits for the PR title: `type(scope): description`
   - Types: feat, fix, docs, style, refactor, perf, test, build, ci, chore
-  - Scopes for system components: dashboard, platform, cilium, kube-ovn, linstor, fluxcd, cluster-api
-  - Scopes for managed apps: postgres, mariadb, redis, kafka, clickhouse, virtual-machine, kubernetes
-  - Scopes for development and maintenance: api, hack, tests, ci, docs, maintenance
+  - Scopes are not an exhaustive list — pick the most specific scope for the change and extend the list when a genuinely new area appears. Examples:
+    - System components: dashboard, platform, operator, cilium, kube-ovn, linstor, fluxcd, cluster-api
+    - Managed apps: postgres, mariadb, redis, kafka, clickhouse, virtual-machine, kubernetes
+    - Development and maintenance: api, hack, tests, ci, docs, maintenance
   - Breaking changes: append `!` after type/scope (`feat(api)!: ...`) or add a `BREAKING CHANGE:` footer
 - If it's a work in progress, consider creating this PR as a draft.
 - Don't hesistate to ask for opinion and review in the community chats, even if it's still a draft.

From 27225f9e8337ae9f11473273aa1939b04cd68a9e Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Mon, 27 Apr 2026 12:46:20 +0300
Subject: [PATCH 106/151] fix(monitoring): use node-level GPU metrics instead
 of namespace-level

DCGM exporter metrics carry the exporter's own namespace
(cozy-gpu-operator), not the workload namespace. Recording rules that
filtered namespace!~"cozy-.*" silently dropped all DCGM series,
producing empty dashboard panels.

Replace namespace-level hardware aggregations with node-level
equivalents (grouped by Hostname), keep namespace-level allocation
rules that use kube_pod_container_resource_requests (which carries the
real workload namespace), and rename pod-level efficiency rules to
gpu-level since DCGM cannot attribute hardware metrics to individual
pods.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 dashboards/gpu/gpu-efficiency.json            |  68 +++---
 dashboards/gpu/gpu-performance.json           |  12 +-
 dashboards/gpu/gpu-tenants.json               | 202 +++---------------
 .../alerts/gpu-recording.rules.yaml           |  94 ++++----
 4 files changed, 110 insertions(+), 266 deletions(-)

diff --git a/dashboards/gpu/gpu-efficiency.json b/dashboards/gpu/gpu-efficiency.json
index 3bd59bc0..f351c964 100644
--- a/dashboards/gpu/gpu-efficiency.json
+++ b/dashboards/gpu/gpu-efficiency.json
@@ -35,12 +35,12 @@
       "id": 2,
       "targets": [
         {
-          "expr": "avg(pod:tensor_saturation:avg5m{namespace=~\"$namespace\"}) * 100",
+          "expr": "avg(gpu:tensor_saturation:avg5m) * 100",
           "refId": "A"
         }
       ],
       "title": "Avg Tensor Saturation",
-      "description": "Mean tensor core saturation across selected namespaces. \u003c10% means GPUs are used inefficiently (tenants could move to CPU or optimize their code).",
+      "description": "Mean tensor core saturation across all GPUs. \u003c10% means GPUs are used inefficiently (workloads could move to CPU or optimize their code). Cluster-wide — DCGM metrics cannot be attributed to workload namespaces.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",
@@ -103,12 +103,12 @@
       "id": 3,
       "targets": [
         {
-          "expr": "avg(pod:util_per_watt:avg5m{namespace=~\"$namespace\"})",
+          "expr": "avg(gpu:util_per_watt:avg5m)",
           "refId": "A"
         }
       ],
       "title": "Avg Utilization per Watt",
-      "description": "NVML utilization % per watt across selected namespaces. Higher value = more efficient workload.",
+      "description": "NVML utilization % per watt across all GPUs. Higher value = more efficient workload. Cluster-wide — DCGM metrics cannot be attributed to workload namespaces.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",
@@ -242,13 +242,13 @@
       "id": 11,
       "targets": [
         {
-          "expr": "DCGM_FI_DEV_GPU_UTIL{namespace=~\"$namespace\", namespace!=\"\"}",
-          "legendFormat": "{{namespace}}/{{pod}}",
+          "expr": "DCGM_FI_DEV_GPU_UTIL",
+          "legendFormat": "{{Hostname}}/{{gpu}}",
           "refId": "A"
         }
       ],
       "title": "NVML GPU Utilization",
-      "description": "Classic utilization metric. Shows activity of any engine (SM, copy, encoder).",
+      "description": "Classic utilization metric. Shows activity of any engine (SM, copy, encoder). Cluster-wide — DCGM namespace is the exporter's own namespace, not the workload namespace.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",
@@ -296,13 +296,13 @@
       "id": 12,
       "targets": [
         {
-          "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace=~\"$namespace\", namespace!=\"\"} * 100",
-          "legendFormat": "{{namespace}}/{{pod}}",
+          "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE * 100",
+          "legendFormat": "{{Hostname}}/{{gpu}}",
           "refId": "A"
         }
       ],
       "title": "Tensor Pipe Active",
-      "description": "Real tensor core load (HMMA). For AI/LLM inference it should be ≥20%, otherwise the workload is unoptimized.",
+      "description": "Real tensor core load (HMMA). For AI/LLM inference it should be ≥20%, otherwise the workload is unoptimized. Cluster-wide — DCGM namespace is the exporter's own namespace, not the workload namespace.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",
@@ -348,7 +348,7 @@
     {
       "type": "row",
       "collapsed": false,
-      "title": "Per-pod ranking",
+      "title": "Per-GPU ranking",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -363,15 +363,15 @@
       "id": 21,
       "targets": [
         {
-          "expr": "topk(20, pod:tensor_saturation:avg5m{namespace=~\"$namespace\"} * 100)",
+          "expr": "topk(20, gpu:tensor_saturation:avg5m * 100)",
           "instant": true,
           "range": false,
           "format": "table",
           "refId": "A"
         }
       ],
-      "title": "Tensor Saturation per pod (5m avg)",
-      "description": "Who is exercising tensor cores and who is not. Sorted descending.",
+      "title": "Tensor Saturation per GPU (5m avg)",
+      "description": "Which GPUs are exercising tensor cores and which are not. Sorted descending. Grouped by Hostname/gpu/UUID — DCGM metrics cannot be attributed to workload namespaces.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",
@@ -390,20 +390,19 @@
           "options": {
             "excludeByName": {
               "DCGM_FI_DRIVER_VERSION": true,
-              "Hostname": true,
               "Time": true,
-              "UUID": true,
               "__name__": true,
               "cluster": true,
               "container": true,
               "device": true,
               "endpoint": true,
-              "gpu": true,
               "gpu_driver_version": true,
               "instance": true,
               "job": true,
               "modelName": true,
+              "namespace": true,
               "pci_bus_id": true,
+              "pod": true,
               "prometheus": true,
               "service": true,
               "tenant": true,
@@ -412,14 +411,16 @@
               "unit": true
             },
             "indexByName": {
-              "Value": 2,
-              "namespace": 0,
-              "pod": 1
+              "Hostname": 0,
+              "UUID": 2,
+              "Value": 3,
+              "gpu": 1
             },
             "renameByName": {
+              "Hostname": "Node",
+              "UUID": "UUID",
               "Value": "Saturation",
-              "namespace": "Namespace",
-              "pod": "Pod"
+              "gpu": "GPU"
             }
           }
         }
@@ -501,15 +502,15 @@
       "id": 22,
       "targets": [
         {
-          "expr": "topk(20, pod:util_per_watt:avg5m{namespace=~\"$namespace\"})",
+          "expr": "topk(20, gpu:util_per_watt:avg5m)",
           "instant": true,
           "range": false,
           "format": "table",
           "refId": "A"
         }
       ],
-      "title": "Utilization / Watt per pod (5m avg)",
-      "description": "How efficiently each pod spends watts. Low value = poor optimization.",
+      "title": "Utilization / Watt per GPU (5m avg)",
+      "description": "How efficiently each GPU spends watts. Low value = poor optimization. Grouped by Hostname/gpu/UUID — DCGM metrics cannot be attributed to workload namespaces.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",
@@ -528,20 +529,19 @@
           "options": {
             "excludeByName": {
               "DCGM_FI_DRIVER_VERSION": true,
-              "Hostname": true,
               "Time": true,
-              "UUID": true,
               "__name__": true,
               "cluster": true,
               "container": true,
               "device": true,
               "endpoint": true,
-              "gpu": true,
               "gpu_driver_version": true,
               "instance": true,
               "job": true,
               "modelName": true,
+              "namespace": true,
               "pci_bus_id": true,
+              "pod": true,
               "prometheus": true,
               "service": true,
               "tenant": true,
@@ -550,14 +550,16 @@
               "unit": true
             },
             "indexByName": {
-              "Value": 2,
-              "namespace": 0,
-              "pod": 1
+              "Hostname": 0,
+              "UUID": 2,
+              "Value": 3,
+              "gpu": 1
             },
             "renameByName": {
+              "Hostname": "Node",
+              "UUID": "UUID",
               "Value": "Util/Watt",
-              "namespace": "Namespace",
-              "pod": "Pod"
+              "gpu": "GPU"
             }
           }
         }
diff --git a/dashboards/gpu/gpu-performance.json b/dashboards/gpu/gpu-performance.json
index e581a76b..f9951577 100644
--- a/dashboards/gpu/gpu-performance.json
+++ b/dashboards/gpu/gpu-performance.json
@@ -273,7 +273,7 @@
       "id": 11,
       "targets": [
         {
-          "expr": "DCGM_FI_DEV_GPU_UTIL{Hostname=~\"$Hostname\", namespace=~\"$namespace\"}",
+          "expr": "DCGM_FI_DEV_GPU_UTIL{Hostname=~\"$Hostname\"}",
           "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}",
           "refId": "A"
         }
@@ -327,7 +327,7 @@
       "id": 12,
       "targets": [
         {
-          "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 100",
+          "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{Hostname=~\"$Hostname\"} * 100",
           "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}",
           "refId": "A"
         }
@@ -381,7 +381,7 @@
       "id": 13,
       "targets": [
         {
-          "expr": "DCGM_FI_PROF_GR_ENGINE_ACTIVE{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 100",
+          "expr": "DCGM_FI_PROF_GR_ENGINE_ACTIVE{Hostname=~\"$Hostname\"} * 100",
           "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}",
           "refId": "A"
         }
@@ -435,7 +435,7 @@
       "id": 14,
       "targets": [
         {
-          "expr": "DCGM_FI_DEV_MEM_COPY_UTIL{Hostname=~\"$Hostname\", namespace=~\"$namespace\"}",
+          "expr": "DCGM_FI_DEV_MEM_COPY_UTIL{Hostname=~\"$Hostname\"}",
           "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}",
           "refId": "A"
         }
@@ -502,7 +502,7 @@
       "id": 21,
       "targets": [
         {
-          "expr": "DCGM_FI_DEV_FB_USED{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 1048576",
+          "expr": "DCGM_FI_DEV_FB_USED{Hostname=~\"$Hostname\"} * 1048576",
           "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}",
           "refId": "A"
         }
@@ -553,7 +553,7 @@
       "id": 22,
       "targets": [
         {
-          "expr": "DCGM_FI_DEV_FB_FREE{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 1048576",
+          "expr": "DCGM_FI_DEV_FB_FREE{Hostname=~\"$Hostname\"} * 1048576",
           "legendFormat": "{{Hostname}}/{{gpu}}",
           "refId": "A"
         }
diff --git a/dashboards/gpu/gpu-tenants.json b/dashboards/gpu/gpu-tenants.json
index d0265c0d..e2c400a8 100644
--- a/dashboards/gpu/gpu-tenants.json
+++ b/dashboards/gpu/gpu-tenants.json
@@ -147,12 +147,12 @@
       "id": 4,
       "targets": [
         {
-          "expr": "avg(namespace:gpu_util:avg{namespace=~\"$namespace\"})",
+          "expr": "cluster:gpu_util:avg",
           "refId": "A"
         }
       ],
-      "title": "Avg tenant util",
-      "description": "Average NVML utilization across all tenant GPUs.",
+      "title": "Avg cluster GPU util",
+      "description": "Average NVML utilization across all GPUs in the cluster. Cluster-wide — DCGM hardware metrics cannot be attributed to workload namespaces.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",
@@ -211,12 +211,12 @@
       "id": 5,
       "targets": [
         {
-          "expr": "sum(namespace:power_watts:sum{namespace=~\"$namespace\"})",
+          "expr": "cluster:gpu_power_watts:sum",
           "refId": "A"
         }
       ],
-      "title": "Total tenant power",
-      "description": "Live power draw summed across tenant workloads.",
+      "title": "Total GPU power",
+      "description": "Live power draw summed across all GPUs in the cluster. Cluster-wide — DCGM hardware metrics cannot be attributed to workload namespaces.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",
@@ -266,12 +266,12 @@
       "id": 6,
       "targets": [
         {
-          "expr": "sum(sum_over_time(namespace:power_watts:sum{namespace=~\"$namespace\"}[24h:1m])) / 60 / 1000",
+          "expr": "sum(sum_over_time(node:power_watts:sum[24h:1m])) / 60 / 1000",
           "refId": "A"
         }
       ],
       "title": "Energy (last 24h)",
-      "description": "Integrated power draw over 24h. Feed into billing/chargeback as kWh.",
+      "description": "Integrated cluster-wide GPU power draw over 24h. Cluster-wide — DCGM hardware metrics cannot be attributed to workload namespaces.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",
@@ -452,13 +452,13 @@
       "id": 12,
       "targets": [
         {
-          "expr": "namespace:fb_used_bytes:sum{namespace=~\"$namespace\"}",
-          "legendFormat": "{{namespace}}",
+          "expr": "node:fb_used_bytes:sum",
+          "legendFormat": "{{Hostname}}",
           "refId": "A"
         }
       ],
-      "title": "VRAM used per namespace",
-      "description": "FB memory actively used by tenant workloads.",
+      "title": "VRAM used per node",
+      "description": "FB memory actively used per node. DCGM hardware metrics cannot be attributed to workload namespaces — shown per-node instead.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",
@@ -513,7 +513,7 @@
     {
       "type": "row",
       "collapsed": false,
-      "title": "Utilization",
+      "title": "Utilization (per-node)",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -528,13 +528,13 @@
       "id": 21,
       "targets": [
         {
-          "expr": "namespace:gpu_util:avg{namespace=~\"$namespace\"}",
-          "legendFormat": "{{namespace}}",
+          "expr": "node:gpu_util:avg",
+          "legendFormat": "{{node}}",
           "refId": "A"
         }
       ],
-      "title": "NVML utilization per namespace",
-      "description": "Per-tenant NVML utilization. Stacked to show fleet-wide pressure over time.",
+      "title": "NVML utilization per node",
+      "description": "Per-node NVML utilization. Stacked to show fleet-wide pressure over time. DCGM hardware metrics cannot be attributed to workload namespaces — shown per-node instead.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",
@@ -584,13 +584,13 @@
       "id": 22,
       "targets": [
         {
-          "expr": "namespace:tensor_active:avg{namespace=~\"$namespace\"} * 100",
-          "legendFormat": "{{namespace}}",
+          "expr": "node:tensor_active:avg * 100",
+          "legendFormat": "{{node}}",
           "refId": "A"
         }
       ],
-      "title": "Tensor saturation per namespace",
-      "description": "Real tensor core load per tenant. Useful to find AI tenants that claim GPUs but leave tensor cores idle.",
+      "title": "Tensor saturation per node",
+      "description": "Real tensor core load per node. Useful to spot nodes with idle tensor cores. DCGM hardware metrics cannot be attributed to workload namespaces — shown per-node instead.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",
@@ -653,13 +653,13 @@
       "id": 31,
       "targets": [
         {
-          "expr": "namespace:power_watts:sum{namespace=~\"$namespace\"}",
-          "legendFormat": "{{namespace}}",
+          "expr": "node:power_watts:sum",
+          "legendFormat": "{{node}}",
           "refId": "A"
         }
       ],
-      "title": "Power draw per namespace",
-      "description": "Per-tenant power draw. Stacked to see cluster-wide energy burn over time.",
+      "title": "Power draw per node",
+      "description": "Per-node GPU power draw. Stacked to see cluster-wide energy burn over time. DCGM hardware metrics cannot be attributed to workload namespaces — shown per-node instead.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",
@@ -783,38 +783,10 @@
           "range": false,
           "format": "table",
           "refId": "gpus"
-        },
-        {
-          "expr": "namespace:gpu_util:avg{namespace=~\"$namespace\"}",
-          "instant": true,
-          "range": false,
-          "format": "table",
-          "refId": "util"
-        },
-        {
-          "expr": "namespace:tensor_active:avg{namespace=~\"$namespace\"} * 100",
-          "instant": true,
-          "range": false,
-          "format": "table",
-          "refId": "tensor"
-        },
-        {
-          "expr": "namespace:power_watts:sum{namespace=~\"$namespace\"}",
-          "instant": true,
-          "range": false,
-          "format": "table",
-          "refId": "power"
-        },
-        {
-          "expr": "namespace:fb_used_bytes:sum{namespace=~\"$namespace\"}",
-          "instant": true,
-          "range": false,
-          "format": "table",
-          "refId": "vram"
         }
       ],
-      "title": "Top consumers",
-      "description": "Live ranking of tenants by power draw. Columns joined from namespace recording rules.",
+      "title": "Top consumers (allocation)",
+      "description": "Live ranking of tenants by GPU allocation. Hardware metrics (util, tensor, power, VRAM) cannot be attributed to workload namespaces — see Fleet and Performance dashboards for per-node/per-GPU hardware views.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",
@@ -828,13 +800,6 @@
       },
       "repeatDirection": "h",
       "transformations": [
-        {
-          "id": "joinByField",
-          "options": {
-            "byField": "namespace",
-            "mode": "outer"
-          }
-        },
         {
           "id": "organize",
           "options": {
@@ -914,18 +879,10 @@
             },
             "indexByName": {
               "Value #gpus": 1,
-              "Value #power": 4,
-              "Value #tensor": 3,
-              "Value #util": 2,
-              "Value #vram": 5,
               "namespace": 0
             },
             "renameByName": {
               "Value #gpus": "GPUs",
-              "Value #power": "Power",
-              "Value #tensor": "Tensor %",
-              "Value #util": "Util %",
-              "Value #vram": "VRAM",
               "namespace": "Namespace"
             }
           }
@@ -937,7 +894,7 @@
         "showTypeIcons": false,
         "sortBy": [
           {
-            "displayName": "Power",
+            "displayName": "GPUs",
             "desc": true
           }
         ],
@@ -946,101 +903,6 @@
           "reducer": []
         },
         "cellHeight": "sm"
-      },
-      "fieldConfig": {
-        "defaults": {},
-        "overrides": [
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Util %"
-            },
-            "properties": [
-              {
-                "id": "unit",
-                "value": "percent"
-              },
-              {
-                "id": "min",
-                "value": 0
-              },
-              {
-                "id": "max",
-                "value": 100
-              },
-              {
-                "id": "decimals",
-                "value": 1
-              },
-              {
-                "id": "custom.cellOptions",
-                "value": {
-                  "mode": "gradient",
-                  "type": "gauge"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Tensor %"
-            },
-            "properties": [
-              {
-                "id": "unit",
-                "value": "percent"
-              },
-              {
-                "id": "min",
-                "value": 0
-              },
-              {
-                "id": "max",
-                "value": 100
-              },
-              {
-                "id": "decimals",
-                "value": 1
-              },
-              {
-                "id": "custom.cellOptions",
-                "value": {
-                  "mode": "gradient",
-                  "type": "gauge"
-                }
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Power"
-            },
-            "properties": [
-              {
-                "id": "unit",
-                "value": "watt"
-              },
-              {
-                "id": "decimals",
-                "value": 0
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "VRAM"
-            },
-            "properties": [
-              {
-                "id": "unit",
-                "value": "bytes"
-              }
-            ]
-          }
-        ]
       }
     },
     {
@@ -1125,13 +987,13 @@
       "id": 52,
       "targets": [
         {
-          "expr": "sum_over_time(namespace:power_watts:sum{namespace=~\"$namespace\"}[24h:1m]) / 60 / 1000",
-          "legendFormat": "{{namespace}}",
+          "expr": "sum_over_time(node:power_watts:sum[24h:1m]) / 60 / 1000",
+          "legendFormat": "{{node}}",
           "refId": "A"
         }
       ],
-      "title": "Energy per namespace (last 24h)",
-      "description": "kWh consumed per tenant over the last 24h. Integrated from 1-minute power samples.",
+      "title": "Energy per node (last 24h)",
+      "description": "kWh consumed per node over the last 24h. Integrated from 1-minute power samples. DCGM hardware metrics cannot be attributed to workload namespaces — shown per-node instead.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",
diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
index 093a2939..b046edf5 100644
--- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
+++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
@@ -42,46 +42,23 @@ spec:
         #     excluding infra pods would under-report raw capacity and
         #     break capacity-planning math.
         #
-        #   * cluster:gpu_util:avg DOES filter infra namespaces because it
-        #     is a tenant-oriented KPI. Admins want to see the mean
-        #     utilization of tenant workloads, not have it diluted by
-        #     GPU Operator DaemonSet pods whose containers hold a GPU
-        #     handle but do no useful compute.
-        #
-        # Do not "normalize" by adding the filter to the hardware rules —
-        # downstream consumers (capacity planning, billing) rely on the
-        # raw totals.
+        # No namespace filter — DCGM exporter metrics carry the exporter's
+        # own namespace, not the workload namespace. Filtering by namespace
+        # here would silently drop all series.
         - record: cluster:gpu_util:avg
-          expr: avg(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"})
+          expr: avg(DCGM_FI_DEV_GPU_UTIL)
         - record: cluster:gpu_power_watts:sum
           expr: sum(DCGM_FI_DEV_POWER_USAGE)
 
-    - name: gpu.recording.namespace.1m
+    - name: gpu.recording.node.1m
       interval: 1m
       params: {}
       rules:
-        # Kube-requested GPU count per namespace — billable view, includes
-        # Pending pods. Use this for GPU-hour reporting via
-        # sum_over_time(...[1h:1m])/60.
-        #
-        # Filters cozy-*/kube-* so the namespace set here matches every
-        # other namespace:* rule below. This keeps the $namespace variable
-        # in the tenant dashboard consistent with panels that read filtered
-        # rules — otherwise picking a system ns from the dropdown leaves
-        # half the panels empty while others show values.
-        #
-        # Trade-off: per-namespace GPU accounting for system workloads is
-        # no longer available here. If it's ever needed (e.g. operator
-        # pods pinning tenant GPUs), add a separate rule like
-        # system:gpu_count:allocated rather than widening this one — the
-        # "billable tenant view" invariant is relied on by tenants.json
-        # and keeping it narrow protects consumers from the asymmetry
-        # that triggered this filter in the first place.
-        #
-        # Note: sum(namespace:gpu_count:allocated) no longer equals
-        # cluster:gpu_count:allocated — the cluster-level rule intentionally
-        # keeps system pods to stay aligned with cluster:gpu_count:total
-        # when computing :free.
+        # Kube-requested GPU count per namespace — billable/allocation view.
+        # This is the only namespace-level rule that works correctly because
+        # kube_pod_container_resource_requests carries the real workload
+        # namespace, unlike DCGM exporter metrics which carry the exporter's
+        # own namespace (cozy-gpu-operator).
         - record: namespace:gpu_count:allocated
           expr: |
             sum by (namespace) (
@@ -90,14 +67,18 @@ spec:
                 kube_pod_status_phase{phase=~"Pending|Running"} == 1
               )
             )
-        - record: namespace:gpu_util:avg
-          expr: avg by (namespace) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"})
-        - record: namespace:tensor_active:avg
-          expr: avg by (namespace) (DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace!="", namespace!~"cozy-.*|kube-.*"})
-        - record: namespace:fb_used_bytes:sum
-          expr: sum by (namespace) (DCGM_FI_DEV_FB_USED{namespace!="", namespace!~"cozy-.*|kube-.*"}) * 1048576
-        - record: namespace:power_watts:sum
-          expr: sum by (namespace) (DCGM_FI_DEV_POWER_USAGE{namespace!="", namespace!~"cozy-.*|kube-.*"})
+        # Hardware metrics aggregated per node (Hostname). DCGM exporter
+        # metrics carry a capital-H "Hostname" label identifying the node
+        # but no usable workload namespace — so hardware telemetry is
+        # aggregated at node/GPU granularity, not namespace.
+        - record: node:gpu_util:avg
+          expr: avg by (Hostname) (DCGM_FI_DEV_GPU_UTIL)
+        - record: node:tensor_active:avg
+          expr: avg by (Hostname) (DCGM_FI_PROF_PIPE_TENSOR_ACTIVE)
+        - record: node:fb_used_bytes:sum
+          expr: sum by (Hostname) (DCGM_FI_DEV_FB_USED) * 1048576
+        - record: node:power_watts:sum
+          expr: sum by (Hostname) (DCGM_FI_DEV_POWER_USAGE)
 
     # Known gap: there is no lower-bound sanity alert for under-reporting.
     # If DCGM ever switches POWER/THERMAL_VIOLATION counter units the other
@@ -113,27 +94,26 @@ spec:
       interval: 1m
       params: {}
       rules:
-        # Tensor hardware saturation — the honest "am I using the GPU" metric
-        # for AI/LLM workloads. Unlike NVML, idle tensor cores are visible.
-        # Stored as a 0..1 ratio to stay consistent with namespace:tensor_active:avg
-        # and DCGM's native units. Consumers multiply by 100 at display time.
-        - record: pod:tensor_saturation:avg5m
+        # Tensor hardware saturation per GPU — the honest "am I using the GPU"
+        # metric for AI/LLM workloads. Aggregated at GPU level (Hostname +
+        # gpu + UUID) because DCGM metrics don't carry workload namespace.
+        # Stored as a 0..1 ratio; consumers multiply by 100 at display time.
+        - record: gpu:tensor_saturation:avg5m
           expr: |
-            avg by (Hostname, gpu, UUID, namespace, pod) (
-              avg_over_time(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m])
+            avg by (Hostname, gpu, UUID) (
+              avg_over_time(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE[5m])
             )
 
-        # Power efficiency — utilization per watt, reveals unoptimized clients.
-        # Explicit on(...) pins the bigop matching set to the labels we
-        # group by — protects against empty results if dcgm-exporter
-        # relabeling ever diverges between the two metrics.
-        - record: pod:util_per_watt:avg5m
+        # Power efficiency per GPU — utilization per watt, reveals
+        # unoptimized workloads. Explicit on(...) pins the matching set
+        # to GPU-identifying labels.
+        - record: gpu:util_per_watt:avg5m
           expr: |
-            avg by (Hostname, gpu, UUID, namespace, pod) (
-              avg_over_time(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m])
-              / on (Hostname, gpu, UUID, namespace, pod)
+            avg by (Hostname, gpu, UUID) (
+              avg_over_time(DCGM_FI_DEV_GPU_UTIL[5m])
+              / on (Hostname, gpu, UUID)
               clamp_min(
-                avg_over_time(DCGM_FI_DEV_POWER_USAGE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]),
+                avg_over_time(DCGM_FI_DEV_POWER_USAGE[5m]),
                 1
               )
             )

From f866c71b684c742ac47e66ceedcca1d7158d6363 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Mon, 27 Apr 2026 12:51:31 +0300
Subject: [PATCH 107/151] fix(gpu-operator): add node relabel to example
 serviceMonitor values

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 .../system/gpu-operator/examples/values-native-talos.yaml     | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/packages/system/gpu-operator/examples/values-native-talos.yaml b/packages/system/gpu-operator/examples/values-native-talos.yaml
index 5a3d7786..ef7b2891 100644
--- a/packages/system/gpu-operator/examples/values-native-talos.yaml
+++ b/packages/system/gpu-operator/examples/values-native-talos.yaml
@@ -44,5 +44,9 @@ spec:
               # documented overhead concerns below 1s; 15s is safe.
               interval: "15s"
               honorLabels: true
+              relabelings:
+                - sourceLabels: [__meta_kubernetes_pod_node_name]
+                  targetLabel: node
+                  action: replace
             config:
               name: dcgm-custom-metrics

From 1baadd75a72ec75921df438671b5e186aed063e2 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Mon, 27 Apr 2026 13:07:04 +0300
Subject: [PATCH 108/151] fix(ci): guard pr.labels access in pr-labeler
 workflow

Address review feedback from myasnikovdaniil on .github/workflows/pr-labeler.yaml:129:
defensive (pr.labels || []) avoids TypeError if the webhook payload
arrives without the labels field on edge cases like stripped edited
events.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 .github/workflows/pr-labeler.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr-labeler.yaml b/.github/workflows/pr-labeler.yaml
index d882bc50..ca3023fd 100644
--- a/.github/workflows/pr-labeler.yaml
+++ b/.github/workflows/pr-labeler.yaml
@@ -126,7 +126,7 @@ jobs:
             const pr = context.payload.pull_request;
             const title = pr.title || '';
             const body  = pr.body  || '';
-            const existing = new Set(pr.labels.map(l => l.name));
+            const existing = new Set((pr.labels || []).map(l => l.name));
             const toAdd = new Set();
 
             // 1. Strip "[Backport release-1.x]" prefix if present.

From 86a1e811ccde200d5206b98df70c4cc29a891ccc Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Mon, 27 Apr 2026 13:07:21 +0300
Subject: [PATCH 109/151] fix(ci): accept hyphenated BREAKING-CHANGE footer in
 pr-labeler

Address review feedback from myasnikovdaniil on .github/workflows/pr-labeler.yaml:155:
Conventional Commits 1.0 spec item 16 treats BREAKING CHANGE: and
BREAKING-CHANGE: as synonymous footers. The hyphen form was silently
ignored before, so PRs that use it would miss kind/breaking-change.

https://www.conventionalcommits.org/en/v1.0.0/#specification

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 .github/workflows/pr-labeler.yaml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pr-labeler.yaml b/.github/workflows/pr-labeler.yaml
index ca3023fd..5a6564a9 100644
--- a/.github/workflows/pr-labeler.yaml
+++ b/.github/workflows/pr-labeler.yaml
@@ -151,8 +151,9 @@ jobs:
               scopeStr = bracket[1];
             }
 
-            // 4. Detect BREAKING CHANGE: footer in body.
-            if (/^BREAKING CHANGE:/m.test(body)) {
+            // 4. Detect BREAKING CHANGE: or BREAKING-CHANGE: footer in body.
+            // Conventional Commits 1.0 spec item 16 treats them as synonymous.
+            if (/^BREAKING[ -]CHANGE:/m.test(body)) {
               breaking = true;
             }
 

From 79744099f63cfa3498362b15f9d2d30024602d0c Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Mon, 27 Apr 2026 13:07:39 +0300
Subject: [PATCH 110/151] chore(ci): warn on unmapped type or scope in
 pr-labeler

Address review feedback from myasnikovdaniil on .github/workflows/pr-labeler.yaml:160,173:
emit core.warning when a Conventional Commits type has no kind/*
mapping or a scope has no area/* mapping. Without the warning, typos
(e.g., "hotfix" instead of "fix") and recurring new scopes silently
fall through to area/uncategorized, masking that the mapping has
drifted.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 .github/workflows/pr-labeler.yaml | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pr-labeler.yaml b/.github/workflows/pr-labeler.yaml
index 5a6564a9..6e6f3017 100644
--- a/.github/workflows/pr-labeler.yaml
+++ b/.github/workflows/pr-labeler.yaml
@@ -158,8 +158,12 @@ jobs:
             }
 
             // 5. Apply kind/* from type.
-            if (type && typeToKind[type]) {
-              toAdd.add(typeToKind[type]);
+            if (type) {
+              if (typeToKind[type]) {
+                toAdd.add(typeToKind[type]);
+              } else {
+                core.warning(`type "${type}" has no kind/* mapping — typo or new type? See .github/workflows/pr-labeler.yaml typeToKind`);
+              }
             }
 
             // 6. Apply area/* from scope. Composite scopes split on comma.
@@ -170,6 +174,8 @@ jobs:
             for (const s of scopes) {
               if (scopeToArea[s]) {
                 toAdd.add(scopeToArea[s]);
+              } else {
+                core.warning(`scope "${s}" has no area/* mapping — consider extending scopeToArea in .github/workflows/pr-labeler.yaml if it recurs`);
               }
             }
 

From d8d870cc2af73925561ae2f9aa936613b47ad0fe Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Mon, 27 Apr 2026 13:24:27 +0300
Subject: [PATCH 111/151] fix(hami): use RollingUpdate strategy for device
 plugin DaemonSet

OnDelete requires manual pod deletion to apply updates. RollingUpdate
with maxUnavailable constraint matches upstream default and is consistent
with all other system packages.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 packages/system/hami/values.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/system/hami/values.yaml b/packages/system/hami/values.yaml
index 64b372a6..76e8d631 100644
--- a/packages/system/hami/values.yaml
+++ b/packages/system/hami/values.yaml
@@ -7,7 +7,7 @@ hami:
   devicePlugin:
     runtimeClassName: nvidia
     updateStrategy:
-      type: OnDelete
+      type: RollingUpdate
     # See https://github.com/Project-HAMi/HAMi/blob/v2.8.1/deployments/charts/hami/values.yaml
     # Each entry: {"name": "node-name", "operatingmode": "hami-core", "devicesplitcount": N, ...}
     nodeConfiguration:

From 36852548e576e153981eab0146614442b5bc9a15 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Mon, 27 Apr 2026 13:29:15 +0300
Subject: [PATCH 112/151] fix(hami): correct label indentation in device-plugin
 monitorservice

Use nindent instead of indent with leading whitespace to prevent
broken YAML rendering when devicePlugin.service.labels is set.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 .../charts/hami/templates/device-plugin/monitorservice.yaml   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml
index 104664ea..88f1b214 100644
--- a/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml
+++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml
@@ -7,8 +7,8 @@ metadata:
   labels:
     app.kubernetes.io/component: hami-device-plugin
     {{- include "hami-vgpu.labels" . | nindent 4 }}
-    {{- if .Values.devicePlugin.service.labels }}  # Use devicePlugin instead of scheduler
-    {{ toYaml .Values.devicePlugin.service.labels | indent 4 }}
+    {{- with .Values.devicePlugin.service.labels }}
+    {{- toYaml . | nindent 4 }}
     {{- end }}
   {{- if .Values.devicePlugin.service.annotations }}  # Use devicePlugin instead of scheduler
   annotations: {{ toYaml .Values.devicePlugin.service.annotations | nindent 4 }}

From f0e033ebcb2fc727bc8fb473507de34edacbf898 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Mon, 27 Apr 2026 13:44:08 +0300
Subject: [PATCH 113/151] style(kubernetes): use with instead of if for hami
 valuesOverride

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 packages/apps/kubernetes/templates/helmreleases/hami.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/apps/kubernetes/templates/helmreleases/hami.yaml b/packages/apps/kubernetes/templates/helmreleases/hami.yaml
index 8733e675..1f7a5358 100644
--- a/packages/apps/kubernetes/templates/helmreleases/hami.yaml
+++ b/packages/apps/kubernetes/templates/helmreleases/hami.yaml
@@ -32,9 +32,9 @@ spec:
     force: true
     remediation:
       retries: -1
-  {{- if .Values.addons.hami.valuesOverride }}
+  {{- with .Values.addons.hami.valuesOverride }}
   values:
-    {{- toYaml .Values.addons.hami.valuesOverride | nindent 4 }}
+    {{- toYaml . | nindent 4 }}
   {{- end }}
 
   dependsOn:

From c53f10475025c6675e5270f359f97f619db7c80f Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Mon, 27 Apr 2026 13:44:12 +0300
Subject: [PATCH 114/151] docs(hami): clarify that parameter defaults come from
 upstream chart

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 packages/system/hami/README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/system/hami/README.md b/packages/system/hami/README.md
index 283f9e80..d648364d 100644
--- a/packages/system/hami/README.md
+++ b/packages/system/hami/README.md
@@ -72,6 +72,8 @@ resources:
 
 ## Parameters
 
+Default values shown below are inherited from the upstream HAMi chart and may change with upstream updates.
+
 | Name | Description | Default |
 | --- | --- | --- |
 | `hami.devicePlugin.runtimeClassName` | RuntimeClass for device plugin pods | `nvidia` |

From 9ba5e587810f44d522d5876e38f12f87c3dde3bb Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Mon, 27 Apr 2026 13:44:17 +0300
Subject: [PATCH 115/151] fix(kubernetes): avoid rendering empty values key in
 gpu-operator HelmRelease

Wrap the values: block in a conditional so the key is omitted entirely
when no defaults or overrides produce content. Previously the template
always emitted values: null, triggering unnecessary FluxCD reconciliation.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 .../apps/kubernetes/templates/helmreleases/gpu-operator.yaml  | 4 +++-
 packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml    | 3 +--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml
index 7a5ecaf1..15849495 100644
--- a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml
+++ b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml
@@ -40,8 +40,10 @@ spec:
   {{- $defaults := fromYaml (include "cozystack.defaultGpuOperatorValues" .) }}
   {{- $overrides := deepCopy (default (dict) .Values.addons.gpuOperator.valuesOverride) }}
   {{- $merged := mergeOverwrite (default (dict) $defaults) $overrides }}
+  {{- if $merged }}
   values:
-    {{- if $merged }}{{ toYaml $merged | nindent 4 }}{{ end }}
+    {{- toYaml $merged | nindent 4 }}
+  {{- end }}
 
   dependsOn:
   {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }}
diff --git a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml
index c3ec30f2..4fa754fb 100644
--- a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml
+++ b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml
@@ -26,9 +26,8 @@ tests:
           enabled: false
           valuesOverride: {}
     asserts:
-      - equal:
+      - notExists:
           path: spec.values
-          value: null
 
   - it: should apply hami defaults when valuesOverride key is omitted
     set:

From 37ecd7b3af9f96ef8cbcb097b8b88a4ffbcef548 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Mon, 27 Apr 2026 13:50:08 +0300
Subject: [PATCH 116/151] feat(kubernetes): make wait-for-kubeconfig image
 overridable for air-gapped registries

Operators in air-gapped or rate-limited environments cannot reach
docker.io and the bundled busybox digest pin gives them no escape
hatch. Add an optional images.waitForKubeconfig chart value that, when
set, replaces the helper's image reference with any registry path
kubelet can pull. Empty value falls back to images/busybox.tag, so the
prior digest-pinned default is preserved.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 api/apps/v1alpha1/kubernetes/types.go         |  9 ++++
 packages/apps/kubernetes/README.md            | 52 ++++++++++---------
 .../apps/kubernetes/templates/_helpers.tpl    | 12 +++--
 .../tests/admin_kubeconfig_wait_test.yaml     | 23 ++++++++
 packages/apps/kubernetes/values.schema.json   | 12 +++++
 packages/apps/kubernetes/values.yaml          |  7 +++
 .../kubernetes-rd/cozyrds/kubernetes.yaml     |  4 +-
 7 files changed, 87 insertions(+), 32 deletions(-)

diff --git a/api/apps/v1alpha1/kubernetes/types.go b/api/apps/v1alpha1/kubernetes/types.go
index 774fcdc5..ea4a2b64 100644
--- a/api/apps/v1alpha1/kubernetes/types.go
+++ b/api/apps/v1alpha1/kubernetes/types.go
@@ -36,6 +36,9 @@ type ConfigSpec struct {
 	// Kubernetes control-plane configuration.
 	// +kubebuilder:default:={}
 	ControlPlane ControlPlane `json:"controlPlane"`
+	// Optional image overrides for air-gapped or rate-limited registries.
+	// +kubebuilder:default:={}
+	Images Images `json:"images"`
 }
 
 type APIServer struct {
@@ -157,6 +160,12 @@ type GatewayAPIAddon struct {
 	Enabled bool `json:"enabled"`
 }
 
+type Images struct {
+	// Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.
+	// +kubebuilder:default:=""
+	WaitForKubeconfig string `json:"waitForKubeconfig,omitempty"`
+}
+
 type IngressNginxAddon struct {
 	// Enable the controller (requires nodes labeled `ingress-nginx`).
 	// +kubebuilder:default:=false
diff --git a/packages/apps/kubernetes/README.md b/packages/apps/kubernetes/README.md
index d62d3a39..ecbbc31b 100644
--- a/packages/apps/kubernetes/README.md
+++ b/packages/apps/kubernetes/README.md
@@ -145,31 +145,33 @@ See the reference for components utilized in this service:
 
 ### Kubernetes Control Plane Configuration
 
-| Name                                                | Description                                      | Type       | Value   |
-| --------------------------------------------------- | ------------------------------------------------ | ---------- | ------- |
-| `controlPlane`                                      | Kubernetes control-plane configuration.          | `object`   | `{}`    |
-| `controlPlane.replicas`                             | Number of control-plane replicas.                | `int`      | `2`     |
-| `controlPlane.apiServer`                            | API Server configuration.                        | `object`   | `{}`    |
-| `controlPlane.apiServer.resources`                  | CPU and memory resources for API Server.         | `object`   | `{}`    |
-| `controlPlane.apiServer.resources.cpu`              | CPU available.                                   | `quantity` | `""`    |
-| `controlPlane.apiServer.resources.memory`           | Memory (RAM) available.                          | `quantity` | `""`    |
-| `controlPlane.apiServer.resourcesPreset`            | Preset if `resources` omitted.                   | `string`   | `large` |
-| `controlPlane.controllerManager`                    | Controller Manager configuration.                | `object`   | `{}`    |
-| `controlPlane.controllerManager.resources`          | CPU and memory resources for Controller Manager. | `object`   | `{}`    |
-| `controlPlane.controllerManager.resources.cpu`      | CPU available.                                   | `quantity` | `""`    |
-| `controlPlane.controllerManager.resources.memory`   | Memory (RAM) available.                          | `quantity` | `""`    |
-| `controlPlane.controllerManager.resourcesPreset`    | Preset if `resources` omitted.                   | `string`   | `micro` |
-| `controlPlane.scheduler`                            | Scheduler configuration.                         | `object`   | `{}`    |
-| `controlPlane.scheduler.resources`                  | CPU and memory resources for Scheduler.          | `object`   | `{}`    |
-| `controlPlane.scheduler.resources.cpu`              | CPU available.                                   | `quantity` | `""`    |
-| `controlPlane.scheduler.resources.memory`           | Memory (RAM) available.                          | `quantity` | `""`    |
-| `controlPlane.scheduler.resourcesPreset`            | Preset if `resources` omitted.                   | `string`   | `micro` |
-| `controlPlane.konnectivity`                         | Konnectivity configuration.                      | `object`   | `{}`    |
-| `controlPlane.konnectivity.server`                  | Konnectivity Server configuration.               | `object`   | `{}`    |
-| `controlPlane.konnectivity.server.resources`        | CPU and memory resources for Konnectivity.       | `object`   | `{}`    |
-| `controlPlane.konnectivity.server.resources.cpu`    | CPU available.                                   | `quantity` | `""`    |
-| `controlPlane.konnectivity.server.resources.memory` | Memory (RAM) available.                          | `quantity` | `""`    |
-| `controlPlane.konnectivity.server.resourcesPreset`  | Preset if `resources` omitted.                   | `string`   | `micro` |
+| Name                                                | Description                                                                                   | Type       | Value   |
+| --------------------------------------------------- | --------------------------------------------------------------------------------------------- | ---------- | ------- |
+| `controlPlane`                                      | Kubernetes control-plane configuration.                                                       | `object`   | `{}`    |
+| `controlPlane.replicas`                             | Number of control-plane replicas.                                                             | `int`      | `2`     |
+| `controlPlane.apiServer`                            | API Server configuration.                                                                     | `object`   | `{}`    |
+| `controlPlane.apiServer.resources`                  | CPU and memory resources for API Server.                                                      | `object`   | `{}`    |
+| `controlPlane.apiServer.resources.cpu`              | CPU available.                                                                                | `quantity` | `""`    |
+| `controlPlane.apiServer.resources.memory`           | Memory (RAM) available.                                                                       | `quantity` | `""`    |
+| `controlPlane.apiServer.resourcesPreset`            | Preset if `resources` omitted.                                                                | `string`   | `large` |
+| `controlPlane.controllerManager`                    | Controller Manager configuration.                                                             | `object`   | `{}`    |
+| `controlPlane.controllerManager.resources`          | CPU and memory resources for Controller Manager.                                              | `object`   | `{}`    |
+| `controlPlane.controllerManager.resources.cpu`      | CPU available.                                                                                | `quantity` | `""`    |
+| `controlPlane.controllerManager.resources.memory`   | Memory (RAM) available.                                                                       | `quantity` | `""`    |
+| `controlPlane.controllerManager.resourcesPreset`    | Preset if `resources` omitted.                                                                | `string`   | `micro` |
+| `controlPlane.scheduler`                            | Scheduler configuration.                                                                      | `object`   | `{}`    |
+| `controlPlane.scheduler.resources`                  | CPU and memory resources for Scheduler.                                                       | `object`   | `{}`    |
+| `controlPlane.scheduler.resources.cpu`              | CPU available.                                                                                | `quantity` | `""`    |
+| `controlPlane.scheduler.resources.memory`           | Memory (RAM) available.                                                                       | `quantity` | `""`    |
+| `controlPlane.scheduler.resourcesPreset`            | Preset if `resources` omitted.                                                                | `string`   | `micro` |
+| `controlPlane.konnectivity`                         | Konnectivity configuration.                                                                   | `object`   | `{}`    |
+| `controlPlane.konnectivity.server`                  | Konnectivity Server configuration.                                                            | `object`   | `{}`    |
+| `controlPlane.konnectivity.server.resources`        | CPU and memory resources for Konnectivity.                                                    | `object`   | `{}`    |
+| `controlPlane.konnectivity.server.resources.cpu`    | CPU available.                                                                                | `quantity` | `""`    |
+| `controlPlane.konnectivity.server.resources.memory` | Memory (RAM) available.                                                                       | `quantity` | `""`    |
+| `controlPlane.konnectivity.server.resourcesPreset`  | Preset if `resources` omitted.                                                                | `string`   | `micro` |
+| `images`                                            | Optional image overrides for air-gapped or rate-limited registries.                           | `object`   | `{}`    |
+| `images.waitForKubeconfig`                          | Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag. | `string`   | `""`    |
 
 
 ## Parameter examples and reference
diff --git a/packages/apps/kubernetes/templates/_helpers.tpl b/packages/apps/kubernetes/templates/_helpers.tpl
index e6ad9dcf..40a8ae7a 100644
--- a/packages/apps/kubernetes/templates/_helpers.tpl
+++ b/packages/apps/kubernetes/templates/_helpers.tpl
@@ -65,18 +65,20 @@ release.cozystack.io/helm-install-timeout annotation) so the
 CrashLoopBackOff surfaces before flux remediation fires and uninstalls
 the Cluster CR.
 
-The pinned busybox image in images/busybox.tag points directly at
+The default image lives in images/busybox.tag and points directly at
 docker.io by digest (not mirrored to ghcr.io like the other .tag files
-here): the payload is a one-shot sh loop, the digest pin makes the
-pull immutable, and the cost of maintaining a private mirror of a tiny
-upstream image that does not move often is not worth it.
+here): the payload is a one-shot sh loop and the digest pin makes the
+pull immutable. Operators in air-gapped or rate-limited environments
+can override it via .Values.images.waitForKubeconfig (any registry
+reference kubelet can pull). When the value is empty the chart falls
+back to the bundled digest pin, preserving the prior default.
 
 Call site owns the surrounding volumes block; the kubeconfig volume
 must exist on the pod and mount at /etc/kubernetes/kubeconfig.
 */}}
 {{- define "kubernetes.waitForAdminKubeconfig" -}}
 - name: wait-for-kubeconfig
-  image: "{{ .Files.Get "images/busybox.tag" | trim }}"
+  image: "{{ default (.Files.Get "images/busybox.tag" | trim) .Values.images.waitForKubeconfig }}"
   command:
   - sh
   - -c
diff --git a/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml b/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml
index e7c03bd1..507e00af 100644
--- a/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml
+++ b/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml
@@ -95,6 +95,29 @@ tests:
             mountPath: /etc/kubernetes/kubeconfig
             readOnly: true
 
+  - it: wait-for-kubeconfig defaults to bundled busybox digest pin
+    template: templates/cluster-autoscaler/deployment.yaml
+    documentSelector:
+      path: kind
+      value: Deployment
+    asserts:
+      - matchRegex:
+          path: spec.template.spec.initContainers[0].image
+          pattern: '^docker\.io/library/busybox:[^@]+@sha256:[0-9a-f]{64}$'
+
+  - it: wait-for-kubeconfig honours images.waitForKubeconfig override
+    template: templates/cluster-autoscaler/deployment.yaml
+    documentSelector:
+      path: kind
+      value: Deployment
+    set:
+      images:
+        waitForKubeconfig: "registry.example.test/library/busybox:1.37.0@sha256:0000000000000000000000000000000000000000000000000000000000000000"
+    asserts:
+      - equal:
+          path: spec.template.spec.initContainers[0].image
+          value: "registry.example.test/library/busybox:1.37.0@sha256:0000000000000000000000000000000000000000000000000000000000000000"
+
   - it: cluster.yaml renders and wires dataStoreName when tenant has etcd
     template: templates/cluster.yaml
     documentSelector:
diff --git a/packages/apps/kubernetes/values.schema.json b/packages/apps/kubernetes/values.schema.json
index 1beeff9c..81e9e2b8 100644
--- a/packages/apps/kubernetes/values.schema.json
+++ b/packages/apps/kubernetes/values.schema.json
@@ -630,6 +630,18 @@
           }
         }
       }
+    },
+    "images": {
+      "description": "Optional image overrides for air-gapped or rate-limited registries.",
+      "type": "object",
+      "default": {},
+      "properties": {
+        "waitForKubeconfig": {
+          "description": "Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.",
+          "type": "string",
+          "default": ""
+        }
+      }
     }
   }
 }
diff --git a/packages/apps/kubernetes/values.yaml b/packages/apps/kubernetes/values.yaml
index a67b5d69..68ba07b0 100644
--- a/packages/apps/kubernetes/values.yaml
+++ b/packages/apps/kubernetes/values.yaml
@@ -197,3 +197,10 @@ controlPlane:
     server:
       resources: {}
       resourcesPreset: "micro"
+
+## @typedef {struct} Images - Optional image overrides for chart-internal helpers.
+## @field {string} [waitForKubeconfig] - Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.
+
+## @param {Images} images - Optional image overrides for air-gapped or rate-limited registries.
+images:
+  waitForKubeconfig: ""
diff --git a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml
index aaa01247..1631c86e 100644
--- a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml
+++ b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml
@@ -22,7 +22,7 @@ spec:
     singular: kubernetes
     plural: kuberneteses
     openAPISchema: |-
-      {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"ephemeralStorage":"20Gi","gpus":[],"instanceType":"u1.medium","maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"]}},"additionalProperties":{"type":"object","required":["ephemeralStorage","instanceType","maxReplicas","minReplicas","resources"],"properties":{"ephemeralStorage":{"description":"Ephemeral storage size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string"}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium"},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `<cluster-name>.<tenant-host>` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","ingressNginx","monitoringAgents","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"large","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}}}}
+      {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"ephemeralStorage":"20Gi","gpus":[],"instanceType":"u1.medium","maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"]}},"additionalProperties":{"type":"object","required":["ephemeralStorage","instanceType","maxReplicas","minReplicas","resources"],"properties":{"ephemeralStorage":{"description":"Ephemeral storage size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string"}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium"},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `<cluster-name>.<tenant-host>` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","ingressNginx","monitoringAgents","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"large","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"images":{"description":"Optional image overrides for air-gapped or rate-limited registries.","type":"object","default":{},"properties":{"waitForKubeconfig":{"description":"Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.","type":"string","default":""}}}}}
   release:
     prefix: kubernetes-
     labels:
@@ -40,7 +40,7 @@ spec:
     tags:
       - compute
     icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODFfMjg0NSkiLz4KPHBhdGggZD0iTTcxLjk5NjggMTlDNzAuMzAzOSAxOS4wMDAyIDY4LjkzMTIgMjAuNTMyMiA2OC45MzE0IDIyLjQyMjFDNjguOTMxNCAyMi40NTExIDY4LjkzNzMgMjIuNDc4OCA2OC45Mzc5IDIyLjUwNzZDNjguOTM1NCAyMi43NjQ0IDY4LjkyMzEgMjMuMDczNyA2OC45MzE0IDIzLjI5NzNDNjguOTcxNyAyNC4zODczIDY5LjIwODIgMjUuMjIxNiA2OS4zNTA2IDI2LjIyNThDNjkuNjA4NCAyOC4zNzUyIDY5LjgyNDUgMzAuMTU2OSA2OS42OTEyIDMxLjgxM0M2OS41NjE1IDMyLjQzNzUgNjkuMTAzNyAzMy4wMDg2IDY4LjY5NTYgMzMuNDA1Nkw2OC42MjM1IDM0LjcwODZDNjYuNzgzOSAzNC44NjE3IDY0LjkzMTkgMzUuMTQyMSA2My4wODIxIDM1LjU2NDFDNTUuMTIyNiAzNy4zNzk4IDQ4LjI2OTUgNDEuNDk5MSA0My4wNTIgNDcuMDYwOUM0Mi43MTM0IDQ2LjgyODggNDIuMTIxMSA0Ni40MDE5IDQxLjk0NSA0Ni4yNzEyQzQxLjM5NzcgNDYuMzQ1NCA0MC44NDQ1IDQ2LjUxNTEgNDAuMTI0MSA0Ni4wOTM1QzM4Ljc1MjIgNDUuMTY1NyAzNy41MDI4IDQzLjg4NTEgMzUuOTkxIDQyLjM0MjRDMzUuMjk4MiA0MS42MDQ0IDM0Ljc5NjYgNDAuOTAxOCAzMy45NzM1IDQwLjE5MDRDMzMuNzg2NiA0MC4wMjg5IDMzLjUwMTQgMzkuODEwNCAzMy4yOTIzIDM5LjY0NDJDMzIuNjQ4OSAzOS4xMjg4IDMxLjg5IDM4Ljg2IDMxLjE1NyAzOC44MzQ4QzMwLjIxNDcgMzguODAyNCAyOS4zMDc1IDM5LjE3MjUgMjguNzEzOCAzOS45MjA2QzI3LjY1ODQgNDEuMjUwNiAyNy45OTYzIDQzLjI4MzMgMjkuNDY3MSA0NC40NjE0QzI5LjQ4MiA0NC40NzM0IDI5LjQ5NzkgNDQuNDgyNyAyOS41MTI5IDQ0LjQ5NDNDMjkuNzE1IDQ0LjY1ODkgMjkuOTYyNSA0NC44Njk4IDMwLjE0ODMgNDUuMDA3NkMzMS4wMjE3IDQ1LjY1NTUgMzEuODE5NSA0NS45ODcyIDMyLjY4OTcgNDYuNTAxNUMzNC41MjMxIDQ3LjYzOTEgMzYuMDQzIDQ4LjU4MjMgMzcuMjQ4NiA0OS43MTk2QzM3LjcxOTQgNTAuMjIzNyAzNy44MDE2IDUxLjExMjIgMzcuODY0MyA1MS40OTY0TDM4Ljg0NjggNTIuMzc4MkMzMy41ODcyIDYwLjMzMDggMzEuMTUzIDcwLjE1MzkgMzIuNTkxNSA4MC4xNjI3TDMxLjMwNzcgODAuNTM3OEMzMC45NjkzIDgwLjk3NjggMzAuNDkxMiA4MS42Njc2IDI5Ljk5MTEgODEuODczOEMyOC40MTM4IDgyLjM3MjkgMjYuNjM4NyA4Mi41NTYyIDI0LjQ5NTYgODIuNzgxOUMyMy40ODk0IDgyLjg2NiAyMi42MjEzIDgyLjgxNTggMjEuNTU0NiA4My4wMTg4QzIxLjMxOTggODMuMDYzNSAyMC45OTI3IDgzLjE0OTEgMjAuNzM1OCA4My4yMDk3QzIwLjcyNjkgODMuMjExNiAyMC43MTg2IDgzLjIxNDIgMjAuNzA5NiA4My4yMTYyQzIwLjY5NTYgODMuMjE5NSAyMC42NzcyIDgzLjIyNjMgMjAuNjYzOCA4My4yMjk0QzE4Ljg1NyA4My42NjggMTcuNjk2MyA4NS4zMzY1IDE4LjA2OTkgODYuOTgwNUMxOC40NDM3IDg4LjYyNDggMjAuMjA4NiA4OS42MjQ4IDIyLjAyNjIgODkuMjMxMkMyMi4wMzkzIDg5LjIyODIgMjIuMDU4NCA4OS4yMjc3IDIyLjA3MiA4OS4yMjQ2QzIyLjA5MjYgODkuMjE5OSAyMi4xMTA2IDg5LjIwOTkgMjIuMTMxIDg5LjIwNDlDMjIuMzg0NCA4OS4xNDkgMjIuNzAxOSA4OS4wODY4IDIyLjkyMzYgODkuMDI3MkMyMy45NzIzIDg4Ljc0NTEgMjQuNzMxOCA4OC4zMzA2IDI1LjY3NDYgODcuOTY3N0MyNy43MDI5IDg3LjIzNjggMjkuMzgyOCA4Ni42MjYyIDMxLjAxOTUgODYuMzg4M0MzMS43MDMgODYuMzM0NSAzMi40MjMyIDg2LjgxMiAzMi43ODE0IDg3LjAxMzRMMzQuMTE3NyA4Ni43ODMxQzM3LjE5MjYgOTYuMzYxMyA0My42MzY2IDEwNC4xMDMgNTEuNzk2MyAxMDguOTYxTDUxLjIzOTYgMTEwLjMwM0M1MS40NDAzIDExMC44MjQgNTEuNjYxNiAxMTEuNTMgNTEuNTEyMSAxMTIuMDQ1QzUwLjkxNzEgMTEzLjU5NSA0OS44OTggMTE1LjIzMSA0OC43Mzc0IDExNy4wNTVDNDguMTc1NSAxMTcuODk4IDQ3LjYwMDQgMTE4LjU1MiA0Ny4wOTM0IDExOS41MTZDNDYuOTcyIDExOS43NDcgNDYuODE3NSAxMjAuMTAyIDQ2LjcwMDQgMTIwLjM0NkM0NS45MTI1IDEyMi4wMzkgNDYuNDkwNCAxMjMuOTkgNDguMDAzOCAxMjQuNzIyQzQ5LjUyNjggMTI1LjQ1OSA1MS40MTcxIDEyNC42ODIgNTIuMjM1MiAxMjIuOTg1QzUyLjIzNjQgMTIyLjk4MiA1Mi4yNDA2IDEyMi45OCA1Mi4yNDE3IDEyMi45NzhDNTIuMjQyNiAxMjIuOTc2IDUyLjI0MDkgMTIyLjk3MyA1Mi4yNDE3IDEyMi45NzFDNTIuMzU4MiAxMjIuNzMxIDUyLjUyMzMgMTIyLjQxNSA1Mi42MjE2IDEyMi4xODhDNTMuMDU2IDEyMS4xODkgNTMuMjAwNSAxMjAuMzMyIDUzLjUwNTkgMTE5LjM2NUM1NC4zMTcgMTE3LjMxOCA1NC43NjI2IDExNS4xNyA1NS44NzkxIDExMy44MzJDNTYuMTg0OSAxMTMuNDY2IDU2LjY4MzMgMTEzLjMyNSA1Ny4yMDAxIDExMy4xODZMNTcuODk0NCAxMTEuOTIyQzY1LjAwOCAxMTQuNjY1IDcyLjk3MDUgMTE1LjQwMiA4MC45MjQ1IDExMy41ODdDODIuNzM5MSAxMTMuMTczIDg0LjQ5MDggMTEyLjYzNyA4Ni4xODQzIDExMS45OTRDODYuMzc5NCAxMTIuMzQyIDg2Ljc0MiAxMTMuMDExIDg2LjgzOTMgMTEzLjE3OUM4Ny4zNjQ0IDExMy4zNTEgODcuOTM3NyAxMTMuNDM5IDg4LjQwNDcgMTE0LjEzM0M4OS4yNDAxIDExNS41NjcgODkuODExNCAxMTcuMjYzIDkwLjUwNzMgMTE5LjMxMkM5MC44MTI4IDEyMC4yNzkgOTAuOTYzOCAxMjEuMTM2IDkxLjM5ODEgMTIyLjEzNkM5MS40OTcxIDEyMi4zNjMgOTEuNjYxNCAxMjIuNjg0IDkxLjc3OCAxMjIuOTI1QzkyLjU5NDQgMTI0LjYyOCA5NC40OTA3IDEyNS40MDcgOTYuMDE1OSAxMjQuNjY5Qzk3LjUyOTIgMTIzLjkzNyA5OC4xMDc3IDEyMS45ODYgOTcuMzE5NCAxMjAuMjkzQzk3LjIwMjMgMTIwLjA0OSA5Ny4wNDEyIDExOS42OTUgOTYuOTE5OCAxMTkuNDY0Qzk2LjQxMjcgMTE4LjQ5OSA5NS44Mzc3IDExNy44NTIgOTUuMjc1OCAxMTcuMDA5Qzk0LjExNTIgMTE1LjE4NSA5My4xNTI2IDExMy42NyA5Mi41NTc1IDExMi4xMkM5Mi4zMDg3IDExMS4zMiA5Mi41OTk1IDExMC44MjMgOTIuNzkzMyAxMTAuMzAzQzkyLjY3NzIgMTEwLjE3IDkyLjQyODggMTA5LjQxNCA5Mi4yODI0IDEwOS4wNTlDMTAwLjc2MiAxMDQuMDI5IDEwNy4wMTcgOTUuOTk4NSAxMDkuOTU1IDg2LjcyMzlDMTEwLjM1MSA4Ni43ODY1IDExMS4wNDEgODYuOTA5MSAxMTEuMjY1IDg2Ljk1NDJDMTExLjcyNiA4Ni42NDg3IDExMi4xNDkgODYuMjUwMSAxMTIuOTgxIDg2LjMxNTlDMTE0LjYxNyA4Ni41NTM3IDExNi4yOTcgODcuMTY0NSAxMTguMzI2IDg3Ljg5NTNDMTE5LjI2OCA4OC4yNTgxIDEyMC4wMjggODguNjc5MyAxMjEuMDc3IDg4Ljk2MTRDMTIxLjI5OCA4OS4wMjEgMTIxLjYxNiA4OS4wNzY2IDEyMS44NjkgODkuMTMyNUMxMjEuODg5IDg5LjEzNzUgMTIxLjkwOCA4OS4xNDc1IDEyMS45MjggODkuMTUyMkMxMjEuOTQyIDg5LjE1NTMgMTIxLjk2MSA4OS4xNTU4IDEyMS45NzQgODkuMTU4OEMxMjMuNzkyIDg5LjU1MiAxMjUuNTU3IDg4LjU1MjYgMTI1LjkzIDg2LjkwODFDMTI2LjMwMyA4NS4yNjQxIDEyNS4xNDMgODMuNTk1MiAxMjMuMzM2IDgzLjE1N0MxMjMuMDc0IDgzLjA5NyAxMjIuNzAxIDgyLjk5NSAxMjIuNDQ2IDgyLjk0NjVDMTIxLjM3OSA4Mi43NDM1IDEyMC41MTEgODIuNzkzNSAxMTkuNTA1IDgyLjcwOTVDMTE3LjM2MSA4Mi40ODM5IDExNS41ODYgODIuMzAwNCAxMTQuMDA5IDgxLjgwMTRDMTEzLjM2NiA4MS41NTA3IDExMi45MDggODAuNzgxOSAxMTIuNjg2IDgwLjQ2NTVMMTExLjQ0OCA4MC4xMDM1QzExMi4wOSA3NS40MzggMTExLjkxNyA3MC41ODI1IDExMC44MDYgNjUuNzI0M0MxMDkuNjg1IDYwLjgyMDggMTA3LjcwNCA1Ni4zMzYxIDEwNS4wNjIgNTIuMzg0OEMxMDUuMzc5IDUyLjA5NDggMTA1Ljk3OSA1MS41NjEyIDEwNi4xNDkgNTEuNDA0M0MxMDYuMTk5IDUwLjg1MTcgMTA2LjE1NiA1MC4yNzIyIDEwNi43MjUgNDkuNjYwM0MxMDcuOTMxIDQ4LjUyMyAxMDkuNDUxIDQ3LjU3OTkgMTExLjI4NCA0Ni40NDIzQzExMi4xNTQgNDUuOTI3OSAxMTIuOTU5IDQ1LjU5NjQgMTEzLjgzMiA0NC45NDg0QzExNC4wMyA0NC44MDE5IDExNC4yOTkgNDQuNTY5OSAxMTQuNTA3IDQ0LjQwMjJDMTE1Ljk3NyA0My4yMjM3IDExNi4zMTYgNDEuMTkxMSAxMTUuMjYgMzkuODYxNEMxMTQuMjA0IDM4LjUzMTcgMTEyLjE1OSAzOC40MDY1IDExMC42ODggMzkuNTg1QzExMC40NzkgMzkuNzUxNiAxMTAuMTk1IDM5Ljk2ODggMTEwLjAwNyA0MC4xMzEyQzEwOS4xODQgNDAuODQyNiAxMDguNjc2IDQxLjU0NTIgMTA3Ljk4MyA0Mi4yODMyQzEwNi40NzEgNDMuODI1OSAxMDUuMjIyIDQ1LjExMyAxMDMuODUgNDYuMDQwOUMxMDMuMjU1IDQ2LjM4ODUgMTAyLjM4NSA0Ni4yNjgyIDEwMS45OSA0Ni4yNDQ5TDEwMC44MjQgNDcuMDgwNkM5NC4xNzUzIDQwLjA3NjMgODUuMTIzNSAzNS41OTgyIDc1LjM3NjYgMzQuNzI4M0M3NS4zNDk0IDM0LjMxNzkgNzUuMzEzNyAzMy41NzYxIDc1LjMwNDYgMzMuMzUyOUM3NC45MDU2IDMyLjk2OTMgNzQuNDIzNSAzMi42NDE4IDc0LjMwMjQgMzEuODEzQzc0LjE2OTEgMzAuMTU2OSA3NC4zOTE3IDI4LjM3NTIgNzQuNjQ5NiAyNi4yMjU4Qzc0Ljc5MTkgMjUuMjIxNiA3NS4wMjg0IDI0LjM4NzMgNzUuMDY4OCAyMy4yOTczQzc1LjA3OCAyMy4wNDk1IDc1LjA2MzIgMjIuNjkgNzUuMDYyMiAyMi40MjIxQzc1LjA2MiAyMC41MzIyIDczLjY4OTggMTguOTk5OCA3MS45OTY4IDE5Wk02OC4xNTg1IDQyLjg4ODZMNjcuMjQ4IDU5LjA0NDdMNjcuMTgyNSA1OS4wNzc2QzY3LjEyMTQgNjAuNTIyOSA2NS45Mzc1IDYxLjY3NyA2NC40ODM5IDYxLjY3N0M2My44ODg0IDYxLjY3NyA2My4zMzg4IDYxLjQ4NDkgNjIuODkyMiA2MS4xNTcxTDYyLjg2NiA2MS4xNzAzTDQ5LjY4MDcgNTEuNzc5NEM1My43MzMxIDQ3Ljc3NTkgNTguOTE2NCA0NC44MTcyIDY0Ljg5IDQzLjQ1NDZDNjUuOTgxMiA0My4yMDU2IDY3LjA3MTkgNDMuMDIwOSA2OC4xNTg1IDQyLjg4ODZaTTc1Ljg0MTcgNDIuODg4NkM4Mi44MTU5IDQzLjc1MDQgODkuMjY1NyA0Ni45MjMyIDk0LjIwODEgNTEuNzg2TDgxLjEwOCA2MS4xMTc2TDgxLjA2MjEgNjEuMDk3OUM3OS44OTk0IDYxLjk1MTIgNzguMjYxMSA2MS43Mzk0IDc3LjM1NDggNjAuNTk3OEM3Ni45ODM1IDYwLjEzMDEgNzYuNzg4NyA1OS41ODAxIDc2Ljc2NTMgNTkuMDI0OUw3Ni43NTIyIDU5LjAxODRMNzUuODQxNyA0Mi44ODg2Wk00NC44OTkxIDU3LjgxNEw1Ni45MzgyIDY4LjYzM0w1Ni45MjUxIDY4LjY5ODhDNTguMDExNyA2OS42NDc5IDU4LjE3MiA3MS4yOTQ5IDU3LjI2NTcgNzIuNDM2OEM1Ni44OTQ0IDcyLjkwNDUgNTYuMzk3NSA3My4yMTgyIDU1Ljg2MzkgNzMuMzY0N0w1NS44NTA4IDczLjQxNzNMNDAuNDE4OCA3Ny44OTIzQzM5LjYzMzQgNzAuNjc2NSA0MS4zMjYxIDYzLjY2MjEgNDQuODk5MSA1Ny44MTRaTTk5LjAwOTQgNTcuODIwNkMxMDAuNzk4IDYwLjczMzYgMTAyLjE1MyA2My45ODcxIDEwMi45NTkgNjcuNTE0M0MxMDMuNzU2IDcwLjk5OTEgMTAzLjk1NiA3NC40Nzc4IDEwMy42MjcgNzcuODM5N0w4OC4xMTY2IDczLjM1MTVMODguMTAzNSA3My4yODU3Qzg2LjcxNDUgNzIuOTA0MyA4NS44NjA5IDcxLjQ4NDggODYuMTg0MyA3MC4wNjExQzg2LjMxNjggNjkuNDc3OCA4Ni42MjQ5IDY4Ljk4NDQgODcuMDQyMyA2OC42MTk4TDg3LjAzNTggNjguNTg2OUw5OS4wMDk0IDU3LjgyMDZaTTY5LjUyNzQgNjkuNDY4OEg3NC40NTk2TDc3LjUyNTEgNzMuMzE4Nkw3Ni40MjQ3IDc4LjEyMjZMNzEuOTk2OCA4MC4yNjE0TDY3LjU1NTggNzguMTE2MUw2Ni40NTU0IDczLjMxMkw2OS41Mjc0IDY5LjQ2ODhaTTg1LjMzOTMgODIuNjQzN0M4NS41NDg5IDgyLjYzMzEgODUuNzU3NiA4Mi42NTIgODUuOTYxNiA4Mi42ODk4TDg1Ljk4NzggODIuNjU2OUwxMDEuOTUgODUuMzY4MkM5OS42MTQyIDkxLjk2MjQgOTUuMTQ0IDk3LjY3NSA4OS4xNzExIDEwMS40OThMODIuOTc0NyA4Ni40NjA2TDgyLjk5NDQgODYuNDM0M0M4Mi40MjUyIDg1LjEwNTUgODIuOTk0OCA4My41NDcyIDg0LjMwNDQgODIuOTEzNUM4NC42Mzk3IDgyLjc1MTMgODQuOTkgODIuNjYxNCA4NS4zMzkzIDgyLjY0MzdaTTU4LjUyOTggODIuNzA5NUM1OS43NDggODIuNzI2NyA2MC44NDA2IDgzLjU3NjEgNjEuMTIzNyA4NC44MjJDNjEuMjU2MiA4NS40MDUyIDYxLjE5MTcgODUuOTgzMSA2MC45NzMgODYuNDkzNUw2MS4wMTg5IDg2LjU1MjdMNTQuODg4IDEwMS40MzlDNDkuMTU1OSA5Ny43NDMyIDQ0LjU5MDQgOTIuMjA5OSA0Mi4xNDgxIDg1LjQyMDhMNTcuOTczMSA4Mi43MjI3TDU3Ljk5OTMgODIuNzU1NkM1OC4xNzYzIDgyLjcyMjkgNTguMzU1OCA4Mi43MDcxIDU4LjUyOTggODIuNzA5NVpNNzEuODk4NiA4OS4yMzEyQzcyLjMyMjkgODkuMjE1NSA3Mi43NTM0IDg5LjMwMyA3My4xNjI3IDg5LjUwMUM3My42OTkyIDg5Ljc2MDYgNzQuMTEzNiA5MC4xNjkyIDc0LjM3NDUgOTAuNjU5Mkg3NC40MzM0TDgyLjIzNDYgMTA0LjgyMUM4MS4yMjIxIDEwNS4xNjIgODAuMTgxMyAxMDUuNDU0IDc5LjExNjcgMTA1LjY5N0M3My4xNTA1IDEwNy4wNTggNjcuMjAzMiAxMDYuNjQ1IDYxLjgxOCAxMDQuODAyTDY5LjU5OTUgOTAuNjY1OEg2OS42MTI2QzcwLjA3OTUgODkuNzg4OCA3MC45NjUgODkuMjY1NiA3MS44OTg2IDg5LjIzMTJaIiBmaWxsPSJ3aGl0ZSIgc3Ryb2tlPSJ3aGl0ZSIgc3Ryb2tlLXdpZHRoPSIwLjI1Ii8+CjxkZWZzPgo8bGluZWFyR3JhZGllbnQgaWQ9InBhaW50MF9saW5lYXJfNjgxXzI4NDUiIHgxPSIxMCIgeTE9IjE1LjUiIHgyPSIxNDQiIHkyPSIxMzEuNSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiPgo8c3RvcCBzdG9wLWNvbG9yPSIjNEQ4N0ZGIi8+CjxzdG9wIG9mZnNldD0iMSIgc3RvcC1jb2xvcj0iIzA1NDdEMCIvPgo8L2xpbmVhckdyYWRpZW50Pgo8L2RlZnM+Cjwvc3ZnPgo=
-    keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "nodeGroups"], ["spec", "nodeGroups", "md0"], ["spec", "nodeGroups", "md0", "minReplicas"], ["spec", "nodeGroups", "md0", "maxReplicas"], ["spec", "nodeGroups", "md0", "instanceType"], ["spec", "nodeGroups", "md0", "ephemeralStorage"], ["spec", "nodeGroups", "md0", "roles"], ["spec", "nodeGroups", "md0", "resources"], ["spec", "nodeGroups", "md0", "gpus"], ["spec", "version"], ["spec", "host"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "fluxcd"], ["spec", "addons", "fluxcd", "enabled"], ["spec", "addons", "fluxcd", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"]]
+    keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "nodeGroups"], ["spec", "nodeGroups", "md0"], ["spec", "nodeGroups", "md0", "minReplicas"], ["spec", "nodeGroups", "md0", "maxReplicas"], ["spec", "nodeGroups", "md0", "instanceType"], ["spec", "nodeGroups", "md0", "ephemeralStorage"], ["spec", "nodeGroups", "md0", "roles"], ["spec", "nodeGroups", "md0", "resources"], ["spec", "nodeGroups", "md0", "gpus"], ["spec", "version"], ["spec", "host"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "fluxcd"], ["spec", "addons", "fluxcd", "enabled"], ["spec", "addons", "fluxcd", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"], ["spec", "images"], ["spec", "images", "waitForKubeconfig"]]
   secrets:
     exclude: []
     include:

From fb1ef59287114fe9ef6eec2351875139d4d84e97 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Mon, 27 Apr 2026 13:50:18 +0300
Subject: [PATCH 117/151] fix(kubernetes): drop undocumented status-beacon
 annotation from awaiting-etcd ConfigMap

The cozystack.io/status-beacon: "true" annotation had no consumer in
the chart, no documented contract, and no convention defined for other
charts to follow. It would have become accidental precedent for
contributors copying the pattern without understanding it.

The ConfigMap itself is self-explanatory: the name <release>-awaiting-etcd,
data.status: "awaiting-etcd", and the human-readable message in
data.message all surface the same operator signal via kubectl get cm.
Drop the annotation; keep the ConfigMap.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 packages/apps/kubernetes/templates/cluster.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/packages/apps/kubernetes/templates/cluster.yaml b/packages/apps/kubernetes/templates/cluster.yaml
index 12da9818..e1494e64 100644
--- a/packages/apps/kubernetes/templates/cluster.yaml
+++ b/packages/apps/kubernetes/templates/cluster.yaml
@@ -106,8 +106,6 @@ metadata:
     app.kubernetes.io/managed-by: {{ .Release.Service }}
     app.kubernetes.io/name: kubernetes
     app.kubernetes.io/instance: {{ .Release.Name }}
-  annotations:
-    cozystack.io/status-beacon: "true"
 data:
   status: "awaiting-etcd"
   message: |

From 7443e223453032d0a907311f3b5865889b713299 Mon Sep 17 00:00:00 2001
From: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
Date: Mon, 27 Apr 2026 17:18:45 +0500
Subject: [PATCH 118/151] build(linstor): include linstor-gui in root image
 build target

The linstor-gui package was added in #2382 with its own per-package
Makefile and Dockerfile, but the root Makefile's `build:` target was
not updated to invoke it. As a result `ghcr.io/cozystack/cozystack/
linstor-gui` has never been published (registry returns NAME_UNKNOWN)
and the chart's `image.tag` was never digest-pinned. Any cluster
deploying the chart hits ImagePullBackOff.

Wire the package into the root build alongside the other system
images. The next CI build will publish the image and the per-package
Makefile will rewrite values.yaml `image.repository`/`image.tag` to a
digest-pinned reference automatically.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
---
 Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Makefile b/Makefile
index 0e94c5e6..5041ac64 100644
--- a/Makefile
+++ b/Makefile
@@ -22,6 +22,7 @@ build: build-deps
 	make -C packages/system/lineage-controller-webhook image
 	make -C packages/system/cilium image
 	make -C packages/system/linstor image
+	make -C packages/system/linstor-gui image
 	make -C packages/system/kubeovn-webhook image
 	make -C packages/system/kubeovn-plunger image
 	make -C packages/system/dashboard image

From a9a66bf066360c340203508fd6072977ff4eb8ea Mon Sep 17 00:00:00 2001
From: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
Date: Mon, 27 Apr 2026 17:31:07 +0500
Subject: [PATCH 119/151] build(linstor): use shared BUILDX_ARGS for
 linstor-gui image build

The per-package Makefile added in #2382 hardcoded buildx flags
(--provenance, --builder, --platform=linux/amd64,linux/arm64,
--push, --load, --label) instead of using the shared $(BUILDX_ARGS)
macro from hack/common-envs.mk.

This broke CI: the runner's default docker driver does not support
multi-platform builds, and the hardcoded multi-arch platform list
crashed `make build` with "Multi-platform build is not supported for
the docker driver."

Replace the hardcoded flags with $(BUILDX_ARGS) to match every other
package (e.g. linstor, dashboard, cilium). $(BUILDX_ARGS) injects
--push, --load, --label, --provenance=false, and only sets --builder
or --platform when the operator explicitly exports BUILDER/PLATFORM.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
---
 packages/system/linstor-gui/Makefile | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/packages/system/linstor-gui/Makefile b/packages/system/linstor-gui/Makefile
index e202f4cc..b408491d 100644
--- a/packages/system/linstor-gui/Makefile
+++ b/packages/system/linstor-gui/Makefile
@@ -12,18 +12,13 @@ image: image-linstor-gui
 
 image-linstor-gui:
 	docker buildx build images/linstor-gui \
-		--provenance false \
-		--builder=$(BUILDER) \
-		--platform=linux/amd64,linux/arm64 \
 		--build-arg LINSTOR_GUI_VERSION=$(LINSTOR_GUI_VERSION) \
 		--tag $(REGISTRY)/linstor-gui:$(call settag,$(LINSTOR_GUI_VERSION)) \
 		--tag $(REGISTRY)/linstor-gui:$(call settag,$(LINSTOR_GUI_VERSION)-$(TAG)) \
 		--cache-from type=registry,ref=$(REGISTRY)/linstor-gui:latest \
 		--cache-to type=inline \
 		--metadata-file images/linstor-gui.json \
-		--push=$(PUSH) \
-		--label "org.opencontainers.image.source=https://github.com/cozystack/cozystack" \
-		--load=$(LOAD)
+		$(BUILDX_ARGS)
 	REPOSITORY="$(REGISTRY)/linstor-gui" \
 		yq -i '.image.repository = strenv(REPOSITORY)' values.yaml
 	TAG="$(call settag,$(LINSTOR_GUI_VERSION))@$$(yq e '."containerimage.digest"' images/linstor-gui.json -o json -r)" \

From 073fb1630d67ace6655deb262d1734fab98e8ce4 Mon Sep 17 00:00:00 2001
From: Andrei Kvapil <kvapss@gmail.com>
Date: Mon, 27 Apr 2026 11:32:17 +0200
Subject: [PATCH 120/151] feat(linstor): bump linstor-csi to v1.10.6 with
 Protocol-C dual-attach fix

Bump linstor-csi to v1.10.6 and add an out-of-tree patch that overrides
DrbdOptions/Net/protocol to C on the resource-definition whenever a
volume is attached to a second node with allow-two-primaries (the
live-migration flow), and reverts it on detach.

Without the patch, any KubeVirt live migration of a VM whose volume
sits on a Protocol-A/B resource (for example a "replicated-async"
StorageClass) ends up in a permanent evacuation loop: drbdadm adjust
on the satellite rejects allow-two-primaries with "Protocol C
required" (errno 139), the migration fails, and KubeVirt retries
until manual intervention.

The override is set at the resource-definition level so it covers
every connection, including diskless TieBreaker peers, and is tagged
with an Aux marker so Detach reverts only the override we installed.
The 001 relocate-after-clone patch was regenerated against v1.10.6
(context shift only, no logic change).

Verified end-to-end on dev5: live migration of a KubeVirt VM whose
PVC sits on a Protocol-A resource-group now succeeds in a single
Attach.

Patch is upstreamed as draft PR piraeusdatastore/linstor-csi#435.

Co-Authored-By: Claude <noreply@anthropic.com>
Signed-off-by: Andrei Kvapil <kvapss@gmail.com>
---
 packages/system/linstor/Makefile              |   2 +-
 .../linstor/images/linstor-csi/Dockerfile     |   2 +-
 .../001-relocate-after-clone-restore.diff     |  62 +-------
 ...2-protocol-c-override-for-dual-attach.diff | 132 ++++++++++++++++++
 4 files changed, 140 insertions(+), 58 deletions(-)
 create mode 100644 packages/system/linstor/images/linstor-csi/patches/002-protocol-c-override-for-dual-attach.diff

diff --git a/packages/system/linstor/Makefile b/packages/system/linstor/Makefile
index 2ad4def9..8d566678 100644
--- a/packages/system/linstor/Makefile
+++ b/packages/system/linstor/Makefile
@@ -5,7 +5,7 @@ include ../../../hack/common-envs.mk
 include ../../../hack/package.mk
 
 LINSTOR_VERSION ?= 1.33.2
-LINSTOR_CSI_VERSION ?= v1.10.5
+LINSTOR_CSI_VERSION ?= v1.10.6
 
 image: image-piraeus-server image-linstor-csi
 
diff --git a/packages/system/linstor/images/linstor-csi/Dockerfile b/packages/system/linstor/images/linstor-csi/Dockerfile
index 6cfd44aa..df5d179d 100644
--- a/packages/system/linstor/images/linstor-csi/Dockerfile
+++ b/packages/system/linstor/images/linstor-csi/Dockerfile
@@ -1,6 +1,6 @@
 FROM golang:1.25 AS builder
 
-ARG VERSION=v1.10.5
+ARG VERSION=v1.10.6
 ARG LINSTOR_WAIT_UNTIL_VERSION=v0.3.1
 ARG TARGETARCH
 ARG TARGETOS
diff --git a/packages/system/linstor/images/linstor-csi/patches/001-relocate-after-clone-restore.diff b/packages/system/linstor/images/linstor-csi/patches/001-relocate-after-clone-restore.diff
index ff0aec0c..0e272670 100644
--- a/packages/system/linstor/images/linstor-csi/patches/001-relocate-after-clone-restore.diff
+++ b/packages/system/linstor/images/linstor-csi/patches/001-relocate-after-clone-restore.diff
@@ -1,25 +1,7 @@
 diff --git a/pkg/client/linstor.go b/pkg/client/linstor.go
-index f544493..98e7fde 100644
+index ac4651d..46d565d 100644
 --- a/pkg/client/linstor.go
 +++ b/pkg/client/linstor.go
-@@ -181,7 +181,7 @@ func LogLevel(s string) func(*Linstor) error {
- func (s *Linstor) ListAllWithStatus(ctx context.Context) ([]volume.VolumeStatus, error) {
- 	var vols []volume.VolumeStatus
- 
--	resourcesByName := make(map[string][]lapi.Resource)
-+	resourcesByName := make(map[string][]lapi.ResourceWithVolumes)
- 
- 	resDefs, err := s.client.ResourceDefinitions.GetAll(ctx, lapi.RDGetAllRequest{WithVolumeDefinitions: true})
- 	if err != nil {
-@@ -194,7 +194,7 @@ func (s *Linstor) ListAllWithStatus(ctx context.Context) ([]volume.VolumeStatus,
- 	}
- 
- 	for i := range allResources {
--		resourcesByName[allResources[i].Name] = append(resourcesByName[allResources[i].Name], allResources[i].Resource)
-+		resourcesByName[allResources[i].Name] = append(resourcesByName[allResources[i].Name], allResources[i])
- 	}
- 
- 	for _, rd := range resDefs {
 @@ -462,6 +462,14 @@ func (s *Linstor) Clone(ctx context.Context, vol, src *volume.Info, params *volu
  		return err
  	}
@@ -35,7 +17,7 @@ index f544493..98e7fde 100644
  	logger.Debug("reconcile extra properties")
  
  	err = s.client.ResourceDefinitions.Modify(ctx, vol.ID, lapi.GenericPropsModify{OverrideProps: vol.Properties})
-@@ -1280,6 +1288,14 @@ func (s *Linstor) VolFromSnap(ctx context.Context, snap *volume.Snapshot, vol *v
+@@ -1297,6 +1305,14 @@ func (s *Linstor) VolFromSnap(ctx context.Context, snap *volume.Snapshot, vol *v
  		return err
  	}
  
@@ -50,7 +32,7 @@ index f544493..98e7fde 100644
  	logger.Debug("reconcile extra properties")
  
  	err = s.client.ResourceDefinitions.Modify(ctx, vol.ID, lapi.GenericPropsModify{OverrideProps: vol.Properties})
-@@ -1470,9 +1486,8 @@ func (s *Linstor) reconcileSnapshotResources(ctx context.Context, snapshot *volu
+@@ -1487,9 +1503,8 @@ func (s *Linstor) reconcileSnapshotResources(ctx context.Context, snapshot *volu
  		return fmt.Errorf("snapshot '%s' not deployed on any node", snap.Name)
  	}
  
@@ -62,7 +44,7 @@ index f544493..98e7fde 100644
  	for _, snapNode := range snap.Nodes {
  		if err := s.NodeAvailable(ctx, snapNode); err != nil {
  			logger.WithField("selected node candidate", snapNode).WithError(err).Debug("node is not available")
-@@ -1480,13 +1495,23 @@ func (s *Linstor) reconcileSnapshotResources(ctx context.Context, snapshot *volu
+@@ -1497,13 +1512,23 @@ func (s *Linstor) reconcileSnapshotResources(ctx context.Context, snapshot *volu
  		}
  
  		if slices.Contains(preferredNodes, snapNode) {
@@ -92,7 +74,7 @@ index f544493..98e7fde 100644
  	}
  
  	if selectedNode == "" {
-@@ -1679,6 +1704,114 @@ func (s *Linstor) reconcileResourcePlacement(ctx context.Context, vol *volume.In
+@@ -1696,6 +1721,114 @@ func (s *Linstor) reconcileResourcePlacement(ctx context.Context, vol *volume.In
  	return nil
  }
  
@@ -207,40 +189,8 @@ index f544493..98e7fde 100644
  // FindSnapsByID searches the snapshot in the backend
  func (s *Linstor) FindSnapsByID(ctx context.Context, id string) ([]*volume.Snapshot, error) {
  	snapshotId, err := volume.ParseSnapshotId(id)
-@@ -2173,7 +2306,7 @@ func (s *Linstor) Status(ctx context.Context, volId string) ([]string, *csi.Volu
- 		"volume": volId,
- 	}).Debug("getting assignments")
- 
--	ress, err := s.client.Resources.GetAll(ctx, volId)
-+	ress, err := s.client.Resources.GetResourceView(ctx, &lapi.ListOpts{Resource: []string{volId}})
- 	if err != nil {
- 		return nil, nil, fmt.Errorf("failed to list resources for '%s': %w", volId, err)
- 	}
-@@ -2261,13 +2394,20 @@ func GetSnapshotRemoteAndReadiness(snap *lapi.Snapshot) (string, bool, error) {
- 	return "", slices.Contains(snap.Flags, lapiconsts.FlagSuccessful), nil
- }
- 
--func NodesAndConditionFromResources(ress []lapi.Resource) ([]string, *csi.VolumeCondition) {
-+func NodesAndConditionFromResources(ress []lapi.ResourceWithVolumes) ([]string, *csi.VolumeCondition) {
- 	var allNodes, abnormalNodes []string
- 
- 	for i := range ress {
- 		res := &ress[i]
- 
--		allNodes = append(allNodes, res.NodeName)
-+		// A resource is a CSI publish target if any of its volumes were created
-+		// by ControllerPublishVolume, identified by the temporary-diskless-attach property.
-+		if slices.ContainsFunc(res.Volumes, func(v lapi.Volume) bool {
-+			createdFor, ok := v.Props[linstor.PropertyCreatedFor]
-+			return ok && createdFor == linstor.CreatedForTemporaryDisklessAttach
-+		}) {
-+			allNodes = append(allNodes, res.NodeName)
-+		}
- 
- 		if res.State == nil {
- 			abnormalNodes = append(abnormalNodes, res.NodeName)
 diff --git a/pkg/volume/parameter.go b/pkg/volume/parameter.go
-index 39acd95..aed18ab 100644
+index 39acd95..54d1dfb 100644
 --- a/pkg/volume/parameter.go
 +++ b/pkg/volume/parameter.go
 @@ -50,6 +50,7 @@ const (
diff --git a/packages/system/linstor/images/linstor-csi/patches/002-protocol-c-override-for-dual-attach.diff b/packages/system/linstor/images/linstor-csi/patches/002-protocol-c-override-for-dual-attach.diff
new file mode 100644
index 00000000..279c636b
--- /dev/null
+++ b/packages/system/linstor/images/linstor-csi/patches/002-protocol-c-override-for-dual-attach.diff
@@ -0,0 +1,132 @@
+diff --git a/pkg/client/linstor.go b/pkg/client/linstor.go
+index ac4651d..9d75c79 100644
+--- a/pkg/client/linstor.go
++++ b/pkg/client/linstor.go
+@@ -653,11 +653,35 @@ func (s *Linstor) Attach(ctx context.Context, volId, node string, rwxBlock bool)
+ 	}
+ 
+ 	if otherResInUse > 0 && rwxBlock {
+-		rdPropsModify := lapi.GenericPropsModify{OverrideProps: map[string]string{
++		rdProps := map[string]string{
+ 			linstor.PropertyAllowTwoPrimaries: "yes",
+-		}}
++		}
+ 
+-		err = s.client.ResourceDefinitions.Modify(ctx, volId, rdPropsModify)
++		// DRBD requires Protocol C whenever allow-two-primaries is enabled.
++		// If the resource is configured with Protocol A or B (typically
++		// through its resource-group), drbdadm adjust on the satellites
++		// fails with "Protocol C required" (errno 139) and live migration
++		// cannot proceed. Override Protocol to C on the resource-definition
++		// itself so it applies to every connection (including diskless
++		// peers, e.g. a TieBreaker). The marker lets Detach revert exactly
++		// the override we installed without disturbing operator-set props.
++		proto, protoErr := s.getEffectiveDrbdProtocol(ctx, volId)
++		if protoErr != nil {
++			s.log.WithError(protoErr).WithField("volume", volId).
++				Warn("failed to determine effective DRBD protocol; skipping Protocol=C override for dual-attach")
++		} else if proto == "A" || proto == "B" {
++			s.log.WithFields(logrus.Fields{
++				"volume":   volId,
++				"protocol": proto,
++			}).Info("installing Protocol=C override on resource-definition for dual-attach")
++
++			rdProps[linstor.PropertyDrbdNetProtocol] = "C"
++			rdProps[linstor.PropertyCsiProtocolOverride] = "yes"
++		}
++
++		err = s.client.ResourceDefinitions.Modify(ctx, volId, lapi.GenericPropsModify{
++			OverrideProps: rdProps,
++		})
+ 		if err != nil {
+ 			return "", err
+ 		}
+@@ -774,11 +798,26 @@ func (s *Linstor) Detach(ctx context.Context, volId, node string) error {
+ 	}
+ 
+ 	if resInUse == 1 {
+-		rdPropsModify := lapi.GenericPropsModify{DeleteProps: []string{
+-			linstor.PropertyAllowTwoPrimaries,
+-		}}
++		deleteProps := []string{linstor.PropertyAllowTwoPrimaries}
++
++		// Drop the Protocol=C override only if Attach installed it (gated by
++		// our marker), so an operator-set Protocol property on the
++		// resource-definition is preserved.
++		rd, getErr := s.client.ResourceDefinitions.Get(ctx, volId)
++		if getErr != nil {
++			log.WithError(getErr).Warn("failed to fetch resource-definition props; skipping Protocol=C override removal")
++		} else if rd.Props[linstor.PropertyCsiProtocolOverride] != "" {
++			log.Info("removing Protocol=C override from resource-definition")
+ 
+-		err = s.client.ResourceDefinitions.Modify(ctx, volId, rdPropsModify)
++			deleteProps = append(deleteProps,
++				linstor.PropertyDrbdNetProtocol,
++				linstor.PropertyCsiProtocolOverride,
++			)
++		}
++
++		err = s.client.ResourceDefinitions.Modify(ctx, volId, lapi.GenericPropsModify{
++			DeleteProps: deleteProps,
++		})
+ 		if err != nil {
+ 			return nil404(err)
+ 		}
+@@ -805,6 +844,33 @@ func (s *Linstor) Detach(ctx context.Context, volId, node string) error {
+ 	return nil404(s.client.Resources.Delete(ctx, volId, node))
+ }
+ 
++// getEffectiveDrbdProtocol returns the DRBD network protocol ("A", "B" or "C")
++// that LINSTOR will hand to drbdadm for the given resource. Resource-definition
++// properties take precedence; otherwise the value is inherited from the
++// resource-group. An empty string is returned when neither level sets the
++// property (LINSTOR's compiled-in default is "C").
++func (s *Linstor) getEffectiveDrbdProtocol(ctx context.Context, volId string) (string, error) {
++	rd, err := s.client.ResourceDefinitions.Get(ctx, volId)
++	if err != nil {
++		return "", err
++	}
++
++	if v, ok := rd.Props[linstor.PropertyDrbdNetProtocol]; ok {
++		return v, nil
++	}
++
++	if rd.ResourceGroupName == "" {
++		return "", nil
++	}
++
++	rg, err := s.client.ResourceGroups.Get(ctx, rd.ResourceGroupName)
++	if err != nil {
++		return "", err
++	}
++
++	return rg.Props[linstor.PropertyDrbdNetProtocol], nil
++}
++
+ // CapacityBytes returns the amount of free space in the storage pool specified by the params and topology.
+ func (s *Linstor) CapacityBytes(ctx context.Context, storagePools []string, overProvision *float64, segments map[string]string) (int64, error) {
+ 	log := s.log.WithField("storage-pools", storagePools).WithField("segments", segments)
+diff --git a/pkg/linstor/const.go b/pkg/linstor/const.go
+index 9a5f79c..c8bc9c3 100644
+--- a/pkg/linstor/const.go
++++ b/pkg/linstor/const.go
+@@ -44,6 +44,19 @@ const (
+ 	// PropertyAllowTwoPrimaries is DRBD option to allow second primary. Mainly used for live-migration.
+ 	PropertyAllowTwoPrimaries = lc.NamespcDrbdNetOptions + "/allow-two-primaries"
+ 
++	// PropertyDrbdNetProtocol is the DRBD network replication protocol option
++	// (A = async, B = semi-sync, C = sync). DRBD requires Protocol C whenever
++	// allow-two-primaries is enabled, otherwise drbdadm adjust fails with
++	// "Protocol C required" (errno 139).
++	PropertyDrbdNetProtocol = lc.NamespcDrbdNetOptions + "/protocol"
++
++	// PropertyCsiProtocolOverride marks a resource-connection where this driver
++	// has installed a temporary Protocol=C override to make a Protocol-A/B
++	// resource compatible with allow-two-primaries during live migration. The
++	// marker lets us safely remove only the overrides we set, without touching
++	// connection-level overrides installed by the operator.
++	PropertyCsiProtocolOverride = lc.NamespcAuxiliary + "/csi-protocol-override"
++
+ 	// CreatedForTemporaryDisklessAttach marks a resource as temporary, i.e. it should be removed after it is no longer
+ 	// needed.
+ 	CreatedForTemporaryDisklessAttach = "temporary-diskless-attach"

From 80631bc9160a4567cce8facdf781799f15455497 Mon Sep 17 00:00:00 2001
From: mattia-eleuteri <mattia@hidora.io>
Date: Mon, 27 Apr 2026 16:29:05 +0200
Subject: [PATCH 121/151] [vm-instance] Set wholeIP annotation conditionally on
 externalMethod
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Render `networking.cozystack.io/wholeIP: "false"` on the Service when
`externalMethod: PortList` is configured (was always `"true"` before).
Combined with cozy-proxy v0.3.0+ which adds per-port filtering for
"false"-annotated services, this makes `externalMethod: PortList`
behave as documented: only ports listed in `externalPorts` are
reachable from the LoadBalancer IP.

Backward-compatible: existing services with `externalMethod: WholeIP`
continue to set `wholeIP: "true"` and behave identically. cozy-proxy
versions older than v0.3.0 ignore Services with `wholeIP: "false"`,
which means PortList Services on older cozy-proxy will lose their
egress IP preservation — but that path was already non-functional
ingress-wise, so this is not a regression for users actually relying
on PortList.

Signed-off-by: mattia-eleuteri <mattia@hidora.io>
---
 docs/changelogs/v1.3.1.md                        | 11 +++++++++++
 packages/apps/vm-instance/templates/service.yaml |  2 +-
 2 files changed, 12 insertions(+), 1 deletion(-)
 create mode 100644 docs/changelogs/v1.3.1.md

diff --git a/docs/changelogs/v1.3.1.md b/docs/changelogs/v1.3.1.md
new file mode 100644
index 00000000..7f9ecfab
--- /dev/null
+++ b/docs/changelogs/v1.3.1.md
@@ -0,0 +1,11 @@
+<!--
+https://github.com/cozystack/cozystack/releases/tag/v1.3.1
+-->
+
+## Bug Fixes
+
+* **[vm-instance] Fix `externalMethod: PortList` not filtering ingress ports**: The `vm-instance` chart now sets `networking.cozystack.io/wholeIP: "false"` on the rendered Service when `externalMethod: PortList` is configured, signaling cozy-proxy to install per-port ingress filtering based on `Service.spec.ports`. Previously the annotation was always `"true"`, which kept cozy-proxy in whole-IP passthrough mode and made `PortList` non-functional. **Requires cozy-proxy v0.3.0 or later** (bundled with this chart). ([@mattia-eleuteri](https://github.com/mattia-eleuteri) in #<TBD>).
+
+---
+
+**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.3.0...v1.3.1
diff --git a/packages/apps/vm-instance/templates/service.yaml b/packages/apps/vm-instance/templates/service.yaml
index b7e3a420..4607ffcf 100644
--- a/packages/apps/vm-instance/templates/service.yaml
+++ b/packages/apps/vm-instance/templates/service.yaml
@@ -9,7 +9,7 @@ metadata:
 {{- if .Values.external }}
     service.kubernetes.io/service-proxy-name: "cozy-proxy"
   annotations:
-    networking.cozystack.io/wholeIP: "true"
+    networking.cozystack.io/wholeIP: {{ ternary "true" "false" (eq .Values.externalMethod "WholeIP") | quote }}
 {{- end }}
 spec:
   type: {{ ternary "LoadBalancer" "ClusterIP" .Values.external }}

From b0afc9a07c8a842915af1fa98ff1e7d17ccb39c8 Mon Sep 17 00:00:00 2001
From: mattia-eleuteri <mattia@hidora.io>
Date: Tue, 28 Apr 2026 08:37:13 +0200
Subject: [PATCH 122/151] [vm-instance] Add externalAllowICMP knob, drop in-PR
 changelog

- Add `externalAllowICMP` value (default true) propagated as
  `networking.cozystack.io/allowICMP` annotation on the rendered Service
  when `externalMethod: PortList`. The cozy-proxy companion (released as
  part of cozystack/cozy-proxy#11 + #12) drops ICMP by default in
  port-filter mode, which breaks ping and PMTU discovery; defaulting the
  chart to "true" preserves user expectations while still allowing
  operators to opt out by setting `externalAllowICMP: false`.

- Remove the v1.3.1.md changelog entry. Project convention is to add
  changelogs in a dedicated "docs: add changelog for vX.Y.Z" commit at
  release time, not as part of feature/fix PRs.

Signed-off-by: mattia-eleuteri <mattia@hidora.io>
---
 api/apps/v1alpha1/vminstance/types.go         |  3 ++
 docs/changelogs/v1.3.1.md                     | 11 ----
 packages/apps/vm-instance/README.md           | 51 ++++++++++---------
 .../apps/vm-instance/templates/service.yaml   |  3 ++
 packages/apps/vm-instance/values.schema.json  |  5 ++
 packages/apps/vm-instance/values.yaml         |  3 ++
 .../vm-instance-rd/cozyrds/vm-instance.yaml   |  4 +-
 7 files changed, 42 insertions(+), 38 deletions(-)
 delete mode 100644 docs/changelogs/v1.3.1.md

diff --git a/api/apps/v1alpha1/vminstance/types.go b/api/apps/v1alpha1/vminstance/types.go
index 2bb059c6..9c48724a 100644
--- a/api/apps/v1alpha1/vminstance/types.go
+++ b/api/apps/v1alpha1/vminstance/types.go
@@ -26,6 +26,9 @@ type ConfigSpec struct {
 	// Ports to forward from outside the cluster.
 	// +kubebuilder:default:={22}
 	ExternalPorts []int `json:"externalPorts,omitempty"`
+	// Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect.
+	// +kubebuilder:default:=true
+	ExternalAllowICMP bool `json:"externalAllowICMP"`
 	// Requested running state of the VirtualMachineInstance
 	// +kubebuilder:default:="Always"
 	RunStrategy RunStrategy `json:"runStrategy"`
diff --git a/docs/changelogs/v1.3.1.md b/docs/changelogs/v1.3.1.md
deleted file mode 100644
index 7f9ecfab..00000000
--- a/docs/changelogs/v1.3.1.md
+++ /dev/null
@@ -1,11 +0,0 @@
-<!--
-https://github.com/cozystack/cozystack/releases/tag/v1.3.1
--->
-
-## Bug Fixes
-
-* **[vm-instance] Fix `externalMethod: PortList` not filtering ingress ports**: The `vm-instance` chart now sets `networking.cozystack.io/wholeIP: "false"` on the rendered Service when `externalMethod: PortList` is configured, signaling cozy-proxy to install per-port ingress filtering based on `Service.spec.ports`. Previously the annotation was always `"true"`, which kept cozy-proxy in whole-IP passthrough mode and made `PortList` non-functional. **Requires cozy-proxy v0.3.0 or later** (bundled with this chart). ([@mattia-eleuteri](https://github.com/mattia-eleuteri) in #<TBD>).
-
----
-
-**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.3.0...v1.3.1
diff --git a/packages/apps/vm-instance/README.md b/packages/apps/vm-instance/README.md
index a2b6603e..9d6a52b2 100644
--- a/packages/apps/vm-instance/README.md
+++ b/packages/apps/vm-instance/README.md
@@ -36,31 +36,32 @@ virtctl ssh <user>@<vm>
 
 ### Common parameters
 
-| Name                | Description                                                                                                                       | Type       | Value       |
-| ------------------- | --------------------------------------------------------------------------------------------------------------------------------- | ---------- | ----------- |
-| `external`          | Enable external access from outside the cluster.                                                                                  | `bool`     | `false`     |
-| `externalMethod`    | Method to pass through traffic to the VM.                                                                                         | `string`   | `PortList`  |
-| `externalPorts`     | Ports to forward from outside the cluster.                                                                                        | `[]int`    | `[22]`      |
-| `runStrategy`       | Requested running state of the VirtualMachineInstance                                                                             | `string`   | `Always`    |
-| `instanceType`      | Virtual Machine instance type.                                                                                                    | `string`   | `u1.medium` |
-| `instanceProfile`   | Virtual Machine preferences profile.                                                                                              | `string`   | `ubuntu`    |
-| `disks`             | List of disks to attach.                                                                                                          | `[]object` | `[]`        |
-| `disks[i].name`     | Disk name.                                                                                                                        | `string`   | `""`        |
-| `disks[i].bus`      | Disk bus type (e.g. "sata").                                                                                                      | `string`   | `""`        |
-| `networks`          | Networks to attach the VM to.                                                                                                     | `[]object` | `[]`        |
-| `networks[i].name`  | Network attachment name.                                                                                                          | `string`   | `""`        |
-| `subnets`           | Deprecated: use networks instead.                                                                                                 | `[]object` | `[]`        |
-| `subnets[i].name`   | Network attachment name.                                                                                                          | `string`   | `""`        |
-| `gpus`              | List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).                                                               | `[]object` | `[]`        |
-| `gpus[i].name`      | The name of the GPU resource to attach.                                                                                           | `string`   | `""`        |
-| `cpuModel`          | Model specifies the CPU model inside the VMI. List of available models https://github.com/libvirt/libvirt/tree/master/src/cpu_map | `string`   | `""`        |
-| `resources`         | Resource configuration for the virtual machine.                                                                                   | `object`   | `{}`        |
-| `resources.cpu`     | Number of CPU cores allocated.                                                                                                    | `quantity` | `""`        |
-| `resources.memory`  | Amount of memory allocated.                                                                                                       | `quantity` | `""`        |
-| `resources.sockets` | Number of CPU sockets (vCPU topology).                                                                                            | `quantity` | `""`        |
-| `sshKeys`           | List of SSH public keys for authentication.                                                                                       | `[]string` | `[]`        |
-| `cloudInit`         | Cloud-init user data.                                                                                                             | `string`   | `""`        |
-| `cloudInitSeed`     | Seed string to generate SMBIOS UUID for the VM.                                                                                   | `string`   | `""`        |
+| Name                | Description                                                                                                                                                                                                   | Type       | Value       |
+| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ----------- |
+| `external`          | Enable external access from outside the cluster.                                                                                                                                                              | `bool`     | `false`     |
+| `externalMethod`    | Method to pass through traffic to the VM.                                                                                                                                                                     | `string`   | `PortList`  |
+| `externalPorts`     | Ports to forward from outside the cluster.                                                                                                                                                                    | `[]int`    | `[22]`      |
+| `externalAllowICMP` | Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect. | `bool`     | `true`      |
+| `runStrategy`       | Requested running state of the VirtualMachineInstance                                                                                                                                                         | `string`   | `Always`    |
+| `instanceType`      | Virtual Machine instance type.                                                                                                                                                                                | `string`   | `u1.medium` |
+| `instanceProfile`   | Virtual Machine preferences profile.                                                                                                                                                                          | `string`   | `ubuntu`    |
+| `disks`             | List of disks to attach.                                                                                                                                                                                      | `[]object` | `[]`        |
+| `disks[i].name`     | Disk name.                                                                                                                                                                                                    | `string`   | `""`        |
+| `disks[i].bus`      | Disk bus type (e.g. "sata").                                                                                                                                                                                  | `string`   | `""`        |
+| `networks`          | Networks to attach the VM to.                                                                                                                                                                                 | `[]object` | `[]`        |
+| `networks[i].name`  | Network attachment name.                                                                                                                                                                                      | `string`   | `""`        |
+| `subnets`           | Deprecated: use networks instead.                                                                                                                                                                             | `[]object` | `[]`        |
+| `subnets[i].name`   | Network attachment name.                                                                                                                                                                                      | `string`   | `""`        |
+| `gpus`              | List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).                                                                                                                                           | `[]object` | `[]`        |
+| `gpus[i].name`      | The name of the GPU resource to attach.                                                                                                                                                                       | `string`   | `""`        |
+| `cpuModel`          | Model specifies the CPU model inside the VMI. List of available models https://github.com/libvirt/libvirt/tree/master/src/cpu_map                                                                             | `string`   | `""`        |
+| `resources`         | Resource configuration for the virtual machine.                                                                                                                                                               | `object`   | `{}`        |
+| `resources.cpu`     | Number of CPU cores allocated.                                                                                                                                                                                | `quantity` | `""`        |
+| `resources.memory`  | Amount of memory allocated.                                                                                                                                                                                   | `quantity` | `""`        |
+| `resources.sockets` | Number of CPU sockets (vCPU topology).                                                                                                                                                                        | `quantity` | `""`        |
+| `sshKeys`           | List of SSH public keys for authentication.                                                                                                                                                                   | `[]string` | `[]`        |
+| `cloudInit`         | Cloud-init user data.                                                                                                                                                                                         | `string`   | `""`        |
+| `cloudInitSeed`     | Seed string to generate SMBIOS UUID for the VM.                                                                                                                                                               | `string`   | `""`        |
 
 
 ## U Series
diff --git a/packages/apps/vm-instance/templates/service.yaml b/packages/apps/vm-instance/templates/service.yaml
index 4607ffcf..cfeffa81 100644
--- a/packages/apps/vm-instance/templates/service.yaml
+++ b/packages/apps/vm-instance/templates/service.yaml
@@ -10,6 +10,9 @@ metadata:
     service.kubernetes.io/service-proxy-name: "cozy-proxy"
   annotations:
     networking.cozystack.io/wholeIP: {{ ternary "true" "false" (eq .Values.externalMethod "WholeIP") | quote }}
+    {{- if eq .Values.externalMethod "PortList" }}
+    networking.cozystack.io/allowICMP: {{ ternary "true" "false" (ne .Values.externalAllowICMP false) | quote }}
+    {{- end }}
 {{- end }}
 spec:
   type: {{ ternary "LoadBalancer" "ClusterIP" .Values.external }}
diff --git a/packages/apps/vm-instance/values.schema.json b/packages/apps/vm-instance/values.schema.json
index 34f7f634..01bd30a9 100644
--- a/packages/apps/vm-instance/values.schema.json
+++ b/packages/apps/vm-instance/values.schema.json
@@ -26,6 +26,11 @@
         "type": "integer"
       }
     },
+    "externalAllowICMP": {
+      "description": "Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect.",
+      "type": "boolean",
+      "default": true
+    },
     "runStrategy": {
       "description": "Requested running state of the VirtualMachineInstance",
       "type": "string",
diff --git a/packages/apps/vm-instance/values.yaml b/packages/apps/vm-instance/values.yaml
index f07b75d3..92e399c2 100644
--- a/packages/apps/vm-instance/values.yaml
+++ b/packages/apps/vm-instance/values.yaml
@@ -31,6 +31,9 @@ externalMethod: PortList
 externalPorts:
 - 22
 
+## @param {bool} externalAllowICMP - Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect.
+externalAllowICMP: true
+
 ## @enum {string} RunStrategy - Requested running state of the VirtualMachineInstance
 ## @value Always - VMI should always be running
 ## @value Halted - VMI should never be running
diff --git a/packages/system/vm-instance-rd/cozyrds/vm-instance.yaml b/packages/system/vm-instance-rd/cozyrds/vm-instance.yaml
index e93fa86b..e56bd9dd 100644
--- a/packages/system/vm-instance-rd/cozyrds/vm-instance.yaml
+++ b/packages/system/vm-instance-rd/cozyrds/vm-instance.yaml
@@ -8,7 +8,7 @@ spec:
     singular: vminstance
     plural: vminstances
     openAPISchema: |-
-      {"title":"Chart Values","type":"object","properties":{"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"externalMethod":{"description":"Method to pass through traffic to the VM.","type":"string","default":"PortList","enum":["PortList","WholeIP"]},"externalPorts":{"description":"Ports to forward from outside the cluster.","type":"array","default":[22],"items":{"type":"integer"}},"runStrategy":{"description":"Requested running state of the VirtualMachineInstance","type":"string","default":"Always","enum":["Always","Halted","Manual","RerunOnFailure","Once"]},"instanceType":{"description":"Virtual Machine instance type.","type":"string","default":"u1.medium"},"instanceProfile":{"description":"Virtual Machine preferences profile.","type":"string","default":"ubuntu","enum":["alpine","centos.7","centos.7.desktop","centos.stream10","centos.stream10.desktop","centos.stream8","centos.stream8.desktop","centos.stream8.dpdk","centos.stream9","centos.stream9.desktop","centos.stream9.dpdk","cirros","fedora","fedora.arm64","opensuse.leap","opensuse.tumbleweed","rhel.10","rhel.10.arm64","rhel.7","rhel.7.desktop","rhel.8","rhel.8.desktop","rhel.8.dpdk","rhel.9","rhel.9.arm64","rhel.9.desktop","rhel.9.dpdk","rhel.9.realtime","sles","ubuntu","windows.10","windows.10.virtio","windows.11","windows.11.virtio","windows.2k16","windows.2k16.virtio","windows.2k19","windows.2k19.virtio","windows.2k22","windows.2k22.virtio","windows.2k25","windows.2k25.virtio",""]},"disks":{"description":"List of disks to attach.","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"bus":{"description":"Disk bus type (e.g. \"sata\").","type":"string"},"name":{"description":"Disk name.","type":"string"}}}},"networks":{"description":"Networks to attach the VM to.","type":"array","default":[],"items":{"type":"object","properties":{"name":{"description":"Network attachment name.","type":"string"}}}},"subnets":{"description":"Deprecated: use networks instead.","type":"array","default":[],"items":{"type":"object","properties":{"name":{"description":"Network attachment name.","type":"string"}}}},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"name":{"description":"The name of the GPU resource to attach.","type":"string"}}}},"cpuModel":{"description":"Model specifies the CPU model inside the VMI. List of available models https://github.com/libvirt/libvirt/tree/master/src/cpu_map","type":"string","default":""},"resources":{"description":"Resource configuration for the virtual machine.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"sockets":{"description":"Number of CPU sockets (vCPU topology).","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"sshKeys":{"description":"List of SSH public keys for authentication.","type":"array","default":[],"items":{"type":"string"}},"cloudInit":{"description":"Cloud-init user data.","type":"string","default":""},"cloudInitSeed":{"description":"Seed string to generate SMBIOS UUID for the VM.","type":"string","default":""}}}
+      {"title":"Chart Values","type":"object","properties":{"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"externalMethod":{"description":"Method to pass through traffic to the VM.","type":"string","default":"PortList","enum":["PortList","WholeIP"]},"externalPorts":{"description":"Ports to forward from outside the cluster.","type":"array","default":[22],"items":{"type":"integer"}},"externalAllowICMP":{"description":"Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect.","type":"boolean","default":true},"runStrategy":{"description":"Requested running state of the VirtualMachineInstance","type":"string","default":"Always","enum":["Always","Halted","Manual","RerunOnFailure","Once"]},"instanceType":{"description":"Virtual Machine instance type.","type":"string","default":"u1.medium"},"instanceProfile":{"description":"Virtual Machine preferences profile.","type":"string","default":"ubuntu","enum":["alpine","centos.7","centos.7.desktop","centos.stream10","centos.stream10.desktop","centos.stream8","centos.stream8.desktop","centos.stream8.dpdk","centos.stream9","centos.stream9.desktop","centos.stream9.dpdk","cirros","fedora","fedora.arm64","opensuse.leap","opensuse.tumbleweed","rhel.10","rhel.10.arm64","rhel.7","rhel.7.desktop","rhel.8","rhel.8.desktop","rhel.8.dpdk","rhel.9","rhel.9.arm64","rhel.9.desktop","rhel.9.dpdk","rhel.9.realtime","sles","ubuntu","windows.10","windows.10.virtio","windows.11","windows.11.virtio","windows.2k16","windows.2k16.virtio","windows.2k19","windows.2k19.virtio","windows.2k22","windows.2k22.virtio","windows.2k25","windows.2k25.virtio",""]},"disks":{"description":"List of disks to attach.","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"bus":{"description":"Disk bus type (e.g. \"sata\").","type":"string"},"name":{"description":"Disk name.","type":"string"}}}},"networks":{"description":"Networks to attach the VM to.","type":"array","default":[],"items":{"type":"object","properties":{"name":{"description":"Network attachment name.","type":"string"}}}},"subnets":{"description":"Deprecated: use networks instead.","type":"array","default":[],"items":{"type":"object","properties":{"name":{"description":"Network attachment name.","type":"string"}}}},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"name":{"description":"The name of the GPU resource to attach.","type":"string"}}}},"cpuModel":{"description":"Model specifies the CPU model inside the VMI. List of available models https://github.com/libvirt/libvirt/tree/master/src/cpu_map","type":"string","default":""},"resources":{"description":"Resource configuration for the virtual machine.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"sockets":{"description":"Number of CPU sockets (vCPU topology).","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"sshKeys":{"description":"List of SSH public keys for authentication.","type":"array","default":[],"items":{"type":"string"}},"cloudInit":{"description":"Cloud-init user data.","type":"string","default":""},"cloudInitSeed":{"description":"Seed string to generate SMBIOS UUID for the VM.","type":"string","default":""}}}
   release:
     prefix: vm-instance-
     labels:
@@ -26,7 +26,7 @@ spec:
     tags:
       - compute
     icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODdfMzQ1NCkiLz4KPGcgY2xpcC1wYXRoPSJ1cmwoI2NsaXAwXzY4N18zNDU0KSI+CjxwYXRoIGQ9Ik04OS41MDM5IDExMS43MDdINTQuNDk3QzU0LjE3MjcgMTExLjcwNyA1NC4wMTA4IDExMS4yMjEgNTQuMzM0OSAxMTEuMDU5TDU3LjI1MjIgMTA4Ljk1MkM2MC4zMzE0IDEwNi42ODMgNjEuOTUyMiAxMDIuNjMxIDYwLjk3OTcgOTguNzQxMkg4My4wMjFDODIuMDQ4NSAxMDIuNjMxIDgzLjY2OTMgMTA2LjY4MyA4Ni43NDg1IDEwOC45NTJMODkuNjY1OCAxMTEuMDU5Qzg5Ljk5IDExMS4yMjEgODkuODI3OSAxMTEuNzA3IDg5LjUwMzkgMTExLjcwN1oiIGZpbGw9IiNCMEI2QkIiLz4KPHBhdGggZD0iTTExMy4zMjggOTguNzQxSDMwLjY3MjVDMjcuNTkzMSA5OC43NDEgMjUgOTYuMTQ4IDI1IDkzLjA2ODdWMzMuMTAzMkMyNSAzMC4wMjM5IDI3LjU5MzEgMjcuNDMwNyAzMC42NzI1IDI3LjQzMDdIMTEzLjMyOEMxMTYuNDA3IDI3LjQzMDcgMTE5IDMwLjAyMzcgMTE5IDMzLjEwMzJWOTMuMDY4N0MxMTkgOTYuMTQ4IDExNi40MDcgOTguNzQxIDExMy4zMjggOTguNzQxWiIgZmlsbD0iI0U4RURFRSIvPgo8cGF0aCBkPSJNMTE5IDg0LjE1NDlIMjVWMzMuMTAzMkMyNSAzMC4wMjM5IDI3LjU5MzEgMjcuNDMwNyAzMC42NzI1IDI3LjQzMDdIMTEzLjMyOEMxMTYuNDA3IDI3LjQzMDcgMTE5IDMwLjAyMzcgMTE5IDMzLjEwMzJMMTE5IDg0LjE1NDlaIiBmaWxsPSIjMDBCM0ZGIi8+CjxwYXRoIGQ9Ik05MC42Mzc0IDExNi41NjlINTMuMzYxNkM1Mi4wNjUxIDExNi41NjkgNTAuOTMwNyAxMTUuNDM1IDUwLjkzMDcgMTE0LjEzOEM1MC45MzA3IDExMi44NDEgNTIuMDY1MSAxMTEuNzA3IDUzLjM2MTYgMTExLjcwN0g5MC42Mzc0QzkxLjkzMzkgMTExLjcwNyA5My4wNjg0IDExMi44NDEgOTMuMDY4NCAxMTQuMTM4QzkzLjA2ODQgMTE1LjQzNSA5MS45MzM5IDExNi41NjkgOTAuNjM3NCAxMTYuNTY5WiIgZmlsbD0iI0U4RURFRSIvPgo8L2c+CjxwYXRoIGQ9Ik03Mi41Mjc1IDUzLjgzNjdDNzIuNDQzMSA1My44MzUxIDcyLjM2MDUgNTMuODEyMiA3Mi4yODczIDUzLjc3MDFMNTYuNDY5OSA0NC43OTM0QzU2LjM5ODMgNDQuNzUxOSA1Ni4zMzg4IDQ0LjY5MjMgNTYuMjk3MyA0NC42MjA3QzU2LjI1NTkgNDQuNTQ5IDU2LjIzMzggNDQuNDY3OCA1Ni4yMzM0IDQ0LjM4NUM1Ni4yMzM0IDQ0LjIxNjkgNTYuMzI1OCA0NC4wNjE3IDU2LjQ2OTkgNDMuOTc4NUw3Mi4xOTEyIDM1LjA2MDlDNzIuMjYzNyAzNS4wMjEgNzIuMzQ1IDM1IDcyLjQyNzcgMzVDNzIuNTEwNSAzNSA3Mi41OTE4IDM1LjAyMSA3Mi42NjQzIDM1LjA2MDlMODguNDg3MiA0NC4wMzk1Qzg4LjU1OTEgNDQuMDgwMSA4OC42MTg4IDQ0LjEzOTIgODguNjYgNDQuMjEwN0M4OC43MDEzIDQ0LjI4MjIgODguNzIyNyA0NC4zNjM1IDg4LjcyMTkgNDQuNDQ2Qzg4LjcyMjUgNDQuNTI4NSA4OC43MDEgNDQuNjA5NyA4OC42NTk4IDQ0LjY4MTJDODguNjE4NSA0NC43NTI2IDg4LjU1ODkgNDQuODExOCA4OC40ODcyIDQ0Ljg1MjVMNzIuNzcxNCA1My43NjgzQzcyLjY5NzIgNTMuODExNCA3Mi42MTMzIDUzLjgzNDkgNzIuNTI3NSA1My44MzY3IiBmaWxsPSJ3aGl0ZSIvPgo8cGF0aCBvcGFjaXR5PSIwLjciIGQ9Ik03MC4yNTUzIDc1LjY1MTdDNzAuMTcxIDc1LjY1MzUgNzAuMDg3OCA3NS42MzE3IDcwLjAxNTEgNzUuNTg4OEw1NC4yNDU4IDY2LjY0MTdDNTQuMTcxNSA2Ni42MDI0IDU0LjEwOTUgNjYuNTQzNiA1NC4wNjYxIDY2LjQ3MTZDNTQuMDIyOCA2Ni4zOTk3IDU0IDY2LjMxNzMgNTQgNjYuMjMzM1Y0OC4yNzhDNTQgNDguMTA4IDU0LjA5MjQgNDcuOTU0NiA1NC4yNDM5IDQ3Ljg2OTZDNTQuMzE3MiA0Ny44MjcxIDU0LjQwMDQgNDcuODA0NyA1NC40ODUxIDQ3LjgwNDdDNTQuNTY5NyA0Ny44MDQ3IDU0LjY1MjkgNDcuODI3MSA1NC43MjYyIDQ3Ljg2OTZMNzAuNDkzNyA1Ni44MTMxQzcwLjU2NDIgNTYuODU2NSA3MC42MjI1IDU2LjkxNyA3MC42NjMyIDU2Ljk4OTFDNzAuNzAzOSA1Ny4wNjEyIDcwLjcyNTcgNTcuMTQyNCA3MC43MjY1IDU3LjIyNTFWNzUuMTgwNUM3MC43MjU5IDc1LjI2MjggNzAuNzA0MiA3NS4zNDM2IDcwLjY2MzUgNzUuNDE1MUM3MC42MjI3IDc1LjQ4NjYgNzAuNTY0MiA3NS41NDY0IDcwLjQ5MzcgNzUuNTg4OEM3MC40MjA2IDc1LjYyOTEgNzAuMzM4NyA3NS42NTA3IDcwLjI1NTMgNzUuNjUxNyIgZmlsbD0id2hpdGUiLz4KPHBhdGggb3BhY2l0eT0iMC40IiBkPSJNNzQuNzE5OCA3NS42NTExQzc0LjYzMzMgNzUuNjUxMiA3NC41NDgyIDc1LjYyOTYgNzQuNDcyMiA3NS41ODgzQzc0LjQwMTYgNzUuNTQ2MSA3NC4zNDMyIDc1LjQ4NjIgNzQuMzAyNyA3NS40MTQ3Qzc0LjI2MjMgNzUuMzQzMSA3NC4yNDExIDc1LjI2MjIgNzQuMjQxMiA3NS4xOFY1Ny4zMzczQzc0LjI0MTIgNTcuMTcxIDc0LjMzMzYgNTcuMDE1OCA3NC40NzIyIDU2LjkyOUw5MC4yMzk3IDQ3Ljk4NTVDOTAuMzExOSA0Ny45NDM4IDkwLjM5MzggNDcuOTIxOSA5MC40NzcxIDQ3LjkyMTlDOTAuNTYwNSA0Ny45MjE5IDkwLjY0MjQgNDcuOTQzOCA5MC43MTQ2IDQ3Ljk4NTVDOTAuNzg3NiA0OC4wMjU1IDkwLjg0ODUgNDguMDg0MiA5MC44OTExIDQ4LjE1NTdDOTAuOTMzNyA0OC4yMjcyIDkwLjk1NjMgNDguMzA4OCA5MC45NTY2IDQ4LjM5MlY2Ni4yMzI4QzkwLjk1NyA2Ni4zMTY0IDkwLjkzNDcgNjYuMzk4NSA5MC44OTIxIDY2LjQ3MDRDOTAuODQ5NSA2Ni41NDI0IDkwLjc4ODEgNjYuNjAxNCA5MC43MTQ2IDY2LjY0MTFMNzQuOTUyNiA3NS41ODgzQzc0Ljg4MjUgNzUuNjMwNyA3NC44MDE4IDc1LjY1MjUgNzQuNzE5OCA3NS42NTExIiBmaWxsPSJ3aGl0ZSIvPgo8ZGVmcz4KPGxpbmVhckdyYWRpZW50IGlkPSJwYWludDBfbGluZWFyXzY4N18zNDU0IiB4MT0iMTYxIiB5MT0iMTgwIiB4Mj0iMy41OTI4NGUtMDciIHkyPSI0Ljk5OTk4IiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSI+CjxzdG9wLz4KPHN0b3Agb2Zmc2V0PSIxIiBzdG9wLWNvbG9yPSIjNTk1NjU2Ii8+CjwvbGluZWFyR3JhZGllbnQ+CjxjbGlwUGF0aCBpZD0iY2xpcDBfNjg3XzM0NTQiPgo8cmVjdCB3aWR0aD0iOTQiIGhlaWdodD0iOTQiIGZpbGw9IndoaXRlIiB0cmFuc2Zvcm09InRyYW5zbGF0ZSgyNSAyNSkiLz4KPC9jbGlwUGF0aD4KPC9kZWZzPgo8L3N2Zz4K
-    keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "external"], ["spec", "externalMethod"], ["spec", "externalPorts"], ["spec", "runStrategy"], ["spec", "instanceType"], ["spec", "instanceProfile"], ["spec", "disks"], ["spec", "networks"], ["spec", "subnets"], ["spec", "gpus"], ["spec", "cpuModel"], ["spec", "resources"], ["spec", "sshKeys"], ["spec", "cloudInit"], ["spec", "cloudInitSeed"]]
+    keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "external"], ["spec", "externalMethod"], ["spec", "externalPorts"], ["spec", "externalAllowICMP"], ["spec", "runStrategy"], ["spec", "instanceType"], ["spec", "instanceProfile"], ["spec", "disks"], ["spec", "networks"], ["spec", "subnets"], ["spec", "gpus"], ["spec", "cpuModel"], ["spec", "resources"], ["spec", "sshKeys"], ["spec", "cloudInit"], ["spec", "cloudInitSeed"]]
   secrets:
     exclude: []
     include: []

From bb51c88f7817c1d862fdb7947deef7798f717d82 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Tue, 28 Apr 2026 11:18:59 +0300
Subject: [PATCH 123/151] fix(monitoring): remove unused namespace variable
 from GPU efficiency dashboard

Address review feedback from coderabbitai on dashboards/gpu/gpu-efficiency.json:839

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 dashboards/gpu/gpu-efficiency.json | 24 ------------------------
 1 file changed, 24 deletions(-)

diff --git a/dashboards/gpu/gpu-efficiency.json b/dashboards/gpu/gpu-efficiency.json
index f351c964..28a24568 100644
--- a/dashboards/gpu/gpu-efficiency.json
+++ b/dashboards/gpu/gpu-efficiency.json
@@ -812,30 +812,6 @@
         "auto": false,
         "auto_min": "10s",
         "auto_count": 30
-      },
-      {
-        "type": "query",
-        "name": "namespace",
-        "label": "Namespace",
-        "skipUrlSync": false,
-        "query": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "$ds_prometheus"
-        },
-        "current": {
-          "selected": true,
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "allowCustomValue": true,
-        "refresh": 2,
-        "sort": 1,
-        "includeAll": true,
-        "auto": false,
-        "auto_min": "10s",
-        "auto_count": 30
       }
     ]
   },

From 452bff45675c55e0c8e01d290dfbb3b1e7f59196 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Tue, 28 Apr 2026 11:19:56 +0300
Subject: [PATCH 124/151] fix(monitoring): remove unused namespace variable
 from GPU performance dashboard

Address review feedback from coderabbitai on dashboards/gpu/gpu-performance.json:277

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 dashboards/gpu/gpu-performance.json | 24 ------------------------
 1 file changed, 24 deletions(-)

diff --git a/dashboards/gpu/gpu-performance.json b/dashboards/gpu/gpu-performance.json
index f9951577..aed9c69a 100644
--- a/dashboards/gpu/gpu-performance.json
+++ b/dashboards/gpu/gpu-performance.json
@@ -950,30 +950,6 @@
         "auto": false,
         "auto_min": "10s",
         "auto_count": 30
-      },
-      {
-        "type": "query",
-        "name": "namespace",
-        "label": "Namespace",
-        "skipUrlSync": false,
-        "query": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "$ds_prometheus"
-        },
-        "current": {
-          "selected": true,
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "allowCustomValue": true,
-        "refresh": 2,
-        "sort": 1,
-        "includeAll": true,
-        "auto": false,
-        "auto_min": "10s",
-        "auto_count": 30
       }
     ]
   },

From 4c697982b284a9d1f1b47d9cbbdbeab6314d253d Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Tue, 28 Apr 2026 11:20:19 +0300
Subject: [PATCH 125/151] fix(monitoring): use Hostname label in GPU tenants
 dashboard legends

Address review feedback from coderabbitai and gemini-code-assist on dashboards/gpu/gpu-tenants.json:532

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 dashboards/gpu/gpu-tenants.json | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/dashboards/gpu/gpu-tenants.json b/dashboards/gpu/gpu-tenants.json
index e2c400a8..6194a15e 100644
--- a/dashboards/gpu/gpu-tenants.json
+++ b/dashboards/gpu/gpu-tenants.json
@@ -529,7 +529,7 @@
       "targets": [
         {
           "expr": "node:gpu_util:avg",
-          "legendFormat": "{{node}}",
+          "legendFormat": "{{Hostname}}",
           "refId": "A"
         }
       ],
@@ -585,7 +585,7 @@
       "targets": [
         {
           "expr": "node:tensor_active:avg * 100",
-          "legendFormat": "{{node}}",
+          "legendFormat": "{{Hostname}}",
           "refId": "A"
         }
       ],
@@ -654,7 +654,7 @@
       "targets": [
         {
           "expr": "node:power_watts:sum",
-          "legendFormat": "{{node}}",
+          "legendFormat": "{{Hostname}}",
           "refId": "A"
         }
       ],
@@ -988,7 +988,7 @@
       "targets": [
         {
           "expr": "sum_over_time(node:power_watts:sum[24h:1m]) / 60 / 1000",
-          "legendFormat": "{{node}}",
+          "legendFormat": "{{Hostname}}",
           "refId": "A"
         }
       ],

From bbf338a57d4dca0286bef4dab991ccbce6a2caa2 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Tue, 28 Apr 2026 11:20:41 +0300
Subject: [PATCH 126/151] fix(gpu-operator): fail fast on missing artifacts in
 driver-compat example

Address review feedback from coderabbitai on packages/system/gpu-operator/examples/nvidia-driver-compat.yaml:77

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 .../examples/nvidia-driver-compat.yaml        | 23 +++++++++++--------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml b/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml
index 7706b221..fc9d5981 100644
--- a/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml
+++ b/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml
@@ -61,16 +61,21 @@ spec:
 
               mkdir -p "$DST/bin"
 
-              if [ -f "$GLIBC_LIB/libnvidia-ml.so.1" ]; then
-                cp -f "$GLIBC_LIB/libnvidia-ml.so.1" "$DST/libnvidia-ml.so.1"
-                echo "Copied libnvidia-ml.so.1"
-              fi
+              [ -f "$GLIBC_LIB/libnvidia-ml.so.1" ] || {
+                echo "missing $GLIBC_LIB/libnvidia-ml.so.1" >&2
+                exit 1
+              }
+              [ -f "$SRC_BIN/nvidia-smi" ] || {
+                echo "missing $SRC_BIN/nvidia-smi" >&2
+                exit 1
+              }
 
-              if [ -f "$SRC_BIN/nvidia-smi" ]; then
-                cp -f "$SRC_BIN/nvidia-smi" "$DST/bin/nvidia-smi"
-                chmod +x "$DST/bin/nvidia-smi"
-                echo "Copied nvidia-smi"
-              fi
+              cp -f "$GLIBC_LIB/libnvidia-ml.so.1" "$DST/libnvidia-ml.so.1"
+              echo "Copied libnvidia-ml.so.1"
+
+              cp -f "$SRC_BIN/nvidia-smi" "$DST/bin/nvidia-smi"
+              chmod +x "$DST/bin/nvidia-smi"
+              echo "Copied nvidia-smi"
 
               mkdir -p /host/run/nvidia/validations
               touch /host/run/nvidia/validations/.driver-ctr-ready

From 5718740ae3edddd9112e483426ebfebba423ed92 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Tue, 28 Apr 2026 11:20:57 +0300
Subject: [PATCH 127/151] docs(gpu-operator): correct gpu-quotas dashboard
 dependencies in README

Address review feedback from coderabbitai on packages/system/gpu-operator/examples/README.md:85

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 packages/system/gpu-operator/examples/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/system/gpu-operator/examples/README.md b/packages/system/gpu-operator/examples/README.md
index f9094549..b6f170c8 100644
--- a/packages/system/gpu-operator/examples/README.md
+++ b/packages/system/gpu-operator/examples/README.md
@@ -81,7 +81,7 @@ What each dashboard needs on top of the upstream DCGM Exporter
 | `gpu-performance` | Per-node, per-GPU deep dive        | `DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION`          |
 | `gpu-efficiency`  | Per-workload util vs tensor active | `DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION` (via `gpu:*_throttle_fraction:rate5m` recording rules) |
 | `gpu-fleet`       | Cluster-wide admin inventory       | `DCGM_FI_DEV_POWER_MGMT_LIMIT` (for the TDP vs draw panel)              |
-| `gpu-quotas`      | Kube-quota vs live usage           | `kube_pod_container_resource_requests`, `kube_pod_status_phase`, `DCGM_FI_DEV_GPU_UTIL` (via `namespace:gpu_count:allocated` / `cluster:gpu_count:*` recording rules) |
+| `gpu-quotas`      | Kube-quota vs live usage           | `kube_pod_container_resource_requests`, `kube_pod_status_phase`, `kube_node_status_allocatable` (via `namespace:gpu_count:allocated` / `cluster:gpu_count:*` recording rules) |
 | `gpu-tenants`     | Per-namespace tenant view          | nothing (works on default counters)                                     |
 
 `DCGM_FI_PROF_PIPE_TENSOR_ACTIVE` and `DCGM_FI_PROF_GR_ENGINE_ACTIVE`

From aba5ae3fcd0f4f4fdceae05f24e9371550076731 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Tue, 28 Apr 2026 11:21:12 +0300
Subject: [PATCH 128/151] fix(monitoring): prevent many-to-many match in
 util-per-watt recording rule

Address review feedback from coderabbitai on packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml:119

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 .../alerts/gpu-recording.rules.yaml                | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
index b046edf5..5e0f9ea0 100644
--- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
+++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml
@@ -109,13 +109,15 @@ spec:
         # to GPU-identifying labels.
         - record: gpu:util_per_watt:avg5m
           expr: |
-            avg by (Hostname, gpu, UUID) (
+            max by (Hostname, gpu, UUID) (
               avg_over_time(DCGM_FI_DEV_GPU_UTIL[5m])
-              / on (Hostname, gpu, UUID)
-              clamp_min(
-                avg_over_time(DCGM_FI_DEV_POWER_USAGE[5m]),
-                1
-              )
+            )
+            / on (Hostname, gpu, UUID)
+            clamp_min(
+              max by (Hostname, gpu, UUID) (
+                avg_over_time(DCGM_FI_DEV_POWER_USAGE[5m])
+              ),
+              1
             )
 
         # Fraction of time power-throttled (TDP cap) — 1.0 = fully throttled.

From 8e6266703da78b28109ec34ed6fd030612dc8c68 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Tue, 28 Apr 2026 11:43:30 +0300
Subject: [PATCH 129/151] Revert "chore: ignore CLAUDE.local.md"

This reverts commit 11f7d3589b3284fca96d0df21a02a5e967dadce7.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 .gitignore | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 64470222..61003de5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -84,4 +84,3 @@ tmp/
 # build revision marker (generated by make image-packages)
 packages/core/platform/.build-revision
 .claude/
-CLAUDE.local.md

From 31de9989f62b2f286fc3e6bf58ca85b1b96e54ad Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Tue, 28 Apr 2026 11:44:04 +0300
Subject: [PATCH 130/151] fix(gpu-operator): add DCGM_FI_DRIVER_VERSION to
 custom metrics CSV

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml b/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml
index 8e57788b..5b82fd44 100644
--- a/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml
+++ b/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml
@@ -16,6 +16,9 @@ data:
     # If line starts with a '#' it is considered a comment
     # DCGM FIELD, Prometheus metric type, help message
 
+    # Identity
+    DCGM_FI_DRIVER_VERSION, label, Driver version.
+
     # Clocks
     DCGM_FI_DEV_SM_CLOCK,  gauge, SM clock frequency (in MHz).
     DCGM_FI_DEV_MEM_CLOCK, gauge, Memory clock frequency (in MHz).

From cacd3714bd24d7a3ec7182b8ec7b1057041fcf0a Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Tue, 28 Apr 2026 11:44:08 +0300
Subject: [PATCH 131/151] fix(monitoring): include phase label in GPU limits
 query for consistency

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 dashboards/gpu/gpu-quotas.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json
index ce1e387e..62928b56 100644
--- a/dashboards/gpu/gpu-quotas.json
+++ b/dashboards/gpu/gpu-quotas.json
@@ -430,7 +430,7 @@
           "refId": "requested"
         },
         {
-          "expr": "kube_pod_container_resource_limits{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left() (kube_pod_status_phase{phase=~\"Running|Pending\"} == 1)",
+          "expr": "kube_pod_container_resource_limits{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left(phase) (kube_pod_status_phase{phase=~\"Running|Pending\"} == 1)",
           "instant": true,
           "range": false,
           "format": "table",

From b0784c0d33d55fefeaaf60cdeabca4bebec1c069 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Tue, 28 Apr 2026 11:44:11 +0300
Subject: [PATCH 132/151] fix(monitoring): generalize GPU temperature
 description in fleet dashboard

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 dashboards/gpu/gpu-fleet.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dashboards/gpu/gpu-fleet.json b/dashboards/gpu/gpu-fleet.json
index a7d3f085..60c12bfb 100644
--- a/dashboards/gpu/gpu-fleet.json
+++ b/dashboards/gpu/gpu-fleet.json
@@ -860,7 +860,7 @@
         }
       ],
       "title": "Max GPU temperature",
-      "description": "Hottest GPU in the fleet right now. A10 sustained target \u003c83°C.",
+      "description": "Hottest GPU in the fleet right now. Adjust thresholds to match your GPU model specifications.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",

From ed6f9bbd1da5d0daeea5fbe38e5aa42e3a528496 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Tue, 28 Apr 2026 12:02:18 +0300
Subject: [PATCH 133/151] fix(monitoring): regenerate gpu-quotas dashboard from
 SDK

Align dashboard JSON with the SDK source of truth.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 dashboards/gpu/gpu-quotas.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json
index 62928b56..ce1e387e 100644
--- a/dashboards/gpu/gpu-quotas.json
+++ b/dashboards/gpu/gpu-quotas.json
@@ -430,7 +430,7 @@
           "refId": "requested"
         },
         {
-          "expr": "kube_pod_container_resource_limits{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left(phase) (kube_pod_status_phase{phase=~\"Running|Pending\"} == 1)",
+          "expr": "kube_pod_container_resource_limits{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left() (kube_pod_status_phase{phase=~\"Running|Pending\"} == 1)",
           "instant": true,
           "range": false,
           "format": "table",

From 84f506116f452889011c9d31c0cd48628265d448 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Tue, 28 Apr 2026 12:07:31 +0300
Subject: [PATCH 134/151] docs(gpu-operator): clarify violation counter unit
 ambiguity in DCGM CSV

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 .../gpu-operator/examples/dcgm-custom-metrics.yaml   | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml b/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml
index 5b82fd44..ef2b0b88 100644
--- a/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml
+++ b/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml
@@ -66,12 +66,12 @@ data:
     DCGM_FI_DEV_ROW_REMAP_FAILURE,           gauge,   Whether remapping of rows has failed.
 
     # Throttle / violation counters (crucial for SLA and tenant monitoring)
-    DCGM_FI_DEV_POWER_VIOLATION,       counter, Throttling duration due to power constraints (in us).
-    DCGM_FI_DEV_THERMAL_VIOLATION,     counter, Throttling duration due to thermal constraints (in us).
-    DCGM_FI_DEV_SYNC_BOOST_VIOLATION,  counter, Throttling duration due to sync-boost constraints (in us).
-    DCGM_FI_DEV_BOARD_LIMIT_VIOLATION, counter, Throttling duration due to board limit constraints (in us).
-    DCGM_FI_DEV_LOW_UTIL_VIOLATION,    counter, Throttling duration due to low utilization (in us).
-    DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (in us).
+    DCGM_FI_DEV_POWER_VIOLATION,       counter, Throttling duration due to power constraints (us per docs; ns on DCGM 3.x).
+    DCGM_FI_DEV_THERMAL_VIOLATION,     counter, Throttling duration due to thermal constraints (us per docs; ns on DCGM 3.x).
+    DCGM_FI_DEV_SYNC_BOOST_VIOLATION,  counter, Throttling duration due to sync-boost constraints (us per docs; ns on DCGM 3.x).
+    DCGM_FI_DEV_BOARD_LIMIT_VIOLATION, counter, Throttling duration due to board limit constraints (us per docs; ns on DCGM 3.x).
+    DCGM_FI_DEV_LOW_UTIL_VIOLATION,    counter, Throttling duration due to low utilization (us per docs; ns on DCGM 3.x).
+    DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (us per docs; ns on DCGM 3.x).
 
     # NVLink — DCGM silently drops this metric on GPUs without NVLink, so
     # enabling it unconditionally is safe and keeps this file reusable

From 2a6653e11ae6a97d992f80e4ae0731995deea08a Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Tue, 28 Apr 2026 12:08:15 +0300
Subject: [PATCH 135/151] docs(monitoring): add panel descriptions to GPU
 quotas dashboard

Regenerated from SDK source.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 dashboards/gpu/gpu-quotas.json | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json
index ce1e387e..865fb95c 100644
--- a/dashboards/gpu/gpu-quotas.json
+++ b/dashboards/gpu/gpu-quotas.json
@@ -38,6 +38,7 @@
         }
       ],
       "title": "GPU allocatable",
+      "description": "Total GPU capacity the cluster can schedule to pods.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",
@@ -91,6 +92,7 @@
         }
       ],
       "title": "GPU requested",
+      "description": "Sum of GPU requests across all pods cluster-wide, including system namespaces.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",
@@ -148,6 +150,7 @@
         }
       ],
       "title": "Allocation ratio",
+      "description": "Percentage of allocatable GPUs currently requested by pods.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",
@@ -213,6 +216,7 @@
         }
       ],
       "title": "Pending pods (GPU)",
+      "description": "Pods requesting GPUs that are stuck in Pending state — indicates capacity shortage.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",
@@ -284,6 +288,7 @@
         }
       ],
       "title": "GPU requested per namespace",
+      "description": "GPU allocation breakdown by namespace — spot top consumers at a glance.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",
@@ -352,6 +357,7 @@
         }
       ],
       "title": "GPU allocated over time",
+      "description": "Requested vs allocatable GPUs over time — shows allocation pressure trends.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",
@@ -438,6 +444,7 @@
         }
       ],
       "title": "Pods requesting GPU",
+      "description": "Per-pod GPU requests and limits with scheduling status — Running or Pending.",
       "transparent": false,
       "datasource": {
         "type": "prometheus",

From d20285836cede23c3d7cfe091e5444359bac4320 Mon Sep 17 00:00:00 2001
From: Andrei Kvapil <kvapss@gmail.com>
Date: Tue, 28 Apr 2026 11:14:27 +0200
Subject: [PATCH 136/151] feat(cozy-proxy): bump to v0.3.0

Pulls in the per-port filtering and allowICMP support that the companion
vm-instance chart fix in #2501 relies on. cozy-proxy v0.3.0 also tightens
the selector to the standard service.kubernetes.io/service-proxy-name=cozy-proxy
label and switches the default ingress mode to port-filter; both are
already covered by the vm-instance chart (label landed in #2357,
wholeIP/allowICMP wired explicitly in #2501), so VM workloads upgrade
transparently.

Out-of-tree consumers using cozy-proxy annotations directly (without the
label, or relying on the absent-annotation passthrough default) are
called out in the upstream v0.3.0 release notes:
https://github.com/cozystack/cozy-proxy/releases/tag/v0.3.0

Signed-off-by: Andrei Kvapil <kvapss@gmail.com>
---
 packages/system/cozy-proxy/charts/cozy-proxy/Chart.yaml  | 4 ++--
 packages/system/cozy-proxy/charts/cozy-proxy/values.yaml | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/system/cozy-proxy/charts/cozy-proxy/Chart.yaml b/packages/system/cozy-proxy/charts/cozy-proxy/Chart.yaml
index 13352680..58f166f6 100644
--- a/packages/system/cozy-proxy/charts/cozy-proxy/Chart.yaml
+++ b/packages/system/cozy-proxy/charts/cozy-proxy/Chart.yaml
@@ -2,5 +2,5 @@ apiVersion: v2
 name: cozy-proxy
 description: A simple kube-proxy addon for 1:1 NAT services in Kubernetes using an NFT backend
 type: application
-version: 0.2.0
-appVersion: 0.2.0
+version: 0.3.0
+appVersion: 0.3.0
diff --git a/packages/system/cozy-proxy/charts/cozy-proxy/values.yaml b/packages/system/cozy-proxy/charts/cozy-proxy/values.yaml
index 8cde5bed..e143e926 100644
--- a/packages/system/cozy-proxy/charts/cozy-proxy/values.yaml
+++ b/packages/system/cozy-proxy/charts/cozy-proxy/values.yaml
@@ -1,6 +1,6 @@
 image:
   repository: ghcr.io/cozystack/cozystack/cozy-proxy
-  tag: v0.2.0
+  tag: v0.3.0
   pullPolicy: IfNotPresent
 
 daemonset:

From b2a8cca3bb8960ab2f058051deb03f0b7cab0dca Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Tue, 28 Apr 2026 12:15:32 +0300
Subject: [PATCH 137/151] fix(monitoring): filter zero-valued series from
 active tenants count

Address review feedback from coderabbitai on dashboards/gpu/gpu-tenants.json:39

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 dashboards/gpu/gpu-tenants.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dashboards/gpu/gpu-tenants.json b/dashboards/gpu/gpu-tenants.json
index 6194a15e..7fef321d 100644
--- a/dashboards/gpu/gpu-tenants.json
+++ b/dashboards/gpu/gpu-tenants.json
@@ -35,7 +35,7 @@
       "id": 2,
       "targets": [
         {
-          "expr": "count(namespace:gpu_count:allocated{namespace=~\"$namespace\"})",
+          "expr": "count(namespace:gpu_count:allocated{namespace=~\"$namespace\"} \u003e 0)",
           "refId": "A"
         }
       ],

From 7257b6aed4bb2b6499168eab467a400196f2e625 Mon Sep 17 00:00:00 2001
From: IvanHunters <xorokhotnikov@gmail.com>
Date: Tue, 28 Apr 2026 12:26:57 +0300
Subject: [PATCH 138/151] fix(api): address review feedback on TenantNamespace
 RBAC

Address review feedback from sircthulhu and CodeRabbit:

1. Return Forbidden instead of NotFound for unauthorized access
   - Get() now returns 403 Forbidden to follow standard K8s RBAC behavior
   - Previously returned 404 NotFound for security-by-obscurity
   - Updated test expectations to match new behavior

2. Propagate field and label selectors in Watch handler
   - Pass opts.FieldSelector and opts.LabelSelector to upstream Watch
   - Add defensive filtering before authorization to prevent RBAC bypass
   - Fixes potential issue with resourceNames restrictions

3. Refactor subject-matching logic to eliminate duplication
   - Extract matchesSubject() helper for Group/User/ServiceAccount checks
   - Remove duplicated code from filterAccessible and hasAccessToNamespace
   - Consolidates ServiceAccount namespace fallback logic

All tests pass successfully.

Signed-off-by: IvanHunters <xorokhotnikov@gmail.com>
---
 pkg/registry/core/tenantnamespace/rest.go     | 94 +++++++++++--------
 .../core/tenantnamespace/rest_test.go         |  6 +-
 2 files changed, 56 insertions(+), 44 deletions(-)

diff --git a/pkg/registry/core/tenantnamespace/rest.go b/pkg/registry/core/tenantnamespace/rest.go
index 7724c0e1..223a6925 100644
--- a/pkg/registry/core/tenantnamespace/rest.go
+++ b/pkg/registry/core/tenantnamespace/rest.go
@@ -17,6 +17,8 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metainternal "k8s.io/apimachinery/pkg/apis/meta/internalversion"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
@@ -130,8 +132,8 @@ func (r *REST) Get(
 		return nil, err
 	}
 	if !hasAccess {
-		// Return NotFound instead of Forbidden to prevent enumeration
-		return nil, apierrors.NewNotFound(r.gvr.GroupResource(), name)
+		// Return Forbidden to follow standard K8s RBAC behavior
+		return nil, apierrors.NewForbidden(r.gvr.GroupResource(), name, fmt.Errorf("access denied"))
 	}
 
 	ns := &corev1.Namespace{}
@@ -155,10 +157,20 @@ func (r *REST) Get(
 
 func (r *REST) Watch(ctx context.Context, opts *metainternal.ListOptions) (watch.Interface, error) {
 	nsList := &corev1.NamespaceList{}
-	nsWatch, err := r.w.Watch(ctx, nsList, &client.ListOptions{Raw: &metav1.ListOptions{
+
+	// Build upstream watch options with field and label selectors
+	rawOpts := &metav1.ListOptions{
 		Watch:           true,
 		ResourceVersion: opts.ResourceVersion,
-	}})
+	}
+	if opts.FieldSelector != nil {
+		rawOpts.FieldSelector = opts.FieldSelector.String()
+	}
+	if opts.LabelSelector != nil {
+		rawOpts.LabelSelector = opts.LabelSelector.String()
+	}
+
+	nsWatch, err := r.w.Watch(ctx, nsList, &client.ListOptions{Raw: rawOpts})
 	if err != nil {
 		return nil, err
 	}
@@ -200,6 +212,18 @@ func (r *REST) Watch(ctx context.Context, opts *metainternal.ListOptions) (watch
 				continue
 			}
 
+			// Apply defensive filtering for field and label selectors
+			if opts.FieldSelector != nil {
+				if !opts.FieldSelector.Matches(fields.Set{"metadata.name": ns.Name}) {
+					continue
+				}
+			}
+			if opts.LabelSelector != nil {
+				if !opts.LabelSelector.Matches(labels.Set(ns.Labels)) {
+					continue
+				}
+			}
+
 			// Check if user has access to this namespace
 			hasAccess, err := r.hasAccessToNamespace(ctx, ns.Name)
 			if err != nil {
@@ -331,6 +355,26 @@ func (r *REST) makeList(src *corev1.NamespaceList, allowed []string) *corev1alph
 	return out
 }
 
+// matchesSubject checks if a RoleBinding subject matches the user's identity.
+// It handles Group, User, and ServiceAccount subjects with proper namespace fallback.
+func matchesSubject(subj rbacv1.Subject, bindingNamespace, username string, groups map[string]struct{}) bool {
+	switch subj.Kind {
+	case "Group":
+		_, ok := groups[subj.Name]
+		return ok
+	case "User":
+		return subj.Name == username
+	case "ServiceAccount":
+		saNamespace := subj.Namespace
+		if saNamespace == "" {
+			saNamespace = bindingNamespace
+		}
+		return username == fmt.Sprintf("system:serviceaccount:%s:%s", saNamespace, subj.Name)
+	default:
+		return false
+	}
+}
+
 func (r *REST) filterAccessible(
 	ctx context.Context,
 	names []string,
@@ -369,26 +413,9 @@ func (r *REST) filterAccessible(
 	subjectLoop:
 		for j := range rbs.Items[i].Subjects {
 			subj := rbs.Items[i].Subjects[j]
-			switch subj.Kind {
-			case "Group":
-				if _, ok = groups[subj.Name]; ok {
-					allowedNameSet[rbs.Items[i].Namespace] = struct{}{}
-					break subjectLoop
-				}
-			case "User":
-				if subj.Name == u.GetName() {
-					allowedNameSet[rbs.Items[i].Namespace] = struct{}{}
-					break subjectLoop
-				}
-			case "ServiceAccount":
-				saNamespace := subj.Namespace
-				if saNamespace == "" {
-					saNamespace = rbs.Items[i].Namespace
-				}
-				if u.GetName() == fmt.Sprintf("system:serviceaccount:%s:%s", saNamespace, subj.Name) {
-					allowedNameSet[rbs.Items[i].Namespace] = struct{}{}
-					break subjectLoop
-				}
+			if matchesSubject(subj, rbs.Items[i].Namespace, u.GetName(), groups) {
+				allowedNameSet[rbs.Items[i].Namespace] = struct{}{}
+				break subjectLoop
 			}
 		}
 	}
@@ -434,23 +461,8 @@ func (r *REST) hasAccessToNamespace(
 	for i := range rbs.Items {
 		for j := range rbs.Items[i].Subjects {
 			subj := rbs.Items[i].Subjects[j]
-			switch subj.Kind {
-			case "Group":
-				if _, ok := groups[subj.Name]; ok {
-					return true, nil
-				}
-			case "User":
-				if subj.Name == u.GetName() {
-					return true, nil
-				}
-			case "ServiceAccount":
-				saNamespace := subj.Namespace
-				if saNamespace == "" {
-					saNamespace = rbs.Items[i].Namespace
-				}
-				if u.GetName() == fmt.Sprintf("system:serviceaccount:%s:%s", saNamespace, subj.Name) {
-					return true, nil
-				}
+			if matchesSubject(subj, rbs.Items[i].Namespace, u.GetName(), groups) {
+				return true, nil
 			}
 		}
 	}
diff --git a/pkg/registry/core/tenantnamespace/rest_test.go b/pkg/registry/core/tenantnamespace/rest_test.go
index eb678949..e1fb1e85 100644
--- a/pkg/registry/core/tenantnamespace/rest_test.go
+++ b/pkg/registry/core/tenantnamespace/rest_test.go
@@ -462,9 +462,9 @@ func TestGet_WithoutAccess(t *testing.T) {
 		t.Errorf("expected nil object, got %v", obj)
 	}
 
-	// Verify it returns NotFound (not Forbidden) to prevent enumeration
-	if !apierrors.IsNotFound(err) {
-		t.Errorf("expected NotFound error, got %v", err)
+	// Verify it returns Forbidden to follow standard K8s RBAC behavior
+	if !apierrors.IsForbidden(err) {
+		t.Errorf("expected Forbidden error, got %v", err)
 	}
 }
 

From 61ed7ad89c0bff66236e6884e6b6cd6038383718 Mon Sep 17 00:00:00 2001
From: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
Date: Tue, 28 Apr 2026 16:33:37 +0500
Subject: [PATCH 139/151] fix(api): address review feedback on TenantNamespace
 watch path

- Hoist user identity extraction out of the Watch goroutine; reuse a
  cached username and groups map across events instead of re-fetching
  them per event. Watch now returns Unauthorized up front when no user
  is present in the context, rather than failing silently per event.
- Switch the per-event access-check error log to structured klog.ErrorS
  to comply with the project Go style guide.
- Strengthen TestGet_WithAccess to assert the concrete *TenantNamespace
  type plus Name, Kind, and APIVersion, so type or metadata regressions
  fail fast.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
---
 pkg/registry/core/tenantnamespace/rest.go     | 35 +++++++++++++++----
 .../core/tenantnamespace/rest_test.go         | 16 +++++++--
 2 files changed, 43 insertions(+), 8 deletions(-)

diff --git a/pkg/registry/core/tenantnamespace/rest.go b/pkg/registry/core/tenantnamespace/rest.go
index 223a6925..4ee97eb0 100644
--- a/pkg/registry/core/tenantnamespace/rest.go
+++ b/pkg/registry/core/tenantnamespace/rest.go
@@ -156,6 +156,18 @@ func (r *REST) Get(
 // -----------------------------------------------------------------------------
 
 func (r *REST) Watch(ctx context.Context, opts *metainternal.ListOptions) (watch.Interface, error) {
+	// Extract user identity once for the lifetime of the watch — it does not
+	// change between events and rebuilding it per event is wasteful.
+	u, ok := request.UserFrom(ctx)
+	if !ok {
+		return nil, apierrors.NewUnauthorized("user missing in context")
+	}
+	username := u.GetName()
+	groups := make(map[string]struct{})
+	for _, group := range u.GetGroups() {
+		groups[group] = struct{}{}
+	}
+
 	nsList := &corev1.NamespaceList{}
 
 	// Build upstream watch options with field and label selectors
@@ -224,10 +236,11 @@ func (r *REST) Watch(ctx context.Context, opts *metainternal.ListOptions) (watch
 				}
 			}
 
-			// Check if user has access to this namespace
-			hasAccess, err := r.hasAccessToNamespace(ctx, ns.Name)
+			// Check if user has access to this namespace using the cached
+			// identity — avoids re-extracting user/groups on every event.
+			hasAccess, err := r.hasAccessToNamespaceForUser(ctx, ns.Name, username, groups)
 			if err != nil {
-				klog.Errorf("Failed to check access for namespace %s in watch: %v", ns.Name, err)
+				klog.ErrorS(err, "Failed to check access for namespace in watch", "namespace", ns.Name)
 				continue
 			}
 			if !hasAccess {
@@ -437,12 +450,22 @@ func (r *REST) hasAccessToNamespace(
 	if !ok {
 		return false, fmt.Errorf("user missing in context")
 	}
-
-	// Check privileged groups
 	groups := make(map[string]struct{})
 	for _, group := range u.GetGroups() {
 		groups[group] = struct{}{}
 	}
+	return r.hasAccessToNamespaceForUser(ctx, namespace, u.GetName(), groups)
+}
+
+// hasAccessToNamespaceForUser is the inner check that does not re-extract user
+// identity from context. Use this in hot paths (e.g. the Watch loop) where the
+// caller has already cached the user name and groups.
+func (r *REST) hasAccessToNamespaceForUser(
+	ctx context.Context,
+	namespace, username string,
+	groups map[string]struct{},
+) (bool, error) {
+	// Check privileged groups
 	if _, ok := groups["system:masters"]; ok {
 		return true, nil
 	}
@@ -461,7 +484,7 @@ func (r *REST) hasAccessToNamespace(
 	for i := range rbs.Items {
 		for j := range rbs.Items[i].Subjects {
 			subj := rbs.Items[i].Subjects[j]
-			if matchesSubject(subj, rbs.Items[i].Namespace, u.GetName(), groups) {
+			if matchesSubject(subj, rbs.Items[i].Namespace, username, groups) {
 				return true, nil
 			}
 		}
diff --git a/pkg/registry/core/tenantnamespace/rest_test.go b/pkg/registry/core/tenantnamespace/rest_test.go
index e1fb1e85..52d2a075 100644
--- a/pkg/registry/core/tenantnamespace/rest_test.go
+++ b/pkg/registry/core/tenantnamespace/rest_test.go
@@ -15,6 +15,8 @@ import (
 	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/apiserver/pkg/endpoints/request"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	corev1alpha1 "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1"
 )
 
 func TestMakeListSortsAlphabetically(t *testing.T) {
@@ -408,8 +410,18 @@ func TestGet_WithAccess(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
-	if obj == nil {
-		t.Fatal("expected object, got nil")
+	tn, ok := obj.(*corev1alpha1.TenantNamespace)
+	if !ok {
+		t.Fatalf("expected *TenantNamespace, got %T", obj)
+	}
+	if tn.Name != "tenant-test" {
+		t.Errorf("expected name %q, got %q", "tenant-test", tn.Name)
+	}
+	if tn.Kind != "TenantNamespace" {
+		t.Errorf("expected Kind=TenantNamespace, got %q", tn.Kind)
+	}
+	if tn.APIVersion != corev1alpha1.SchemeGroupVersion.String() {
+		t.Errorf("expected APIVersion=%q, got %q", corev1alpha1.SchemeGroupVersion.String(), tn.APIVersion)
 	}
 }
 

From 1dc02d44f4ea15253ffd799e85a54ed79ae87fd1 Mon Sep 17 00:00:00 2001
From: Timofei Larkin <lllamnyp@gmail.com>
Date: Mon, 27 Apr 2026 10:27:46 +0300
Subject: [PATCH 140/151] refactor(lineage-controller-webhook): align with
 cozystack-api shape
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Collapse the chart to a single Deployment shape that mirrors
cozystack-api: 2 replicas, soft (preferred) nodeAffinity to
node-role.kubernetes.io/control-plane via Exists, permissive
tolerations, soft podAntiAffinity on hostname, an unconditional
PodDisruptionBudget with maxUnavailable: 1, and a Service with
spec.trafficDistribution: PreferClose. The same shape works on
Talos / kubeadm / k3s and on managed Kubernetes / Cozy-in-Cozy
tenant clusters without per-distro overrides — fixing #2417 by
making the soft control-plane affinity gracefully fall back to
worker scheduling when no control-plane nodes are visible.

Drop the DaemonSet path entirely. The previous PR's deployment.enabled
toggle, the workload-kind switch in templates/workload.yaml, the
fail-guard for localK8sAPIEndpoint+nodeAffinity=[], the conditional
PDB, and the values-shape knobs for nodeAffinity and tolerations all
go away as a consequence. Override the standard Deployment fields
through the usual component-values mechanism if a non-default topology
is ever needed.

Mark localK8sAPIEndpoint.enabled deprecated and flip the default to
false. The flag injects KUBERNETES_SERVICE_HOST=status.hostIP, which
is only valid when the pod is actually scheduled on an apiserver-
bearing node. With the new soft control-plane affinity, the pod can
land off-control-plane and crash-loop. The latency motivation for the
flag is real but pending separate webhook performance work; once
addressed, the flag can be removed.

Revert all changes to packages/core/platform/images/migrations/migrations/20.
The earlier ds/...-or-deploy/... fallback was over-engineered: per
run-migrations.sh, migration 20 fires only when CURRENT < 20 (i.e.
direct upgrades from pre-0.37 to 1.3+), and that path is unsupported
anyway. The original ds/... rollout-status line is dead code on every
supported install and upgrade path.

Add helm-unittest coverage for: the default Deployment shape (replicas,
soft nodeAffinity, soft podAntiAffinity, tolerations); no env vars at
the default localK8sAPIEndpoint.enabled=false; PDB rendered
unconditionally with maxUnavailable: 1, including at replicas=1 (the
no-op case); replicas value drives spec.replicas; and that
localK8sAPIEndpoint.enabled=true does inject the env vars. The Service
test continues to assert trafficDistribution: PreferClose with no
internalTrafficPolicy.

Add a slim README documenting the topology, parameters, and the
deprecation note for localK8sAPIEndpoint.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
Signed-off-by: Timofei Larkin <lllamnyp@gmail.com>
---
 .../platform/templates/bundles/system.yaml    |  3 -
 .../lineage-controller-webhook/Makefile       |  5 +
 .../lineage-controller-webhook/README.md      | 67 +++++++++++++
 .../templates/deployment.yaml                 | 49 ---------
 .../templates/pdb.yaml                        |  4 +-
 .../templates/service.yaml                    |  2 +-
 .../{daemonset.yaml => workload.yaml}         | 27 +++--
 .../tests/service_test.yaml                   | 30 ++++++
 .../tests/workload_test.yaml                  | 99 +++++++++++++++++++
 .../lineage-controller-webhook/values.yaml    | 18 ++--
 10 files changed, 230 insertions(+), 74 deletions(-)
 create mode 100644 packages/system/lineage-controller-webhook/README.md
 delete mode 100644 packages/system/lineage-controller-webhook/templates/deployment.yaml
 rename packages/system/lineage-controller-webhook/templates/{daemonset.yaml => workload.yaml} (67%)
 create mode 100644 packages/system/lineage-controller-webhook/tests/service_test.yaml
 create mode 100644 packages/system/lineage-controller-webhook/tests/workload_test.yaml

diff --git a/packages/core/platform/templates/bundles/system.yaml b/packages/core/platform/templates/bundles/system.yaml
index 43ea7266..6c587c82 100644
--- a/packages/core/platform/templates/bundles/system.yaml
+++ b/packages/core/platform/templates/bundles/system.yaml
@@ -105,9 +105,6 @@
 {{- /* cozystack-api DaemonSet */ -}}
 {{- $apiValues := dict "cozystackAPI" (dict "nodeSelector" $genericNodeSelector) -}}
 {{- $_ := set $cozystackEngineComponents "cozystack-api" (dict "values" $apiValues) -}}
-{{- /* lineage-controller-webhook DaemonSet */ -}}
-{{- $lineageValues := dict "lineageControllerWebhook" (dict "nodeSelector" $genericNodeSelector) -}}
-{{- $_ := set $cozystackEngineComponents "lineage-controller-webhook" (dict "values" $lineageValues) -}}
 {{- end -}}
 {{- if .Values.authentication.oidc.enabled }}
 {{include "cozystack.platform.package" (list "cozystack.cozystack-engine" "oidc" $ $cozystackEngineComponents) }}
diff --git a/packages/system/lineage-controller-webhook/Makefile b/packages/system/lineage-controller-webhook/Makefile
index d5bada31..cdc1cf01 100644
--- a/packages/system/lineage-controller-webhook/Makefile
+++ b/packages/system/lineage-controller-webhook/Makefile
@@ -4,8 +4,13 @@ NAMESPACE=cozy-system
 include ../../../hack/common-envs.mk
 include ../../../hack/package.mk
 
+.PHONY: test
+
 image: image-lineage-controller-webhook
 
+test:
+	helm unittest .
+
 image-lineage-controller-webhook:
 	docker buildx build -f images/lineage-controller-webhook/Dockerfile ../../.. \
 		--tag $(REGISTRY)/lineage-controller-webhook:$(call settag,$(TAG)) \
diff --git a/packages/system/lineage-controller-webhook/README.md b/packages/system/lineage-controller-webhook/README.md
new file mode 100644
index 00000000..1e43c9c8
--- /dev/null
+++ b/packages/system/lineage-controller-webhook/README.md
@@ -0,0 +1,67 @@
+# lineage-controller-webhook
+
+Cozystack system package for the **lineage controller webhook** — a mutating
+admission webhook that stamps "lineage" labels onto tenant workloads, linking
+each resource back to the Cozystack `Application` that ultimately owns it.
+
+The webhook intercepts CREATE and UPDATE on `pods`, `secrets`, `services`,
+`persistentvolumeclaims`, `ingresses.networking.k8s.io`, and
+`workloadmonitors.cozystack.io` outside system namespaces. For each request it
+walks the ownership graph upward (Kubernetes `ownerReferences`, then the
+`HelmRelease` Flux installed the resource with, then the Cozystack
+`Application`-derived CRD that produced the HelmRelease) and writes the
+discovered application's group, kind and name as labels
+(`apps.cozystack.io/application.{group,kind,name}`) on the incoming object.
+Those labels let the aggregated API server, the Cozystack dashboard, and other
+lineage-aware consumers reason about which application a resource belongs to.
+
+The webhook serves TLS on port 9443 with a cert-manager issued certificate, and
+runs with `failurePolicy: Fail` — every CREATE/UPDATE on the resources above
+is gated on the webhook being reachable.
+
+## Topology
+
+The chart ships a single shape, modelled on `cozystack-api`:
+
+- **Deployment** with two replicas (override via `replicas`).
+- **Soft `nodeAffinity`** preferring `node-role.kubernetes.io/control-plane`
+  (`Exists`, so it matches both Talos's empty value and k3s/kubeadm's `"true"`).
+  Soft means: the pod lands on a control-plane node when one is reachable, and
+  on any worker otherwise — no override needed for managed Kubernetes,
+  Cozy-in-Cozy tenants, or any other cluster where control-plane nodes aren't
+  visible.
+- **Permissive `tolerations`** (`[{operator: Exists}]`) so the pod can land on
+  tainted control-plane nodes when the soft affinity is satisfiable.
+- **Soft `podAntiAffinity`** on `kubernetes.io/hostname` so replicas spread
+  across nodes when possible (best-effort).
+- **`PodDisruptionBudget`** with `maxUnavailable: 1`, unconditional. At
+  `replicas: 1` it's a useful no-op (allows full drain); at `replicas >= 2`
+  it caps disruption to one pod.
+- **`Service` with `spec.trafficDistribution: PreferClose`** so the apiserver
+  prefers a webhook endpoint on its own node when one exists, and transparently
+  falls over to a remote endpoint otherwise. Requires Kubernetes ≥ 1.31; on
+  older clusters the field is silently ignored and traffic uses default
+  cluster-wide distribution.
+
+## Parameters
+
+All keys live under the top-level `lineageControllerWebhook:` map.
+
+| Name                          | Description                                               | Value                                                                            |
+| ----------------------------- | --------------------------------------------------------- | -------------------------------------------------------------------------------- |
+| `image`                       | Container image (digest-pinned)                           | `ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.3.0@sha256:e898…6fb0` |
+| `debug`                       | Enable `--zap-log-level=debug` instead of `info`          | `false`                                                                          |
+| `replicas`                    | Deployment replica count                                  | `2`                                                                              |
+| `localK8sAPIEndpoint.enabled` | **Deprecated.** See note below.                           | `false`                                                                          |
+
+### `localK8sAPIEndpoint.enabled` (deprecated)
+
+When enabled, this injects `KUBERNETES_SERVICE_HOST=status.hostIP` and
+`KUBERNETES_SERVICE_PORT=6443` so the webhook talks to the kube-apiserver on
+its own node. It was originally added to avoid latency on the
+webhook-to-apiserver path, but it is only valid when the pod is actually
+scheduled on an apiserver-bearing node — which the chart's soft control-plane
+affinity no longer guarantees. With this flag enabled and the pod scheduled
+off a control-plane node, the controller will crash-loop dialing a non-
+apiserver hostIP. Slated for removal once the latency motivation is addressed
+in the webhook itself; **leave disabled**.
diff --git a/packages/system/lineage-controller-webhook/templates/deployment.yaml b/packages/system/lineage-controller-webhook/templates/deployment.yaml
deleted file mode 100644
index 87717fd3..00000000
--- a/packages/system/lineage-controller-webhook/templates/deployment.yaml
+++ /dev/null
@@ -1,49 +0,0 @@
-{{- if .Values.lineageControllerWebhook.deployment.enabled }}
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: lineage-controller-webhook
-  labels:
-    app: lineage-controller-webhook
-spec:
-  replicas: {{ .Values.lineageControllerWebhook.deployment.replicas }}
-  selector:
-    matchLabels:
-      app: lineage-controller-webhook
-  template:
-    metadata:
-      labels:
-        app: lineage-controller-webhook
-    spec:
-      serviceAccountName: lineage-controller-webhook
-      affinity:
-        podAntiAffinity:
-          preferredDuringSchedulingIgnoredDuringExecution:
-            - weight: 100
-              podAffinityTerm:
-                topologyKey: kubernetes.io/hostname
-                labelSelector:
-                  matchLabels:
-                    app: lineage-controller-webhook
-      containers:
-      - name: lineage-controller-webhook
-        image: "{{ .Values.lineageControllerWebhook.image }}"
-        args:
-        {{- if .Values.lineageControllerWebhook.debug }}
-        - --zap-log-level=debug
-        {{- else }}
-        - --zap-log-level=info
-        {{- end }}
-        ports:
-        - name: webhook
-          containerPort: 9443
-        volumeMounts:
-          - name: webhook-certs
-            mountPath: /tmp/k8s-webhook-server/serving-certs
-            readOnly: true
-      volumes:
-        - name: webhook-certs
-          secret:
-            secretName: lineage-controller-webhook-cert
-            defaultMode: 0400
-{{- end }}
\ No newline at end of file
diff --git a/packages/system/lineage-controller-webhook/templates/pdb.yaml b/packages/system/lineage-controller-webhook/templates/pdb.yaml
index b0a9d4e4..0786c8ff 100644
--- a/packages/system/lineage-controller-webhook/templates/pdb.yaml
+++ b/packages/system/lineage-controller-webhook/templates/pdb.yaml
@@ -1,11 +1,9 @@
-{{- if .Values.lineageControllerWebhook.deployment.enabled }}
 apiVersion: policy/v1
 kind: PodDisruptionBudget
 metadata:
   name: lineage-controller-webhook
 spec:
-  minAvailable: {{ if gt (int .Values.lineageControllerWebhook.deployment.replicas) 1 }}1{{ else }}0{{ end }}
+  maxUnavailable: 1
   selector:
     matchLabels:
       app: lineage-controller-webhook
-{{- end }}
\ No newline at end of file
diff --git a/packages/system/lineage-controller-webhook/templates/service.yaml b/packages/system/lineage-controller-webhook/templates/service.yaml
index c5df90fb..fbb3ad54 100644
--- a/packages/system/lineage-controller-webhook/templates/service.yaml
+++ b/packages/system/lineage-controller-webhook/templates/service.yaml
@@ -5,7 +5,7 @@ metadata:
   labels:
     app: lineage-controller-webhook
 spec:
-  internalTrafficPolicy: Local
+  trafficDistribution: PreferClose
   type: ClusterIP
   ports:
     - port: 443
diff --git a/packages/system/lineage-controller-webhook/templates/daemonset.yaml b/packages/system/lineage-controller-webhook/templates/workload.yaml
similarity index 67%
rename from packages/system/lineage-controller-webhook/templates/daemonset.yaml
rename to packages/system/lineage-controller-webhook/templates/workload.yaml
index 536ea24a..097919c6 100644
--- a/packages/system/lineage-controller-webhook/templates/daemonset.yaml
+++ b/packages/system/lineage-controller-webhook/templates/workload.yaml
@@ -1,11 +1,11 @@
-{{- if not .Values.lineageControllerWebhook.deployment.enabled }}
 apiVersion: apps/v1
-kind: DaemonSet
+kind: Deployment
 metadata:
   name: lineage-controller-webhook
   labels:
     app: lineage-controller-webhook
 spec:
+  replicas: {{ .Values.lineageControllerWebhook.replicas }}
   selector:
     matchLabels:
       app: lineage-controller-webhook
@@ -14,13 +14,25 @@ spec:
       labels:
         app: lineage-controller-webhook
     spec:
-      {{- with .Values.lineageControllerWebhook.nodeSelector }}
-      nodeSelector:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
+      serviceAccountName: lineage-controller-webhook
       tolerations:
       - operator: Exists
-      serviceAccountName: lineage-controller-webhook
+      affinity:
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            preference:
+              matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            podAffinityTerm:
+              topologyKey: kubernetes.io/hostname
+              labelSelector:
+                matchLabels:
+                  app: lineage-controller-webhook
       containers:
       - name: lineage-controller-webhook
         image: "{{ .Values.lineageControllerWebhook.image }}"
@@ -52,4 +64,3 @@ spec:
           secret:
             secretName: lineage-controller-webhook-cert
             defaultMode: 0400
-{{- end }}
\ No newline at end of file
diff --git a/packages/system/lineage-controller-webhook/tests/service_test.yaml b/packages/system/lineage-controller-webhook/tests/service_test.yaml
new file mode 100644
index 00000000..3cdbddd8
--- /dev/null
+++ b/packages/system/lineage-controller-webhook/tests/service_test.yaml
@@ -0,0 +1,30 @@
+suite: lineage-controller-webhook service
+templates:
+  - templates/service.yaml
+
+release:
+  name: lineage-controller-webhook
+  namespace: cozy-system
+
+tests:
+  - it: renders a ClusterIP service on 443 -> 9443 with PreferClose traffic distribution
+    asserts:
+      - isKind:
+          of: Service
+      - equal:
+          path: spec.type
+          value: ClusterIP
+      - equal:
+          path: spec.trafficDistribution
+          value: PreferClose
+      - notExists:
+          path: spec.internalTrafficPolicy
+      - equal:
+          path: spec.ports[0].port
+          value: 443
+      - equal:
+          path: spec.ports[0].targetPort
+          value: 9443
+      - equal:
+          path: spec.selector.app
+          value: lineage-controller-webhook
diff --git a/packages/system/lineage-controller-webhook/tests/workload_test.yaml b/packages/system/lineage-controller-webhook/tests/workload_test.yaml
new file mode 100644
index 00000000..bcb24014
--- /dev/null
+++ b/packages/system/lineage-controller-webhook/tests/workload_test.yaml
@@ -0,0 +1,99 @@
+suite: lineage-controller-webhook workload + PDB
+templates:
+  - templates/workload.yaml
+  - templates/pdb.yaml
+
+release:
+  name: lineage-controller-webhook
+  namespace: cozy-system
+
+tests:
+  # ---- Default Deployment shape ----------------------------------------------
+
+  - it: defaults render a 2-replica Deployment with soft control-plane nodeAffinity, soft podAntiAffinity, and Exists tolerations
+    template: templates/workload.yaml
+    asserts:
+      - isKind:
+          of: Deployment
+      - equal:
+          path: spec.replicas
+          value: 2
+      - equal:
+          path: spec.template.spec.tolerations
+          value:
+            - operator: Exists
+      - equal:
+          path: spec.template.spec.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution
+          value:
+            - weight: 100
+              preference:
+                matchExpressions:
+                  - key: node-role.kubernetes.io/control-plane
+                    operator: Exists
+      - notExists:
+          path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution
+      - equal:
+          path: spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.topologyKey
+          value: kubernetes.io/hostname
+
+  - it: defaults inject no localK8sAPIEndpoint env vars (knob is opt-in)
+    template: templates/workload.yaml
+    asserts:
+      - notExists:
+          path: spec.template.spec.containers[0].env
+
+  # ---- PDB --------------------------------------------------------------------
+
+  - it: PDB renders unconditionally with maxUnavailable=1 at the default replica count
+    template: templates/pdb.yaml
+    asserts:
+      - isKind:
+          of: PodDisruptionBudget
+      - equal:
+          path: spec.maxUnavailable
+          value: 1
+      - notExists:
+          path: spec.minAvailable
+
+  - it: PDB still renders at replicas=1 (no-op, but consistent shape)
+    set:
+      lineageControllerWebhook.replicas: 1
+    template: templates/pdb.yaml
+    asserts:
+      - isKind:
+          of: PodDisruptionBudget
+      - equal:
+          path: spec.maxUnavailable
+          value: 1
+
+  # ---- Replica override -------------------------------------------------------
+
+  - it: replicas value drives spec.replicas
+    set:
+      lineageControllerWebhook.replicas: 5
+    template: templates/workload.yaml
+    asserts:
+      - equal:
+          path: spec.replicas
+          value: 5
+
+  # ---- Deprecated localK8sAPIEndpoint knob -----------------------------------
+
+  - it: localK8sAPIEndpoint.enabled=true injects KUBERNETES_SERVICE_HOST and PORT env vars
+    set:
+      lineageControllerWebhook.localK8sAPIEndpoint.enabled: true
+    template: templates/workload.yaml
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: KUBERNETES_SERVICE_HOST
+            valueFrom:
+              fieldRef:
+                apiVersion: v1
+                fieldPath: status.hostIP
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: KUBERNETES_SERVICE_PORT
+            value: "6443"
diff --git a/packages/system/lineage-controller-webhook/values.yaml b/packages/system/lineage-controller-webhook/values.yaml
index 2b84f4e1..e56d332d 100644
--- a/packages/system/lineage-controller-webhook/values.yaml
+++ b/packages/system/lineage-controller-webhook/values.yaml
@@ -1,15 +1,13 @@
 lineageControllerWebhook:
   image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.3.0@sha256:e8984709686a5eaf19b89da378d7b8c688ea5607e0783a88d9c9e4ccfca96fb0
   debug: false
+  replicas: 2
+  # DEPRECATED. Injects KUBERNETES_SERVICE_HOST=status.hostIP and
+  # KUBERNETES_SERVICE_PORT=6443 so the webhook talks to the apiserver on its
+  # own node — only valid when the pod is actually scheduled on an apiserver-
+  # bearing node. Now that the chart's nodeAffinity to control-plane is soft
+  # (preferred, not required), the pod can land off-control-plane and crash-
+  # loop dialing a non-apiserver hostIP. Slated for removal once webhook
+  # performance work obviates the locality motivation. Leave disabled.
   localK8sAPIEndpoint:
-    # incompatable with Deployment mode
-    enabled: true
-  # nodeSelector for the DaemonSet
-  # Talos uses empty value: "node-role.kubernetes.io/control-plane": ""
-  # Generic k8s (k3s, kubeadm) uses: "node-role.kubernetes.io/control-plane": "true"
-  nodeSelector:
-    node-role.kubernetes.io/control-plane: ""
-  # if enabled, replace DaemonSet by Deployment
-  deployment:
     enabled: false
-    replicas: 1

From 3a8359fa73f5cd96d8363c48d31414d8cadf3447 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Tue, 28 Apr 2026 18:59:43 +0300
Subject: [PATCH 141/151] chore(kubernetes): regenerate deepcopy after merge

Run `make generate` to pick up the Images type added in main.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 .../v1alpha1/kubernetes/zz_generated.deepcopy.go | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go b/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go
index 3f1cb5ce..e021fbaa 100644
--- a/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go
+++ b/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go
@@ -136,6 +136,7 @@ func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) {
 	}
 	in.Addons.DeepCopyInto(&out.Addons)
 	in.ControlPlane.DeepCopyInto(&out.ControlPlane)
+	out.Images = in.Images
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigSpec.
@@ -277,6 +278,21 @@ func (in *HAMiAddon) DeepCopy() *HAMiAddon {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Images) DeepCopyInto(out *Images) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Images.
+func (in *Images) DeepCopy() *Images {
+	if in == nil {
+		return nil
+	}
+	out := new(Images)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *IngressNginxAddon) DeepCopyInto(out *IngressNginxAddon) {
 	*out = *in

From 80b86b0c1a39266488b84473724a54b0d3ad7d9c Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Tue, 28 Apr 2026 19:02:35 +0300
Subject: [PATCH 142/151] chore(kubernetes): regenerate CRD after merge
 conflict resolution

Per-package `make generate` corrects keysOrder placement and
openAPISchema formatting in the kubernetes CRD definition.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 api/apps/v1alpha1/kubernetes/types.go                 | 2 +-
 packages/system/kubernetes-rd/cozyrds/kubernetes.yaml | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/api/apps/v1alpha1/kubernetes/types.go b/api/apps/v1alpha1/kubernetes/types.go
index e0143445..88a3c97e 100644
--- a/api/apps/v1alpha1/kubernetes/types.go
+++ b/api/apps/v1alpha1/kubernetes/types.go
@@ -6,7 +6,7 @@ package kubernetes
 
 import (
 	resource "k8s.io/apimachinery/pkg/api/resource"
-	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1"
 	k8sRuntime "k8s.io/apimachinery/pkg/runtime"
 )
 
diff --git a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml
index b4ada238..f81eaff0 100644
--- a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml
+++ b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml
@@ -22,7 +22,7 @@ spec:
     singular: kubernetes
     plural: kuberneteses
     openAPISchema: |-
-      {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"ephemeralStorage":"20Gi","gpus":[],"instanceType":"u1.medium","maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"]}},"additionalProperties":{"type":"object","required":["ephemeralStorage","instanceType","maxReplicas","minReplicas","resources"],"properties":{"ephemeralStorage":{"description":"Ephemeral storage size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string"}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium"},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `<cluster-name>.<tenant-host>` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","hami","ingressNginx","monitoringAgents","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"hami":{"description":"HAMi GPU virtualization middleware.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable HAMi (requires GPU Operator).","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"large","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"images":{"description":"Optional image overrides for air-gapped or rate-limited registries.","type":"object","default":{},"properties":{"waitForKubeconfig":{"description":"Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.","type":"string","default":""}}}}}
+      {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"ephemeralStorage":"20Gi","gpus":[],"instanceType":"u1.medium","maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"]}},"additionalProperties":{"type":"object","required":["ephemeralStorage","instanceType","maxReplicas","minReplicas","resources"],"properties":{"ephemeralStorage":{"description":"Ephemeral storage size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string"}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium"},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `<cluster-name>.<tenant-host>` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","hami","ingressNginx","monitoringAgents","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"hami":{"description":"HAMi GPU virtualization middleware.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable HAMi (requires GPU Operator).","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"large","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"images":{"description":"Optional image overrides for air-gapped or rate-limited registries.","type":"object","default":{},"properties":{"waitForKubeconfig":{"description":"Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.","type":"string","default":""}}}}}
   release:
     prefix: kubernetes-
     labels:
@@ -40,7 +40,7 @@ spec:
     tags:
       - compute
     icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODFfMjg0NSkiLz4KPHBhdGggZD0iTTcxLjk5NjggMTlDNzAuMzAzOSAxOS4wMDAyIDY4LjkzMTIgMjAuNTMyMiA2OC45MzE0IDIyLjQyMjFDNjguOTMxNCAyMi40NTExIDY4LjkzNzMgMjIuNDc4OCA2OC45Mzc5IDIyLjUwNzZDNjguOTM1NCAyMi43NjQ0IDY4LjkyMzEgMjMuMDczNyA2OC45MzE0IDIzLjI5NzNDNjguOTcxNyAyNC4zODczIDY5LjIwODIgMjUuMjIxNiA2OS4zNTA2IDI2LjIyNThDNjkuNjA4NCAyOC4zNzUyIDY5LjgyNDUgMzAuMTU2OSA2OS42OTEyIDMxLjgxM0M2OS41NjE1IDMyLjQzNzUgNjkuMTAzNyAzMy4wMDg2IDY4LjY5NTYgMzMuNDA1Nkw2OC42MjM1IDM0LjcwODZDNjYuNzgzOSAzNC44NjE3IDY0LjkzMTkgMzUuMTQyMSA2My4wODIxIDM1LjU2NDFDNTUuMTIyNiAzNy4zNzk4IDQ4LjI2OTUgNDEuNDk5MSA0My4wNTIgNDcuMDYwOUM0Mi43MTM0IDQ2LjgyODggNDIuMTIxMSA0Ni40MDE5IDQxLjk0NSA0Ni4yNzEyQzQxLjM5NzcgNDYuMzQ1NCA0MC44NDQ1IDQ2LjUxNTEgNDAuMTI0MSA0Ni4wOTM1QzM4Ljc1MjIgNDUuMTY1NyAzNy41MDI4IDQzLjg4NTEgMzUuOTkxIDQyLjM0MjRDMzUuMjk4MiA0MS42MDQ0IDM0Ljc5NjYgNDAuOTAxOCAzMy45NzM1IDQwLjE5MDRDMzMuNzg2NiA0MC4wMjg5IDMzLjUwMTQgMzkuODEwNCAzMy4yOTIzIDM5LjY0NDJDMzIuNjQ4OSAzOS4xMjg4IDMxLjg5IDM4Ljg2IDMxLjE1NyAzOC44MzQ4QzMwLjIxNDcgMzguODAyNCAyOS4zMDc1IDM5LjE3MjUgMjguNzEzOCAzOS45MjA2QzI3LjY1ODQgNDEuMjUwNiAyNy45OTYzIDQzLjI4MzMgMjkuNDY3MSA0NC40NjE0QzI5LjQ4MiA0NC40NzM0IDI5LjQ5NzkgNDQuNDgyNyAyOS41MTI5IDQ0LjQ5NDNDMjkuNzE1IDQ0LjY1ODkgMjkuOTYyNSA0NC44Njk4IDMwLjE0ODMgNDUuMDA3NkMzMS4wMjE3IDQ1LjY1NTUgMzEuODE5NSA0NS45ODcyIDMyLjY4OTcgNDYuNTAxNUMzNC41MjMxIDQ3LjYzOTEgMzYuMDQzIDQ4LjU4MjMgMzcuMjQ4NiA0OS43MTk2QzM3LjcxOTQgNTAuMjIzNyAzNy44MDE2IDUxLjExMjIgMzcuODY0MyA1MS40OTY0TDM4Ljg0NjggNTIuMzc4MkMzMy41ODcyIDYwLjMzMDggMzEuMTUzIDcwLjE1MzkgMzIuNTkxNSA4MC4xNjI3TDMxLjMwNzcgODAuNTM3OEMzMC45NjkzIDgwLjk3NjggMzAuNDkxMiA4MS42Njc2IDI5Ljk5MTEgODEuODczOEMyOC40MTM4IDgyLjM3MjkgMjYuNjM4NyA4Mi41NTYyIDI0LjQ5NTYgODIuNzgxOUMyMy40ODk0IDgyLjg2NiAyMi42MjEzIDgyLjgxNTggMjEuNTU0NiA4My4wMTg4QzIxLjMxOTggODMuMDYzNSAyMC45OTI3IDgzLjE0OTEgMjAuNzM1OCA4My4yMDk3QzIwLjcyNjkgODMuMjExNiAyMC43MTg2IDgzLjIxNDIgMjAuNzA5NiA4My4yMTYyQzIwLjY5NTYgODMuMjE5NSAyMC42NzcyIDgzLjIyNjMgMjAuNjYzOCA4My4yMjk0QzE4Ljg1NyA4My42NjggMTcuNjk2MyA4NS4zMzY1IDE4LjA2OTkgODYuOTgwNUMxOC40NDM3IDg4LjYyNDggMjAuMjA4NiA4OS42MjQ4IDIyLjAyNjIgODkuMjMxMkMyMi4wMzkzIDg5LjIyODIgMjIuMDU4NCA4OS4yMjc3IDIyLjA3MiA4OS4yMjQ2QzIyLjA5MjYgODkuMjE5OSAyMi4xMTA2IDg5LjIwOTkgMjIuMTMxIDg5LjIwNDlDMjIuMzg0NCA4OS4xNDkgMjIuNzAxOSA4OS4wODY4IDIyLjkyMzYgODkuMDI3MkMyMy45NzIzIDg4Ljc0NTEgMjQuNzMxOCA4OC4zMzA2IDI1LjY3NDYgODcuOTY3N0MyNy43MDI5IDg3LjIzNjggMjkuMzgyOCA4Ni42MjYyIDMxLjAxOTUgODYuMzg4M0MzMS43MDMgODYuMzM0NSAzMi40MjMyIDg2LjgxMiAzMi43ODE0IDg3LjAxMzRMMzQuMTE3NyA4Ni43ODMxQzM3LjE5MjYgOTYuMzYxMyA0My42MzY2IDEwNC4xMDMgNTEuNzk2MyAxMDguOTYxTDUxLjIzOTYgMTEwLjMwM0M1MS40NDAzIDExMC44MjQgNTEuNjYxNiAxMTEuNTMgNTEuNTEyMSAxMTIuMDQ1QzUwLjkxNzEgMTEzLjU5NSA0OS44OTggMTE1LjIzMSA0OC43Mzc0IDExNy4wNTVDNDguMTc1NSAxMTcuODk4IDQ3LjYwMDQgMTE4LjU1MiA0Ny4wOTM0IDExOS41MTZDNDYuOTcyIDExOS43NDcgNDYuODE3NSAxMjAuMTAyIDQ2LjcwMDQgMTIwLjM0NkM0NS45MTI1IDEyMi4wMzkgNDYuNDkwNCAxMjMuOTkgNDguMDAzOCAxMjQuNzIyQzQ5LjUyNjggMTI1LjQ1OSA1MS40MTcxIDEyNC42ODIgNTIuMjM1MiAxMjIuOTg1QzUyLjIzNjQgMTIyLjk4MiA1Mi4yNDA2IDEyMi45OCA1Mi4yNDE3IDEyMi45NzhDNTIuMjQyNiAxMjIuOTc2IDUyLjI0MDkgMTIyLjk3MyA1Mi4yNDE3IDEyMi45NzFDNTIuMzU4MiAxMjIuNzMxIDUyLjUyMzMgMTIyLjQxNSA1Mi42MjE2IDEyMi4xODhDNTMuMDU2IDEyMS4xODkgNTMuMjAwNSAxMjAuMzMyIDUzLjUwNTkgMTE5LjM2NUM1NC4zMTcgMTE3LjMxOCA1NC43NjI2IDExNS4xNyA1NS44NzkxIDExMy44MzJDNTYuMTg0OSAxMTMuNDY2IDU2LjY4MzMgMTEzLjMyNSA1Ny4yMDAxIDExMy4xODZMNTcuODk0NCAxMTEuOTIyQzY1LjAwOCAxMTQuNjY1IDcyLjk3MDUgMTE1LjQwMiA4MC45MjQ1IDExMy41ODdDODIuNzM5MSAxMTMuMTczIDg0LjQ5MDggMTEyLjYzNyA4Ni4xODQzIDExMS45OTRDODYuMzc5NCAxMTIuMzQyIDg2Ljc0MiAxMTMuMDExIDg2LjgzOTMgMTEzLjE3OUM4Ny4zNjQ0IDExMy4zNTEgODcuOTM3NyAxMTMuNDM5IDg4LjQwNDcgMTE0LjEzM0M4OS4yNDAxIDExNS41NjcgODkuODExNCAxMTcuMjYzIDkwLjUwNzMgMTE5LjMxMkM5MC44MTI4IDEyMC4yNzkgOTAuOTYzOCAxMjEuMTM2IDkxLjM5ODEgMTIyLjEzNkM5MS40OTcxIDEyMi4zNjMgOTEuNjYxNCAxMjIuNjg0IDkxLjc3OCAxMjIuOTI1QzkyLjU5NDQgMTI0LjYyOCA5NC40OTA3IDEyNS40MDcgOTYuMDE1OSAxMjQuNjY5Qzk3LjUyOTIgMTIzLjkzNyA5OC4xMDc3IDEyMS45ODYgOTcuMzE5NCAxMjAuMjkzQzk3LjIwMjMgMTIwLjA0OSA5Ny4wNDEyIDExOS42OTUgOTYuOTE5OCAxMTkuNDY0Qzk2LjQxMjcgMTE4LjQ5OSA5NS44Mzc3IDExNy44NTIgOTUuMjc1OCAxMTcuMDA5Qzk0LjExNTIgMTE1LjE4NSA5My4xNTI2IDExMy42NyA5Mi41NTc1IDExMi4xMkM5Mi4zMDg3IDExMS4zMiA5Mi41OTk1IDExMC44MjMgOTIuNzkzMyAxMTAuMzAzQzkyLjY3NzIgMTEwLjE3IDkyLjQyODggMTA5LjQxNCA5Mi4yODI0IDEwOS4wNTlDMTAwLjc2MiAxMDQuMDI5IDEwNy4wMTcgOTUuOTk4NSAxMDkuOTU1IDg2LjcyMzlDMTEwLjM1MSA4Ni43ODY1IDExMS4wNDEgODYuOTA5MSAxMTEuMjY1IDg2Ljk1NDJDMTExLjcyNiA4Ni42NDg3IDExMi4xNDkgODYuMjUwMSAxMTIuOTgxIDg2LjMxNTlDMTE0LjYxNyA4Ni41NTM3IDExNi4yOTcgODcuMTY0NSAxMTguMzI2IDg3Ljg5NTNDMTE5LjI2OCA4OC4yNTgxIDEyMC4wMjggODguNjc5MyAxMjEuMDc3IDg4Ljk2MTRDMTIxLjI5OCA4OS4wMjEgMTIxLjYxNiA4OS4wNzY2IDEyMS44NjkgODkuMTMyNUMxMjEuODg5IDg5LjEzNzUgMTIxLjkwOCA4OS4xNDc1IDEyMS45MjggODkuMTUyMkMxMjEuOTQyIDg5LjE1NTMgMTIxLjk2MSA4OS4xNTU4IDEyMS45NzQgODkuMTU4OEMxMjMuNzkyIDg5LjU1MiAxMjUuNTU3IDg4LjU1MjYgMTI1LjkzIDg2LjkwODFDMTI2LjMwMyA4NS4yNjQxIDEyNS4xNDMgODMuNTk1MiAxMjMuMzM2IDgzLjE1N0MxMjMuMDc0IDgzLjA5NyAxMjIuNzAxIDgyLjk5NSAxMjIuNDQ2IDgyLjk0NjVDMTIxLjM3OSA4Mi43NDM1IDEyMC41MTEgODIuNzkzNSAxMTkuNTA1IDgyLjcwOTVDMTE3LjM2MSA4Mi40ODM5IDExNS41ODYgODIuMzAwNCAxMTQuMDA5IDgxLjgwMTRDMTEzLjM2NiA4MS41NTA3IDExMi45MDggODAuNzgxOSAxMTIuNjg2IDgwLjQ2NTVMMTExLjQ0OCA4MC4xMDM1QzExMi4wOSA3NS40MzggMTExLjkxNyA3MC41ODI1IDExMC44MDYgNjUuNzI0M0MxMDkuNjg1IDYwLjgyMDggMTA3LjcwNCA1Ni4zMzYxIDEwNS4wNjIgNTIuMzg0OEMxMDUuMzc5IDUyLjA5NDggMTA1Ljk3OSA1MS41NjEyIDEwNi4xNDkgNTEuNDA0M0MxMDYuMTk5IDUwLjg1MTcgMTA2LjE1NiA1MC4yNzIyIDEwNi43MjUgNDkuNjYwM0MxMDcuOTMxIDQ4LjUyMyAxMDkuNDUxIDQ3LjU3OTkgMTExLjI4NCA0Ni40NDIzQzExMi4xNTQgNDUuOTI3OSAxMTIuOTU5IDQ1LjU5NjQgMTEzLjgzMiA0NC45NDg0QzExNC4wMyA0NC44MDE5IDExNC4yOTkgNDQuNTY5OSAxMTQuNTA3IDQ0LjQwMjJDMTE1Ljk3NyA0My4yMjM3IDExNi4zMTYgNDEuMTkxMSAxMTUuMjYgMzkuODYxNEMxMTQuMjA0IDM4LjUzMTcgMTEyLjE1OSAzOC40MDY1IDExMC42ODggMzkuNTg1QzExMC40NzkgMzkuNzUxNiAxMTAuMTk1IDM5Ljk2ODggMTEwLjAwNyA0MC4xMzEyQzEwOS4xODQgNDAuODQyNiAxMDguNjc2IDQxLjU0NTIgMTA3Ljk4MyA0Mi4yODMyQzEwNi40NzEgNDMuODI1OSAxMDUuMjIyIDQ1LjExMyAxMDMuODUgNDYuMDQwOUMxMDMuMjU1IDQ2LjM4ODUgMTAyLjM4NSA0Ni4yNjgyIDEwMS45OSA0Ni4yNDQ5TDEwMC44MjQgNDcuMDgwNkM5NC4xNzUzIDQwLjA3NjMgODUuMTIzNSAzNS41OTgyIDc1LjM3NjYgMzQuNzI4M0M3NS4zNDk0IDM0LjMxNzkgNzUuMzEzNyAzMy41NzYxIDc1LjMwNDYgMzMuMzUyOUM3NC45MDU2IDMyLjk2OTMgNzQuNDIzNSAzMi42NDE4IDc0LjMwMjQgMzEuODEzQzc0LjE2OTEgMzAuMTU2OSA3NC4zOTE3IDI4LjM3NTIgNzQuNjQ5NiAyNi4yMjU4Qzc0Ljc5MTkgMjUuMjIxNiA3NS4wMjg0IDI0LjM4NzMgNzUuMDY4OCAyMy4yOTczQzc1LjA3OCAyMy4wNDk1IDc1LjA2MzIgMjIuNjkgNzUuMDYyMiAyMi40MjIxQzc1LjA2MiAyMC41MzIyIDczLjY4OTggMTguOTk5OCA3MS45OTY4IDE5Wk02OC4xNTg1IDQyLjg4ODZMNjcuMjQ4IDU5LjA0NDdMNjcuMTgyNSA1OS4wNzc2QzY3LjEyMTQgNjAuNTIyOSA2NS45Mzc1IDYxLjY3NyA2NC40ODM5IDYxLjY3N0M2My44ODg0IDYxLjY3NyA2My4zMzg4IDYxLjQ4NDkgNjIuODkyMiA2MS4xNTcxTDYyLjg2NiA2MS4xNzAzTDQ5LjY4MDcgNTEuNzc5NEM1My43MzMxIDQ3Ljc3NTkgNTguOTE2NCA0NC44MTcyIDY0Ljg5IDQzLjQ1NDZDNjUuOTgxMiA0My4yMDU2IDY3LjA3MTkgNDMuMDIwOSA2OC4xNTg1IDQyLjg4ODZaTTc1Ljg0MTcgNDIuODg4NkM4Mi44MTU5IDQzLjc1MDQgODkuMjY1NyA0Ni45MjMyIDk0LjIwODEgNTEuNzg2TDgxLjEwOCA2MS4xMTc2TDgxLjA2MjEgNjEuMDk3OUM3OS44OTk0IDYxLjk1MTIgNzguMjYxMSA2MS43Mzk0IDc3LjM1NDggNjAuNTk3OEM3Ni45ODM1IDYwLjEzMDEgNzYuNzg4NyA1OS41ODAxIDc2Ljc2NTMgNTkuMDI0OUw3Ni43NTIyIDU5LjAxODRMNzUuODQxNyA0Mi44ODg2Wk00NC44OTkxIDU3LjgxNEw1Ni45MzgyIDY4LjYzM0w1Ni45MjUxIDY4LjY5ODhDNTguMDExNyA2OS42NDc5IDU4LjE3MiA3MS4yOTQ5IDU3LjI2NTcgNzIuNDM2OEM1Ni44OTQ0IDcyLjkwNDUgNTYuMzk3NSA3My4yMTgyIDU1Ljg2MzkgNzMuMzY0N0w1NS44NTA4IDczLjQxNzNMNDAuNDE4OCA3Ny44OTIzQzM5LjYzMzQgNzAuNjc2NSA0MS4zMjYxIDYzLjY2MjEgNDQuODk5MSA1Ny44MTRaTTk5LjAwOTQgNTcuODIwNkMxMDAuNzk4IDYwLjczMzYgMTAyLjE1MyA2My45ODcxIDEwMi45NTkgNjcuNTE0M0MxMDMuNzU2IDcwLjk5OTEgMTAzLjk1NiA3NC40Nzc4IDEwMy42MjcgNzcuODM5N0w4OC4xMTY2IDczLjM1MTVMODguMTAzNSA3My4yODU3Qzg2LjcxNDUgNzIuOTA0MyA4NS44NjA5IDcxLjQ4NDggODYuMTg0MyA3MC4wNjExQzg2LjMxNjggNjkuNDc3OCA4Ni42MjQ5IDY4Ljk4NDQgODcuMDQyMyA2OC42MTk4TDg3LjAzNTggNjguNTg2OUw5OS4wMDk0IDU3LjgyMDZaTTY5LjUyNzQgNjkuNDY4OEg3NC40NTk2TDc3LjUyNTEgNzMuMzE4Nkw3Ni40MjQ3IDc4LjEyMjZMNzEuOTk2OCA4MC4yNjE0TDY3LjU1NTggNzguMTE2MUw2Ni40NTU0IDczLjMxMkw2OS41Mjc0IDY5LjQ2ODhaTTg1LjMzOTMgODIuNjQzN0M4NS41NDg5IDgyLjYzMzEgODUuNzU3NiA4Mi42NTIgODUuOTYxNiA4Mi42ODk4TDg1Ljk4NzggODIuNjU2OUwxMDEuOTUgODUuMzY4MkM5OS42MTQyIDkxLjk2MjQgOTUuMTQ0IDk3LjY3NSA4OS4xNzExIDEwMS40OThMODIuOTc0NyA4Ni40NjA2TDgyLjk5NDQgODYuNDM0M0M4Mi40MjUyIDg1LjEwNTUgODIuOTk0OCA4My41NDcyIDg0LjMwNDQgODIuOTEzNUM4NC42Mzk3IDgyLjc1MTMgODQuOTkgODIuNjYxNCA4NS4zMzkzIDgyLjY0MzdaTTU4LjUyOTggODIuNzA5NUM1OS43NDggODIuNzI2NyA2MC44NDA2IDgzLjU3NjEgNjEuMTIzNyA4NC44MjJDNjEuMjU2MiA4NS40MDUyIDYxLjE5MTcgODUuOTgzMSA2MC45NzMgODYuNDkzNUw2MS4wMTg5IDg2LjU1MjdMNTQuODg4IDEwMS40MzlDNDkuMTU1OSA5Ny43NDMyIDQ0LjU5MDQgOTIuMjA5OSA0Mi4xNDgxIDg1LjQyMDhMNTcuOTczMSA4Mi43MjI3TDU3Ljk5OTMgODIuNzU1NkM1OC4xNzYzIDgyLjcyMjkgNTguMzU1OCA4Mi43MDcxIDU4LjUyOTggODIuNzA5NVpNNzEuODk4NiA4OS4yMzEyQzcyLjMyMjkgODkuMjE1NSA3Mi43NTM0IDg5LjMwMyA3My4xNjI3IDg5LjUwMUM3My42OTkyIDg5Ljc2MDYgNzQuMTEzNiA5MC4xNjkyIDc0LjM3NDUgOTAuNjU5Mkg3NC40MzM0TDgyLjIzNDYgMTA0LjgyMUM4MS4yMjIxIDEwNS4xNjIgODAuMTgxMyAxMDUuNDU0IDc5LjExNjcgMTA1LjY5N0M3My4xNTA1IDEwNy4wNTggNjcuMjAzMiAxMDYuNjQ1IDYxLjgxOCAxMDQuODAyTDY5LjU5OTUgOTAuNjY1OEg2OS42MTI2QzcwLjA3OTUgODkuNzg4OCA3MC45NjUgODkuMjY1NiA3MS44OTg2IDg5LjIzMTJaIiBmaWxsPSJ3aGl0ZSIgc3Ryb2tlPSJ3aGl0ZSIgc3Ryb2tlLXdpZHRoPSIwLjI1Ii8+CjxkZWZzPgo8bGluZWFyR3JhZGllbnQgaWQ9InBhaW50MF9saW5lYXJfNjgxXzI4NDUiIHgxPSIxMCIgeTE9IjE1LjUiIHgyPSIxNDQiIHkyPSIxMzEuNSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiPgo8c3RvcCBzdG9wLWNvbG9yPSIjNEQ4N0ZGIi8+CjxzdG9wIG9mZnNldD0iMSIgc3RvcC1jb2xvcj0iIzA1NDdEMCIvPgo8L2xpbmVhckdyYWRpZW50Pgo8L2RlZnM+Cjwvc3ZnPgo=
-    keysOrder: [["apiVersion"],["appVersion"],["kind"],["metadata"],["metadata","name"],["spec","storageClass"],["spec","nodeGroups"],["spec","nodeGroups","md0"],["spec","nodeGroups","md0","minReplicas"],["spec","nodeGroups","md0","maxReplicas"],["spec","nodeGroups","md0","instanceType"],["spec","nodeGroups","md0","ephemeralStorage"],["spec","nodeGroups","md0","roles"],["spec","nodeGroups","md0","resources"],["spec","nodeGroups","md0","gpus"],["spec","version"],["spec","host"],["spec","addons"],["spec","addons","certManager"],["spec","addons","certManager","enabled"],["spec","addons","certManager","valuesOverride"],["spec","addons","cilium"],["spec","addons","cilium","valuesOverride"],["spec","addons","gatewayAPI"],["spec","addons","gatewayAPI","enabled"],["spec","addons","ingressNginx"],["spec","addons","ingressNginx","enabled"],["spec","addons","ingressNginx","exposeMethod"],["spec","addons","ingressNginx","hosts"],["spec","addons","ingressNginx","valuesOverride"],["spec","addons","gpuOperator"],["spec","addons","gpuOperator","enabled"],["spec","addons","gpuOperator","valuesOverride"],["spec","addons","fluxcd"],["spec","addons","fluxcd","enabled"],["spec","addons","fluxcd","valuesOverride"],["spec","addons","monitoringAgents"],["spec","addons","monitoringAgents","enabled"],["spec","addons","monitoringAgents","valuesOverride"],["spec","addons","verticalPodAutoscaler"],["spec","addons","verticalPodAutoscaler","valuesOverride"],["spec","addons","velero"],["spec","addons","velero","enabled"],["spec","addons","velero","valuesOverride"],["spec","addons","coredns"],["spec","addons","coredns","valuesOverride"],["spec","controlPlane"],["spec","controlPlane","replicas"],["spec","controlPlane","apiServer"],["spec","controlPlane","apiServer","resources"],["spec","controlPlane","apiServer","resourcesPreset"],["spec","controlPlane","controllerManager"],["spec","controlPlane","controllerManager","resources"],["spec","controlPlane","controllerManager","resourcesPreset"],["spec","controlPlane","scheduler"],["spec","controlPlane","scheduler","resources"],["spec","controlPlane","scheduler","resourcesPreset"],["spec","controlPlane","konnectivity"],["spec","controlPlane","konnectivity","server"],["spec","controlPlane","konnectivity","server","resources"],["spec","controlPlane","konnectivity","server","resourcesPreset"],["spec","addons","hami"],["spec","addons","hami","enabled"],["spec","addons","hami","valuesOverride"],["spec","images"],["spec","images","waitForKubeconfig"]]
+    keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "nodeGroups"], ["spec", "nodeGroups", "md0"], ["spec", "nodeGroups", "md0", "minReplicas"], ["spec", "nodeGroups", "md0", "maxReplicas"], ["spec", "nodeGroups", "md0", "instanceType"], ["spec", "nodeGroups", "md0", "ephemeralStorage"], ["spec", "nodeGroups", "md0", "roles"], ["spec", "nodeGroups", "md0", "resources"], ["spec", "nodeGroups", "md0", "gpus"], ["spec", "version"], ["spec", "host"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "hami"], ["spec", "addons", "hami", "enabled"], ["spec", "addons", "hami", "valuesOverride"], ["spec", "addons", "fluxcd"], ["spec", "addons", "fluxcd", "enabled"], ["spec", "addons", "fluxcd", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"], ["spec", "images"], ["spec", "images", "waitForKubeconfig"]]
   secrets:
     exclude: []
     include:

From 805b3b17cbc4d89206203bc24b45c1412c29a2c8 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Tue, 28 Apr 2026 19:11:26 +0300
Subject: [PATCH 143/151] fix(kubernetes): add CI values to GPU Operator HAMi
 tests

Tests need _namespace.etcd from values-ci.yaml after the merge
introduced an etcd-namespace guard on the gpu-operator HelmRelease
condition.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml
index 4fa754fb..44528470 100644
--- a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml
+++ b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml
@@ -1,6 +1,8 @@
 suite: GPU Operator HelmRelease HAMi integration tests
 templates:
   - templates/helmreleases/gpu-operator.yaml
+values:
+  - values-ci.yaml
 tests:
   - it: should disable devicePlugin when hami is enabled
     set:

From 4541c20e3403a5eda85a15ae3a02b80e6cc2ebbd Mon Sep 17 00:00:00 2001
From: "cozystack-ci[bot]"
 <274107086+cozystack-ci[bot]@users.noreply.github.com>
Date: Wed, 29 Apr 2026 04:38:30 +0000
Subject: [PATCH 144/151] docs: add changelog for v1.3.1

Signed-off-by: cozystack-ci[bot] <274107086+cozystack-ci[bot]@users.noreply.github.com>
---
 docs/changelogs/v1.3.1.md | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100644 docs/changelogs/v1.3.1.md

diff --git a/docs/changelogs/v1.3.1.md b/docs/changelogs/v1.3.1.md
new file mode 100644
index 00000000..55e6a796
--- /dev/null
+++ b/docs/changelogs/v1.3.1.md
@@ -0,0 +1,38 @@
+# v1.3.1 (2026-04-29)
+
+This is a patch release containing fixes and minor improvements.
+
+## Fixes and Improvements
+
+* **[linstor-gui] Add package for LINBIT linstor-gui web UI**: [linstor-gui] Add package for LINBIT linstor-gui web UI ([**@myasnikovdaniil**](https://github.com/cozystack/cozystack/pull/2382) in #2382)
+* **fix(kamaji): increase memory limits and add startup probe**: fix(kamaji): increase memory limits and add startup probe ([**@IvanHunters**](https://github.com/cozystack/cozystack/pull/2421) in #2421)
+* **fix(backups): move velero-configmap Role to velero chart**: fix(backups): move velero-configmap Role to velero chart ([**@myasnikovdaniil**](https://github.com/cozystack/cozystack/pull/2459) in #2459)
+* **fix(etcd): remove destructive post-upgrade cert-regeneration hook**: fix(etcd): remove destructive post-upgrade cert-regeneration hook ([**@myasnikovdaniil**](https://github.com/cozystack/cozystack/pull/2462) in #2462)
+* **[Backport release-1.3] fix(backups): move velero-configmap Role to velero chart**: [Backport release-1.3] fix(backups): move velero-configmap Role to velero chart ([**@IvanHunters**](https://github.com/cozystack/cozystack/pull/2467) in #2467)
+* **fix(api): prevent IDOR in TenantNamespace Get and Watch handlers**: fix(api): prevent IDOR in TenantNamespace Get and Watch handlers ([**@IvanHunters**](https://github.com/cozystack/cozystack/pull/2471) in #2471)
+* **[Backport release-1.3] fix(kamaji): increase memory limits and add startup probe**: [Backport release-1.3] fix(kamaji): increase memory limits and add startup probe ([**@app/github-actions**](https://github.com/cozystack/cozystack/pull/2491) in #2491)
+* **feat(linstor): bump linstor-csi to v1.10.6 with Protocol-C dual-attach fix**: feat(linstor): bump linstor-csi to v1.10.6 with Protocol-C dual-attach fix ([**@kvaps**](https://github.com/cozystack/cozystack/pull/2496) in #2496)
+* **build(linstor): include linstor-gui in root image build target**: build(linstor): include linstor-gui in root image build target ([**@myasnikovdaniil**](https://github.com/cozystack/cozystack/pull/2498) in #2498)
+* **[Backport release-1.3] feat(linstor): bump linstor-csi to v1.10.6 with Protocol-C dual-attach fix**: [Backport release-1.3] feat(linstor): bump linstor-csi to v1.10.6 with Protocol-C dual-attach fix ([**@app/github-actions**](https://github.com/cozystack/cozystack/pull/2505) in #2505)
+* **[Backport release-1.3] fix(etcd): remove destructive post-upgrade cert-regeneration hook**: [Backport release-1.3] fix(etcd): remove destructive post-upgrade cert-regeneration hook ([**@app/github-actions**](https://github.com/cozystack/cozystack/pull/2511) in #2511)
+* **[Backport release-1.3] build(linstor): include linstor-gui in root image build target**: [Backport release-1.3] build(linstor): include linstor-gui in root image build target ([**@app/github-actions**](https://github.com/cozystack/cozystack/pull/2518) in #2518)
+* **[Backport release-1.3] fix(api): prevent IDOR in TenantNamespace Get and Watch handlers**: [Backport release-1.3] fix(api): prevent IDOR in TenantNamespace Get and Watch handlers ([**@app/github-actions**](https://github.com/cozystack/cozystack/pull/2524) in #2524)
+* **fix namespaces creation**: fix namespaces creation ([**@kvaps**](https://github.com/cozystack/cozystack/pull/435) in #435)
+
+## Documentation
+
+* **[website] [docs] Update managed apps reference for v1.3.0**: Documentation updates ([**@myasnikovdaniil**](https://github.com/cozystack/website/pull/507) in cozystack/website#507)
+* **[website] feat(blog): add Cozystack 1.3 release announcement**: Documentation updates ([**@tym83**](https://github.com/cozystack/website/pull/512) in cozystack/website#512)
+* **[website] fix(telemetry): use April 2026 data as placeholder for quarter and year periods**: Documentation updates ([**@tym83**](https://github.com/cozystack/website/pull/511) in cozystack/website#511)
+* **[website] fix(oss-health): correct April 2026 telemetry snapshot and pause fetch cron**: Documentation updates ([**@tym83**](https://github.com/cozystack/website/pull/508) in cozystack/website#508)
+
+## Contributors
+
+We thank the following contributors for this release:
+
+* [**@IvanHunters**](https://github.com/IvanHunters)
+* [**@app/github-actions**](https://github.com/app/github-actions)
+* [**@kvaps**](https://github.com/kvaps)
+* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil)
+
+**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.3.0...v1.3.1

From 875a940033fea97a4800797989b0e004bbb8eca4 Mon Sep 17 00:00:00 2001
From: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
Date: Wed, 29 Apr 2026 12:22:08 +0500
Subject: [PATCH 145/151] docs(changelog): rewrite v1.3.1 with the actual
 release contents

The AI-generated v1.3.1 changelog (#2480) was generated from
git log v1.3.0..main rather than git log v1.3.0..v1.3.1, because
the workflow checked out main while the v1.3.1 tag points to
release-1.3. As a result the changelog included:

- 8 PRs that were merged to main but never shipped in v1.3.1
- 6 backport PRs that were merged to release-1.3 *after* v1.3.1
  was tagged
- Both originals and their backports as separate duplicate entries
- A 2024 PR (#435) that has nothing to do with this range
- Generic "Documentation updates" placeholders for website entries
- Title duplicated as both the bold label and the description
  (`* **fix(...): X**: fix(...): X (...)`)
- The cozystack-ci bot listed as a human contributor

The actual v1.3.1 release range (v1.3.0..v1.3.1) contains exactly
one user-facing change:

  41bcb0be [Backport release-1.3] fix(backups): move
           velero-configmap Role to velero chart (#2467)

which is the backport of #2459 (myasnikovdaniil) shipped via
#2467 (IvanHunters). This commit replaces the contents of
docs/changelogs/v1.3.1.md with that one entry, the matching
two-person contributors list, and the standard footer.

The workflow + docs fixes that prevent this regression for future
patch releases will land in a separate PR against main.

Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
---
 docs/changelogs/v1.3.1.md | 34 ++++++++--------------------------
 1 file changed, 8 insertions(+), 26 deletions(-)

diff --git a/docs/changelogs/v1.3.1.md b/docs/changelogs/v1.3.1.md
index 55e6a796..e8318ff4 100644
--- a/docs/changelogs/v1.3.1.md
+++ b/docs/changelogs/v1.3.1.md
@@ -1,38 +1,20 @@
-# v1.3.1 (2026-04-29)
+<!--
+https://github.com/cozystack/cozystack/releases/tag/v1.3.1
+-->
 
-This is a patch release containing fixes and minor improvements.
+# v1.3.1 (2026-04-23)
 
-## Fixes and Improvements
+A patch release with a single bug fix in the backup subsystem.
 
-* **[linstor-gui] Add package for LINBIT linstor-gui web UI**: [linstor-gui] Add package for LINBIT linstor-gui web UI ([**@myasnikovdaniil**](https://github.com/cozystack/cozystack/pull/2382) in #2382)
-* **fix(kamaji): increase memory limits and add startup probe**: fix(kamaji): increase memory limits and add startup probe ([**@IvanHunters**](https://github.com/cozystack/cozystack/pull/2421) in #2421)
-* **fix(backups): move velero-configmap Role to velero chart**: fix(backups): move velero-configmap Role to velero chart ([**@myasnikovdaniil**](https://github.com/cozystack/cozystack/pull/2459) in #2459)
-* **fix(etcd): remove destructive post-upgrade cert-regeneration hook**: fix(etcd): remove destructive post-upgrade cert-regeneration hook ([**@myasnikovdaniil**](https://github.com/cozystack/cozystack/pull/2462) in #2462)
-* **[Backport release-1.3] fix(backups): move velero-configmap Role to velero chart**: [Backport release-1.3] fix(backups): move velero-configmap Role to velero chart ([**@IvanHunters**](https://github.com/cozystack/cozystack/pull/2467) in #2467)
-* **fix(api): prevent IDOR in TenantNamespace Get and Watch handlers**: fix(api): prevent IDOR in TenantNamespace Get and Watch handlers ([**@IvanHunters**](https://github.com/cozystack/cozystack/pull/2471) in #2471)
-* **[Backport release-1.3] fix(kamaji): increase memory limits and add startup probe**: [Backport release-1.3] fix(kamaji): increase memory limits and add startup probe ([**@app/github-actions**](https://github.com/cozystack/cozystack/pull/2491) in #2491)
-* **feat(linstor): bump linstor-csi to v1.10.6 with Protocol-C dual-attach fix**: feat(linstor): bump linstor-csi to v1.10.6 with Protocol-C dual-attach fix ([**@kvaps**](https://github.com/cozystack/cozystack/pull/2496) in #2496)
-* **build(linstor): include linstor-gui in root image build target**: build(linstor): include linstor-gui in root image build target ([**@myasnikovdaniil**](https://github.com/cozystack/cozystack/pull/2498) in #2498)
-* **[Backport release-1.3] feat(linstor): bump linstor-csi to v1.10.6 with Protocol-C dual-attach fix**: [Backport release-1.3] feat(linstor): bump linstor-csi to v1.10.6 with Protocol-C dual-attach fix ([**@app/github-actions**](https://github.com/cozystack/cozystack/pull/2505) in #2505)
-* **[Backport release-1.3] fix(etcd): remove destructive post-upgrade cert-regeneration hook**: [Backport release-1.3] fix(etcd): remove destructive post-upgrade cert-regeneration hook ([**@app/github-actions**](https://github.com/cozystack/cozystack/pull/2511) in #2511)
-* **[Backport release-1.3] build(linstor): include linstor-gui in root image build target**: [Backport release-1.3] build(linstor): include linstor-gui in root image build target ([**@app/github-actions**](https://github.com/cozystack/cozystack/pull/2518) in #2518)
-* **[Backport release-1.3] fix(api): prevent IDOR in TenantNamespace Get and Watch handlers**: [Backport release-1.3] fix(api): prevent IDOR in TenantNamespace Get and Watch handlers ([**@app/github-actions**](https://github.com/cozystack/cozystack/pull/2524) in #2524)
-* **fix namespaces creation**: fix namespaces creation ([**@kvaps**](https://github.com/cozystack/cozystack/pull/435) in #435)
+## Fixes
 
-## Documentation
-
-* **[website] [docs] Update managed apps reference for v1.3.0**: Documentation updates ([**@myasnikovdaniil**](https://github.com/cozystack/website/pull/507) in cozystack/website#507)
-* **[website] feat(blog): add Cozystack 1.3 release announcement**: Documentation updates ([**@tym83**](https://github.com/cozystack/website/pull/512) in cozystack/website#512)
-* **[website] fix(telemetry): use April 2026 data as placeholder for quarter and year periods**: Documentation updates ([**@tym83**](https://github.com/cozystack/website/pull/511) in cozystack/website#511)
-* **[website] fix(oss-health): correct April 2026 telemetry snapshot and pause fetch cron**: Documentation updates ([**@tym83**](https://github.com/cozystack/website/pull/508) in cozystack/website#508)
+* **fix(backups): move velero-configmap Role to velero chart**: The `backupstrategy-controller` (a default package) declared a Role/RoleBinding scoped to the `cozy-velero` namespace for managing `ResourceModifier` ConfigMaps. On bundles where Velero was not enabled, that namespace did not exist and the HelmRelease failed with `namespaces "cozy-velero" not found`, blocking installation. The Role/RoleBinding now lives in the velero chart, so it is created only when velero is actually deployed. ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2459, backport #2467)
 
 ## Contributors
 
-We thank the following contributors for this release:
+Thanks to everyone who contributed to this patch release:
 
 * [**@IvanHunters**](https://github.com/IvanHunters)
-* [**@app/github-actions**](https://github.com/app/github-actions)
-* [**@kvaps**](https://github.com/kvaps)
 * [**@myasnikovdaniil**](https://github.com/myasnikovdaniil)
 
 **Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.3.0...v1.3.1

From 1e0d8acb3553e0702a4c96a5775d4aadd6338c6b Mon Sep 17 00:00:00 2001
From: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
Date: Wed, 29 Apr 2026 12:46:18 +0500
Subject: [PATCH 146/151] docs(changelog): add 1.3.1 entries for backports
 landed after auto-gen

The auto-generated changelog only listed #2459/#2467 (velero-configmap
Role move). Five additional PRs were backported and merged into
release-1.3 between then and the v1.3.1 tag (2026-04-28):

- #2471/#2524 - fix(api): IDOR in TenantNamespace Get/Watch
- #2496/#2505 - feat(linstor): linstor-csi v1.10.6 (Protocol-C dual-attach)
- #2462/#2511 - fix(etcd): remove destructive post-upgrade hook
- #2421/#2491 - fix(kamaji): memory limits + startup probe
- #2498/#2518 - build(linstor): wire linstor-gui into root build target

Update the release date to match the actual tag (2026-04-28), rewrite
the intro paragraph, and add @kvaps to contributors.

Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
---
 docs/changelogs/v1.3.1.md | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/docs/changelogs/v1.3.1.md b/docs/changelogs/v1.3.1.md
index e8318ff4..0f149d0d 100644
--- a/docs/changelogs/v1.3.1.md
+++ b/docs/changelogs/v1.3.1.md
@@ -2,19 +2,36 @@
 https://github.com/cozystack/cozystack/releases/tag/v1.3.1
 -->
 
-# v1.3.1 (2026-04-23)
+# v1.3.1 (2026-04-28)
 
-A patch release with a single bug fix in the backup subsystem.
+Patch release covering a TenantNamespace IDOR fix in the API, a destructive `post-upgrade` hook removed from the etcd chart, kamaji controller stability, a `linstor-csi` bump that fixes live migration on Protocol-A/B DRBD resources, the missing `linstor-gui` build wiring, and a velero RBAC fix that unblocked installs on bundles without Velero.
+
+## Security
+
+* **fix(api): prevent IDOR in TenantNamespace Get and Watch handlers**: Two IDOR (Insecure Direct Object Reference) vulnerabilities allowed authenticated users to read TenantNamespace metadata they had no RoleBinding for. The `Get` and `Watch` handlers now go through a new `hasAccessToNamespace()` helper that lists RoleBindings scoped only to the target namespace (orders of magnitude faster than the previous all-cluster scan), returns `NotFound` instead of leaking existence on unauthorized access, and applies the same check on the `Watch` filter path. Includes regression tests for the unauthorized paths. ([**@IvanHunters**](https://github.com/IvanHunters) in #2471, backport #2524)
+
+## Features
+
+* **feat(linstor): bump linstor-csi to v1.10.6 with Protocol-C dual-attach fix**: Live migration of KubeVirt VMs on Protocol-A/B (async) DRBD volumes no longer fails with `Protocol C required`. `linstor-csi` v1.10.6 now installs a `Protocol=C` override on the resource-definition during dual-attach and reverts it on detach, so `replicated-async` StorageClasses and other Protocol-A/B resource groups support live migration without manual `drbdadm adjust` intervention. ([**@kvaps**](https://github.com/kvaps) in #2496, backport #2505)
 
 ## Fixes
 
 * **fix(backups): move velero-configmap Role to velero chart**: The `backupstrategy-controller` (a default package) declared a Role/RoleBinding scoped to the `cozy-velero` namespace for managing `ResourceModifier` ConfigMaps. On bundles where Velero was not enabled, that namespace did not exist and the HelmRelease failed with `namespaces "cozy-velero" not found`, blocking installation. The Role/RoleBinding now lives in the velero chart, so it is created only when velero is actually deployed. ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2459, backport #2467)
 
+* **fix(etcd): remove destructive post-upgrade cert-regeneration hook**: The etcd chart ran a `post-upgrade` Helm hook on every upgrade that deleted etcd TLS Secrets (`etcd-ca-tls`, `etcd-peer-ca-tls`, `etcd-client-tls`, `etcd-peer-tls`, `etcd-server-tls`) and then deleted etcd pods, forcing cert-manager to re-issue the entire etcd CA chain. On clusters with Kamaji-managed tenant control planes this put every tenant `kube-apiserver` into CrashLoopBackOff until each DataStore was manually re-reconciled. The hook was a one-shot `2.6.0 → 2.6.1` migration that became a permanent footgun once chart versioning moved to `0.0.0+<git-hash>` (always `< 2.6.1` per semver) and after the underlying `rotationPolicy: Always` issue was fixed in `47d81f70`. The hook is now removed entirely. ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2462, backport #2511)
+
+* **fix(kamaji): increase memory limits and add startup probe**: The kamaji controller frequently entered CrashLoopBackOff due to OOMKills (exit 137) within ~20–25 seconds of startup, with the readiness probe failing while the controller was still finishing initialization. Memory limit raised from 500Mi to 512Mi, request from 100Mi to 256Mi, and a 60-second startup probe (12 attempts × 5s periods) is added so the controller has room to boot before liveness/readiness probes engage. ([**@IvanHunters**](https://github.com/IvanHunters) in #2421, backport #2491)
+
+## Build
+
+* **build(linstor): include linstor-gui in root image build target**: The `linstor-gui` package (added in #2382) was never wired into the root `Makefile`'s `build:` target, so CI never built or published the image. `ghcr.io/cozystack/cozystack/linstor-gui` returned `NAME_UNKNOWN` and `values.yaml` stayed pinned to `tag: 2.3.0` without a digest. The missing build line is added so the next CI run publishes the image and the per-package `Makefile` digest-pins `values.yaml` automatically. ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2498, backport #2518)
+
 ## Contributors
 
 Thanks to everyone who contributed to this patch release:
 
 * [**@IvanHunters**](https://github.com/IvanHunters)
+* [**@kvaps**](https://github.com/kvaps)
 * [**@myasnikovdaniil**](https://github.com/myasnikovdaniil)
 
 **Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.3.0...v1.3.1

From ba8c1a0535d2e0462fb8269df0750ea6f57fdd0b Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Wed, 29 Apr 2026 11:13:12 +0300
Subject: [PATCH 147/151] docs(hami): drop HAMi#173 reference from glibc
 tracking issues
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

HAMi#173 was closed as "not planned" and only suggests a typo fix
(< 2.3.0 → < 2.30); it does not establish the actual 2.34 boundary.
HAMi-core#174 (symbol-level cause) and HAMi#1190 (empirical per-glibc
behavior) cover the same ground accurately.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 packages/system/hami/README.md | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/packages/system/hami/README.md b/packages/system/hami/README.md
index d648364d..aa80d48a 100644
--- a/packages/system/hami/README.md
+++ b/packages/system/hami/README.md
@@ -38,9 +38,8 @@ Most modern ML/AI base images (CUDA 12.x, PyTorch 2.x, TensorFlow 2.x) use Ubunt
 
 **Upstream tracking issues:**
 
-- [HAMi-core#174](https://github.com/Project-HAMi/HAMi-core/issues/174) — `_dl_sym` removal breaks HAMi-core on glibc >= 2.34
-- [HAMi#1190](https://github.com/Project-HAMi/HAMi/issues/1190) — degraded isolation across glibc versions
-- [HAMi#173](https://github.com/Project-HAMi/HAMi/issues/173) — documentation incorrectly states glibc < 2.30 (actual boundary is 2.34)
+- [HAMi-core#174](https://github.com/Project-HAMi/HAMi-core/issues/174) — `_dl_sym` removal in glibc 2.34 breaks HAMi-core's CUDA symbol resolution at the symbol level
+- [HAMi#1190](https://github.com/Project-HAMi/HAMi/issues/1190) — maintainer thread confirming the empirical per-glibc-version isolation behavior shown in the table above
 
 ### musl libc (Alpine) incompatibility
 

From 9876acec2fafbadeb09741f70e01250171a42a48 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Wed, 29 Apr 2026 11:13:24 +0300
Subject: [PATCH 148/151] docs(hami): clarify gpu-operator devicePlugin
 override behavior

The HAMi-driven default of disabling gpu-operator's device plugin is
applied via valuesOverride and can be re-enabled by users for advanced
topologies (mixed HAMi / vanilla NVIDIA device plugin pools). Document
this explicitly so the README aligns with the existing template merge
order.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 packages/system/hami/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/system/hami/README.md b/packages/system/hami/README.md
index aa80d48a..68669ac7 100644
--- a/packages/system/hami/README.md
+++ b/packages/system/hami/README.md
@@ -57,7 +57,7 @@ addons:
     enabled: true
 ```
 
-When HAMi is enabled, GPU Operator's built-in device plugin is automatically disabled to avoid conflicts.
+When HAMi is enabled, GPU Operator's built-in device plugin is automatically disabled to avoid conflicts. This default is preserved by setting `addons.gpuOperator.valuesOverride.gpu-operator.devicePlugin.enabled: false`; advanced topologies that partition GPU pools (e.g. some nodes use HAMi while others run the standard NVIDIA device plugin via node selectors) can re-enable it explicitly through `valuesOverride`.
 
 ### Requesting fractional GPU resources
 

From d5caffcc8de0177b248899b03391c312d20dde54 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Wed, 29 Apr 2026 11:13:42 +0300
Subject: [PATCH 149/151] fix(kubernetes): gate HAMi HelmRelease on
 _namespace.etcd

Match the pattern used by every other tenant-cluster addon HelmRelease
(gpu-operator, cilium, fluxcd, ...): only render when the upstream etcd
namespace is ready, not just when the addon is enabled. Add values-ci.yaml
to the HAMi test suite so the new gate has the fixture it needs.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 packages/apps/kubernetes/templates/helmreleases/hami.yaml | 2 +-
 packages/apps/kubernetes/tests/hami_test.yaml             | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/apps/kubernetes/templates/helmreleases/hami.yaml b/packages/apps/kubernetes/templates/helmreleases/hami.yaml
index 1f7a5358..f1538c7c 100644
--- a/packages/apps/kubernetes/templates/helmreleases/hami.yaml
+++ b/packages/apps/kubernetes/templates/helmreleases/hami.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.addons.hami.enabled }}
+{{- if and .Values.addons.hami.enabled .Values._namespace.etcd }}
 {{- if not .Values.addons.gpuOperator.enabled }}
 {{- fail "addons.hami requires addons.gpuOperator to be enabled" }}
 {{- end }}
diff --git a/packages/apps/kubernetes/tests/hami_test.yaml b/packages/apps/kubernetes/tests/hami_test.yaml
index bdbe990f..27f7c75b 100644
--- a/packages/apps/kubernetes/tests/hami_test.yaml
+++ b/packages/apps/kubernetes/tests/hami_test.yaml
@@ -1,6 +1,8 @@
 suite: HAMi HelmRelease tests
 templates:
   - templates/helmreleases/hami.yaml
+values:
+  - values-ci.yaml
 tests:
   - it: should not render when hami is disabled
     set:

From 9b5848ed268bfe3bc04b42bae38cc724c00a3bf0 Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Wed, 29 Apr 2026 11:13:55 +0300
Subject: [PATCH 150/151] fix(platform): register HAMi as
 kubernetes-application component
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The HelmRelease at packages/apps/kubernetes/templates/helmreleases/hami.yaml
references chartRef.name 'cozystack-kubernetes-application-kubevirt-kubernetes-hami',
but that component was missing from the kubernetes-application PackageSource.
The HelmRelease would sit in Stalled: ArtifactNotFound at install time.

Add kubernetes-hami next to kubernetes-gpu-operator under variant kubevirt,
mirroring the existing pattern for every other tenant-cluster addon. The
standalone cozystack.hami PackageSource is retained — same shape as
gpu-operator, which is registered both standalone (for bundles/iaas) and
as a kubernetes-application component (for tenant-cluster HelmReleases).

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 packages/core/platform/sources/kubernetes-application.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/core/platform/sources/kubernetes-application.yaml b/packages/core/platform/sources/kubernetes-application.yaml
index 9787cb19..088383ad 100644
--- a/packages/core/platform/sources/kubernetes-application.yaml
+++ b/packages/core/platform/sources/kubernetes-application.yaml
@@ -52,6 +52,8 @@ spec:
           path: system/cilium
         - name: kubernetes-gpu-operator
           path: system/gpu-operator
+        - name: kubernetes-hami
+          path: system/hami
         - name: kubernetes-vertical-pod-autoscaler
           path: system/vertical-pod-autoscaler
         - name: kubernetes-prometheus-operator-crds

From 5d58d0f340f7157370305cc469317f4fa21944af Mon Sep 17 00:00:00 2001
From: Arsolitt <arsolitt@gmail.com>
Date: Wed, 29 Apr 2026 11:14:41 +0300
Subject: [PATCH 151/151] chore(hami): document and partially automate vendor
 patches in Makefile
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The `update:` recipe now reproduces the top-level vendoring overrides
(remove broken hami-dra subchart, clear Chart.yaml dependencies, strip
dra/hami-dra/podSecurityPolicy from upstream values.yaml) after
`helm pull`, so they no longer silently disappear on the next bump.

Template-level patches (DRA guards in scheduler/*, indent fix in
device-plugin/monitorservice.yaml) are documented with rationale and
commit references — they remain a manual step because automating them
would be fragile across upstream template restructures.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
---
 packages/system/hami/Makefile | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/packages/system/hami/Makefile b/packages/system/hami/Makefile
index b7a169c2..83663a66 100644
--- a/packages/system/hami/Makefile
+++ b/packages/system/hami/Makefile
@@ -4,8 +4,40 @@ export NAMESPACE=cozy-$(NAME)
 include ../../../hack/common-envs.mk
 include ../../../hack/package.mk
 
+# When bumping the HAMi version, run `make update` and then review
+# the resulting diff in `charts/hami/`. The recipe below reproduces the
+# top-level vendoring overrides automatically:
+#
+#   1. Removes the broken hami-dra subchart. Upstream's NVIDIA DRA driver
+#      path requires kubelet DRA support that cozystack does not enable
+#      and has no upstream fix tracked. See commit 3c5521e.
+#   2. Empties Chart.yaml dependencies and drops Chart.lock so Helm does
+#      not try to re-pull hami-dra at build time. See commit 2734dc0.
+#   3. Strips dra/hami-dra/podSecurityPolicy blocks from the upstream
+#      values.yaml since the corresponding code paths are gone. PSP is
+#      removed from Kubernetes 1.25+ and is unused by cozystack.
+#
+# Template-level patches are NOT reproduced automatically:
+#
+#   * Scheduler templates have `{{- if .Values.dra.enabled }}` blocks
+#     that need to be removed because the dra value is gone (commit
+#     2734dc0 stripped them).
+#   * device-plugin/monitorservice.yaml uses `indent` with leading
+#     whitespace; it must be rewritten to `nindent` for the labels block
+#     to render correctly when devicePlugin.service.labels is set
+#     (commit 3685254).
+#
+# After `make update`, run `git diff -- charts/hami/templates/` and
+# replay these template patches against the new upstream version, then
+# verify with `helm unittest`. If upstream restructured the affected
+# files, the patches may need to be redesigned rather than reapplied.
+
 update:
 	rm -rf charts
 	helm repo add hami-charts https://project-hami.github.io/HAMi/
 	helm repo update hami-charts
 	helm pull hami-charts/hami --untar --untardir charts
+	rm -rf charts/hami/charts/hami-dra
+	yq --inplace '.dependencies = []' charts/hami/Chart.yaml
+	rm -f charts/hami/Chart.lock
+	yq --inplace 'del(.dra) | del(.["hami-dra"]) | del(.podSecurityPolicy)' charts/hami/values.yaml