fix(harbor): merge BucketInfo at values root, drop targetPath

The previous fix (a21c18f9) sourced BucketInfo via Flux valuesFrom with
`targetPath: bucket.bucketInfo`. Flux runs values with `targetPath`
through Helm's `strvals.ParseInto`, which splits the value on commas as
list separators. The COSI BucketInfo JSON is comma-rich, so values
resolution bailed:

  could not resolve Secret chart values reference
  'tenant-X/harbor-X-registry-bucket' with key 'BucketInfo':
  key "\"spec\":{\"bucketName\":\"bucket-1785...\""
  has no value (cannot end with ,)

Drop `targetPath`. With only `valuesKey: BucketInfo`, helm-controller
unmarshals the value as YAML and merges at the chart's values root, so
JSON commas stay nested instead of being split. The system chart's
bucket-secret.yaml now reads `.Values.spec.bucketName` /
`.Values.spec.secretS3.{accessKeyID,accessSecretKey,endpoint}`,
guarded by nested `with` blocks so the registry-s3 Secret renders only
when secretS3 has been populated.

Gating semantics from the previous fix are preserved: with the default
`optional: false`, helm-controller still refuses to compose values
until the COSI BucketAccess controller writes `BucketInfo` into the
Secret, and the value's content still drives the HR config-digest.

Verified via `helm template` against the system chart with three
scenarios: missing `spec` and `spec` without `secretS3` render
nothing; full BucketInfo renders all five S3 env keys correctly.

Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
Assisted-By: Claude <noreply@anthropic.com>

.github/workflows/codegen-drift.yml

@@ -1,61 +0,0 @@
-name: Codegen Drift Check
-
-on:
-  pull_request:
-    types: [opened, synchronize, reopened]
-    paths:
-      - 'api/**'
-      - 'pkg/apis/**'
-      - 'pkg/generated/**'
-      - 'internal/crdinstall/manifests/**'
-      - 'packages/system/cozystack-controller/definitions/**'
-      - 'packages/system/application-definition-crd/definition/**'
-      - 'packages/system/backup-controller/definitions/**'
-      - 'packages/system/backupstrategy-controller/definitions/**'
-      - 'hack/update-codegen.sh'
-      - 'hack/boilerplate.go.txt'
-      - 'Makefile'
-      - 'go.mod'
-      - 'go.sum'
-      - '.github/workflows/codegen-drift.yml'
-
-concurrency:
-  group: codegen-drift-${{ github.workflow }}-${{ github.event.pull_request.number }}
-  cancel-in-progress: true
-
-jobs:
-  codegen-drift:
-    name: Verify generated code is up to date
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Set up Go
-        uses: actions/setup-go@v5
-        with:
-          go-version-file: go.mod
-          cache: true
-
-      - name: Pre-fetch k8s.io/code-generator module
-        # hack/update-codegen.sh sources kube_codegen.sh from the Go module cache.
-        # The module is not declared in go.mod, so fetch it explicitly at the
-        # version pinned in the script.
-        run: |
-          version=$(grep -oP 'code-generator@\Kv[0-9.]+' hack/update-codegen.sh)
-          tmpdir=$(mktemp -d)
-          cd "$tmpdir"
-          go mod init codegen-fetch
-          go get "k8s.io/code-generator@${version}"
-
-      - name: Run make generate
-        run: make generate
-
-      - name: Fail on drift
-        run: |
-          if [ -n "$(git status --porcelain)" ]; then
-            echo "::error::'make generate' produced changes. Run 'make generate' locally and commit the result."
-            git status --short
-            git diff --color=always
-            exit 1
-          fi

.github/workflows/pull-requests.yaml

@@ -227,27 +227,18 @@ jobs:
           until make SANDBOX_NAME=$SANDBOX_NAME prepare-env; do
             attempt=$((attempt + 1))
             if [ $attempt -ge 3 ]; then
-              echo "❌ Attempt $attempt failed, exiting..."
+              echo "Attempt $attempt failed, exiting..."
               exit 1
             fi
-            echo "❌ Attempt $attempt failed, retrying..."
+            echo "Attempt $attempt failed, retrying..."
           done
-          echo "✅ The task completed successfully after $attempt attempts"
+          echo "Prepare environment completed after $attempt attempts"
 
       # ▸ Install Cozystack
       - name: Install Cozystack into sandbox
         run: |
           cd /tmp/$SANDBOX_NAME
-          attempt=0
-          until make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME install-cozystack; do
-            attempt=$((attempt + 1))
-            if [ $attempt -ge 3 ]; then
-              echo "❌ Attempt $attempt failed, exiting..."
-              exit 1
-            fi
-            echo "❌ Attempt $attempt failed, retrying..."
-          done
-          echo "✅ The task completed successfully after $attempt attempts"
+          make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME install-cozystack
 
       - name: Run OpenAPI tests
         run: |
@@ -262,23 +253,18 @@ jobs:
           failed_tests=""
           for app in $(ls hack/e2e-apps/*.bats | xargs -n1 basename | cut -d. -f1); do
             echo "::group::Testing $app"
-            attempt=0
-            success=false
-            until [ $attempt -ge 3 ]; do
-              if make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-apps-$app; then
-                success=true
-                break
-              fi
-              attempt=$((attempt + 1))
-              echo "❌ Attempt $attempt failed, retrying..."
-            done
-            if [ "$success" = true ]; then
+            if make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-apps-$app; then
+              echo "::endgroup::"
               echo "✅ Test $app completed successfully"
             else
-              echo "❌ Test $app failed after $attempt attempts"
+              echo "::endgroup::"
+              echo "❌ Test $app failed (no retry — see diagnostics below)"
               failed_tests="$failed_tests $app"
+              echo "::group::Diagnostics for $app"
+              kubectl get hr -A -o wide 2>&1 | tail -50 || true
+              kubectl get events -A --sort-by=.lastTimestamp 2>&1 | tail -30 || true
+              echo "::endgroup::"
             fi
-            echo "::endgroup::"
           done
           if [ -n "$failed_tests" ]; then
             echo "❌ Failed tests:$failed_tests"

.pre-commit-config.yaml

@@ -1,17 +1,6 @@
 repos:
 - repo: local
   hooks:
-    - id: run-make-generate-root
-      name: Run 'make generate' at repo root
-      entry: |
-        flock -x .git/pre-commit.lock sh -c '
-          echo "Running make generate at repo root"
-          make generate || exit $?
-          git diff --color=always | cat
-        '
-      language: system
-      files: ^(api/|pkg/apis/|pkg/generated/|internal/crdinstall/manifests/|packages/system/cozystack-controller/definitions/|packages/system/application-definition-crd/definition/|packages/system/backup-controller/definitions/|packages/system/backupstrategy-controller/definitions/|hack/update-codegen\.sh$|hack/boilerplate\.go\.txt$|Makefile$)
-      pass_filenames: false
     - id: run-make-generate
       name: Run 'make generate' in all app directories
       entry: |

Makefile

@@ -22,7 +22,6 @@ build: build-deps
 	make -C packages/system/lineage-controller-webhook image
 	make -C packages/system/cilium image
 	make -C packages/system/linstor image
-	make -C packages/system/linstor-gui image
 	make -C packages/system/kubeovn-webhook image
 	make -C packages/system/kubeovn-plunger image
 	make -C packages/system/dashboard image

api/apps/v1alpha1/kubernetes/types.go

@@ -69,9 +69,6 @@ type Addons struct {
 	// NVIDIA GPU Operator.
 	// +kubebuilder:default:={}
 	GpuOperator GPUOperatorAddon `json:"gpuOperator"`
-	// HAMi GPU virtualization middleware.
-	// +kubebuilder:default:={}
-	Hami HAMiAddon `json:"hami"`
 	// Ingress-NGINX controller.
 	// +kubebuilder:default:={}
 	IngressNginx IngressNginxAddon `json:"ingressNginx"`
@@ -163,15 +160,6 @@ type GatewayAPIAddon struct {
 	Enabled bool `json:"enabled"`
 }
 
-type HAMiAddon struct {
-	// Enable HAMi (requires GPU Operator).
-	// +kubebuilder:default:=false
-	Enabled bool `json:"enabled"`
-	// Custom Helm values overrides.
-	// +kubebuilder:default:={}
-	ValuesOverride k8sRuntime.RawExtension `json:"valuesOverride"`
-}
-
 type Images struct {
 	// Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.
 	// +kubebuilder:default:=""

api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go

@@ -49,7 +49,6 @@ func (in *Addons) DeepCopyInto(out *Addons) {
 	in.Fluxcd.DeepCopyInto(&out.Fluxcd)
 	out.GatewayAPI = in.GatewayAPI
 	in.GpuOperator.DeepCopyInto(&out.GpuOperator)
-	in.Hami.DeepCopyInto(&out.Hami)
 	in.IngressNginx.DeepCopyInto(&out.IngressNginx)
 	in.MonitoringAgents.DeepCopyInto(&out.MonitoringAgents)
 	in.Velero.DeepCopyInto(&out.Velero)
@@ -136,7 +135,6 @@ func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) {
 	}
 	in.Addons.DeepCopyInto(&out.Addons)
 	in.ControlPlane.DeepCopyInto(&out.ControlPlane)
-	out.Images = in.Images
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigSpec.
@@ -262,37 +260,6 @@ func (in *GatewayAPIAddon) DeepCopy() *GatewayAPIAddon {
 	return out
 }
 
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *HAMiAddon) DeepCopyInto(out *HAMiAddon) {
-	*out = *in
-	in.ValuesOverride.DeepCopyInto(&out.ValuesOverride)
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HAMiAddon.
-func (in *HAMiAddon) DeepCopy() *HAMiAddon {
-	if in == nil {
-		return nil
-	}
-	out := new(HAMiAddon)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *Images) DeepCopyInto(out *Images) {
-	*out = *in
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Images.
-func (in *Images) DeepCopy() *Images {
-	if in == nil {
-		return nil
-	}
-	out := new(Images)
-	in.DeepCopyInto(out)
-	return out
-}
-
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *IngressNginxAddon) DeepCopyInto(out *IngressNginxAddon) {
 	*out = *in

api/apps/v1alpha1/postgresql/types.go

@@ -92,9 +92,6 @@ type Bootstrap struct {
 	// Timestamp (RFC3339) for point-in-time recovery; empty means latest.
 	// +kubebuilder:default:=""
 	RecoveryTime string `json:"recoveryTime,omitempty"`
-	// Barman server name (S3 path prefix) used by the original cluster when writing backups. Set this only when the original cluster had an explicit barmanObjectStore.serverName that differed from its Kubernetes resource name.
-	// +kubebuilder:default:=""
-	ServerName string `json:"serverName,omitempty"`
 }
 
 type Database struct {

api/apps/v1alpha1/vminstance/types.go

@@ -26,9 +26,6 @@ type ConfigSpec struct {
 	// Ports to forward from outside the cluster.
 	// +kubebuilder:default:={22}
 	ExternalPorts []int `json:"externalPorts,omitempty"`
-	// Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect.
-	// +kubebuilder:default:=true
-	ExternalAllowICMP bool `json:"externalAllowICMP"`
 	// Requested running state of the VirtualMachineInstance
 	// +kubebuilder:default:="Always"
 	RunStrategy RunStrategy `json:"runStrategy"`

api/backups/v1alpha1/zz_generated.deepcopy.go

@@ -620,11 +620,6 @@ func (in *RestoreJobSpec) DeepCopyInto(out *RestoreJobSpec) {
 		*out = new(v1.TypedLocalObjectReference)
 		(*in).DeepCopyInto(*out)
 	}
-	if in.Options != nil {
-		in, out := &in.Options, &out.Options
-		*out = new(runtime.RawExtension)
-		(*in).DeepCopyInto(*out)
-	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestoreJobSpec.

cmd/cozystack-operator/main.go

@@ -83,6 +83,11 @@ func main() {
 	var disableTelemetry bool
 	var telemetryEndpoint string
 	var telemetryInterval string
+	var helmReleaseInterval string
+	var helmReleaseRetryInterval string
+	var helmReleaseInstallTimeout string
+	var helmReleaseUpgradeTimeout string
+	var helmReleaseMaxHistory int
 	var cozyValuesSecretName string
 	var cozyValuesSecretNamespace string
 	var cozyValuesNamespaceSelector string
@@ -107,6 +112,28 @@ func main() {
 		"Endpoint for sending telemetry data")
 	flag.StringVar(&telemetryInterval, "telemetry-interval", "15m",
 		"Interval between telemetry data collection (e.g. 15m, 1h)")
+	flag.StringVar(&helmReleaseInterval, "helmrelease-interval", "5m",
+		"Reconcile interval applied to HelmReleases created by the Package reconciler. "+
+			"Lower values speed up dependency-blocked retries (e.g. during E2E install) at the cost of "+
+			"controller load. Production default 5m matches existing behaviour.")
+	flag.StringVar(&helmReleaseRetryInterval, "helmrelease-retry-interval", "30s",
+		"Retry interval applied to Install.Strategy and Upgrade.Strategy of HelmReleases created "+
+			"by the Package reconciler. With Strategy.Name=RetryOnFailure, this controls how long the "+
+			"controller waits between failed install/upgrade attempts. Decoupled from --helmrelease-interval "+
+			"(which is the healthy reconcile cadence) so failures recover fast without polling healthy "+
+			"releases at the same fast cadence.")
+	flag.StringVar(&helmReleaseInstallTimeout, "helmrelease-install-timeout", "10m",
+		"Timeout for the Helm install action of HelmReleases created by the Package reconciler "+
+			"(Spec.Install.Timeout). Bounds how long an individual Kubernetes operation (Job/hook/wait) "+
+			"may take during install.")
+	flag.StringVar(&helmReleaseUpgradeTimeout, "helmrelease-upgrade-timeout", "10m",
+		"Timeout for the Helm upgrade action of HelmReleases created by the Package reconciler "+
+			"(Spec.Upgrade.Timeout). Bounds how long an individual Kubernetes operation (Job/hook/wait) "+
+			"may take during upgrade.")
+	flag.IntVar(&helmReleaseMaxHistory, "helmrelease-max-history", 5,
+		"Number of release revisions Helm keeps for HelmReleases created by the Package reconciler "+
+			"(Spec.MaxHistory). 0 means unlimited; 5 matches Helm's default. Lower values reduce "+
+			"per-release Secret accumulation in clusters that bounce HRs frequently (e.g. E2E sandboxes).")
 	flag.StringVar(&platformSourceURL, "platform-source-url", "", "Platform source URL (oci:// or https://). If specified, generates OCIRepository or GitRepository resource.")
 	flag.StringVar(&platformSourceName, "platform-source-name", "cozystack-platform", "Name for the generated platform source resource and PackageSource")
 	flag.StringVar(&platformSourceRef, "platform-source-ref", "", "Reference specification as key=value pairs (e.g., 'branch=main' or 'digest=sha256:...,tag=v1.0'). For OCI: digest, semver, semverFilter, tag. For Git: branch, tag, semver, name, commit.")
@@ -122,6 +149,31 @@ func main() {
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
+	hrIntervalDuration, err := time.ParseDuration(helmReleaseInterval)
+	if err != nil {
+		setupLog.Error(err, "invalid --helmrelease-interval value", "value", helmReleaseInterval)
+		os.Exit(1)
+	}
+	hrRetryIntervalDuration, err := time.ParseDuration(helmReleaseRetryInterval)
+	if err != nil {
+		setupLog.Error(err, "invalid --helmrelease-retry-interval value", "value", helmReleaseRetryInterval)
+		os.Exit(1)
+	}
+	hrInstallTimeoutDuration, err := time.ParseDuration(helmReleaseInstallTimeout)
+	if err != nil {
+		setupLog.Error(err, "invalid --helmrelease-install-timeout value", "value", helmReleaseInstallTimeout)
+		os.Exit(1)
+	}
+	hrUpgradeTimeoutDuration, err := time.ParseDuration(helmReleaseUpgradeTimeout)
+	if err != nil {
+		setupLog.Error(err, "invalid --helmrelease-upgrade-timeout value", "value", helmReleaseUpgradeTimeout)
+		os.Exit(1)
+	}
+	if helmReleaseMaxHistory < 0 {
+		setupLog.Error(fmt.Errorf("--helmrelease-max-history must be >= 0"), "invalid value", "value", helmReleaseMaxHistory)
+		os.Exit(1)
+	}
+
 	config := ctrl.GetConfigOrDie()
 
 	// Create a direct client (without cache) for pre-start operations
@@ -258,8 +310,13 @@ func main() {
 
 	// Setup Package reconciler
 	if err := (&operator.PackageReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
+		Client:                    mgr.GetClient(),
+		Scheme:                    mgr.GetScheme(),
+		HelmReleaseInterval:       hrIntervalDuration,
+		HelmReleaseRetryInterval:  hrRetryIntervalDuration,
+		HelmReleaseInstallTimeout: hrInstallTimeoutDuration,
+		HelmReleaseUpgradeTimeout: hrUpgradeTimeoutDuration,
+		HelmReleaseMaxHistory:     helmReleaseMaxHistory,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Package")
 		os.Exit(1)

dashboards/gpu/gpu-efficiency.json

@@ -1,819 +0,0 @@
-{
-  "uid": "gpu-efficiency",
-  "title": "GPU Efficiency Score",
-  "description": "Tensor saturation, util/watt and throttling — reveals inefficient GPU workloads",
-  "tags": [
-    "gpu",
-    "efficiency",
-    "finops"
-  ],
-  "timezone": "browser",
-  "editable": true,
-  "graphTooltip": 1,
-  "time": {
-    "from": "now-1h",
-    "to": "now"
-  },
-  "fiscalYearStartMonth": 0,
-  "schemaVersion": 42,
-  "panels": [
-    {
-      "type": "row",
-      "collapsed": false,
-      "title": "Overall efficiency metrics",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 0
-      },
-      "id": 1,
-      "panels": []
-    },
-    {
-      "type": "stat",
-      "id": 2,
-      "targets": [
-        {
-          "expr": "avg(gpu:tensor_saturation:avg5m) * 100",
-          "refId": "A"
-        }
-      ],
-      "title": "Avg Tensor Saturation",
-      "description": "Mean tensor core saturation across all GPUs. \u003c10% means GPUs are used inefficiently (workloads could move to CPU or optimize their code). Cluster-wide — DCGM metrics cannot be attributed to workload namespaces.",
-      "transparent": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "gridPos": {
-        "h": 5,
-        "w": 8,
-        "x": 0,
-        "y": 1
-      },
-      "repeatDirection": "h",
-      "options": {
-        "graphMode": "area",
-        "colorMode": "background",
-        "justifyMode": "auto",
-        "textMode": "value",
-        "wideLayout": true,
-        "showPercentChange": false,
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ]
-        },
-        "percentChangeColorMode": "standard",
-        "orientation": ""
-      },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "percent",
-          "min": 0,
-          "max": 100,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "value": null,
-                "color": "red"
-              },
-              {
-                "value": 10,
-                "color": "orange"
-              },
-              {
-                "value": 30,
-                "color": "yellow"
-              },
-              {
-                "value": 60,
-                "color": "green"
-              }
-            ]
-          }
-        },
-        "overrides": []
-      }
-    },
-    {
-      "type": "stat",
-      "id": 3,
-      "targets": [
-        {
-          "expr": "avg(gpu:util_per_watt:avg5m)",
-          "refId": "A"
-        }
-      ],
-      "title": "Avg Utilization per Watt",
-      "description": "NVML utilization % per watt across all GPUs. Higher value = more efficient workload. Cluster-wide — DCGM metrics cannot be attributed to workload namespaces.",
-      "transparent": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "gridPos": {
-        "h": 5,
-        "w": 8,
-        "x": 8,
-        "y": 1
-      },
-      "repeatDirection": "h",
-      "options": {
-        "graphMode": "area",
-        "colorMode": "background",
-        "justifyMode": "auto",
-        "textMode": "value",
-        "wideLayout": true,
-        "showPercentChange": false,
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ]
-        },
-        "percentChangeColorMode": "standard",
-        "orientation": ""
-      },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "none",
-          "decimals": 3,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "value": null,
-                "color": "red"
-              },
-              {
-                "value": 0.5,
-                "color": "yellow"
-              },
-              {
-                "value": 1,
-                "color": "green"
-              }
-            ]
-          }
-        },
-        "overrides": []
-      }
-    },
-    {
-      "type": "stat",
-      "id": 4,
-      "targets": [
-        {
-          "expr": "avg(gpu:power_throttle_fraction:rate5m)",
-          "refId": "A"
-        }
-      ],
-      "title": "Avg Power Throttling",
-      "description": "Fraction of time GPUs hit the TDP cap and lose performance. \u003e5% means tenants underutilize billed FLOPS. Cluster-wide — rule aggregates by (Hostname, gpu, UUID) and drops namespace label.",
-      "transparent": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "gridPos": {
-        "h": 5,
-        "w": 8,
-        "x": 16,
-        "y": 1
-      },
-      "repeatDirection": "h",
-      "options": {
-        "graphMode": "area",
-        "colorMode": "background",
-        "justifyMode": "auto",
-        "textMode": "value",
-        "wideLayout": true,
-        "showPercentChange": false,
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ]
-        },
-        "percentChangeColorMode": "standard",
-        "orientation": ""
-      },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "percentunit",
-          "decimals": 2,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "value": null,
-                "color": "green"
-              },
-              {
-                "value": 0.05,
-                "color": "yellow"
-              },
-              {
-                "value": 0.2,
-                "color": "red"
-              }
-            ]
-          }
-        },
-        "overrides": []
-      }
-    },
-    {
-      "type": "row",
-      "collapsed": false,
-      "title": "NVML vs Tensor (mismatch detector)",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 6
-      },
-      "id": 10,
-      "panels": []
-    },
-    {
-      "type": "timeseries",
-      "id": 11,
-      "targets": [
-        {
-          "expr": "DCGM_FI_DEV_GPU_UTIL",
-          "legendFormat": "{{Hostname}}/{{gpu}}",
-          "refId": "A"
-        }
-      ],
-      "title": "NVML GPU Utilization",
-      "description": "Classic utilization metric. Shows activity of any engine (SM, copy, encoder). Cluster-wide — DCGM namespace is the exporter's own namespace, not the workload namespace.",
-      "transparent": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 7
-      },
-      "repeatDirection": "h",
-      "options": {
-        "legend": {
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": false,
-          "calcs": [
-            "mean",
-            "max"
-          ]
-        },
-        "tooltip": {
-          "mode": "multi",
-          "sort": ""
-        }
-      },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "percent",
-          "min": 0,
-          "max": 100,
-          "custom": {
-            "drawStyle": "line",
-            "lineInterpolation": "linear",
-            "fillOpacity": 10,
-            "showPoints": "never"
-          }
-        },
-        "overrides": []
-      }
-    },
-    {
-      "type": "timeseries",
-      "id": 12,
-      "targets": [
-        {
-          "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE * 100",
-          "legendFormat": "{{Hostname}}/{{gpu}}",
-          "refId": "A"
-        }
-      ],
-      "title": "Tensor Pipe Active",
-      "description": "Real tensor core load (HMMA). For AI/LLM inference it should be ≥20%, otherwise the workload is unoptimized. Cluster-wide — DCGM namespace is the exporter's own namespace, not the workload namespace.",
-      "transparent": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 7
-      },
-      "repeatDirection": "h",
-      "options": {
-        "legend": {
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": false,
-          "calcs": [
-            "mean",
-            "max"
-          ]
-        },
-        "tooltip": {
-          "mode": "multi",
-          "sort": ""
-        }
-      },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "percent",
-          "min": 0,
-          "max": 100,
-          "custom": {
-            "drawStyle": "line",
-            "lineInterpolation": "linear",
-            "fillOpacity": 10,
-            "showPoints": "never"
-          }
-        },
-        "overrides": []
-      }
-    },
-    {
-      "type": "row",
-      "collapsed": false,
-      "title": "Per-GPU ranking",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 15
-      },
-      "id": 20,
-      "panels": []
-    },
-    {
-      "type": "table",
-      "id": 21,
-      "targets": [
-        {
-          "expr": "topk(20, gpu:tensor_saturation:avg5m * 100)",
-          "instant": true,
-          "range": false,
-          "format": "table",
-          "refId": "A"
-        }
-      ],
-      "title": "Tensor Saturation per GPU (5m avg)",
-      "description": "Which GPUs are exercising tensor cores and which are not. Sorted descending. Grouped by Hostname/gpu/UUID — DCGM metrics cannot be attributed to workload namespaces.",
-      "transparent": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 16
-      },
-      "repeatDirection": "h",
-      "transformations": [
-        {
-          "id": "organize",
-          "options": {
-            "excludeByName": {
-              "DCGM_FI_DRIVER_VERSION": true,
-              "Time": true,
-              "__name__": true,
-              "cluster": true,
-              "container": true,
-              "device": true,
-              "endpoint": true,
-              "gpu_driver_version": true,
-              "instance": true,
-              "job": true,
-              "modelName": true,
-              "namespace": true,
-              "pci_bus_id": true,
-              "pod": true,
-              "prometheus": true,
-              "service": true,
-              "tenant": true,
-              "tier": true,
-              "uid": true,
-              "unit": true
-            },
-            "indexByName": {
-              "Hostname": 0,
-              "UUID": 2,
-              "Value": 3,
-              "gpu": 1
-            },
-            "renameByName": {
-              "Hostname": "Node",
-              "UUID": "UUID",
-              "Value": "Saturation",
-              "gpu": "GPU"
-            }
-          }
-        }
-      ],
-      "options": {
-        "frameIndex": 0,
-        "showHeader": true,
-        "showTypeIcons": false,
-        "sortBy": [
-          {
-            "displayName": "Saturation",
-            "desc": true
-          }
-        ],
-        "footer": {
-          "show": false,
-          "reducer": []
-        },
-        "cellHeight": "sm"
-      },
-      "fieldConfig": {
-        "defaults": {},
-        "overrides": [
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Saturation"
-            },
-            "properties": [
-              {
-                "id": "unit",
-                "value": "percent"
-              },
-              {
-                "id": "min",
-                "value": 0
-              },
-              {
-                "id": "max",
-                "value": 100
-              },
-              {
-                "id": "custom.cellOptions",
-                "value": {
-                  "mode": "gradient",
-                  "type": "gauge"
-                }
-              },
-              {
-                "id": "thresholds",
-                "value": {
-                  "mode": "absolute",
-                  "steps": [
-                    {
-                      "color": "red"
-                    },
-                    {
-                      "color": "orange",
-                      "value": 10
-                    },
-                    {
-                      "color": "yellow",
-                      "value": 30
-                    },
-                    {
-                      "color": "green",
-                      "value": 60
-                    }
-                  ]
-                }
-              }
-            ]
-          }
-        ]
-      }
-    },
-    {
-      "type": "table",
-      "id": 22,
-      "targets": [
-        {
-          "expr": "topk(20, gpu:util_per_watt:avg5m)",
-          "instant": true,
-          "range": false,
-          "format": "table",
-          "refId": "A"
-        }
-      ],
-      "title": "Utilization / Watt per GPU (5m avg)",
-      "description": "How efficiently each GPU spends watts. Low value = poor optimization. Grouped by Hostname/gpu/UUID — DCGM metrics cannot be attributed to workload namespaces.",
-      "transparent": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 16
-      },
-      "repeatDirection": "h",
-      "transformations": [
-        {
-          "id": "organize",
-          "options": {
-            "excludeByName": {
-              "DCGM_FI_DRIVER_VERSION": true,
-              "Time": true,
-              "__name__": true,
-              "cluster": true,
-              "container": true,
-              "device": true,
-              "endpoint": true,
-              "gpu_driver_version": true,
-              "instance": true,
-              "job": true,
-              "modelName": true,
-              "namespace": true,
-              "pci_bus_id": true,
-              "pod": true,
-              "prometheus": true,
-              "service": true,
-              "tenant": true,
-              "tier": true,
-              "uid": true,
-              "unit": true
-            },
-            "indexByName": {
-              "Hostname": 0,
-              "UUID": 2,
-              "Value": 3,
-              "gpu": 1
-            },
-            "renameByName": {
-              "Hostname": "Node",
-              "UUID": "UUID",
-              "Value": "Util/Watt",
-              "gpu": "GPU"
-            }
-          }
-        }
-      ],
-      "options": {
-        "frameIndex": 0,
-        "showHeader": true,
-        "showTypeIcons": false,
-        "sortBy": [
-          {
-            "displayName": "Util/Watt",
-            "desc": true
-          }
-        ],
-        "footer": {
-          "show": false,
-          "reducer": []
-        },
-        "cellHeight": "sm"
-      },
-      "fieldConfig": {
-        "defaults": {},
-        "overrides": [
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Util/Watt"
-            },
-            "properties": [
-              {
-                "id": "unit",
-                "value": "none"
-              },
-              {
-                "id": "decimals",
-                "value": 3
-              },
-              {
-                "id": "min",
-                "value": 0
-              },
-              {
-                "id": "max",
-                "value": 1.5
-              },
-              {
-                "id": "custom.cellOptions",
-                "value": {
-                  "mode": "gradient",
-                  "type": "gauge"
-                }
-              },
-              {
-                "id": "thresholds",
-                "value": {
-                  "mode": "absolute",
-                  "steps": [
-                    {
-                      "color": "red"
-                    },
-                    {
-                      "color": "yellow",
-                      "value": 0.5
-                    },
-                    {
-                      "color": "green",
-                      "value": 1
-                    }
-                  ]
-                }
-              }
-            ]
-          }
-        ]
-      }
-    },
-    {
-      "type": "row",
-      "collapsed": false,
-      "title": "Throttling",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 24
-      },
-      "id": 30,
-      "panels": []
-    },
-    {
-      "type": "timeseries",
-      "id": 31,
-      "targets": [
-        {
-          "expr": "gpu:power_throttle_fraction:rate5m",
-          "legendFormat": "{{Hostname}}/{{gpu}}",
-          "refId": "A"
-        }
-      ],
-      "title": "Power throttle fraction per GPU",
-      "description": "Fraction of time the GPU was power-throttled. 1.0 = always throttled. Cluster-wide — rule aggregates by (Hostname, gpu, UUID) and drops namespace label.",
-      "transparent": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 25
-      },
-      "repeatDirection": "h",
-      "options": {
-        "legend": {
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": false,
-          "calcs": [
-            "mean",
-            "max"
-          ]
-        },
-        "tooltip": {
-          "mode": "multi",
-          "sort": ""
-        }
-      },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "percentunit",
-          "min": 0,
-          "max": 1,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "value": null,
-                "color": "green"
-              },
-              {
-                "value": 0.05,
-                "color": "yellow"
-              },
-              {
-                "value": 0.2,
-                "color": "red"
-              }
-            ]
-          },
-          "custom": {
-            "drawStyle": "line",
-            "lineInterpolation": "linear",
-            "fillOpacity": 25,
-            "showPoints": "never"
-          }
-        },
-        "overrides": []
-      }
-    },
-    {
-      "type": "timeseries",
-      "id": 32,
-      "targets": [
-        {
-          "expr": "gpu:thermal_throttle_fraction:rate5m",
-          "legendFormat": "{{Hostname}}/{{gpu}}",
-          "refId": "A"
-        }
-      ],
-      "title": "Thermal throttle fraction per GPU",
-      "description": "Fraction of time the GPU was thermal-throttled. Cluster-wide — rule aggregates by (Hostname, gpu, UUID) and drops namespace label.",
-      "transparent": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 25
-      },
-      "repeatDirection": "h",
-      "options": {
-        "legend": {
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": false,
-          "calcs": [
-            "mean",
-            "max"
-          ]
-        },
-        "tooltip": {
-          "mode": "multi",
-          "sort": ""
-        }
-      },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "percentunit",
-          "min": 0,
-          "max": 1,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "value": null,
-                "color": "green"
-              },
-              {
-                "value": 0.05,
-                "color": "yellow"
-              },
-              {
-                "value": 0.2,
-                "color": "red"
-              }
-            ]
-          },
-          "custom": {
-            "drawStyle": "line",
-            "lineInterpolation": "linear",
-            "fillOpacity": 25,
-            "showPoints": "never"
-          }
-        },
-        "overrides": []
-      }
-    }
-  ],
-  "templating": {
-    "list": [
-      {
-        "type": "datasource",
-        "name": "ds_prometheus",
-        "label": "Prometheus",
-        "skipUrlSync": false,
-        "query": "prometheus",
-        "current": {
-          "selected": false,
-          "text": "default",
-          "value": "default"
-        },
-        "multi": false,
-        "allowCustomValue": true,
-        "includeAll": false,
-        "regex": "",
-        "auto": false,
-        "auto_min": "10s",
-        "auto_count": 30
-      }
-    ]
-  },
-  "annotations": {}
-}

dashboards/gpu/gpu-performance.json

@@ -1,957 +0,0 @@
-{
-  "uid": "gpu-performance",
-  "title": "GPU Performance",
-  "tags": [
-    "gpu",
-    "dcgm"
-  ],
-  "timezone": "browser",
-  "editable": true,
-  "graphTooltip": 1,
-  "time": {
-    "from": "now-1h",
-    "to": "now"
-  },
-  "fiscalYearStartMonth": 0,
-  "schemaVersion": 42,
-  "panels": [
-    {
-      "type": "row",
-      "collapsed": false,
-      "title": "Overview",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 0
-      },
-      "id": 1,
-      "panels": []
-    },
-    {
-      "type": "stat",
-      "id": 2,
-      "targets": [
-        {
-          "expr": "cluster:gpu_count:total",
-          "refId": "A"
-        }
-      ],
-      "title": "Total GPUs",
-      "transparent": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 0,
-        "y": 1
-      },
-      "repeatDirection": "h",
-      "options": {
-        "graphMode": "none",
-        "colorMode": "value",
-        "justifyMode": "auto",
-        "textMode": "value",
-        "wideLayout": true,
-        "showPercentChange": false,
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ]
-        },
-        "percentChangeColorMode": "standard",
-        "orientation": ""
-      },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "short",
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "value": null,
-                "color": "blue"
-              }
-            ]
-          }
-        },
-        "overrides": []
-      }
-    },
-    {
-      "type": "stat",
-      "id": 3,
-      "targets": [
-        {
-          "expr": "cluster:gpu_count:allocated",
-          "refId": "A"
-        }
-      ],
-      "title": "Allocated",
-      "transparent": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 6,
-        "y": 1
-      },
-      "repeatDirection": "h",
-      "options": {
-        "graphMode": "none",
-        "colorMode": "value",
-        "justifyMode": "auto",
-        "textMode": "value",
-        "wideLayout": true,
-        "showPercentChange": false,
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ]
-        },
-        "percentChangeColorMode": "standard",
-        "orientation": ""
-      },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "short",
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "value": null,
-                "color": "green"
-              },
-              {
-                "value": 1,
-                "color": "yellow"
-              }
-            ]
-          }
-        },
-        "overrides": []
-      }
-    },
-    {
-      "type": "stat",
-      "id": 4,
-      "targets": [
-        {
-          "expr": "cluster:gpu_util:avg",
-          "refId": "A"
-        }
-      ],
-      "title": "Average utilization",
-      "transparent": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 12,
-        "y": 1
-      },
-      "repeatDirection": "h",
-      "options": {
-        "graphMode": "area",
-        "colorMode": "value",
-        "justifyMode": "auto",
-        "textMode": "value",
-        "wideLayout": true,
-        "showPercentChange": false,
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ]
-        },
-        "percentChangeColorMode": "standard",
-        "orientation": ""
-      },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "percent",
-          "min": 0,
-          "max": 100,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "value": null,
-                "color": "blue"
-              },
-              {
-                "value": 30,
-                "color": "green"
-              },
-              {
-                "value": 80,
-                "color": "orange"
-              }
-            ]
-          }
-        },
-        "overrides": []
-      }
-    },
-    {
-      "type": "stat",
-      "id": 5,
-      "targets": [
-        {
-          "expr": "cluster:gpu_power_watts:sum",
-          "refId": "A"
-        }
-      ],
-      "title": "Power draw",
-      "transparent": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 18,
-        "y": 1
-      },
-      "repeatDirection": "h",
-      "options": {
-        "graphMode": "area",
-        "colorMode": "value",
-        "justifyMode": "auto",
-        "textMode": "value",
-        "wideLayout": true,
-        "showPercentChange": false,
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ]
-        },
-        "percentChangeColorMode": "standard",
-        "orientation": ""
-      },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "watt",
-          "decimals": 0,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "value": null,
-                "color": "green"
-              }
-            ]
-          }
-        },
-        "overrides": []
-      }
-    },
-    {
-      "type": "row",
-      "collapsed": false,
-      "title": "Utilization",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 5
-      },
-      "id": 10,
-      "panels": []
-    },
-    {
-      "type": "timeseries",
-      "id": 11,
-      "targets": [
-        {
-          "expr": "DCGM_FI_DEV_GPU_UTIL{Hostname=~\"$Hostname\"}",
-          "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}",
-          "refId": "A"
-        }
-      ],
-      "title": "GPU Utilization (NVML)",
-      "transparent": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 6
-      },
-      "repeatDirection": "h",
-      "options": {
-        "legend": {
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": false,
-          "calcs": [
-            "mean",
-            "max"
-          ]
-        },
-        "tooltip": {
-          "mode": "multi",
-          "sort": ""
-        }
-      },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "percent",
-          "min": 0,
-          "max": 100,
-          "custom": {
-            "drawStyle": "line",
-            "lineInterpolation": "linear",
-            "fillOpacity": 10,
-            "showPoints": "never",
-            "spanNulls": false
-          }
-        },
-        "overrides": []
-      }
-    },
-    {
-      "type": "timeseries",
-      "id": 12,
-      "targets": [
-        {
-          "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{Hostname=~\"$Hostname\"} * 100",
-          "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}",
-          "refId": "A"
-        }
-      ],
-      "title": "Tensor Pipe Active (realistic load for LLM/AI)",
-      "transparent": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 6
-      },
-      "repeatDirection": "h",
-      "options": {
-        "legend": {
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": false,
-          "calcs": [
-            "mean",
-            "max"
-          ]
-        },
-        "tooltip": {
-          "mode": "multi",
-          "sort": ""
-        }
-      },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "percent",
-          "min": 0,
-          "max": 100,
-          "custom": {
-            "drawStyle": "line",
-            "lineInterpolation": "linear",
-            "fillOpacity": 10,
-            "showPoints": "never",
-            "spanNulls": false
-          }
-        },
-        "overrides": []
-      }
-    },
-    {
-      "type": "timeseries",
-      "id": 13,
-      "targets": [
-        {
-          "expr": "DCGM_FI_PROF_GR_ENGINE_ACTIVE{Hostname=~\"$Hostname\"} * 100",
-          "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}",
-          "refId": "A"
-        }
-      ],
-      "title": "Graphics Engine Active",
-      "transparent": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 14
-      },
-      "repeatDirection": "h",
-      "options": {
-        "legend": {
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": false,
-          "calcs": [
-            "mean",
-            "max"
-          ]
-        },
-        "tooltip": {
-          "mode": "multi",
-          "sort": ""
-        }
-      },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "percent",
-          "min": 0,
-          "max": 100,
-          "custom": {
-            "drawStyle": "line",
-            "lineInterpolation": "linear",
-            "fillOpacity": 10,
-            "showPoints": "never",
-            "spanNulls": false
-          }
-        },
-        "overrides": []
-      }
-    },
-    {
-      "type": "timeseries",
-      "id": 14,
-      "targets": [
-        {
-          "expr": "DCGM_FI_DEV_MEM_COPY_UTIL{Hostname=~\"$Hostname\"}",
-          "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}",
-          "refId": "A"
-        }
-      ],
-      "title": "Memory Copy Utilization",
-      "transparent": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 14
-      },
-      "repeatDirection": "h",
-      "options": {
-        "legend": {
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": false,
-          "calcs": [
-            "mean",
-            "max"
-          ]
-        },
-        "tooltip": {
-          "mode": "multi",
-          "sort": ""
-        }
-      },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "percent",
-          "min": 0,
-          "max": 100,
-          "custom": {
-            "drawStyle": "line",
-            "lineInterpolation": "linear",
-            "fillOpacity": 10,
-            "showPoints": "never",
-            "spanNulls": false
-          }
-        },
-        "overrides": []
-      }
-    },
-    {
-      "type": "row",
-      "collapsed": false,
-      "title": "Memory",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 22
-      },
-      "id": 20,
-      "panels": []
-    },
-    {
-      "type": "timeseries",
-      "id": 21,
-      "targets": [
-        {
-          "expr": "DCGM_FI_DEV_FB_USED{Hostname=~\"$Hostname\"} * 1048576",
-          "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}",
-          "refId": "A"
-        }
-      ],
-      "title": "VRAM Used",
-      "transparent": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 23
-      },
-      "repeatDirection": "h",
-      "options": {
-        "legend": {
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": false,
-          "calcs": [
-            "mean",
-            "max"
-          ]
-        },
-        "tooltip": {
-          "mode": "multi",
-          "sort": ""
-        }
-      },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "bytes",
-          "custom": {
-            "drawStyle": "line",
-            "lineInterpolation": "linear",
-            "fillOpacity": 25,
-            "showPoints": "never"
-          }
-        },
-        "overrides": []
-      }
-    },
-    {
-      "type": "timeseries",
-      "id": 22,
-      "targets": [
-        {
-          "expr": "DCGM_FI_DEV_FB_FREE{Hostname=~\"$Hostname\"} * 1048576",
-          "legendFormat": "{{Hostname}}/{{gpu}}",
-          "refId": "A"
-        }
-      ],
-      "title": "VRAM Free",
-      "transparent": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 23
-      },
-      "repeatDirection": "h",
-      "options": {
-        "legend": {
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": false,
-          "calcs": [
-            "mean",
-            "min"
-          ]
-        },
-        "tooltip": {
-          "mode": "multi",
-          "sort": ""
-        }
-      },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "bytes",
-          "custom": {
-            "drawStyle": "line",
-            "lineInterpolation": "linear",
-            "fillOpacity": 10,
-            "showPoints": "never"
-          }
-        },
-        "overrides": []
-      }
-    },
-    {
-      "type": "row",
-      "collapsed": false,
-      "title": "Power \u0026 Temperature",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 31
-      },
-      "id": 30,
-      "panels": []
-    },
-    {
-      "type": "timeseries",
-      "id": 31,
-      "targets": [
-        {
-          "expr": "DCGM_FI_DEV_POWER_USAGE{Hostname=~\"$Hostname\"}",
-          "legendFormat": "{{Hostname}}/{{gpu}}",
-          "refId": "A"
-        }
-      ],
-      "title": "Power Usage per GPU",
-      "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.",
-      "transparent": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 32
-      },
-      "repeatDirection": "h",
-      "options": {
-        "legend": {
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": false,
-          "calcs": [
-            "mean",
-            "max"
-          ]
-        },
-        "tooltip": {
-          "mode": "multi",
-          "sort": ""
-        }
-      },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "watt",
-          "custom": {
-            "drawStyle": "line",
-            "lineInterpolation": "linear",
-            "fillOpacity": 10,
-            "showPoints": "never"
-          }
-        },
-        "overrides": []
-      }
-    },
-    {
-      "type": "timeseries",
-      "id": 32,
-      "targets": [
-        {
-          "expr": "DCGM_FI_DEV_GPU_TEMP{Hostname=~\"$Hostname\"}",
-          "legendFormat": "{{Hostname}}/{{gpu}}",
-          "refId": "A"
-        }
-      ],
-      "title": "GPU Temperature",
-      "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.",
-      "transparent": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 32
-      },
-      "repeatDirection": "h",
-      "options": {
-        "legend": {
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": false,
-          "calcs": [
-            "mean",
-            "max"
-          ]
-        },
-        "tooltip": {
-          "mode": "multi",
-          "sort": ""
-        }
-      },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "celsius",
-          "min": 20,
-          "max": 100,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "value": null,
-                "color": "green"
-              },
-              {
-                "value": 75,
-                "color": "yellow"
-              },
-              {
-                "value": 85,
-                "color": "red"
-              }
-            ]
-          },
-          "custom": {
-            "drawStyle": "line",
-            "lineInterpolation": "linear",
-            "fillOpacity": 10,
-            "showPoints": "never"
-          }
-        },
-        "overrides": []
-      }
-    },
-    {
-      "type": "row",
-      "collapsed": false,
-      "title": "Health",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 40
-      },
-      "id": 40,
-      "panels": []
-    },
-    {
-      "type": "stat",
-      "id": 41,
-      "targets": [
-        {
-          "expr": "max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS{Hostname=~\"$Hostname\"})",
-          "legendFormat": "{{Hostname}}/{{gpu}}",
-          "refId": "A"
-        }
-      ],
-      "title": "XID errors (latest)",
-      "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.",
-      "transparent": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "gridPos": {
-        "h": 5,
-        "w": 8,
-        "x": 0,
-        "y": 41
-      },
-      "repeatDirection": "h",
-      "options": {
-        "graphMode": "none",
-        "colorMode": "background",
-        "justifyMode": "auto",
-        "textMode": "value_and_name",
-        "wideLayout": true,
-        "showPercentChange": false,
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ]
-        },
-        "percentChangeColorMode": "standard",
-        "orientation": ""
-      },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "short",
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "value": null,
-                "color": "green"
-              },
-              {
-                "value": 1,
-                "color": "red"
-              }
-            ]
-          }
-        },
-        "overrides": []
-      }
-    },
-    {
-      "type": "timeseries",
-      "id": 42,
-      "targets": [
-        {
-          "expr": "rate(DCGM_FI_DEV_POWER_VIOLATION{Hostname=~\"$Hostname\"}[5m])",
-          "legendFormat": "{{Hostname}}/{{gpu}}",
-          "refId": "A"
-        }
-      ],
-      "title": "Power Violation (µs/s throttled due to power)",
-      "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.",
-      "transparent": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "gridPos": {
-        "h": 5,
-        "w": 8,
-        "x": 8,
-        "y": 41
-      },
-      "repeatDirection": "h",
-      "options": {
-        "legend": {
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": false,
-          "calcs": []
-        },
-        "tooltip": {
-          "mode": "multi",
-          "sort": ""
-        }
-      },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "µs",
-          "custom": {
-            "drawStyle": "line",
-            "lineInterpolation": "linear",
-            "fillOpacity": 10,
-            "showPoints": "never"
-          }
-        },
-        "overrides": []
-      }
-    },
-    {
-      "type": "timeseries",
-      "id": 43,
-      "targets": [
-        {
-          "expr": "rate(DCGM_FI_DEV_THERMAL_VIOLATION{Hostname=~\"$Hostname\"}[5m])",
-          "legendFormat": "{{Hostname}}/{{gpu}}",
-          "refId": "A"
-        }
-      ],
-      "title": "Thermal Violation (µs/s throttled due to thermals)",
-      "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.",
-      "transparent": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "gridPos": {
-        "h": 5,
-        "w": 8,
-        "x": 16,
-        "y": 41
-      },
-      "repeatDirection": "h",
-      "options": {
-        "legend": {
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": false,
-          "calcs": []
-        },
-        "tooltip": {
-          "mode": "multi",
-          "sort": ""
-        }
-      },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "µs",
-          "custom": {
-            "drawStyle": "line",
-            "lineInterpolation": "linear",
-            "fillOpacity": 10,
-            "showPoints": "never"
-          }
-        },
-        "overrides": []
-      }
-    }
-  ],
-  "templating": {
-    "list": [
-      {
-        "type": "datasource",
-        "name": "ds_prometheus",
-        "label": "Prometheus",
-        "skipUrlSync": false,
-        "query": "prometheus",
-        "current": {
-          "selected": false,
-          "text": "default",
-          "value": "default"
-        },
-        "multi": false,
-        "allowCustomValue": true,
-        "includeAll": false,
-        "regex": "",
-        "auto": false,
-        "auto_min": "10s",
-        "auto_count": 30
-      },
-      {
-        "type": "query",
-        "name": "Hostname",
-        "label": "Host",
-        "skipUrlSync": false,
-        "query": "label_values(DCGM_FI_DEV_GPU_UTIL, Hostname)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "$ds_prometheus"
-        },
-        "current": {
-          "selected": true,
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "allowCustomValue": true,
-        "refresh": 2,
-        "sort": 1,
-        "includeAll": true,
-        "auto": false,
-        "auto_min": "10s",
-        "auto_count": 30
-      }
-    ]
-  },
-  "annotations": {}
-}

dashboards/gpu/gpu-quotas.json

@@ -1,637 +0,0 @@
-{
-  "uid": "gpu-quotas",
-  "title": "GPU Quotas \u0026 Allocation",
-  "tags": [
-    "gpu",
-    "quotas"
-  ],
-  "timezone": "browser",
-  "editable": true,
-  "graphTooltip": 1,
-  "time": {
-    "from": "now-6h",
-    "to": "now"
-  },
-  "fiscalYearStartMonth": 0,
-  "schemaVersion": 42,
-  "panels": [
-    {
-      "type": "row",
-      "collapsed": false,
-      "title": "Allocation overview",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 0
-      },
-      "id": 1,
-      "panels": []
-    },
-    {
-      "type": "stat",
-      "id": 2,
-      "targets": [
-        {
-          "expr": "sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"})",
-          "refId": "A"
-        }
-      ],
-      "title": "GPU allocatable",
-      "description": "Total GPU capacity the cluster can schedule to pods.",
-      "transparent": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 0,
-        "y": 1
-      },
-      "repeatDirection": "h",
-      "options": {
-        "graphMode": "none",
-        "colorMode": "value",
-        "justifyMode": "auto",
-        "textMode": "value",
-        "wideLayout": true,
-        "showPercentChange": false,
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ]
-        },
-        "percentChangeColorMode": "standard",
-        "orientation": ""
-      },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "short",
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "value": null,
-                "color": "blue"
-              }
-            ]
-          }
-        },
-        "overrides": []
-      }
-    },
-    {
-      "type": "stat",
-      "id": 3,
-      "targets": [
-        {
-          "expr": "cluster:gpu_count:allocated",
-          "refId": "A"
-        }
-      ],
-      "title": "GPU requested",
-      "description": "Sum of GPU requests across all pods cluster-wide, including system namespaces.",
-      "transparent": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 6,
-        "y": 1
-      },
-      "repeatDirection": "h",
-      "options": {
-        "graphMode": "none",
-        "colorMode": "value",
-        "justifyMode": "auto",
-        "textMode": "value",
-        "wideLayout": true,
-        "showPercentChange": false,
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ]
-        },
-        "percentChangeColorMode": "standard",
-        "orientation": ""
-      },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "short",
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "value": null,
-                "color": "green"
-              },
-              {
-                "value": 1,
-                "color": "yellow"
-              }
-            ]
-          }
-        },
-        "overrides": []
-      }
-    },
-    {
-      "type": "gauge",
-      "id": 4,
-      "targets": [
-        {
-          "expr": "cluster:gpu_count:allocated / sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"}) * 100",
-          "refId": "A"
-        }
-      ],
-      "title": "Allocation ratio",
-      "description": "Percentage of allocatable GPUs currently requested by pods.",
-      "transparent": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 12,
-        "y": 1
-      },
-      "repeatDirection": "h",
-      "options": {
-        "showThresholdLabels": false,
-        "showThresholdMarkers": true,
-        "sizing": "auto",
-        "minVizWidth": 75,
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ]
-        },
-        "minVizHeight": 75,
-        "orientation": ""
-      },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "percent",
-          "min": 0,
-          "max": 100,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "value": null,
-                "color": "blue"
-              },
-              {
-                "value": 40,
-                "color": "green"
-              },
-              {
-                "value": 80,
-                "color": "yellow"
-              },
-              {
-                "value": 95,
-                "color": "red"
-              }
-            ]
-          }
-        },
-        "overrides": []
-      }
-    },
-    {
-      "type": "stat",
-      "id": 5,
-      "targets": [
-        {
-          "expr": "count((sum by (namespace, pod) (kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}) \u003e 0) * on(namespace, pod) group_left() (kube_pod_status_phase{phase=\"Pending\"} == 1)) or vector(0)",
-          "refId": "A"
-        }
-      ],
-      "title": "Pending pods (GPU)",
-      "description": "Pods requesting GPUs that are stuck in Pending state — indicates capacity shortage.",
-      "transparent": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 18,
-        "y": 1
-      },
-      "repeatDirection": "h",
-      "options": {
-        "graphMode": "none",
-        "colorMode": "background",
-        "justifyMode": "auto",
-        "textMode": "value",
-        "wideLayout": true,
-        "showPercentChange": false,
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ]
-        },
-        "percentChangeColorMode": "standard",
-        "orientation": ""
-      },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "short",
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "value": null,
-                "color": "green"
-              },
-              {
-                "value": 1,
-                "color": "red"
-              }
-            ]
-          }
-        },
-        "overrides": []
-      }
-    },
-    {
-      "type": "row",
-      "collapsed": false,
-      "title": "Per namespace",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 5
-      },
-      "id": 10,
-      "panels": []
-    },
-    {
-      "type": "bargauge",
-      "id": 11,
-      "targets": [
-        {
-          "expr": "sum by (namespace) (namespace:gpu_count:allocated{namespace=~\"$namespace\"})",
-          "legendFormat": "{{namespace}}",
-          "refId": "A"
-        }
-      ],
-      "title": "GPU requested per namespace",
-      "description": "GPU allocation breakdown by namespace — spot top consumers at a glance.",
-      "transparent": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 6
-      },
-      "repeatDirection": "h",
-      "options": {
-        "displayMode": "gradient",
-        "valueMode": "color",
-        "namePlacement": "auto",
-        "showUnfilled": true,
-        "sizing": "auto",
-        "minVizWidth": 8,
-        "minVizHeight": 16,
-        "legend": {
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": false,
-          "calcs": []
-        },
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ]
-        },
-        "maxVizHeight": 300,
-        "orientation": "horizontal"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "short",
-          "min": 0,
-          "max": 8,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "value": null,
-                "color": "green"
-              }
-            ]
-          }
-        },
-        "overrides": []
-      }
-    },
-    {
-      "type": "timeseries",
-      "id": 12,
-      "targets": [
-        {
-          "expr": "sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"})",
-          "legendFormat": "Allocatable (total)",
-          "refId": "A"
-        },
-        {
-          "expr": "sum(namespace:gpu_count:allocated{namespace=~\"$namespace\"})",
-          "legendFormat": "Requested",
-          "refId": "B"
-        }
-      ],
-      "title": "GPU allocated over time",
-      "description": "Requested vs allocatable GPUs over time — shows allocation pressure trends.",
-      "transparent": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 6
-      },
-      "repeatDirection": "h",
-      "options": {
-        "legend": {
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": false,
-          "calcs": []
-        },
-        "tooltip": {
-          "mode": "multi",
-          "sort": ""
-        }
-      },
-      "fieldConfig": {
-        "defaults": {
-          "unit": "short",
-          "custom": {
-            "drawStyle": "line",
-            "lineInterpolation": "stepAfter",
-            "fillOpacity": 10,
-            "showPoints": "never"
-          }
-        },
-        "overrides": [
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Allocatable (total)"
-            },
-            "properties": [
-              {
-                "id": "color",
-                "value": {
-                  "fixedColor": "blue",
-                  "mode": "fixed"
-                }
-              }
-            ]
-          }
-        ]
-      }
-    },
-    {
-      "type": "row",
-      "collapsed": false,
-      "title": "Pods with GPU",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 14
-      },
-      "id": 20,
-      "panels": []
-    },
-    {
-      "type": "table",
-      "id": 21,
-      "targets": [
-        {
-          "expr": "kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left(phase) (kube_pod_status_phase{phase=~\"Running|Pending\"} == 1)",
-          "instant": true,
-          "range": false,
-          "format": "table",
-          "refId": "requested"
-        },
-        {
-          "expr": "kube_pod_container_resource_limits{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left() (kube_pod_status_phase{phase=~\"Running|Pending\"} == 1)",
-          "instant": true,
-          "range": false,
-          "format": "table",
-          "refId": "limits"
-        }
-      ],
-      "title": "Pods requesting GPU",
-      "description": "Per-pod GPU requests and limits with scheduling status — Running or Pending.",
-      "transparent": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds_prometheus"
-      },
-      "gridPos": {
-        "h": 10,
-        "w": 24,
-        "x": 0,
-        "y": 15
-      },
-      "repeatDirection": "h",
-      "transformations": [
-        {
-          "id": "joinByField",
-          "options": {
-            "byField": "pod",
-            "mode": "outer"
-          }
-        },
-        {
-          "id": "organize",
-          "options": {
-            "excludeByName": {
-              "Time": true,
-              "Time 1": true,
-              "Time 2": true,
-              "__name__": true,
-              "__name__ 1": true,
-              "__name__ 2": true,
-              "cluster": true,
-              "cluster 1": true,
-              "cluster 2": true,
-              "container 2": true,
-              "endpoint": true,
-              "endpoint 1": true,
-              "endpoint 2": true,
-              "instance": true,
-              "instance 1": true,
-              "instance 2": true,
-              "job": true,
-              "job 1": true,
-              "job 2": true,
-              "namespace 2": true,
-              "node 2": true,
-              "prometheus": true,
-              "prometheus 1": true,
-              "prometheus 2": true,
-              "resource": true,
-              "resource 1": true,
-              "resource 2": true,
-              "service": true,
-              "service 1": true,
-              "service 2": true,
-              "tenant": true,
-              "tenant 1": true,
-              "tenant 2": true,
-              "tier": true,
-              "tier 1": true,
-              "tier 2": true,
-              "uid": true,
-              "uid 1": true,
-              "uid 2": true,
-              "unit": true,
-              "unit 1": true,
-              "unit 2": true
-            },
-            "indexByName": {
-              "Value #limits": 5,
-              "Value #requested": 4,
-              "container 1": 2,
-              "namespace 1": 0,
-              "node 1": 3,
-              "phase": 6,
-              "pod": 1
-            },
-            "renameByName": {
-              "Value #limits": "Limit",
-              "Value #requested": "Req",
-              "container 1": "Container",
-              "namespace 1": "Namespace",
-              "node 1": "Node",
-              "phase": "Status"
-            }
-          }
-        }
-      ],
-      "options": {
-        "frameIndex": 0,
-        "showHeader": true,
-        "showTypeIcons": false,
-        "footer": {
-          "show": false,
-          "reducer": []
-        },
-        "cellHeight": "sm"
-      },
-      "fieldConfig": {
-        "defaults": {},
-        "overrides": [
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Status"
-            },
-            "properties": [
-              {
-                "id": "custom.cellOptions",
-                "value": {
-                  "mode": "basic",
-                  "type": "color-background"
-                }
-              },
-              {
-                "id": "mappings",
-                "value": [
-                  {
-                    "options": {
-                      "Failed": {
-                        "color": "red",
-                        "index": 2
-                      },
-                      "Pending": {
-                        "color": "orange",
-                        "index": 1
-                      },
-                      "Running": {
-                        "color": "green",
-                        "index": 0
-                      }
-                    },
-                    "type": "value"
-                  }
-                ]
-              }
-            ]
-          }
-        ]
-      }
-    }
-  ],
-  "templating": {
-    "list": [
-      {
-        "type": "datasource",
-        "name": "ds_prometheus",
-        "label": "Prometheus",
-        "skipUrlSync": false,
-        "query": "prometheus",
-        "current": {
-          "selected": false,
-          "text": "default",
-          "value": "default"
-        },
-        "multi": false,
-        "allowCustomValue": true,
-        "includeAll": false,
-        "regex": "",
-        "auto": false,
-        "auto_min": "10s",
-        "auto_count": 30
-      },
-      {
-        "type": "query",
-        "name": "namespace",
-        "label": "Namespace",
-        "skipUrlSync": false,
-        "query": "label_values(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}, namespace)",
-        "datasource": {
-          "type": "prometheus",
-          "uid": "$ds_prometheus"
-        },
-        "current": {
-          "selected": true,
-          "text": "All",
-          "value": "$__all"
-        },
-        "multi": true,
-        "allowCustomValue": true,
-        "refresh": 2,
-        "sort": 1,
-        "includeAll": true,
-        "auto": false,
-        "auto_min": "10s",
-        "auto_count": 30
-      }
-    ]
-  },
-  "annotations": {}
-}

docs/changelogs/v1.3.1.md

@@ -1,37 +0,0 @@
-<!--
-https://github.com/cozystack/cozystack/releases/tag/v1.3.1
--->
-
-# v1.3.1 (2026-04-28)
-
-Patch release covering a TenantNamespace IDOR fix in the API, a destructive `post-upgrade` hook removed from the etcd chart, kamaji controller stability, a `linstor-csi` bump that fixes live migration on Protocol-A/B DRBD resources, the missing `linstor-gui` build wiring, and a velero RBAC fix that unblocked installs on bundles without Velero.
-
-## Security
-
-* **fix(api): prevent IDOR in TenantNamespace Get and Watch handlers**: Two IDOR (Insecure Direct Object Reference) vulnerabilities allowed authenticated users to read TenantNamespace metadata they had no RoleBinding for. The `Get` and `Watch` handlers now go through a new `hasAccessToNamespace()` helper that lists RoleBindings scoped only to the target namespace (orders of magnitude faster than the previous all-cluster scan), returns `NotFound` instead of leaking existence on unauthorized access, and applies the same check on the `Watch` filter path. Includes regression tests for the unauthorized paths. ([**@IvanHunters**](https://github.com/IvanHunters) in #2471, backport #2524)
-
-## Features
-
-* **feat(linstor): bump linstor-csi to v1.10.6 with Protocol-C dual-attach fix**: Live migration of KubeVirt VMs on Protocol-A/B (async) DRBD volumes no longer fails with `Protocol C required`. `linstor-csi` v1.10.6 now installs a `Protocol=C` override on the resource-definition during dual-attach and reverts it on detach, so `replicated-async` StorageClasses and other Protocol-A/B resource groups support live migration without manual `drbdadm adjust` intervention. ([**@kvaps**](https://github.com/kvaps) in #2496, backport #2505)
-
-## Fixes
-
-* **fix(backups): move velero-configmap Role to velero chart**: The `backupstrategy-controller` (a default package) declared a Role/RoleBinding scoped to the `cozy-velero` namespace for managing `ResourceModifier` ConfigMaps. On bundles where Velero was not enabled, that namespace did not exist and the HelmRelease failed with `namespaces "cozy-velero" not found`, blocking installation. The Role/RoleBinding now lives in the velero chart, so it is created only when velero is actually deployed. ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2459, backport #2467)
-
-* **fix(etcd): remove destructive post-upgrade cert-regeneration hook**: The etcd chart ran a `post-upgrade` Helm hook on every upgrade that deleted etcd TLS Secrets (`etcd-ca-tls`, `etcd-peer-ca-tls`, `etcd-client-tls`, `etcd-peer-tls`, `etcd-server-tls`) and then deleted etcd pods, forcing cert-manager to re-issue the entire etcd CA chain. On clusters with Kamaji-managed tenant control planes this put every tenant `kube-apiserver` into CrashLoopBackOff until each DataStore was manually re-reconciled. The hook was a one-shot `2.6.0 → 2.6.1` migration that became a permanent footgun once chart versioning moved to `0.0.0+<git-hash>` (always `< 2.6.1` per semver) and after the underlying `rotationPolicy: Always` issue was fixed in `47d81f70`. The hook is now removed entirely. ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2462, backport #2511)
-
-* **fix(kamaji): increase memory limits and add startup probe**: The kamaji controller frequently entered CrashLoopBackOff due to OOMKills (exit 137) within ~20–25 seconds of startup, with the readiness probe failing while the controller was still finishing initialization. Memory limit raised from 500Mi to 512Mi, request from 100Mi to 256Mi, and a 60-second startup probe (12 attempts × 5s periods) is added so the controller has room to boot before liveness/readiness probes engage. ([**@IvanHunters**](https://github.com/IvanHunters) in #2421, backport #2491)
-
-## Build
-
-* **build(linstor): include linstor-gui in root image build target**: The `linstor-gui` package (added in #2382) was never wired into the root `Makefile`'s `build:` target, so CI never built or published the image. `ghcr.io/cozystack/cozystack/linstor-gui` returned `NAME_UNKNOWN` and `values.yaml` stayed pinned to `tag: 2.3.0` without a digest. The missing build line is added so the next CI run publishes the image and the per-package `Makefile` digest-pins `values.yaml` automatically. ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2498, backport #2518)
-
-## Contributors
-
-Thanks to everyone who contributed to this patch release:
-
-* [**@IvanHunters**](https://github.com/IvanHunters)
-* [**@kvaps**](https://github.com/kvaps)
-* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil)
-
-**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.3.0...v1.3.1

hack/check-gpu-recording-rules.bats

@@ -1,206 +0,0 @@
-#!/usr/bin/env bats
-# -----------------------------------------------------------------------------
-# Cross-validation between GPU recording rules, the dashboards that consume
-# them, and the DCGM Exporter metric set the cluster actually scrapes. Catches:
-#
-#   1. dangling references — a dashboard query mentions a recording rule name
-#      that doesn't exist in gpu-recording.rules.yaml. This is the bug the
-#      pre-merge review caught: gpu-efficiency.json shipped panels keyed on
-#      pod:tensor_saturation:avg5m without the rule being defined, so the
-#      panel showed "No data" everywhere.
-#
-#   2. typos in rule names — same bug class, manifested as a single-character
-#      difference between rule and reference.
-#
-#   3. undeclared DCGM metrics — a dashboard query or recording rule mentions
-#      a DCGM_FI_* metric that is neither in the upstream default CSV nor in
-#      the project's custom CSV (dcgm-custom-metrics.yaml), meaning DCGM
-#      Exporter would never emit it and the panel silently shows "No data".
-#      Example regression: gpu-fleet.json shipped a TDP panel referencing
-#      DCGM_FI_DEV_POWER_MGMT_LIMIT before the custom CSV declared it.
-#
-# Scope: only dashboards listed in packages/system/monitoring/dashboards-infra.list
-# under the "gpu/" prefix (i.e. shipped to production), not every JSON file in
-# dashboards/gpu/. Untracked drafts stay out of scope on purpose — adding one
-# to dashboards-infra.list is what brings it under the test.
-#
-# Reverse direction (rule defined but never consumed) is intentionally NOT
-# enforced: some rules exist for ad-hoc PromQL or upcoming dashboards. Treat
-# unused rules as an editorial concern, not a regression.
-#
-# Title syntax constraints from cozytest.sh's awk parser:
-#   - Titles delimited by ASCII double quotes; embedded quotes truncate.
-#   - Only [A-Za-z0-9] from the title survives into the function name; titles
-#     differing only in punctuation collapse to the same function.
-#
-# Run with: hack/cozytest.sh hack/check-gpu-recording-rules.bats
-# -----------------------------------------------------------------------------
-
-REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)"
-RULES_FILE="$REPO_ROOT/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml"
-DASHBOARDS_LIST="$REPO_ROOT/packages/system/monitoring/dashboards-infra.list"
-DASHBOARDS_DIR="$REPO_ROOT/dashboards"
-DCGM_DEFAULT_CSV="$REPO_ROOT/hack/dcgm-default-counters.csv"
-DCGM_CUSTOM_CSV="$REPO_ROOT/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml"
-
-# Extract the set of "- record: NAME" entries from the rules YAML.
-# Outputs one rule name per line, sorted and deduplicated.
-extract_rules() {
-  awk '/^[[:space:]]*-[[:space:]]*record:[[:space:]]/ {
-    sub(/^[[:space:]]*-[[:space:]]*record:[[:space:]]*/, "")
-    sub(/[[:space:]]*$/, "")
-    print
-  }' "$RULES_FILE" | sort -u
-}
-
-# Extract the set of recording-rule references from a dashboard JSON.
-# A recording-rule reference is matched by the pattern
-#   <segment>:<segment>(:<segment>)+
-# where each <segment> is [a-z0-9_]. Raw DCGM metrics (DCGM_FI_*),
-# kube-state-metrics (kube_*) and similar uppercase / single-word metric
-# names do not match because the leading segment must be lowercase and the
-# whole expression must contain at least two ':' characters.
-extract_refs() {
-  json_file=$1
-  # Prometheus convention allows 2-segment rule names (level:metric); this
-  # regex is tuned to the 3+ segment convention used in this repo
-  # (level:metric:op — e.g. cluster:gpu_count:total). Update if future
-  # rules use 2 segments, otherwise they will be silently skipped.
-  grep -hoE '[a-z][a-z0-9_]*:[a-z0-9_]+:[a-z0-9_]+' "$json_file" | sort -u
-}
-
-# Resolve "gpu/foo" -> "$DASHBOARDS_DIR/gpu/foo.json"
-list_tracked_gpu_dashboards() {
-  awk '/^gpu\// { print $0 ".json" }' "$DASHBOARDS_LIST"
-}
-
-# Extract the set of DCGM_FI_* metric names declared in a CSV file. Handles
-# both the upstream-style default CSV (unindented) and the ConfigMap-style
-# custom CSV (YAML-indented). A declaration line starts — after any leading
-# whitespace — with "DCGM_FI_<NAME>," ; comment lines begin with "#" and are
-# skipped. Uses POSIX awk's match()+RSTART/RLENGTH so no GNU extensions
-# are required.
-extract_csv_metrics() {
-  file=$1
-  awk '
-    {
-      line = $0
-      sub(/^[[:space:]]+/, "", line)
-      if (line ~ /^#/) next
-      if (match(line, /^DCGM_FI_[A-Z0-9_]+/)) {
-        print substr(line, RSTART, RLENGTH)
-      }
-    }
-  ' "$file" | sort -u
-}
-
-# Extract the set of DCGM_FI_* metric references from a text file (dashboard
-# JSON or rules YAML). A DCGM metric name has at least two underscore-delimited
-# segments after the "DCGM_FI_" prefix (e.g. DCGM_FI_DEV_GPU_UTIL, DCGM_FI_PROF_
-# PIPE_TENSOR_ACTIVE, DCGM_FI_DRIVER_VERSION). Requiring two segments keeps
-# the matcher from latching onto glob stubs like "DCGM_FI_DEV_*_VIOLATION" that
-# appear in comments.
-extract_dcgm_refs() {
-  file=$1
-  grep -hoE 'DCGM_FI_[A-Z0-9]+(_[A-Z0-9]+)+' "$file" | sort -u
-}
-
-@test "every recording rule reference in tracked GPU dashboards has a matching record" {
-  TMP=$(mktemp -d)
-  trap 'rm -rf "$TMP"' EXIT
-
-  extract_rules > "$TMP/rules.txt"
-  [ -s "$TMP/rules.txt" ] || { echo "no recording rules extracted from $RULES_FILE" >&2; exit 1; }
-
-  list_tracked_gpu_dashboards > "$TMP/dashboards.txt"
-  [ -s "$TMP/dashboards.txt" ] || { echo "no gpu/* dashboards listed in $DASHBOARDS_LIST" >&2; exit 1; }
-
-  failed=0
-  while IFS= read -r dashboard_rel; do
-    dashboard="$DASHBOARDS_DIR/$dashboard_rel"
-    if [ ! -f "$dashboard" ]; then
-      echo "ERROR: dashboard listed but file missing: $dashboard" >&2
-      failed=1
-      continue
-    fi
-
-    extract_refs "$dashboard" > "$TMP/refs.txt"
-    # comm -23: lines unique to refs.txt (referenced but not defined)
-    # Both inputs must be sorted; extract_* helpers already sort.
-    comm -23 "$TMP/refs.txt" "$TMP/rules.txt" > "$TMP/missing.txt"
-    if [ -s "$TMP/missing.txt" ]; then
-      echo "ERROR: $dashboard_rel references undefined recording rules:" >&2
-      sed 's/^/  - /' "$TMP/missing.txt" >&2
-      failed=1
-    fi
-  done < "$TMP/dashboards.txt"
-
-  [ "$failed" -eq 0 ]
-}
-
-@test "every DCGM metric referenced in tracked dashboards and rules is declared" {
-  TMP=$(mktemp -d)
-  trap 'rm -rf "$TMP"' EXIT
-
-  [ -f "$DCGM_DEFAULT_CSV" ] || { echo "missing $DCGM_DEFAULT_CSV" >&2; exit 1; }
-  [ -f "$DCGM_CUSTOM_CSV" ]  || { echo "missing $DCGM_CUSTOM_CSV"  >&2; exit 1; }
-
-  {
-    extract_csv_metrics "$DCGM_DEFAULT_CSV"
-    extract_csv_metrics "$DCGM_CUSTOM_CSV"
-  } | sort -u > "$TMP/declared.txt"
-  [ -s "$TMP/declared.txt" ] || { echo "no DCGM metrics extracted from CSVs" >&2; exit 1; }
-
-  list_tracked_gpu_dashboards > "$TMP/dashboards.txt"
-  [ -s "$TMP/dashboards.txt" ] || { echo "no gpu/* dashboards listed in $DASHBOARDS_LIST" >&2; exit 1; }
-
-  failed=0
-
-  # Dashboard coverage — every dashboard's DCGM references must resolve.
-  while IFS= read -r dashboard_rel; do
-    dashboard="$DASHBOARDS_DIR/$dashboard_rel"
-    [ -f "$dashboard" ] || continue  # handled by the existence test
-    extract_dcgm_refs "$dashboard" > "$TMP/refs.txt"
-    [ -s "$TMP/refs.txt" ] || continue  # dashboard relies entirely on recording rules
-    comm -23 "$TMP/refs.txt" "$TMP/declared.txt" > "$TMP/missing.txt"
-    if [ -s "$TMP/missing.txt" ]; then
-      echo "ERROR: $dashboard_rel references DCGM metrics not declared in any CSV:" >&2
-      sed 's/^/  - /' "$TMP/missing.txt" >&2
-      failed=1
-    fi
-  done < "$TMP/dashboards.txt"
-
-  # Rules coverage — recording rules consume DCGM directly, so their set
-  # must be declared too, otherwise derived series on every dashboard
-  # collapse to empty.
-  extract_dcgm_refs "$RULES_FILE" > "$TMP/rule-refs.txt"
-  if [ -s "$TMP/rule-refs.txt" ]; then
-    comm -23 "$TMP/rule-refs.txt" "$TMP/declared.txt" > "$TMP/rule-missing.txt"
-    if [ -s "$TMP/rule-missing.txt" ]; then
-      echo "ERROR: gpu-recording.rules.yaml references DCGM metrics not declared in any CSV:" >&2
-      sed 's/^/  - /' "$TMP/rule-missing.txt" >&2
-      failed=1
-    fi
-  fi
-
-  [ "$failed" -eq 0 ]
-}
-
-@test "every tracked GPU dashboard listed in dashboards-infra.list exists on disk" {
-  TMP=$(mktemp -d)
-  trap 'rm -rf "$TMP"' EXIT
-
-  list_tracked_gpu_dashboards > "$TMP/dashboards.txt"
-  [ -s "$TMP/dashboards.txt" ] || { echo "no gpu/* dashboards listed in $DASHBOARDS_LIST" >&2; exit 1; }
-
-  failed=0
-  while IFS= read -r dashboard_rel; do
-    dashboard="$DASHBOARDS_DIR/$dashboard_rel"
-    if [ ! -f "$dashboard" ]; then
-      echo "ERROR: $dashboard_rel listed in dashboards-infra.list but $dashboard does not exist" >&2
-      failed=1
-    fi
-  done < "$TMP/dashboards.txt"
-
-  [ "$failed" -eq 0 ]
-}

hack/cozyreport-summary.sh

@@ -0,0 +1,66 @@
+#!/bin/sh
+# Emit a human-readable summary of "what is broken" to a single file.
+# Reads the live cluster (not the report dir) so it can use kubectl JSONPath.
+# Usage: cozyreport-summary.sh > summary.txt
+set -eu
+
+echo "# Cozystack E2E Diagnostic Summary"
+echo "Generated: $(date -Iseconds)"
+echo
+
+echo "## HelmReleases not Ready"
+echo
+kubectl get hr -A --no-headers 2>/dev/null \
+  | awk '$4 != "True" {printf "  %s/%s — %s\n", $1, $2, $5}' \
+  | head -40
+echo
+
+echo "## Pods not Running/Succeeded"
+echo
+kubectl get pod -A --no-headers 2>/dev/null \
+  | awk '$4 !~ /Running|Succeeded|Completed/ {printf "  %s/%s — %s (restarts=%s, age=%s)\n", $1, $2, $4, $5, $6}' \
+  | head -40
+echo
+
+echo "## ImagePullBackOff / ErrImagePull"
+echo
+kubectl get pod -A --no-headers 2>/dev/null \
+  | awk '$4 ~ /ImagePullBackOff|ErrImagePull/ {printf "  %s/%s — %s\n", $1, $2, $4}'
+echo
+
+echo "## OOMKilled in last 30 min"
+echo
+kubectl get events -A --field-selector reason=OOMKilling --sort-by=.lastTimestamp 2>/dev/null \
+  | tail -20
+echo
+
+echo "## Recent Warning events (top 30)"
+echo
+kubectl get events -A --field-selector type=Warning --sort-by=.lastTimestamp 2>/dev/null \
+  | tail -30
+echo
+
+echo "## cert-manager: Certificates not Ready"
+echo
+if kubectl get crd certificates.cert-manager.io >/dev/null 2>&1; then
+  kubectl get certificates.cert-manager.io -A --no-headers 2>/dev/null \
+    | awk '$3 != "True" {printf "  %s/%s — Ready=%s\n", $1, $2, $3}'
+fi
+echo
+
+echo "## Flux Sources not Ready"
+echo
+for kind in helmrepositories.source.toolkit.fluxcd.io ocirepositories.source.toolkit.fluxcd.io gitrepositories.source.toolkit.fluxcd.io; do
+  kubectl get $kind -A --no-headers 2>/dev/null \
+    | awk -v k="${kind%%.*}" '$4 != "True" {printf "  %s %s/%s — Ready=%s\n", k, $1, $2, $4}'
+done
+echo
+
+echo "## Storage: PVCs not Bound, PVs not Bound"
+echo
+kubectl get pvc -A --no-headers 2>/dev/null | awk '$3 != "Bound" {printf "  PVC %s/%s — %s\n", $1, $2, $3}'
+kubectl get pv --no-headers 2>/dev/null    | awk '$5 != "Bound" {printf "  PV %s — %s\n", $1, $5}'
+echo
+
+echo "## Node Conditions"
+kubectl get nodes -o custom-columns=NAME:.metadata.name,READY:.status.conditions[?\(@.type==\"Ready\"\)].status,DISK:.status.conditions[?\(@.type==\"DiskPressure\"\)].status,MEM:.status.conditions[?\(@.type==\"MemoryPressure\"\)].status 2>/dev/null

hack/cozyreport.sh

@@ -9,10 +9,13 @@ command -V kubectl >/dev/null || exit $?
 command -V tar >/dev/null || exit $?
 
 # -- cozystack module
-
 echo "Collecting Cozystack information..."
 mkdir -p $REPORT_DIR/cozystack
 kubectl get deploy -n cozy-system cozystack -o jsonpath='{.spec.template.spec.containers[0].image}' > $REPORT_DIR/cozystack/image.txt 2>&1
+if kubectl get deploy -n cozy-system cozystack-operator >/dev/null 2>&1; then
+  kubectl logs -n cozy-system deploy/cozystack-operator --tail=2000 > $REPORT_DIR/cozystack/operator.log 2>&1
+  kubectl logs -n cozy-system deploy/cozystack-operator --tail=2000 --previous > $REPORT_DIR/cozystack/operator-previous.log 2>&1 || true
+fi
 kubectl get cm -n cozy-system --no-headers | awk '$1 ~ /^cozystack/' |
   while read NAME _; do
     DIR=$REPORT_DIR/cozystack/configs
@@ -20,6 +23,46 @@ kubectl get cm -n cozy-system --no-headers | awk '$1 ~ /^cozystack/' |
     kubectl get cm -n cozy-system $NAME -o yaml > $DIR/$NAME.yaml 2>&1
   done
 
+# -- flux module
+echo "Collecting Flux controller state..."
+mkdir -p $REPORT_DIR/flux
+for ctrl in helm-controller source-controller notification-controller kustomize-controller; do
+  if kubectl get deploy -n cozy-fluxcd $ctrl >/dev/null 2>&1; then
+    kubectl logs -n cozy-fluxcd deploy/$ctrl --tail=2000 > $REPORT_DIR/flux/$ctrl.log 2>&1
+    kubectl logs -n cozy-fluxcd deploy/$ctrl --tail=2000 --previous > $REPORT_DIR/flux/$ctrl-previous.log 2>&1 || true
+  fi
+done
+
+echo "Collecting Flux sources..."
+for kind in helmrepositories.source.toolkit.fluxcd.io ocirepositories.source.toolkit.fluxcd.io gitrepositories.source.toolkit.fluxcd.io externalartifacts.source.toolkit.fluxcd.io; do
+  short=${kind%%.*}
+  kubectl get $kind -A > $REPORT_DIR/flux/$short.txt 2>&1
+  kubectl get $kind -A -o yaml > $REPORT_DIR/flux/$short.yaml 2>&1
+done
+
+# -- cert-manager module
+if kubectl get crd certificates.cert-manager.io >/dev/null 2>&1; then
+  echo "Collecting cert-manager state..."
+  DIR=$REPORT_DIR/cert-manager
+  mkdir -p $DIR
+  kubectl get certificates.cert-manager.io -A > $DIR/certificates.txt 2>&1
+  kubectl get certificaterequests.cert-manager.io -A > $DIR/certificaterequests.txt 2>&1
+  kubectl get orders.acme.cert-manager.io -A > $DIR/orders.txt 2>&1
+  kubectl get challenges.acme.cert-manager.io -A > $DIR/challenges.txt 2>&1
+  # Per non-Ready cert: full yaml + describe
+  kubectl get certificates.cert-manager.io -A --no-headers 2>/dev/null | awk '$3 != "True"' | \
+    while read NAMESPACE NAME _; do
+      cdir=$DIR/certificates/$NAMESPACE/$NAME
+      mkdir -p $cdir
+      kubectl get certificates.cert-manager.io -n $NAMESPACE $NAME -o yaml > $cdir/cert.yaml 2>&1
+      kubectl describe certificates.cert-manager.io -n $NAMESPACE $NAME > $cdir/describe.txt 2>&1
+    done
+  if kubectl get deploy -n cozy-cert-manager cert-manager >/dev/null 2>&1; then
+    kubectl logs -n cozy-cert-manager deploy/cert-manager --tail=2000 > $DIR/cert-manager.log 2>&1
+    kubectl logs -n cozy-cert-manager deploy/cert-manager-webhook --tail=2000 > $DIR/cert-manager-webhook.log 2>&1
+  fi
+fi
+
 # -- kubernetes module
 
 echo "Collecting Kubernetes information..."
@@ -46,6 +89,13 @@ kubectl get ns --no-headers | awk '$2 != "Active"' |
     kubectl describe ns $NAME > $DIR/describe.txt 2>&1
   done
 
+echo "Collecting events..."
+kubectl get events -A --sort-by=.lastTimestamp > $REPORT_DIR/kubernetes/events.txt 2>&1
+# Filter to warning-class and recent for quick triage
+kubectl get events -A --sort-by=.lastTimestamp \
+  -o jsonpath='{range .items[?(@.type!="Normal")]}{.lastTimestamp}{"\t"}{.involvedObject.namespace}/{.involvedObject.kind}/{.involvedObject.name}{"\t"}{.reason}{"\t"}{.message}{"\n"}{end}' \
+  > $REPORT_DIR/kubernetes/events-warnings.txt 2>&1
+
 echo "Collecting helmreleases..."
 kubectl get hr -A > $REPORT_DIR/kubernetes/helmreleases.txt 2>&1
 kubectl get hr -A --no-headers | awk '$4 != "True"' | \
@@ -54,6 +104,13 @@ kubectl get hr -A --no-headers | awk '$4 != "True"' | \
     mkdir -p $DIR
     kubectl get hr -n $NAMESPACE $NAME -o yaml > $DIR/hr.yaml 2>&1
     kubectl describe hr -n $NAMESPACE $NAME > $DIR/describe.txt 2>&1
+    # Helm storage secrets: latest revision per release.
+    kubectl get secret -n $NAMESPACE -l owner=helm,name=$NAME --sort-by='.metadata.creationTimestamp' --no-headers 2>/dev/null | \
+      tail -1 | awk '{print $1}' | while read SECRET; do
+        [ -z "$SECRET" ] && continue
+        kubectl get secret -n $NAMESPACE $SECRET -o jsonpath='{.data.release}' 2>/dev/null \
+          | base64 -d | base64 -d | gzip -d > $DIR/helm-release.json 2>&1 || true
+      done
   done
 
 echo "Collecting packages..."
@@ -76,6 +133,23 @@ kubectl get packagesources --no-headers | awk '$3 != "True"' | \
     kubectl describe packagesource $NAME > $DIR/describe.txt 2>&1
   done
 
+echo "Collecting cozystack apps..."
+DIR=$REPORT_DIR/cozystack-apps
+mkdir -p $DIR
+for kind in applications.apps.cozystack.io applicationdefinitions.apps.cozystack.io tenants.apps.cozystack.io; do
+  short=${kind%%.*}
+  if kubectl get crd $kind >/dev/null 2>&1; then
+    kubectl get $kind -A > $DIR/$short.txt 2>&1
+    kubectl get $kind -A --no-headers 2>/dev/null | awk 'NF >= 3 && $NF != "True" && $NF != "Ready"' | \
+      while read NAMESPACE NAME _; do
+        d=$DIR/$short/$NAMESPACE/$NAME
+        mkdir -p $d
+        kubectl get $kind -n $NAMESPACE $NAME -o yaml > $d/$short.yaml 2>&1
+        kubectl describe $kind -n $NAMESPACE $NAME > $d/describe.txt 2>&1
+      done
+  fi
+done
+
 echo "Collecting pods..."
 kubectl get pod -A -o wide > $REPORT_DIR/kubernetes/pods.txt 2>&1
 kubectl get pod -A --no-headers | awk '$4 !~ /Running|Succeeded|Completed/' |
@@ -157,8 +231,28 @@ if kubectl get deploy -n cozy-linstor linstor-controller >/dev/null 2>&1; then
   kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor --no-color r l > $DIR/resources.txt 2>&1
 fi
 
+# -- sandbox-host module
+
+echo "Collecting sandbox host context..."
+DIR=$REPORT_DIR/sandbox-host
+mkdir -p $DIR
+df -h > $DIR/df.txt 2>&1
+free -m > $DIR/free.txt 2>&1
+ps auxww > $DIR/ps.txt 2>&1
+dmesg | tail -200 > $DIR/dmesg.txt 2>&1 || true
+if [ -f /workspace/talosconfig ]; then
+  for node in 192.168.123.11 192.168.123.12 192.168.123.13; do
+    talosctl --talosconfig /workspace/talosconfig -n $node dmesg --tail=200 > $DIR/talos-$node-dmesg.txt 2>&1 || true
+    talosctl --talosconfig /workspace/talosconfig -n $node logs kubelet --tail=500 > $DIR/talos-$node-kubelet.log 2>&1 || true
+    talosctl --talosconfig /workspace/talosconfig -n $node logs containerd --tail=500 > $DIR/talos-$node-containerd.log 2>&1 || true
+  done
+fi
+
 # -- finalization
 
+echo "Generating summary..."
+hack/cozyreport-summary.sh > $REPORT_DIR/summary.txt 2>&1 || true
+
 echo "Creating archive..."
 tar -czf $REPORT_NAME.tgz -C $REPORT_PDIR .
 echo "Report created: $REPORT_NAME.tgz"

hack/dcgm-default-counters.csv

@@ -1,104 +0,0 @@
-# Snapshot of the upstream DCGM Exporter default-counters.csv used as a
-# fixture by hack/check-gpu-recording-rules.bats. The test verifies that
-# every DCGM_FI_* metric referenced by a tracked GPU dashboard is either
-# declared here (upstream defaults) or in
-# packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml
-# (the project's custom CSV).
-#
-# Source: https://github.com/NVIDIA/dcgm-exporter/blob/4.1.1-4.0.4/etc/default-counters.csv
-# Pinned to the DCGM Exporter image tag shipped by the gpu-operator
-# chart under packages/system/gpu-operator/charts/gpu-operator/values.yaml
-# (dcgmExporter.version = 4.1.1-4.0.4-ubuntu22.04). When that image is
-# bumped, refresh this file from the matching tag in the NVIDIA/dcgm-exporter
-# repo and re-run `./hack/cozytest.sh hack/check-gpu-recording-rules.bats`.
-
-# Format
-# If line starts with a '#' it is considered a comment
-# DCGM FIELD, Prometheus metric type, help message
-
-# Clocks
-DCGM_FI_DEV_SM_CLOCK,  gauge, SM clock frequency (in MHz).
-DCGM_FI_DEV_MEM_CLOCK, gauge, Memory clock frequency (in MHz).
-
-# Temperature
-DCGM_FI_DEV_MEMORY_TEMP, gauge, Memory temperature (in C).
-DCGM_FI_DEV_GPU_TEMP,    gauge, GPU temperature (in C).
-
-# Power
-DCGM_FI_DEV_POWER_USAGE,              gauge, Power draw (in W).
-DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION, counter, Total energy consumption since boot (in mJ).
-
-# PCIE
-# DCGM_FI_PROF_PCIE_TX_BYTES,  counter, Total number of bytes transmitted through PCIe TX via NVML.
-# DCGM_FI_PROF_PCIE_RX_BYTES,  counter, Total number of bytes received through PCIe RX via NVML.
-DCGM_FI_DEV_PCIE_REPLAY_COUNTER, counter, Total number of PCIe retries.
-
-# Utilization (the sample period varies depending on the product)
-DCGM_FI_DEV_GPU_UTIL,      gauge, GPU utilization (in %).
-DCGM_FI_DEV_MEM_COPY_UTIL, gauge, Memory utilization (in %).
-DCGM_FI_DEV_ENC_UTIL,      gauge, Encoder utilization (in %).
-DCGM_FI_DEV_DEC_UTIL ,     gauge, Decoder utilization (in %).
-
-# Errors and violations
-DCGM_FI_DEV_XID_ERRORS,              gauge,   Value of the last XID error encountered.
-# DCGM_FI_DEV_POWER_VIOLATION,       counter, Throttling duration due to power constraints (in us).
-# DCGM_FI_DEV_THERMAL_VIOLATION,     counter, Throttling duration due to thermal constraints (in us).
-# DCGM_FI_DEV_SYNC_BOOST_VIOLATION,  counter, Throttling duration due to sync-boost constraints (in us).
-# DCGM_FI_DEV_BOARD_LIMIT_VIOLATION, counter, Throttling duration due to board limit constraints (in us).
-# DCGM_FI_DEV_LOW_UTIL_VIOLATION,    counter, Throttling duration due to low utilization (in us).
-# DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (in us).
-
-# Memory usage
-DCGM_FI_DEV_FB_FREE, gauge, Framebuffer memory free (in MiB).
-DCGM_FI_DEV_FB_USED, gauge, Framebuffer memory used (in MiB).
-
-# ECC
-# DCGM_FI_DEV_ECC_SBE_VOL_TOTAL, counter, Total number of single-bit volatile ECC errors.
-# DCGM_FI_DEV_ECC_DBE_VOL_TOTAL, counter, Total number of double-bit volatile ECC errors.
-# DCGM_FI_DEV_ECC_SBE_AGG_TOTAL, counter, Total number of single-bit persistent ECC errors.
-# DCGM_FI_DEV_ECC_DBE_AGG_TOTAL, counter, Total number of double-bit persistent ECC errors.
-
-# Retired pages
-# DCGM_FI_DEV_RETIRED_SBE,     counter, Total number of retired pages due to single-bit errors.
-# DCGM_FI_DEV_RETIRED_DBE,     counter, Total number of retired pages due to double-bit errors.
-# DCGM_FI_DEV_RETIRED_PENDING, counter, Total number of pages pending retirement.
-
-# NVLink
-# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_TOTAL, counter, Total number of NVLink flow-control CRC errors.
-# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_TOTAL, counter, Total number of NVLink data CRC errors.
-# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_TOTAL,   counter, Total number of NVLink retries.
-# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_TOTAL, counter, Total number of NVLink recovery errors.
-DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL,            counter, Total number of NVLink bandwidth counters for all lanes.
-# DCGM_FI_DEV_NVLINK_BANDWIDTH_L0,               counter, The number of bytes of active NVLink rx or tx data including both header and payload.
-
-# VGPU License status
-DCGM_FI_DEV_VGPU_LICENSE_STATUS, gauge, vGPU License status
-
-# Remapped rows
-DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for uncorrectable errors
-DCGM_FI_DEV_CORRECTABLE_REMAPPED_ROWS,   counter, Number of remapped rows for correctable errors
-DCGM_FI_DEV_ROW_REMAP_FAILURE,           gauge,   Whether remapping of rows has failed
-
-# Static configuration information. These appear as labels on the other metrics
-DCGM_FI_DRIVER_VERSION,        label, Driver Version
-# DCGM_FI_NVML_VERSION,          label, NVML Version
-# DCGM_FI_DEV_BRAND,             label, Device Brand
-# DCGM_FI_DEV_SERIAL,            label, Device Serial Number
-# DCGM_FI_DEV_OEM_INFOROM_VER,   label, OEM inforom version
-# DCGM_FI_DEV_ECC_INFOROM_VER,   label, ECC inforom version
-# DCGM_FI_DEV_POWER_INFOROM_VER, label, Power management object inforom version
-# DCGM_FI_DEV_INFOROM_IMAGE_VER, label, Inforom image version
-# DCGM_FI_DEV_VBIOS_VERSION,     label, VBIOS version of the device
-
-# Datacenter Profiling (DCP) metrics
-# NOTE: supported on Nvidia datacenter Volta GPUs and newer
-DCGM_FI_PROF_GR_ENGINE_ACTIVE,   gauge, Ratio of time the graphics engine is active.
-# DCGM_FI_PROF_SM_ACTIVE,          gauge, The ratio of cycles an SM has at least 1 warp assigned.
-# DCGM_FI_PROF_SM_OCCUPANCY,       gauge, The ratio of number of warps resident on an SM.
-DCGM_FI_PROF_PIPE_TENSOR_ACTIVE, gauge, Ratio of cycles the tensor (HMMA) pipe is active.
-DCGM_FI_PROF_DRAM_ACTIVE,        gauge, Ratio of cycles the device memory interface is active sending or receiving data.
-# DCGM_FI_PROF_PIPE_FP64_ACTIVE,   gauge, Ratio of cycles the fp64 pipes are active.
-# DCGM_FI_PROF_PIPE_FP32_ACTIVE,   gauge, Ratio of cycles the fp32 pipes are active.
-# DCGM_FI_PROF_PIPE_FP16_ACTIVE,   gauge, Ratio of cycles the fp16 pipes are active.
-DCGM_FI_PROF_PCIE_TX_BYTES,      gauge, The rate of data transmitted over the PCIe bus - including both protocol headers and data payloads - in bytes per second.
-DCGM_FI_PROF_PCIE_RX_BYTES,      gauge, The rate of data received over the PCIe bus - including both protocol headers and data payloads - in bytes per second.

hack/e2e-apps/bucket.bats

@@ -20,8 +20,11 @@ EOF
 
   # Wait for the bucket to be ready
   kubectl -n tenant-test wait hr bucket-${name} --timeout=100s --for=condition=ready
+  timeout 60 sh -ec "until kubectl -n tenant-test get bucketclaims.objectstorage.k8s.io bucket-${name} >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait bucketclaims.objectstorage.k8s.io bucket-${name} --timeout=300s --for=jsonpath='{.status.bucketReady}'
+  timeout 60 sh -ec "until kubectl -n tenant-test get bucketaccesses.objectstorage.k8s.io bucket-${name}-admin >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait bucketaccesses.objectstorage.k8s.io bucket-${name}-admin --timeout=300s --for=jsonpath='{.status.accessGranted}'
+  timeout 60 sh -ec "until kubectl -n tenant-test get bucketaccesses.objectstorage.k8s.io bucket-${name}-viewer >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait bucketaccesses.objectstorage.k8s.io bucket-${name}-viewer --timeout=300s --for=jsonpath='{.status.accessGranted}'
 
   # Get admin (readwrite) credentials

hack/e2e-apps/clickhouse.bats

@@ -35,9 +35,12 @@ spec:
   resources: {}
   resourcesPreset: "nano"
 EOF
-  sleep 5
+  # Wait for the operator to materialise the HelmRelease before kubectl wait
+  # kicks in (kubectl wait errors immediately if the object does not exist yet).
+  timeout 60 sh -ec "until kubectl -n tenant-test get hr clickhouse-$name >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait hr clickhouse-$name --timeout=20s --for=condition=ready
   timeout 180 sh -ec "until kubectl -n tenant-test get svc chendpoint-clickhouse-$name -o jsonpath='{.spec.ports[*].port}' | grep -q '8123 9000'; do sleep 10; done"
+  timeout 60 sh -ec "until kubectl -n tenant-test get statefulset.apps/chi-clickhouse-$name-clickhouse-0-0 >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait statefulset.apps/chi-clickhouse-$name-clickhouse-0-0 --timeout=120s --for=jsonpath='{.status.replicas}'=1
   timeout 80 sh -ec "until kubectl -n tenant-test get endpoints chi-clickhouse-$name-clickhouse-0-0 -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done"
   timeout 100 sh -ec "until kubectl -n tenant-test get svc chi-clickhouse-$name-clickhouse-0-0 -o jsonpath='{.spec.ports[*].port}' | grep -q '9000 8123 9009'; do sleep 10; done"

hack/e2e-apps/external-dns.bats

@@ -18,7 +18,9 @@ spec:
     - example.com
 EOF
 
-  sleep 5
+  # Wait for the operator to materialise the HelmRelease before kubectl wait
+  # kicks in (kubectl wait errors immediately if the object does not exist yet).
+  timeout 60 sh -ec "until kubectl -n tenant-test get hr ${name} >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait hr ${name} --timeout=120s --for=condition=ready
   kubectl -n tenant-test wait hr ${name}-system --timeout=120s --for=condition=ready
   timeout 60 sh -ec "until kubectl -n tenant-test get deploy -l app.kubernetes.io/instance=${name}-system -o jsonpath='{.items[0].status.readyReplicas}' | grep -q '1'; do sleep 5; done"
@@ -41,7 +43,9 @@ spec:
     - example.org
 EOF
 
-  sleep 5
+  # Wait for the operator to materialise the HelmRelease before kubectl wait
+  # kicks in (kubectl wait errors immediately if the object does not exist yet).
+  timeout 60 sh -ec "until kubectl -n tenant-test get hr ${name} >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait hr ${name} --timeout=120s --for=condition=ready
   kubectl -n tenant-test wait hr ${name}-system --timeout=120s --for=condition=ready
   timeout 60 sh -ec "until kubectl -n tenant-test get deploy -l app.kubernetes.io/instance=${name}-system -o jsonpath='{.items[0].status.readyReplicas}' | grep -q '1'; do sleep 5; done"

hack/e2e-apps/foundationdb.bats

@@ -42,7 +42,9 @@ spec:
   imageType: "unified"
   automaticReplacements: true
 EOF
-  sleep 15
+  # Wait for the operator to materialise the HelmRelease before kubectl wait
+  # kicks in (kubectl wait errors immediately if the object does not exist yet).
+  timeout 60 sh -ec "until kubectl -n tenant-test get hr foundationdb-$name >/dev/null 2>&1; do sleep 2; done"
 
   # Wait for HelmRelease to be ready
   kubectl -n tenant-test wait hr foundationdb-$name --timeout=300s --for=condition=ready

hack/e2e-apps/harbor.bats

@@ -38,12 +38,16 @@ spec:
     size: 1Gi
     replicas: 1
 EOF
-  sleep 5
+  # Wait for the operator to materialise the HelmRelease before kubectl wait
+  # kicks in (kubectl wait errors immediately if the object does not exist yet).
+  timeout 60 sh -ec "until kubectl -n tenant-test get hr $release >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait hr $release --timeout=60s --for=condition=ready
 
   # Wait for COSI to provision bucket
+  timeout 60 sh -ec "until kubectl -n tenant-test get bucketclaims.objectstorage.k8s.io $release-registry >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait bucketclaims.objectstorage.k8s.io $release-registry \
     --timeout=300s --for=jsonpath='{.status.bucketReady}'=true
+  timeout 60 sh -ec "until kubectl -n tenant-test get bucketaccesses.objectstorage.k8s.io $release-registry >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait bucketaccesses.objectstorage.k8s.io $release-registry \
     --timeout=60s --for=jsonpath='{.status.accessGranted}'=true
 
@@ -64,8 +68,11 @@ EOF
     kubectl -n tenant-test get secret $release-registry-bucket -o jsonpath='{.data.BucketInfo}' 2>&1 | base64 -d 2>&1 || true
     false
   }
+  timeout 60 sh -ec "until kubectl -n tenant-test get deploy $release-core >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait deploy $release-core --timeout=120s --for=condition=available
+  timeout 60 sh -ec "until kubectl -n tenant-test get deploy $release-registry >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait deploy $release-registry --timeout=120s --for=condition=available
+  timeout 60 sh -ec "until kubectl -n tenant-test get deploy $release-portal >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait deploy $release-portal --timeout=120s --for=condition=available
   kubectl -n tenant-test get secret $release-credentials -o jsonpath='{.data.admin-password}' | base64 --decode | grep -q '.'
   kubectl -n tenant-test get secret $release-credentials -o jsonpath='{.data.url}' | base64 --decode | grep -q 'https://'

hack/e2e-apps/kafka.bats

@@ -39,8 +39,11 @@ spec:
       partitions: 1
       replicas: 2
 EOF
-  sleep 5
+  # Wait for the operator to materialise the HelmRelease before kubectl wait
+  # kicks in (kubectl wait errors immediately if the object does not exist yet).
+  timeout 60 sh -ec "until kubectl -n tenant-test get hr kafka-$name >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait hr kafka-$name --timeout=30s --for=condition=ready
+  timeout 60 sh -ec "until kubectl -n tenant-test get kafkas test >/dev/null 2>&1; do sleep 2; done"
   kubectl wait kafkas -n tenant-test test --timeout=300s --for=condition=ready
   timeout 60 sh -ec "until kubectl -n tenant-test get pvc data-kafka-$name-zookeeper-0; do sleep 10; done"
   kubectl -n tenant-test wait pvc data-kafka-$name-zookeeper-0 --timeout=50s --for=jsonpath='{.status.phase}'=Bound

hack/e2e-apps/mariadb.bats

@@ -35,13 +35,17 @@ spec:
   resources: {}
   resourcesPreset: "nano"
 EOF
-  sleep 5
+  # Wait for the operator to materialise the HelmRelease before kubectl wait
+  # kicks in (kubectl wait errors immediately if the object does not exist yet).
+  timeout 60 sh -ec "until kubectl -n tenant-test get hr mariadb-$name >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait hr mariadb-$name --timeout=30s --for=condition=ready
   timeout 80 sh -ec "until kubectl -n tenant-test get svc mariadb-$name -o jsonpath='{.spec.ports[0].port}' | grep -q '3306'; do sleep 10; done"
   timeout 80 sh -ec "until kubectl -n tenant-test get endpoints mariadb-$name -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done"
+  timeout 60 sh -ec "until kubectl -n tenant-test get statefulset.apps/mariadb-$name >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait statefulset.apps/mariadb-$name --timeout=110s --for=jsonpath='{.status.replicas}'=2
   timeout 80 sh -ec "until kubectl -n tenant-test get svc mariadb-$name-metrics -o jsonpath='{.spec.ports[0].port}' | grep -q '9104'; do sleep 10; done"
   timeout 40 sh -ec "until kubectl -n tenant-test get endpoints mariadb-$name-metrics -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done"
+  timeout 60 sh -ec "until kubectl -n tenant-test get deployment.apps/mariadb-$name-metrics >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait deployment.apps/mariadb-$name-metrics --timeout=90s --for=jsonpath='{.status.replicas}'=1
   kubectl -n tenant-test delete mariadbs.apps.cozystack.io $name
 }

hack/e2e-apps/mongodb.bats

@@ -26,7 +26,9 @@ spec:
   backup:
     enabled: false
 EOF
-  sleep 5
+  # Wait for the operator to materialise the HelmRelease before kubectl wait
+  # kicks in (kubectl wait errors immediately if the object does not exist yet).
+  timeout 60 sh -ec "until kubectl -n tenant-test get hr mongodb-$name >/dev/null 2>&1; do sleep 2; done"
   # Wait for HelmRelease
   kubectl -n tenant-test wait hr mongodb-$name --timeout=60s --for=condition=ready
   # Wait for MongoDB service (port 27017)
@@ -34,6 +36,7 @@ EOF
   # Wait for endpoints
   timeout 180 sh -ec "until kubectl -n tenant-test get endpoints mongodb-$name-rs0 -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done"
   # Wait for StatefulSet replicas
+  timeout 60 sh -ec "until kubectl -n tenant-test get statefulset.apps/mongodb-$name-rs0 >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait statefulset.apps/mongodb-$name-rs0 --timeout=300s --for=jsonpath='{.status.replicas}'=1
   # Cleanup
   kubectl -n tenant-test delete mongodbs.apps.cozystack.io $name

hack/e2e-apps/openbao.bats

@@ -18,7 +18,9 @@ spec:
   external: false
   ui: true
 EOF
-  sleep 5
+  # Wait for the operator to materialise the HelmRelease before kubectl wait
+  # kicks in (kubectl wait errors immediately if the object does not exist yet).
+  timeout 60 sh -ec "until kubectl -n tenant-test get hr openbao-$name >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait hr openbao-$name --timeout=60s --for=condition=ready
   kubectl -n tenant-test wait hr openbao-$name-system --timeout=120s --for=condition=ready
 
@@ -53,7 +55,9 @@ EOF
   kubectl -n tenant-test exec openbao-$name-0 -- bao operator unseal "$unseal_key"
 
   # Now wait for pod to become ready (readiness probe checks seal status)
+  timeout 60 sh -ec "until kubectl -n tenant-test get sts openbao-$name >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait sts openbao-$name --timeout=90s --for=jsonpath='{.status.readyReplicas}'=1
+  timeout 60 sh -ec "until kubectl -n tenant-test get pvc data-openbao-$name-0 >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait pvc data-openbao-$name-0 --timeout=50s --for=jsonpath='{.status.phase}'=Bound
   kubectl -n tenant-test delete openbao.apps.cozystack.io $name
   kubectl -n tenant-test delete pvc data-openbao-$name-0 --ignore-not-found

hack/e2e-apps/postgres.bats

@@ -40,8 +40,11 @@ spec:
   resources: {}
   resourcesPreset: "nano"
 EOF
-  sleep 5
+  # Wait for the operator to materialise the HelmRelease before kubectl wait
+  # kicks in (kubectl wait errors immediately if the object does not exist yet).
+  timeout 60 sh -ec "until kubectl -n tenant-test get hr postgres-$name >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait hr postgres-$name --timeout=100s --for=condition=ready
+  timeout 60 sh -ec "until kubectl -n tenant-test get job.batch postgres-$name-init-job >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait job.batch postgres-$name-init-job --timeout=50s --for=condition=Complete
   timeout 40 sh -ec "until kubectl -n tenant-test get svc postgres-$name-r -o jsonpath='{.spec.ports[0].port}' | grep -q '5432'; do sleep 10; done"
   timeout 40 sh -ec "until kubectl -n tenant-test get svc postgres-$name-ro -o jsonpath='{.spec.ports[0].port}' | grep -q '5432'; do sleep 10; done"

hack/e2e-apps/qdrant.bats

@@ -17,10 +17,14 @@ spec:
   resources: {}
   external: false
 EOF
-  sleep 5
+  # Wait for the operator to materialise the HelmRelease before kubectl wait
+  # kicks in (kubectl wait errors immediately if the object does not exist yet).
+  timeout 60 sh -ec "until kubectl -n tenant-test get hr qdrant-$name >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait hr qdrant-$name --timeout=60s --for=condition=ready
   kubectl -n tenant-test wait hr qdrant-$name-system --timeout=120s --for=condition=ready
+  timeout 60 sh -ec "until kubectl -n tenant-test get sts qdrant-$name >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait sts qdrant-$name --timeout=90s --for=jsonpath='{.status.readyReplicas}'=1
+  timeout 60 sh -ec "until kubectl -n tenant-test get pvc qdrant-storage-qdrant-$name-0 >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait pvc qdrant-storage-qdrant-$name-0 --timeout=50s --for=jsonpath='{.status.phase}'=Bound
   kubectl -n tenant-test delete qdrant.apps.cozystack.io $name
 }

hack/e2e-apps/redis.bats

@@ -18,10 +18,15 @@ spec:
   resources: {}
   resourcesPreset: "nano"
 EOF
-  sleep 5
+  # Wait for the operator to materialise the HelmRelease before kubectl wait
+  # kicks in (kubectl wait errors immediately if the object does not exist yet).
+  timeout 60 sh -ec "until kubectl -n tenant-test get hr redis-$name >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait hr redis-$name --timeout=20s --for=condition=ready
+  timeout 60 sh -ec "until kubectl -n tenant-test get pvc redisfailover-persistent-data-rfr-redis-$name-0 >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait pvc redisfailover-persistent-data-rfr-redis-$name-0 --timeout=50s --for=jsonpath='{.status.phase}'=Bound
+  timeout 60 sh -ec "until kubectl -n tenant-test get deploy rfs-redis-$name >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait deploy rfs-redis-$name --timeout=90s --for=condition=available
+  timeout 60 sh -ec "until kubectl -n tenant-test get sts rfr-redis-$name >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait sts rfr-redis-$name --timeout=90s --for=jsonpath='{.status.replicas}'=2
   kubectl -n tenant-test delete redis.apps.cozystack.io $name
 }

hack/e2e-apps/run-kubernetes.sh

@@ -227,6 +227,11 @@ EOF
     exit 1
   fi
 
+  # TODO(e2e-replace-fixed-timeouts): genuine retry loop. This validates an
+  # external HTTP path (MetalLB-advertised LB IP -> in-tenant ingress ->
+  # backend pod) which is not visible to the Kubernetes API as a single
+  # condition, so kubectl wait cannot replace it. The 20x3s = 60s budget is
+  # capped with `lb_ok=false` then asserted below.
   lb_ok=false
   for i in $(seq 1 20); do
     echo "Attempt $i"

hack/e2e-apps/vminstance.bats

@@ -2,8 +2,15 @@
 
 @test "Create a VM Disk" {
   name='test'
-  kubectl -n tenant-test delete vminstances.apps.cozystack.io $name --ignore-not-found --timeout=2m || true
-  kubectl -n tenant-test delete vmdisks.apps.cozystack.io $name --ignore-not-found --timeout=2m || true
+  # Delete any leftover from a previous run and BLOCK until removal completes.
+  # `kubectl apply` on a resource still in finalizer-drain silently no-ops
+  # ("Detected changes to resource ... which is currently being deleted"),
+  # which then races a NotFound on the downstream HR wait. The previous
+  # `|| true` swallowed timeout errors here and let the test continue with
+  # the old VMDisk still draining. Bumped timeout to 3m and removed `|| true`
+  # so a true delete failure surfaces immediately.
+  kubectl -n tenant-test delete vminstances.apps.cozystack.io $name --ignore-not-found --timeout=3m
+  kubectl -n tenant-test delete vmdisks.apps.cozystack.io $name --ignore-not-found --timeout=3m
   kubectl apply -f - <<EOF
 apiVersion: apps.cozystack.io/v1alpha1
 kind: VMDisk
@@ -18,16 +25,22 @@ spec:
   storage: 5Gi
   storageClass: replicated
 EOF
-  sleep 5
+  # Wait for the operator to materialise the HelmRelease before kubectl wait
+  # kicks in (kubectl wait errors immediately if the object does not exist yet).
+  timeout 60 sh -ec "until kubectl -n tenant-test get hr vm-disk-$name >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait hr vm-disk-$name --timeout=5s --for=condition=ready
+  timeout 120 sh -ec "until kubectl -n tenant-test get dv vm-disk-$name >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait dv vm-disk-$name --timeout=250s --for=condition=ready
+  timeout 120 sh -ec "until kubectl -n tenant-test get pvc vm-disk-$name >/dev/null 2>&1; do sleep 2; done"
   kubectl -n tenant-test wait pvc vm-disk-$name --timeout=200s --for=jsonpath='{.status.phase}'=Bound
 }
 
 @test "Create a VM Instance" {
   diskName='test'
   name='test'
-  kubectl -n tenant-test delete vminstances.apps.cozystack.io $name --ignore-not-found --timeout=2m || true
+  # Same delete-finalizer-drain race as in "Create a VM Disk" above —
+  # block until removal completes, surface timeouts loudly.
+  kubectl -n tenant-test delete vminstances.apps.cozystack.io $name --ignore-not-found --timeout=3m
   kubectl apply -f - <<EOF
 apiVersion: apps.cozystack.io/v1alpha1
 kind: VMInstance
@@ -59,10 +72,17 @@ spec:
           - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPht0dPk5qQ+54g1hSX7A6AUxXJW5T6n/3d7Ga2F8gTF test@test
   cloudInitSeed: ""
 EOF
-  sleep 5
-  timeout 20 sh -ec "until kubectl -n tenant-test get vmi vm-instance-$name -o jsonpath='{.status.interfaces[0].ipAddress}' | grep -q '[0-9]'; do sleep 5; done"
+  # Wait for the operator to materialise the HelmRelease before downstream
+  # waits proceed (kubectl wait errors immediately if the HR does not exist).
+  timeout 60 sh -ec "until kubectl -n tenant-test get hr vm-instance-$name >/dev/null 2>&1; do sleep 2; done"
+  # Nested KubeVirt VM startup (virt-launcher + libvirt + cloud-init DHCP)
+  # routinely takes 30-60s under runner load; the previous 20s was unrealistic
+  # and produced flakes. 120s is a comfortable upper bound for nested virt.
+  timeout 120 sh -ec "until kubectl -n tenant-test get vmi vm-instance-$name -o jsonpath='{.status.interfaces[0].ipAddress}' | grep -q '[0-9]'; do sleep 2; done"
   kubectl -n tenant-test wait hr vm-instance-$name --timeout=5s --for=condition=ready
-  kubectl -n tenant-test wait vm vm-instance-$name --timeout=20s --for=condition=ready
-  kubectl -n tenant-test delete vminstances.apps.cozystack.io $name 
-  kubectl -n tenant-test delete vmdisks.apps.cozystack.io $diskName 
+  # VM ready follows IP assignment closely; 60s gives buffer for the qemu-guest-agent.
+  timeout 120 sh -ec "until kubectl -n tenant-test get vm vm-instance-$name >/dev/null 2>&1; do sleep 2; done"
+  kubectl -n tenant-test wait vm vm-instance-$name --timeout=60s --for=condition=ready
+  kubectl -n tenant-test delete vminstances.apps.cozystack.io $name --timeout=3m
+  kubectl -n tenant-test delete vmdisks.apps.cozystack.io $diskName --timeout=3m
 }

hack/e2e-install-cozystack.bats

@@ -7,12 +7,52 @@
   fi
 }
 
+@test "Pre-pull platform images" {
+  # Cluster-member workloads (OVN raft, LINSTOR) fail if replicas start at
+  # different times due to image-pull stagger across nodes. Pre-pull these
+  # images to every node so all replicas start with images already cached.
+  #
+  # Source images directly from the rendered charts so version bumps stay in
+  # sync automatically. yq walks every PodSpec-shaped object and emits the
+  # images of each container — this scopes the result to images the kubelet
+  # actually pulls (skips configmap fields and CRD examples that happen to
+  # contain an `image:` key). Add a chart here when a new peer-sensitive
+  # workload is found.
+  {
+    helm template packages/system/kubeovn
+    helm template packages/system/linstor
+  } | yq -N '
+      (..|select(has("containers"))|.containers[]|.image),
+      (..|select(has("initContainers"))|.initContainers[]|.image)
+    ' | hack/e2e-prepull-images.sh
+}
+
 @test "Install Cozystack" {
-  # Install cozy-installer chart (operator installs CRDs on startup via --install-crds)
+  # Pre-create the cozy-system namespace with the labels the operator pod needs.
+  # The chart no longer ships a Namespace resource (helm v3's --create-namespace
+  # collides with chart-defined Namespace). On Talos clusters that enforce PSA
+  # baseline-by-default, the cozystack-operator pod (hostNetwork=true) is
+  # rejected unless the namespace is labelled enforce=privileged BEFORE the
+  # pod is scheduled. A pre-install hook can't bootstrap that label because
+  # the hook itself runs in the same namespace and faces the same PSA gate.
+  # So apply the namespace from the caller (this bats / production install docs)
+  # — same pattern as kube-prometheus-stack, argo-cd, cert-manager.
+  kubectl apply -f - <<'EOF'
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: cozy-system
+  labels:
+    cozystack.io/system: "true"
+    cozystack.io/deletion-protected: "true"
+    pod-security.kubernetes.io/enforce: privileged
+EOF
+
+  # Install cozy-installer chart (operator installs CRDs on startup via --install-crds).
   helm upgrade installer packages/core/installer \
     --install \
     --namespace cozy-system \
-    --create-namespace \
+    --set cozystackOperator.helmReleaseInterval=30s \
     --wait \
     --timeout 2m
 
@@ -51,11 +91,31 @@ spec:
             - cozystack.external-dns-application
 EOF
 
+  # Launch storage + LB configuration in the background. It waits for its
+  # own prerequisites (linstor-controller deploy, MetalLB CRDs) and finishes
+  # while the parallel HR wait below is still running, so the cost overlaps
+  # with the platform reconcile instead of compounding it.
+  hack/e2e-post-install-prep.sh > /tmp/post-install-prep.log 2>&1 &
+  POST_PREP_PID=$!
+
   # Wait until HelmReleases appear & reconcile them
   timeout 180 sh -ec 'until [ $(kubectl get hr -A --no-headers 2>/dev/null | wc -l) -gt 10 ]; do sleep 1; done'
+  # TODO(e2e-replace-fixed-timeouts): genuine sleep. The threshold of 10 is a
+  # heuristic for "enough HRs visible to start waiting"; the snapshot below
+  # uses whatever HRs have appeared by then. There is no objective k8s API
+  # signal for "all platform HRs have been emitted" without hard-coding the
+  # expected list, so the 5s pad lets a few late-arrivals join the snapshot.
   sleep 5
   kubectl get hr -A | awk 'NR>1 {print "kubectl wait --timeout=15m --for=condition=ready -n "$1" hr/"$2" &"} END {print "wait"}' | sh -ex
 
+  echo "Waiting for post-install-prep to complete"
+  if ! wait $POST_PREP_PID; then
+    cat /tmp/post-install-prep.log >&2
+    echo "post-install-prep failed" >&2
+    exit 1
+  fi
+  cat /tmp/post-install-prep.log
+
   # Fail the test if any HelmRelease is not Ready
   if kubectl get hr -A | grep -v " True " | grep -v NAME; then
     kubectl get hr -A
@@ -69,80 +129,8 @@ EOF
   kubectl wait deployment/capi-controller-manager deployment/capi-kamaji-controller-manager deployment/capi-kubeadm-bootstrap-controller-manager deployment/capi-operator-cluster-api-operator deployment/capk-controller-manager -n cozy-cluster-api --timeout=2m --for=condition=available
 }
 
-@test "Wait for LINSTOR and configure storage" {
-  # Linstor controller and nodes
-  kubectl wait deployment/linstor-controller -n cozy-linstor --timeout=5m --for=condition=available
-  timeout 60 sh -ec 'until [ $(kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor node list | grep -c Online) -eq 3 ]; do sleep 1; done'
-
-  created_pools=$(kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor sp l -s data --pastable | awk '$2 == "data" {printf " " $4} END{printf " "}')
-  for node in srv1 srv2 srv3; do
-    case $created_pools in
-      *" $node "*) echo "Storage pool 'data' already exists on node $node"; continue;;
-    esac
-    kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor ps cdp zfs ${node} /dev/vdc --pool-name data --storage-pool data
-  done
-
-  # Storage classes
-  kubectl apply -f - <<'EOF'
----
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  name: local
-  annotations:
-    storageclass.kubernetes.io/is-default-class: "true"
-provisioner: linstor.csi.linbit.com
-parameters:
-  linstor.csi.linbit.com/storagePool: "data"
-  linstor.csi.linbit.com/layerList: "storage"
-  linstor.csi.linbit.com/allowRemoteVolumeAccess: "false"
-volumeBindingMode: WaitForFirstConsumer
-allowVolumeExpansion: true
----
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  name: replicated
-provisioner: linstor.csi.linbit.com
-parameters:
-  linstor.csi.linbit.com/storagePool: "data"
-  linstor.csi.linbit.com/autoPlace: "3"
-  linstor.csi.linbit.com/layerList: "drbd storage"
-  linstor.csi.linbit.com/allowRemoteVolumeAccess: "true"
-  property.linstor.csi.linbit.com/DrbdOptions/auto-quorum: suspend-io
-  property.linstor.csi.linbit.com/DrbdOptions/Resource/on-no-data-accessible: suspend-io
-  property.linstor.csi.linbit.com/DrbdOptions/Resource/on-suspended-primary-outdated: force-secondary
-  property.linstor.csi.linbit.com/DrbdOptions/Net/rr-conflict: retry-connect
-volumeBindingMode: Immediate
-allowVolumeExpansion: true
-EOF
-}
-
-@test "Wait for MetalLB and configure address pool" {
-  # MetalLB address pool
-  kubectl apply -f - <<'EOF'
----
-apiVersion: metallb.io/v1beta1
-kind: L2Advertisement
-metadata:
-  name: cozystack
-  namespace: cozy-metallb
-spec:
-  ipAddressPools: [cozystack]
----
-apiVersion: metallb.io/v1beta1
-kind: IPAddressPool
-metadata:
-  name: cozystack
-  namespace: cozy-metallb
-spec:
-  addresses: [192.168.123.200-192.168.123.250]
-  autoAssign: true
-  avoidBuggyIPs: false
-EOF
-}
-
 @test "Check Cozystack API service" {
+  timeout 60 sh -ec 'until kubectl get apiservices/v1alpha1.apps.cozystack.io apiservices/v1alpha1.core.cozystack.io >/dev/null 2>&1; do sleep 2; done'
   kubectl wait --for=condition=Available apiservices/v1alpha1.apps.cozystack.io apiservices/v1alpha1.core.cozystack.io --timeout=2m
 }
 
@@ -174,15 +162,22 @@ EOF
   kubectl wait deploy/root-ingress-controller -n tenant-root --timeout=5m --for=condition=available
 
   # etcd statefulset
+  timeout 60 sh -ec 'until kubectl get sts/etcd -n tenant-root >/dev/null 2>&1; do sleep 2; done'
   kubectl wait sts/etcd -n tenant-root --for=jsonpath='{.status.readyReplicas}'=3 --timeout=5m
 
   # VictoriaMetrics components
+  timeout 60 sh -ec 'until kubectl get vmalert/vmalert-shortterm -n tenant-root >/dev/null 2>&1; do sleep 2; done'
+  timeout 60 sh -ec 'until kubectl get vmalertmanager/alertmanager -n tenant-root >/dev/null 2>&1; do sleep 2; done'
   kubectl wait vmalert/vmalert-shortterm vmalertmanager/alertmanager -n tenant-root --for=jsonpath='{.status.updateStatus}'=operational --timeout=15m
+  timeout 60 sh -ec 'until kubectl get vlclusters/generic -n tenant-root >/dev/null 2>&1; do sleep 2; done'
   kubectl wait vlclusters/generic -n tenant-root --for=jsonpath='{.status.updateStatus}'=operational --timeout=5m
+  timeout 60 sh -ec 'until kubectl get vmcluster/shortterm vmcluster/longterm -n tenant-root >/dev/null 2>&1; do sleep 2; done'
   kubectl wait vmcluster/shortterm vmcluster/longterm -n tenant-root --for=jsonpath='{.status.updateStatus}'=operational --timeout=5m
 
   # Grafana
+  timeout 60 sh -ec 'until kubectl get clusters.postgresql.cnpg.io/grafana-db -n tenant-root >/dev/null 2>&1; do sleep 2; done'
   kubectl wait clusters.postgresql.cnpg.io/grafana-db -n tenant-root --for=condition=ready --timeout=5m
+  timeout 60 sh -ec 'until kubectl get deploy/grafana-deployment -n tenant-root >/dev/null 2>&1; do sleep 2; done'
   kubectl wait deploy/grafana-deployment -n tenant-root --for=condition=available --timeout=5m
 
   # Verify Grafana via ingress
@@ -273,6 +268,7 @@ spec:
   seaweedfs: false
 EOF
   kubectl wait hr/tenant-test -n tenant-root --timeout=1m --for=condition=ready
+  timeout 60 sh -ec 'until kubectl get namespace tenant-test >/dev/null 2>&1; do sleep 2; done'
   kubectl wait namespace tenant-test --timeout=20s --for=jsonpath='{.status.phase}'=Active
   # Wait for ResourceQuota to appear and assert values
   timeout 60 sh -ec 'until [ "$(kubectl get quota -n tenant-test --no-headers 2>/dev/null | wc -l)" -ge 1 ]; do sleep 1; done'

hack/e2e-post-install-prep.sh

@@ -0,0 +1,96 @@
+#!/bin/sh
+# Runs LINSTOR pool + StorageClass + MetalLB pool configuration as soon as
+# their respective prerequisites are reachable. Designed to run in the
+# background during the platform HR reconcile wait, so its wall-clock cost
+# overlaps with the wait instead of compounding it.
+set -eu
+
+echo "[post-install-prep] waiting for linstor HelmRelease object to exist"
+timeout 60 sh -ec 'until kubectl get hr/linstor -n cozy-linstor >/dev/null 2>&1; do sleep 2; done'
+
+echo "[post-install-prep] waiting for linstor HelmRelease to be Ready"
+kubectl wait helmrelease/linstor -n cozy-linstor --for=condition=Ready --timeout=15m
+
+echo "[post-install-prep] waiting for linstor-controller deployment to exist"
+timeout 300 sh -ec 'until kubectl get deploy/linstor-controller -n cozy-linstor >/dev/null 2>&1; do sleep 2; done'
+
+echo "[post-install-prep] waiting for linstor-controller to be Available"
+kubectl wait deployment/linstor-controller -n cozy-linstor --timeout=5m --for=condition=available
+
+echo "[post-install-prep] waiting for 3 LINSTOR nodes Online"
+# TODO(e2e-replace-fixed-timeouts): genuine poll. LINSTOR node membership is
+# reported by the linstor binary inside the controller pod, not via a
+# Kubernetes API condition, so kubectl wait cannot subscribe to it.
+timeout 60 sh -ec 'until [ $(kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor node list | grep -c Online) -eq 3 ]; do sleep 1; done'
+
+echo "[post-install-prep] creating LINSTOR storage pools (parallel across nodes)"
+created_pools=$(kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor sp l -s data --pastable | awk '$2 == "data" {printf " " $4} END{printf " "}')
+for node in srv1 srv2 srv3; do
+  case $created_pools in
+    *" $node "*) echo "  pool 'data' already exists on $node"; continue;;
+  esac
+  kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor ps cdp zfs ${node} /dev/vdc --pool-name data --storage-pool data &
+done
+wait
+
+echo "[post-install-prep] applying StorageClasses"
+kubectl apply -f - <<'EOF'
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: local
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "true"
+provisioner: linstor.csi.linbit.com
+parameters:
+  linstor.csi.linbit.com/storagePool: "data"
+  linstor.csi.linbit.com/layerList: "storage"
+  linstor.csi.linbit.com/allowRemoteVolumeAccess: "false"
+volumeBindingMode: WaitForFirstConsumer
+allowVolumeExpansion: true
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: replicated
+provisioner: linstor.csi.linbit.com
+parameters:
+  linstor.csi.linbit.com/storagePool: "data"
+  linstor.csi.linbit.com/autoPlace: "3"
+  linstor.csi.linbit.com/layerList: "drbd storage"
+  linstor.csi.linbit.com/allowRemoteVolumeAccess: "true"
+  property.linstor.csi.linbit.com/DrbdOptions/auto-quorum: suspend-io
+  property.linstor.csi.linbit.com/DrbdOptions/Resource/on-no-data-accessible: suspend-io
+  property.linstor.csi.linbit.com/DrbdOptions/Resource/on-suspended-primary-outdated: force-secondary
+  property.linstor.csi.linbit.com/DrbdOptions/Net/rr-conflict: retry-connect
+volumeBindingMode: Immediate
+allowVolumeExpansion: true
+EOF
+
+echo "[post-install-prep] waiting for MetalLB CRDs"
+timeout 300 sh -ec 'until kubectl get crd ipaddresspools.metallb.io l2advertisements.metallb.io >/dev/null 2>&1; do sleep 2; done'
+
+echo "[post-install-prep] applying MetalLB IPAddressPool"
+kubectl apply -f - <<'EOF'
+---
+apiVersion: metallb.io/v1beta1
+kind: L2Advertisement
+metadata:
+  name: cozystack
+  namespace: cozy-metallb
+spec:
+  ipAddressPools: [cozystack]
+---
+apiVersion: metallb.io/v1beta1
+kind: IPAddressPool
+metadata:
+  name: cozystack
+  namespace: cozy-metallb
+spec:
+  addresses: [192.168.123.200-192.168.123.250]
+  autoAssign: true
+  avoidBuggyIPs: false
+EOF
+
+echo "[post-install-prep] done"

hack/e2e-prepull-images.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+# Pre-pull images to all cluster nodes.
+#
+# Reads image references from stdin, one per line. Empty lines and lines
+# starting with '#' are ignored.
+#
+# Some workloads (OVN raft, LINSTOR) are sensitive to peer-readiness at
+# startup: if nodes pull the image at different speeds, one replica boots
+# before its peers are reachable, exhausts its raft connection retries, and
+# the HelmRelease install times out. Pre-pulling eliminates the pull stagger
+# so all replicas start within milliseconds of each other.
+#
+# Implementation: a DaemonSet with one regular container per image (parallel
+# pulls — total time = max of any one image rather than sum). kubectl rollout
+# status blocks until all pods are Ready (= all containers running = all
+# images cached on every node), then we delete the DaemonSet.
+
+set -euo pipefail
+
+mapfile -t images < <(grep -Ev '^[[:space:]]*(#|$)' | sort -u)
+
+if [[ ${#images[@]} -eq 0 ]]; then
+  echo "e2e-prepull-images: no images on stdin, nothing to do" >&2
+  exit 0
+fi
+
+echo "Pre-pulling ${#images[@]} image(s):"
+printf '  %s\n' "${images[@]}"
+
+containers=""
+i=0
+for image in "${images[@]}"; do
+  containers+="
+      - name: pull-${i}
+        image: ${image}
+        command: [\"sleep\", \"infinity\"]
+        imagePullPolicy: IfNotPresent
+        resources:
+          requests:
+            cpu: 1m
+            memory: 1Mi"
+  i=$((i + 1))
+done
+
+kubectl apply -f - <<EOF
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: e2e-image-prepuller
+  namespace: kube-system
+  labels:
+    app: e2e-image-prepuller
+spec:
+  selector:
+    matchLabels:
+      app: e2e-image-prepuller
+  template:
+    metadata:
+      labels:
+        app: e2e-image-prepuller
+    spec:
+      # hostNetwork bypasses the CNI: this script runs BEFORE Cozystack
+      # installs kube-ovn, so the cluster has no working CNI yet and a normal
+      # pod would stay ContainerCreating with NetworkPluginNotReady, never
+      # reaching image-pull. With hostNetwork the pod sandbox is created on
+      # the host's namespace and the kubelet pulls images right away.
+      # Same pattern the cozystack-operator pod uses for the same reason.
+      hostNetwork: true
+      # No need for an SA token — this pod just sleeps while images pull.
+      automountServiceAccountToken: false
+      tolerations:
+      # Reach every node including control-plane and nodes still coming up.
+      - operator: Exists
+      containers:${containers}
+EOF
+
+echo "Waiting for e2e-image-prepuller DaemonSet to become ready..."
+kubectl rollout status daemonset/e2e-image-prepuller -n kube-system --timeout=10m
+
+kubectl delete daemonset e2e-image-prepuller -n kube-system --ignore-not-found
+echo "Image pre-pull complete."

hack/e2e-test-openapi.bats

@@ -14,8 +14,13 @@
 
 @test "Test OpenAPI v2 endpoint (protobuf)" {
   (
-    kubectl proxy --port=21234 & sleep 0.5
-    trap "kill $!" EXIT
+    kubectl proxy --port=21234 &
+    proxy_pid=$!
+    trap "kill $proxy_pid" EXIT
+    # Wait for the proxy to actually be listening rather than guessing with
+    # a fixed sleep. nc -z is non-destructive and exits 0 the moment the
+    # listener accepts a connection.
+    timeout 10 sh -ec 'until nc -z localhost 21234; do sleep 0.1; done'
     curl -sS --fail 'http://localhost:21234/openapi/v2?timeout=32s' -H 'Accept: application/com.github.proto-openapi.spec.v2@v1.0+protobuf' > /dev/null
   )
 }

internal/operator/package_reconciler.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"strings"
+	"time"
 
 	cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
 	helmv2 "github.com/fluxcd/helm-controller/api/v2"
@@ -58,7 +59,12 @@ func parseCRDPolicy(install *cozyv1alpha1.ComponentInstall) helmv2.CRDsPolicy {
 // PackageReconciler reconciles Package resources
 type PackageReconciler struct {
 	client.Client
-	Scheme *runtime.Scheme
+	Scheme                    *runtime.Scheme
+	HelmReleaseInterval       time.Duration
+	HelmReleaseRetryInterval  time.Duration
+	HelmReleaseInstallTimeout time.Duration
+	HelmReleaseUpgradeTimeout time.Duration
+	HelmReleaseMaxHistory     int
 }
 
 // +kubebuilder:rbac:groups=cozystack.io,resources=packages,verbs=get;list;watch;create;update;patch;delete
@@ -214,22 +220,30 @@ func (r *PackageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 				Labels:    labels,
 			},
 			Spec: helmv2.HelmReleaseSpec{
-				Interval: metav1.Duration{Duration: 5 * 60 * 1000000000}, // 5m
+				Interval:   metav1.Duration{Duration: r.HelmReleaseInterval},
+				MaxHistory: &r.HelmReleaseMaxHistory,
 				ChartRef: &helmv2.CrossNamespaceSourceReference{
 					Kind:      "ExternalArtifact",
 					Name:      artifactName,
 					Namespace: "cozy-system",
 				},
 				Install: &helmv2.Install{
-					Timeout: &metav1.Duration{Duration: 10 * 60 * 1000000000}, // 10m
-					Remediation: &helmv2.InstallRemediation{
-						Retries: -1,
+					Timeout: &metav1.Duration{Duration: r.HelmReleaseInstallTimeout},
+					// Strategy=RetryOnFailure (with RetryInterval) replaces the previous
+					// Remediation{Retries:-1} setup. Functionally equivalent ("retry forever
+					// on failure"), but decouples retry timing from spec.Interval so failed
+					// installs recover at HelmReleaseRetryInterval (default 30s) without
+					// also polling healthy releases at the same fast cadence.
+					Strategy: &helmv2.InstallStrategy{
+						Name:          string(helmv2.ActionStrategyRetryOnFailure),
+						RetryInterval: &metav1.Duration{Duration: r.HelmReleaseRetryInterval},
 					},
 				},
 				Upgrade: &helmv2.Upgrade{
-					Timeout: &metav1.Duration{Duration: 10 * 60 * 1000000000}, // 10m
-					Remediation: &helmv2.UpgradeRemediation{
-						Retries: -1,
+					Timeout: &metav1.Duration{Duration: r.HelmReleaseUpgradeTimeout},
+					Strategy: &helmv2.UpgradeStrategy{
+						Name:          string(helmv2.ActionStrategyRetryOnFailure),
+						RetryInterval: &metav1.Duration{Duration: r.HelmReleaseRetryInterval},
 					},
 					CRDs: parseCRDPolicy(component.Install),
 				},

packages/apps/harbor/templates/harbor.yaml

@@ -65,9 +65,15 @@ spec:
     name: {{ .Release.Name }}-credentials
     valuesKey: redis-password
     targetPath: harbor.redis.external.password
+  # BucketInfo is the JSON blob populated by the COSI BucketAccess controller.
+  # Merge it at the values root (no targetPath) so Flux unmarshals it as YAML
+  # instead of running it through Helm's strvals parser, which would split the
+  # JSON on commas. This also gates reconciliation on the Secret being ready and
+  # forces a config-digest change when its contents change.
+  - kind: Secret
+    name: {{ .Release.Name }}-registry-bucket
+    valuesKey: BucketInfo
   values:
-    bucket:
-      secretName: {{ .Release.Name }}-registry-bucket
     db:
       replicas: {{ .Values.database.replicas }}
       size: {{ .Values.database.size }}

packages/apps/kubernetes/README.md

@@ -128,9 +128,6 @@ See the reference for components utilized in this service:
 | `addons.gpuOperator`                          | NVIDIA GPU Operator.                                                        | `object`   | `{}`      |
 | `addons.gpuOperator.enabled`                  | Enable GPU Operator.                                                        | `bool`     | `false`   |
 | `addons.gpuOperator.valuesOverride`           | Custom Helm values overrides.                                               | `object`   | `{}`      |
-| `addons.hami`                                 | HAMi GPU virtualization middleware.                                         | `object`   | `{}`      |
-| `addons.hami.enabled`                         | Enable HAMi (requires GPU Operator).                                        | `bool`     | `false`   |
-| `addons.hami.valuesOverride`                  | Custom Helm values overrides.                                               | `object`   | `{}`      |
 | `addons.fluxcd`                               | FluxCD GitOps operator.                                                     | `object`   | `{}`      |
 | `addons.fluxcd.enabled`                       | Enable FluxCD.                                                              | `bool`     | `false`   |
 | `addons.fluxcd.valuesOverride`                | Custom Helm values overrides.                                               | `object`   | `{}`      |

packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml

@@ -1,11 +1,3 @@
-{{- define "cozystack.defaultGpuOperatorValues" -}}
-{{- if .Values.addons.hami.enabled }}
-gpu-operator:
-  devicePlugin:
-    enabled: false
-{{- end }}
-{{- end }}
-
 {{- if and .Values.addons.gpuOperator.enabled .Values._namespace.etcd }}
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
@@ -37,12 +29,9 @@ spec:
     force: true
     remediation:
       retries: -1
-  {{- $defaults := fromYaml (include "cozystack.defaultGpuOperatorValues" .) }}
-  {{- $overrides := deepCopy (default (dict) .Values.addons.gpuOperator.valuesOverride) }}
-  {{- $merged := mergeOverwrite (default (dict) $defaults) $overrides }}
-  {{- if $merged }}
+  {{- with .Values.addons.gpuOperator.valuesOverride }}
   values:
-    {{- toYaml $merged | nindent 4 }}
+    {{- toYaml . | nindent 4 }}
   {{- end }}
 
   dependsOn:

packages/apps/kubernetes/templates/helmreleases/hami.yaml

@@ -1,49 +0,0 @@
-{{- if and .Values.addons.hami.enabled .Values._namespace.etcd }}
-{{- if not .Values.addons.gpuOperator.enabled }}
-{{- fail "addons.hami requires addons.gpuOperator to be enabled" }}
-{{- end }}
-apiVersion: helm.toolkit.fluxcd.io/v2
-kind: HelmRelease
-metadata:
-  name: {{ .Release.Name }}-hami
-  labels:
-    cozystack.io/repository: system
-    cozystack.io/target-cluster-name: {{ .Release.Name }}
-    sharding.fluxcd.io/key: tenants
-spec:
-  releaseName: hami
-  chartRef:
-    kind: ExternalArtifact
-    name: cozystack-kubernetes-application-kubevirt-kubernetes-hami
-    namespace: cozy-system
-  kubeConfig:
-    secretRef:
-      name: {{ .Release.Name }}-admin-kubeconfig
-      key: super-admin.svc
-  targetNamespace: cozy-hami
-  storageNamespace: cozy-hami
-  interval: 5m
-  timeout: 10m
-  install:
-    createNamespace: true
-    remediation:
-      retries: -1
-  upgrade:
-    force: true
-    remediation:
-      retries: -1
-  {{- with .Values.addons.hami.valuesOverride }}
-  values:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-
-  dependsOn:
-  {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }}
-  - name: {{ .Release.Name }}
-    namespace: {{ .Release.Namespace }}
-  {{- end }}
-  - name: {{ .Release.Name }}-cilium
-    namespace: {{ .Release.Namespace }}
-  - name: {{ .Release.Name }}-gpu-operator
-    namespace: {{ .Release.Namespace }}
-{{- end }}

packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml

@@ -1,99 +0,0 @@
-suite: GPU Operator HelmRelease HAMi integration tests
-templates:
-  - templates/helmreleases/gpu-operator.yaml
-values:
-  - values-ci.yaml
-tests:
-  - it: should disable devicePlugin when hami is enabled
-    set:
-      addons:
-        gpuOperator:
-          enabled: true
-          valuesOverride: {}
-        hami:
-          enabled: true
-          valuesOverride: {}
-    asserts:
-      - equal:
-          path: spec.values.gpu-operator.devicePlugin.enabled
-          value: false
-
-  - it: should not have values when hami is disabled and no overrides
-    set:
-      addons:
-        gpuOperator:
-          enabled: true
-          valuesOverride: {}
-        hami:
-          enabled: false
-          valuesOverride: {}
-    asserts:
-      - notExists:
-          path: spec.values
-
-  - it: should apply hami defaults when valuesOverride key is omitted
-    set:
-      addons:
-        gpuOperator:
-          enabled: true
-        hami:
-          enabled: true
-    asserts:
-      - equal:
-          path: spec.values.gpu-operator.devicePlugin.enabled
-          value: false
-
-  - it: should allow user overrides to merge with hami defaults
-    set:
-      addons:
-        gpuOperator:
-          enabled: true
-          valuesOverride:
-            gpu-operator:
-              driver:
-                enabled: false
-        hami:
-          enabled: true
-          valuesOverride: {}
-    asserts:
-      - equal:
-          path: spec.values.gpu-operator.devicePlugin.enabled
-          value: false
-      - equal:
-          path: spec.values.gpu-operator.driver.enabled
-          value: false
-
-  - it: should let user explicitly override devicePlugin.enabled to true with hami enabled
-    set:
-      addons:
-        gpuOperator:
-          enabled: true
-          valuesOverride:
-            gpu-operator:
-              devicePlugin:
-                enabled: true
-              driver:
-                enabled: false
-        hami:
-          enabled: true
-          valuesOverride: {}
-    asserts:
-      - equal:
-          path: spec.values.gpu-operator.devicePlugin.enabled
-          value: true
-      - equal:
-          path: spec.values.gpu-operator.driver.enabled
-          value: false
-
-  - it: should not render when gpuOperator is disabled
-    set:
-      addons:
-        gpuOperator:
-          enabled: false
-          valuesOverride: {}
-        hami:
-          enabled: false
-          valuesOverride: {}
-    asserts:
-      - hasDocuments:
-          count: 0

packages/apps/kubernetes/tests/hami_test.yaml

@@ -1,153 +0,0 @@
-suite: HAMi HelmRelease tests
-templates:
-  - templates/helmreleases/hami.yaml
-values:
-  - values-ci.yaml
-tests:
-  - it: should not render when hami is disabled
-    set:
-      addons:
-        hami:
-          enabled: false
-          valuesOverride: {}
-        gpuOperator:
-          enabled: true
-          valuesOverride: {}
-    asserts:
-      - hasDocuments:
-          count: 0
-
-  - it: should render HelmRelease when hami is enabled
-    set:
-      addons:
-        hami:
-          enabled: true
-          valuesOverride: {}
-        gpuOperator:
-          enabled: true
-          valuesOverride: {}
-    asserts:
-      - hasDocuments:
-          count: 1
-      - isKind:
-          of: HelmRelease
-
-  - it: should fail when gpuOperator is not enabled
-    set:
-      addons:
-        hami:
-          enabled: true
-          valuesOverride: {}
-        gpuOperator:
-          enabled: false
-          valuesOverride: {}
-    asserts:
-      - failedTemplate:
-          errorMessage: "addons.hami requires addons.gpuOperator to be enabled"
-
-  - it: should have correct metadata labels
-    set:
-      addons:
-        hami:
-          enabled: true
-          valuesOverride: {}
-        gpuOperator:
-          enabled: true
-          valuesOverride: {}
-    asserts:
-      - equal:
-          path: metadata.labels["cozystack.io/repository"]
-          value: system
-      - equal:
-          path: metadata.labels["sharding.fluxcd.io/key"]
-          value: tenants
-
-  - it: should use ExternalArtifact chartRef
-    set:
-      addons:
-        hami:
-          enabled: true
-          valuesOverride: {}
-        gpuOperator:
-          enabled: true
-          valuesOverride: {}
-    asserts:
-      - equal:
-          path: spec.chartRef.kind
-          value: ExternalArtifact
-      - equal:
-          path: spec.chartRef.namespace
-          value: cozy-system
-
-  - it: should target cozy-hami namespace
-    set:
-      addons:
-        hami:
-          enabled: true
-          valuesOverride: {}
-        gpuOperator:
-          enabled: true
-          valuesOverride: {}
-    asserts:
-      - equal:
-          path: spec.targetNamespace
-          value: cozy-hami
-      - equal:
-          path: spec.storageNamespace
-          value: cozy-hami
-
-  - it: should depend on gpu-operator and cilium
-    release:
-      name: test
-      namespace: test-ns
-    set:
-      addons:
-        hami:
-          enabled: true
-          valuesOverride: {}
-        gpuOperator:
-          enabled: true
-          valuesOverride: {}
-    asserts:
-      - contains:
-          path: spec.dependsOn
-          content:
-            name: test-cilium
-            namespace: test-ns
-      - contains:
-          path: spec.dependsOn
-          content:
-            name: test-gpu-operator
-            namespace: test-ns
-
-  - it: should not render spec.values when valuesOverride is empty
-    set:
-      addons:
-        hami:
-          enabled: true
-          valuesOverride: {}
-        gpuOperator:
-          enabled: true
-          valuesOverride: {}
-    asserts:
-      - hasDocuments:
-          count: 1
-      - notExists:
-          path: spec.values
-
-  - it: should pass through valuesOverride
-    set:
-      addons:
-        hami:
-          enabled: true
-          valuesOverride:
-            hami:
-              devicePlugin:
-                deviceSplitCount: 5
-        gpuOperator:
-          enabled: true
-          valuesOverride: {}
-    asserts:
-      - equal:
-          path: spec.values.hami.devicePlugin.deviceSplitCount
-          value: 5

packages/apps/kubernetes/values.schema.json

@@ -149,7 +149,6 @@
         "fluxcd",
         "gatewayAPI",
         "gpuOperator",
-        "hami",
         "ingressNginx",
         "monitoringAgents",
         "velero",
@@ -269,28 +268,6 @@
             }
           }
         },
-        "hami": {
-          "description": "HAMi GPU virtualization middleware.",
-          "type": "object",
-          "default": {},
-          "required": [
-            "enabled",
-            "valuesOverride"
-          ],
-          "properties": {
-            "enabled": {
-              "description": "Enable HAMi (requires GPU Operator).",
-              "type": "boolean",
-              "default": false
-            },
-            "valuesOverride": {
-              "description": "Custom Helm values overrides.",
-              "type": "object",
-              "default": {},
-              "x-kubernetes-preserve-unknown-fields": true
-            }
-          }
-        },
         "ingressNginx": {
           "description": "Ingress-NGINX controller.",
           "type": "object",

packages/apps/kubernetes/values.yaml

@@ -94,10 +94,6 @@ host: ""
 ## @field {bool} enabled - Enable FluxCD.
 ## @field {object} valuesOverride - Custom Helm values overrides.
 
-## @typedef {struct} HAMiAddon - HAMi GPU virtualization middleware.
-## @field {bool} enabled - Enable HAMi (requires GPU Operator).
-## @field {object} valuesOverride - Custom Helm values overrides.
-
 ## @typedef {struct} MonitoringAgentsAddon - Monitoring agents (Fluent Bit, VMAgents).
 ## @field {bool} enabled - Enable monitoring agents.
 ## @field {object} valuesOverride - Custom Helm values overrides.
@@ -118,7 +114,6 @@ host: ""
 ## @field {GatewayAPIAddon} gatewayAPI - Gateway API addon.
 ## @field {IngressNginxAddon} ingressNginx - Ingress-NGINX controller.
 ## @field {GPUOperatorAddon} gpuOperator - NVIDIA GPU Operator.
-## @field {HAMiAddon} hami - HAMi GPU virtualization middleware.
 ## @field {FluxCDAddon} fluxcd - FluxCD GitOps operator.
 ## @field {MonitoringAgentsAddon} monitoringAgents - Monitoring agents.
 ## @field {VerticalPodAutoscalerAddon} verticalPodAutoscaler - Vertical Pod Autoscaler.
@@ -142,9 +137,6 @@ addons:
   gpuOperator:
     enabled: false
     valuesOverride: {}
-  hami:
-    enabled: false
-    valuesOverride: {}
   fluxcd:
     enabled: false
     valuesOverride: {}

packages/apps/postgres/README.md

@@ -133,13 +133,12 @@ See:
 
 ### Bootstrap (recovery) parameters
 
-| Name                     | Description                                                                                                                                                                                                                  | Type     | Value   |
-| ------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------- |
-| `bootstrap`              | Bootstrap configuration.                                                                                                                                                                                                     | `object` | `{}`    |
-| `bootstrap.enabled`      | Whether to restore from a backup.                                                                                                                                                                                            | `bool`   | `false` |
-| `bootstrap.recoveryTime` | Timestamp (RFC3339) for point-in-time recovery; empty means latest.                                                                                                                                                          | `string` | `""`    |
-| `bootstrap.oldName`      | Previous cluster name before deletion.                                                                                                                                                                                       | `string` | `""`    |
-| `bootstrap.serverName`   | Barman server name (S3 path prefix) used by the original cluster when writing backups. Set this only when the original cluster had an explicit barmanObjectStore.serverName that differed from its Kubernetes resource name. | `string` | `""`    |
+| Name                     | Description                                                         | Type     | Value   |
+| ------------------------ | ------------------------------------------------------------------- | -------- | ------- |
+| `bootstrap`              | Bootstrap configuration.                                            | `object` | `{}`    |
+| `bootstrap.enabled`      | Whether to restore from a backup.                                   | `bool`   | `false` |
+| `bootstrap.recoveryTime` | Timestamp (RFC3339) for point-in-time recovery; empty means latest. | `string` | `""`    |
+| `bootstrap.oldName`      | Previous cluster name before deletion.                              | `string` | `""`    |
 
 
 ## Parameter examples and reference

packages/apps/postgres/templates/db.yaml

@@ -32,9 +32,6 @@ spec:
     - name: {{ .Values.bootstrap.oldName }}
       barmanObjectStore:
         destinationPath: {{ .Values.backup.destinationPath }}
-        {{- if .Values.bootstrap.serverName }}
-        serverName: {{ .Values.bootstrap.serverName }}
-        {{- end }}
         endpointURL: {{ .Values.backup.endpointURL }}
         s3Credentials:
           accessKeyId:

packages/apps/postgres/values.schema.json

@@ -254,11 +254,6 @@
           "description": "Timestamp (RFC3339) for point-in-time recovery; empty means latest.",
           "type": "string",
           "default": ""
-        },
-        "serverName": {
-          "description": "Barman server name (S3 path prefix) used by the original cluster when writing backups. Set this only when the original cluster had an explicit barmanObjectStore.serverName that differed from its Kubernetes resource name.",
-          "type": "string",
-          "default": ""
         }
       }
     }

packages/apps/postgres/values.yaml

@@ -154,7 +154,6 @@ backup:
 ## @field {bool} enabled - Whether to restore from a backup.
 ## @field {string} [recoveryTime] - Timestamp (RFC3339) for point-in-time recovery; empty means latest.
 ## @field {string} oldName - Previous cluster name before deletion.
-## @field {string} [serverName] - Barman server name (S3 path prefix) used by the original cluster when writing backups. Set this only when the original cluster had an explicit barmanObjectStore.serverName that differed from its Kubernetes resource name.
 
 ## @param {Bootstrap} bootstrap - Bootstrap configuration.
 bootstrap:
@@ -162,4 +161,3 @@ bootstrap:
   # example: 2020-11-26 15:22:00.00000+00
   recoveryTime: ""
   oldName: ""
-  serverName: ""

packages/apps/tenant/Makefile

@@ -3,6 +3,3 @@ include ../../../hack/package.mk
 generate:
 	cozyvalues-gen -m 'tenant' -v values.yaml -s values.schema.json -r README.md -g ../../../api/apps/v1alpha1/tenant/types.go
 	../../../hack/update-crd.sh
-
-test:
-	helm unittest .

packages/apps/tenant/templates/cilium-lb-pool.yaml

@@ -1,28 +0,0 @@
-{{- $exposeMode := (index .Values._cluster "expose-mode") | default "externalIPs" }}
-{{- $exposeIngress := (index .Values._cluster "expose-ingress") | default "tenant-root" }}
-{{- $exposeIPs := (index .Values._cluster "expose-external-ips") | default "" | nospace }}
-{{- $ipsList := list }}
-{{- range splitList "," $exposeIPs }}
-  {{- $ip := . | trim }}
-  {{- if $ip }}
-    {{- $ipsList = append $ipsList $ip }}
-  {{- end }}
-{{- end }}
-{{- $isPublishingIngressLB := and
-      (eq $exposeMode "loadBalancer")
-      (eq $exposeIngress .Release.Namespace)
-      .Values.ingress }}
-{{- if and $isPublishingIngressLB $ipsList }}
-apiVersion: cilium.io/v2
-kind: CiliumLoadBalancerIPPool
-metadata:
-  name: {{ trimPrefix "tenant-" .Release.Namespace }}-exposure
-spec:
-  blocks:
-  {{- range $ipsList }}
-  - cidr: {{ . }}{{ if not (contains "/" .) }}/{{ if contains ":" . }}128{{ else }}32{{ end }}{{ end }}
-  {{- end }}
-  serviceSelector:
-    matchLabels:
-      "io.kubernetes.service.namespace": {{ .Release.Namespace | quote }}
-{{- end }}

packages/apps/tenant/tests/exposure_test.yaml

@@ -1,141 +0,0 @@
-suite: tenant CiliumLoadBalancerIPPool rendering for publishing.exposure=loadBalancer
-templates:
-  - templates/cilium-lb-pool.yaml
-
-release:
-  name: tenant-root
-  namespace: tenant-root
-
-tests:
-  - it: default exposure (externalIPs) renders no pool
-    set:
-      ingress: true
-      _cluster:
-        expose-ingress: tenant-root
-        expose-external-ips: "192.0.2.10,192.0.2.11"
-    asserts:
-      - hasDocuments:
-          count: 0
-
-  - it: loadBalancer mode in publishing tenant with ingress=true renders v2 pool with namespace-only selector
-    set:
-      ingress: true
-      _cluster:
-        expose-ingress: tenant-root
-        expose-external-ips: "192.0.2.10,192.0.2.11"
-        expose-mode: loadBalancer
-    asserts:
-      - hasDocuments:
-          count: 1
-      - equal:
-          path: apiVersion
-          value: cilium.io/v2
-      - equal:
-          path: kind
-          value: CiliumLoadBalancerIPPool
-      - equal:
-          path: metadata.name
-          value: root-exposure
-      - equal:
-          path: spec.blocks
-          value:
-            - cidr: 192.0.2.10/32
-            - cidr: 192.0.2.11/32
-      - equal:
-          path: spec.serviceSelector.matchLabels["io.kubernetes.service.namespace"]
-          value: tenant-root
-      - notExists:
-          path: spec.serviceSelector.matchLabels["app.kubernetes.io/name"]
-
-  - it: loadBalancer mode with IPv6 emits /128 CIDR
-    set:
-      ingress: true
-      _cluster:
-        expose-ingress: tenant-root
-        expose-external-ips: "2001:db8::1"
-        expose-mode: loadBalancer
-    asserts:
-      - equal:
-          path: spec.blocks
-          value:
-            - cidr: 2001:db8::1/128
-
-  - it: loadBalancer mode with mixed IPv4 and IPv6 emits correct CIDR per family
-    set:
-      ingress: true
-      _cluster:
-        expose-ingress: tenant-root
-        expose-external-ips: "192.0.2.10,2001:db8::1"
-        expose-mode: loadBalancer
-    asserts:
-      - equal:
-          path: spec.blocks
-          value:
-            - cidr: 192.0.2.10/32
-            - cidr: 2001:db8::1/128
-
-  - it: loadBalancer mode accepts pre-CIDR input without double-suffixing
-    set:
-      ingress: true
-      _cluster:
-        expose-ingress: tenant-root
-        expose-external-ips: "192.0.2.10/32,2001:db8::1/128"
-        expose-mode: loadBalancer
-    asserts:
-      - equal:
-          path: spec.blocks
-          value:
-            - cidr: 192.0.2.10/32
-            - cidr: 2001:db8::1/128
-
-  - it: loadBalancer mode filters out empty entries from externalIPs (trailing, leading, repeated commas)
-    set:
-      ingress: true
-      _cluster:
-        expose-ingress: tenant-root
-        expose-external-ips: "192.0.2.10,,192.0.2.11,"
-        expose-mode: loadBalancer
-    asserts:
-      - hasDocuments:
-          count: 1
-      - equal:
-          path: spec.blocks
-          value:
-            - cidr: 192.0.2.10/32
-            - cidr: 192.0.2.11/32
-
-  - it: loadBalancer mode with ingress=false in publishing tenant renders no pool
-    set:
-      ingress: false
-      _cluster:
-        expose-ingress: tenant-root
-        expose-external-ips: "192.0.2.10"
-        expose-mode: loadBalancer
-    asserts:
-      - hasDocuments:
-          count: 0
-
-  - it: loadBalancer mode in a non-publishing tenant renders no pool
-    set:
-      ingress: true
-      _cluster:
-        expose-ingress: tenant-root
-        expose-external-ips: "192.0.2.10"
-        expose-mode: loadBalancer
-    release:
-      name: tenant-u1
-      namespace: tenant-u1
-    asserts:
-      - hasDocuments:
-          count: 0
-
-  - it: loadBalancer mode in publishing tenant with empty externalIPs renders no pool
-    set:
-      ingress: true
-      _cluster:
-        expose-ingress: tenant-root
-        expose-external-ips: ""
-        expose-mode: loadBalancer
-    asserts:
-      - hasDocuments:
-          count: 0

packages/apps/vm-instance/README.md

@@ -36,32 +36,31 @@ virtctl ssh <user>@<vm>
 
 ### Common parameters
 
-| Name                | Description                                                                                                                                                                                                   | Type       | Value       |
-| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ----------- |
-| `external`          | Enable external access from outside the cluster.                                                                                                                                                              | `bool`     | `false`     |
-| `externalMethod`    | Method to pass through traffic to the VM.                                                                                                                                                                     | `string`   | `PortList`  |
-| `externalPorts`     | Ports to forward from outside the cluster.                                                                                                                                                                    | `[]int`    | `[22]`      |
-| `externalAllowICMP` | Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect. | `bool`     | `true`      |
-| `runStrategy`       | Requested running state of the VirtualMachineInstance                                                                                                                                                         | `string`   | `Always`    |
-| `instanceType`      | Virtual Machine instance type.                                                                                                                                                                                | `string`   | `u1.medium` |
-| `instanceProfile`   | Virtual Machine preferences profile.                                                                                                                                                                          | `string`   | `ubuntu`    |
-| `disks`             | List of disks to attach.                                                                                                                                                                                      | `[]object` | `[]`        |
-| `disks[i].name`     | Disk name.                                                                                                                                                                                                    | `string`   | `""`        |
-| `disks[i].bus`      | Disk bus type (e.g. "sata").                                                                                                                                                                                  | `string`   | `""`        |
-| `networks`          | Networks to attach the VM to.                                                                                                                                                                                 | `[]object` | `[]`        |
-| `networks[i].name`  | Network attachment name.                                                                                                                                                                                      | `string`   | `""`        |
-| `subnets`           | Deprecated: use networks instead.                                                                                                                                                                             | `[]object` | `[]`        |
-| `subnets[i].name`   | Network attachment name.                                                                                                                                                                                      | `string`   | `""`        |
-| `gpus`              | List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).                                                                                                                                           | `[]object` | `[]`        |
-| `gpus[i].name`      | The name of the GPU resource to attach.                                                                                                                                                                       | `string`   | `""`        |
-| `cpuModel`          | Model specifies the CPU model inside the VMI. List of available models https://github.com/libvirt/libvirt/tree/master/src/cpu_map                                                                             | `string`   | `""`        |
-| `resources`         | Resource configuration for the virtual machine.                                                                                                                                                               | `object`   | `{}`        |
-| `resources.cpu`     | Number of CPU cores allocated.                                                                                                                                                                                | `quantity` | `""`        |
-| `resources.memory`  | Amount of memory allocated.                                                                                                                                                                                   | `quantity` | `""`        |
-| `resources.sockets` | Number of CPU sockets (vCPU topology).                                                                                                                                                                        | `quantity` | `""`        |
-| `sshKeys`           | List of SSH public keys for authentication.                                                                                                                                                                   | `[]string` | `[]`        |
-| `cloudInit`         | Cloud-init user data.                                                                                                                                                                                         | `string`   | `""`        |
-| `cloudInitSeed`     | Seed string to generate SMBIOS UUID for the VM.                                                                                                                                                               | `string`   | `""`        |
+| Name                | Description                                                                                                                       | Type       | Value       |
+| ------------------- | --------------------------------------------------------------------------------------------------------------------------------- | ---------- | ----------- |
+| `external`          | Enable external access from outside the cluster.                                                                                  | `bool`     | `false`     |
+| `externalMethod`    | Method to pass through traffic to the VM.                                                                                         | `string`   | `PortList`  |
+| `externalPorts`     | Ports to forward from outside the cluster.                                                                                        | `[]int`    | `[22]`      |
+| `runStrategy`       | Requested running state of the VirtualMachineInstance                                                                             | `string`   | `Always`    |
+| `instanceType`      | Virtual Machine instance type.                                                                                                    | `string`   | `u1.medium` |
+| `instanceProfile`   | Virtual Machine preferences profile.                                                                                              | `string`   | `ubuntu`    |
+| `disks`             | List of disks to attach.                                                                                                          | `[]object` | `[]`        |
+| `disks[i].name`     | Disk name.                                                                                                                        | `string`   | `""`        |
+| `disks[i].bus`      | Disk bus type (e.g. "sata").                                                                                                      | `string`   | `""`        |
+| `networks`          | Networks to attach the VM to.                                                                                                     | `[]object` | `[]`        |
+| `networks[i].name`  | Network attachment name.                                                                                                          | `string`   | `""`        |
+| `subnets`           | Deprecated: use networks instead.                                                                                                 | `[]object` | `[]`        |
+| `subnets[i].name`   | Network attachment name.                                                                                                          | `string`   | `""`        |
+| `gpus`              | List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).                                                               | `[]object` | `[]`        |
+| `gpus[i].name`      | The name of the GPU resource to attach.                                                                                           | `string`   | `""`        |
+| `cpuModel`          | Model specifies the CPU model inside the VMI. List of available models https://github.com/libvirt/libvirt/tree/master/src/cpu_map | `string`   | `""`        |
+| `resources`         | Resource configuration for the virtual machine.                                                                                   | `object`   | `{}`        |
+| `resources.cpu`     | Number of CPU cores allocated.                                                                                                    | `quantity` | `""`        |
+| `resources.memory`  | Amount of memory allocated.                                                                                                       | `quantity` | `""`        |
+| `resources.sockets` | Number of CPU sockets (vCPU topology).                                                                                            | `quantity` | `""`        |
+| `sshKeys`           | List of SSH public keys for authentication.                                                                                       | `[]string` | `[]`        |
+| `cloudInit`         | Cloud-init user data.                                                                                                             | `string`   | `""`        |
+| `cloudInitSeed`     | Seed string to generate SMBIOS UUID for the VM.                                                                                   | `string`   | `""`        |
 
 
 ## U Series

packages/apps/vm-instance/templates/service.yaml

@@ -9,10 +9,7 @@ metadata:
 {{- if .Values.external }}
     service.kubernetes.io/service-proxy-name: "cozy-proxy"
   annotations:
-    networking.cozystack.io/wholeIP: {{ ternary "true" "false" (eq .Values.externalMethod "WholeIP") | quote }}
-    {{- if eq .Values.externalMethod "PortList" }}
-    networking.cozystack.io/allowICMP: {{ ternary "true" "false" (ne .Values.externalAllowICMP false) | quote }}
-    {{- end }}
+    networking.cozystack.io/wholeIP: "true"
 {{- end }}
 spec:
   type: {{ ternary "LoadBalancer" "ClusterIP" .Values.external }}

packages/apps/vm-instance/values.schema.json

@@ -26,11 +26,6 @@
         "type": "integer"
       }
     },
-    "externalAllowICMP": {
-      "description": "Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect.",
-      "type": "boolean",
-      "default": true
-    },
     "runStrategy": {
       "description": "Requested running state of the VirtualMachineInstance",
       "type": "string",

packages/apps/vm-instance/values.yaml

@@ -31,9 +31,6 @@ externalMethod: PortList
 externalPorts:
 - 22
 
-## @param {bool} externalAllowICMP - Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect.
-externalAllowICMP: true
-
 ## @enum {string} RunStrategy - Requested running state of the VirtualMachineInstance
 ## @value Always - VMI should always be running
 ## @value Halted - VMI should never be running

packages/core/installer/templates/cozystack-operator.yaml

@@ -4,16 +4,6 @@
 {{- end -}}
 ---
 apiVersion: v1
-kind: Namespace
-metadata:
-  name: cozy-system
-  labels:
-    cozystack.io/system: "true"
-    pod-security.kubernetes.io/enforce: privileged
-  annotations:
-    helm.sh/resource-policy: keep
----
-apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: cozystack
@@ -68,6 +58,21 @@ spec:
         {{- if .Values.cozystackOperator.disableTelemetry }}
         - --disable-telemetry
         {{- end }}
+        {{- if .Values.cozystackOperator.helmReleaseInterval }}
+        - --helmrelease-interval={{ .Values.cozystackOperator.helmReleaseInterval }}
+        {{- end }}
+        {{- if .Values.cozystackOperator.helmReleaseRetryInterval }}
+        - --helmrelease-retry-interval={{ .Values.cozystackOperator.helmReleaseRetryInterval }}
+        {{- end }}
+        {{- if .Values.cozystackOperator.helmReleaseInstallTimeout }}
+        - --helmrelease-install-timeout={{ .Values.cozystackOperator.helmReleaseInstallTimeout }}
+        {{- end }}
+        {{- if .Values.cozystackOperator.helmReleaseUpgradeTimeout }}
+        - --helmrelease-upgrade-timeout={{ .Values.cozystackOperator.helmReleaseUpgradeTimeout }}
+        {{- end }}
+        {{- if .Values.cozystackOperator.helmReleaseMaxHistory }}
+        - --helmrelease-max-history={{ .Values.cozystackOperator.helmReleaseMaxHistory }}
+        {{- end }}
         - --platform-source-name=cozystack-platform
         - --platform-source-url={{ .Values.cozystackOperator.platformSourceUrl }}
         {{- if .Values.cozystackOperator.platformSourceRef }}

packages/core/installer/values.yaml

@@ -4,6 +4,25 @@ cozystackOperator:
   image: ghcr.io/cozystack/cozystack/cozystack-operator:v1.3.0@sha256:62574f12486bb40c901cf5ed484cca264405ce5810196d86555cbb27cce1ba48
   platformSourceUrl: 'oci://ghcr.io/cozystack/cozystack/cozystack-packages'
   platformSourceRef: 'digest=sha256:a0b9ef938446b3132d3d22ad2262beb1027c48c9037b6c2346fdc2f19acd3036'
+  # When non-empty, overrides the operator's --helmrelease-interval flag
+  # (operator default: 5m). E2E sets this to 30s; production should leave empty.
+  helmReleaseInterval: ""
+  # When non-empty, overrides the operator's --helmrelease-retry-interval flag
+  # (operator default: 30s). Controls how fast failed install/upgrade attempts
+  # are retried. Decoupled from helmReleaseInterval (healthy reconcile cadence).
+  helmReleaseRetryInterval: ""
+  # When non-empty, overrides the operator's --helmrelease-install-timeout flag
+  # (operator default: 10m). Bounds individual k8s operations (job/hook/wait)
+  # during Helm install. Charts with long-running install hooks may need longer.
+  helmReleaseInstallTimeout: ""
+  # When non-empty, overrides the operator's --helmrelease-upgrade-timeout flag
+  # (operator default: 10m). Same semantics as helmReleaseInstallTimeout, for
+  # the Helm upgrade action.
+  helmReleaseUpgradeTimeout: ""
+  # When non-empty, overrides the operator's --helmrelease-max-history flag
+  # (operator default: 5). Number of release revisions Helm keeps per HR.
+  # Use 0 for unlimited; lower values reduce Secret accumulation in test clusters.
+  helmReleaseMaxHistory: ""
 # Generic variant configuration (only used when cozystackOperator.variant=generic)
 cozystack:
   # Kubernetes API server host (IP only, no protocol/port)

packages/core/platform/sources/hami.yaml

@@ -1,24 +0,0 @@
----
-apiVersion: cozystack.io/v1alpha1
-kind: PackageSource
-metadata:
-  name: cozystack.hami
-spec:
-  sourceRef:
-    kind: OCIRepository
-    name: cozystack-packages
-    namespace: cozy-system
-    path: /
-  variants:
-  - name: default
-    dependsOn:
-    - cozystack.gpu-operator
-    components:
-    - name: hami
-      path: system/hami
-      valuesFiles:
-      - values.yaml
-      install:
-        privileged: true
-        namespace: cozy-hami
-        releaseName: hami

packages/core/platform/sources/kubernetes-application.yaml

@@ -52,8 +52,6 @@ spec:
           path: system/cilium
         - name: kubernetes-gpu-operator
           path: system/gpu-operator
-        - name: kubernetes-hami
-          path: system/hami
         - name: kubernetes-vertical-pod-autoscaler
           path: system/vertical-pod-autoscaler
         - name: kubernetes-prometheus-operator-crds

packages/core/platform/templates/apps.yaml

@@ -30,7 +30,6 @@ stringData:
       expose-services: {{ .Values.publishing.exposedServices | join "," | quote }}
       expose-ingress: {{ .Values.publishing.ingressName | quote }}
       expose-external-ips: {{ .Values.publishing.externalIPs | join "," | quote }}
-      expose-mode: {{ .Values.publishing.exposure | default "externalIPs" | quote }}
       cluster-domain: {{ .Values.networking.clusterDomain | quote }}
       api-server-endpoint: {{ .Values.publishing.apiServerEndpoint | quote }}
       {{- with .Values.branding }}

packages/core/platform/templates/bundles/iaas.yaml

@@ -10,7 +10,6 @@
 {{include "cozystack.platform.package.default" (list "cozystack.kubevirt-cdi" $) }}
 {{include "cozystack.platform.package.optional.default" (list "cozystack.vm-default-images" $) }}
 {{include "cozystack.platform.package.optional.default" (list "cozystack.gpu-operator" $) }}
-{{include "cozystack.platform.package.optional.default" (list "cozystack.hami" $) }}
 {{include "cozystack.platform.package.default" (list "cozystack.kamaji" $) }}
 {{include "cozystack.platform.package.default" (list "cozystack.capi-operator" $) }}
 {{include "cozystack.platform.package.default" (list "cozystack.capi-provider-bootstrap-kubeadm" $) }}

packages/core/platform/templates/bundles/system.yaml

@@ -105,6 +105,9 @@
 {{- /* cozystack-api DaemonSet */ -}}
 {{- $apiValues := dict "cozystackAPI" (dict "nodeSelector" $genericNodeSelector) -}}
 {{- $_ := set $cozystackEngineComponents "cozystack-api" (dict "values" $apiValues) -}}
+{{- /* lineage-controller-webhook DaemonSet */ -}}
+{{- $lineageValues := dict "lineageControllerWebhook" (dict "nodeSelector" $genericNodeSelector) -}}
+{{- $_ := set $cozystackEngineComponents "lineage-controller-webhook" (dict "values" $lineageValues) -}}
 {{- end -}}
 {{- if .Values.authentication.oidc.enabled }}
 {{include "cozystack.platform.package" (list "cozystack.cozystack-engine" "oidc" $ $cozystackEngineComponents) }}

packages/core/platform/values.yaml

@@ -45,41 +45,6 @@ publishing:
     - cdi-uploadproxy
   apiServerEndpoint: "" # example: "https://api.example.org"
   externalIPs: []
-  # Exposure mode for the ingress-nginx Service. When "externalIPs" (current
-  # default) is selected, the Service is created as ClusterIP with
-  # Service.spec.externalIPs set from publishing.externalIPs. When
-  # "loadBalancer" is selected, the Service is type: LoadBalancer and a
-  # CiliumLoadBalancerIPPool makes those same addresses allocatable via LB IPAM.
-  #
-  # Service.spec.externalIPs is deprecated upstream in Kubernetes v1.36
-  # (KEP-5707). The AllowServiceExternalIPs feature gate is expected to default
-  # to false around v1.40 and the implementation removed around v1.43 — switch
-  # to "loadBalancer" before upgrading past v1.40.
-  #
-  # Caveats for the "loadBalancer" mode:
-  #   - publishing.externalIPs must contain at least one non-empty address,
-  #     otherwise the chart render fails with an explicit error (a LoadBalancer
-  #     Service without a pool would sit in <pending> forever).
-  #   - The ingress-nginx Service is created with externalTrafficPolicy: Local
-  #     to preserve the client source IP. Traffic arriving on a node that does
-  #     not host an ingress-nginx pod is dropped, so the external IP must be
-  #     routed to a node that runs the ingress pod (floating IP / keepalived /
-  #     upstream router / podAntiAffinity).
-  #   - Cilium does NOT announce the IP on its own unless L2 announcements or
-  #     BGP are enabled in the Cilium values (disabled by default in Cozystack).
-  #     This mode assumes the operator already routes the externalIPs to a
-  #     cluster node; enabling announcements is out of scope for this setting.
-  #   - Switching this value on a running cluster causes the ingress-nginx
-  #     Service to be recreated (the HelmRelease has upgrade.force: true and
-  #     the Service kind changes between ClusterIP and LoadBalancer). Expect a
-  #     brief interruption of ingress traffic during the flip.
-  #
-  # Scope: this setting only controls the ingress-nginx Service. Other
-  # cozystack components that currently write Service.spec.externalIPs directly
-  # (e.g. the vpn app at packages/apps/vpn/templates/service.yaml) are NOT
-  # migrated by flipping this value and must be addressed separately before
-  # the AllowServiceExternalIPs feature gate flips to off in ~v1.40.
-  exposure: externalIPs # "externalIPs" or "loadBalancer"
   certificates:
     solver: http01 # "http01" or "dns01"
     issuerName: letsencrypt-prod

packages/extra/etcd/Makefile

@@ -5,6 +5,3 @@ include ../../../hack/package.mk
 generate:
 	cozyvalues-gen -v values.yaml -s values.schema.json -r README.md
 	../../../hack/update-crd.sh
-
-test:
-	helm unittest .

packages/extra/etcd/templates/hook/job.yaml

@@ -0,0 +1,39 @@
+{{- $shouldUpdateCerts := true }}
+{{- $configMap := lookup "v1" "ConfigMap" .Release.Namespace "etcd-deployed-version" }}
+{{- if $configMap }}
+  {{- $deployedVersion := index $configMap "data" "version" }}
+  {{- if $deployedVersion | semverCompare ">= 2.6.1" }}
+    {{- $shouldUpdateCerts = false }}
+  {{- end }}
+{{- end }}
+
+{{- if $shouldUpdateCerts }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: etcd-hook
+  annotations:
+    helm.sh/hook: post-upgrade
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+spec:
+  template:
+    metadata:
+      labels:
+        policy.cozystack.io/allow-to-apiserver: "true"
+    spec:
+      serviceAccountName: etcd-hook
+      containers:
+        - name: kubectl
+          image: docker.io/alpine/k8s:1.33.4
+          command:
+          - sh
+          args:
+            - -exc
+            - |-
+              kubectl --namespace={{ .Release.Namespace }} delete secrets etcd-ca-tls etcd-peer-ca-tls 
+              sleep 10
+              kubectl --namespace={{ .Release.Namespace }} delete secrets etcd-client-tls etcd-peer-tls etcd-server-tls
+              kubectl --namespace={{ .Release.Namespace }} delete pods --selector=app.kubernetes.io/instance=etcd,app.kubernetes.io/managed-by=etcd-operator,app.kubernetes.io/name=etcd,cozystack.io/service=etcd
+      restartPolicy: Never
+{{- end }}

packages/extra/etcd/templates/hook/role.yaml

@@ -0,0 +1,26 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  annotations:
+    helm.sh/hook: post-upgrade
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+  name: etcd-hook
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch

packages/extra/etcd/templates/hook/rolebinding.yaml

@@ -0,0 +1,15 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: etcd-hook
+  annotations:
+    helm.sh/hook: post-upgrade
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: etcd-hook
+subjects:
+  - kind: ServiceAccount
+    name: etcd-hook
+    namespace: {{ .Release.Namespace | quote }}

packages/extra/etcd/templates/hook/serviceaccount.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: etcd-hook
+  annotations:
+    helm.sh/hook: post-upgrade
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded

packages/extra/etcd/templates/version.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: etcd-deployed-version
+data:
+  version: {{ .Chart.Version }}

packages/extra/etcd/tests/no-post-upgrade-hook_test.yaml

@@ -1,34 +0,0 @@
-suite: etcd chart does not ship a destructive post-upgrade cert-regeneration hook
-
-release:
-  name: etcd
-  namespace: tenant-root
-
-templates:
-  - templates/check-release-name.yaml
-  - templates/dashboard-resourcemap.yaml
-  - templates/datastore.yaml
-  - templates/etcd-defrag.yaml
-  - templates/hook/job.yaml
-  - templates/podscrape.yaml
-  - templates/prometheus-rules.yaml
-  - templates/version.yaml
-
-tests:
-  - it: renders no Job named etcd-hook
-    documentSelector:
-      path: metadata.name
-      value: etcd-hook
-      skipEmptyTemplates: true
-    asserts:
-      - hasDocuments:
-          count: 0
-
-  - it: renders no ConfigMap named etcd-deployed-version
-    documentSelector:
-      path: metadata.name
-      value: etcd-deployed-version
-      skipEmptyTemplates: true
-    asserts:
-      - hasDocuments:
-          count: 0

packages/extra/ingress/Makefile

@@ -10,6 +10,3 @@ get-cloudflare-ips:
 generate:
 	cozyvalues-gen -v values.yaml -s values.schema.json -r README.md
 	../../../hack/update-crd.sh
-
-test:
-	helm unittest .

packages/extra/ingress/README.md

@@ -14,17 +14,3 @@
 | `resources.memory` | Memory (RAM) available to each replica.                                                                                                 | `quantity` | `""`    |
 | `resourcesPreset`  | Default sizing preset used when `resources` is omitted.                                                                                 | `string`   | `micro` |
 
-
-## Exposure mode
-
-The ingress Service type is driven by the cluster-wide `publishing.exposure` value in the platform chart, not by any key in this package. Two modes exist:
-
-- `externalIPs` (default) has three rendered shapes:
-  - Release namespace matches `publishing.ingressName` AND `publishing.externalIPs` is non-empty → Service is `ClusterIP` with `Service.spec.externalIPs` set from that list and `externalTrafficPolicy: Cluster`.
-  - Release namespace matches `publishing.ingressName` but `publishing.externalIPs` is empty → Service falls back to `type: LoadBalancer` with `externalTrafficPolicy: Local`.
-  - Release namespace does not match `publishing.ingressName` (non-root tenants) → Service is `type: LoadBalancer` with `externalTrafficPolicy: Local`.
-  `Service.spec.externalIPs` is deprecated upstream in Kubernetes v1.36 (KEP-5707); plan migration before v1.40.
-- `loadBalancer` — Service is `type: LoadBalancer` with `externalTrafficPolicy: Local`, and a `CiliumLoadBalancerIPPool` makes the addresses in `publishing.externalIPs` allocatable via Cilium LB IPAM. Requires `publishing.externalIPs` to contain at least one non-empty address (render fails otherwise) and assumes the addresses are already routed to a cluster node (floating IP / upstream router). See the inline comment on `publishing.exposure` in the platform chart for full caveats, including the note that switching the value on a running cluster causes the ingress Service to be recreated.
-
-This setting only migrates ingress-nginx away from `Service.spec.externalIPs`. Other cozystack components that use the same deprecated field (e.g. the `vpn` app) must be migrated separately before Kubernetes v1.40 flips the `AllowServiceExternalIPs` feature gate off.
-

packages/extra/ingress/templates/nginx-ingress.yaml

@@ -1,19 +1,5 @@
 {{- $exposeIngress := (index .Values._cluster "expose-ingress") | default "tenant-root" }}
 {{- $exposeExternalIPs := (index .Values._cluster "expose-external-ips") | default "" | nospace }}
-{{- $exposeMode := (index .Values._cluster "expose-mode") | default "externalIPs" }}
-{{- $exposeIPsList := list }}
-{{- range splitList "," $exposeExternalIPs }}
-  {{- $ip := . | trim }}
-  {{- if $ip }}
-    {{- $exposeIPsList = append $exposeIPsList $ip }}
-  {{- end }}
-{{- end }}
-{{- if not (has $exposeMode (list "externalIPs" "loadBalancer")) }}
-{{- fail (printf "unknown publishing.exposure mode %q: must be \"externalIPs\" or \"loadBalancer\"" $exposeMode) }}
-{{- end }}
-{{- if and (eq $exposeMode "loadBalancer") (eq $exposeIngress .Release.Namespace) (not $exposeIPsList) }}
-{{- fail "publishing.exposure=loadBalancer requires publishing.externalIPs to contain at least one non-empty address: the CiliumLoadBalancerIPPool has nothing to advertise and the Service would stay in <pending>." }}
-{{- end }}
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
@@ -55,12 +41,9 @@ spec:
           enabled: false
         {{- end }}
         service:
-          {{- if and (eq $exposeIngress .Release.Namespace) (eq $exposeMode "loadBalancer") }}
-          type: LoadBalancer
-          externalTrafficPolicy: Local
-          {{- else if and (eq $exposeIngress .Release.Namespace) $exposeIPsList }}
+          {{- if and (eq $exposeIngress .Release.Namespace) $exposeExternalIPs }}
           externalIPs:
-            {{- toYaml $exposeIPsList | nindent 12 }}
+            {{- toYaml (splitList "," $exposeExternalIPs) | nindent 12 }}
           type: ClusterIP
           externalTrafficPolicy: Cluster
           {{- else }}

packages/extra/ingress/tests/exposure_test.yaml

@@ -1,165 +0,0 @@
-suite: ingress exposure modes
-templates:
-  - templates/nginx-ingress.yaml
-
-release:
-  name: ingress
-  namespace: tenant-root
-
-tests:
-  - it: default exposure (externalIPs) renders ClusterIP Service with spec.externalIPs
-    set:
-      _cluster:
-        expose-ingress: tenant-root
-        expose-external-ips: "192.0.2.10,192.0.2.11"
-    asserts:
-      - template: templates/nginx-ingress.yaml
-        equal:
-          path: spec.values.ingress-nginx.controller.service.type
-          value: ClusterIP
-      - template: templates/nginx-ingress.yaml
-        equal:
-          path: spec.values.ingress-nginx.controller.service.externalTrafficPolicy
-          value: Cluster
-      - template: templates/nginx-ingress.yaml
-        equal:
-          path: spec.values.ingress-nginx.controller.service.externalIPs
-          value:
-            - 192.0.2.10
-            - 192.0.2.11
-      - template: templates/nginx-ingress.yaml
-        notExists:
-          path: spec.values.ingress-nginx.controller.service.labels
-
-  - it: legacy config without expose-mode falls back to externalIPs behavior
-    set:
-      _cluster:
-        expose-ingress: tenant-root
-        expose-external-ips: "192.0.2.10"
-    asserts:
-      - template: templates/nginx-ingress.yaml
-        equal:
-          path: spec.values.ingress-nginx.controller.service.type
-          value: ClusterIP
-
-  - it: externalIPs mode in a namespace other than publishing.ingressName renders LoadBalancer fallback without externalIPs
-    set:
-      _cluster:
-        expose-ingress: tenant-root
-        expose-external-ips: "192.0.2.10"
-    release:
-      namespace: tenant-other
-    asserts:
-      - template: templates/nginx-ingress.yaml
-        equal:
-          path: spec.values.ingress-nginx.controller.service.type
-          value: LoadBalancer
-      - template: templates/nginx-ingress.yaml
-        equal:
-          path: spec.values.ingress-nginx.controller.service.externalTrafficPolicy
-          value: Local
-      - template: templates/nginx-ingress.yaml
-        notExists:
-          path: spec.values.ingress-nginx.controller.service.externalIPs
-
-  - it: loadBalancer mode renders LoadBalancer Service without externalIPs on the Service
-    set:
-      _cluster:
-        expose-ingress: tenant-root
-        expose-external-ips: "192.0.2.10,192.0.2.11"
-        expose-mode: loadBalancer
-    asserts:
-      - template: templates/nginx-ingress.yaml
-        equal:
-          path: spec.values.ingress-nginx.controller.service.type
-          value: LoadBalancer
-      - template: templates/nginx-ingress.yaml
-        equal:
-          path: spec.values.ingress-nginx.controller.service.externalTrafficPolicy
-          value: Local
-      - template: templates/nginx-ingress.yaml
-        notExists:
-          path: spec.values.ingress-nginx.controller.service.labels
-      - template: templates/nginx-ingress.yaml
-        notExists:
-          path: spec.values.ingress-nginx.controller.service.externalIPs
-
-  - it: loadBalancer mode without externalIPs fails chart render with explicit message
-    set:
-      _cluster:
-        expose-ingress: tenant-root
-        expose-external-ips: ""
-        expose-mode: loadBalancer
-    asserts:
-      - template: templates/nginx-ingress.yaml
-        failedTemplate:
-          errorMessage: "publishing.exposure=loadBalancer requires publishing.externalIPs to contain at least one non-empty address: the CiliumLoadBalancerIPPool has nothing to advertise and the Service would stay in <pending>."
-
-  - it: unknown exposure mode is rejected with a clear error (case-sensitive enum)
-    set:
-      _cluster:
-        expose-ingress: tenant-root
-        expose-external-ips: "192.0.2.10"
-        expose-mode: LoadBalancer
-    asserts:
-      - template: templates/nginx-ingress.yaml
-        failedTemplate:
-          errorMessage: "unknown publishing.exposure mode \"LoadBalancer\": must be \"externalIPs\" or \"loadBalancer\""
-
-  - it: another typo in exposure mode also fails
-    set:
-      _cluster:
-        expose-ingress: tenant-root
-        expose-external-ips: "192.0.2.10"
-        expose-mode: loadbalancer
-    asserts:
-      - template: templates/nginx-ingress.yaml
-        failedTemplate:
-          errorMessage: "unknown publishing.exposure mode \"loadbalancer\": must be \"externalIPs\" or \"loadBalancer\""
-
-  - it: loadBalancer mode in a namespace other than publishing.ingressName falls back to LoadBalancer Service without pool
-    set:
-      _cluster:
-        expose-ingress: tenant-root
-        expose-external-ips: "192.0.2.10"
-        expose-mode: loadBalancer
-    release:
-      namespace: tenant-other
-    asserts:
-      - template: templates/nginx-ingress.yaml
-        equal:
-          path: spec.values.ingress-nginx.controller.service.type
-          value: LoadBalancer
-      - template: templates/nginx-ingress.yaml
-        equal:
-          path: spec.values.ingress-nginx.controller.service.externalTrafficPolicy
-          value: Local
-      - template: templates/nginx-ingress.yaml
-        notExists:
-          path: spec.values.ingress-nginx.controller.service.labels
-      - template: templates/nginx-ingress.yaml
-        notExists:
-          path: spec.values.ingress-nginx.controller.service.externalIPs
-
-  - it: loadBalancer mode with only-empty externalIPs fails chart render (comma-only input)
-    set:
-      _cluster:
-        expose-ingress: tenant-root
-        expose-external-ips: ",,"
-        expose-mode: loadBalancer
-    asserts:
-      - template: templates/nginx-ingress.yaml
-        failedTemplate:
-          errorMessage: "publishing.exposure=loadBalancer requires publishing.externalIPs to contain at least one non-empty address: the CiliumLoadBalancerIPPool has nothing to advertise and the Service would stay in <pending>."
-
-  - it: externalIPs mode also filters out empty entries (trailing comma)
-    set:
-      _cluster:
-        expose-ingress: tenant-root
-        expose-external-ips: "192.0.2.10,"
-    asserts:
-      - template: templates/nginx-ingress.yaml
-        equal:
-          path: spec.values.ingress-nginx.controller.service.externalIPs
-          value:
-            - 192.0.2.10

packages/system/cozy-proxy/charts/cozy-proxy/Chart.yaml

@@ -2,5 +2,5 @@ apiVersion: v2
 name: cozy-proxy
 description: A simple kube-proxy addon for 1:1 NAT services in Kubernetes using an NFT backend
 type: application
-version: 0.3.0
-appVersion: 0.3.0
+version: 0.2.0
+appVersion: 0.2.0

packages/system/cozy-proxy/charts/cozy-proxy/values.yaml

@@ -1,6 +1,6 @@
 image:
   repository: ghcr.io/cozystack/cozystack/cozy-proxy
-  tag: v0.3.0
+  tag: v0.2.0
   pullPolicy: IfNotPresent
 
 daemonset:

packages/system/gpu-operator/examples/README.md

@@ -1,121 +0,0 @@
-# GPU operator — native pod workload on Talos (reference)
-
-The files in this directory are **not** templates. They are reference
-artifacts that document one working configuration for running GPU
-workloads directly in pods on a Talos-based Cozystack cluster, together
-with the DCGM metrics needed by the `gpu/gpu-performance` Grafana
-dashboard.
-
-The out-of-the-box `values-talos.yaml` for this package targets the
-sandbox (VFIO passthrough to KubeVirt VMs) scenario. The files here
-illustrate an alternative — running CUDA workloads in regular pods with
-the NVIDIA device plugin — and the workarounds it currently requires on
-Talos.
-
-## Files
-
-- [`values-native-talos.yaml`](./values-native-talos.yaml) — Cozystack
-  `Package` values that disable sandbox workloads, enable the device
-  plugin, point `hostPaths.driverInstallDir` at the staging location
-  used by the compat DaemonSet, and wire DCGM to the custom metrics
-  ConfigMap.
-- [`dcgm-custom-metrics.yaml`](./dcgm-custom-metrics.yaml) — `ConfigMap`
-  with a DCGM metrics CSV that adds profiling, ECC, throttling and
-  energy counters on top of the upstream defaults. The CSV is a
-  superset needed for full coverage of the `gpu/gpu-performance`
-  dashboard. Which parts are actually required depends on which
-  dashboards you ship — see the table below.
-- [`nvidia-driver-compat.yaml`](./nvidia-driver-compat.yaml) — DaemonSet
-  that stages `libnvidia-ml.so.1` and `nvidia-smi` from the Talos glibc
-  tree into a path where the NVIDIA GPU Operator validator expects
-  them. See the "Why the compat DaemonSet exists" section below.
-
-## Why these are reference, not templates
-
-Shipping these as first-class templates would silently impose
-assumptions that do not hold for every user:
-
-- Whether the NVIDIA Talos system extension is installed on the nodes.
-- Whether GPUs are exposed directly to pods or passed through to VMs.
-- The exact path the installed driver ends up at (depends on the
-  extension version and Talos release).
-
-The sandbox-oriented `values-talos.yaml` remains the default. Operators
-who want native pod GPU workloads can start from this directory and
-adapt as needed.
-
-## Why the compat DaemonSet exists
-
-The NVIDIA GPU Operator validator checks for `libnvidia-ml.so.1` and
-`bin/nvidia-smi` in the path given by `hostPaths.driverInstallDir`.
-Talos installs them under `/usr/local/glibc/usr/lib/` and
-`/usr/local/bin/`, which the validator does not look at. Until upstream
-addresses [NVIDIA/gpu-operator#1687][1], the DaemonSet copies those
-files into a directory the validator does inspect and creates the
-`.driver-ctr-ready` flag file so the validator proceeds.
-
-[1]: https://github.com/NVIDIA/gpu-operator/issues/1687
-
-The compat DaemonSet runs privileged and bind-mounts host paths, so
-the target namespace must allow privileged pods. On clusters that
-enforce the Kubernetes Pod Security Standards at `baseline` or
-`restricted`, label the namespace with
-`pod-security.kubernetes.io/enforce: privileged` (and the matching
-`audit`/`warn` labels if the admission webhook is configured to
-surface violations) before applying the manifest.
-
-## Dashboards and what DCGM metrics they need
-
-Five GPU dashboards live under `gpu/*` in
-`packages/system/monitoring/dashboards-infra.list`. All of them share
-`packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml` as
-their source of aggregated series. The recording rules are safe to
-ship on any cluster — they evaluate to empty series when DCGM is not
-scraped, or when optional counters are missing.
-
-What each dashboard needs on top of the upstream DCGM Exporter
-[`default-counters.csv`][default-csv]:
-
-| Dashboard         | Scope                              | Needs beyond defaults                                                   |
-| ----------------- | ---------------------------------- | ----------------------------------------------------------------------- |
-| `gpu-performance` | Per-node, per-GPU deep dive        | `DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION`          |
-| `gpu-efficiency`  | Per-workload util vs tensor active | `DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION` (via `gpu:*_throttle_fraction:rate5m` recording rules) |
-| `gpu-fleet`       | Cluster-wide admin inventory       | `DCGM_FI_DEV_POWER_MGMT_LIMIT` (for the TDP vs draw panel)              |
-| `gpu-quotas`      | Kube-quota vs live usage           | `kube_pod_container_resource_requests`, `kube_pod_status_phase`, `kube_node_status_allocatable` (via `namespace:gpu_count:allocated` / `cluster:gpu_count:*` recording rules) |
-| `gpu-tenants`     | Per-namespace tenant view          | nothing (works on default counters)                                     |
-
-`DCGM_FI_PROF_PIPE_TENSOR_ACTIVE` and `DCGM_FI_PROF_GR_ENGINE_ACTIVE`
-are already in the upstream default set for the pinned DCGM Exporter
-version, so the tensor-saturation and engine-active panels work without
-any CSV override. The three counters listed in the table — throttling
-violations and the power management limit — are the only extras the
-tracked dashboards need. The recording rules in
-`gpu-recording.rules.yaml` consume utilization, FB used, power,
-temperature and the tensor-active profiling counter from the default
-set, plus `DCGM_FI_DEV_POWER_VIOLATION` and
-`DCGM_FI_DEV_THERMAL_VIOLATION` — used by the
-`gpu.recording.efficiency.1m` group to derive the
-`gpu:power_throttle_fraction:rate5m` and
-`gpu:thermal_throttle_fraction:rate5m` series consumed by the
-throttling panels on the efficiency and fleet dashboards.
-
-The `gpu.recording.throttle.validation.5m` group additionally ships the
-`GPUThrottleFractionOverOne` alert (severity `warning`) as a regression
-detector: it fires when either throttle-fraction series exceeds 1.0,
-which would indicate that DCGM changed the scale/divisor of the
-underlying violation counters and the recording rules need to be
-re-derived.
-
-## Verification status
-
-The minimum-CSV claims above are verified by
-`hack/check-gpu-recording-rules.bats`, which cross-checks every
-`DCGM_FI_*` reference in the tracked GPU dashboards and recording rules
-against the union of the upstream default set (snapshotted at
-`hack/dcgm-default-counters.csv` for the pinned DCGM Exporter version)
-and the custom CSV in `dcgm-custom-metrics.yaml`. When the DCGM
-Exporter image in `packages/system/gpu-operator/charts/gpu-operator/values.yaml`
-is bumped, refresh the snapshot from the matching tag of the
-[`NVIDIA/dcgm-exporter`][default-csv] repository and rerun the test.
-
-[default-csv]: https://github.com/NVIDIA/dcgm-exporter/blob/main/etc/default-counters.csv

packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml

@@ -1,88 +0,0 @@
-# Custom DCGM Exporter metrics CSV. Referenced from
-# examples/values-native-talos.yaml via dcgmExporter.config.name.
-#
-# Extends the upstream default set with profiling counters, ECC, page
-# retirement, row remap, energy and throttling violations — everything
-# the gpu/gpu-performance dashboard and the GPU recording rules in
-# monitoring-agents expect.
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: dcgm-custom-metrics
-  namespace: cozy-gpu-operator
-data:
-  dcgm-metrics.csv: |
-    # Format
-    # If line starts with a '#' it is considered a comment
-    # DCGM FIELD, Prometheus metric type, help message
-
-    # Identity
-    DCGM_FI_DRIVER_VERSION, label, Driver version.
-
-    # Clocks
-    DCGM_FI_DEV_SM_CLOCK,  gauge, SM clock frequency (in MHz).
-    DCGM_FI_DEV_MEM_CLOCK, gauge, Memory clock frequency (in MHz).
-
-    # Temperature
-    DCGM_FI_DEV_MEMORY_TEMP, gauge, Memory temperature (in C).
-    DCGM_FI_DEV_GPU_TEMP,    gauge, GPU temperature (in C).
-
-    # Power
-    DCGM_FI_DEV_POWER_USAGE,              gauge, Power draw (in W).
-    DCGM_FI_DEV_POWER_MGMT_LIMIT,         gauge, Current power management limit (in W).
-    DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION, counter, Total energy consumption since boot (in mJ).
-
-    # PCIE
-    DCGM_FI_DEV_PCIE_REPLAY_COUNTER, counter, Total number of PCIe retries.
-
-    # Utilization (the sample period varies depending on the product)
-    DCGM_FI_DEV_GPU_UTIL,      gauge, GPU utilization (in %).
-    DCGM_FI_DEV_MEM_COPY_UTIL, gauge, Memory utilization (in %).
-    DCGM_FI_DEV_ENC_UTIL,      gauge, Encoder utilization (in %).
-    DCGM_FI_DEV_DEC_UTIL,      gauge, Decoder utilization (in %).
-
-    # Errors and violations
-    DCGM_FI_DEV_XID_ERRORS, gauge, Value of the last XID error encountered.
-
-    # Memory usage
-    DCGM_FI_DEV_FB_FREE,     gauge, Framebuffer memory free (in MiB).
-    DCGM_FI_DEV_FB_USED,     gauge, Framebuffer memory used (in MiB).
-    DCGM_FI_DEV_FB_RESERVED, gauge, Framebuffer memory reserved (in MiB).
-
-    # ECC (supported on datacenter-class GPUs such as A10)
-    DCGM_FI_DEV_ECC_SBE_VOL_TOTAL, counter, Total number of single-bit volatile ECC errors.
-    DCGM_FI_DEV_ECC_DBE_VOL_TOTAL, counter, Total number of double-bit volatile ECC errors.
-    DCGM_FI_DEV_ECC_SBE_AGG_TOTAL, counter, Total number of single-bit persistent ECC errors.
-    DCGM_FI_DEV_ECC_DBE_AGG_TOTAL, counter, Total number of double-bit persistent ECC errors.
-
-    # Retired pages
-    DCGM_FI_DEV_RETIRED_SBE,     counter, Total number of retired pages due to single-bit errors.
-    DCGM_FI_DEV_RETIRED_DBE,     counter, Total number of retired pages due to double-bit errors.
-    DCGM_FI_DEV_RETIRED_PENDING, counter, Total number of pages pending retirement.
-
-    # Row remapping (not applicable to GDDR6 but left for datacenter GPUs)
-    DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for uncorrectable errors.
-    DCGM_FI_DEV_CORRECTABLE_REMAPPED_ROWS,   counter, Number of remapped rows for correctable errors.
-    DCGM_FI_DEV_ROW_REMAP_FAILURE,           gauge,   Whether remapping of rows has failed.
-
-    # Throttle / violation counters (crucial for SLA and tenant monitoring)
-    DCGM_FI_DEV_POWER_VIOLATION,       counter, Throttling duration due to power constraints (us per docs; ns on DCGM 3.x).
-    DCGM_FI_DEV_THERMAL_VIOLATION,     counter, Throttling duration due to thermal constraints (us per docs; ns on DCGM 3.x).
-    DCGM_FI_DEV_SYNC_BOOST_VIOLATION,  counter, Throttling duration due to sync-boost constraints (us per docs; ns on DCGM 3.x).
-    DCGM_FI_DEV_BOARD_LIMIT_VIOLATION, counter, Throttling duration due to board limit constraints (us per docs; ns on DCGM 3.x).
-    DCGM_FI_DEV_LOW_UTIL_VIOLATION,    counter, Throttling duration due to low utilization (us per docs; ns on DCGM 3.x).
-    DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (us per docs; ns on DCGM 3.x).
-
-    # NVLink — DCGM silently drops this metric on GPUs without NVLink, so
-    # enabling it unconditionally is safe and keeps this file reusable
-    # across GPU families.
-    DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL, counter, Total number of NVLink bandwidth counters for all lanes.
-
-    # DCP (profiling) metrics — supported from Ampere onwards.
-    DCGM_FI_PROF_GR_ENGINE_ACTIVE,   gauge, Ratio of time the graphics engine is active.
-    DCGM_FI_PROF_SM_ACTIVE,          gauge, The ratio of cycles an SM has at least 1 warp assigned.
-    DCGM_FI_PROF_SM_OCCUPANCY,       gauge, The ratio of number of warps resident on an SM.
-    DCGM_FI_PROF_PIPE_TENSOR_ACTIVE, gauge, Ratio of cycles the tensor (HMMA) pipe is active.
-    DCGM_FI_PROF_DRAM_ACTIVE,        gauge, Ratio of cycles the device memory interface is active sending or receiving data.
-    DCGM_FI_PROF_PCIE_TX_BYTES,      counter, The number of bytes of active PCIe tx (transmit) data including both header and payload.
-    DCGM_FI_PROF_PCIE_RX_BYTES,      counter, The number of bytes of active PCIe rx (read) data including both header and payload.

packages/system/gpu-operator/examples/nvidia-driver-compat.yaml

@@ -1,109 +0,0 @@
-# Workaround for https://github.com/NVIDIA/gpu-operator/issues/1687
-#
-# On Talos, the NVIDIA system extension installs driver files under
-# /usr/local/glibc/usr/lib/ and /usr/local/bin/. The GPU Operator
-# validator only looks for libnvidia-ml.so.1 and bin/nvidia-smi under
-# the path configured as hostPaths.driverInstallDir. This DaemonSet
-# stages the required files into /var/nvidia-driver on each node and
-# creates the .driver-ctr-ready flag so the validator proceeds.
-#
-# Paired with examples/values-native-talos.yaml, which sets
-# hostPaths.driverInstallDir to /var/nvidia-driver.
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: nvidia-driver-compat
-  namespace: cozy-gpu-operator
-  labels:
-    app: nvidia-driver-compat
-spec:
-  selector:
-    matchLabels:
-      app: nvidia-driver-compat
-  template:
-    metadata:
-      labels:
-        app: nvidia-driver-compat
-    # DaemonSet rather than a one-shot Job: staging is idempotent per-node,
-    # must re-run after every Talos reboot (hostPath contents are wiped on
-    # reboot when the system extension re-installs), and the pause container
-    # keeps the pod visible for operator observability (kubectl get pods).
-    spec:
-      priorityClassName: system-node-critical
-      # Restrict to GPU nodes only. Without this the DaemonSet schedules onto
-      # every node (including control-plane and CPU-only workers) and burns
-      # resources on hosts where the compat tree is meaningless.
-      # The label is published by Node Feature Discovery / GPU Operator's NFD
-      # subchart; if NFD is disabled, replace this with whatever label your
-      # cluster uses to mark GPU hosts.
-      nodeSelector:
-        nvidia.com/gpu.present: "true"
-      # Tolerate-all is intentionally broad: the nodeSelector above already
-      # confines scheduling to GPU nodes, so the "Exists" toleration only
-      # matters when those nodes carry extra taints (dedicated=gpu:NoSchedule,
-      # nvidia.com/gpu:NoSchedule from the GPU Operator's default policy,
-      # etc.). This mirrors the stance of the upstream nvidia-driver-daemonset
-      # and nvidia-device-plugin DaemonSets — blanket tolerations plus a
-      # narrow nodeSelector.
-      tolerations:
-        - operator: Exists
-      initContainers:
-        - name: create-driver-tree
-          image: busybox:1.37
-          command:
-            - sh
-            - -c
-            - |
-              set -e
-              GLIBC_LIB="/host/usr/local/glibc/usr/lib"
-              SRC_BIN="/host/usr/local/bin"
-              DST="/host/var/nvidia-driver"
-
-              mkdir -p "$DST/bin"
-
-              [ -f "$GLIBC_LIB/libnvidia-ml.so.1" ] || {
-                echo "missing $GLIBC_LIB/libnvidia-ml.so.1" >&2
-                exit 1
-              }
-              [ -f "$SRC_BIN/nvidia-smi" ] || {
-                echo "missing $SRC_BIN/nvidia-smi" >&2
-                exit 1
-              }
-
-              cp -f "$GLIBC_LIB/libnvidia-ml.so.1" "$DST/libnvidia-ml.so.1"
-              echo "Copied libnvidia-ml.so.1"
-
-              cp -f "$SRC_BIN/nvidia-smi" "$DST/bin/nvidia-smi"
-              chmod +x "$DST/bin/nvidia-smi"
-              echo "Copied nvidia-smi"
-
-              mkdir -p /host/run/nvidia/validations
-              touch /host/run/nvidia/validations/.driver-ctr-ready
-              echo "Created driver-ctr-ready flag"
-              echo "Done"
-          securityContext:
-            privileged: true
-          resources:
-            requests:
-              cpu: 10m
-              memory: 16Mi
-            limits:
-              cpu: 100m
-              memory: 64Mi
-          volumeMounts:
-            - name: host-root
-              mountPath: /host
-      containers:
-        - name: pause
-          image: registry.k8s.io/pause:3.10
-          resources:
-            requests:
-              cpu: 10m
-              memory: 8Mi
-            limits:
-              cpu: 50m
-              memory: 16Mi
-      volumes:
-        - name: host-root
-          hostPath:
-            path: /

packages/system/gpu-operator/examples/values-native-talos.yaml

@@ -1,52 +0,0 @@
-# Cozystack Package values for running GPU workloads natively in pods
-# on Talos. This is the counterpart to values-talos.yaml, which targets
-# the sandbox (VFIO passthrough) scenario.
-#
-# Prerequisites:
-#   - NVIDIA Talos system extension installed on GPU nodes.
-#   - examples/nvidia-driver-compat.yaml deployed to stage driver files
-#     where the gpu-operator validator looks for them.
-#   - examples/dcgm-custom-metrics.yaml applied so DCGM Exporter exports
-#     the full set of metrics used by the dashboard and recording rules.
-apiVersion: cozystack.io/v1alpha1
-kind: Package
-metadata:
-  name: cozystack.gpu-operator
-spec:
-  variant: default
-  components:
-    gpu-operator:
-      values:
-        gpu-operator:
-          # The compat DaemonSet stages libnvidia-ml.so.1 and nvidia-smi
-          # under /var/nvidia-driver. Point the validator at that path.
-          hostPaths:
-            driverInstallDir: "/var/nvidia-driver"
-          # Disable the sandbox path — workloads run in pods, not VMs.
-          sandboxWorkloads:
-            enabled: false
-          vfioManager:
-            enabled: false
-          # The Talos extension provides the driver and runtime hooks,
-          # so the operator's own toolkit and driver components must be
-          # switched off to avoid conflicting installations.
-          toolkit:
-            enabled: false
-          devicePlugin:
-            enabled: true
-          # Export full set of DCGM metrics using the ConfigMap in
-          # examples/dcgm-custom-metrics.yaml.
-          dcgmExporter:
-            serviceMonitor:
-              enabled: true
-              # Matches the 1m evaluation cadence of GPU recording rules with
-              # 4x oversampling headroom. DCGM_FI_PROF_* counters have
-              # documented overhead concerns below 1s; 15s is safe.
-              interval: "15s"
-              honorLabels: true
-              relabelings:
-                - sourceLabels: [__meta_kubernetes_pod_node_name]
-                  targetLabel: node
-                  action: replace
-            config:
-              name: dcgm-custom-metrics

packages/system/hami/Chart.yaml

@@ -1,3 +0,0 @@
-apiVersion: v2
-name: cozy-hami
-version: 0.0.0 # Placeholder, the actual version will be automatically set during the build process

packages/system/hami/Makefile

@@ -1,43 +0,0 @@
-export NAME=hami
-export NAMESPACE=cozy-$(NAME)
-
-include ../../../hack/common-envs.mk
-include ../../../hack/package.mk
-
-# When bumping the HAMi version, run `make update` and then review
-# the resulting diff in `charts/hami/`. The recipe below reproduces the
-# top-level vendoring overrides automatically:
-#
-#   1. Removes the broken hami-dra subchart. Upstream's NVIDIA DRA driver
-#      path requires kubelet DRA support that cozystack does not enable
-#      and has no upstream fix tracked. See commit 3c5521e.
-#   2. Empties Chart.yaml dependencies and drops Chart.lock so Helm does
-#      not try to re-pull hami-dra at build time. See commit 2734dc0.
-#   3. Strips dra/hami-dra/podSecurityPolicy blocks from the upstream
-#      values.yaml since the corresponding code paths are gone. PSP is
-#      removed from Kubernetes 1.25+ and is unused by cozystack.
-#
-# Template-level patches are NOT reproduced automatically:
-#
-#   * Scheduler templates have `{{- if .Values.dra.enabled }}` blocks
-#     that need to be removed because the dra value is gone (commit
-#     2734dc0 stripped them).
-#   * device-plugin/monitorservice.yaml uses `indent` with leading
-#     whitespace; it must be rewritten to `nindent` for the labels block
-#     to render correctly when devicePlugin.service.labels is set
-#     (commit 3685254).
-#
-# After `make update`, run `git diff -- charts/hami/templates/` and
-# replay these template patches against the new upstream version, then
-# verify with `helm unittest`. If upstream restructured the affected
-# files, the patches may need to be redesigned rather than reapplied.
-
-update:
-	rm -rf charts
-	helm repo add hami-charts https://project-hami.github.io/HAMi/
-	helm repo update hami-charts
-	helm pull hami-charts/hami --untar --untardir charts
-	rm -rf charts/hami/charts/hami-dra
-	yq --inplace '.dependencies = []' charts/hami/Chart.yaml
-	rm -f charts/hami/Chart.lock
-	yq --inplace 'del(.dra) | del(.["hami-dra"]) | del(.podSecurityPolicy)' charts/hami/values.yaml

packages/system/hami/README.md

@@ -1,82 +0,0 @@
-# HAMi — GPU Virtualization Middleware
-
-[HAMi](https://github.com/Project-HAMi/HAMi) (Heterogeneous AI Computing Virtualization Middleware) is a CNCF Sandbox project that enables fractional GPU sharing in Kubernetes. It allows workloads to request specific amounts of GPU memory and compute cores instead of claiming entire GPUs.
-
-## Architecture
-
-HAMi consists of four components:
-
-- **MutatingWebhook** — intercepts pod creation, injects `schedulerName: hami-scheduler`
-- **Scheduler Extender** — extends kube-scheduler with GPU-aware Filter and Bind logic
-- **Device Plugin** (DaemonSet) — registers vGPU resources via the Kubernetes Device Plugin API
-- **HAMi-core** (`libvgpu.so`) — `LD_PRELOAD` library injected into workload containers, intercepts CUDA API calls to enforce memory and compute isolation
-
-## Prerequisites
-
-- GPU Operator must be enabled (`addons.gpuOperator.enabled: true`)
-- NVIDIA driver >= 440 on host nodes
-- nvidia-container-toolkit configured as the default container runtime
-- GPU nodes labeled with `gpu=on`
-
-## Known Limitations
-
-### glibc < 2.34 requirement for workload containers
-
-HAMi-core uses `LD_PRELOAD` to intercept `dlsym()` for CUDA symbol resolution. The fallback code path relies on `_dl_sym`, a private glibc internal symbol that was removed in glibc 2.34 when libdl and libpthread were merged into libc.so.
-
-**This limitation affects workload containers only**, not the host OS or HAMi's own components.
-
-| Distribution    | glibc | Result                                       |
-| --------------- | ----- | -------------------------------------------- |
-| Ubuntu 18.04    | 2.27  | Full isolation (memory + compute)            |
-| Ubuntu 20.04    | 2.31  | Full isolation (memory + compute)            |
-| Ubuntu 22.04    | 2.35  | Memory isolation works, compute breaks       |
-| Ubuntu 24.04    | 2.39  | Both memory and compute isolation break      |
-| Alpine (musl)   | N/A   | Completely incompatible (`dlvsym` absent)    |
-
-Most modern ML/AI base images (CUDA 12.x, PyTorch 2.x, TensorFlow 2.x) use Ubuntu 22.04+ with glibc >= 2.35, which means compute isolation will not work with these images until the upstream fix is merged.
-
-**Upstream tracking issues:**
-
-- [HAMi-core#174](https://github.com/Project-HAMi/HAMi-core/issues/174) — `_dl_sym` removal in glibc 2.34 breaks HAMi-core's CUDA symbol resolution at the symbol level
-- [HAMi#1190](https://github.com/Project-HAMi/HAMi/issues/1190) — maintainer thread confirming the empirical per-glibc-version isolation behavior shown in the table above
-
-### musl libc (Alpine) incompatibility
-
-HAMi-core is completely incompatible with musl libc. The `dlvsym()` function used by HAMi-core is a glibc extension not available in musl. Only glibc-based container images (Debian, Ubuntu, RHEL, etc.) can use HAMi GPU isolation.
-
-## Usage
-
-Enable HAMi in your tenant Kubernetes cluster values:
-
-```yaml
-addons:
-  gpuOperator:
-    enabled: true
-  hami:
-    enabled: true
-```
-
-When HAMi is enabled, GPU Operator's built-in device plugin is automatically disabled to avoid conflicts. This default is preserved by setting `addons.gpuOperator.valuesOverride.gpu-operator.devicePlugin.enabled: false`; advanced topologies that partition GPU pools (e.g. some nodes use HAMi while others run the standard NVIDIA device plugin via node selectors) can re-enable it explicitly through `valuesOverride`.
-
-### Requesting fractional GPU resources
-
-```yaml
-resources:
-  limits:
-    nvidia.com/gpu: 1
-    nvidia.com/gpumem: 3000     # 3000 MB of GPU memory
-    nvidia.com/gpucores: 30     # 30% of GPU compute cores
-```
-
-## Parameters
-
-Default values shown below are inherited from the upstream HAMi chart and may change with upstream updates.
-
-| Name | Description | Default |
-| --- | --- | --- |
-| `hami.devicePlugin.runtimeClassName` | RuntimeClass for device plugin pods | `nvidia` |
-| `hami.devicePlugin.deviceSplitCount` | Max virtual GPUs per physical GPU | `10` |
-| `hami.devicePlugin.deviceMemoryScaling` | Memory overcommit factor (> 1.0 enables overcommit) | `1` |
-| `hami.scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy` | Node packing strategy (`binpack` or `spread`) | `binpack` |
-| `hami.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy` | GPU packing strategy (`binpack` or `spread`) | `spread` |

packages/system/hami/charts/hami/Chart.yaml

@@ -1,18 +0,0 @@
-apiVersion: v2
-appVersion: 2.8.1
-dependencies: []
-description: Heterogeneous AI Computing Virtualization Middleware
-keywords:
-- vgpu
-- gpu
-kubeVersion: '>= 1.18.0-0'
-maintainers:
-- email: archlitchi@gmail.com
-  name: limengxuan
-- email: xiaozhang0210@hotmail.com
-  name: zhangxiao
-name: hami
-sources:
-- https://github.com/Project-HAMi/HAMi
-type: application
-version: 2.8.1

packages/system/hami/charts/hami/README.md

@@ -1,237 +0,0 @@
-# HAMi Helm Chart Values Documentation
-
-This document provides detailed descriptions of all configurable values parameters for the HAMi Helm Chart.
-
-## Global Configuration
-
-| Parameter | Description | Default Value |
-|-----------|-------------|---------------|
-| `global.imageRegistry` | Global Docker image registry | `""` |
-| `global.imagePullSecrets` | Global Docker image pull secrets | `[]` |
-| `global.imageTag` | Image tag | `"v2.8.1"` |
-| `global.gpuHookPath` | GPU Hook path | `/usr/local` |
-| `global.labels` | Global labels | `{}` |
-| `global.annotations` | Global annotations | `{}` |
-| `global.managedNodeSelectorEnable` | Whether to enable managed node selector | `false` |
-| `global.managedNodeSelector.usage` | Managed node selector usage | `"gpu"` |
-| `nameOverride` | Name override | `""` |
-| `fullnameOverride` | Full name override | `""` |
-| `namespaceOverride` | Namespace override | `""` |
-
-## Resource Name Configuration
-
-### NVIDIA GPU Resources
-| Parameter | Description | Default Value |
-|-----------|-------------|---------------|
-| `resourceName` | GPU resource name | `"nvidia.com/gpu"` |
-| `resourceMem` | GPU memory resource name | `"nvidia.com/gpumem"` |
-| `resourceMemPercentage` | GPU memory percentage resource name | `"nvidia.com/gpumem-percentage"` |
-| `resourceCores` | GPU core resource name | `"nvidia.com/gpucores"` |
-| `resourcePriority` | GPU priority resource name | `"nvidia.com/priority"` |
-
-### Cambricon MLU Resources
-| Parameter | Description | Default Value |
-|-----------|-------------|---------------|
-| `mluResourceName` | MLU resource name | `"cambricon.com/vmlu"` |
-| `mluResourceMem` | MLU memory resource name | `"cambricon.com/mlu.smlu.vmemory"` |
-| `mluResourceCores` | MLU core resource name | `"cambricon.com/mlu.smlu.vcore"` |
-
-### Hygon DCU Resources
-| Parameter | Description | Default Value |
-|-----------|-------------|---------------|
-| `dcuResourceName` | DCU resource name | `"hygon.com/dcunum"` |
-| `dcuResourceMem` | DCU memory resource name | `"hygon.com/dcumem"` |
-| `dcuResourceCores` | DCU core resource name | `"hygon.com/dcucores"` |
-
-### Metax GPU Resources
-| Parameter | Description | Default Value |
-|-----------|-------------|---------------|
-| `metaxResourceName` | GPU resource name | `"metax-tech.com/sgpu"` |
-| `metaxResourceCore` | GPU core resource name | `"metax-tech.com/vcore"` |
-| `metaxResourceMem` | GPU memory resource name | `"metax-tech.com/vmemory"` |
-| `metaxsGPUTopologyAware` | GPU topology awareness | `"false"` |
-
-### Enflame GCU Resources
-| Parameter | Description | Default Value |
-|-----------|-------------|---------------|
-| `enflameResourceNameVGCU` | vGCU resource name | `"enflame.com/vgcu"` |
-| `enflameResourceNameVGCUPercentage` | vGCU percentage resource name | `"enflame.com/vgcu-percentage"` |
-
-### Kunlunxin XPU Resources
-| Parameter | Description | Default Value |
-|-----------|-------------|---------------|
-| `kunlunResourceName` | XPU resource name | `"kunlunxin.com/xpu"` |
-
-## Scheduler Configuration
-
-| Parameter | Description | Default Value |
-|-----------|-------------|---------------|
-| `schedulerName` | Scheduler name | `"hami-scheduler"` |
-| `scheduler.nodeName` | Define node name, scheduler will schedule to this node | `""` |
-| `scheduler.overwriteEnv` | Whether to overwrite environment variables | `"false"` |
-| `scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy` | Node scheduler policy | `binpack` |
-| `scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy` | GPU scheduler policy | `spread` |
-| `scheduler.metricsBindAddress` | Metrics bind address | `":9395"` |
-| `scheduler.forceOverwriteDefaultScheduler` | Whether to force overwrite default scheduler | `true` |
-| `scheduler.livenessProbe` | Whether to enable liveness probe | `false` |
-| `scheduler.leaderElect` | Whether to enable leader election | `true` |
-| `scheduler.replicas` | Number of replicas | `1` |
-
-### Kube Scheduler Configuration
-
-| Parameter | Description | Default Value |
-|-----------|-------------|---------------|
-| `scheduler.kubeScheduler.enabled` | Whether to run kube-scheduler container in scheduler pod | `true` |
-| `scheduler.kubeScheduler.image.registry` | Kube scheduler image registry | `"registry.cn-hangzhou.aliyuncs.com"` |
-| `scheduler.kubeScheduler.image.repository` | Kube scheduler image repository | `"google_containers/kube-scheduler"` |
-| `scheduler.kubeScheduler.image.tag` | Kube scheduler image tag | `""` |
-| `scheduler.kubeScheduler.image.pullPolicy` | Kube scheduler image pull policy | `IfNotPresent` |
-| `scheduler.kubeScheduler.image.pullSecrets` | Kube scheduler image pull secrets | `[]` |
-| `scheduler.kubeScheduler.extraNewArgs` | Extra new arguments | `["--config=/config/config.yaml", "-v=4"]` |
-| `scheduler.kubeScheduler.extraArgs` | Extra arguments | `["--policy-config-file=/config/config.json", "-v=4"]` |
-
-### Extender Configuration
-
-| Parameter | Description | Default Value |
-|-----------|-------------|---------------|
-| `scheduler.extender.image.registry` | Scheduler extender image registry | `"docker.io"` |
-| `scheduler.extender.image.repository` | Scheduler extender image repository | `"projecthami/hami"` |
-| `scheduler.extender.image.tag` | Scheduler extender image tag | `""` |
-| `scheduler.extender.image.pullPolicy` | Scheduler extender image pull policy | `IfNotPresent` |
-| `scheduler.extender.image.pullSecrets` | Scheduler extender image pull secrets | `[]` |
-| `scheduler.extender.extraArgs` | Scheduler extender extra arguments | `["--debug", "-v=4"]` |
-
-### Admission Webhook Configuration
-
-| Parameter | Description | Default Value |
-|-----------|-------------|---------------|
-| `scheduler.admissionWebhook.enabled` | Whether to enable admission webhook | `true` |
-| `scheduler.admissionWebhook.customURL.enabled` | Whether to enable custom URL | `false` |
-| `scheduler.admissionWebhook.customURL.host` | Custom URL host | `127.0.0.1` |
-| `scheduler.admissionWebhook.customURL.port` | Custom URL port | `31998` |
-| `scheduler.admissionWebhook.customURL.path` | Custom URL path | `/webhook` |
-| `scheduler.admissionWebhook.reinvocationPolicy` | Reinvocation policy | `Never` |
-| `scheduler.admissionWebhook.failurePolicy` | Failure policy | `Ignore` |
-
-### TLS Certificate Configuration
-
-| Parameter | Description | Default Value |
-|-----------|-------------|---------------|
-| `scheduler.certManager.enabled` | Whether to use cert-manager to generate self-signed certificates | `false` |
-| `scheduler.patch.enabled` | Whether to use kube-webhook-certgen to generate self-signed certificates | `true` |
-| `scheduler.patch.image.registry` | Certgen image registry | `"docker.io"` |
-| `scheduler.patch.image.repository` | Certgen image repository | `"jettech/kube-webhook-certgen"` |
-| `scheduler.patch.image.tag` | Certgen image tag | `"v1.5.2"` |
-| `scheduler.patch.image.pullPolicy` | Certgen image pull policy | `IfNotPresent` |
-| `scheduler.patch.imageNew.registry` | New certgen image registry | `"docker.io"` |
-| `scheduler.patch.imageNew.repository` | New certgen image repository | `"liangjw/kube-webhook-certgen"` |
-| `scheduler.patch.imageNew.tag` | New certgen image tag | `"v1.1.1"` |
-
-### Scheduler Service Configuration
-
-| Parameter | Description | Default Value |
-|-----------|-------------|---------------|
-| `scheduler.service.type` | Service type | `NodePort` |
-| `scheduler.service.httpPort` | HTTP port | `443` |
-| `scheduler.service.schedulerPort` | Scheduler NodePort | `31998` |
-| `scheduler.service.monitorPort` | Monitor port | `31993` |
-| `scheduler.service.monitorTargetPort` | Monitor target port | `9395` |
-
-## Device Plugin Configuration
-
-| Parameter | Description | Default Value |
-|-----------|-------------|---------------|
-| `devicePlugin.image.registry` | Device plugin image registry | `"docker.io"` |
-| `devicePlugin.image.repository` | Device plugin image repository | `"projecthami/hami"` |
-| `devicePlugin.image.tag` | Device plugin image tag | `""` |
-| `devicePlugin.image.pullPolicy` | Device plugin image pull policy | `IfNotPresent` |
-| `devicePlugin.image.pullSecrets` | Device plugin image pull secrets | `[]` |
-
-### Monitor Configuration
-
-| Parameter | Description | Default Value |
-|-----------|-------------|---------------|
-| `devicePlugin.monitor.image.registry` | Monitor image registry | `"docker.io"` |
-| `devicePlugin.monitor.image.repository` | Monitor image repository | `"projecthami/hami"` |
-| `devicePlugin.monitor.image.tag` | Monitor image tag | `""` |
-| `devicePlugin.monitor.image.pullPolicy` | Monitor image pull policy | `IfNotPresent` |
-| `devicePlugin.monitor.image.pullSecrets` | Monitor image pull secrets | `[]` |
-| `devicePlugin.monitor.ctrPath` | Container path | `/usr/local/vgpu/containers` |
-| `devicePlugin.monitor.extraArgs` | Monitor extra arguments | `["-v=4"]` |
-| `devicePlugin.monitor.extraEnvs` | Monitor extra environments | `{}` |
-
-### Device Plugin Other Configuration
-
-| Parameter | Description | Default Value |
-|-----------|-------------|---------------|
-| `devicePlugin.deviceSplitCount` | Integer type, default value: 10. Maximum number of tasks assigned to a single GPU device | `10` |
-| `devicePlugin.deviceMemoryScaling` | Device memory scaling ratio | `1` |
-| `devicePlugin.deviceCoreScaling` | Device core scaling ratio | `1` |
-| `devicePlugin.runtimeClassName` | Runtime class name | `""` |
-| `devicePlugin.createRuntimeClass` | Whether to create runtime class | `false` |
-| `devicePlugin.migStrategy` | String type, "none" means ignore MIG functionality, "mixed" means allocate MIG devices through independent resources | `"none"` |
-| `devicePlugin.disablecorelimit` | String type, "true" means disable core limit, "false" means enable core limit | `"false"` |
-| `devicePlugin.passDeviceSpecsEnabled` | Whether to enable passing device specs | `false` |
-| `devicePlugin.extraArgs` | Device plugin extra arguments | `["-v=4"]` |
-| `devicePlugin.nodeConfiguration.config` | Node configuration for device plugin by json | An example of default configuration. |
-| `devicePlugin.nodeConfiguration.externalConfigName` | Node configuration for device plugin by external congimap | `""` |
-| `devicePlugin.extraEnvs` | Device plugin extra environments | `{}` |
-
-### Device Plugin Service Configuration
-
-| Parameter | Description | Default Value |
-|-----------|-------------|---------------|
-| `devicePlugin.service.type` | Service type | `NodePort` |
-| `devicePlugin.service.httpPort` | HTTP port | `31992` |
-
-### Device Plugin Deployment Configuration
-
-| Parameter | Description | Default Value |
-|-----------|-------------|---------------|
-| `devicePlugin.pluginPath` | Plugin path | `/var/lib/kubelet/device-plugins` |
-| `devicePlugin.libPath` | Library path | `/usr/local/vgpu` |
-| `devicePlugin.nvidiaNodeSelector` | NVIDIA node selector | `{"gpu": "on"}` |
-| `devicePlugin.updateStrategy.type` | Update strategy type | `RollingUpdate` |
-| `devicePlugin.updateStrategy.rollingUpdate.maxUnavailable` | Maximum unavailable count | `1` |
-
-## Device Configuration
-
-### AWS Neuron
-| Parameter | Description | Default Value |
-|-----------|-------------|---------------|
-| `devices.awsneuron.customresources` | Custom resources | `["aws.amazon.com/neuron", "aws.amazon.com/neuroncore"]` |
-
-### Kunlunxin
-| Parameter | Description | Default Value |
-|-----------|-------------|---------------|
-| `devices.kunlun.enabled` | Whether to enable | `true` |
-| `devices.kunlun.customresources` | Custom resources | `["kunlunxin.com/xpu"]` |
-
-### Mthreads
-| Parameter | Description | Default Value |
-|-----------|-------------|---------------|
-| `devices.mthreads.enabled` | Whether to enable | `true` |
-| `devices.mthreads.customresources` | Custom resources | `["mthreads.com/vgpu"]` |
-
-### NVIDIA
-| Parameter | Description | Default Value |
-|-----------|-------------|---------------|
-| `devices.nvidia.gpuCorePolicy` | GPU core policy | `default` |
-| `devices.nvidia.libCudaLogLevel` | CUDA library log level | `1` |
-
-### Huawei Ascend
-| Parameter | Description | Default Value |
-|-----------|-------------|---------------|
-| `devices.ascend.enabled` | Whether to enable | `false` |
-| `devices.ascend.image` | Image | `""` |
-| `devices.ascend.imagePullPolicy` | Image pull policy | `IfNotPresent` |
-| `devices.ascend.extraArgs` | Extra arguments | `[]` |
-| `devices.ascend.nodeSelector` | Node selector | `{"ascend": "on"}` |
-| `devices.ascend.tolerations` | Tolerations | `[]` |
-| `devices.ascend.customresources` | Custom resources | `["huawei.com/Ascend910A", "huawei.com/Ascend910A-memory", ...]` |
-
-### Iluvatar
-| Parameter | Description | Default Value |
-|-----------|-------------|---------------|
-| `devices.iluvatar.enabled` | Whether to enable | `false` |
-| `devices.iluvatar.customresources` | Custom resources | `["iluvatar.ai/BI-V150-vgpu", "iluvatar.ai/BI-V150.vMem","iluvatar.ai/BI-V150.vCore", ...]` |

packages/system/hami/charts/hami/templates/NOTES.txt

@@ -1,3 +0,0 @@
-** Please be patient while the chart is being deployed **
-Resource name: {{ .Values.resourceName }}
-

packages/system/hami/charts/hami/templates/_commons.tpl

@@ -1,49 +0,0 @@
-{{/*
-Return the proper image name
-{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" $) }}
-*/}}
-{{- define "common.images.image" -}}
-{{- $registryName := .imageRoot.registry -}}
-{{- $repositoryName := .imageRoot.repository -}}
-{{- $tag := .imageRoot.tag | toString -}}
-{{- if .global }}
-    {{- if .global.imageRegistry }}
-     {{- $registryName = .global.imageRegistry -}}
-    {{- end -}}
-{{- end -}}
-{{- if .tag }}
-     {{- $tag = .tag | toString -}}
-{{- end -}}
-{{- if $registryName }}
-{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}}
-{{- else -}}
-{{- printf "%s:%s" $repositoryName $tag -}}
-{{- end -}}
-{{- end -}}
-
-{{/*
-Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead)
-{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }}
-*/}}
-{{- define "common.images.pullSecrets" -}}
-  {{- $pullSecrets := list }}
-
-  {{- if .global }}
-    {{- range .global.imagePullSecrets -}}
-      {{- $pullSecrets = append $pullSecrets . -}}
-    {{- end -}}
-  {{- end -}}
-
-  {{- range .images -}}
-    {{- range .pullSecrets -}}
-      {{- $pullSecrets = append $pullSecrets . -}}
-    {{- end -}}
-  {{- end -}}
-
-  {{- if (not (empty $pullSecrets)) }}
-imagePullSecrets:
-    {{- range $pullSecrets }}
-  - name: {{ . }}
-    {{- end }}
-  {{- end }}
-{{- end -}}

packages/system/hami/charts/hami/templates/_helpers.tpl

@@ -1,163 +0,0 @@
-{{/*
-Expand the name of the chart.
-*/}}
-{{- define "hami-vgpu.name" -}}
-{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
-{{/*
-Create a default fully qualified app name.
-We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
-If release name contains chart name it will be used as a full name.
-*/}}
-{{- define "hami-vgpu.fullname" -}}
-{{- if .Values.fullnameOverride -}}
-{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
-{{- else }}
-{{- $name := default .Chart.Name .Values.nameOverride -}}
-{{- if contains $name .Release.Name }}
-{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
-{{- else -}}
-{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-{{- end -}}
-{{- end -}}
-
-{{/*
-Allow the release namespace to be overridden for multi-namespace deployments in combined charts
-*/}}
-{{- define "hami-vgpu.namespace" -}}
-  {{- if .Values.namespaceOverride -}}
-    {{- .Values.namespaceOverride -}}
-  {{- else -}}
-    {{- .Release.Namespace -}}
-  {{- end -}}
-{{- end -}}
-
-{{/*
-The app name for Scheduler
-*/}}
-{{- define "hami-vgpu.scheduler" -}}
-{{- printf "%s-scheduler" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
-{{/*
-The app name for DevicePlugin
-*/}}
-{{- define "hami-vgpu.device-plugin" -}}
-{{- printf "%s-device-plugin" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
-{{/*
-  The app name for MockDevicePlugin
-  */}}
-{{- define "hami-vgpu.mock-device-plugin" -}}
-{{- printf "%s-mock-device-plugin" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
-{{/*
-The tls secret name for Scheduler
-*/}}
-{{- define "hami-vgpu.scheduler.tls" -}}
-{{- printf "%s-scheduler-tls" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
-{{/*
-The webhook name
-*/}}
-{{- define "hami-vgpu.scheduler.webhook" -}}
-{{- printf "%s-webhook" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
-{{/*
-Create chart name and version as used by the chart label.
-*/}}
-{{- define "hami-vgpu.chart" -}}
-{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
-{{- end }}
-
-{{/*
-Common labels
-*/}}
-{{- define "hami-vgpu.labels" -}}
-helm.sh/chart: {{ include "hami-vgpu.chart" . }}
-{{ include "hami-vgpu.selectorLabels" . }}
-{{- if .Chart.AppVersion }}
-app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
-{{- end }}
-app.kubernetes.io/managed-by: {{ .Release.Service }}
-{{- end }}
-
-{{/*
-Selector labels
-*/}}
-{{- define "hami-vgpu.selectorLabels" -}}
-app.kubernetes.io/name: {{ include "hami-vgpu.name" . }}
-app.kubernetes.io/instance: {{ .Release.Name }}
-{{- end }}
-
-
-{{/*
-    Resolve the tag for kubeScheduler.
-*/}}
-{{- define "resolvedKubeSchedulerTag" -}}
-{{- if .Values.scheduler.kubeScheduler.image.tag }}
-{{- .Values.scheduler.kubeScheduler.image.tag | trim -}}
-{{- else }}
-{{- include "strippedKubeVersion" . | trim -}}
-{{- end }}
-{{- end }}
-
-{{/*
-    Return the stripped Kubernetes version string by removing extra parts after semantic version number.
-    v1.31.1+k3s1 -> v1.31.1
-    v1.30.8-eks-2d5f260 -> v1.30.8
-    v1.31.1 -> v1.31.1
-*/}}
-{{- define "strippedKubeVersion" -}}
-{{ regexReplaceAll "^(v[0-9]+\\.[0-9]+\\.[0-9]+)(.*)$" .Capabilities.KubeVersion.Version "$1" }}
-{{- end -}}
-
-{{- define "hami.scheduler.kubeScheduler.image" -}}
-{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.kubeScheduler.image "global" .Values.global "tag" (include "resolvedKubeSchedulerTag" .)) }}
-{{- end -}}
-
-{{- define "hami.scheduler.extender.image" -}}
-{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.extender.image "global" .Values.global "tag" .Values.global.imageTag) }}
-{{- end -}}
-
-{{- define "hami.devicePlugin.image" -}}
-{{ include "common.images.image" (dict "imageRoot" .Values.devicePlugin.image "global" .Values.global "tag" .Values.global.imageTag) }}
-{{- end -}}
-
-{{- define "hami.mockDevicePlugin.image" -}}
-{{ include "common.images.image" (dict "imageRoot" .Values.mockDevicePlugin.image "global" .Values.global "tag" .Values.mockDevicePlugin.tag) }}
-{{- end -}}
-
-{{- define "hami.devicePlugin.monitor.image" -}}
-{{ include "common.images.image" (dict "imageRoot" .Values.devicePlugin.monitor.image "global" .Values.global "tag" .Values.global.imageTag) }}
-{{- end -}}
-
-{{- define "hami.scheduler.patch.image" -}}
-{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.patch.image "global" .Values.global) }}
-{{- end -}}
-
-{{- define "hami.scheduler.patch.new.image" -}}
-{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.patch.imageNew "global" .Values.global) }}
-{{- end -}}
-
-{{- define "hami.scheduler.extender.imagePullSecrets" -}}
-{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.extender.image) "global" .Values.global) }}
-{{- end -}}
-
-{{- define "hami.devicePlugin.imagePullSecrets" -}}
-{{ include "common.images.pullSecrets" (dict "images" (list .Values.devicePlugin.image) "global" .Values.global) }}
-{{- end -}}
-
-{{- define "hami.scheduler.patch.imagePullSecrets" -}}
-{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.patch.image) "global" .Values.global) }}
-{{- end -}}
-
-{{- define "hami.scheduler.patch.new.imagePullSecrets" -}}
-{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.patch.imageNew) "global" .Values.global) }}
-{{- end -}}

packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml

@@ -1,13 +0,0 @@
-{{- if and .Values.devicePlugin.enabled (not .Values.devicePlugin.nodeConfiguration.externalConfigName) -}}
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: {{ include "hami-vgpu.device-plugin" . }}
-  namespace: {{ include "hami-vgpu.namespace" . }}
-  labels:
-    app.kubernetes.io/component: hami-device-plugin
-    {{- include "hami-vgpu.labels" . | nindent 4 }}
-data:
-  config.json: |
-{{- .Values.devicePlugin.nodeConfiguration.config | nindent 4 }}
-{{- end }}

packages/system/hami/charts/hami/templates/device-plugin/daemonsetmock.yaml

@@ -1,55 +0,0 @@
-{{- if .Values.mockDevicePlugin.enabled }}
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: {{ include "hami-vgpu.mock-device-plugin" . }}
-  namespace: {{ include "hami-vgpu.namespace" . }}
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/component: hami-mock-device-plugin
-      {{- include "hami-vgpu.selectorLabels" . | nindent 6 }}
-  template:
-    metadata:
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ""
-      labels:
-        app.kubernetes.io/component: hami-mock-device-plugin
-        {{- include "hami-vgpu.selectorLabels" . | nindent 8 }}
-    spec:
-      serviceAccountName: {{ include "hami-vgpu.mock-device-plugin" . }}
-      tolerations:
-      - key: CriticalAddonsOnly
-        operator: Exists
-      containers:
-      - image: {{ include "hami.mockDevicePlugin.image" . }}
-        imagePullPolicy: {{ .Values.mockDevicePlugin.image.pullPolicy }}
-        name: hami-mock-dp-cntr
-        env:
-          - name: NODE_NAME
-            valueFrom:
-              fieldRef:
-                fieldPath: spec.nodeName
-        command:
-          - ./k8s-device-plugin
-          - -v=5
-          - --device-config-file=/device-config.yaml
-        volumeMounts:
-          - name: dp
-            mountPath: /var/lib/kubelet/device-plugins
-          - name: sys
-            mountPath: /sys
-          - name: device-config
-            mountPath: /device-config.yaml
-            subPath: device-config.yaml
-      volumes:
-        - name: dp
-          hostPath:
-            path: /var/lib/kubelet/device-plugins
-        - name: sys
-          hostPath:
-            path: /sys
-        - name: device-config
-          configMap:
-            name: {{ include "hami-vgpu.scheduler" . }}-device
-{{- end -}}

packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml

@@ -1,262 +0,0 @@
-{{- if .Values.devicePlugin.enabled }}
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: {{ include "hami-vgpu.device-plugin" . }}
-  namespace: {{ include "hami-vgpu.namespace" . }}
-  labels:
-    app.kubernetes.io/component: hami-device-plugin
-    {{- include "hami-vgpu.labels" . | nindent 4 }}
-    {{- with .Values.global.labels }}
-    {{- toYaml . | nindent 4 }}
-    {{- end }}
-  {{- if .Values.global.annotations }}
-  annotations: {{ toYaml .Values.global.annotations | nindent 4}}
-  {{- end }}
-spec:
-  updateStrategy:
-    {{- with .Values.devicePlugin.updateStrategy }}
-    {{- toYaml . | nindent 4 }}
-    {{- end }}
-  selector:
-    matchLabels:
-      app.kubernetes.io/component: hami-device-plugin
-      {{- include "hami-vgpu.selectorLabels" . | nindent 6 }}
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/component: hami-device-plugin
-        hami.io/webhook: ignore
-        {{- include "hami-vgpu.selectorLabels" . | nindent 8 }}
-      annotations:
-        {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
-        checksum/hami-scheduler-newversion-config: {{ include (print $.Template.BasePath "/scheduler/configmapnew.yaml") . | sha256sum }}
-        {{- else }}
-        checksum/hami-scheduler-config: {{ include (print $.Template.BasePath "/scheduler/configmap.yaml") . | sha256sum }}
-        {{- end }}
-        checksum/hami-scheduler-device-config: {{ include (print $.Template.BasePath "/scheduler/device-configmap.yaml") . | sha256sum }}
-      {{- if .Values.devicePlugin.podAnnotations }}
-        {{- toYaml .Values.devicePlugin.podAnnotations | nindent 8 }}
-      {{- end }}
-    spec:
-      {{- if .Values.devicePlugin.runtimeClassName }}
-      runtimeClassName: {{ .Values.devicePlugin.runtimeClassName }}
-      {{- end }}
-      serviceAccountName: {{ include "hami-vgpu.device-plugin" . }}
-      priorityClassName: system-node-critical
-      hostPID: true
-      hostNetwork: true
-      {{- include "hami.devicePlugin.imagePullSecrets" . | nindent 6 }}
-      {{- if .Values.devicePlugin.gpuOperatorToolkitReady.enabled }}
-      initContainers:
-        - name: toolkit-validation
-          image: {{ include "hami.devicePlugin.image" . }}
-          imagePullPolicy: {{ .Values.devicePlugin.image.pullPolicy }}
-          securityContext:
-            privileged: true
-            runAsUser: 0
-          command: ["sh", "-c"]
-          args:
-            - |
-              echo "Waiting for NVIDIA Toolkit to be ready..."
-              until [ -f {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }}/toolkit-ready ]; do 
-                echo "Waiting for {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }}/toolkit-ready..."
-                sleep 5
-              done
-              echo "NVIDIA Toolkit is ready!"
-          volumeMounts:
-            - name: nvidia-validations
-              mountPath: {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath | quote }}
-              mountPropagation: HostToContainer
-              readOnly: true
-      {{- end }}
-      containers:
-        - name: device-plugin
-          image: {{ include "hami.devicePlugin.image" . }}
-          imagePullPolicy: {{ .Values.devicePlugin.image.pullPolicy }}
-          lifecycle:
-            postStart:
-              exec:
-                command: ["/bin/sh","-c", {{ printf "/k8s-vgpu/bin/vgpu-init.sh %s/vgpu/" .Values.global.gpuHookPath | quote }}]
-          command:
-            - nvidia-device-plugin
-            - --config-file=/device-config.yaml
-            - --mig-strategy={{ .Values.devicePlugin.migStrategy }}
-            - --disable-core-limit={{ .Values.devicePlugin.disablecorelimit }}
-            {{- range .Values.devicePlugin.extraArgs }}
-            - {{ . }}
-            {{- end }}
-          env:
-            - name: NODE_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-            - name: NVIDIA_MIG_MONITOR_DEVICES
-              value: all
-            - name: DEVICE_LIST_STRATEGY
-              value: {{ .Values.devicePlugin.deviceListStrategy }}
-            - name: HOOK_PATH
-              value: {{ .Values.global.gpuHookPath }}
-            {{- if typeIs "bool" .Values.devicePlugin.passDeviceSpecsEnabled }}
-            - name: PASS_DEVICE_SPECS
-              value: {{ .Values.devicePlugin.passDeviceSpecsEnabled | quote }}
-            {{- end }}
-            {{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }}
-            - name: NVIDIA_DRIVER_ROOT
-              value: {{ .Values.devicePlugin.nvidiaDriverRoot }}
-            {{- end }}
-            {{- if typeIs "string" .Values.devicePlugin.nvidiaHookPath }}
-            - name: NVIDIA_CDI_HOOK_PATH
-              value: {{ .Values.devicePlugin.nvidiaHookPath }}
-            {{- end }}
-            {{- if typeIs "bool" .Values.devicePlugin.gdrcopyEnabled }}
-            - name: GDRCOPY_ENABLED
-              value: {{ .Values.devicePlugin.gdrcopyEnabled | quote }}
-            {{- end }}
-            {{- if typeIs "bool" .Values.devicePlugin.gdsEnabled }}
-            - name: GDS_ENABLED
-              value: {{ .Values.devicePlugin.gdsEnabled | quote }}
-            {{- end }}
-            {{- if typeIs "bool" .Values.devicePlugin.mofedEnabled }}
-            - name: MOFED_ENABLED
-              value: {{ .Values.devicePlugin.mofedEnabled | quote }}
-            {{- end }}
-            {{- if eq (.Values.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy | default "spread")  "topology-aware" }}
-            - name: ENABLE_TOPOLOGY_SCORE
-              value: "true"
-            {{- end }}
-            {{- with .Values.devicePlugin.extraEnvs }}
-            {{- . | toYaml | nindent 12 }}
-            {{- end }}
-          securityContext:
-            privileged: true
-            allowPrivilegeEscalation: true
-            capabilities:
-              drop: ["ALL"]
-              add: ["SYS_ADMIN"]
-          resources:
-          {{- toYaml .Values.devicePlugin.resources | nindent 12 }}
-          volumeMounts:
-            - name: device-plugin
-              mountPath: /var/lib/kubelet/device-plugins
-            - name: lib
-              mountPath: {{ printf "%s%s" .Values.global.gpuHookPath "/vgpu" }}
-            - name: usrbin
-              mountPath: /usrbin
-            - name: deviceconfig
-              mountPath: /config
-            - name: hosttmp
-              mountPath: /tmp
-            - name: device-config
-              mountPath: /device-config.yaml
-              subPath: device-config.yaml
-            - name: cdi-root
-              mountPath: /var/run/cdi
-            {{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }}
-            # We always mount the driver root at /driver-root in the container.
-            # This is required for CDI detection to work correctly.
-            - name: driver-root
-              mountPath: /driver-root
-              readOnly: true
-            {{- end }}
-        - name: vgpu-monitor
-          image: {{ include "hami.devicePlugin.monitor.image" . }}
-          imagePullPolicy: {{ .Values.devicePlugin.monitor.image.pullPolicy }}
-          command:
-            - "vGPUmonitor"
-            {{- range .Values.devicePlugin.monitor.extraArgs }}
-            - {{ . }}
-            {{- end }}
-          securityContext:
-            allowPrivilegeEscalation: false
-            capabilities:
-              drop: ["ALL"]
-              add: ["SYS_ADMIN"]
-          env:
-            - name: NODE_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-            - name: NVIDIA_VISIBLE_DEVICES
-              value: "all"
-            - name: NVIDIA_MIG_MONITOR_DEVICES
-              value: "all"
-            - name: HOOK_PATH
-              value: "{{ .Values.global.gpuHookPath }}/vgpu"
-            - name: HAMI_RESYNC_INTERVAL
-              value: {{ .Values.devicePlugin.monitor.resyncInterval | default "5m" | quote }}
-            {{- with .Values.devicePlugin.monitor.extraEnvs }}
-            {{- . | toYaml | nindent 12 }}
-            {{- end }}
-          resources:
-          {{- toYaml .Values.devicePlugin.monitor.resources | nindent 12 }}
-          volumeMounts:
-            - name: ctrs
-              mountPath: {{ .Values.devicePlugin.monitor.ctrPath }}
-            - name: dockers
-              mountPath: /run/docker
-            - name: containerds
-              mountPath: /run/containerd
-            - name: sysinfo
-              mountPath: /sysinfo
-            - name: hostvar
-              mountPath: /hostvar
-            - name: hosttmp
-              mountPath: /tmp
-      volumes:
-        - name: ctrs
-          hostPath:
-            path: {{ .Values.devicePlugin.monitor.ctrPath }}
-        - name: hosttmp
-          hostPath:
-            path: /tmp
-        - name: dockers
-          hostPath:
-            path: /run/docker
-        - name: containerds
-          hostPath:
-            path: /run/containerd
-        - name: device-plugin
-          hostPath:
-            path: {{ .Values.devicePlugin.pluginPath }}
-        - name: lib
-          hostPath:
-            path: {{ .Values.devicePlugin.libPath }}
-        {{- if .Values.devicePlugin.gpuOperatorToolkitReady.enabled }}
-        - name: nvidia-validations
-          hostPath:
-            path: {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }}
-            type: DirectoryOrCreate
-        {{- end }}
-        {{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }}
-        - name: driver-root
-          hostPath:
-            path: {{ .Values.devicePlugin.nvidiaDriverRoot }}
-            type: Directory
-        {{- end }}
-        - name: cdi-root
-          hostPath:
-            path: /var/run/cdi
-            type: DirectoryOrCreate
-        - name: usrbin
-          hostPath:
-            path: /usr/bin
-        - name: sysinfo
-          hostPath:
-            path: /sys
-        - name: hostvar
-          hostPath:
-            path: /var
-        - name: deviceconfig
-          configMap:
-            name: {{ .Values.devicePlugin.nodeConfiguration.externalConfigName | default (include "hami-vgpu.device-plugin" .) }}
-        - name: device-config
-          configMap:
-            name: {{ include "hami-vgpu.scheduler" . }}-device
-      {{- if .Values.devicePlugin.nvidiaNodeSelector }}
-      nodeSelector: {{ toYaml .Values.devicePlugin.nvidiaNodeSelector | nindent 8 }}
-      {{- end }}
-      {{- if .Values.devicePlugin.tolerations }}
-      tolerations: {{ toYaml .Values.devicePlugin.tolerations | nindent 8 }}
-      {{- end }}
-{{- end }}

packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml

@@ -1,28 +0,0 @@
-{{- if .Values.devicePlugin.enabled -}}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name:  {{ include "hami-vgpu.device-plugin" . }}-monitor
-rules:
-  - apiGroups:
-      - ""
-    resources:
-      - pods
-    verbs:
-      - get
-      - create
-      - watch
-      - list
-      - update
-      - patch
-  - apiGroups:
-      - ""
-    resources:
-      - nodes
-    verbs:
-      - get
-      - update
-      - list
-      - patch
-{{- end -}}
-

packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml

@@ -1,17 +0,0 @@
-{{- if .Values.devicePlugin.enabled -}}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: {{ include "hami-vgpu.device-plugin" . }}
-  labels:
-    app.kubernetes.io/component: "hami-device-plugin"
-    {{- include "hami-vgpu.labels" . | nindent 4 }}
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: {{ include "hami-vgpu.device-plugin" . }}-monitor
-subjects:
-  - kind: ServiceAccount
-    name: {{ include "hami-vgpu.device-plugin" . }}
-    namespace: {{ include "hami-vgpu.namespace" . }}
-{{- end -}}

packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml

@@ -1,29 +0,0 @@
-{{- if .Values.devicePlugin.enabled -}}
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ include "hami-vgpu.device-plugin" . }}-monitor
-  namespace: {{ include "hami-vgpu.namespace" . }}
-  labels:
-    app.kubernetes.io/component: hami-device-plugin
-    {{- include "hami-vgpu.labels" . | nindent 4 }}
-    {{- with .Values.devicePlugin.service.labels }}
-    {{- toYaml . | nindent 4 }}
-    {{- end }}
-  {{- if .Values.devicePlugin.service.annotations }}  # Use devicePlugin instead of scheduler
-  annotations: {{ toYaml .Values.devicePlugin.service.annotations | nindent 4 }}
-  {{- end }}
-spec:
-  type: {{ .Values.devicePlugin.service.type | default "NodePort" }}  # Default type is NodePort
-  ports:
-    - name: monitorport
-      port: {{ .Values.devicePlugin.service.httpPort | default 31992 }}  # Default HTTP port is 31992
-      targetPort: 9394
-      {{- if eq (.Values.devicePlugin.service.type | default "NodePort") "NodePort" }}  # If type is NodePort, set nodePort
-      nodePort: {{ .Values.devicePlugin.service.httpPort | default 31992 }}
-      {{- end }}
-      protocol: TCP
-  selector:
-    app.kubernetes.io/component: hami-device-plugin
-    {{- include "hami-vgpu.selectorLabels" . | nindent 4 }}
-{{- end -}}

packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml

@@ -1,10 +0,0 @@
-{{- if .Values.devicePlugin.enabled -}}
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: {{ include "hami-vgpu.device-plugin" . }}
-  namespace: {{ include "hami-vgpu.namespace" . }}
-  labels:
-    app.kubernetes.io/component: "hami-device-plugin"
-    {{- include "hami-vgpu.labels" . | nindent 4 }}
-{{- end -}}

packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml

@@ -1,9 +0,0 @@
-{{- if and .Values.devicePlugin.enabled .Values.devicePlugin.createRuntimeClass .Values.devicePlugin.runtimeClassName -}}
-apiVersion: node.k8s.io/v1
-kind: RuntimeClass
-metadata:
-  name: {{ .Values.devicePlugin.runtimeClassName }}
-  annotations:
-    helm.sh/hook: pre-install,pre-upgrade
-handler: nvidia
-{{- end -}}

packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml

@@ -1,31 +0,0 @@
-{{- if .Values.scheduler.admissionWebhook.enabled -}}
-{{- if .Values.scheduler.certManager.enabled }}
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: {{ include "hami-vgpu.scheduler" . }}-serving-cert
-  namespace: {{ include "hami-vgpu.namespace" . }}
-  labels:
-    app.kubernetes.io/component: hami-scheduler
-    {{- include "hami-vgpu.labels" . | nindent 4 }}
-spec:
-  dnsNames:
-    - {{ include "hami-vgpu.scheduler" . }}.{{ include "hami-vgpu.namespace" . }}.svc
-    - {{ include "hami-vgpu.scheduler" . }}.{{ include "hami-vgpu.namespace" . }}.svc.cluster.local
-  issuerRef:
-    kind: Issuer
-    name: {{ include "hami-vgpu.scheduler" . }}-selfsigned-issuer
-  secretName: {{ include "hami-vgpu.scheduler.tls" . }}
----
-apiVersion: cert-manager.io/v1
-kind: Issuer
-metadata:
-  name: {{ include "hami-vgpu.scheduler" . }}-selfsigned-issuer
-  namespace: {{ include "hami-vgpu.namespace" . }}
-  labels:
-    app.kubernetes.io/component: hami-scheduler
-    {{- include "hami-vgpu.labels" . | nindent 4 }}
-spec:
-  selfSigned: {}
-{{- end }}
-{{- end }}