From 51c2ec0ad41722f34d6cb8a5d111ceac021c4464 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Wed, 22 Apr 2026 13:25:16 +0300 Subject: [PATCH 01/27] feat(hami): add HAMi GPU virtualization system chart Add HAMi v2.8.1 (CNCF Sandbox) as a new system package for fractional GPU sharing in tenant Kubernetes clusters. The chart enables workloads to request specific amounts of GPU memory and compute cores instead of claiming entire GPUs. Vendored upstream chart includes device plugin, scheduler extender, mutating webhook, and DRA subchart. Wrapper values configure nvidia RuntimeClass and clear hardcoded node config from upstream defaults. Assisted-By: Claude Signed-off-by: Arsolitt --- packages/system/hami/Chart.yaml | 3 + packages/system/hami/Makefile | 11 + packages/system/hami/charts/hami/Chart.lock | 6 + packages/system/hami/charts/hami/Chart.yaml | 22 + packages/system/hami/charts/hami/README.md | 237 +++++++++ .../charts/hami/charts/hami-dra/.helmignore | 24 + .../charts/hami/charts/hami-dra/Chart.yaml | 18 + .../charts/hami-dra/templates/_commons.tpl | 84 ++++ .../charts/hami-dra/templates/_helpers.tpl | 73 +++ .../hami-dra-driver/deviceclass.yaml | 11 + .../nvidia-dra-driver-daemonset.yaml | 145 ++++++ .../templates/hami-dra-driver/rbac.yaml | 110 ++++ .../monitor/monitor-clusterrole.yaml | 11 + .../monitor/monitor-clusterrolebinding.yaml | 15 + .../templates/monitor/monitor-deployment.yaml | 62 +++ .../templates/monitor/monitor-service.yaml | 26 + .../monitor/monitor-serviceaccount.yaml | 8 + .../templates/webhook/cert-manager.yaml | 29 ++ .../templates/webhook/cert-secret.yaml | 13 + .../templates/webhook/device-config.yaml | 394 +++++++++++++++ .../webhook/mutatingwebhookconfiguration.yaml | 28 ++ .../validatingwebhookconfiguration.yaml | 29 ++ .../webhook/webhook-clusterrole.yaml | 14 + .../webhook/webhook-clusterrolebinding.yaml | 12 + .../templates/webhook/webhook-deployment.yaml | 59 +++ .../templates/webhook/webhook-service.yaml | 15 + .../webhook/webhook-serviceaccount.yaml | 5 + .../charts/hami/charts/hami-dra/values.yaml | 166 ++++++ .../hami/charts/hami/templates/NOTES.txt | 3 + .../hami/charts/hami/templates/_commons.tpl | 49 ++ .../hami/charts/hami/templates/_helpers.tpl | 163 ++++++ .../templates/device-plugin/configmap.yaml | 13 + .../device-plugin/daemonsetmock.yaml | 55 ++ .../device-plugin/daemonsetnvidia.yaml | 262 ++++++++++ .../templates/device-plugin/monitorrole.yaml | 29 ++ .../device-plugin/monitorrolebinding.yaml | 17 + .../device-plugin/monitorservice.yaml | 29 ++ .../device-plugin/monitorserviceaccount.yaml | 10 + .../device-plugin/runtime-class.yaml | 11 + .../hami/templates/scheduler/certmanager.yaml | 31 ++ .../hami/templates/scheduler/clusterrole.yaml | 36 ++ .../scheduler/clusterrolebinding.yaml | 49 ++ .../hami/templates/scheduler/configmap.yaml | 142 ++++++ .../templates/scheduler/configmapnew.yaml | 102 ++++ .../hami/templates/scheduler/deployment.yaml | 227 +++++++++ .../templates/scheduler/device-configmap.yaml | 410 +++++++++++++++ .../scheduler/job-patch/clusterrole.yaml | 30 ++ .../job-patch/clusterrolebinding.yaml | 22 + .../scheduler/job-patch/job-createSecret.yaml | 70 +++ .../scheduler/job-patch/job-patchWebhook.yaml | 65 +++ .../templates/scheduler/job-patch/psp.yaml | 40 ++ .../templates/scheduler/job-patch/role.yaml | 23 + .../scheduler/job-patch/rolebinding.yaml | 23 + .../scheduler/job-patch/serviceaccount.yaml | 15 + .../charts/hami/templates/scheduler/role.yaml | 14 + .../hami/templates/scheduler/rolebinding.yaml | 33 ++ .../hami/templates/scheduler/service.yaml | 36 ++ .../templates/scheduler/serviceaccount.yaml | 18 + .../hami/templates/scheduler/webhook.yaml | 57 +++ packages/system/hami/charts/hami/values.yaml | 476 ++++++++++++++++++ packages/system/hami/values.yaml | 8 + 61 files changed, 4198 insertions(+) create mode 100644 packages/system/hami/Chart.yaml create mode 100644 packages/system/hami/Makefile create mode 100644 packages/system/hami/charts/hami/Chart.lock create mode 100644 packages/system/hami/charts/hami/Chart.yaml create mode 100644 packages/system/hami/charts/hami/README.md create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/.helmignore create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/_commons.tpl create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/_helpers.tpl create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/deviceclass.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/nvidia-dra-driver-daemonset.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/rbac.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrole.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrolebinding.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-deployment.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-service.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-serviceaccount.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-manager.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-secret.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/device-config.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/mutatingwebhookconfiguration.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/validatingwebhookconfiguration.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrole.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrolebinding.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-deployment.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-service.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-serviceaccount.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/values.yaml create mode 100644 packages/system/hami/charts/hami/templates/NOTES.txt create mode 100644 packages/system/hami/charts/hami/templates/_commons.tpl create mode 100644 packages/system/hami/charts/hami/templates/_helpers.tpl create mode 100644 packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml create mode 100644 packages/system/hami/charts/hami/templates/device-plugin/daemonsetmock.yaml create mode 100644 packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml create mode 100644 packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml create mode 100644 packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml create mode 100644 packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml create mode 100644 packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml create mode 100644 packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/configmap.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/deployment.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/role.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/service.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/webhook.yaml create mode 100644 packages/system/hami/charts/hami/values.yaml create mode 100644 packages/system/hami/values.yaml diff --git a/packages/system/hami/Chart.yaml b/packages/system/hami/Chart.yaml new file mode 100644 index 00000000..3fcf4c5d --- /dev/null +++ b/packages/system/hami/Chart.yaml @@ -0,0 +1,3 @@ +apiVersion: v2 +name: cozy-hami +version: 0.0.0 # Placeholder, the actual version will be automatically set during the build process diff --git a/packages/system/hami/Makefile b/packages/system/hami/Makefile new file mode 100644 index 00000000..b7a169c2 --- /dev/null +++ b/packages/system/hami/Makefile @@ -0,0 +1,11 @@ +export NAME=hami +export NAMESPACE=cozy-$(NAME) + +include ../../../hack/common-envs.mk +include ../../../hack/package.mk + +update: + rm -rf charts + helm repo add hami-charts https://project-hami.github.io/HAMi/ + helm repo update hami-charts + helm pull hami-charts/hami --untar --untardir charts diff --git a/packages/system/hami/charts/hami/Chart.lock b/packages/system/hami/charts/hami/Chart.lock new file mode 100644 index 00000000..25856c07 --- /dev/null +++ b/packages/system/hami/charts/hami/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: hami-dra + repository: https://project-hami.github.io/HAMi-DRA/ + version: 0.1.0 +digest: sha256:374551539570bcee82c1c4dea93eac59e6f410b6d5967d188efd630e203d43bb +generated: "2026-04-17T04:06:47.689117678Z" diff --git a/packages/system/hami/charts/hami/Chart.yaml b/packages/system/hami/charts/hami/Chart.yaml new file mode 100644 index 00000000..aba7e95c --- /dev/null +++ b/packages/system/hami/charts/hami/Chart.yaml @@ -0,0 +1,22 @@ +apiVersion: v2 +appVersion: 2.8.1 +dependencies: +- condition: dra.enabled + name: hami-dra + repository: https://project-hami.github.io/HAMi-DRA/ + version: 0.1.0 +description: Heterogeneous AI Computing Virtualization Middleware +keywords: +- vgpu +- gpu +kubeVersion: '>= 1.18.0-0' +maintainers: +- email: archlitchi@gmail.com + name: limengxuan +- email: xiaozhang0210@hotmail.com + name: zhangxiao +name: hami +sources: +- https://github.com/Project-HAMi/HAMi +type: application +version: 2.8.1 diff --git a/packages/system/hami/charts/hami/README.md b/packages/system/hami/charts/hami/README.md new file mode 100644 index 00000000..e8217290 --- /dev/null +++ b/packages/system/hami/charts/hami/README.md @@ -0,0 +1,237 @@ +# HAMi Helm Chart Values Documentation + +This document provides detailed descriptions of all configurable values parameters for the HAMi Helm Chart. + +## Global Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `global.imageRegistry` | Global Docker image registry | `""` | +| `global.imagePullSecrets` | Global Docker image pull secrets | `[]` | +| `global.imageTag` | Image tag | `"v2.8.1"` | +| `global.gpuHookPath` | GPU Hook path | `/usr/local` | +| `global.labels` | Global labels | `{}` | +| `global.annotations` | Global annotations | `{}` | +| `global.managedNodeSelectorEnable` | Whether to enable managed node selector | `false` | +| `global.managedNodeSelector.usage` | Managed node selector usage | `"gpu"` | +| `nameOverride` | Name override | `""` | +| `fullnameOverride` | Full name override | `""` | +| `namespaceOverride` | Namespace override | `""` | + +## Resource Name Configuration + +### NVIDIA GPU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `resourceName` | GPU resource name | `"nvidia.com/gpu"` | +| `resourceMem` | GPU memory resource name | `"nvidia.com/gpumem"` | +| `resourceMemPercentage` | GPU memory percentage resource name | `"nvidia.com/gpumem-percentage"` | +| `resourceCores` | GPU core resource name | `"nvidia.com/gpucores"` | +| `resourcePriority` | GPU priority resource name | `"nvidia.com/priority"` | + +### Cambricon MLU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `mluResourceName` | MLU resource name | `"cambricon.com/vmlu"` | +| `mluResourceMem` | MLU memory resource name | `"cambricon.com/mlu.smlu.vmemory"` | +| `mluResourceCores` | MLU core resource name | `"cambricon.com/mlu.smlu.vcore"` | + +### Hygon DCU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `dcuResourceName` | DCU resource name | `"hygon.com/dcunum"` | +| `dcuResourceMem` | DCU memory resource name | `"hygon.com/dcumem"` | +| `dcuResourceCores` | DCU core resource name | `"hygon.com/dcucores"` | + +### Metax GPU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `metaxResourceName` | GPU resource name | `"metax-tech.com/sgpu"` | +| `metaxResourceCore` | GPU core resource name | `"metax-tech.com/vcore"` | +| `metaxResourceMem` | GPU memory resource name | `"metax-tech.com/vmemory"` | +| `metaxsGPUTopologyAware` | GPU topology awareness | `"false"` | + +### Enflame GCU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `enflameResourceNameVGCU` | vGCU resource name | `"enflame.com/vgcu"` | +| `enflameResourceNameVGCUPercentage` | vGCU percentage resource name | `"enflame.com/vgcu-percentage"` | + +### Kunlunxin XPU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `kunlunResourceName` | XPU resource name | `"kunlunxin.com/xpu"` | + +## Scheduler Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `schedulerName` | Scheduler name | `"hami-scheduler"` | +| `scheduler.nodeName` | Define node name, scheduler will schedule to this node | `""` | +| `scheduler.overwriteEnv` | Whether to overwrite environment variables | `"false"` | +| `scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy` | Node scheduler policy | `binpack` | +| `scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy` | GPU scheduler policy | `spread` | +| `scheduler.metricsBindAddress` | Metrics bind address | `":9395"` | +| `scheduler.forceOverwriteDefaultScheduler` | Whether to force overwrite default scheduler | `true` | +| `scheduler.livenessProbe` | Whether to enable liveness probe | `false` | +| `scheduler.leaderElect` | Whether to enable leader election | `true` | +| `scheduler.replicas` | Number of replicas | `1` | + +### Kube Scheduler Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `scheduler.kubeScheduler.enabled` | Whether to run kube-scheduler container in scheduler pod | `true` | +| `scheduler.kubeScheduler.image.registry` | Kube scheduler image registry | `"registry.cn-hangzhou.aliyuncs.com"` | +| `scheduler.kubeScheduler.image.repository` | Kube scheduler image repository | `"google_containers/kube-scheduler"` | +| `scheduler.kubeScheduler.image.tag` | Kube scheduler image tag | `""` | +| `scheduler.kubeScheduler.image.pullPolicy` | Kube scheduler image pull policy | `IfNotPresent` | +| `scheduler.kubeScheduler.image.pullSecrets` | Kube scheduler image pull secrets | `[]` | +| `scheduler.kubeScheduler.extraNewArgs` | Extra new arguments | `["--config=/config/config.yaml", "-v=4"]` | +| `scheduler.kubeScheduler.extraArgs` | Extra arguments | `["--policy-config-file=/config/config.json", "-v=4"]` | + +### Extender Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `scheduler.extender.image.registry` | Scheduler extender image registry | `"docker.io"` | +| `scheduler.extender.image.repository` | Scheduler extender image repository | `"projecthami/hami"` | +| `scheduler.extender.image.tag` | Scheduler extender image tag | `""` | +| `scheduler.extender.image.pullPolicy` | Scheduler extender image pull policy | `IfNotPresent` | +| `scheduler.extender.image.pullSecrets` | Scheduler extender image pull secrets | `[]` | +| `scheduler.extender.extraArgs` | Scheduler extender extra arguments | `["--debug", "-v=4"]` | + +### Admission Webhook Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `scheduler.admissionWebhook.enabled` | Whether to enable admission webhook | `true` | +| `scheduler.admissionWebhook.customURL.enabled` | Whether to enable custom URL | `false` | +| `scheduler.admissionWebhook.customURL.host` | Custom URL host | `127.0.0.1` | +| `scheduler.admissionWebhook.customURL.port` | Custom URL port | `31998` | +| `scheduler.admissionWebhook.customURL.path` | Custom URL path | `/webhook` | +| `scheduler.admissionWebhook.reinvocationPolicy` | Reinvocation policy | `Never` | +| `scheduler.admissionWebhook.failurePolicy` | Failure policy | `Ignore` | + +### TLS Certificate Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `scheduler.certManager.enabled` | Whether to use cert-manager to generate self-signed certificates | `false` | +| `scheduler.patch.enabled` | Whether to use kube-webhook-certgen to generate self-signed certificates | `true` | +| `scheduler.patch.image.registry` | Certgen image registry | `"docker.io"` | +| `scheduler.patch.image.repository` | Certgen image repository | `"jettech/kube-webhook-certgen"` | +| `scheduler.patch.image.tag` | Certgen image tag | `"v1.5.2"` | +| `scheduler.patch.image.pullPolicy` | Certgen image pull policy | `IfNotPresent` | +| `scheduler.patch.imageNew.registry` | New certgen image registry | `"docker.io"` | +| `scheduler.patch.imageNew.repository` | New certgen image repository | `"liangjw/kube-webhook-certgen"` | +| `scheduler.patch.imageNew.tag` | New certgen image tag | `"v1.1.1"` | + +### Scheduler Service Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `scheduler.service.type` | Service type | `NodePort` | +| `scheduler.service.httpPort` | HTTP port | `443` | +| `scheduler.service.schedulerPort` | Scheduler NodePort | `31998` | +| `scheduler.service.monitorPort` | Monitor port | `31993` | +| `scheduler.service.monitorTargetPort` | Monitor target port | `9395` | + +## Device Plugin Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devicePlugin.image.registry` | Device plugin image registry | `"docker.io"` | +| `devicePlugin.image.repository` | Device plugin image repository | `"projecthami/hami"` | +| `devicePlugin.image.tag` | Device plugin image tag | `""` | +| `devicePlugin.image.pullPolicy` | Device plugin image pull policy | `IfNotPresent` | +| `devicePlugin.image.pullSecrets` | Device plugin image pull secrets | `[]` | + +### Monitor Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devicePlugin.monitor.image.registry` | Monitor image registry | `"docker.io"` | +| `devicePlugin.monitor.image.repository` | Monitor image repository | `"projecthami/hami"` | +| `devicePlugin.monitor.image.tag` | Monitor image tag | `""` | +| `devicePlugin.monitor.image.pullPolicy` | Monitor image pull policy | `IfNotPresent` | +| `devicePlugin.monitor.image.pullSecrets` | Monitor image pull secrets | `[]` | +| `devicePlugin.monitor.ctrPath` | Container path | `/usr/local/vgpu/containers` | +| `devicePlugin.monitor.extraArgs` | Monitor extra arguments | `["-v=4"]` | +| `devicePlugin.monitor.extraEnvs` | Monitor extra environments | `{}` | + +### Device Plugin Other Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devicePlugin.deviceSplitCount` | Integer type, default value: 10. Maximum number of tasks assigned to a single GPU device | `10` | +| `devicePlugin.deviceMemoryScaling` | Device memory scaling ratio | `1` | +| `devicePlugin.deviceCoreScaling` | Device core scaling ratio | `1` | +| `devicePlugin.runtimeClassName` | Runtime class name | `""` | +| `devicePlugin.createRuntimeClass` | Whether to create runtime class | `false` | +| `devicePlugin.migStrategy` | String type, "none" means ignore MIG functionality, "mixed" means allocate MIG devices through independent resources | `"none"` | +| `devicePlugin.disablecorelimit` | String type, "true" means disable core limit, "false" means enable core limit | `"false"` | +| `devicePlugin.passDeviceSpecsEnabled` | Whether to enable passing device specs | `false` | +| `devicePlugin.extraArgs` | Device plugin extra arguments | `["-v=4"]` | +| `devicePlugin.nodeConfiguration.config` | Node configuration for device plugin by json | An example of default configuration. | +| `devicePlugin.nodeConfiguration.externalConfigName` | Node configuration for device plugin by external congimap | `""` | +| `devicePlugin.extraEnvs` | Device plugin extra environments | `{}` | + +### Device Plugin Service Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devicePlugin.service.type` | Service type | `NodePort` | +| `devicePlugin.service.httpPort` | HTTP port | `31992` | + +### Device Plugin Deployment Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devicePlugin.pluginPath` | Plugin path | `/var/lib/kubelet/device-plugins` | +| `devicePlugin.libPath` | Library path | `/usr/local/vgpu` | +| `devicePlugin.nvidiaNodeSelector` | NVIDIA node selector | `{"gpu": "on"}` | +| `devicePlugin.updateStrategy.type` | Update strategy type | `RollingUpdate` | +| `devicePlugin.updateStrategy.rollingUpdate.maxUnavailable` | Maximum unavailable count | `1` | + +## Device Configuration + +### AWS Neuron +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.awsneuron.customresources` | Custom resources | `["aws.amazon.com/neuron", "aws.amazon.com/neuroncore"]` | + +### Kunlunxin +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.kunlun.enabled` | Whether to enable | `true` | +| `devices.kunlun.customresources` | Custom resources | `["kunlunxin.com/xpu"]` | + +### Mthreads +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.mthreads.enabled` | Whether to enable | `true` | +| `devices.mthreads.customresources` | Custom resources | `["mthreads.com/vgpu"]` | + +### NVIDIA +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.nvidia.gpuCorePolicy` | GPU core policy | `default` | +| `devices.nvidia.libCudaLogLevel` | CUDA library log level | `1` | + +### Huawei Ascend +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.ascend.enabled` | Whether to enable | `false` | +| `devices.ascend.image` | Image | `""` | +| `devices.ascend.imagePullPolicy` | Image pull policy | `IfNotPresent` | +| `devices.ascend.extraArgs` | Extra arguments | `[]` | +| `devices.ascend.nodeSelector` | Node selector | `{"ascend": "on"}` | +| `devices.ascend.tolerations` | Tolerations | `[]` | +| `devices.ascend.customresources` | Custom resources | `["huawei.com/Ascend910A", "huawei.com/Ascend910A-memory", ...]` | + +### Iluvatar +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.iluvatar.enabled` | Whether to enable | `false` | +| `devices.iluvatar.customresources` | Custom resources | `["iluvatar.ai/BI-V150-vgpu", "iluvatar.ai/BI-V150.vMem","iluvatar.ai/BI-V150.vCore", ...]` | diff --git a/packages/system/hami/charts/hami/charts/hami-dra/.helmignore b/packages/system/hami/charts/hami/charts/hami-dra/.helmignore new file mode 100644 index 00000000..898df488 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/.helmignore @@ -0,0 +1,24 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ + diff --git a/packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml b/packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml new file mode 100644 index 00000000..d01f678f --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml @@ -0,0 +1,18 @@ +apiVersion: v2 +appVersion: 0.1.0 +description: A Helm chart for HAMi DRA +home: https://github.com/Project-HAMi/HAMi-DRA +keywords: +- webhook +- kubernetes +- admission-controller +- hami +- dra +maintainers: +- name: hami-dra + url: https://github.com/Project-HAMi/HAMi-DRA +name: hami-dra +sources: +- https://github.com/Project-HAMi/HAMi-DRA +type: application +version: 0.1.0 diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/_commons.tpl b/packages/system/hami/charts/hami/charts/hami-dra/templates/_commons.tpl new file mode 100644 index 00000000..02fbed20 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/_commons.tpl @@ -0,0 +1,84 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* +Return the proper image name +{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" $) }} +*/}} +{{- define "common.images.image" -}} +{{- $registryName := .imageRoot.registry -}} +{{- $repositoryName := .imageRoot.repository -}} +{{- $tag := .imageRoot.tag | toString -}} +{{- if .global }} + {{- if .global.imageRegistry }} + {{- $registryName = .global.imageRegistry -}} + {{- end -}} +{{- end -}} +{{- if .tag }} + {{- if .tag.imageTag }} + {{- $tag = .tag.imageTag -}} + {{- end -}} +{{- end -}} +{{- if $registryName }} +{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- else -}} +{{- printf "%s:%s" $repositoryName $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead) +{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }} +*/}} +{{- define "common.images.pullSecrets" -}} + {{- $pullSecrets := list }} + + {{- if .global }} + {{- range .global.imagePullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- range .images -}} + {{- range .pullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- if (not (empty $pullSecrets)) }} +imagePullSecrets: + {{- range $pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end -}} + +{{/* +Renders a value that contains template. +Usage: +{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $) }} +*/}} +{{- define "common.tplvalues.render" -}} + {{- if typeIs "string" .value }} + {{- tpl .value .context }} + {{- else }} + {{- tpl (.value | toYaml) .context }} + {{- end }} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "common.names.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/_helpers.tpl b/packages/system/hami/charts/hami/charts/hami-dra/templates/_helpers.tpl new file mode 100644 index 00000000..99beec77 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/_helpers.tpl @@ -0,0 +1,73 @@ +{{- define "hami.dra.webhook.fullname" -}} +{{- printf "%s-%s" (include "common.names.fullname" .) "webhook" | trunc 63 | trimSuffix "-" -}} +{{- end }} + +{{- define "hami.dra.webhook.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.webhook.image "global" .Values.global) }} +{{- end -}} + +{{- define "hami.dra.webhook.imagePullSecrets" -}} +{{ include "common.images.pullSecrets" (dict "images" (list .Values.webhook.image) "global" .Values.global) }} +{{- end -}} + +{{- define "hami.dra.driver.nvidia.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.drivers.nvidia.image "global" .Values.global) }} +{{- end -}} + +{{- define "hami.dra.driver.nvidia.imagePullSecrets" -}} +{{ include "common.images.pullSecrets" (dict "images" (list .Values.drivers.nvidia.image) "global" .Values.global) }} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "hami-dra-webhook.labels" -}} +helm.sh/chart: {{ .Chart.Name }} +{{ include "hami-dra-webhook.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "hami-dra-webhook.selectorLabels" -}} +app.kubernetes.io/name: {{ .Release.Name }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: webhook +{{- end }} + +{{- define "hami.dra.monitor.fullname" -}} +{{- printf "%s-%s" (include "common.names.fullname" .) "monitor" | trunc 63 | trimSuffix "-" -}} +{{- end }} + +{{- define "hami.dra.monitor.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.monitor.image "global" .Values.global) }} +{{- end -}} + +{{- define "hami.dra.monitor.imagePullSecrets" -}} +{{ include "common.images.pullSecrets" (dict "images" (list .Values.monitor.image) "global" .Values.global) }} +{{- end -}} + +{{/* +Common labels for monitor +*/}} +{{- define "hami-dra-monitor.labels" -}} +helm.sh/chart: {{ .Chart.Name }} +{{ include "hami-dra-monitor.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels for monitor +*/}} +{{- define "hami-dra-monitor.selectorLabels" -}} +app.kubernetes.io/name: {{ .Release.Name }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: monitor +{{- end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/deviceclass.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/deviceclass.yaml new file mode 100644 index 00000000..bcb6d2c5 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/deviceclass.yaml @@ -0,0 +1,11 @@ +{{- if .Values.drivers.nvidia.enabled }} +apiVersion: resource.k8s.io/v1 +kind: DeviceClass +metadata: + name: hami-core-gpu.project-hami.io +spec: + selectors: + - cel: + expression: |- + device.driver == "hami-core-gpu.project-hami.io" && device.attributes["hami-core-gpu.project-hami.io"].type == "hami-gpu" +{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/nvidia-dra-driver-daemonset.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/nvidia-dra-driver-daemonset.yaml new file mode 100644 index 00000000..a3fb0d01 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/nvidia-dra-driver-daemonset.yaml @@ -0,0 +1,145 @@ +{{- if .Values.drivers.nvidia.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: hami-dra-driver-kubelet-plugin + namespace: {{ .Release.Namespace }} +spec: + revisionHistoryLimit: 10 + selector: + matchLabels: + hami-dra-driver-component: kubelet-plugin + template: + metadata: + labels: + app.kubernetes.io/instance: hami-dra-driver + app.kubernetes.io/name: hami-dra-driver + hami-dra-driver-component: kubelet-plugin + spec: + {{- include "hami.dra.driver.nvidia.imagePullSecrets" . | nindent 6 }} + initContainers: + - name: init-container + image: {{ include "hami.dra.driver.nvidia.image" . }} + securityContext: + privileged: true + command: [bash, /usr/bin/kubelet-plugin-prestart.sh] + env: + - name: NVIDIA_DRIVER_ROOT + value: {{ if .Values.drivers.nvidia.containerDriver }}/run/nvidia/driver{{ else }}/{{ end }} + - name: NVIDIA_VISIBLE_DEVICES + value: void + - name: KUBELET_REGISTRAR_DIRECTORY_PATH + value: /var/lib/kubelet/plugins_registry + - name: KUBELET_PLUGINS_DIRECTORY_PATH + value: /var/lib/kubelet/plugins + volumeMounts: + - name: driver-root-parent + mountPath: /driver-root-parent + readOnly: true + containers: + - args: + - |- + # Conditionally mask the params file to prevent this container from + # recreating any missing GPU device nodes. This is necessary, for + # example, when running under nvkind to limit the set GPUs governed + # by the plugin even though it has cgroup access to all of them. + if [ "${MASK_NVIDIA_DRIVER_PARAMS}" = "true" ]; then + cp /proc/driver/nvidia/params root/gpu-params + sed -i 's/^ModifyDeviceFiles: 1$/ModifyDeviceFiles: 0/' root/gpu-params + mount --bind root/gpu-params /proc/driver/nvidia/params + fi + hami-kubelet-plugin -v 6 + command: + - bash + - -c + env: + - name: MASK_NVIDIA_DRIVER_PARAMS + value: "" + - name: NVIDIA_DRIVER_ROOT + value: {{ if .Values.drivers.nvidia.containerDriver }}/run/nvidia/driver{{ else }}/{{ end }} + - name: NVIDIA_VISIBLE_DEVICES + value: void + - name: CDI_ROOT + value: /var/run/cdi + - name: NVIDIA_MIG_CONFIG_DEVICES + value: all + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: KUBELET_REGISTRAR_DIRECTORY_PATH + value: /var/lib/kubelet/plugins_registry + - name: KUBELET_PLUGINS_DIRECTORY_PATH + value: /var/lib/kubelet/plugins + - name: IMAGE_NAME + value: {{ include "hami.dra.driver.nvidia.image" . }} + lifecycle: + postStart: + exec: + command: ["/bin/bash", "-c", "/usr/bin/vgpu-init.sh /usr/local/vgpu/"] + image: {{ include "hami.dra.driver.nvidia.image" . }} + imagePullPolicy: {{ .Values.drivers.nvidia.image.pullPolicy | default "IfNotPresent" }} + name: gpus + securityContext: + privileged: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /var/lib/kubelet/plugins_registry + name: plugins-registry + - mountPath: /var/lib/kubelet/plugins + mountPropagation: Bidirectional + name: plugins + - mountPath: /var/run/cdi + name: cdi + - mountPath: /driver-root + mountPropagation: HostToContainer + name: driver-root + readOnly: true + - mountPath: /usr/local/vgpu + name: host-vgpu + - mountPath: /tmp + name: host-tmp + dnsPolicy: ClusterFirst + serviceAccount: hami-dra-driver-service-account + serviceAccountName: hami-dra-driver-service-account + volumes: + - hostPath: + path: /var/lib/kubelet/plugins_registry + type: "" + name: plugins-registry + - hostPath: + path: /var/lib/kubelet/plugins + type: "" + name: plugins + - hostPath: + path: /var/run/cdi + type: "" + name: cdi + - hostPath: + path: {{if .Values.drivers.nvidia.containerDriver }}/run/nvidia{{ else }}/{{ end }} + type: DirectoryOrCreate + name: driver-root-parent + - hostPath: + path: {{if .Values.drivers.nvidia.containerDriver }}/run/nvidia/driver{{ else }}/{{ end }} + type: DirectoryOrCreate + name: driver-root + - hostPath: + path: /dev + type: "" + name: host-dev + - hostPath: + path: /usr/local/vgpu + type: DirectoryOrCreate + name: host-vgpu + - hostPath: + path: /tmp + type: "" + name: host-tmp +{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/rbac.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/rbac.yaml new file mode 100644 index 00000000..435c37cc --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/rbac.yaml @@ -0,0 +1,110 @@ +{{- if .Values.drivers.nvidia.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hami-dra-driver-service-account + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +items: +- apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: hami-dra-driver-role + namespace: {{ .Release.Namespace }} + rules: + - apiGroups: + - apps + resources: + - daemonsets + - deployments + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +kind: List +metadata: + resourceVersion: "" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: hami-dra-driver-role-binding + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: hami-dra-driver-role +subjects: +- kind: ServiceAccount + name: hami-dra-driver-service-account + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: hami-dra-driver-clusterrole +rules: +- apiGroups: + - resource.k8s.io + resources: + - resourceclaims + verbs: + - get + - list + - watch +- apiGroups: + - resource.k8s.io + resources: + - resourceclaimtemplates + verbs: + - get + - list + - watch + - create + - update + - delete +- apiGroups: + - resource.k8s.io + resources: + - resourceslices + verbs: + - get + - list + - watch + - create + - update + - delete +- apiGroups: + - resource.k8s.io + resources: + - resourceclaims/status + verbs: + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: hami-dra-driver-clusterrole-binding-hami-dra-driver +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hami-dra-driver-clusterrole +subjects: +- kind: ServiceAccount + name: hami-dra-driver-service-account + namespace: {{ .Release.Namespace }} +{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrole.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrole.yaml new file mode 100644 index 00000000..45c24863 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrole.yaml @@ -0,0 +1,11 @@ +{{- if .Values.monitor.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "hami.dra.monitor.fullname" . }} +rules: +- apiGroups: ["resource.k8s.io"] + resources: ["resourceslices", "resourceclaims"] + verbs: ["get", "list", "watch"] +{{- end }} + diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrolebinding.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrolebinding.yaml new file mode 100644 index 00000000..766d3098 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrolebinding.yaml @@ -0,0 +1,15 @@ +{{- if .Values.monitor.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami.dra.monitor.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "hami.dra.monitor.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "hami.dra.monitor.fullname" . }} + namespace: {{ .Release.Namespace }} +{{- end }} + diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-deployment.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-deployment.yaml new file mode 100644 index 00000000..b544b376 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-deployment.yaml @@ -0,0 +1,62 @@ +{{- if .Values.monitor.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "hami.dra.monitor.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "hami.dra.monitor.fullname" . }} + {{- include "hami-dra-monitor.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.monitor.replicas | default 1 }} + selector: + matchLabels: + {{- include "hami-dra-monitor.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "hami-dra-monitor.selectorLabels" . | nindent 8 }} + spec: + {{- include "hami.dra.monitor.imagePullSecrets" . | nindent 6 }} + serviceAccountName: {{ include "hami.dra.monitor.fullname" . }} + containers: + - name: monitor + image: {{ include "hami.dra.monitor.image" . }} + imagePullPolicy: {{ .Values.monitor.image.pullPolicy | default "IfNotPresent" }} + command: + - /bin/monitor + - --v={{ .Values.monitor.logLevel | default 2 }} + - --metrics-bind-address={{ .Values.monitor.metricsBindAddress | default ":8080" }} + - --health-probe-bind-address={{ .Values.monitor.healthProbeBindAddress | default ":8000" }} + - --kube-api-qps={{ .Values.monitor.kubeAPIQPS | default 40.0 }} + - --kube-api-burst={{ .Values.monitor.kubeAPIBurst | default 60 }} + - --collect-interval={{ .Values.monitor.collectInterval | default "30s" }} + ports: + - containerPort: 8080 + name: metrics + protocol: TCP + - containerPort: 8000 + name: health + protocol: TCP + {{- if .Values.monitor.resources }} + resources: + {{- toYaml .Values.monitor.resources | nindent 12 }} + {{- end }} + livenessProbe: + httpGet: + path: /healthz + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8000 + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 +{{- end }} + diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-service.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-service.yaml new file mode 100644 index 00000000..d740cc38 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-service.yaml @@ -0,0 +1,26 @@ +{{- if .Values.monitor.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "hami.dra.monitor.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "hami-dra-monitor.selectorLabels" . | nindent 4 }} +spec: + type: {{ .Values.monitor.service.type | default "ClusterIP" }} + selector: + {{- include "hami-dra-monitor.selectorLabels" . | nindent 4 }} + ports: + - port: 8080 + targetPort: 8080 + name: metrics + protocol: TCP + {{- if and (eq (.Values.monitor.service.type | default "ClusterIP") "NodePort") .Values.monitor.service.nodePort.metrics }} + nodePort: {{ .Values.monitor.service.nodePort.metrics }} + {{- end }} + - port: 8000 + targetPort: 8000 + name: health + protocol: TCP +{{- end }} + diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-serviceaccount.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-serviceaccount.yaml new file mode 100644 index 00000000..c61ad454 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-serviceaccount.yaml @@ -0,0 +1,8 @@ +{{- if .Values.monitor.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "hami.dra.monitor.fullname" . }} + namespace: {{ .Release.Namespace }} +{{- end }} + diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-manager.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-manager.yaml new file mode 100644 index 00000000..dd31c71d --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-manager.yaml @@ -0,0 +1,29 @@ +{{- if .Values.certs.certManager.enabled }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ .Release.Name }}-dra-webhook-serving-cert + namespace: {{ .Release.Namespace }} + labels: + {{- include "hami-dra-webhook.labels" . | nindent 4 }} +spec: + dnsNames: + - {{ .Release.Name }}-dra-webhook.{{ .Release.Namespace }}.svc + - {{ .Release.Name }}-dra-webhook.{{ .Release.Namespace }}.svc.cluster.local + issuerRef: + kind: Issuer + name: {{ .Release.Name }}-dra-webhook-selfsigned-issuer + secretName: {{ .Release.Name }}-dra-webhook-tls + privateKey: + rotationPolicy: {{ .Values.certs.certManager.privateKeyRotationPolicy | default "Never" }} +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ .Release.Name }}-dra-webhook-selfsigned-issuer + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/component: hami-dra-webhook +spec: + selfSigned: {} +{{- end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-secret.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-secret.yaml new file mode 100644 index 00000000..eadbb517 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-secret.yaml @@ -0,0 +1,13 @@ +{{- if eq .Values.certs.mode "custom" }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "hami-dra-webhook.tlsSecretName" . }} + namespace: {{ .Release.Namespace }} +type: kubernetes.io/tls +data: + tls.crt: | + {{- .Values.certs.custom.crt | b64enc | indent 4 }} + tls.key: | + {{- .Values.certs.custom.key | b64enc | indent 4 }} +{{- end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/device-config.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/device-config.yaml new file mode 100644 index 00000000..97283f3e --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/device-config.yaml @@ -0,0 +1,394 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "hami.dra.webhook.fullname" . }}-device-config + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-dra-webhook.selectorLabels" . | nindent 4 }} +data: + device-config.yaml: |- + {{- if .Files.Glob "files/device-config.yaml" }} + {{- .Files.Get "files/device-config.yaml" | nindent 4}} + {{- else }} + nvidia: + resourceCountName: {{ .Values.resourceName }} + resourceMemoryName: {{ .Values.resourceMem }} + resourceMemoryPercentageName: {{ .Values.resourceMemPercentage }} + resourceCoreName: {{ .Values.resourceCores }} + resourcePriorityName: {{ .Values.resourcePriority }} + overwriteEnv: false + defaultMemory: 0 + defaultCores: 0 + defaultGPUNum: 1 + knownMigGeometries: + - models: [ "A30" ] + allowedGeometries: + - + - name: 1g.6gb + core: 25 + memory: 6144 + count: 4 + - + - name: 2g.12gb + core: 50 + memory: 12288 + count: 2 + - + - name: 4g.24gb + core: 100 + memory: 24576 + count: 1 + - models: [ "A100-SXM4-40GB", "A100-40GB-PCIe", "A100-PCIE-40GB"] + allowedGeometries: + - + - name: 1g.5gb + core: 14 + memory: 5120 + count: 7 + - + - name: 1g.5gb + core: 14 + memory: 5120 + count: 1 + - name: 2g.10gb + core: 28 + memory: 10240 + count: 3 + - + - name: 3g.20gb + core: 42 + memory: 20480 + count: 2 + - + - name: 7g.40gb + core: 100 + memory: 40960 + count: 1 + - models: [ "A100-SXM4-80GB", "A100-80GB-PCIe", "A100-PCIE-80GB"] + allowedGeometries: + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 7 + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 1 + - name: 2g.20gb + core: 28 + memory: 20480 + count: 3 + - + - name: 3g.40gb + core: 42 + memory: 40960 + count: 2 + - + - name: 7g.79gb + core: 100 + memory: 80896 + count: 1 + - models: [ "H100-PCIE-80GB", "H100-SXM5-80GB"] + allowedGeometries: + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 7 + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 1 + - name: 2g.20gb + core: 28 + memory: 20480 + count: 3 + - + - name: 3g.40gb + core: 42 + memory: 40960 + count: 2 + - + - name: 7g.80gb + core: 100 + memory: 81920 + count: 1 + - models: [ "H100-PCIE-94GB", "H100-SXM5-94GB"] + allowedGeometries: + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 7 + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 1 + - name: 2g.24gb + core: 28 + memory: 24576 + count: 3 + - + - name: 3g.47gb + core: 42 + memory: 48128 + count: 2 + - + - name: 7g.94gb + core: 100 + memory: 96256 + count: 1 + - models: [ "H20", "H100 on GH200"] + allowedGeometries: + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 7 + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 1 + - name: 2g.24gb + core: 28 + memory: 24576 + count: 3 + - + - name: 3g.48gb + core: 42 + memory: 49152 + count: 2 + - + - name: 7g.96gb + core: 100 + memory: 98304 + count: 1 + - models: [ "H200 NVL", "H200-SXM5"] + allowedGeometries: + - + - name: 1g.18gb + core: 14 + memory: 18432 + count: 7 + - + - name: 1g.18gb + core: 14 + memory: 18432 + count: 1 + - name: 2g.35gb + core: 28 + memory: 35840 + count: 3 + - + - name: 3g.71gb + core: 42 + memory: 72704 + count: 2 + - + - name: 7g.141gb + core: 100 + memory: 144384 + count: 1 + - models: [ "B200" ] + allowedGeometries: + - + - name: 1g.23gb + core: 14 + memory: 23552 + count: 7 + - + - name: 1g.23gb + core: 14 + memory: 23552 + count: 1 + - name: 2g.45gb + core: 28 + memory: 46080 + count: 3 + - + - name: 3g.90gb + core: 42 + memory: 92160 + count: 2 + - + - name: 7g.180gb + core: 100 + memory: 184320 + count: 1 + cambricon: + resourceCountName: {{ .Values.mluResourceName }} + resourceMemoryName: {{ .Values.mluResourceMem }} + resourceCoreName: {{ .Values.mluResourceCores }} + hygon: + resourceCountName: {{ .Values.dcuResourceName }} + resourceMemoryName: {{ .Values.dcuResourceMem }} + resourceCoreName: {{ .Values.dcuResourceCores }} + metax: + resourceCountName: "metax-tech.com/gpu" + resourceVCountName: {{ .Values.metaxResourceName }} + resourceVMemoryName: {{ .Values.metaxResourceMem }} + resourceVCoreName: {{ .Values.metaxResourceCore }} + sgpuTopologyAware: {{ .Values.metaxsGPUTopologyAware }} + enflame: + resourceNameGCU: "enflame.com/gcu" + resourceNameVGCU: {{ .Values.enflameResourceNameVGCU }} + resourceNameVGCUPercentage: {{ .Values.enflameResourceNameVGCUPercentage }} + mthreads: + resourceCountName: "mthreads.com/vgpu" + resourceMemoryName: "mthreads.com/sgpu-memory" + resourceCoreName: "mthreads.com/sgpu-core" + iluvatars: + - chipName: MR-V100 + commonWord: MR-V100 + resourceCountName: iluvatar.ai/MR-V100-vgpu + resourceMemoryName: iluvatar.ai/MR-V100.vMem + resourceCoreName: iluvatar.ai/MR-V100.vCore + - chipName: MR-V50 + commonWord: MR-V50 + resourceCountName: iluvatar.ai/MR-V50-vgpu + resourceMemoryName: iluvatar.ai/MR-V50.vMem + resourceCoreName: iluvatar.ai/MR-V50.vCore + - chipName: BI-V150 + commonWord: BI-V150 + resourceCountName: iluvatar.ai/BI-V150-vgpu + resourceMemoryName: iluvatar.ai/BI-V150.vMem + resourceCoreName: iluvatar.ai/BI-V150.vCore + - chipName: BI-V100 + commonWord: BI-V100 + resourceCountName: iluvatar.ai/BI-V100-vgpu + resourceMemoryName: iluvatar.ai/BI-V100.vMem + resourceCoreName: iluvatar.ai/BI-V100.vCore + kunlun: + resourceCountName: {{ .Values.kunlunResourceName }} + resourceVCountName: {{ .Values.kunlunResourceVCountName }} + resourceVMemoryName: {{ .Values.kunlunResourceVMemoryName }} + awsneuron: + resourceCountName: "aws.amazon.com/neuron" + resourceCoreName: "aws.amazon.com/neuroncore" + amd: + resourceCountName: "amd.com/gpu" + vnpus: + - chipName: 910A + commonWord: Ascend910A + resourceName: huawei.com/Ascend910A + resourceMemoryName: huawei.com/Ascend910A-memory + memoryAllocatable: 32768 + memoryCapacity: 32768 + aiCore: 30 + templates: + - name: vir02 + memory: 2184 + aiCore: 2 + - name: vir04 + memory: 4369 + aiCore: 4 + - name: vir08 + memory: 8738 + aiCore: 8 + - name: vir16 + memory: 17476 + aiCore: 16 + - chipName: 910B2 + commonWord: Ascend910B2 + resourceName: huawei.com/Ascend910B2 + resourceMemoryName: huawei.com/Ascend910B2-memory + memoryAllocatable: 65536 + memoryCapacity: 65536 + aiCore: 24 + aiCPU: 6 + templates: + - name: vir03_1c_8g + memory: 8192 + aiCore: 3 + aiCPU: 1 + - name: vir06_1c_16g + memory: 16384 + aiCore: 6 + aiCPU: 1 + - name: vir12_3c_32g + memory: 32768 + aiCore: 12 + aiCPU: 3 + - chipName: 910B3 + commonWord: Ascend910B3 + resourceName: huawei.com/Ascend910B3 + resourceMemoryName: huawei.com/Ascend910B3-memory + memoryAllocatable: 65536 + memoryCapacity: 65536 + aiCore: 20 + aiCPU: 7 + templates: + - name: vir05_1c_16g + memory: 16384 + aiCore: 5 + aiCPU: 1 + - name: vir10_3c_32g + memory: 32768 + aiCore: 10 + aiCPU: 3 + - chipName: 910B4-1 + commonWord: Ascend910B4-1 + resourceName: huawei.com/Ascend910B4-1 + resourceMemoryName: huawei.com/Ascend910B4-1-memory + memoryAllocatable: 65536 + memoryCapacity: 65536 + aiCore: 20 + aiCPU: 7 + templates: + # NOTE: Names of vnpu template for 910B4-1 are fixed by Ascend runtime and must not be changed. + # The memory is used for scheduling so the correct values must be set. + # Template vir05_1c_8g actually provides 16GB memory, + - name: vir05_1c_8g + memory: 16384 + aiCore: 5 + aiCPU: 1 + # Template vir10_3c_16g actually provides 32GB memory + - name: vir10_3c_16g + memory: 32768 + aiCore: 10 + aiCPU: 3 + - chipName: 910B4 + commonWord: Ascend910B4 + resourceName: huawei.com/Ascend910B4 + resourceMemoryName: huawei.com/Ascend910B4-memory + memoryAllocatable: 32768 + memoryCapacity: 32768 + aiCore: 20 + aiCPU: 7 + templates: + - name: vir05_1c_8g + memory: 8192 + aiCore: 5 + aiCPU: 1 + - name: vir10_3c_16g + memory: 16384 + aiCore: 10 + aiCPU: 3 + - chipName: 310P3 + commonWord: Ascend310P + resourceName: huawei.com/Ascend310P + resourceMemoryName: huawei.com/Ascend310P-memory + memoryAllocatable: 21527 + memoryCapacity: 24576 + aiCore: 8 + aiCPU: 7 + templates: + - name: vir01 + memory: 3072 + aiCore: 1 + aiCPU: 1 + - name: vir02 + memory: 6144 + aiCore: 2 + aiCPU: 2 + - name: vir04 + memory: 12288 + aiCore: 4 + aiCPU: 4 + {{ end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/mutatingwebhookconfiguration.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/mutatingwebhookconfiguration.yaml new file mode 100644 index 00000000..c763b6aa --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/mutatingwebhookconfiguration.yaml @@ -0,0 +1,28 @@ +{{- if .Values.webhook.config.mutating.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: {{ .Release.Name }}-mutatingwebhookconfiguration + annotations: + cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ .Release.Name }}-dra-webhook-serving-cert +webhooks: + - name: mutate.hami.io + admissionReviewVersions: ["v1"] + clientConfig: + service: + name: {{ .Release.Name }}-dra-webhook + namespace: {{ .Release.Namespace }} + path: /mutate + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - pods + scope: '*' + sideEffects: None + timeoutSeconds: 10 +{{- end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/validatingwebhookconfiguration.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/validatingwebhookconfiguration.yaml new file mode 100644 index 00000000..acdcbb96 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/validatingwebhookconfiguration.yaml @@ -0,0 +1,29 @@ +{{- if .Values.webhook.config.validating.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: {{ .Release.Name }}-validatingwebhookconfiguration + annotations: + cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ .Release.Name }}-dra-webhook-serving-cert +webhooks: + - name: validate.hami.io + admissionReviewVersions: ["v1"] + clientConfig: + service: + name: {{ .Release.Name }}-dra-webhook + namespace: {{ .Release.Namespace }} + path: /validate + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - DELETE + resources: + - pods + scope: '*' + sideEffects: None + timeoutSeconds: 10 + failurePolicy: Ignore +{{- end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrole.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrole.yaml new file mode 100644 index 00000000..28916a36 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrole.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "hami.dra.webhook.fullname" . }} +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] +- apiGroups: ["admissionregistration.k8s.io"] + resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"] + verbs: ["get", "list", "watch"] +- apiGroups: ["resource.k8s.io"] + resources: ["resourceclaims"] + verbs: ["*"] diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrolebinding.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrolebinding.yaml new file mode 100644 index 00000000..daad52f2 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrolebinding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami.dra.webhook.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "hami.dra.webhook.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "hami.dra.webhook.fullname" . }} + namespace: {{ .Release.Namespace }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-deployment.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-deployment.yaml new file mode 100644 index 00000000..8e3cf90c --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-deployment.yaml @@ -0,0 +1,59 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "hami.dra.webhook.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "hami.dra.webhook.fullname" . }} + {{- include "hami-dra-webhook.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "hami-dra-webhook.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "hami-dra-webhook.selectorLabels" . | nindent 8 }} + spec: + {{- include "hami.dra.webhook.imagePullSecrets" . | nindent 6 }} + serviceAccountName: {{ include "hami.dra.webhook.fullname" . }} + containers: + - name: webhook + image: {{ include "hami.dra.webhook.image" . }} + imagePullPolicy: {{ .Values.webhook.image.pullPolicy | default "IfNotPresent" }} + command: + - /bin/webhook + - --v=5 + - --bind-address=0.0.0.0 + - --secure-port=8443 + - --cert-dir=/tls + - --tls-cert-file-name=tls.crt + - --tls-private-key-file-name=tls.key + - --metrics-bind-address=:8080 + - --health-probe-bind-address=:8000 + - --device-config-file=/device-config.yaml + ports: + - containerPort: 8443 + name: webhook + protocol: TCP + - containerPort: 8080 + name: metrics + protocol: TCP + - containerPort: 8000 + name: health + protocol: TCP + volumeMounts: + - name: device-config + mountPath: /device-config.yaml + subPath: device-config.yaml + - name: tls-config + mountPath: /tls + readOnly: true + volumes: + - name: device-config + configMap: + name: {{ include "hami.dra.webhook.fullname" . }}-device-config + - name: tls-config + secret: + secretName: {{ if .Values.certs.certManager.enabled }}{{ .Release.Name }}-dra-webhook-tls{{ else }}{{ .Release.Name }}-tls{{ end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-service.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-service.yaml new file mode 100644 index 00000000..290174ec --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-dra-webhook + namespace: {{ .Release.Namespace }} + labels: + {{- include "hami-dra-webhook.selectorLabels" . | nindent 4 }} +spec: + type: "ClusterIP" + selector: + {{- include "hami-dra-webhook.selectorLabels" . | nindent 4 }} + ports: + - port: 443 + targetPort: 8443 + name: webhook diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-serviceaccount.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-serviceaccount.yaml new file mode 100644 index 00000000..b0dc2fb2 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-serviceaccount.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "hami.dra.webhook.fullname" . }} + namespace: {{ .Release.Namespace }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/values.yaml b/packages/system/hami/charts/hami/charts/hami-dra/values.yaml new file mode 100644 index 00000000..ebbbfed7 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/values.yaml @@ -0,0 +1,166 @@ +## @section Global configuration +## @param global.imageRegistry Global Docker image registry +## @param global.imagePullSecrets Global Docker image pull secrets +global: + ## @param global.imageRegistry Global Docker image registry + imageRegistry: "" + ## E.g. + ## imagePullSecrets: + ## - myRegistryKeySecretName + ## @param global.imagePullSecrets Global Docker image pull secrets + imagePullSecrets: [] + +#Nvidia GPU Parameters +resourceName: "nvidia.com/gpu" +resourceMem: "nvidia.com/gpumem" +resourceMemPercentage: "nvidia.com/gpumem-percentage" +resourceCores: "nvidia.com/gpucores" +resourcePriority: "nvidia.com/priority" + +#MLU Parameters +mluResourceName: "cambricon.com/vmlu" +mluResourceMem: "cambricon.com/mlu.smlu.vmemory" +mluResourceCores: "cambricon.com/mlu.smlu.vcore" + +#Hygon DCU Parameters +dcuResourceName: "hygon.com/dcunum" +dcuResourceMem: "hygon.com/dcumem" +dcuResourceCores: "hygon.com/dcucores" + +#Metax sGPU Parameters +metaxResourceName: "metax-tech.com/sgpu" +metaxResourceCore: "metax-tech.com/vcore" +metaxResourceMem: "metax-tech.com/vmemory" +metaxsGPUTopologyAware: "false" + +#Enflame VGCU Parameters +enflameResourceNameVGCU: "enflame.com/vgcu" +enflameResourceNameVGCUPercentage: "enflame.com/vgcu-percentage" + +#Kunlun XPU Parameters +kunlunResourceName: "kunlunxin.com/xpu" +kunlunResourceVCountName: "kunlunxin.com/vxpu" +kunlunResourceVMemoryName: "kunlunxin.com/vxpu-memory" + +# Webhook deployment configuration +webhook: + image: + registry: ghcr.io + repository: project-hami/hami-dra-webhook + tag: "v0.1.0" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## @param resources controllerManager resource requests and limits + resources: + # If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 128Mi + ## @param nodeSelector controllerManager node labels for pod assignment + args: + webhookName: "" # Default: {{ .Release.Name }}-hami-dra-webhook + secretName: "" # Default: {{ .Release.Name }}-hami-dra-webhook-tls + # Webhook configuration + config: + mutating: + enabled: true + validating: + enabled: true + +# Monitor deployment configuration +monitor: + enabled: true + replicas: 1 + image: + registry: ghcr.io + repository: project-hami/hami-dra-monitor + tag: "v0.1.0" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## @param resources monitor resource requests and limits + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 128Mi + ## Monitor configuration + logLevel: 2 + metricsBindAddress: ":8080" + healthProbeBindAddress: ":8000" + kubeAPIQPS: 40.0 + kubeAPIBurst: 60 + collectInterval: "30s" + ## Monitor service configuration + service: + ## Service type: ClusterIP, NodePort, or LoadBalancer + ## Defaults to ClusterIP + type: ClusterIP + ## NodePort configuration (only used when type is NodePort) + nodePort: + ## Metrics port (NodePort). If not specified, Kubernetes will assign a random port + metrics: "" + +# Certificate configuration +certs: + # Custom certificate (used when mode is "custom") + custom: + crt: "" + key: "" + # Cert-manager configuration (used when mode is "cert-manager") + certManager: + enabled: true + # Private key rotation policy: "Never" or "Always" + # In cert-manager >= v1.18.0, the default changed from "Never" to "Always" + # Setting to "Never" avoids frequent certificate rotation for webhooks + privateKeyRotationPolicy: "Never" + issuer: + clusterIssuerName: "" # Required when type is "clusterIssuer" + +# DRA Drivers configuration +drivers: + nvidia: + enabled: true + # If you are using gpu driver on host, you need to set this to false + containerDriver: true + image: + registry: ghcr.io + repository: projecthami/k8s-dra-driver + tag: "v0.0.1-dev" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] diff --git a/packages/system/hami/charts/hami/templates/NOTES.txt b/packages/system/hami/charts/hami/templates/NOTES.txt new file mode 100644 index 00000000..15bb1218 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/NOTES.txt @@ -0,0 +1,3 @@ +** Please be patient while the chart is being deployed ** +Resource name: {{ .Values.resourceName }} + diff --git a/packages/system/hami/charts/hami/templates/_commons.tpl b/packages/system/hami/charts/hami/templates/_commons.tpl new file mode 100644 index 00000000..78a262c9 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/_commons.tpl @@ -0,0 +1,49 @@ +{{/* +Return the proper image name +{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" $) }} +*/}} +{{- define "common.images.image" -}} +{{- $registryName := .imageRoot.registry -}} +{{- $repositoryName := .imageRoot.repository -}} +{{- $tag := .imageRoot.tag | toString -}} +{{- if .global }} + {{- if .global.imageRegistry }} + {{- $registryName = .global.imageRegistry -}} + {{- end -}} +{{- end -}} +{{- if .tag }} + {{- $tag = .tag | toString -}} +{{- end -}} +{{- if $registryName }} +{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- else -}} +{{- printf "%s:%s" $repositoryName $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead) +{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }} +*/}} +{{- define "common.images.pullSecrets" -}} + {{- $pullSecrets := list }} + + {{- if .global }} + {{- range .global.imagePullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- range .images -}} + {{- range .pullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- if (not (empty $pullSecrets)) }} +imagePullSecrets: + {{- range $pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end -}} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/_helpers.tpl b/packages/system/hami/charts/hami/templates/_helpers.tpl new file mode 100644 index 00000000..ffcc61da --- /dev/null +++ b/packages/system/hami/charts/hami/templates/_helpers.tpl @@ -0,0 +1,163 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "hami-vgpu.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "hami-vgpu.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "hami-vgpu.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +The app name for Scheduler +*/}} +{{- define "hami-vgpu.scheduler" -}} +{{- printf "%s-scheduler" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +The app name for DevicePlugin +*/}} +{{- define "hami-vgpu.device-plugin" -}} +{{- printf "%s-device-plugin" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* + The app name for MockDevicePlugin + */}} +{{- define "hami-vgpu.mock-device-plugin" -}} +{{- printf "%s-mock-device-plugin" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +The tls secret name for Scheduler +*/}} +{{- define "hami-vgpu.scheduler.tls" -}} +{{- printf "%s-scheduler-tls" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +The webhook name +*/}} +{{- define "hami-vgpu.scheduler.webhook" -}} +{{- printf "%s-webhook" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "hami-vgpu.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "hami-vgpu.labels" -}} +helm.sh/chart: {{ include "hami-vgpu.chart" . }} +{{ include "hami-vgpu.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "hami-vgpu.selectorLabels" -}} +app.kubernetes.io/name: {{ include "hami-vgpu.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + + +{{/* + Resolve the tag for kubeScheduler. +*/}} +{{- define "resolvedKubeSchedulerTag" -}} +{{- if .Values.scheduler.kubeScheduler.image.tag }} +{{- .Values.scheduler.kubeScheduler.image.tag | trim -}} +{{- else }} +{{- include "strippedKubeVersion" . | trim -}} +{{- end }} +{{- end }} + +{{/* + Return the stripped Kubernetes version string by removing extra parts after semantic version number. + v1.31.1+k3s1 -> v1.31.1 + v1.30.8-eks-2d5f260 -> v1.30.8 + v1.31.1 -> v1.31.1 +*/}} +{{- define "strippedKubeVersion" -}} +{{ regexReplaceAll "^(v[0-9]+\\.[0-9]+\\.[0-9]+)(.*)$" .Capabilities.KubeVersion.Version "$1" }} +{{- end -}} + +{{- define "hami.scheduler.kubeScheduler.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.kubeScheduler.image "global" .Values.global "tag" (include "resolvedKubeSchedulerTag" .)) }} +{{- end -}} + +{{- define "hami.scheduler.extender.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.extender.image "global" .Values.global "tag" .Values.global.imageTag) }} +{{- end -}} + +{{- define "hami.devicePlugin.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.devicePlugin.image "global" .Values.global "tag" .Values.global.imageTag) }} +{{- end -}} + +{{- define "hami.mockDevicePlugin.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.mockDevicePlugin.image "global" .Values.global "tag" .Values.mockDevicePlugin.tag) }} +{{- end -}} + +{{- define "hami.devicePlugin.monitor.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.devicePlugin.monitor.image "global" .Values.global "tag" .Values.global.imageTag) }} +{{- end -}} + +{{- define "hami.scheduler.patch.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.patch.image "global" .Values.global) }} +{{- end -}} + +{{- define "hami.scheduler.patch.new.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.patch.imageNew "global" .Values.global) }} +{{- end -}} + +{{- define "hami.scheduler.extender.imagePullSecrets" -}} +{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.extender.image) "global" .Values.global) }} +{{- end -}} + +{{- define "hami.devicePlugin.imagePullSecrets" -}} +{{ include "common.images.pullSecrets" (dict "images" (list .Values.devicePlugin.image) "global" .Values.global) }} +{{- end -}} + +{{- define "hami.scheduler.patch.imagePullSecrets" -}} +{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.patch.image) "global" .Values.global) }} +{{- end -}} + +{{- define "hami.scheduler.patch.new.imagePullSecrets" -}} +{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.patch.imageNew) "global" .Values.global) }} +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml b/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml new file mode 100644 index 00000000..b36347f2 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml @@ -0,0 +1,13 @@ +{{- if and .Values.devicePlugin.enabled (not .Values.devicePlugin.nodeConfiguration.externalConfigName) (not .Values.dra.enabled) -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "hami-vgpu.device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-device-plugin + {{- include "hami-vgpu.labels" . | nindent 4 }} +data: + config.json: | +{{- .Values.devicePlugin.nodeConfiguration.config | nindent 4 }} +{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/device-plugin/daemonsetmock.yaml b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetmock.yaml new file mode 100644 index 00000000..a0dc4ff3 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetmock.yaml @@ -0,0 +1,55 @@ +{{- if .Values.mockDevicePlugin.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "hami-vgpu.mock-device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +spec: + selector: + matchLabels: + app.kubernetes.io/component: hami-mock-device-plugin + {{- include "hami-vgpu.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" + labels: + app.kubernetes.io/component: hami-mock-device-plugin + {{- include "hami-vgpu.selectorLabels" . | nindent 8 }} + spec: + serviceAccountName: {{ include "hami-vgpu.mock-device-plugin" . }} + tolerations: + - key: CriticalAddonsOnly + operator: Exists + containers: + - image: {{ include "hami.mockDevicePlugin.image" . }} + imagePullPolicy: {{ .Values.mockDevicePlugin.image.pullPolicy }} + name: hami-mock-dp-cntr + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + command: + - ./k8s-device-plugin + - -v=5 + - --device-config-file=/device-config.yaml + volumeMounts: + - name: dp + mountPath: /var/lib/kubelet/device-plugins + - name: sys + mountPath: /sys + - name: device-config + mountPath: /device-config.yaml + subPath: device-config.yaml + volumes: + - name: dp + hostPath: + path: /var/lib/kubelet/device-plugins + - name: sys + hostPath: + path: /sys + - name: device-config + configMap: + name: {{ include "hami-vgpu.scheduler" . }}-device +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml new file mode 100644 index 00000000..577dc0f9 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml @@ -0,0 +1,262 @@ +{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "hami-vgpu.device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-device-plugin + {{- include "hami-vgpu.labels" . | nindent 4 }} + {{- with .Values.global.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.global.annotations }} + annotations: {{ toYaml .Values.global.annotations | nindent 4}} + {{- end }} +spec: + updateStrategy: + {{- with .Values.devicePlugin.updateStrategy }} + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/component: hami-device-plugin + {{- include "hami-vgpu.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app.kubernetes.io/component: hami-device-plugin + hami.io/webhook: ignore + {{- include "hami-vgpu.selectorLabels" . | nindent 8 }} + annotations: + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + checksum/hami-scheduler-newversion-config: {{ include (print $.Template.BasePath "/scheduler/configmapnew.yaml") . | sha256sum }} + {{- else }} + checksum/hami-scheduler-config: {{ include (print $.Template.BasePath "/scheduler/configmap.yaml") . | sha256sum }} + {{- end }} + checksum/hami-scheduler-device-config: {{ include (print $.Template.BasePath "/scheduler/device-configmap.yaml") . | sha256sum }} + {{- if .Values.devicePlugin.podAnnotations }} + {{- toYaml .Values.devicePlugin.podAnnotations | nindent 8 }} + {{- end }} + spec: + {{- if .Values.devicePlugin.runtimeClassName }} + runtimeClassName: {{ .Values.devicePlugin.runtimeClassName }} + {{- end }} + serviceAccountName: {{ include "hami-vgpu.device-plugin" . }} + priorityClassName: system-node-critical + hostPID: true + hostNetwork: true + {{- include "hami.devicePlugin.imagePullSecrets" . | nindent 6 }} + {{- if .Values.devicePlugin.gpuOperatorToolkitReady.enabled }} + initContainers: + - name: toolkit-validation + image: {{ include "hami.devicePlugin.image" . }} + imagePullPolicy: {{ .Values.devicePlugin.image.pullPolicy }} + securityContext: + privileged: true + runAsUser: 0 + command: ["sh", "-c"] + args: + - | + echo "Waiting for NVIDIA Toolkit to be ready..." + until [ -f {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }}/toolkit-ready ]; do + echo "Waiting for {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }}/toolkit-ready..." + sleep 5 + done + echo "NVIDIA Toolkit is ready!" + volumeMounts: + - name: nvidia-validations + mountPath: {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath | quote }} + mountPropagation: HostToContainer + readOnly: true + {{- end }} + containers: + - name: device-plugin + image: {{ include "hami.devicePlugin.image" . }} + imagePullPolicy: {{ .Values.devicePlugin.image.pullPolicy }} + lifecycle: + postStart: + exec: + command: ["/bin/sh","-c", {{ printf "/k8s-vgpu/bin/vgpu-init.sh %s/vgpu/" .Values.global.gpuHookPath | quote }}] + command: + - nvidia-device-plugin + - --config-file=/device-config.yaml + - --mig-strategy={{ .Values.devicePlugin.migStrategy }} + - --disable-core-limit={{ .Values.devicePlugin.disablecorelimit }} + {{- range .Values.devicePlugin.extraArgs }} + - {{ . }} + {{- end }} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: NVIDIA_MIG_MONITOR_DEVICES + value: all + - name: DEVICE_LIST_STRATEGY + value: {{ .Values.devicePlugin.deviceListStrategy }} + - name: HOOK_PATH + value: {{ .Values.global.gpuHookPath }} + {{- if typeIs "bool" .Values.devicePlugin.passDeviceSpecsEnabled }} + - name: PASS_DEVICE_SPECS + value: {{ .Values.devicePlugin.passDeviceSpecsEnabled | quote }} + {{- end }} + {{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }} + - name: NVIDIA_DRIVER_ROOT + value: {{ .Values.devicePlugin.nvidiaDriverRoot }} + {{- end }} + {{- if typeIs "string" .Values.devicePlugin.nvidiaHookPath }} + - name: NVIDIA_CDI_HOOK_PATH + value: {{ .Values.devicePlugin.nvidiaHookPath }} + {{- end }} + {{- if typeIs "bool" .Values.devicePlugin.gdrcopyEnabled }} + - name: GDRCOPY_ENABLED + value: {{ .Values.devicePlugin.gdrcopyEnabled | quote }} + {{- end }} + {{- if typeIs "bool" .Values.devicePlugin.gdsEnabled }} + - name: GDS_ENABLED + value: {{ .Values.devicePlugin.gdsEnabled | quote }} + {{- end }} + {{- if typeIs "bool" .Values.devicePlugin.mofedEnabled }} + - name: MOFED_ENABLED + value: {{ .Values.devicePlugin.mofedEnabled | quote }} + {{- end }} + {{- if eq (.Values.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy | default "spread") "topology-aware" }} + - name: ENABLE_TOPOLOGY_SCORE + value: "true" + {{- end }} + {{- with .Values.devicePlugin.extraEnvs }} + {{- . | toYaml | nindent 12 }} + {{- end }} + securityContext: + privileged: true + allowPrivilegeEscalation: true + capabilities: + drop: ["ALL"] + add: ["SYS_ADMIN"] + resources: + {{- toYaml .Values.devicePlugin.resources | nindent 12 }} + volumeMounts: + - name: device-plugin + mountPath: /var/lib/kubelet/device-plugins + - name: lib + mountPath: {{ printf "%s%s" .Values.global.gpuHookPath "/vgpu" }} + - name: usrbin + mountPath: /usrbin + - name: deviceconfig + mountPath: /config + - name: hosttmp + mountPath: /tmp + - name: device-config + mountPath: /device-config.yaml + subPath: device-config.yaml + - name: cdi-root + mountPath: /var/run/cdi + {{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }} + # We always mount the driver root at /driver-root in the container. + # This is required for CDI detection to work correctly. + - name: driver-root + mountPath: /driver-root + readOnly: true + {{- end }} + - name: vgpu-monitor + image: {{ include "hami.devicePlugin.monitor.image" . }} + imagePullPolicy: {{ .Values.devicePlugin.monitor.image.pullPolicy }} + command: + - "vGPUmonitor" + {{- range .Values.devicePlugin.monitor.extraArgs }} + - {{ . }} + {{- end }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + add: ["SYS_ADMIN"] + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: NVIDIA_VISIBLE_DEVICES + value: "all" + - name: NVIDIA_MIG_MONITOR_DEVICES + value: "all" + - name: HOOK_PATH + value: "{{ .Values.global.gpuHookPath }}/vgpu" + - name: HAMI_RESYNC_INTERVAL + value: {{ .Values.devicePlugin.monitor.resyncInterval | default "5m" | quote }} + {{- with .Values.devicePlugin.monitor.extraEnvs }} + {{- . | toYaml | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.devicePlugin.monitor.resources | nindent 12 }} + volumeMounts: + - name: ctrs + mountPath: {{ .Values.devicePlugin.monitor.ctrPath }} + - name: dockers + mountPath: /run/docker + - name: containerds + mountPath: /run/containerd + - name: sysinfo + mountPath: /sysinfo + - name: hostvar + mountPath: /hostvar + - name: hosttmp + mountPath: /tmp + volumes: + - name: ctrs + hostPath: + path: {{ .Values.devicePlugin.monitor.ctrPath }} + - name: hosttmp + hostPath: + path: /tmp + - name: dockers + hostPath: + path: /run/docker + - name: containerds + hostPath: + path: /run/containerd + - name: device-plugin + hostPath: + path: {{ .Values.devicePlugin.pluginPath }} + - name: lib + hostPath: + path: {{ .Values.devicePlugin.libPath }} + {{- if .Values.devicePlugin.gpuOperatorToolkitReady.enabled }} + - name: nvidia-validations + hostPath: + path: {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }} + type: DirectoryOrCreate + {{- end }} + {{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }} + - name: driver-root + hostPath: + path: {{ .Values.devicePlugin.nvidiaDriverRoot }} + type: Directory + {{- end }} + - name: cdi-root + hostPath: + path: /var/run/cdi + type: DirectoryOrCreate + - name: usrbin + hostPath: + path: /usr/bin + - name: sysinfo + hostPath: + path: /sys + - name: hostvar + hostPath: + path: /var + - name: deviceconfig + configMap: + name: {{ .Values.devicePlugin.nodeConfiguration.externalConfigName | default (include "hami-vgpu.device-plugin" .) }} + - name: device-config + configMap: + name: {{ include "hami-vgpu.scheduler" . }}-device + {{- if .Values.devicePlugin.nvidiaNodeSelector }} + nodeSelector: {{ toYaml .Values.devicePlugin.nvidiaNodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.devicePlugin.tolerations }} + tolerations: {{ toYaml .Values.devicePlugin.tolerations | nindent 8 }} + {{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml new file mode 100644 index 00000000..c51ff6c6 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "hami-vgpu.device-plugin" . }}-monitor +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - create + - watch + - list + - update + - patch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - update + - list + - patch +{{- end -}} + + diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml new file mode 100644 index 00000000..93a118dd --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.device-plugin" . }} + labels: + app.kubernetes.io/component: "hami-device-plugin" + {{- include "hami-vgpu.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "hami-vgpu.device-plugin" . }}-monitor +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml new file mode 100644 index 00000000..24daca9b --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "hami-vgpu.device-plugin" . }}-monitor + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-device-plugin + {{- include "hami-vgpu.labels" . | nindent 4 }} + {{- if .Values.devicePlugin.service.labels }} # Use devicePlugin instead of scheduler + {{ toYaml .Values.devicePlugin.service.labels | indent 4 }} + {{- end }} + {{- if .Values.devicePlugin.service.annotations }} # Use devicePlugin instead of scheduler + annotations: {{ toYaml .Values.devicePlugin.service.annotations | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.devicePlugin.service.type | default "NodePort" }} # Default type is NodePort + ports: + - name: monitorport + port: {{ .Values.devicePlugin.service.httpPort | default 31992 }} # Default HTTP port is 31992 + targetPort: 9394 + {{- if eq (.Values.devicePlugin.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort + nodePort: {{ .Values.devicePlugin.service.httpPort | default 31992 }} + {{- end }} + protocol: TCP + selector: + app.kubernetes.io/component: hami-device-plugin + {{- include "hami-vgpu.selectorLabels" . | nindent 4 }} +{{- end -}} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml new file mode 100644 index 00000000..b10d2cb9 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml @@ -0,0 +1,10 @@ +{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "hami-vgpu.device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: "hami-device-plugin" + {{- include "hami-vgpu.labels" . | nindent 4 }} +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml b/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml new file mode 100644 index 00000000..cd3d14cc --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml @@ -0,0 +1,11 @@ +{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}} +{{- if and .Values.devicePlugin.createRuntimeClass .Values.devicePlugin.runtimeClassName }} +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: {{ .Values.devicePlugin.runtimeClassName }} + annotations: + helm.sh/hook: pre-install,pre-upgrade +handler: nvidia +{{- end }} +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml b/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml new file mode 100644 index 00000000..b6edb605 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml @@ -0,0 +1,31 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} +{{- if .Values.scheduler.certManager.enabled }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-serving-cert + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} +spec: + dnsNames: + - {{ include "hami-vgpu.scheduler" . }}.{{ include "hami-vgpu.namespace" . }}.svc + - {{ include "hami-vgpu.scheduler" . }}.{{ include "hami-vgpu.namespace" . }}.svc.cluster.local + issuerRef: + kind: Issuer + name: {{ include "hami-vgpu.scheduler" . }}-selfsigned-issuer + secretName: {{ include "hami-vgpu.scheduler.tls" . }} +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-selfsigned-issuer + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} +spec: + selfSigned: {} +{{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml b/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml new file mode 100644 index 00000000..9c510c00 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml @@ -0,0 +1,36 @@ +{{- if not .Values.dra.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["pods", "configmaps"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: [""] + resources: ["pods/binding"] + verbs: ["create"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "patch", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "get", "list"] + - apiGroups: [""] + resources: ["resourcequotas"] + verbs: ["get", "list", "watch"] +--- +{{- if .Values.mockDevicePlugin.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "hami-vgpu.mock-device-plugin" . }} +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "update", "list", "patch"] +{{- end -}} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml new file mode 100644 index 00000000..69d75986 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml @@ -0,0 +1,49 @@ +{{- if not .Values.dra.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-kube + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-scheduler +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-volume + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:volume-scheduler +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "hami-vgpu.scheduler" . }} +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml b/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml new file mode 100644 index 00000000..75889146 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml @@ -0,0 +1,142 @@ +{{- if and .Values.scheduler.kubeScheduler.enabled (not .Values.dra.enabled) -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} +data: + config.json: | + { + "kind": "Policy", + "apiVersion": "v1", + "extenders": [ + { + {{- if .Values.scheduler.admissionWebhook.enabled }} + "urlPrefix": "https://127.0.0.1:443", + "enableHttps": true, + "tlsConfig": { + "insecure": true + }, + {{- else }} + "urlPrefix": "http://127.0.0.1:80", + "enableHttps": false, + {{- end }} + "filterVerb": "filter", + "bindVerb": "bind", + "weight": 1, + "nodeCacheCapable": true, + "httpTimeout": 30000000000, + "managedResources": [ + {{- range .Values.devices.amd.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- if .Values.devices.ascend.enabled }} + {{- range .Values.devices.ascend.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- end }} + {{- if .Values.devices.mthreads.enabled }} + {{- range .Values.devices.mthreads.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- end }} + {{- if .Values.devices.enflame.enabled }} + {{- range .Values.devices.enflame.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- end }} + {{- if .Values.devices.kunlun.enabled }} + {{- range .Values.devices.kunlun.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- end }} + {{- range .Values.devices.awsneuron.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- if .Values.devices.iluvatar.enabled }} + {{- range .Values.devices.iluvatar.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- end }} + { + "name": "{{ .Values.resourceName }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.resourceMem }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.resourceCores }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.resourceMemPercentage }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.resourcePriority }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.mluResourceName }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.dcuResourceName }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.dcuResourceMem }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.dcuResourceCores }}", + "ignoredByScheduler": true + }, + { + "name": "metax-tech.com/gpu", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.metaxResourceName }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.metaxResourceCore }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.metaxResourceMem }}", + "ignoredByScheduler": true + } + ], + "ignoreable": false + } + ] + } +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml b/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml new file mode 100644 index 00000000..e2a91d8b --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml @@ -0,0 +1,102 @@ +{{- if and .Values.scheduler.kubeScheduler.enabled (not .Values.dra.enabled) -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-newversion + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} +data: + config.yaml: | + {{- if gt (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 25}} + apiVersion: kubescheduler.config.k8s.io/v1 + {{- else }} + apiVersion: kubescheduler.config.k8s.io/v1beta2 + {{- end }} + kind: KubeSchedulerConfiguration + leaderElection: + leaderElect: false + profiles: + - schedulerName: {{ .Values.schedulerName }} + extenders: + {{- if .Values.scheduler.admissionWebhook.enabled }} + - urlPrefix: "https://127.0.0.1:443" + enableHTTPS: true + tlsConfig: + insecure: true + {{- else }} + - urlPrefix: "http://127.0.0.1:80" + enableHTTPS: false + {{- end }} + filterVerb: filter + bindVerb: bind + nodeCacheCapable: true + weight: 1 + httpTimeout: 30s + managedResources: + - name: {{ .Values.resourceName }} + ignoredByScheduler: true + - name: {{ .Values.resourceMem }} + ignoredByScheduler: true + - name: {{ .Values.resourceCores }} + ignoredByScheduler: true + - name: {{ .Values.resourceMemPercentage }} + ignoredByScheduler: true + - name: {{ .Values.resourcePriority }} + ignoredByScheduler: true + - name: {{ .Values.mluResourceName }} + ignoredByScheduler: true + - name: {{ .Values.dcuResourceName }} + ignoredByScheduler: true + - name: {{ .Values.dcuResourceMem }} + ignoredByScheduler: true + - name: {{ .Values.dcuResourceCores }} + ignoredByScheduler: true + - name: "metax-tech.com/gpu" + ignoredByScheduler: true + - name: {{ .Values.metaxResourceName }} + ignoredByScheduler: true + - name: {{ .Values.metaxResourceCore }} + ignoredByScheduler: true + - name: {{ .Values.metaxResourceMem }} + ignoredByScheduler: true + {{- if .Values.devices.ascend.enabled }} + {{- range .Values.devices.ascend.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- end }} + {{- if .Values.devices.mthreads.enabled }} + {{- range .Values.devices.mthreads.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- end }} + {{- if .Values.devices.enflame.enabled }} + {{- range .Values.devices.enflame.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- end }} + {{- if .Values.devices.kunlun.enabled }} + {{- range .Values.devices.kunlun.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- end }} + {{- range .Values.devices.awsneuron.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- if .Values.devices.iluvatar.enabled }} + {{- range .Values.devices.iluvatar.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- end }} + {{- range .Values.devices.amd.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml b/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml new file mode 100644 index 00000000..6364cae3 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml @@ -0,0 +1,227 @@ +{{- if not .Values.dra.enabled -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} + {{- with .Values.global.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.global.annotations }} + annotations: {{ toYaml .Values.global.annotations | nindent 4}} + {{- end }} +spec: + {{- if .Values.scheduler.leaderElect }} + replicas: {{ .Values.scheduler.replicas }} + {{- else }} + replicas: 1 + {{- end }} + selector: + matchLabels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.selectorLabels" . | nindent 8 }} + hami.io/webhook: ignore + annotations: + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + checksum/hami-scheduler-newversion-config: {{ include (print $.Template.BasePath "/scheduler/configmapnew.yaml") . | sha256sum }} + {{- else }} + checksum/hami-scheduler-config: {{ include (print $.Template.BasePath "/scheduler/configmap.yaml") . | sha256sum }} + {{- end }} + checksum/hami-scheduler-device-config: {{ include (print $.Template.BasePath "/scheduler/device-configmap.yaml") . | sha256sum }} + {{- if .Values.scheduler.podAnnotations }} + {{- toYaml .Values.scheduler.podAnnotations | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "hami-vgpu.scheduler" . }} + priorityClassName: system-node-critical + {{- include "hami.scheduler.extender.imagePullSecrets" . | nindent 6 }} + containers: + {{- if .Values.scheduler.kubeScheduler.enabled }} + - name: kube-scheduler + image: {{ include "hami.scheduler.kubeScheduler.image" . }} + imagePullPolicy: {{ .Values.scheduler.kubeScheduler.image.pullPolicy }} + command: + - kube-scheduler + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + {{- range .Values.scheduler.kubeScheduler.extraNewArgs }} + - {{ . }} + {{- end }} + {{- else }} + - --scheduler-name={{ .Values.schedulerName }} + {{- range .Values.scheduler.kubeScheduler.extraArgs }} + - {{ . }} + {{- end }} + {{- end }} + - --leader-elect={{ .Values.scheduler.leaderElect }} + - --leader-elect-resource-name={{ .Values.schedulerName }} + - --leader-elect-resource-namespace={{ include "hami-vgpu.namespace" . }} + resources: + {{- toYaml .Values.scheduler.kubeScheduler.resources | nindent 12 }} + volumeMounts: + - name: scheduler-config + mountPath: /config + {{- end }} + {{- if .Values.scheduler.livenessProbe }} + livenessProbe: + failureThreshold: 8 + httpGet: + path: /healthz + port: 10259 + scheme: HTTPS + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 15 + {{- end }} + - name: vgpu-scheduler-extender + image: {{ include "hami.scheduler.extender.image" . }} + imagePullPolicy: {{ .Values.scheduler.extender.image.pullPolicy }} + env: + {{- if .Values.scheduler.nodeLockExpire }} + - name: HAMI_NODELOCK_EXPIRE + value: "{{ .Values.scheduler.nodeLockExpire }}" + {{- end }} + {{- if .Values.global.managedNodeSelectorEnable }} + {{- range $key, $value := .Values.global.managedNodeSelector }} + - name: NODE_SELECTOR_{{ $key | upper | replace "-" "_" }} + value: "{{ $value }}" + {{- end }} + {{- end }} + command: + - scheduler + {{- if .Values.scheduler.admissionWebhook.enabled }} + - --http_bind=0.0.0.0:443 + - --cert_file=/tls/tls.crt + - --key_file=/tls/tls.key + {{- else }} + - --http_bind=0.0.0.0:80 + {{- end }} + - --scheduler-name={{ .Values.schedulerName }} + - --metrics-bind-address={{ .Values.scheduler.metricsBindAddress }} + - --node-scheduler-policy={{ .Values.scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy }} + - --gpu-scheduler-policy={{ .Values.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy }} + - --force-overwrite-default-scheduler={{ .Values.scheduler.forceOverwriteDefaultScheduler}} + - --device-config-file=/device-config.yaml + - --leader-elect={{ .Values.scheduler.leaderElect }} + - --leader-elect-resource-name={{ .Values.schedulerName }} + - --leader-elect-resource-namespace={{ include "hami-vgpu.namespace" . }} + {{- if .Values.devices.ascend.enabled }} + - --enable-ascend=true + {{- end }} + {{- if .Values.devices.iluvatar.enabled }} + - --enable-iluvatar=true + {{- end }} + {{- if .Values.scheduler.nodeLabelSelector }} + - --node-label-selector={{- $first := true -}} + {{- range $key, $value := .Values.scheduler.nodeLabelSelector -}} + {{- if not $first }},{{ end -}} + {{- $key }}={{ $value -}} + {{- $first = false -}} + {{- end -}} + {{- end }} + {{- range .Values.scheduler.extender.extraArgs }} + - {{ . }} + {{- end }} + ports: + {{- if .Values.scheduler.admissionWebhook.enabled }} + - name: https + containerPort: 443 + protocol: TCP + {{- else }} + - name: http + containerPort: 80 + protocol: TCP + {{- end }} + - name: metrics + containerPort: {{ last (splitList ":" .Values.scheduler.metricsBindAddress) | int }} + protocol: TCP + resources: + {{- toYaml .Values.scheduler.extender.resources | nindent 12 }} + volumeMounts: + - name: device-config + mountPath: /device-config.yaml + subPath: device-config.yaml + {{- if .Values.scheduler.admissionWebhook.enabled }} + - name: tls-config + mountPath: /tls + {{- end }} + {{- if .Values.scheduler.livenessProbe }} + livenessProbe: + httpGet: + path: /healthz + {{- if .Values.scheduler.admissionWebhook.enabled }} + port: https + scheme: HTTPS + {{- else }} + port: http + scheme: HTTP + {{- end }} + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 3 + timeoutSeconds: 5 + {{- end }} + {{- if .Values.scheduler.leaderElect }} + readinessProbe: + httpGet: + path: /readyz + {{- if .Values.scheduler.admissionWebhook.enabled }} + port: https + scheme: HTTPS + {{- else }} + port: http + scheme: HTTP + {{- end }} + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 3 + timeoutSeconds: 5 + {{- end }} + {{- if .Values.scheduler.leaderElect }} + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app.kubernetes.io/component + operator: In + values: + - hami-scheduler + topologyKey: "kubernetes.io/hostname" + {{- end }} + volumes: + {{- if .Values.scheduler.admissionWebhook.enabled }} + - name: tls-config + secret: + secretName: {{ template "hami-vgpu.scheduler.tls" . }} + {{- end }} + {{- if .Values.scheduler.kubeScheduler.enabled }} + - name: scheduler-config + configMap: + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + name: {{ template "hami-vgpu.scheduler" . }}-newversion + {{- else }} + name: {{ template "hami-vgpu.scheduler" . }} + {{- end }} + {{- end }} + - name: device-config + configMap: + name: {{ include "hami-vgpu.scheduler" . }}-device + {{- if .Values.scheduler.nodeSelector }} + nodeSelector: {{ toYaml .Values.scheduler.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.scheduler.tolerations }} + tolerations: {{ toYaml .Values.scheduler.tolerations | nindent 8 }} + {{- end }} + {{- if .Values.scheduler.nodeName }} + nodeName: {{ .Values.scheduler.nodeName }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml b/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml new file mode 100644 index 00000000..ccb945ac --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml @@ -0,0 +1,410 @@ +{{- if not .Values.dra.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-device + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} +data: + device-config.yaml: |- + {{- if .Files.Glob "files/device-config.yaml" }} + {{- .Files.Get "files/device-config.yaml" | nindent 4}} + {{- else }} + nvidia: + resourceCountName: {{ .Values.resourceName }} + resourceMemoryName: {{ .Values.resourceMem }} + resourceMemoryPercentageName: {{ .Values.resourceMemPercentage }} + resourceCoreName: {{ .Values.resourceCores }} + resourcePriorityName: {{ .Values.resourcePriority }} + overwriteEnv: false + defaultMemory: 0 + defaultCores: 0 + defaultGPUNum: 1 + memoryFactor: 1 + deviceSplitCount: {{ .Values.devicePlugin.deviceSplitCount }} + deviceMemoryScaling: {{ .Values.devicePlugin.deviceMemoryScaling }} + deviceCoreScaling: {{ .Values.devicePlugin.deviceCoreScaling }} + gpuCorePolicy: {{ .Values.devices.nvidia.gpuCorePolicy }} + libCudaLogLevel: {{ .Values.devices.nvidia.libCudaLogLevel }} + runtimeClassName: "{{ .Values.devicePlugin.runtimeClassName }}" + knownMigGeometries: + - models: [ "A30" ] + allowedGeometries: + - + - name: 1g.6gb + core: 25 + memory: 6144 + count: 4 + - + - name: 2g.12gb + core: 50 + memory: 12288 + count: 2 + - + - name: 4g.24gb + core: 100 + memory: 24576 + count: 1 + - models: [ "A100-SXM4-40GB", "A100-40GB-PCIe", "A100-PCIE-40GB"] + allowedGeometries: + - + - name: 1g.5gb + core: 14 + memory: 5120 + count: 7 + - + - name: 1g.5gb + core: 14 + memory: 5120 + count: 1 + - name: 2g.10gb + core: 28 + memory: 10240 + count: 3 + - + - name: 3g.20gb + core: 42 + memory: 20480 + count: 2 + - + - name: 7g.40gb + core: 100 + memory: 40960 + count: 1 + - models: [ "A100-SXM4-80GB", "A100-80GB-PCIe", "A100-PCIE-80GB"] + allowedGeometries: + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 7 + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 1 + - name: 2g.20gb + core: 28 + memory: 20480 + count: 3 + - + - name: 3g.40gb + core: 42 + memory: 40960 + count: 2 + - + - name: 7g.79gb + core: 100 + memory: 80896 + count: 1 + - models: [ "H100-PCIE-80GB", "H100-SXM5-80GB"] + allowedGeometries: + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 7 + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 1 + - name: 2g.20gb + core: 28 + memory: 20480 + count: 3 + - + - name: 3g.40gb + core: 42 + memory: 40960 + count: 2 + - + - name: 7g.80gb + core: 100 + memory: 81920 + count: 1 + - models: [ "H100-PCIE-94GB", "H100-SXM5-94GB"] + allowedGeometries: + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 7 + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 1 + - name: 2g.24gb + core: 28 + memory: 24576 + count: 3 + - + - name: 3g.47gb + core: 42 + memory: 48128 + count: 2 + - + - name: 7g.94gb + core: 100 + memory: 96256 + count: 1 + - models: [ "H20", "H100 on GH200"] + allowedGeometries: + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 7 + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 1 + - name: 2g.24gb + core: 28 + memory: 24576 + count: 3 + - + - name: 3g.48gb + core: 42 + memory: 49152 + count: 2 + - + - name: 7g.96gb + core: 100 + memory: 98304 + count: 1 + - models: [ "H200 NVL", "H200-SXM5"] + allowedGeometries: + - + - name: 1g.18gb + core: 14 + memory: 18432 + count: 7 + - + - name: 1g.18gb + core: 14 + memory: 18432 + count: 1 + - name: 2g.35gb + core: 28 + memory: 35840 + count: 3 + - + - name: 3g.71gb + core: 42 + memory: 72704 + count: 2 + - + - name: 7g.141gb + core: 100 + memory: 144384 + count: 1 + - models: [ "B200" ] + allowedGeometries: + - + - name: 1g.23gb + core: 14 + memory: 23552 + count: 7 + - + - name: 1g.23gb + core: 14 + memory: 23552 + count: 1 + - name: 2g.45gb + core: 28 + memory: 46080 + count: 3 + - + - name: 3g.90gb + core: 42 + memory: 92160 + count: 2 + - + - name: 7g.180gb + core: 100 + memory: 184320 + count: 1 + cambricon: + resourceCountName: {{ .Values.mluResourceName }} + resourceMemoryName: {{ .Values.mluResourceMem }} + resourceCoreName: {{ .Values.mluResourceCores }} + hygon: + resourceCountName: {{ .Values.dcuResourceName }} + resourceMemoryName: {{ .Values.dcuResourceMem }} + resourceCoreName: {{ .Values.dcuResourceCores }} + memoryFactor: 1 + metax: + resourceCountName: "metax-tech.com/gpu" + resourceVCountName: {{ .Values.metaxResourceName }} + resourceVMemoryName: {{ .Values.metaxResourceMem }} + resourceVCoreName: {{ .Values.metaxResourceCore }} + sgpuTopologyAware: {{ .Values.metaxsGPUTopologyAware }} + enflame: + resourceNameGCU: "enflame.com/gcu" + resourceNameVGCU: {{ .Values.enflameResourceNameVGCU }} + resourceNameVGCUPercentage: {{ .Values.enflameResourceNameVGCUPercentage }} + mthreads: + resourceCountName: "mthreads.com/vgpu" + resourceMemoryName: "mthreads.com/sgpu-memory" + resourceCoreName: "mthreads.com/sgpu-core" + iluvatars: + - chipName: MR-V100 + commonWord: MR-V100 + resourceCountName: iluvatar.ai/MR-V100-vgpu + resourceMemoryName: iluvatar.ai/MR-V100.vMem + resourceCoreName: iluvatar.ai/MR-V100.vCore + - chipName: MR-V50 + commonWord: MR-V50 + resourceCountName: iluvatar.ai/MR-V50-vgpu + resourceMemoryName: iluvatar.ai/MR-V50.vMem + resourceCoreName: iluvatar.ai/MR-V50.vCore + - chipName: BI-V150 + commonWord: BI-V150 + resourceCountName: iluvatar.ai/BI-V150-vgpu + resourceMemoryName: iluvatar.ai/BI-V150.vMem + resourceCoreName: iluvatar.ai/BI-V150.vCore + - chipName: BI-V100 + commonWord: BI-V100 + resourceCountName: iluvatar.ai/BI-V100-vgpu + resourceMemoryName: iluvatar.ai/BI-V100.vMem + resourceCoreName: iluvatar.ai/BI-V100.vCore + kunlun: + resourceCountName: {{ .Values.kunlunResourceName }} + resourceVCountName: {{ .Values.kunlunResourceVCountName }} + resourceVMemoryName: {{ .Values.kunlunResourceVMemoryName }} + awsneuron: + resourceCountName: "aws.amazon.com/neuron" + resourceCoreName: "aws.amazon.com/neuroncore" + amd: + resourceCountName: "amd.com/gpu" + vnpus: + - chipName: 910A + commonWord: Ascend910A + resourceName: huawei.com/Ascend910A + resourceMemoryName: huawei.com/Ascend910A-memory + memoryAllocatable: 32768 + memoryCapacity: 32768 + memoryFactor: 1 + aiCore: 30 + templates: + - name: vir02 + memory: 2184 + aiCore: 2 + - name: vir04 + memory: 4369 + aiCore: 4 + - name: vir08 + memory: 8738 + aiCore: 8 + - name: vir16 + memory: 17476 + aiCore: 16 + - chipName: 910B2 + commonWord: Ascend910B2 + resourceName: huawei.com/Ascend910B2 + resourceMemoryName: huawei.com/Ascend910B2-memory + memoryAllocatable: 65536 + memoryCapacity: 65536 + memoryFactor: 1 + aiCore: 24 + aiCPU: 6 + templates: + - name: vir03_1c_8g + memory: 8192 + aiCore: 3 + aiCPU: 1 + - name: vir06_1c_16g + memory: 16384 + aiCore: 6 + aiCPU: 1 + - name: vir12_3c_32g + memory: 32768 + aiCore: 12 + aiCPU: 3 + - chipName: 910B3 + commonWord: Ascend910B3 + resourceName: huawei.com/Ascend910B3 + resourceMemoryName: huawei.com/Ascend910B3-memory + memoryAllocatable: 65536 + memoryCapacity: 65536 + memoryFactor: 1 + aiCore: 20 + aiCPU: 7 + templates: + - name: vir05_1c_16g + memory: 16384 + aiCore: 5 + aiCPU: 1 + - name: vir10_3c_32g + memory: 32768 + aiCore: 10 + aiCPU: 3 + - chipName: 910B4-1 + commonWord: Ascend910B4-1 + resourceName: huawei.com/Ascend910B4-1 + resourceMemoryName: huawei.com/Ascend910B4-1-memory + memoryAllocatable: 65536 + memoryCapacity: 65536 + memoryFactor: 1 + aiCore: 20 + aiCPU: 7 + templates: + # NOTE: Names of vnpu template for 910B4-1 are fixed by Ascend runtime and must not be changed. + # The memory is used for scheduling so the correct values must be set. + # Template vir05_1c_8g actually provides 16GB memory, + - name: vir05_1c_8g + memory: 16384 + aiCore: 5 + aiCPU: 1 + # Template vir10_3c_16g actually provides 32GB memory + - name: vir10_3c_16g + memory: 32768 + aiCore: 10 + aiCPU: 3 + - chipName: 910B4 + commonWord: Ascend910B4 + resourceName: huawei.com/Ascend910B4 + resourceMemoryName: huawei.com/Ascend910B4-memory + memoryAllocatable: 32768 + memoryCapacity: 32768 + memoryFactor: 1 + aiCore: 20 + aiCPU: 7 + templates: + - name: vir05_1c_8g + memory: 8192 + aiCore: 5 + aiCPU: 1 + - name: vir10_3c_16g + memory: 16384 + aiCore: 10 + aiCPU: 3 + - chipName: 310P3 + commonWord: Ascend310P + resourceName: huawei.com/Ascend310P + resourceMemoryName: huawei.com/Ascend310P-memory + memoryAllocatable: 21527 + memoryCapacity: 24576 + memoryFactor: 1 + aiCore: 8 + aiCPU: 7 + templates: + - name: vir01 + memory: 3072 + aiCore: 1 + aiCPU: 1 + - name: vir02 + memory: 6144 + aiCore: 2 + aiCPU: 2 + - name: vir04 + memory: 12288 + aiCore: 4 + aiCPU: 4 + {{ end }} +{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml new file mode 100644 index 00000000..b089ae82 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml @@ -0,0 +1,30 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} +{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +rules: + - apiGroups: + - admissionregistration.k8s.io + resources: + #- validatingwebhookconfigurations + - mutatingwebhookconfigurations + verbs: + - get + - update +{{- if .Values.podSecurityPolicy.enabled }} + - apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ include "hami-vgpu.fullname" . }}-admission +{{- end }} +{{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml new file mode 100644 index 00000000..64f01ecf --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} +{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "hami-vgpu.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.fullname" . }}-admission + namespace: {{ include "hami-vgpu.namespace" . }} +{{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml new file mode 100644 index 00000000..400168a7 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml @@ -0,0 +1,70 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} +{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission-create + namespace: {{ include "hami-vgpu.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission-create + {{- if .Values.scheduler.patch.podAnnotations }} + annotations: {{ toYaml .Values.scheduler.patch.podAnnotations | nindent 8 }} + {{- end }} + labels: + {{- include "hami-vgpu.labels" . | nindent 8 }} + app.kubernetes.io/component: admission-webhook + hami.io/webhook: ignore + spec: + {{- if .Values.scheduler.patch.priorityClassName }} + priorityClassName: {{ .Values.scheduler.patch.priorityClassName }} + {{- end }} + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + {{- include "hami.scheduler.patch.new.imagePullSecrets" . | nindent 6 }} + {{- else }} + {{- include "hami.scheduler.patch.imagePullSecrets" . | nindent 6 }} + {{- end }} + containers: + - name: create + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + image: {{ include "hami.scheduler.patch.new.image" . }} + imagePullPolicy: {{ .Values.scheduler.patch.imageNew.pullPolicy }} + {{- else }} + image: {{ include "hami.scheduler.patch.image" . }} + imagePullPolicy: {{ .Values.scheduler.patch.image.pullPolicy }} + {{- end }} + args: + - create + - --cert-name=tls.crt + - --key-name=tls.key + {{- if .Values.scheduler.admissionWebhook.customURL.enabled }} + - --host={{ printf "%s.%s.svc,127.0.0.1,%s" (include "hami-vgpu.scheduler" .) (include "hami-vgpu.namespace" .) .Values.scheduler.admissionWebhook.customURL.host}} + {{- else }} + - --host={{ printf "%s.%s.svc,127.0.0.1" (include "hami-vgpu.scheduler" .) (include "hami-vgpu.namespace" .) }} + {{- end }} + - --namespace={{ include "hami-vgpu.namespace" . }} + - --secret-name={{ include "hami-vgpu.scheduler.tls" . }} + restartPolicy: OnFailure + serviceAccountName: {{ include "hami-vgpu.fullname" . }}-admission + {{- if .Values.scheduler.patch.nodeSelector }} + nodeSelector: {{ toYaml .Values.scheduler.patch.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.scheduler.patch.tolerations }} + tolerations: {{ toYaml .Values.scheduler.patch.tolerations | nindent 8 }} + {{- end }} + securityContext: + runAsNonRoot: true + runAsUser: {{ .Values.scheduler.patch.runAsUser }} +{{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml new file mode 100644 index 00000000..71e97f6b --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml @@ -0,0 +1,65 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} +{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission-patch + namespace: {{ include "hami-vgpu.namespace" . }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission-patch + {{- if .Values.scheduler.patch.podAnnotations }} + annotations: {{ toYaml .Values.scheduler.patch.podAnnotations | nindent 8 }} + {{- end }} + labels: + {{- include "hami-vgpu.labels" . | nindent 8 }} + app.kubernetes.io/component: admission-webhook + hami.io/webhook: ignore + spec: + {{- if .Values.scheduler.patch.priorityClassName }} + priorityClassName: {{ .Values.scheduler.patch.priorityClassName }} + {{- end }} + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + {{- include "hami.scheduler.patch.new.imagePullSecrets" . | nindent 6 }} + {{- else }} + {{- include "hami.scheduler.patch.imagePullSecrets" . | nindent 6 }} + {{- end }} + containers: + - name: patch + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + image: {{ include "hami.scheduler.patch.new.image" . }} + imagePullPolicy: {{ .Values.scheduler.patch.imageNew.pullPolicy }} + {{- else }} + image: {{ include "hami.scheduler.patch.image" . }} + imagePullPolicy: {{ .Values.scheduler.patch.image.pullPolicy }} + {{- end }} + args: + - patch + - --webhook-name={{ include "hami-vgpu.scheduler.webhook" . }} + - --namespace={{ include "hami-vgpu.namespace" . }} + - --patch-validating=false + - --secret-name={{ include "hami-vgpu.scheduler.tls" . }} + restartPolicy: OnFailure + serviceAccountName: {{ include "hami-vgpu.fullname" . }}-admission + {{- if .Values.scheduler.patch.nodeSelector }} + nodeSelector: {{ toYaml .Values.scheduler.patch.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.scheduler.patch.tolerations }} + tolerations: {{ toYaml .Values.scheduler.patch.tolerations | nindent 8 }} + {{- end }} + securityContext: + runAsNonRoot: true + runAsUser: {{ .Values.scheduler.patch.runAsUser }} +{{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml new file mode 100644 index 00000000..a48270e7 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml @@ -0,0 +1,40 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} +{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +{{- if .Values.podSecurityPolicy.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +spec: + allowPrivilegeEscalation: false + fsGroup: + ranges: + - max: 65535 + min: 1 + rule: MustRunAs + requiredDropCapabilities: + - ALL + runAsUser: + rule: MustRunAsNonRoot + seLinux: + rule: RunAsAny + supplementalGroups: + ranges: + - max: 65535 + min: 1 + rule: MustRunAs + volumes: + - configMap + - emptyDir + - projected + - secret + - downwardAPI +{{- end }} +{{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml new file mode 100644 index 00000000..74e45681 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml @@ -0,0 +1,23 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} +{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission + namespace: {{ include "hami-vgpu.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create +{{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml new file mode 100644 index 00000000..32da5ba4 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml @@ -0,0 +1,23 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} +{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission + namespace: {{ include "hami-vgpu.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "hami-vgpu.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.fullname" . }}-admission + namespace: {{ include "hami-vgpu.namespace" . }} +{{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml new file mode 100644 index 00000000..d2eb41de --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) }} +{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission + namespace: {{ include "hami-vgpu.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +{{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/role.yaml b/packages/system/hami/charts/hami/templates/scheduler/role.yaml new file mode 100644 index 00000000..6e681967 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/role.yaml @@ -0,0 +1,14 @@ +{{- if not .Values.dra.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create", "list", "watch", "get", "update", "patch"] +{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml new file mode 100644 index 00000000..934f8223 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml @@ -0,0 +1,33 @@ +{{- if not .Values.dra.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "hami-vgpu.scheduler" . }} +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +--- +{{- if .Values.mockDevicePlugin.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.mock-device-plugin" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "hami-vgpu.mock-device-plugin" . }} +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.mock-device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +{{- end -}} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/service.yaml b/packages/system/hami/charts/hami/templates/scheduler/service.yaml new file mode 100644 index 00000000..3e18bc98 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/service.yaml @@ -0,0 +1,36 @@ +{{- if not .Values.dra.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} + {{- if .Values.scheduler.service.labels }} + {{ toYaml .Values.scheduler.service.labels | indent 4 }} + {{- end }} + {{- if .Values.scheduler.service.annotations }} + annotations: {{ toYaml .Values.scheduler.service.annotations | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.scheduler.service.type | default "NodePort" }} # Default type is NodePort + ports: + - name: http + port: {{ .Values.scheduler.service.httpPort | default 443 }} # Default HTTP port is 443 + targetPort: {{ .Values.scheduler.service.httpTargetPort | default 443 }} + {{- if eq (.Values.scheduler.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort + nodePort: {{ .Values.scheduler.service.schedulerPort | default 31998 }} + {{- end }} + protocol: TCP + - name: monitor + port: {{ .Values.scheduler.service.monitorPort | default 31993 }} # Default monitoring port is 31993 + targetPort: {{ .Values.scheduler.service.monitorTargetPort | default 9395 }} + {{- if eq (.Values.scheduler.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort + nodePort: {{ .Values.scheduler.service.monitorPort | default 31993 }} + {{- end }} + protocol: TCP + selector: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.selectorLabels" . | nindent 4 }} +{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml b/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml new file mode 100644 index 00000000..929bf59a --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml @@ -0,0 +1,18 @@ +{{- if not .Values.dra.enabled -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +--- +{{- if .Values.mockDevicePlugin.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "hami-vgpu.mock-device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +{{- end -}} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml b/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml new file mode 100644 index 00000000..148feb5c --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml @@ -0,0 +1,57 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + {{- if .Values.scheduler.certManager.enabled }} + annotations: + cert-manager.io/inject-ca-from: {{ include "hami-vgpu.namespace" . }}/{{ include "hami-vgpu.scheduler" . }}-serving-cert + {{- end }} + name: {{ include "hami-vgpu.scheduler.webhook" . }} +webhooks: + - admissionReviewVersions: + - v1beta1 + clientConfig: + {{- if .Values.scheduler.admissionWebhook.customURL.enabled }} + url: https://{{ .Values.scheduler.admissionWebhook.customURL.host}}:{{.Values.scheduler.admissionWebhook.customURL.port}}{{.Values.scheduler.admissionWebhook.customURL.path}} + {{- else }} + service: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + path: /webhook + port: {{ .Values.scheduler.service.httpPort }} + {{- end }} + failurePolicy: {{ .Values.scheduler.admissionWebhook.failurePolicy }} + matchPolicy: Equivalent + name: vgpu.hami.io + namespaceSelector: + matchExpressions: + - key: hami.io/webhook + operator: NotIn + values: + - ignore + {{- if .Values.scheduler.admissionWebhook.whitelistNamespaces }} + - key: kubernetes.io/metadata.name + operator: NotIn + values: + {{- toYaml .Values.scheduler.admissionWebhook.whitelistNamespaces | nindent 10 }} + {{- end }} + objectSelector: + matchExpressions: + - key: hami.io/webhook + operator: NotIn + values: + - ignore + reinvocationPolicy: {{ .Values.scheduler.admissionWebhook.reinvocationPolicy }} + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - pods + scope: '*' + sideEffects: None + timeoutSeconds: 10 +{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/values.yaml b/packages/system/hami/charts/hami/values.yaml new file mode 100644 index 00000000..9fbeca70 --- /dev/null +++ b/packages/system/hami/charts/hami/values.yaml @@ -0,0 +1,476 @@ +## @section Global configuration +## @param global.imageRegistry Global Docker image registry +## @param global.imagePullSecrets Global Docker image pull secrets +global: + ## @param global.imageRegistry Global Docker image registry + imageRegistry: "" + ## E.g. + ## imagePullSecrets: + ## - myRegistryKeySecretName + ## @param global.imagePullSecrets Global Docker image pull secrets + imagePullSecrets: [] + imageTag: "v2.8.1" + gpuHookPath: /usr/local + labels: {} + annotations: {} + managedNodeSelectorEnable: false + managedNodeSelector: + usage: "gpu" + +nameOverride: "" +fullnameOverride: "" +namespaceOverride: "" + +#Nvidia GPU Parameters +resourceName: "nvidia.com/gpu" +resourceMem: "nvidia.com/gpumem" +resourceMemPercentage: "nvidia.com/gpumem-percentage" +resourceCores: "nvidia.com/gpucores" +resourcePriority: "nvidia.com/priority" + +#MLU Parameters +mluResourceName: "cambricon.com/vmlu" +mluResourceMem: "cambricon.com/mlu.smlu.vmemory" +mluResourceCores: "cambricon.com/mlu.smlu.vcore" + +#Hygon DCU Parameters +dcuResourceName: "hygon.com/dcunum" +dcuResourceMem: "hygon.com/dcumem" +dcuResourceCores: "hygon.com/dcucores" + +#Metax sGPU Parameters +metaxResourceName: "metax-tech.com/sgpu" +metaxResourceCore: "metax-tech.com/vcore" +metaxResourceMem: "metax-tech.com/vmemory" +metaxsGPUTopologyAware: "false" + +#Enflame VGCU Parameters +enflameResourceNameVGCU: "enflame.com/vgcu" +enflameResourceNameVGCUPercentage: "enflame.com/vgcu-percentage" + +#Kunlun XPU Parameters +kunlunResourceName: "kunlunxin.com/xpu" +kunlunResourceVCountName: "kunlunxin.com/vxpu" +kunlunResourceVMemoryName: "kunlunxin.com/vxpu-memory" + +schedulerName: "hami-scheduler" + +podSecurityPolicy: + enabled: false + +scheduler: + # @param nodeName defines the node name and the nvidia-vgpu-scheduler-scheduler will schedule to the node. + # if we install the nvidia-vgpu-scheduler-scheduler as default scheduler, we need to remove the k8s default + # scheduler pod from the cluster first, we must specify node name to skip the schedule workflow. + nodeName: "" + #nodeLabelSelector: + # "gpu": "on" + overwriteEnv: "false" + defaultSchedulerPolicy: + nodeSchedulerPolicy: binpack + gpuSchedulerPolicy: spread + metricsBindAddress: ":9395" + # If set to false, When Pod.Spec.SchedulerName equals to the const DefaultSchedulerName in k8s.io/api/core/v1 package, webhook will not overwrite it + forceOverwriteDefaultScheduler: true + livenessProbe: false + leaderElect: true + # when leaderElect is true, replicas is available, otherwise replicas is 1. + replicas: 1 + kubeScheduler: + # @param enabled indicate whether to run kube-scheduler container in the scheduler pod, it's true by default. + enabled: true + ## @param image.registry kube scheduler image registry + ## @param image.repository kube scheduler image repository + ## @param image.tag kube scheduler image tag (immutable tags are recommended) + ## @param image.pullPolicy kube scheduler image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + image: + registry: "registry.cn-hangzhou.aliyuncs.com" + repository: "google_containers/kube-scheduler" + tag: "" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + resources: {} + # If you do want to specify resources, uncomment the following lines, adjust them as necessary. + # and remove the curly braces after 'resources:'. +# limits: +# cpu: 1000m +# memory: 1000Mi +# requests: +# cpu: 100m +# memory: 100Mi + extraNewArgs: + - --config=/config/config.yaml + - -v=4 + extraArgs: + - --policy-config-file=/config/config.json + - -v=4 + extender: + ## @param image.registry scheduler extender image registry + ## @param image.repository scheduler extender image repository + ## @param image.tag scheduler extender image tag (immutable tags are recommended) + ## @param image.pullPolicy scheduler extender image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + image: + registry: "docker.io" + repository: "projecthami/hami" + tag: "" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + resources: {} + # If you do want to specify resources, uncomment the following lines, adjust them as necessary, + # and remove the curly braces after 'resources:'. +# limits: +# cpu: 1000m +# memory: 1000Mi +# requests: +# cpu: 100m +# memory: 100Mi + extraArgs: + - --debug + - -v=4 + nodeLockExpire: "5m" + podAnnotations: {} + tolerations: [] + #serviceAccountName: "hami-vgpu-scheduler-sa" + admissionWebhook: + # If set to false, the admission webhook is not installed and any pods that should use HAMi must be + # configured to use the right 'schedulerName' and other device-specific configurations. + enabled: true + customURL: + enabled: false + # must be an endpoint using https. + # should generate host certs here + host: 127.0.0.1 # hostname or ip, can be your node'IP if you want to use https://:/ + port: 31998 + path: /webhook + whitelistNamespaces: + # Specify the namespaces that the webhook will not be applied to. + # - default + # - kube-system + # - istio-system + reinvocationPolicy: Never + failurePolicy: Ignore + ## TLS Certificate Option 1: Use cert-manager to generate self-signed certificate. + ## If enabled, always takes precedence over options 2. + certManager: + enabled: false + ## TLS Certificate Option 2: Use kube-webhook-certgen to generate self-signed certificate. + ## If true and certManager.enabled is false, Helm will automatically create a self-signed cert and secret for you. + patch: + enabled: true + ## @param image.registry scheduler extender image registry + ## @param image.repository scheduler extender image repository + ## @param image.tag scheduler extender image tag (immutable tags are recommended) + ## @param image.pullPolicy scheduler extender image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + image: + registry: "docker.io" + repository: "jettech/kube-webhook-certgen" + tag: "v1.5.2" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## @param image.registry scheduler extender image registry + ## @param image.repository scheduler extender image repository + ## @param image.tag scheduler extender image tag (immutable tags are recommended) + ## @param image.pullPolicy scheduler extender image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + imageNew: + registry: "docker.io" + repository: "liangjw/kube-webhook-certgen" + tag: "v1.1.1" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + priorityClassName: "" + podAnnotations: {} + nodeSelector: {} + tolerations: [] + runAsUser: 2000 + service: + type: NodePort # Default type is NodePort, can be changed to ClusterIP + httpPort: 443 # HTTP port + schedulerPort: 31998 # NodePort for HTTP + monitorPort: 31993 # Monitoring port + monitorTargetPort: 9395 + httpTargetPort: 443 + labels: {} + annotations: {} + +devicePlugin: + enabled: true + gpuOperatorToolkitReady: + enabled: false + hostPath: "/run/nvidia/validations" + ## @param image.registry devicePlugin image registry + ## @param image.repository devicePlugin image repository + ## @param image.tag devicePlugin image tag (immutable tags are recommended) + ## @param image.pullPolicy devicePlugin image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + image: + registry: "docker.io" + repository: "projecthami/hami" + tag: "" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + monitor: + ## @param image.registry monitor image registry + ## @param image.repository monitor image repository + ## @param image.tag monitor image tag (immutable tags are recommended) + ## @param image.pullPolicy monitor image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + image: + registry: "docker.io" + repository: "projecthami/hami" + tag: "" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ctrPath: /usr/local/vgpu/containers + resyncInterval: "5m" + extraArgs: + - -v=4 + extraEnvs: {} + resources: {} + # If you do want to specify resources, uncomment the following lines, adjust them as necessary. + # and remove the curly braces after 'resources:'. +# limits: +# cpu: 1000m +# memory: 1000Mi +# requests: +# cpu: 100m +# memory: 100Mi + + deviceSplitCount: 10 + deviceMemoryScaling: 1 + deviceCoreScaling: 1 + # Node configuration for device plugin, Priority: externalConfigName > config > default config + nodeConfiguration: + # If you want to use a custom config.json, you can set the content here. + # If this is set, it will override the default config.json(An example is as follows). + config: | + { + "nodeconfig": [ + { + "name": "your-node-name", + "operatingmode": "hami-core", + "devicememoryscaling": 1, + "devicesplitcount": 10, + "migstrategy": "none", + "filterdevices": { + "uuid": [], + "index": [] + } + } + ] + } + # If you want to use an existing ConfigMap, you can set the name here. + # If this is set, the chart will not create the ConfigMap and will use the existing one. + externalConfigName: "" + # The runtime class name to be used by the device plugin, and added to the pod.spec.runtimeClassName of applications utilizing NVIDIA GPUs + runtimeClassName: "" + # Whether to create runtime class, name comes from runtimeClassName when it is set + createRuntimeClass: false + migStrategy: "none" + disablecorelimit: "false" + passDeviceSpecsEnabled: false + deviceListStrategy: "envvar" + nvidiaHookPath: null + nvidiaDriverRoot: null + gdrcopyEnabled: null + gdsEnabled: null + mofedEnabled: null + + extraArgs: + - -v=4 + extraEnvs: {} + service: + type: NodePort # Default type is NodePort, can be changed to ClusterIP + httpPort: 31992 + labels: {} + annotations: {} + + pluginPath: /var/lib/kubelet/device-plugins + libPath: /usr/local/vgpu + + podAnnotations: {} + nvidiaNodeSelector: + gpu: "on" + tolerations: [] + # The updateStrategy for DevicePlugin DaemonSet. + # If you want to update the DaemonSet by manual, set type as "OnDelete". + # We recommend use OnDelete update strategy because DevicePlugin pod restart will cause business pod restart, this behavior is destructive. + # Otherwise, you can use RollingUpdate update strategy to rolling update DevicePlugin pod. + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + + resources: {} + # If you do want to specify resources, uncomment the following lines, adjust them as necessary. + # and remove the curly braces after 'resources:'. +# limits: +# cpu: 1000m +# memory: 1000Mi +# requests: +# cpu: 100m +# memory: 100Mi + +mockDevicePlugin: + enabled: false + image: + registry: "docker.io" + repository: "projecthami/mock-device-plugin" + tag: "1.0.1" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] +# If this option is enabled, the DRA will be installed and the scheduler extender will not be installed. +dra: + enabled: false + +# Configuration for the hami-dra subchart, which includes DRA drivers. +# This is part of config for DRA, please refer to the DRA documentation for more details. +hami-dra: + monitor: + enabled: true + drivers: + nvidia: + enabled: true + # If you are using gpu driver on host, you need to set this to false + containerDriver: true + image: + repository: projecthami/k8s-dra-driver + tag: "v0.0.1-dev" + +devices: + amd: + customresources: + - amd.com/gpu + - amd.com/gpu-memory + awsneuron: + customresources: + - aws.amazon.com/neuron + - aws.amazon.com/neuroncore + kunlun: + enabled: true + customresources: + - kunlunxin.com/xpu + - kunlunxin.com/vxpu + - kunlunxin.com/vxpu-memory + enflame: + enabled: true + customresources: + - enflame.com/vgcu + - enflame.com/vgcu-percentage + - enflame.com/gcu + mthreads: + enabled: true + customresources: + - mthreads.com/vgpu + nvidia: + gpuCorePolicy: default + libCudaLogLevel: 1 + ascend: + enabled: false + image: "" + imagePullPolicy: IfNotPresent + extraArgs: [] + nodeSelector: + ascend: "on" + tolerations: [] + customresources: + - huawei.com/Ascend910A + - huawei.com/Ascend910A-memory + - huawei.com/Ascend910B2 + - huawei.com/Ascend910B2-memory + - huawei.com/Ascend910B3 + - huawei.com/Ascend910B3-memory + - huawei.com/Ascend910B4 + - huawei.com/Ascend910B4-memory + - huawei.com/Ascend910B4-1 + - huawei.com/Ascend910B4-1-memory + - huawei.com/Ascend310P + - huawei.com/Ascend310P-memory + iluvatar: + enabled: false + customresources: + - iluvatar.ai/BI-V100-vgpu + - iluvatar.ai/BI-V100.vCore + - iluvatar.ai/BI-V100.vMem + - iluvatar.ai/BI-V150-vgpu + - iluvatar.ai/BI-V150.vCore + - iluvatar.ai/BI-V150.vMem + - iluvatar.ai/MR-V100-vgpu + - iluvatar.ai/MR-V100.vCore + - iluvatar.ai/MR-V100.vMem + - iluvatar.ai/MR-V50-vgpu + - iluvatar.ai/MR-V50.vCore + - iluvatar.ai/MR-V50.vMem diff --git a/packages/system/hami/values.yaml b/packages/system/hami/values.yaml new file mode 100644 index 00000000..1155e2cb --- /dev/null +++ b/packages/system/hami/values.yaml @@ -0,0 +1,8 @@ +hami: + devicePlugin: + runtimeClassName: nvidia + nodeConfiguration: + config: | + { + "nodeconfig": [] + } From a3e2fbd742d0c9f8bf6a255836913f080f42b826 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Wed, 22 Apr 2026 13:25:23 +0300 Subject: [PATCH 02/27] feat(kubernetes): integrate HAMi as optional addon Add HAMi HelmRelease to the kubernetes app as a toggleable addon. When enabled, GPU Operator's built-in device plugin is automatically disabled via a values merge to avoid conflicts with HAMi's own device plugin. The HelmRelease fails fast if GPU Operator is not enabled, since HAMi depends on it for driver management and container toolkit. Assisted-By: Claude Signed-off-by: Arsolitt --- .../templates/helmreleases/gpu-operator.yaml | 13 ++++- .../templates/helmreleases/hami.yaml | 49 +++++++++++++++++++ packages/apps/kubernetes/values.schema.json | 23 +++++++++ packages/apps/kubernetes/values.yaml | 8 +++ 4 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 packages/apps/kubernetes/templates/helmreleases/hami.yaml diff --git a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml index 5ef48912..995aa430 100644 --- a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml @@ -1,3 +1,11 @@ +{{- define "cozystack.defaultGpuOperatorValues" -}} +{{- if and (hasKey .Values.addons "hami") .Values.addons.hami.enabled }} +gpu-operator: + devicePlugin: + enabled: false +{{- end }} +{{- end }} + {{- if .Values.addons.gpuOperator.enabled }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease @@ -29,7 +37,10 @@ spec: force: true remediation: retries: -1 - {{- with .Values.addons.gpuOperator.valuesOverride }} + {{- $defaults := fromYaml (include "cozystack.defaultGpuOperatorValues" .) }} + {{- $overrides := deepCopy (default (dict) .Values.addons.gpuOperator.valuesOverride) }} + {{- $merged := mergeOverwrite (default (dict) $defaults) $overrides }} + {{- with $merged }} values: {{- toYaml . | nindent 4 }} {{- end }} diff --git a/packages/apps/kubernetes/templates/helmreleases/hami.yaml b/packages/apps/kubernetes/templates/helmreleases/hami.yaml new file mode 100644 index 00000000..1f7a5358 --- /dev/null +++ b/packages/apps/kubernetes/templates/helmreleases/hami.yaml @@ -0,0 +1,49 @@ +{{- if .Values.addons.hami.enabled }} +{{- if not .Values.addons.gpuOperator.enabled }} +{{- fail "addons.hami requires addons.gpuOperator to be enabled" }} +{{- end }} +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: {{ .Release.Name }}-hami + labels: + cozystack.io/repository: system + cozystack.io/target-cluster-name: {{ .Release.Name }} + sharding.fluxcd.io/key: tenants +spec: + releaseName: hami + chartRef: + kind: ExternalArtifact + name: cozystack-kubernetes-application-kubevirt-kubernetes-hami + namespace: cozy-system + kubeConfig: + secretRef: + name: {{ .Release.Name }}-admin-kubeconfig + key: super-admin.svc + targetNamespace: cozy-hami + storageNamespace: cozy-hami + interval: 5m + timeout: 10m + install: + createNamespace: true + remediation: + retries: -1 + upgrade: + force: true + remediation: + retries: -1 + {{- with .Values.addons.hami.valuesOverride }} + values: + {{- toYaml . | nindent 4 }} + {{- end }} + + dependsOn: + {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }} + - name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} + {{- end }} + - name: {{ .Release.Name }}-cilium + namespace: {{ .Release.Namespace }} + - name: {{ .Release.Name }}-gpu-operator + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/packages/apps/kubernetes/values.schema.json b/packages/apps/kubernetes/values.schema.json index 1beeff9c..3d7a17dd 100644 --- a/packages/apps/kubernetes/values.schema.json +++ b/packages/apps/kubernetes/values.schema.json @@ -149,6 +149,7 @@ "fluxcd", "gatewayAPI", "gpuOperator", + "hami", "ingressNginx", "monitoringAgents", "velero", @@ -268,6 +269,28 @@ } } }, + "hami": { + "description": "HAMi GPU virtualization middleware. Enables fractional GPU sharing (memory and compute isolation) for workloads. Requires GPU Operator. Note: workload containers must use glibc < 2.34 (not musl/Alpine) for full compute isolation.", + "type": "object", + "default": {}, + "required": [ + "enabled", + "valuesOverride" + ], + "properties": { + "enabled": { + "description": "Enable HAMi.", + "type": "boolean", + "default": false + }, + "valuesOverride": { + "description": "Custom Helm values overrides.", + "type": "object", + "default": {}, + "x-kubernetes-preserve-unknown-fields": true + } + } + }, "ingressNginx": { "description": "Ingress-NGINX controller.", "type": "object", diff --git a/packages/apps/kubernetes/values.yaml b/packages/apps/kubernetes/values.yaml index a67b5d69..f601b979 100644 --- a/packages/apps/kubernetes/values.yaml +++ b/packages/apps/kubernetes/values.yaml @@ -94,6 +94,10 @@ host: "" ## @field {bool} enabled - Enable FluxCD. ## @field {object} valuesOverride - Custom Helm values overrides. +## @typedef {struct} HAMiAddon - HAMi GPU virtualization middleware. +## @field {bool} enabled - Enable HAMi (requires GPU Operator). +## @field {object} valuesOverride - Custom Helm values overrides. + ## @typedef {struct} MonitoringAgentsAddon - Monitoring agents (Fluent Bit, VMAgents). ## @field {bool} enabled - Enable monitoring agents. ## @field {object} valuesOverride - Custom Helm values overrides. @@ -114,6 +118,7 @@ host: "" ## @field {GatewayAPIAddon} gatewayAPI - Gateway API addon. ## @field {IngressNginxAddon} ingressNginx - Ingress-NGINX controller. ## @field {GPUOperatorAddon} gpuOperator - NVIDIA GPU Operator. +## @field {HAMiAddon} hami - HAMi GPU virtualization middleware. ## @field {FluxCDAddon} fluxcd - FluxCD GitOps operator. ## @field {MonitoringAgentsAddon} monitoringAgents - Monitoring agents. ## @field {VerticalPodAutoscalerAddon} verticalPodAutoscaler - Vertical Pod Autoscaler. @@ -137,6 +142,9 @@ addons: gpuOperator: enabled: false valuesOverride: {} + hami: + enabled: false + valuesOverride: {} fluxcd: enabled: false valuesOverride: {} From 273eb7811a36fb69141d80360705f081a3276194 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Wed, 22 Apr 2026 13:25:29 +0300 Subject: [PATCH 03/27] docs(hami): document glibc < 2.34 limitation and upstream issues HAMi-core relies on _dl_sym (private glibc symbol removed in 2.34) for CUDA interception. This breaks compute isolation on modern container images using Ubuntu 22.04+ and makes Alpine/musl completely incompatible. Include upstream issue references and a compatibility matrix so users can make informed decisions about base image selection. Assisted-By: Claude Signed-off-by: Arsolitt --- packages/system/hami/README.md | 81 ++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 packages/system/hami/README.md diff --git a/packages/system/hami/README.md b/packages/system/hami/README.md new file mode 100644 index 00000000..283f9e80 --- /dev/null +++ b/packages/system/hami/README.md @@ -0,0 +1,81 @@ +# HAMi — GPU Virtualization Middleware + +[HAMi](https://github.com/Project-HAMi/HAMi) (Heterogeneous AI Computing Virtualization Middleware) is a CNCF Sandbox project that enables fractional GPU sharing in Kubernetes. It allows workloads to request specific amounts of GPU memory and compute cores instead of claiming entire GPUs. + +## Architecture + +HAMi consists of four components: + +- **MutatingWebhook** — intercepts pod creation, injects `schedulerName: hami-scheduler` +- **Scheduler Extender** — extends kube-scheduler with GPU-aware Filter and Bind logic +- **Device Plugin** (DaemonSet) — registers vGPU resources via the Kubernetes Device Plugin API +- **HAMi-core** (`libvgpu.so`) — `LD_PRELOAD` library injected into workload containers, intercepts CUDA API calls to enforce memory and compute isolation + +## Prerequisites + +- GPU Operator must be enabled (`addons.gpuOperator.enabled: true`) +- NVIDIA driver >= 440 on host nodes +- nvidia-container-toolkit configured as the default container runtime +- GPU nodes labeled with `gpu=on` + +## Known Limitations + +### glibc < 2.34 requirement for workload containers + +HAMi-core uses `LD_PRELOAD` to intercept `dlsym()` for CUDA symbol resolution. The fallback code path relies on `_dl_sym`, a private glibc internal symbol that was removed in glibc 2.34 when libdl and libpthread were merged into libc.so. + +**This limitation affects workload containers only**, not the host OS or HAMi's own components. + +| Distribution | glibc | Result | +| --------------- | ----- | -------------------------------------------- | +| Ubuntu 18.04 | 2.27 | Full isolation (memory + compute) | +| Ubuntu 20.04 | 2.31 | Full isolation (memory + compute) | +| Ubuntu 22.04 | 2.35 | Memory isolation works, compute breaks | +| Ubuntu 24.04 | 2.39 | Both memory and compute isolation break | +| Alpine (musl) | N/A | Completely incompatible (`dlvsym` absent) | + +Most modern ML/AI base images (CUDA 12.x, PyTorch 2.x, TensorFlow 2.x) use Ubuntu 22.04+ with glibc >= 2.35, which means compute isolation will not work with these images until the upstream fix is merged. + +**Upstream tracking issues:** + +- [HAMi-core#174](https://github.com/Project-HAMi/HAMi-core/issues/174) — `_dl_sym` removal breaks HAMi-core on glibc >= 2.34 +- [HAMi#1190](https://github.com/Project-HAMi/HAMi/issues/1190) — degraded isolation across glibc versions +- [HAMi#173](https://github.com/Project-HAMi/HAMi/issues/173) — documentation incorrectly states glibc < 2.30 (actual boundary is 2.34) + +### musl libc (Alpine) incompatibility + +HAMi-core is completely incompatible with musl libc. The `dlvsym()` function used by HAMi-core is a glibc extension not available in musl. Only glibc-based container images (Debian, Ubuntu, RHEL, etc.) can use HAMi GPU isolation. + +## Usage + +Enable HAMi in your tenant Kubernetes cluster values: + +```yaml +addons: + gpuOperator: + enabled: true + hami: + enabled: true +``` + +When HAMi is enabled, GPU Operator's built-in device plugin is automatically disabled to avoid conflicts. + +### Requesting fractional GPU resources + +```yaml +resources: + limits: + nvidia.com/gpu: 1 + nvidia.com/gpumem: 3000 # 3000 MB of GPU memory + nvidia.com/gpucores: 30 # 30% of GPU compute cores +``` + +## Parameters + +| Name | Description | Default | +| --- | --- | --- | +| `hami.devicePlugin.runtimeClassName` | RuntimeClass for device plugin pods | `nvidia` | +| `hami.devicePlugin.deviceSplitCount` | Max virtual GPUs per physical GPU | `10` | +| `hami.devicePlugin.deviceMemoryScaling` | Memory overcommit factor (> 1.0 enables overcommit) | `1` | +| `hami.scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy` | Node packing strategy (`binpack` or `spread`) | `binpack` | +| `hami.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy` | GPU packing strategy (`binpack` or `spread`) | `spread` | From 43035562efff34f18c3c439026fe2a233dc8d17b Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Wed, 22 Apr 2026 13:25:35 +0300 Subject: [PATCH 04/27] test(kubernetes): add helm-unittest tests for HAMi integration Cover HelmRelease rendering, gpuOperator dependency validation, ExternalArtifact chartRef, namespace targeting, dependency chain, valuesOverride passthrough, and automatic devicePlugin disable in GPU Operator when HAMi is active. Assisted-By: Claude Signed-off-by: Arsolitt --- .../tests/gpu_operator_hami_test.yaml | 80 +++++++++++ packages/apps/kubernetes/tests/hami_test.yaml | 136 ++++++++++++++++++ 2 files changed, 216 insertions(+) create mode 100644 packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml create mode 100644 packages/apps/kubernetes/tests/hami_test.yaml diff --git a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml new file mode 100644 index 00000000..33256ae6 --- /dev/null +++ b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml @@ -0,0 +1,80 @@ +suite: GPU Operator HelmRelease HAMi integration tests +templates: + - templates/helmreleases/gpu-operator.yaml +tests: + - it: should disable devicePlugin when hami is enabled + set: + addons: + gpuOperator: + enabled: true + valuesOverride: {} + hami: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.values.gpu-operator.devicePlugin.enabled + value: false + + - it: should not have values when hami is disabled and no overrides + set: + addons: + gpuOperator: + enabled: true + valuesOverride: {} + hami: + enabled: false + valuesOverride: {} + asserts: + - notExists: + path: spec.values + + - it: should allow user overrides to merge with hami defaults + set: + addons: + gpuOperator: + enabled: true + valuesOverride: + gpu-operator: + driver: + enabled: false + hami: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.values.gpu-operator.devicePlugin.enabled + value: false + - equal: + path: spec.values.gpu-operator.driver.enabled + value: false + + - it: should let user override devicePlugin back to true if needed + set: + addons: + gpuOperator: + enabled: true + valuesOverride: + gpu-operator: + devicePlugin: + enabled: true + hami: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.values.gpu-operator.devicePlugin.enabled + value: true + + - it: should not render when gpuOperator is disabled + set: + addons: + gpuOperator: + enabled: false + valuesOverride: {} + hami: + enabled: false + valuesOverride: {} + asserts: + - hasDocuments: + count: 0 diff --git a/packages/apps/kubernetes/tests/hami_test.yaml b/packages/apps/kubernetes/tests/hami_test.yaml new file mode 100644 index 00000000..e9d32236 --- /dev/null +++ b/packages/apps/kubernetes/tests/hami_test.yaml @@ -0,0 +1,136 @@ +suite: HAMi HelmRelease tests +templates: + - templates/helmreleases/hami.yaml +tests: + - it: should not render when hami is disabled + set: + addons: + hami: + enabled: false + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - hasDocuments: + count: 0 + + - it: should render HelmRelease when hami is enabled + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - hasDocuments: + count: 1 + - isKind: + of: HelmRelease + + - it: should fail when gpuOperator is not enabled + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: false + valuesOverride: {} + asserts: + - failedTemplate: + errorMessage: "addons.hami requires addons.gpuOperator to be enabled" + + - it: should have correct metadata labels + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: metadata.labels["cozystack.io/repository"] + value: system + - equal: + path: metadata.labels["sharding.fluxcd.io/key"] + value: tenants + + - it: should use ExternalArtifact chartRef + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.chartRef.kind + value: ExternalArtifact + - equal: + path: spec.chartRef.namespace + value: cozy-system + + - it: should target cozy-hami namespace + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.targetNamespace + value: cozy-hami + - equal: + path: spec.storageNamespace + value: cozy-hami + + - it: should depend on gpu-operator and cilium + release: + name: test + namespace: test-ns + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - contains: + path: spec.dependsOn + content: + name: test-cilium + namespace: test-ns + - contains: + path: spec.dependsOn + content: + name: test-gpu-operator + namespace: test-ns + + - it: should pass through valuesOverride + set: + addons: + hami: + enabled: true + valuesOverride: + hami: + devicePlugin: + deviceSplitCount: 5 + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.values.hami.devicePlugin.deviceSplitCount + value: 5 From 3c5521ee992b2dce74ff8956a6d8c581763bf640 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Thu, 23 Apr 2026 22:16:30 +0300 Subject: [PATCH 05/27] fix(hami): remove broken hami-dra subchart The hami-dra subchart renders resources even when dra.enabled=false because Helm dependency conditions don't work for vendored subcharts. The subchart also contains duplicate YAML label keys (app.kubernetes.io/component, app.kubernetes.io/name) and references unpublished container images (v0.0.1-dev). DRA support can be re-added when the upstream project stabilizes it. Signed-off-by: Arsolitt --- .../charts/hami/charts/hami-dra/.helmignore | 24 -- .../charts/hami/charts/hami-dra/Chart.yaml | 18 - .../charts/hami-dra/templates/_commons.tpl | 84 ---- .../charts/hami-dra/templates/_helpers.tpl | 73 ---- .../hami-dra-driver/deviceclass.yaml | 11 - .../nvidia-dra-driver-daemonset.yaml | 145 ------- .../templates/hami-dra-driver/rbac.yaml | 110 ----- .../monitor/monitor-clusterrole.yaml | 11 - .../monitor/monitor-clusterrolebinding.yaml | 15 - .../templates/monitor/monitor-deployment.yaml | 62 --- .../templates/monitor/monitor-service.yaml | 26 -- .../monitor/monitor-serviceaccount.yaml | 8 - .../templates/webhook/cert-manager.yaml | 29 -- .../templates/webhook/cert-secret.yaml | 13 - .../templates/webhook/device-config.yaml | 394 ------------------ .../webhook/mutatingwebhookconfiguration.yaml | 28 -- .../validatingwebhookconfiguration.yaml | 29 -- .../webhook/webhook-clusterrole.yaml | 14 - .../webhook/webhook-clusterrolebinding.yaml | 12 - .../templates/webhook/webhook-deployment.yaml | 59 --- .../templates/webhook/webhook-service.yaml | 15 - .../webhook/webhook-serviceaccount.yaml | 5 - .../charts/hami/charts/hami-dra/values.yaml | 166 -------- 23 files changed, 1351 deletions(-) delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/.helmignore delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/_commons.tpl delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/_helpers.tpl delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/deviceclass.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/nvidia-dra-driver-daemonset.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/rbac.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrole.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrolebinding.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-deployment.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-service.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-serviceaccount.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-manager.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-secret.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/device-config.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/mutatingwebhookconfiguration.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/validatingwebhookconfiguration.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrole.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrolebinding.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-deployment.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-service.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-serviceaccount.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/values.yaml diff --git a/packages/system/hami/charts/hami/charts/hami-dra/.helmignore b/packages/system/hami/charts/hami/charts/hami-dra/.helmignore deleted file mode 100644 index 898df488..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/.helmignore +++ /dev/null @@ -1,24 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ - diff --git a/packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml b/packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml deleted file mode 100644 index d01f678f..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: v2 -appVersion: 0.1.0 -description: A Helm chart for HAMi DRA -home: https://github.com/Project-HAMi/HAMi-DRA -keywords: -- webhook -- kubernetes -- admission-controller -- hami -- dra -maintainers: -- name: hami-dra - url: https://github.com/Project-HAMi/HAMi-DRA -name: hami-dra -sources: -- https://github.com/Project-HAMi/HAMi-DRA -type: application -version: 0.1.0 diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/_commons.tpl b/packages/system/hami/charts/hami/charts/hami-dra/templates/_commons.tpl deleted file mode 100644 index 02fbed20..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/_commons.tpl +++ /dev/null @@ -1,84 +0,0 @@ -{{/* vim: set filetype=mustache: */}} - -{{/* -Return the proper image name -{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" $) }} -*/}} -{{- define "common.images.image" -}} -{{- $registryName := .imageRoot.registry -}} -{{- $repositoryName := .imageRoot.repository -}} -{{- $tag := .imageRoot.tag | toString -}} -{{- if .global }} - {{- if .global.imageRegistry }} - {{- $registryName = .global.imageRegistry -}} - {{- end -}} -{{- end -}} -{{- if .tag }} - {{- if .tag.imageTag }} - {{- $tag = .tag.imageTag -}} - {{- end -}} -{{- end -}} -{{- if $registryName }} -{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} -{{- else -}} -{{- printf "%s:%s" $repositoryName $tag -}} -{{- end -}} -{{- end -}} - -{{/* -Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead) -{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }} -*/}} -{{- define "common.images.pullSecrets" -}} - {{- $pullSecrets := list }} - - {{- if .global }} - {{- range .global.imagePullSecrets -}} - {{- $pullSecrets = append $pullSecrets . -}} - {{- end -}} - {{- end -}} - - {{- range .images -}} - {{- range .pullSecrets -}} - {{- $pullSecrets = append $pullSecrets . -}} - {{- end -}} - {{- end -}} - - {{- if (not (empty $pullSecrets)) }} -imagePullSecrets: - {{- range $pullSecrets }} - - name: {{ . }} - {{- end }} - {{- end }} -{{- end -}} - -{{/* -Renders a value that contains template. -Usage: -{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $) }} -*/}} -{{- define "common.tplvalues.render" -}} - {{- if typeIs "string" .value }} - {{- tpl .value .context }} - {{- else }} - {{- tpl (.value | toYaml) .context }} - {{- end }} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "common.names.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/_helpers.tpl b/packages/system/hami/charts/hami/charts/hami-dra/templates/_helpers.tpl deleted file mode 100644 index 99beec77..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/_helpers.tpl +++ /dev/null @@ -1,73 +0,0 @@ -{{- define "hami.dra.webhook.fullname" -}} -{{- printf "%s-%s" (include "common.names.fullname" .) "webhook" | trunc 63 | trimSuffix "-" -}} -{{- end }} - -{{- define "hami.dra.webhook.image" -}} -{{ include "common.images.image" (dict "imageRoot" .Values.webhook.image "global" .Values.global) }} -{{- end -}} - -{{- define "hami.dra.webhook.imagePullSecrets" -}} -{{ include "common.images.pullSecrets" (dict "images" (list .Values.webhook.image) "global" .Values.global) }} -{{- end -}} - -{{- define "hami.dra.driver.nvidia.image" -}} -{{ include "common.images.image" (dict "imageRoot" .Values.drivers.nvidia.image "global" .Values.global) }} -{{- end -}} - -{{- define "hami.dra.driver.nvidia.imagePullSecrets" -}} -{{ include "common.images.pullSecrets" (dict "images" (list .Values.drivers.nvidia.image) "global" .Values.global) }} -{{- end -}} - -{{/* -Common labels -*/}} -{{- define "hami-dra-webhook.labels" -}} -helm.sh/chart: {{ .Chart.Name }} -{{ include "hami-dra-webhook.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "hami-dra-webhook.selectorLabels" -}} -app.kubernetes.io/name: {{ .Release.Name }} -app.kubernetes.io/instance: {{ .Release.Name }} -app.kubernetes.io/component: webhook -{{- end }} - -{{- define "hami.dra.monitor.fullname" -}} -{{- printf "%s-%s" (include "common.names.fullname" .) "monitor" | trunc 63 | trimSuffix "-" -}} -{{- end }} - -{{- define "hami.dra.monitor.image" -}} -{{ include "common.images.image" (dict "imageRoot" .Values.monitor.image "global" .Values.global) }} -{{- end -}} - -{{- define "hami.dra.monitor.imagePullSecrets" -}} -{{ include "common.images.pullSecrets" (dict "images" (list .Values.monitor.image) "global" .Values.global) }} -{{- end -}} - -{{/* -Common labels for monitor -*/}} -{{- define "hami-dra-monitor.labels" -}} -helm.sh/chart: {{ .Chart.Name }} -{{ include "hami-dra-monitor.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels for monitor -*/}} -{{- define "hami-dra-monitor.selectorLabels" -}} -app.kubernetes.io/name: {{ .Release.Name }} -app.kubernetes.io/instance: {{ .Release.Name }} -app.kubernetes.io/component: monitor -{{- end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/deviceclass.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/deviceclass.yaml deleted file mode 100644 index bcb6d2c5..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/deviceclass.yaml +++ /dev/null @@ -1,11 +0,0 @@ -{{- if .Values.drivers.nvidia.enabled }} -apiVersion: resource.k8s.io/v1 -kind: DeviceClass -metadata: - name: hami-core-gpu.project-hami.io -spec: - selectors: - - cel: - expression: |- - device.driver == "hami-core-gpu.project-hami.io" && device.attributes["hami-core-gpu.project-hami.io"].type == "hami-gpu" -{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/nvidia-dra-driver-daemonset.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/nvidia-dra-driver-daemonset.yaml deleted file mode 100644 index a3fb0d01..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/nvidia-dra-driver-daemonset.yaml +++ /dev/null @@ -1,145 +0,0 @@ -{{- if .Values.drivers.nvidia.enabled }} -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: hami-dra-driver-kubelet-plugin - namespace: {{ .Release.Namespace }} -spec: - revisionHistoryLimit: 10 - selector: - matchLabels: - hami-dra-driver-component: kubelet-plugin - template: - metadata: - labels: - app.kubernetes.io/instance: hami-dra-driver - app.kubernetes.io/name: hami-dra-driver - hami-dra-driver-component: kubelet-plugin - spec: - {{- include "hami.dra.driver.nvidia.imagePullSecrets" . | nindent 6 }} - initContainers: - - name: init-container - image: {{ include "hami.dra.driver.nvidia.image" . }} - securityContext: - privileged: true - command: [bash, /usr/bin/kubelet-plugin-prestart.sh] - env: - - name: NVIDIA_DRIVER_ROOT - value: {{ if .Values.drivers.nvidia.containerDriver }}/run/nvidia/driver{{ else }}/{{ end }} - - name: NVIDIA_VISIBLE_DEVICES - value: void - - name: KUBELET_REGISTRAR_DIRECTORY_PATH - value: /var/lib/kubelet/plugins_registry - - name: KUBELET_PLUGINS_DIRECTORY_PATH - value: /var/lib/kubelet/plugins - volumeMounts: - - name: driver-root-parent - mountPath: /driver-root-parent - readOnly: true - containers: - - args: - - |- - # Conditionally mask the params file to prevent this container from - # recreating any missing GPU device nodes. This is necessary, for - # example, when running under nvkind to limit the set GPUs governed - # by the plugin even though it has cgroup access to all of them. - if [ "${MASK_NVIDIA_DRIVER_PARAMS}" = "true" ]; then - cp /proc/driver/nvidia/params root/gpu-params - sed -i 's/^ModifyDeviceFiles: 1$/ModifyDeviceFiles: 0/' root/gpu-params - mount --bind root/gpu-params /proc/driver/nvidia/params - fi - hami-kubelet-plugin -v 6 - command: - - bash - - -c - env: - - name: MASK_NVIDIA_DRIVER_PARAMS - value: "" - - name: NVIDIA_DRIVER_ROOT - value: {{ if .Values.drivers.nvidia.containerDriver }}/run/nvidia/driver{{ else }}/{{ end }} - - name: NVIDIA_VISIBLE_DEVICES - value: void - - name: CDI_ROOT - value: /var/run/cdi - - name: NVIDIA_MIG_CONFIG_DEVICES - value: all - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: KUBELET_REGISTRAR_DIRECTORY_PATH - value: /var/lib/kubelet/plugins_registry - - name: KUBELET_PLUGINS_DIRECTORY_PATH - value: /var/lib/kubelet/plugins - - name: IMAGE_NAME - value: {{ include "hami.dra.driver.nvidia.image" . }} - lifecycle: - postStart: - exec: - command: ["/bin/bash", "-c", "/usr/bin/vgpu-init.sh /usr/local/vgpu/"] - image: {{ include "hami.dra.driver.nvidia.image" . }} - imagePullPolicy: {{ .Values.drivers.nvidia.image.pullPolicy | default "IfNotPresent" }} - name: gpus - securityContext: - privileged: true - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - volumeMounts: - - mountPath: /var/lib/kubelet/plugins_registry - name: plugins-registry - - mountPath: /var/lib/kubelet/plugins - mountPropagation: Bidirectional - name: plugins - - mountPath: /var/run/cdi - name: cdi - - mountPath: /driver-root - mountPropagation: HostToContainer - name: driver-root - readOnly: true - - mountPath: /usr/local/vgpu - name: host-vgpu - - mountPath: /tmp - name: host-tmp - dnsPolicy: ClusterFirst - serviceAccount: hami-dra-driver-service-account - serviceAccountName: hami-dra-driver-service-account - volumes: - - hostPath: - path: /var/lib/kubelet/plugins_registry - type: "" - name: plugins-registry - - hostPath: - path: /var/lib/kubelet/plugins - type: "" - name: plugins - - hostPath: - path: /var/run/cdi - type: "" - name: cdi - - hostPath: - path: {{if .Values.drivers.nvidia.containerDriver }}/run/nvidia{{ else }}/{{ end }} - type: DirectoryOrCreate - name: driver-root-parent - - hostPath: - path: {{if .Values.drivers.nvidia.containerDriver }}/run/nvidia/driver{{ else }}/{{ end }} - type: DirectoryOrCreate - name: driver-root - - hostPath: - path: /dev - type: "" - name: host-dev - - hostPath: - path: /usr/local/vgpu - type: DirectoryOrCreate - name: host-vgpu - - hostPath: - path: /tmp - type: "" - name: host-tmp -{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/rbac.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/rbac.yaml deleted file mode 100644 index 435c37cc..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/rbac.yaml +++ /dev/null @@ -1,110 +0,0 @@ -{{- if .Values.drivers.nvidia.enabled }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: hami-dra-driver-service-account - namespace: {{ .Release.Namespace }} ---- -apiVersion: v1 -items: -- apiVersion: rbac.authorization.k8s.io/v1 - kind: Role - metadata: - name: hami-dra-driver-role - namespace: {{ .Release.Namespace }} - rules: - - apiGroups: - - apps - resources: - - daemonsets - - deployments - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -kind: List -metadata: - resourceVersion: "" ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: hami-dra-driver-role-binding - namespace: {{ .Release.Namespace }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: hami-dra-driver-role -subjects: -- kind: ServiceAccount - name: hami-dra-driver-service-account - namespace: {{ .Release.Namespace }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: hami-dra-driver-clusterrole -rules: -- apiGroups: - - resource.k8s.io - resources: - - resourceclaims - verbs: - - get - - list - - watch -- apiGroups: - - resource.k8s.io - resources: - - resourceclaimtemplates - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - resource.k8s.io - resources: - - resourceslices - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - resource.k8s.io - resources: - - resourceclaims/status - verbs: - - update -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - watch - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: hami-dra-driver-clusterrole-binding-hami-dra-driver -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: hami-dra-driver-clusterrole -subjects: -- kind: ServiceAccount - name: hami-dra-driver-service-account - namespace: {{ .Release.Namespace }} -{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrole.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrole.yaml deleted file mode 100644 index 45c24863..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrole.yaml +++ /dev/null @@ -1,11 +0,0 @@ -{{- if .Values.monitor.enabled }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "hami.dra.monitor.fullname" . }} -rules: -- apiGroups: ["resource.k8s.io"] - resources: ["resourceslices", "resourceclaims"] - verbs: ["get", "list", "watch"] -{{- end }} - diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrolebinding.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrolebinding.yaml deleted file mode 100644 index 766d3098..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrolebinding.yaml +++ /dev/null @@ -1,15 +0,0 @@ -{{- if .Values.monitor.enabled }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "hami.dra.monitor.fullname" . }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "hami.dra.monitor.fullname" . }} -subjects: - - kind: ServiceAccount - name: {{ include "hami.dra.monitor.fullname" . }} - namespace: {{ .Release.Namespace }} -{{- end }} - diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-deployment.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-deployment.yaml deleted file mode 100644 index b544b376..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-deployment.yaml +++ /dev/null @@ -1,62 +0,0 @@ -{{- if .Values.monitor.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "hami.dra.monitor.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: - app.kubernetes.io/name: {{ include "hami.dra.monitor.fullname" . }} - {{- include "hami-dra-monitor.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.monitor.replicas | default 1 }} - selector: - matchLabels: - {{- include "hami-dra-monitor.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - {{- include "hami-dra-monitor.selectorLabels" . | nindent 8 }} - spec: - {{- include "hami.dra.monitor.imagePullSecrets" . | nindent 6 }} - serviceAccountName: {{ include "hami.dra.monitor.fullname" . }} - containers: - - name: monitor - image: {{ include "hami.dra.monitor.image" . }} - imagePullPolicy: {{ .Values.monitor.image.pullPolicy | default "IfNotPresent" }} - command: - - /bin/monitor - - --v={{ .Values.monitor.logLevel | default 2 }} - - --metrics-bind-address={{ .Values.monitor.metricsBindAddress | default ":8080" }} - - --health-probe-bind-address={{ .Values.monitor.healthProbeBindAddress | default ":8000" }} - - --kube-api-qps={{ .Values.monitor.kubeAPIQPS | default 40.0 }} - - --kube-api-burst={{ .Values.monitor.kubeAPIBurst | default 60 }} - - --collect-interval={{ .Values.monitor.collectInterval | default "30s" }} - ports: - - containerPort: 8080 - name: metrics - protocol: TCP - - containerPort: 8000 - name: health - protocol: TCP - {{- if .Values.monitor.resources }} - resources: - {{- toYaml .Values.monitor.resources | nindent 12 }} - {{- end }} - livenessProbe: - httpGet: - path: /healthz - port: 8000 - initialDelaySeconds: 10 - periodSeconds: 10 - timeoutSeconds: 5 - failureThreshold: 3 - readinessProbe: - httpGet: - path: /readyz - port: 8000 - initialDelaySeconds: 5 - periodSeconds: 5 - timeoutSeconds: 3 - failureThreshold: 3 -{{- end }} - diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-service.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-service.yaml deleted file mode 100644 index d740cc38..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-service.yaml +++ /dev/null @@ -1,26 +0,0 @@ -{{- if .Values.monitor.enabled }} -apiVersion: v1 -kind: Service -metadata: - name: {{ include "hami.dra.monitor.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: - {{- include "hami-dra-monitor.selectorLabels" . | nindent 4 }} -spec: - type: {{ .Values.monitor.service.type | default "ClusterIP" }} - selector: - {{- include "hami-dra-monitor.selectorLabels" . | nindent 4 }} - ports: - - port: 8080 - targetPort: 8080 - name: metrics - protocol: TCP - {{- if and (eq (.Values.monitor.service.type | default "ClusterIP") "NodePort") .Values.monitor.service.nodePort.metrics }} - nodePort: {{ .Values.monitor.service.nodePort.metrics }} - {{- end }} - - port: 8000 - targetPort: 8000 - name: health - protocol: TCP -{{- end }} - diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-serviceaccount.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-serviceaccount.yaml deleted file mode 100644 index c61ad454..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-serviceaccount.yaml +++ /dev/null @@ -1,8 +0,0 @@ -{{- if .Values.monitor.enabled }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "hami.dra.monitor.fullname" . }} - namespace: {{ .Release.Namespace }} -{{- end }} - diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-manager.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-manager.yaml deleted file mode 100644 index dd31c71d..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-manager.yaml +++ /dev/null @@ -1,29 +0,0 @@ -{{- if .Values.certs.certManager.enabled }} -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: {{ .Release.Name }}-dra-webhook-serving-cert - namespace: {{ .Release.Namespace }} - labels: - {{- include "hami-dra-webhook.labels" . | nindent 4 }} -spec: - dnsNames: - - {{ .Release.Name }}-dra-webhook.{{ .Release.Namespace }}.svc - - {{ .Release.Name }}-dra-webhook.{{ .Release.Namespace }}.svc.cluster.local - issuerRef: - kind: Issuer - name: {{ .Release.Name }}-dra-webhook-selfsigned-issuer - secretName: {{ .Release.Name }}-dra-webhook-tls - privateKey: - rotationPolicy: {{ .Values.certs.certManager.privateKeyRotationPolicy | default "Never" }} ---- -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - name: {{ .Release.Name }}-dra-webhook-selfsigned-issuer - namespace: {{ .Release.Namespace }} - labels: - app.kubernetes.io/component: hami-dra-webhook -spec: - selfSigned: {} -{{- end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-secret.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-secret.yaml deleted file mode 100644 index eadbb517..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-secret.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- if eq .Values.certs.mode "custom" }} -apiVersion: v1 -kind: Secret -metadata: - name: {{ include "hami-dra-webhook.tlsSecretName" . }} - namespace: {{ .Release.Namespace }} -type: kubernetes.io/tls -data: - tls.crt: | - {{- .Values.certs.custom.crt | b64enc | indent 4 }} - tls.key: | - {{- .Values.certs.custom.key | b64enc | indent 4 }} -{{- end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/device-config.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/device-config.yaml deleted file mode 100644 index 97283f3e..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/device-config.yaml +++ /dev/null @@ -1,394 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "hami.dra.webhook.fullname" . }}-device-config - namespace: {{ .Release.Namespace }} - labels: - app.kubernetes.io/component: hami-scheduler - {{- include "hami-dra-webhook.selectorLabels" . | nindent 4 }} -data: - device-config.yaml: |- - {{- if .Files.Glob "files/device-config.yaml" }} - {{- .Files.Get "files/device-config.yaml" | nindent 4}} - {{- else }} - nvidia: - resourceCountName: {{ .Values.resourceName }} - resourceMemoryName: {{ .Values.resourceMem }} - resourceMemoryPercentageName: {{ .Values.resourceMemPercentage }} - resourceCoreName: {{ .Values.resourceCores }} - resourcePriorityName: {{ .Values.resourcePriority }} - overwriteEnv: false - defaultMemory: 0 - defaultCores: 0 - defaultGPUNum: 1 - knownMigGeometries: - - models: [ "A30" ] - allowedGeometries: - - - - name: 1g.6gb - core: 25 - memory: 6144 - count: 4 - - - - name: 2g.12gb - core: 50 - memory: 12288 - count: 2 - - - - name: 4g.24gb - core: 100 - memory: 24576 - count: 1 - - models: [ "A100-SXM4-40GB", "A100-40GB-PCIe", "A100-PCIE-40GB"] - allowedGeometries: - - - - name: 1g.5gb - core: 14 - memory: 5120 - count: 7 - - - - name: 1g.5gb - core: 14 - memory: 5120 - count: 1 - - name: 2g.10gb - core: 28 - memory: 10240 - count: 3 - - - - name: 3g.20gb - core: 42 - memory: 20480 - count: 2 - - - - name: 7g.40gb - core: 100 - memory: 40960 - count: 1 - - models: [ "A100-SXM4-80GB", "A100-80GB-PCIe", "A100-PCIE-80GB"] - allowedGeometries: - - - - name: 1g.10gb - core: 14 - memory: 10240 - count: 7 - - - - name: 1g.10gb - core: 14 - memory: 10240 - count: 1 - - name: 2g.20gb - core: 28 - memory: 20480 - count: 3 - - - - name: 3g.40gb - core: 42 - memory: 40960 - count: 2 - - - - name: 7g.79gb - core: 100 - memory: 80896 - count: 1 - - models: [ "H100-PCIE-80GB", "H100-SXM5-80GB"] - allowedGeometries: - - - - name: 1g.10gb - core: 14 - memory: 10240 - count: 7 - - - - name: 1g.10gb - core: 14 - memory: 10240 - count: 1 - - name: 2g.20gb - core: 28 - memory: 20480 - count: 3 - - - - name: 3g.40gb - core: 42 - memory: 40960 - count: 2 - - - - name: 7g.80gb - core: 100 - memory: 81920 - count: 1 - - models: [ "H100-PCIE-94GB", "H100-SXM5-94GB"] - allowedGeometries: - - - - name: 1g.12gb - core: 14 - memory: 12288 - count: 7 - - - - name: 1g.12gb - core: 14 - memory: 12288 - count: 1 - - name: 2g.24gb - core: 28 - memory: 24576 - count: 3 - - - - name: 3g.47gb - core: 42 - memory: 48128 - count: 2 - - - - name: 7g.94gb - core: 100 - memory: 96256 - count: 1 - - models: [ "H20", "H100 on GH200"] - allowedGeometries: - - - - name: 1g.12gb - core: 14 - memory: 12288 - count: 7 - - - - name: 1g.12gb - core: 14 - memory: 12288 - count: 1 - - name: 2g.24gb - core: 28 - memory: 24576 - count: 3 - - - - name: 3g.48gb - core: 42 - memory: 49152 - count: 2 - - - - name: 7g.96gb - core: 100 - memory: 98304 - count: 1 - - models: [ "H200 NVL", "H200-SXM5"] - allowedGeometries: - - - - name: 1g.18gb - core: 14 - memory: 18432 - count: 7 - - - - name: 1g.18gb - core: 14 - memory: 18432 - count: 1 - - name: 2g.35gb - core: 28 - memory: 35840 - count: 3 - - - - name: 3g.71gb - core: 42 - memory: 72704 - count: 2 - - - - name: 7g.141gb - core: 100 - memory: 144384 - count: 1 - - models: [ "B200" ] - allowedGeometries: - - - - name: 1g.23gb - core: 14 - memory: 23552 - count: 7 - - - - name: 1g.23gb - core: 14 - memory: 23552 - count: 1 - - name: 2g.45gb - core: 28 - memory: 46080 - count: 3 - - - - name: 3g.90gb - core: 42 - memory: 92160 - count: 2 - - - - name: 7g.180gb - core: 100 - memory: 184320 - count: 1 - cambricon: - resourceCountName: {{ .Values.mluResourceName }} - resourceMemoryName: {{ .Values.mluResourceMem }} - resourceCoreName: {{ .Values.mluResourceCores }} - hygon: - resourceCountName: {{ .Values.dcuResourceName }} - resourceMemoryName: {{ .Values.dcuResourceMem }} - resourceCoreName: {{ .Values.dcuResourceCores }} - metax: - resourceCountName: "metax-tech.com/gpu" - resourceVCountName: {{ .Values.metaxResourceName }} - resourceVMemoryName: {{ .Values.metaxResourceMem }} - resourceVCoreName: {{ .Values.metaxResourceCore }} - sgpuTopologyAware: {{ .Values.metaxsGPUTopologyAware }} - enflame: - resourceNameGCU: "enflame.com/gcu" - resourceNameVGCU: {{ .Values.enflameResourceNameVGCU }} - resourceNameVGCUPercentage: {{ .Values.enflameResourceNameVGCUPercentage }} - mthreads: - resourceCountName: "mthreads.com/vgpu" - resourceMemoryName: "mthreads.com/sgpu-memory" - resourceCoreName: "mthreads.com/sgpu-core" - iluvatars: - - chipName: MR-V100 - commonWord: MR-V100 - resourceCountName: iluvatar.ai/MR-V100-vgpu - resourceMemoryName: iluvatar.ai/MR-V100.vMem - resourceCoreName: iluvatar.ai/MR-V100.vCore - - chipName: MR-V50 - commonWord: MR-V50 - resourceCountName: iluvatar.ai/MR-V50-vgpu - resourceMemoryName: iluvatar.ai/MR-V50.vMem - resourceCoreName: iluvatar.ai/MR-V50.vCore - - chipName: BI-V150 - commonWord: BI-V150 - resourceCountName: iluvatar.ai/BI-V150-vgpu - resourceMemoryName: iluvatar.ai/BI-V150.vMem - resourceCoreName: iluvatar.ai/BI-V150.vCore - - chipName: BI-V100 - commonWord: BI-V100 - resourceCountName: iluvatar.ai/BI-V100-vgpu - resourceMemoryName: iluvatar.ai/BI-V100.vMem - resourceCoreName: iluvatar.ai/BI-V100.vCore - kunlun: - resourceCountName: {{ .Values.kunlunResourceName }} - resourceVCountName: {{ .Values.kunlunResourceVCountName }} - resourceVMemoryName: {{ .Values.kunlunResourceVMemoryName }} - awsneuron: - resourceCountName: "aws.amazon.com/neuron" - resourceCoreName: "aws.amazon.com/neuroncore" - amd: - resourceCountName: "amd.com/gpu" - vnpus: - - chipName: 910A - commonWord: Ascend910A - resourceName: huawei.com/Ascend910A - resourceMemoryName: huawei.com/Ascend910A-memory - memoryAllocatable: 32768 - memoryCapacity: 32768 - aiCore: 30 - templates: - - name: vir02 - memory: 2184 - aiCore: 2 - - name: vir04 - memory: 4369 - aiCore: 4 - - name: vir08 - memory: 8738 - aiCore: 8 - - name: vir16 - memory: 17476 - aiCore: 16 - - chipName: 910B2 - commonWord: Ascend910B2 - resourceName: huawei.com/Ascend910B2 - resourceMemoryName: huawei.com/Ascend910B2-memory - memoryAllocatable: 65536 - memoryCapacity: 65536 - aiCore: 24 - aiCPU: 6 - templates: - - name: vir03_1c_8g - memory: 8192 - aiCore: 3 - aiCPU: 1 - - name: vir06_1c_16g - memory: 16384 - aiCore: 6 - aiCPU: 1 - - name: vir12_3c_32g - memory: 32768 - aiCore: 12 - aiCPU: 3 - - chipName: 910B3 - commonWord: Ascend910B3 - resourceName: huawei.com/Ascend910B3 - resourceMemoryName: huawei.com/Ascend910B3-memory - memoryAllocatable: 65536 - memoryCapacity: 65536 - aiCore: 20 - aiCPU: 7 - templates: - - name: vir05_1c_16g - memory: 16384 - aiCore: 5 - aiCPU: 1 - - name: vir10_3c_32g - memory: 32768 - aiCore: 10 - aiCPU: 3 - - chipName: 910B4-1 - commonWord: Ascend910B4-1 - resourceName: huawei.com/Ascend910B4-1 - resourceMemoryName: huawei.com/Ascend910B4-1-memory - memoryAllocatable: 65536 - memoryCapacity: 65536 - aiCore: 20 - aiCPU: 7 - templates: - # NOTE: Names of vnpu template for 910B4-1 are fixed by Ascend runtime and must not be changed. - # The memory is used for scheduling so the correct values must be set. - # Template vir05_1c_8g actually provides 16GB memory, - - name: vir05_1c_8g - memory: 16384 - aiCore: 5 - aiCPU: 1 - # Template vir10_3c_16g actually provides 32GB memory - - name: vir10_3c_16g - memory: 32768 - aiCore: 10 - aiCPU: 3 - - chipName: 910B4 - commonWord: Ascend910B4 - resourceName: huawei.com/Ascend910B4 - resourceMemoryName: huawei.com/Ascend910B4-memory - memoryAllocatable: 32768 - memoryCapacity: 32768 - aiCore: 20 - aiCPU: 7 - templates: - - name: vir05_1c_8g - memory: 8192 - aiCore: 5 - aiCPU: 1 - - name: vir10_3c_16g - memory: 16384 - aiCore: 10 - aiCPU: 3 - - chipName: 310P3 - commonWord: Ascend310P - resourceName: huawei.com/Ascend310P - resourceMemoryName: huawei.com/Ascend310P-memory - memoryAllocatable: 21527 - memoryCapacity: 24576 - aiCore: 8 - aiCPU: 7 - templates: - - name: vir01 - memory: 3072 - aiCore: 1 - aiCPU: 1 - - name: vir02 - memory: 6144 - aiCore: 2 - aiCPU: 2 - - name: vir04 - memory: 12288 - aiCore: 4 - aiCPU: 4 - {{ end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/mutatingwebhookconfiguration.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/mutatingwebhookconfiguration.yaml deleted file mode 100644 index c763b6aa..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/mutatingwebhookconfiguration.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{- if .Values.webhook.config.mutating.enabled }} -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - name: {{ .Release.Name }}-mutatingwebhookconfiguration - annotations: - cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ .Release.Name }}-dra-webhook-serving-cert -webhooks: - - name: mutate.hami.io - admissionReviewVersions: ["v1"] - clientConfig: - service: - name: {{ .Release.Name }}-dra-webhook - namespace: {{ .Release.Namespace }} - path: /mutate - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - CREATE - resources: - - pods - scope: '*' - sideEffects: None - timeoutSeconds: 10 -{{- end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/validatingwebhookconfiguration.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/validatingwebhookconfiguration.yaml deleted file mode 100644 index acdcbb96..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/validatingwebhookconfiguration.yaml +++ /dev/null @@ -1,29 +0,0 @@ -{{- if .Values.webhook.config.validating.enabled }} -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: {{ .Release.Name }}-validatingwebhookconfiguration - annotations: - cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ .Release.Name }}-dra-webhook-serving-cert -webhooks: - - name: validate.hami.io - admissionReviewVersions: ["v1"] - clientConfig: - service: - name: {{ .Release.Name }}-dra-webhook - namespace: {{ .Release.Namespace }} - path: /validate - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - DELETE - resources: - - pods - scope: '*' - sideEffects: None - timeoutSeconds: 10 - failurePolicy: Ignore -{{- end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrole.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrole.yaml deleted file mode 100644 index 28916a36..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrole.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "hami.dra.webhook.fullname" . }} -rules: -- apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list", "watch"] -- apiGroups: ["admissionregistration.k8s.io"] - resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"] - verbs: ["get", "list", "watch"] -- apiGroups: ["resource.k8s.io"] - resources: ["resourceclaims"] - verbs: ["*"] diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrolebinding.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrolebinding.yaml deleted file mode 100644 index daad52f2..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrolebinding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "hami.dra.webhook.fullname" . }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "hami.dra.webhook.fullname" . }} -subjects: - - kind: ServiceAccount - name: {{ include "hami.dra.webhook.fullname" . }} - namespace: {{ .Release.Namespace }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-deployment.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-deployment.yaml deleted file mode 100644 index 8e3cf90c..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-deployment.yaml +++ /dev/null @@ -1,59 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "hami.dra.webhook.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: - app.kubernetes.io/name: {{ include "hami.dra.webhook.fullname" . }} - {{- include "hami-dra-webhook.labels" . | nindent 4 }} -spec: - replicas: 1 - selector: - matchLabels: - {{- include "hami-dra-webhook.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - {{- include "hami-dra-webhook.selectorLabels" . | nindent 8 }} - spec: - {{- include "hami.dra.webhook.imagePullSecrets" . | nindent 6 }} - serviceAccountName: {{ include "hami.dra.webhook.fullname" . }} - containers: - - name: webhook - image: {{ include "hami.dra.webhook.image" . }} - imagePullPolicy: {{ .Values.webhook.image.pullPolicy | default "IfNotPresent" }} - command: - - /bin/webhook - - --v=5 - - --bind-address=0.0.0.0 - - --secure-port=8443 - - --cert-dir=/tls - - --tls-cert-file-name=tls.crt - - --tls-private-key-file-name=tls.key - - --metrics-bind-address=:8080 - - --health-probe-bind-address=:8000 - - --device-config-file=/device-config.yaml - ports: - - containerPort: 8443 - name: webhook - protocol: TCP - - containerPort: 8080 - name: metrics - protocol: TCP - - containerPort: 8000 - name: health - protocol: TCP - volumeMounts: - - name: device-config - mountPath: /device-config.yaml - subPath: device-config.yaml - - name: tls-config - mountPath: /tls - readOnly: true - volumes: - - name: device-config - configMap: - name: {{ include "hami.dra.webhook.fullname" . }}-device-config - - name: tls-config - secret: - secretName: {{ if .Values.certs.certManager.enabled }}{{ .Release.Name }}-dra-webhook-tls{{ else }}{{ .Release.Name }}-tls{{ end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-service.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-service.yaml deleted file mode 100644 index 290174ec..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ .Release.Name }}-dra-webhook - namespace: {{ .Release.Namespace }} - labels: - {{- include "hami-dra-webhook.selectorLabels" . | nindent 4 }} -spec: - type: "ClusterIP" - selector: - {{- include "hami-dra-webhook.selectorLabels" . | nindent 4 }} - ports: - - port: 443 - targetPort: 8443 - name: webhook diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-serviceaccount.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-serviceaccount.yaml deleted file mode 100644 index b0dc2fb2..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-serviceaccount.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "hami.dra.webhook.fullname" . }} - namespace: {{ .Release.Namespace }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/values.yaml b/packages/system/hami/charts/hami/charts/hami-dra/values.yaml deleted file mode 100644 index ebbbfed7..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/values.yaml +++ /dev/null @@ -1,166 +0,0 @@ -## @section Global configuration -## @param global.imageRegistry Global Docker image registry -## @param global.imagePullSecrets Global Docker image pull secrets -global: - ## @param global.imageRegistry Global Docker image registry - imageRegistry: "" - ## E.g. - ## imagePullSecrets: - ## - myRegistryKeySecretName - ## @param global.imagePullSecrets Global Docker image pull secrets - imagePullSecrets: [] - -#Nvidia GPU Parameters -resourceName: "nvidia.com/gpu" -resourceMem: "nvidia.com/gpumem" -resourceMemPercentage: "nvidia.com/gpumem-percentage" -resourceCores: "nvidia.com/gpucores" -resourcePriority: "nvidia.com/priority" - -#MLU Parameters -mluResourceName: "cambricon.com/vmlu" -mluResourceMem: "cambricon.com/mlu.smlu.vmemory" -mluResourceCores: "cambricon.com/mlu.smlu.vcore" - -#Hygon DCU Parameters -dcuResourceName: "hygon.com/dcunum" -dcuResourceMem: "hygon.com/dcumem" -dcuResourceCores: "hygon.com/dcucores" - -#Metax sGPU Parameters -metaxResourceName: "metax-tech.com/sgpu" -metaxResourceCore: "metax-tech.com/vcore" -metaxResourceMem: "metax-tech.com/vmemory" -metaxsGPUTopologyAware: "false" - -#Enflame VGCU Parameters -enflameResourceNameVGCU: "enflame.com/vgcu" -enflameResourceNameVGCUPercentage: "enflame.com/vgcu-percentage" - -#Kunlun XPU Parameters -kunlunResourceName: "kunlunxin.com/xpu" -kunlunResourceVCountName: "kunlunxin.com/vxpu" -kunlunResourceVMemoryName: "kunlunxin.com/vxpu-memory" - -# Webhook deployment configuration -webhook: - image: - registry: ghcr.io - repository: project-hami/hami-dra-webhook - tag: "v0.1.0" - ## Specify a imagePullPolicy - ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' - ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images - ## - pullPolicy: Always - ## Optionally specify an array of imagePullSecrets. - ## Secrets must be manually created in the namespace. - ## Example: - ## pullSecrets: - ## - myRegistryKeySecretName - ## - pullSecrets: [] - ## @param resources controllerManager resource requests and limits - resources: - # If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 128Mi - ## @param nodeSelector controllerManager node labels for pod assignment - args: - webhookName: "" # Default: {{ .Release.Name }}-hami-dra-webhook - secretName: "" # Default: {{ .Release.Name }}-hami-dra-webhook-tls - # Webhook configuration - config: - mutating: - enabled: true - validating: - enabled: true - -# Monitor deployment configuration -monitor: - enabled: true - replicas: 1 - image: - registry: ghcr.io - repository: project-hami/hami-dra-monitor - tag: "v0.1.0" - ## Specify a imagePullPolicy - ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' - ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images - ## - pullPolicy: Always - ## Optionally specify an array of imagePullSecrets. - ## Secrets must be manually created in the namespace. - ## Example: - ## pullSecrets: - ## - myRegistryKeySecretName - ## - pullSecrets: [] - ## @param resources monitor resource requests and limits - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 128Mi - ## Monitor configuration - logLevel: 2 - metricsBindAddress: ":8080" - healthProbeBindAddress: ":8000" - kubeAPIQPS: 40.0 - kubeAPIBurst: 60 - collectInterval: "30s" - ## Monitor service configuration - service: - ## Service type: ClusterIP, NodePort, or LoadBalancer - ## Defaults to ClusterIP - type: ClusterIP - ## NodePort configuration (only used when type is NodePort) - nodePort: - ## Metrics port (NodePort). If not specified, Kubernetes will assign a random port - metrics: "" - -# Certificate configuration -certs: - # Custom certificate (used when mode is "custom") - custom: - crt: "" - key: "" - # Cert-manager configuration (used when mode is "cert-manager") - certManager: - enabled: true - # Private key rotation policy: "Never" or "Always" - # In cert-manager >= v1.18.0, the default changed from "Never" to "Always" - # Setting to "Never" avoids frequent certificate rotation for webhooks - privateKeyRotationPolicy: "Never" - issuer: - clusterIssuerName: "" # Required when type is "clusterIssuer" - -# DRA Drivers configuration -drivers: - nvidia: - enabled: true - # If you are using gpu driver on host, you need to set this to false - containerDriver: true - image: - registry: ghcr.io - repository: projecthami/k8s-dra-driver - tag: "v0.0.1-dev" - ## Specify a imagePullPolicy - ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' - ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images - ## - pullPolicy: IfNotPresent - ## Optionally specify an array of imagePullSecrets. - ## Secrets must be manually created in the namespace. - ## Example: - ## pullSecrets: - ## - myRegistryKeySecretName - ## - pullSecrets: [] From 385ea7a17f8cfc4526c50db036c105c63a5817fc Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Thu, 23 Apr 2026 22:16:34 +0300 Subject: [PATCH 06/27] feat(platform): register HAMi as optional system package Add PackageSource for HAMi with dependency on gpu-operator. Include HAMi in the iaas bundle as an optional package, allowing host-level deployment alongside the existing kubernetes addon path. Signed-off-by: Arsolitt --- packages/core/platform/sources/hami.yaml | 24 +++++++++++++++++++ .../core/platform/templates/bundles/iaas.yaml | 1 + 2 files changed, 25 insertions(+) create mode 100644 packages/core/platform/sources/hami.yaml diff --git a/packages/core/platform/sources/hami.yaml b/packages/core/platform/sources/hami.yaml new file mode 100644 index 00000000..0184e405 --- /dev/null +++ b/packages/core/platform/sources/hami.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: cozystack.io/v1alpha1 +kind: PackageSource +metadata: + name: cozystack.hami +spec: + sourceRef: + kind: OCIRepository + name: cozystack-packages + namespace: cozy-system + path: / + variants: + - name: default + dependsOn: + - cozystack.gpu-operator + components: + - name: hami + path: system/hami + valuesFiles: + - values.yaml + install: + privileged: true + namespace: cozy-hami + releaseName: hami diff --git a/packages/core/platform/templates/bundles/iaas.yaml b/packages/core/platform/templates/bundles/iaas.yaml index ced0a322..66ec98b2 100644 --- a/packages/core/platform/templates/bundles/iaas.yaml +++ b/packages/core/platform/templates/bundles/iaas.yaml @@ -10,6 +10,7 @@ {{include "cozystack.platform.package.default" (list "cozystack.kubevirt-cdi" $) }} {{include "cozystack.platform.package.optional.default" (list "cozystack.vm-default-images" $) }} {{include "cozystack.platform.package.optional.default" (list "cozystack.gpu-operator" $) }} +{{include "cozystack.platform.package.optional.default" (list "cozystack.hami" $) }} {{include "cozystack.platform.package.default" (list "cozystack.kamaji" $) }} {{include "cozystack.platform.package.default" (list "cozystack.capi-operator" $) }} {{include "cozystack.platform.package.default" (list "cozystack.capi-provider-bootstrap-kubeadm" $) }} From 3b3540448975391ad7582db58d83ce5b18b15129 Mon Sep 17 00:00:00 2001 From: Denis Chernosov Date: Fri, 24 Apr 2026 11:32:20 +0400 Subject: [PATCH 07/27] fix(cozystack-engine): add managed Kubernetes without visible control-plane nodes support to lineage-controller-webhook (#2417) Signed-off-by: Denis Chernosov --- .../templates/daemonset.yaml | 2 + .../templates/deployment.yaml | 49 +++++++++++++++++++ .../templates/pdb.yaml | 11 +++++ .../lineage-controller-webhook/values.yaml | 5 ++ 4 files changed, 67 insertions(+) create mode 100644 packages/system/lineage-controller-webhook/templates/deployment.yaml create mode 100644 packages/system/lineage-controller-webhook/templates/pdb.yaml diff --git a/packages/system/lineage-controller-webhook/templates/daemonset.yaml b/packages/system/lineage-controller-webhook/templates/daemonset.yaml index 860aee6e..536ea24a 100644 --- a/packages/system/lineage-controller-webhook/templates/daemonset.yaml +++ b/packages/system/lineage-controller-webhook/templates/daemonset.yaml @@ -1,3 +1,4 @@ +{{- if not .Values.lineageControllerWebhook.deployment.enabled }} apiVersion: apps/v1 kind: DaemonSet metadata: @@ -51,3 +52,4 @@ spec: secret: secretName: lineage-controller-webhook-cert defaultMode: 0400 +{{- end }} \ No newline at end of file diff --git a/packages/system/lineage-controller-webhook/templates/deployment.yaml b/packages/system/lineage-controller-webhook/templates/deployment.yaml new file mode 100644 index 00000000..87717fd3 --- /dev/null +++ b/packages/system/lineage-controller-webhook/templates/deployment.yaml @@ -0,0 +1,49 @@ +{{- if .Values.lineageControllerWebhook.deployment.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: lineage-controller-webhook + labels: + app: lineage-controller-webhook +spec: + replicas: {{ .Values.lineageControllerWebhook.deployment.replicas }} + selector: + matchLabels: + app: lineage-controller-webhook + template: + metadata: + labels: + app: lineage-controller-webhook + spec: + serviceAccountName: lineage-controller-webhook + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app: lineage-controller-webhook + containers: + - name: lineage-controller-webhook + image: "{{ .Values.lineageControllerWebhook.image }}" + args: + {{- if .Values.lineageControllerWebhook.debug }} + - --zap-log-level=debug + {{- else }} + - --zap-log-level=info + {{- end }} + ports: + - name: webhook + containerPort: 9443 + volumeMounts: + - name: webhook-certs + mountPath: /tmp/k8s-webhook-server/serving-certs + readOnly: true + volumes: + - name: webhook-certs + secret: + secretName: lineage-controller-webhook-cert + defaultMode: 0400 +{{- end }} \ No newline at end of file diff --git a/packages/system/lineage-controller-webhook/templates/pdb.yaml b/packages/system/lineage-controller-webhook/templates/pdb.yaml new file mode 100644 index 00000000..b0a9d4e4 --- /dev/null +++ b/packages/system/lineage-controller-webhook/templates/pdb.yaml @@ -0,0 +1,11 @@ +{{- if .Values.lineageControllerWebhook.deployment.enabled }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: lineage-controller-webhook +spec: + minAvailable: {{ if gt (int .Values.lineageControllerWebhook.deployment.replicas) 1 }}1{{ else }}0{{ end }} + selector: + matchLabels: + app: lineage-controller-webhook +{{- end }} \ No newline at end of file diff --git a/packages/system/lineage-controller-webhook/values.yaml b/packages/system/lineage-controller-webhook/values.yaml index c5671548..2b84f4e1 100644 --- a/packages/system/lineage-controller-webhook/values.yaml +++ b/packages/system/lineage-controller-webhook/values.yaml @@ -2,9 +2,14 @@ lineageControllerWebhook: image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.3.0@sha256:e8984709686a5eaf19b89da378d7b8c688ea5607e0783a88d9c9e4ccfca96fb0 debug: false localK8sAPIEndpoint: + # incompatable with Deployment mode enabled: true # nodeSelector for the DaemonSet # Talos uses empty value: "node-role.kubernetes.io/control-plane": "" # Generic k8s (k3s, kubeadm) uses: "node-role.kubernetes.io/control-plane": "true" nodeSelector: node-role.kubernetes.io/control-plane: "" + # if enabled, replace DaemonSet by Deployment + deployment: + enabled: false + replicas: 1 From ab7deb2b055c2a677c4c84c1bb22271306ebf372 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Fri, 24 Apr 2026 13:03:49 +0300 Subject: [PATCH 08/27] chore(kubernetes): regenerate after HAMi addon integration Run make generate to update generated types, schema, README, and CRD definitions for the new HAMi addon. Signed-off-by: Arsolitt --- api/apps/v1alpha1/kubernetes/types.go | 12 ++++++++++++ packages/apps/kubernetes/README.md | 3 +++ packages/apps/kubernetes/values.schema.json | 4 ++-- .../system/kubernetes-rd/cozyrds/kubernetes.yaml | 4 ++-- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/api/apps/v1alpha1/kubernetes/types.go b/api/apps/v1alpha1/kubernetes/types.go index 774fcdc5..39a05567 100644 --- a/api/apps/v1alpha1/kubernetes/types.go +++ b/api/apps/v1alpha1/kubernetes/types.go @@ -66,6 +66,9 @@ type Addons struct { // NVIDIA GPU Operator. // +kubebuilder:default:={} GpuOperator GPUOperatorAddon `json:"gpuOperator"` + // HAMi GPU virtualization middleware. + // +kubebuilder:default:={} + Hami HAMiAddon `json:"hami"` // Ingress-NGINX controller. // +kubebuilder:default:={} IngressNginx IngressNginxAddon `json:"ingressNginx"` @@ -157,6 +160,15 @@ type GatewayAPIAddon struct { Enabled bool `json:"enabled"` } +type HAMiAddon struct { + // Enable HAMi (requires GPU Operator). + // +kubebuilder:default:=false + Enabled bool `json:"enabled"` + // Custom Helm values overrides. + // +kubebuilder:default:={} + ValuesOverride k8sRuntime.RawExtension `json:"valuesOverride"` +} + type IngressNginxAddon struct { // Enable the controller (requires nodes labeled `ingress-nginx`). // +kubebuilder:default:=false diff --git a/packages/apps/kubernetes/README.md b/packages/apps/kubernetes/README.md index d62d3a39..d705af64 100644 --- a/packages/apps/kubernetes/README.md +++ b/packages/apps/kubernetes/README.md @@ -128,6 +128,9 @@ See the reference for components utilized in this service: | `addons.gpuOperator` | NVIDIA GPU Operator. | `object` | `{}` | | `addons.gpuOperator.enabled` | Enable GPU Operator. | `bool` | `false` | | `addons.gpuOperator.valuesOverride` | Custom Helm values overrides. | `object` | `{}` | +| `addons.hami` | HAMi GPU virtualization middleware. | `object` | `{}` | +| `addons.hami.enabled` | Enable HAMi (requires GPU Operator). | `bool` | `false` | +| `addons.hami.valuesOverride` | Custom Helm values overrides. | `object` | `{}` | | `addons.fluxcd` | FluxCD GitOps operator. | `object` | `{}` | | `addons.fluxcd.enabled` | Enable FluxCD. | `bool` | `false` | | `addons.fluxcd.valuesOverride` | Custom Helm values overrides. | `object` | `{}` | diff --git a/packages/apps/kubernetes/values.schema.json b/packages/apps/kubernetes/values.schema.json index 3d7a17dd..1e457f1f 100644 --- a/packages/apps/kubernetes/values.schema.json +++ b/packages/apps/kubernetes/values.schema.json @@ -270,7 +270,7 @@ } }, "hami": { - "description": "HAMi GPU virtualization middleware. Enables fractional GPU sharing (memory and compute isolation) for workloads. Requires GPU Operator. Note: workload containers must use glibc < 2.34 (not musl/Alpine) for full compute isolation.", + "description": "HAMi GPU virtualization middleware.", "type": "object", "default": {}, "required": [ @@ -279,7 +279,7 @@ ], "properties": { "enabled": { - "description": "Enable HAMi.", + "description": "Enable HAMi (requires GPU Operator).", "type": "boolean", "default": false }, diff --git a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml index 5e9e8f94..cdf900e1 100644 --- a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml +++ b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml @@ -8,7 +8,7 @@ spec: singular: kubernetes plural: kuberneteses openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"ephemeralStorage":"20Gi","gpus":[],"instanceType":"u1.medium","maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"]}},"additionalProperties":{"type":"object","required":["ephemeralStorage","instanceType","maxReplicas","minReplicas","resources"],"properties":{"ephemeralStorage":{"description":"Ephemeral storage size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string"}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium"},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `.` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","ingressNginx","monitoringAgents","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"large","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}}}} + {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"ephemeralStorage":"20Gi","gpus":[],"instanceType":"u1.medium","maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"]}},"additionalProperties":{"type":"object","required":["ephemeralStorage","instanceType","maxReplicas","minReplicas","resources"],"properties":{"ephemeralStorage":{"description":"Ephemeral storage size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string"}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium"},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `.` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","hami","ingressNginx","monitoringAgents","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"hami":{"description":"HAMi GPU virtualization middleware.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable HAMi (requires GPU Operator).","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"large","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}}}} release: prefix: kubernetes- labels: @@ -26,7 +26,7 @@ spec: tags: - compute icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODFfMjg0NSkiLz4KPHBhdGggZD0iTTcxLjk5NjggMTlDNzAuMzAzOSAxOS4wMDAyIDY4LjkzMTIgMjAuNTMyMiA2OC45MzE0IDIyLjQyMjFDNjguOTMxNCAyMi40NTExIDY4LjkzNzMgMjIuNDc4OCA2OC45Mzc5IDIyLjUwNzZDNjguOTM1NCAyMi43NjQ0IDY4LjkyMzEgMjMuMDczNyA2OC45MzE0IDIzLjI5NzNDNjguOTcxNyAyNC4zODczIDY5LjIwODIgMjUuMjIxNiA2OS4zNTA2IDI2LjIyNThDNjkuNjA4NCAyOC4zNzUyIDY5LjgyNDUgMzAuMTU2OSA2OS42OTEyIDMxLjgxM0M2OS41NjE1IDMyLjQzNzUgNjkuMTAzNyAzMy4wMDg2IDY4LjY5NTYgMzMuNDA1Nkw2OC42MjM1IDM0LjcwODZDNjYuNzgzOSAzNC44NjE3IDY0LjkzMTkgMzUuMTQyMSA2My4wODIxIDM1LjU2NDFDNTUuMTIyNiAzNy4zNzk4IDQ4LjI2OTUgNDEuNDk5MSA0My4wNTIgNDcuMDYwOUM0Mi43MTM0IDQ2LjgyODggNDIuMTIxMSA0Ni40MDE5IDQxLjk0NSA0Ni4yNzEyQzQxLjM5NzcgNDYuMzQ1NCA0MC44NDQ1IDQ2LjUxNTEgNDAuMTI0MSA0Ni4wOTM1QzM4Ljc1MjIgNDUuMTY1NyAzNy41MDI4IDQzLjg4NTEgMzUuOTkxIDQyLjM0MjRDMzUuMjk4MiA0MS42MDQ0IDM0Ljc5NjYgNDAuOTAxOCAzMy45NzM1IDQwLjE5MDRDMzMuNzg2NiA0MC4wMjg5IDMzLjUwMTQgMzkuODEwNCAzMy4yOTIzIDM5LjY0NDJDMzIuNjQ4OSAzOS4xMjg4IDMxLjg5IDM4Ljg2IDMxLjE1NyAzOC44MzQ4QzMwLjIxNDcgMzguODAyNCAyOS4zMDc1IDM5LjE3MjUgMjguNzEzOCAzOS45MjA2QzI3LjY1ODQgNDEuMjUwNiAyNy45OTYzIDQzLjI4MzMgMjkuNDY3MSA0NC40NjE0QzI5LjQ4MiA0NC40NzM0IDI5LjQ5NzkgNDQuNDgyNyAyOS41MTI5IDQ0LjQ5NDNDMjkuNzE1IDQ0LjY1ODkgMjkuOTYyNSA0NC44Njk4IDMwLjE0ODMgNDUuMDA3NkMzMS4wMjE3IDQ1LjY1NTUgMzEuODE5NSA0NS45ODcyIDMyLjY4OTcgNDYuNTAxNUMzNC41MjMxIDQ3LjYzOTEgMzYuMDQzIDQ4LjU4MjMgMzcuMjQ4NiA0OS43MTk2QzM3LjcxOTQgNTAuMjIzNyAzNy44MDE2IDUxLjExMjIgMzcuODY0MyA1MS40OTY0TDM4Ljg0NjggNTIuMzc4MkMzMy41ODcyIDYwLjMzMDggMzEuMTUzIDcwLjE1MzkgMzIuNTkxNSA4MC4xNjI3TDMxLjMwNzcgODAuNTM3OEMzMC45NjkzIDgwLjk3NjggMzAuNDkxMiA4MS42Njc2IDI5Ljk5MTEgODEuODczOEMyOC40MTM4IDgyLjM3MjkgMjYuNjM4NyA4Mi41NTYyIDI0LjQ5NTYgODIuNzgxOUMyMy40ODk0IDgyLjg2NiAyMi42MjEzIDgyLjgxNTggMjEuNTU0NiA4My4wMTg4QzIxLjMxOTggODMuMDYzNSAyMC45OTI3IDgzLjE0OTEgMjAuNzM1OCA4My4yMDk3QzIwLjcyNjkgODMuMjExNiAyMC43MTg2IDgzLjIxNDIgMjAuNzA5NiA4My4yMTYyQzIwLjY5NTYgODMuMjE5NSAyMC42NzcyIDgzLjIyNjMgMjAuNjYzOCA4My4yMjk0QzE4Ljg1NyA4My42NjggMTcuNjk2MyA4NS4zMzY1IDE4LjA2OTkgODYuOTgwNUMxOC40NDM3IDg4LjYyNDggMjAuMjA4NiA4OS42MjQ4IDIyLjAyNjIgODkuMjMxMkMyMi4wMzkzIDg5LjIyODIgMjIuMDU4NCA4OS4yMjc3IDIyLjA3MiA4OS4yMjQ2QzIyLjA5MjYgODkuMjE5OSAyMi4xMTA2IDg5LjIwOTkgMjIuMTMxIDg5LjIwNDlDMjIuMzg0NCA4OS4xNDkgMjIuNzAxOSA4OS4wODY4IDIyLjkyMzYgODkuMDI3MkMyMy45NzIzIDg4Ljc0NTEgMjQuNzMxOCA4OC4zMzA2IDI1LjY3NDYgODcuOTY3N0MyNy43MDI5IDg3LjIzNjggMjkuMzgyOCA4Ni42MjYyIDMxLjAxOTUgODYuMzg4M0MzMS43MDMgODYuMzM0NSAzMi40MjMyIDg2LjgxMiAzMi43ODE0IDg3LjAxMzRMMzQuMTE3NyA4Ni43ODMxQzM3LjE5MjYgOTYuMzYxMyA0My42MzY2IDEwNC4xMDMgNTEuNzk2MyAxMDguOTYxTDUxLjIzOTYgMTEwLjMwM0M1MS40NDAzIDExMC44MjQgNTEuNjYxNiAxMTEuNTMgNTEuNTEyMSAxMTIuMDQ1QzUwLjkxNzEgMTEzLjU5NSA0OS44OTggMTE1LjIzMSA0OC43Mzc0IDExNy4wNTVDNDguMTc1NSAxMTcuODk4IDQ3LjYwMDQgMTE4LjU1MiA0Ny4wOTM0IDExOS41MTZDNDYuOTcyIDExOS43NDcgNDYuODE3NSAxMjAuMTAyIDQ2LjcwMDQgMTIwLjM0NkM0NS45MTI1IDEyMi4wMzkgNDYuNDkwNCAxMjMuOTkgNDguMDAzOCAxMjQuNzIyQzQ5LjUyNjggMTI1LjQ1OSA1MS40MTcxIDEyNC42ODIgNTIuMjM1MiAxMjIuOTg1QzUyLjIzNjQgMTIyLjk4MiA1Mi4yNDA2IDEyMi45OCA1Mi4yNDE3IDEyMi45NzhDNTIuMjQyNiAxMjIuOTc2IDUyLjI0MDkgMTIyLjk3MyA1Mi4yNDE3IDEyMi45NzFDNTIuMzU4MiAxMjIuNzMxIDUyLjUyMzMgMTIyLjQxNSA1Mi42MjE2IDEyMi4xODhDNTMuMDU2IDEyMS4xODkgNTMuMjAwNSAxMjAuMzMyIDUzLjUwNTkgMTE5LjM2NUM1NC4zMTcgMTE3LjMxOCA1NC43NjI2IDExNS4xNyA1NS44NzkxIDExMy44MzJDNTYuMTg0OSAxMTMuNDY2IDU2LjY4MzMgMTEzLjMyNSA1Ny4yMDAxIDExMy4xODZMNTcuODk0NCAxMTEuOTIyQzY1LjAwOCAxMTQuNjY1IDcyLjk3MDUgMTE1LjQwMiA4MC45MjQ1IDExMy41ODdDODIuNzM5MSAxMTMuMTczIDg0LjQ5MDggMTEyLjYzNyA4Ni4xODQzIDExMS45OTRDODYuMzc5NCAxMTIuMzQyIDg2Ljc0MiAxMTMuMDExIDg2LjgzOTMgMTEzLjE3OUM4Ny4zNjQ0IDExMy4zNTEgODcuOTM3NyAxMTMuNDM5IDg4LjQwNDcgMTE0LjEzM0M4OS4yNDAxIDExNS41NjcgODkuODExNCAxMTcuMjYzIDkwLjUwNzMgMTE5LjMxMkM5MC44MTI4IDEyMC4yNzkgOTAuOTYzOCAxMjEuMTM2IDkxLjM5ODEgMTIyLjEzNkM5MS40OTcxIDEyMi4zNjMgOTEuNjYxNCAxMjIuNjg0IDkxLjc3OCAxMjIuOTI1QzkyLjU5NDQgMTI0LjYyOCA5NC40OTA3IDEyNS40MDcgOTYuMDE1OSAxMjQuNjY5Qzk3LjUyOTIgMTIzLjkzNyA5OC4xMDc3IDEyMS45ODYgOTcuMzE5NCAxMjAuMjkzQzk3LjIwMjMgMTIwLjA0OSA5Ny4wNDEyIDExOS42OTUgOTYuOTE5OCAxMTkuNDY0Qzk2LjQxMjcgMTE4LjQ5OSA5NS44Mzc3IDExNy44NTIgOTUuMjc1OCAxMTcuMDA5Qzk0LjExNTIgMTE1LjE4NSA5My4xNTI2IDExMy42NyA5Mi41NTc1IDExMi4xMkM5Mi4zMDg3IDExMS4zMiA5Mi41OTk1IDExMC44MjMgOTIuNzkzMyAxMTAuMzAzQzkyLjY3NzIgMTEwLjE3IDkyLjQyODggMTA5LjQxNCA5Mi4yODI0IDEwOS4wNTlDMTAwLjc2MiAxMDQuMDI5IDEwNy4wMTcgOTUuOTk4NSAxMDkuOTU1IDg2LjcyMzlDMTEwLjM1MSA4Ni43ODY1IDExMS4wNDEgODYuOTA5MSAxMTEuMjY1IDg2Ljk1NDJDMTExLjcyNiA4Ni42NDg3IDExMi4xNDkgODYuMjUwMSAxMTIuOTgxIDg2LjMxNTlDMTE0LjYxNyA4Ni41NTM3IDExNi4yOTcgODcuMTY0NSAxMTguMzI2IDg3Ljg5NTNDMTE5LjI2OCA4OC4yNTgxIDEyMC4wMjggODguNjc5MyAxMjEuMDc3IDg4Ljk2MTRDMTIxLjI5OCA4OS4wMjEgMTIxLjYxNiA4OS4wNzY2IDEyMS44NjkgODkuMTMyNUMxMjEuODg5IDg5LjEzNzUgMTIxLjkwOCA4OS4xNDc1IDEyMS45MjggODkuMTUyMkMxMjEuOTQyIDg5LjE1NTMgMTIxLjk2MSA4OS4xNTU4IDEyMS45NzQgODkuMTU4OEMxMjMuNzkyIDg5LjU1MiAxMjUuNTU3IDg4LjU1MjYgMTI1LjkzIDg2LjkwODFDMTI2LjMwMyA4NS4yNjQxIDEyNS4xNDMgODMuNTk1MiAxMjMuMzM2IDgzLjE1N0MxMjMuMDc0IDgzLjA5NyAxMjIuNzAxIDgyLjk5NSAxMjIuNDQ2IDgyLjk0NjVDMTIxLjM3OSA4Mi43NDM1IDEyMC41MTEgODIuNzkzNSAxMTkuNTA1IDgyLjcwOTVDMTE3LjM2MSA4Mi40ODM5IDExNS41ODYgODIuMzAwNCAxMTQuMDA5IDgxLjgwMTRDMTEzLjM2NiA4MS41NTA3IDExMi45MDggODAuNzgxOSAxMTIuNjg2IDgwLjQ2NTVMMTExLjQ0OCA4MC4xMDM1QzExMi4wOSA3NS40MzggMTExLjkxNyA3MC41ODI1IDExMC44MDYgNjUuNzI0M0MxMDkuNjg1IDYwLjgyMDggMTA3LjcwNCA1Ni4zMzYxIDEwNS4wNjIgNTIuMzg0OEMxMDUuMzc5IDUyLjA5NDggMTA1Ljk3OSA1MS41NjEyIDEwNi4xNDkgNTEuNDA0M0MxMDYuMTk5IDUwLjg1MTcgMTA2LjE1NiA1MC4yNzIyIDEwNi43MjUgNDkuNjYwM0MxMDcuOTMxIDQ4LjUyMyAxMDkuNDUxIDQ3LjU3OTkgMTExLjI4NCA0Ni40NDIzQzExMi4xNTQgNDUuOTI3OSAxMTIuOTU5IDQ1LjU5NjQgMTEzLjgzMiA0NC45NDg0QzExNC4wMyA0NC44MDE5IDExNC4yOTkgNDQuNTY5OSAxMTQuNTA3IDQ0LjQwMjJDMTE1Ljk3NyA0My4yMjM3IDExNi4zMTYgNDEuMTkxMSAxMTUuMjYgMzkuODYxNEMxMTQuMjA0IDM4LjUzMTcgMTEyLjE1OSAzOC40MDY1IDExMC42ODggMzkuNTg1QzExMC40NzkgMzkuNzUxNiAxMTAuMTk1IDM5Ljk2ODggMTEwLjAwNyA0MC4xMzEyQzEwOS4xODQgNDAuODQyNiAxMDguNjc2IDQxLjU0NTIgMTA3Ljk4MyA0Mi4yODMyQzEwNi40NzEgNDMuODI1OSAxMDUuMjIyIDQ1LjExMyAxMDMuODUgNDYuMDQwOUMxMDMuMjU1IDQ2LjM4ODUgMTAyLjM4NSA0Ni4yNjgyIDEwMS45OSA0Ni4yNDQ5TDEwMC44MjQgNDcuMDgwNkM5NC4xNzUzIDQwLjA3NjMgODUuMTIzNSAzNS41OTgyIDc1LjM3NjYgMzQuNzI4M0M3NS4zNDk0IDM0LjMxNzkgNzUuMzEzNyAzMy41NzYxIDc1LjMwNDYgMzMuMzUyOUM3NC45MDU2IDMyLjk2OTMgNzQuNDIzNSAzMi42NDE4IDc0LjMwMjQgMzEuODEzQzc0LjE2OTEgMzAuMTU2OSA3NC4zOTE3IDI4LjM3NTIgNzQuNjQ5NiAyNi4yMjU4Qzc0Ljc5MTkgMjUuMjIxNiA3NS4wMjg0IDI0LjM4NzMgNzUuMDY4OCAyMy4yOTczQzc1LjA3OCAyMy4wNDk1IDc1LjA2MzIgMjIuNjkgNzUuMDYyMiAyMi40MjIxQzc1LjA2MiAyMC41MzIyIDczLjY4OTggMTguOTk5OCA3MS45OTY4IDE5Wk02OC4xNTg1IDQyLjg4ODZMNjcuMjQ4IDU5LjA0NDdMNjcuMTgyNSA1OS4wNzc2QzY3LjEyMTQgNjAuNTIyOSA2NS45Mzc1IDYxLjY3NyA2NC40ODM5IDYxLjY3N0M2My44ODg0IDYxLjY3NyA2My4zMzg4IDYxLjQ4NDkgNjIuODkyMiA2MS4xNTcxTDYyLjg2NiA2MS4xNzAzTDQ5LjY4MDcgNTEuNzc5NEM1My43MzMxIDQ3Ljc3NTkgNTguOTE2NCA0NC44MTcyIDY0Ljg5IDQzLjQ1NDZDNjUuOTgxMiA0My4yMDU2IDY3LjA3MTkgNDMuMDIwOSA2OC4xNTg1IDQyLjg4ODZaTTc1Ljg0MTcgNDIuODg4NkM4Mi44MTU5IDQzLjc1MDQgODkuMjY1NyA0Ni45MjMyIDk0LjIwODEgNTEuNzg2TDgxLjEwOCA2MS4xMTc2TDgxLjA2MjEgNjEuMDk3OUM3OS44OTk0IDYxLjk1MTIgNzguMjYxMSA2MS43Mzk0IDc3LjM1NDggNjAuNTk3OEM3Ni45ODM1IDYwLjEzMDEgNzYuNzg4NyA1OS41ODAxIDc2Ljc2NTMgNTkuMDI0OUw3Ni43NTIyIDU5LjAxODRMNzUuODQxNyA0Mi44ODg2Wk00NC44OTkxIDU3LjgxNEw1Ni45MzgyIDY4LjYzM0w1Ni45MjUxIDY4LjY5ODhDNTguMDExNyA2OS42NDc5IDU4LjE3MiA3MS4yOTQ5IDU3LjI2NTcgNzIuNDM2OEM1Ni44OTQ0IDcyLjkwNDUgNTYuMzk3NSA3My4yMTgyIDU1Ljg2MzkgNzMuMzY0N0w1NS44NTA4IDczLjQxNzNMNDAuNDE4OCA3Ny44OTIzQzM5LjYzMzQgNzAuNjc2NSA0MS4zMjYxIDYzLjY2MjEgNDQuODk5MSA1Ny44MTRaTTk5LjAwOTQgNTcuODIwNkMxMDAuNzk4IDYwLjczMzYgMTAyLjE1MyA2My45ODcxIDEwMi45NTkgNjcuNTE0M0MxMDMuNzU2IDcwLjk5OTEgMTAzLjk1NiA3NC40Nzc4IDEwMy42MjcgNzcuODM5N0w4OC4xMTY2IDczLjM1MTVMODguMTAzNSA3My4yODU3Qzg2LjcxNDUgNzIuOTA0MyA4NS44NjA5IDcxLjQ4NDggODYuMTg0MyA3MC4wNjExQzg2LjMxNjggNjkuNDc3OCA4Ni42MjQ5IDY4Ljk4NDQgODcuMDQyMyA2OC42MTk4TDg3LjAzNTggNjguNTg2OUw5OS4wMDk0IDU3LjgyMDZaTTY5LjUyNzQgNjkuNDY4OEg3NC40NTk2TDc3LjUyNTEgNzMuMzE4Nkw3Ni40MjQ3IDc4LjEyMjZMNzEuOTk2OCA4MC4yNjE0TDY3LjU1NTggNzguMTE2MUw2Ni40NTU0IDczLjMxMkw2OS41Mjc0IDY5LjQ2ODhaTTg1LjMzOTMgODIuNjQzN0M4NS41NDg5IDgyLjYzMzEgODUuNzU3NiA4Mi42NTIgODUuOTYxNiA4Mi42ODk4TDg1Ljk4NzggODIuNjU2OUwxMDEuOTUgODUuMzY4MkM5OS42MTQyIDkxLjk2MjQgOTUuMTQ0IDk3LjY3NSA4OS4xNzExIDEwMS40OThMODIuOTc0NyA4Ni40NjA2TDgyLjk5NDQgODYuNDM0M0M4Mi40MjUyIDg1LjEwNTUgODIuOTk0OCA4My41NDcyIDg0LjMwNDQgODIuOTEzNUM4NC42Mzk3IDgyLjc1MTMgODQuOTkgODIuNjYxNCA4NS4zMzkzIDgyLjY0MzdaTTU4LjUyOTggODIuNzA5NUM1OS43NDggODIuNzI2NyA2MC44NDA2IDgzLjU3NjEgNjEuMTIzNyA4NC44MjJDNjEuMjU2MiA4NS40MDUyIDYxLjE5MTcgODUuOTgzMSA2MC45NzMgODYuNDkzNUw2MS4wMTg5IDg2LjU1MjdMNTQuODg4IDEwMS40MzlDNDkuMTU1OSA5Ny43NDMyIDQ0LjU5MDQgOTIuMjA5OSA0Mi4xNDgxIDg1LjQyMDhMNTcuOTczMSA4Mi43MjI3TDU3Ljk5OTMgODIuNzU1NkM1OC4xNzYzIDgyLjcyMjkgNTguMzU1OCA4Mi43MDcxIDU4LjUyOTggODIuNzA5NVpNNzEuODk4NiA4OS4yMzEyQzcyLjMyMjkgODkuMjE1NSA3Mi43NTM0IDg5LjMwMyA3My4xNjI3IDg5LjUwMUM3My42OTkyIDg5Ljc2MDYgNzQuMTEzNiA5MC4xNjkyIDc0LjM3NDUgOTAuNjU5Mkg3NC40MzM0TDgyLjIzNDYgMTA0LjgyMUM4MS4yMjIxIDEwNS4xNjIgODAuMTgxMyAxMDUuNDU0IDc5LjExNjcgMTA1LjY5N0M3My4xNTA1IDEwNy4wNTggNjcuMjAzMiAxMDYuNjQ1IDYxLjgxOCAxMDQuODAyTDY5LjU5OTUgOTAuNjY1OEg2OS42MTI2QzcwLjA3OTUgODkuNzg4OCA3MC45NjUgODkuMjY1NiA3MS44OTg2IDg5LjIzMTJaIiBmaWxsPSJ3aGl0ZSIgc3Ryb2tlPSJ3aGl0ZSIgc3Ryb2tlLXdpZHRoPSIwLjI1Ii8+CjxkZWZzPgo8bGluZWFyR3JhZGllbnQgaWQ9InBhaW50MF9saW5lYXJfNjgxXzI4NDUiIHgxPSIxMCIgeTE9IjE1LjUiIHgyPSIxNDQiIHkyPSIxMzEuNSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiPgo8c3RvcCBzdG9wLWNvbG9yPSIjNEQ4N0ZGIi8+CjxzdG9wIG9mZnNldD0iMSIgc3RvcC1jb2xvcj0iIzA1NDdEMCIvPgo8L2xpbmVhckdyYWRpZW50Pgo8L2RlZnM+Cjwvc3ZnPgo= - keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "nodeGroups"], ["spec", "nodeGroups", "md0"], ["spec", "nodeGroups", "md0", "minReplicas"], ["spec", "nodeGroups", "md0", "maxReplicas"], ["spec", "nodeGroups", "md0", "instanceType"], ["spec", "nodeGroups", "md0", "ephemeralStorage"], ["spec", "nodeGroups", "md0", "roles"], ["spec", "nodeGroups", "md0", "resources"], ["spec", "nodeGroups", "md0", "gpus"], ["spec", "version"], ["spec", "host"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "fluxcd"], ["spec", "addons", "fluxcd", "enabled"], ["spec", "addons", "fluxcd", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"]] + keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "nodeGroups"], ["spec", "nodeGroups", "md0"], ["spec", "nodeGroups", "md0", "minReplicas"], ["spec", "nodeGroups", "md0", "maxReplicas"], ["spec", "nodeGroups", "md0", "instanceType"], ["spec", "nodeGroups", "md0", "ephemeralStorage"], ["spec", "nodeGroups", "md0", "roles"], ["spec", "nodeGroups", "md0", "resources"], ["spec", "nodeGroups", "md0", "gpus"], ["spec", "version"], ["spec", "host"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "hami"], ["spec", "addons", "hami", "enabled"], ["spec", "addons", "hami", "valuesOverride"], ["spec", "addons", "fluxcd"], ["spec", "addons", "fluxcd", "enabled"], ["spec", "addons", "fluxcd", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"]] secrets: exclude: [] include: From 2734dc0bcb96aaa087cbff6b2ee261e1bde8bffa Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Fri, 24 Apr 2026 14:00:48 +0300 Subject: [PATCH 09/27] fix(hami): clean up leftover hami-dra references and harden defaults Remove broken hami-dra subchart dependency from vendored chart (Chart.yaml, Chart.lock, values.yaml) and strip DRA condition guards from all templates since the subchart was already deleted. Override devicePlugin updateStrategy to OnDelete to prevent destructive rolling updates of GPU workloads. Align gpu-operator template with project pattern (unconditional values emission). Add nodeConfiguration format documentation and test for conflicting valuesOverride scenario. Signed-off-by: Arsolitt --- .../templates/helmreleases/gpu-operator.yaml | 6 ++--- .../tests/gpu_operator_hami_test.yaml | 25 ++++++++++++++++++- packages/system/hami/charts/hami/Chart.lock | 6 ----- packages/system/hami/charts/hami/Chart.yaml | 6 +---- .../templates/device-plugin/configmap.yaml | 2 +- .../device-plugin/daemonsetnvidia.yaml | 2 +- .../templates/device-plugin/monitorrole.yaml | 2 +- .../device-plugin/monitorrolebinding.yaml | 2 +- .../device-plugin/monitorservice.yaml | 2 +- .../device-plugin/monitorserviceaccount.yaml | 2 +- .../device-plugin/runtime-class.yaml | 4 +-- .../hami/templates/scheduler/certmanager.yaml | 2 +- .../hami/templates/scheduler/clusterrole.yaml | 2 -- .../scheduler/clusterrolebinding.yaml | 4 +-- .../hami/templates/scheduler/configmap.yaml | 2 +- .../templates/scheduler/configmapnew.yaml | 2 +- .../hami/templates/scheduler/deployment.yaml | 4 +-- .../templates/scheduler/device-configmap.yaml | 4 +-- .../scheduler/job-patch/clusterrole.yaml | 4 +-- .../job-patch/clusterrolebinding.yaml | 4 +-- .../scheduler/job-patch/job-createSecret.yaml | 4 +-- .../scheduler/job-patch/job-patchWebhook.yaml | 4 +-- .../templates/scheduler/job-patch/psp.yaml | 4 +-- .../templates/scheduler/job-patch/role.yaml | 4 +-- .../scheduler/job-patch/rolebinding.yaml | 4 +-- .../scheduler/job-patch/serviceaccount.yaml | 4 +-- .../charts/hami/templates/scheduler/role.yaml | 4 +-- .../hami/templates/scheduler/rolebinding.yaml | 2 -- .../hami/templates/scheduler/service.yaml | 4 +-- .../templates/scheduler/serviceaccount.yaml | 2 -- .../hami/templates/scheduler/webhook.yaml | 2 +- packages/system/hami/charts/hami/values.yaml | 18 ------------- packages/system/hami/values.yaml | 4 +++ 33 files changed, 55 insertions(+), 92 deletions(-) delete mode 100644 packages/system/hami/charts/hami/Chart.lock diff --git a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml index 995aa430..7a5ecaf1 100644 --- a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml @@ -1,5 +1,5 @@ {{- define "cozystack.defaultGpuOperatorValues" -}} -{{- if and (hasKey .Values.addons "hami") .Values.addons.hami.enabled }} +{{- if .Values.addons.hami.enabled }} gpu-operator: devicePlugin: enabled: false @@ -40,10 +40,8 @@ spec: {{- $defaults := fromYaml (include "cozystack.defaultGpuOperatorValues" .) }} {{- $overrides := deepCopy (default (dict) .Values.addons.gpuOperator.valuesOverride) }} {{- $merged := mergeOverwrite (default (dict) $defaults) $overrides }} - {{- with $merged }} values: - {{- toYaml . | nindent 4 }} - {{- end }} + {{- if $merged }}{{ toYaml $merged | nindent 4 }}{{ end }} dependsOn: {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }} diff --git a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml index 33256ae6..11de743a 100644 --- a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml +++ b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml @@ -26,8 +26,9 @@ tests: enabled: false valuesOverride: {} asserts: - - notExists: + - equal: path: spec.values + value: null - it: should allow user overrides to merge with hami defaults set: @@ -66,6 +67,28 @@ tests: path: spec.values.gpu-operator.devicePlugin.enabled value: true + - it: should let user explicitly override devicePlugin.enabled to true with hami enabled + set: + addons: + gpuOperator: + enabled: true + valuesOverride: + gpu-operator: + devicePlugin: + enabled: true + driver: + enabled: false + hami: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.values.gpu-operator.devicePlugin.enabled + value: true + - equal: + path: spec.values.gpu-operator.driver.enabled + value: false + - it: should not render when gpuOperator is disabled set: addons: diff --git a/packages/system/hami/charts/hami/Chart.lock b/packages/system/hami/charts/hami/Chart.lock deleted file mode 100644 index 25856c07..00000000 --- a/packages/system/hami/charts/hami/Chart.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: hami-dra - repository: https://project-hami.github.io/HAMi-DRA/ - version: 0.1.0 -digest: sha256:374551539570bcee82c1c4dea93eac59e6f410b6d5967d188efd630e203d43bb -generated: "2026-04-17T04:06:47.689117678Z" diff --git a/packages/system/hami/charts/hami/Chart.yaml b/packages/system/hami/charts/hami/Chart.yaml index aba7e95c..55f32ab6 100644 --- a/packages/system/hami/charts/hami/Chart.yaml +++ b/packages/system/hami/charts/hami/Chart.yaml @@ -1,10 +1,6 @@ apiVersion: v2 appVersion: 2.8.1 -dependencies: -- condition: dra.enabled - name: hami-dra - repository: https://project-hami.github.io/HAMi-DRA/ - version: 0.1.0 +dependencies: [] description: Heterogeneous AI Computing Virtualization Middleware keywords: - vgpu diff --git a/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml b/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml index b36347f2..db2ddacb 100644 --- a/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml +++ b/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.devicePlugin.enabled (not .Values.devicePlugin.nodeConfiguration.externalConfigName) (not .Values.dra.enabled) -}} +{{- if and .Values.devicePlugin.enabled (not .Values.devicePlugin.nodeConfiguration.externalConfigName) -}} apiVersion: v1 kind: ConfigMap metadata: diff --git a/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml index 577dc0f9..1f4c24f3 100644 --- a/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml +++ b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) }} +{{- if .Values.devicePlugin.enabled }} apiVersion: apps/v1 kind: DaemonSet metadata: diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml index c51ff6c6..377f6a86 100644 --- a/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}} +{{- if .Values.devicePlugin.enabled -}} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml index 93a118dd..2f0a14ba 100644 --- a/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}} +{{- if .Values.devicePlugin.enabled -}} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml index 24daca9b..0fb156cc 100644 --- a/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}} +{{- if .Values.devicePlugin.enabled -}} apiVersion: v1 kind: Service metadata: diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml index b10d2cb9..6e3c2a44 100644 --- a/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}} +{{- if .Values.devicePlugin.enabled -}} apiVersion: v1 kind: ServiceAccount metadata: diff --git a/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml b/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml index cd3d14cc..ebcc2c9c 100644 --- a/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml +++ b/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml @@ -1,5 +1,4 @@ -{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}} -{{- if and .Values.devicePlugin.createRuntimeClass .Values.devicePlugin.runtimeClassName }} +{{- if and .Values.devicePlugin.enabled .Values.devicePlugin.createRuntimeClass .Values.devicePlugin.runtimeClassName -}} apiVersion: node.k8s.io/v1 kind: RuntimeClass metadata: @@ -7,5 +6,4 @@ metadata: annotations: helm.sh/hook: pre-install,pre-upgrade handler: nvidia -{{- end }} {{- end -}} diff --git a/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml b/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml index b6edb605..e6d28721 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} +{{- if .Values.scheduler.admissionWebhook.enabled -}} {{- if .Values.scheduler.certManager.enabled }} apiVersion: cert-manager.io/v1 kind: Certificate diff --git a/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml b/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml index 9c510c00..81c4fddf 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml @@ -1,4 +1,3 @@ -{{- if not .Values.dra.enabled -}} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -33,4 +32,3 @@ rules: resources: ["nodes"] verbs: ["get", "update", "list", "patch"] {{- end -}} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml index 69d75986..aa0f3c66 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml @@ -1,4 +1,3 @@ -{{- if not .Values.dra.enabled -}} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -45,5 +44,4 @@ roleRef: subjects: - kind: ServiceAccount name: {{ include "hami-vgpu.scheduler" . }} - namespace: {{ include "hami-vgpu.namespace" . }} -{{- end }} \ No newline at end of file + namespace: {{ include "hami-vgpu.namespace" . }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml b/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml index 75889146..109ecbce 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.scheduler.kubeScheduler.enabled (not .Values.dra.enabled) -}} +{{- if .Values.scheduler.kubeScheduler.enabled -}} apiVersion: v1 kind: ConfigMap metadata: diff --git a/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml b/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml index e2a91d8b..6f6db097 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.scheduler.kubeScheduler.enabled (not .Values.dra.enabled) -}} +{{- if .Values.scheduler.kubeScheduler.enabled -}} apiVersion: v1 kind: ConfigMap metadata: diff --git a/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml b/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml index 6364cae3..12911a42 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml @@ -1,4 +1,3 @@ -{{- if not .Values.dra.enabled -}} apiVersion: apps/v1 kind: Deployment metadata: @@ -223,5 +222,4 @@ spec: {{- end }} {{- if .Values.scheduler.nodeName }} nodeName: {{ .Values.scheduler.nodeName }} - {{- end }} -{{- end }} \ No newline at end of file + {{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml b/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml index ccb945ac..8738a06f 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml @@ -1,4 +1,3 @@ -{{- if not .Values.dra.enabled -}} apiVersion: v1 kind: ConfigMap metadata: @@ -406,5 +405,4 @@ data: memory: 12288 aiCore: 4 aiCPU: 4 - {{ end }} -{{- end }} \ No newline at end of file + {{ end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml index b089ae82..412b1094 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml @@ -1,5 +1,4 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} -{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -27,4 +26,3 @@ rules: - {{ include "hami-vgpu.fullname" . }}-admission {{- end }} {{- end }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml index 64f01ecf..2b82f926 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml @@ -1,5 +1,4 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} -{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -19,4 +18,3 @@ subjects: name: {{ include "hami-vgpu.fullname" . }}-admission namespace: {{ include "hami-vgpu.namespace" . }} {{- end }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml index 400168a7..0e36d95f 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml @@ -1,5 +1,4 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} -{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} apiVersion: batch/v1 kind: Job metadata: @@ -67,4 +66,3 @@ spec: runAsNonRoot: true runAsUser: {{ .Values.scheduler.patch.runAsUser }} {{- end }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml index 71e97f6b..ce52042c 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml @@ -1,5 +1,4 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} -{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} apiVersion: batch/v1 kind: Job metadata: @@ -62,4 +61,3 @@ spec: runAsNonRoot: true runAsUser: {{ .Values.scheduler.patch.runAsUser }} {{- end }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml index a48270e7..1e9cd7da 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml @@ -1,5 +1,4 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} -{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} {{- if .Values.podSecurityPolicy.enabled }} apiVersion: policy/v1beta1 kind: PodSecurityPolicy @@ -37,4 +36,3 @@ spec: - downwardAPI {{- end }} {{- end }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml index 74e45681..56682f8e 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml @@ -1,5 +1,4 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} -{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: @@ -20,4 +19,3 @@ rules: - get - create {{- end }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml index 32da5ba4..7239b128 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml @@ -1,5 +1,4 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} -{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: @@ -20,4 +19,3 @@ subjects: name: {{ include "hami-vgpu.fullname" . }}-admission namespace: {{ include "hami-vgpu.namespace" . }} {{- end }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml index d2eb41de..857c6a8d 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml @@ -1,5 +1,4 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) }} -{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} apiVersion: v1 kind: ServiceAccount metadata: @@ -12,4 +11,3 @@ metadata: {{- include "hami-vgpu.labels" . | nindent 4 }} app.kubernetes.io/component: admission-webhook {{- end }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/role.yaml b/packages/system/hami/charts/hami/templates/scheduler/role.yaml index 6e681967..90987a4f 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/role.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/role.yaml @@ -1,4 +1,3 @@ -{{- if not .Values.dra.enabled -}} apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: @@ -10,5 +9,4 @@ metadata: rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] - verbs: ["create", "list", "watch", "get", "update", "patch"] -{{- end }} \ No newline at end of file + verbs: ["create", "list", "watch", "get", "update", "patch"] \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml index 934f8223..96e175a1 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml @@ -1,4 +1,3 @@ -{{- if not .Values.dra.enabled -}} apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: @@ -30,4 +29,3 @@ subjects: name: {{ include "hami-vgpu.mock-device-plugin" . }} namespace: {{ include "hami-vgpu.namespace" . }} {{- end -}} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/service.yaml b/packages/system/hami/charts/hami/templates/scheduler/service.yaml index 3e18bc98..70378c5c 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/service.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/service.yaml @@ -1,4 +1,3 @@ -{{- if not .Values.dra.enabled }} apiVersion: v1 kind: Service metadata: @@ -32,5 +31,4 @@ spec: protocol: TCP selector: app.kubernetes.io/component: hami-scheduler - {{- include "hami-vgpu.selectorLabels" . | nindent 4 }} -{{- end }} \ No newline at end of file + {{- include "hami-vgpu.selectorLabels" . | nindent 4 }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml b/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml index 929bf59a..0435c003 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml @@ -1,4 +1,3 @@ -{{- if not .Values.dra.enabled -}} apiVersion: v1 kind: ServiceAccount metadata: @@ -15,4 +14,3 @@ metadata: name: {{ include "hami-vgpu.mock-device-plugin" . }} namespace: {{ include "hami-vgpu.namespace" . }} {{- end -}} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml b/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml index 148feb5c..7296eb4e 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} +{{- if .Values.scheduler.admissionWebhook.enabled -}} apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: diff --git a/packages/system/hami/charts/hami/values.yaml b/packages/system/hami/charts/hami/values.yaml index 9fbeca70..02feeada 100644 --- a/packages/system/hami/charts/hami/values.yaml +++ b/packages/system/hami/charts/hami/values.yaml @@ -392,24 +392,6 @@ mockDevicePlugin: ## - myRegistryKeySecretName ## pullSecrets: [] -# If this option is enabled, the DRA will be installed and the scheduler extender will not be installed. -dra: - enabled: false - -# Configuration for the hami-dra subchart, which includes DRA drivers. -# This is part of config for DRA, please refer to the DRA documentation for more details. -hami-dra: - monitor: - enabled: true - drivers: - nvidia: - enabled: true - # If you are using gpu driver on host, you need to set this to false - containerDriver: true - image: - repository: projecthami/k8s-dra-driver - tag: "v0.0.1-dev" - devices: amd: customresources: diff --git a/packages/system/hami/values.yaml b/packages/system/hami/values.yaml index 1155e2cb..1d1407b7 100644 --- a/packages/system/hami/values.yaml +++ b/packages/system/hami/values.yaml @@ -1,6 +1,10 @@ hami: devicePlugin: runtimeClassName: nvidia + updateStrategy: + type: OnDelete + # See upstream charts/hami/values.yaml for nodeConfiguration format. + # Each entry: {"name": "node-name", "operatingmode": "hami-core", "devicesplitcount": N, ...} nodeConfiguration: config: | { From 37d5ff0c6f0566ca18495e871c5ad626f7976ca1 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Fri, 24 Apr 2026 14:07:17 +0300 Subject: [PATCH 10/27] fix(hami): align templates with project patterns and clean dead code Unconditionally emit values in hami.yaml matching the project pattern. Remove duplicate test case and add coverage for omitted valuesOverride key. Delete dead PSP template and RBAC rules (policy/v1beta1 removed in K8s 1.25). Override kube-scheduler image registry to registry.k8s.io to avoid Chinese registry for international users. Signed-off-by: Arsolitt --- .../templates/helmreleases/hami.yaml | 4 +- .../tests/gpu_operator_hami_test.yaml | 29 ++++++-------- .../scheduler/job-patch/clusterrole.yaml | 7 ---- .../templates/scheduler/job-patch/psp.yaml | 38 ------------------- packages/system/hami/charts/hami/values.yaml | 3 -- packages/system/hami/values.yaml | 5 +++ 6 files changed, 18 insertions(+), 68 deletions(-) delete mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml diff --git a/packages/apps/kubernetes/templates/helmreleases/hami.yaml b/packages/apps/kubernetes/templates/helmreleases/hami.yaml index 1f7a5358..ac4714da 100644 --- a/packages/apps/kubernetes/templates/helmreleases/hami.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/hami.yaml @@ -32,10 +32,8 @@ spec: force: true remediation: retries: -1 - {{- with .Values.addons.hami.valuesOverride }} values: - {{- toYaml . | nindent 4 }} - {{- end }} + {{- if .Values.addons.hami.valuesOverride }}{{ toYaml .Values.addons.hami.valuesOverride | nindent 4 }}{{ end }} dependsOn: {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }} diff --git a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml index 11de743a..c3ec30f2 100644 --- a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml +++ b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml @@ -30,6 +30,18 @@ tests: path: spec.values value: null + - it: should apply hami defaults when valuesOverride key is omitted + set: + addons: + gpuOperator: + enabled: true + hami: + enabled: true + asserts: + - equal: + path: spec.values.gpu-operator.devicePlugin.enabled + value: false + - it: should allow user overrides to merge with hami defaults set: addons: @@ -50,23 +62,6 @@ tests: path: spec.values.gpu-operator.driver.enabled value: false - - it: should let user override devicePlugin back to true if needed - set: - addons: - gpuOperator: - enabled: true - valuesOverride: - gpu-operator: - devicePlugin: - enabled: true - hami: - enabled: true - valuesOverride: {} - asserts: - - equal: - path: spec.values.gpu-operator.devicePlugin.enabled - value: true - - it: should let user explicitly override devicePlugin.enabled to true with hami enabled set: addons: diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml index 412b1094..77e891cc 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml @@ -18,11 +18,4 @@ rules: verbs: - get - update -{{- if .Values.podSecurityPolicy.enabled }} - - apiGroups: ['extensions'] - resources: ['podsecuritypolicies'] - verbs: ['use'] - resourceNames: - - {{ include "hami-vgpu.fullname" . }}-admission -{{- end }} {{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml deleted file mode 100644 index 1e9cd7da..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml +++ /dev/null @@ -1,38 +0,0 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} -{{- if .Values.podSecurityPolicy.enabled }} -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - name: {{ include "hami-vgpu.fullname" . }}-admission - annotations: - "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded - labels: - {{- include "hami-vgpu.labels" . | nindent 4 }} - app.kubernetes.io/component: admission-webhook -spec: - allowPrivilegeEscalation: false - fsGroup: - ranges: - - max: 65535 - min: 1 - rule: MustRunAs - requiredDropCapabilities: - - ALL - runAsUser: - rule: MustRunAsNonRoot - seLinux: - rule: RunAsAny - supplementalGroups: - ranges: - - max: 65535 - min: 1 - rule: MustRunAs - volumes: - - configMap - - emptyDir - - projected - - secret - - downwardAPI -{{- end }} -{{- end }} diff --git a/packages/system/hami/charts/hami/values.yaml b/packages/system/hami/charts/hami/values.yaml index 02feeada..01caca0b 100644 --- a/packages/system/hami/charts/hami/values.yaml +++ b/packages/system/hami/charts/hami/values.yaml @@ -55,9 +55,6 @@ kunlunResourceVMemoryName: "kunlunxin.com/vxpu-memory" schedulerName: "hami-scheduler" -podSecurityPolicy: - enabled: false - scheduler: # @param nodeName defines the node name and the nvidia-vgpu-scheduler-scheduler will schedule to the node. # if we install the nvidia-vgpu-scheduler-scheduler as default scheduler, we need to remove the k8s default diff --git a/packages/system/hami/values.yaml b/packages/system/hami/values.yaml index 1d1407b7..f45d2998 100644 --- a/packages/system/hami/values.yaml +++ b/packages/system/hami/values.yaml @@ -1,4 +1,9 @@ hami: + scheduler: + kubeScheduler: + image: + registry: registry.k8s.io + repository: kube-scheduler devicePlugin: runtimeClassName: nvidia updateStrategy: From 6a9c310a4bfdb8bab3cd4f52724205adfbd2219e Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Fri, 24 Apr 2026 15:31:24 +0300 Subject: [PATCH 11/27] fix(hami): conditional values emission and cosmetic template cleanup Only emit values key in hami.yaml when valuesOverride has content, matching gpu-operator pattern. Add test verifying empty valuesOverride does not produce spec.values. Fix trailing whitespace and missing newlines in vendored chart templates. Signed-off-by: Arsolitt --- .../kubernetes/templates/helmreleases/hami.yaml | 4 +++- packages/apps/kubernetes/tests/hami_test.yaml | 15 +++++++++++++++ .../hami/charts/hami/templates/_commons.tpl | 2 +- .../hami/templates/device-plugin/configmap.yaml | 2 +- .../hami/templates/device-plugin/monitorrole.yaml | 3 +-- .../templates/device-plugin/monitorservice.yaml | 2 +- .../templates/scheduler/clusterrolebinding.yaml | 2 +- .../hami/templates/scheduler/deployment.yaml | 2 +- .../templates/scheduler/device-configmap.yaml | 2 +- .../charts/hami/templates/scheduler/role.yaml | 2 +- .../charts/hami/templates/scheduler/service.yaml | 2 +- .../charts/hami/templates/scheduler/webhook.yaml | 2 +- 12 files changed, 28 insertions(+), 12 deletions(-) diff --git a/packages/apps/kubernetes/templates/helmreleases/hami.yaml b/packages/apps/kubernetes/templates/helmreleases/hami.yaml index ac4714da..8733e675 100644 --- a/packages/apps/kubernetes/templates/helmreleases/hami.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/hami.yaml @@ -32,8 +32,10 @@ spec: force: true remediation: retries: -1 + {{- if .Values.addons.hami.valuesOverride }} values: - {{- if .Values.addons.hami.valuesOverride }}{{ toYaml .Values.addons.hami.valuesOverride | nindent 4 }}{{ end }} + {{- toYaml .Values.addons.hami.valuesOverride | nindent 4 }} + {{- end }} dependsOn: {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }} diff --git a/packages/apps/kubernetes/tests/hami_test.yaml b/packages/apps/kubernetes/tests/hami_test.yaml index e9d32236..bdbe990f 100644 --- a/packages/apps/kubernetes/tests/hami_test.yaml +++ b/packages/apps/kubernetes/tests/hami_test.yaml @@ -118,6 +118,21 @@ tests: name: test-gpu-operator namespace: test-ns + - it: should not render spec.values when valuesOverride is empty + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - hasDocuments: + count: 1 + - notExists: + path: spec.values + - it: should pass through valuesOverride set: addons: diff --git a/packages/system/hami/charts/hami/templates/_commons.tpl b/packages/system/hami/charts/hami/templates/_commons.tpl index 78a262c9..b68018e4 100644 --- a/packages/system/hami/charts/hami/templates/_commons.tpl +++ b/packages/system/hami/charts/hami/templates/_commons.tpl @@ -46,4 +46,4 @@ imagePullSecrets: - name: {{ . }} {{- end }} {{- end }} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml b/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml index db2ddacb..74631e24 100644 --- a/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml +++ b/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml @@ -10,4 +10,4 @@ metadata: data: config.json: | {{- .Values.devicePlugin.nodeConfiguration.config | nindent 4 }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml index 377f6a86..6ac757f2 100644 --- a/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml @@ -25,5 +25,4 @@ rules: - list - patch {{- end -}} - - + diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml index 0fb156cc..104664ea 100644 --- a/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml @@ -26,4 +26,4 @@ spec: selector: app.kubernetes.io/component: hami-device-plugin {{- include "hami-vgpu.selectorLabels" . | nindent 4 }} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml index aa0f3c66..a81d425c 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml @@ -44,4 +44,4 @@ roleRef: subjects: - kind: ServiceAccount name: {{ include "hami-vgpu.scheduler" . }} - namespace: {{ include "hami-vgpu.namespace" . }} \ No newline at end of file + namespace: {{ include "hami-vgpu.namespace" . }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml b/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml index 12911a42..1d89e189 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml @@ -222,4 +222,4 @@ spec: {{- end }} {{- if .Values.scheduler.nodeName }} nodeName: {{ .Values.scheduler.nodeName }} - {{- end }} \ No newline at end of file + {{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml b/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml index 8738a06f..873b813b 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml @@ -405,4 +405,4 @@ data: memory: 12288 aiCore: 4 aiCPU: 4 - {{ end }} \ No newline at end of file + {{ end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/role.yaml b/packages/system/hami/charts/hami/templates/scheduler/role.yaml index 90987a4f..5f7d7e44 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/role.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/role.yaml @@ -9,4 +9,4 @@ metadata: rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] - verbs: ["create", "list", "watch", "get", "update", "patch"] \ No newline at end of file + verbs: ["create", "list", "watch", "get", "update", "patch"] diff --git a/packages/system/hami/charts/hami/templates/scheduler/service.yaml b/packages/system/hami/charts/hami/templates/scheduler/service.yaml index 70378c5c..d7538fed 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/service.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/service.yaml @@ -31,4 +31,4 @@ spec: protocol: TCP selector: app.kubernetes.io/component: hami-scheduler - {{- include "hami-vgpu.selectorLabels" . | nindent 4 }} \ No newline at end of file + {{- include "hami-vgpu.selectorLabels" . | nindent 4 }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml b/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml index 7296eb4e..db9f8029 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml @@ -54,4 +54,4 @@ webhooks: scope: '*' sideEffects: None timeoutSeconds: 10 -{{- end }} \ No newline at end of file +{{- end }} From 1131e2f113a8fda8402bf9c92014bff9aaf29f25 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Fri, 24 Apr 2026 15:33:07 +0300 Subject: [PATCH 12/27] docs(hami): reference upstream repo for nodeConfiguration format Signed-off-by: Arsolitt --- packages/system/hami/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/system/hami/values.yaml b/packages/system/hami/values.yaml index f45d2998..64b372a6 100644 --- a/packages/system/hami/values.yaml +++ b/packages/system/hami/values.yaml @@ -8,7 +8,7 @@ hami: runtimeClassName: nvidia updateStrategy: type: OnDelete - # See upstream charts/hami/values.yaml for nodeConfiguration format. + # See https://github.com/Project-HAMi/HAMi/blob/v2.8.1/deployments/charts/hami/values.yaml # Each entry: {"name": "node-name", "operatingmode": "hami-core", "devicesplitcount": N, ...} nodeConfiguration: config: | From 3eeda2ba3596851393fd62d3d352cdd1017be267 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Fri, 24 Apr 2026 16:12:34 +0300 Subject: [PATCH 13/27] chore(kubernetes): regenerate code after hami addon addition Signed-off-by: Arsolitt --- .../kubernetes/zz_generated.deepcopy.go | 17 +++++++++++++++++ api/backups/v1alpha1/zz_generated.deepcopy.go | 5 +++++ 2 files changed, 22 insertions(+) diff --git a/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go b/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go index 2cbe3ba1..3f1cb5ce 100644 --- a/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go +++ b/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go @@ -49,6 +49,7 @@ func (in *Addons) DeepCopyInto(out *Addons) { in.Fluxcd.DeepCopyInto(&out.Fluxcd) out.GatewayAPI = in.GatewayAPI in.GpuOperator.DeepCopyInto(&out.GpuOperator) + in.Hami.DeepCopyInto(&out.Hami) in.IngressNginx.DeepCopyInto(&out.IngressNginx) in.MonitoringAgents.DeepCopyInto(&out.MonitoringAgents) in.Velero.DeepCopyInto(&out.Velero) @@ -260,6 +261,22 @@ func (in *GatewayAPIAddon) DeepCopy() *GatewayAPIAddon { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HAMiAddon) DeepCopyInto(out *HAMiAddon) { + *out = *in + in.ValuesOverride.DeepCopyInto(&out.ValuesOverride) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HAMiAddon. +func (in *HAMiAddon) DeepCopy() *HAMiAddon { + if in == nil { + return nil + } + out := new(HAMiAddon) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IngressNginxAddon) DeepCopyInto(out *IngressNginxAddon) { *out = *in diff --git a/api/backups/v1alpha1/zz_generated.deepcopy.go b/api/backups/v1alpha1/zz_generated.deepcopy.go index 89f6171f..61b8f839 100644 --- a/api/backups/v1alpha1/zz_generated.deepcopy.go +++ b/api/backups/v1alpha1/zz_generated.deepcopy.go @@ -620,6 +620,11 @@ func (in *RestoreJobSpec) DeepCopyInto(out *RestoreJobSpec) { *out = new(v1.TypedLocalObjectReference) (*in).DeepCopyInto(*out) } + if in.Options != nil { + in, out := &in.Options, &out.Options + *out = new(runtime.RawExtension) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestoreJobSpec. From d8d870cc2af73925561ae2f9aa936613b47ad0fe Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Mon, 27 Apr 2026 13:24:27 +0300 Subject: [PATCH 14/27] fix(hami): use RollingUpdate strategy for device plugin DaemonSet OnDelete requires manual pod deletion to apply updates. RollingUpdate with maxUnavailable constraint matches upstream default and is consistent with all other system packages. Signed-off-by: Arsolitt --- packages/system/hami/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/system/hami/values.yaml b/packages/system/hami/values.yaml index 64b372a6..76e8d631 100644 --- a/packages/system/hami/values.yaml +++ b/packages/system/hami/values.yaml @@ -7,7 +7,7 @@ hami: devicePlugin: runtimeClassName: nvidia updateStrategy: - type: OnDelete + type: RollingUpdate # See https://github.com/Project-HAMi/HAMi/blob/v2.8.1/deployments/charts/hami/values.yaml # Each entry: {"name": "node-name", "operatingmode": "hami-core", "devicesplitcount": N, ...} nodeConfiguration: From 36852548e576e153981eab0146614442b5bc9a15 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Mon, 27 Apr 2026 13:29:15 +0300 Subject: [PATCH 15/27] fix(hami): correct label indentation in device-plugin monitorservice Use nindent instead of indent with leading whitespace to prevent broken YAML rendering when devicePlugin.service.labels is set. Signed-off-by: Arsolitt --- .../charts/hami/templates/device-plugin/monitorservice.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml index 104664ea..88f1b214 100644 --- a/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml @@ -7,8 +7,8 @@ metadata: labels: app.kubernetes.io/component: hami-device-plugin {{- include "hami-vgpu.labels" . | nindent 4 }} - {{- if .Values.devicePlugin.service.labels }} # Use devicePlugin instead of scheduler - {{ toYaml .Values.devicePlugin.service.labels | indent 4 }} + {{- with .Values.devicePlugin.service.labels }} + {{- toYaml . | nindent 4 }} {{- end }} {{- if .Values.devicePlugin.service.annotations }} # Use devicePlugin instead of scheduler annotations: {{ toYaml .Values.devicePlugin.service.annotations | nindent 4 }} From f0e033ebcb2fc727bc8fb473507de34edacbf898 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Mon, 27 Apr 2026 13:44:08 +0300 Subject: [PATCH 16/27] style(kubernetes): use with instead of if for hami valuesOverride Signed-off-by: Arsolitt --- packages/apps/kubernetes/templates/helmreleases/hami.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apps/kubernetes/templates/helmreleases/hami.yaml b/packages/apps/kubernetes/templates/helmreleases/hami.yaml index 8733e675..1f7a5358 100644 --- a/packages/apps/kubernetes/templates/helmreleases/hami.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/hami.yaml @@ -32,9 +32,9 @@ spec: force: true remediation: retries: -1 - {{- if .Values.addons.hami.valuesOverride }} + {{- with .Values.addons.hami.valuesOverride }} values: - {{- toYaml .Values.addons.hami.valuesOverride | nindent 4 }} + {{- toYaml . | nindent 4 }} {{- end }} dependsOn: From c53f10475025c6675e5270f359f97f619db7c80f Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Mon, 27 Apr 2026 13:44:12 +0300 Subject: [PATCH 17/27] docs(hami): clarify that parameter defaults come from upstream chart Signed-off-by: Arsolitt --- packages/system/hami/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/system/hami/README.md b/packages/system/hami/README.md index 283f9e80..d648364d 100644 --- a/packages/system/hami/README.md +++ b/packages/system/hami/README.md @@ -72,6 +72,8 @@ resources: ## Parameters +Default values shown below are inherited from the upstream HAMi chart and may change with upstream updates. + | Name | Description | Default | | --- | --- | --- | | `hami.devicePlugin.runtimeClassName` | RuntimeClass for device plugin pods | `nvidia` | From 9ba5e587810f44d522d5876e38f12f87c3dde3bb Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Mon, 27 Apr 2026 13:44:17 +0300 Subject: [PATCH 18/27] fix(kubernetes): avoid rendering empty values key in gpu-operator HelmRelease Wrap the values: block in a conditional so the key is omitted entirely when no defaults or overrides produce content. Previously the template always emitted values: null, triggering unnecessary FluxCD reconciliation. Signed-off-by: Arsolitt --- .../apps/kubernetes/templates/helmreleases/gpu-operator.yaml | 4 +++- packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml index 7a5ecaf1..15849495 100644 --- a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml @@ -40,8 +40,10 @@ spec: {{- $defaults := fromYaml (include "cozystack.defaultGpuOperatorValues" .) }} {{- $overrides := deepCopy (default (dict) .Values.addons.gpuOperator.valuesOverride) }} {{- $merged := mergeOverwrite (default (dict) $defaults) $overrides }} + {{- if $merged }} values: - {{- if $merged }}{{ toYaml $merged | nindent 4 }}{{ end }} + {{- toYaml $merged | nindent 4 }} + {{- end }} dependsOn: {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }} diff --git a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml index c3ec30f2..4fa754fb 100644 --- a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml +++ b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml @@ -26,9 +26,8 @@ tests: enabled: false valuesOverride: {} asserts: - - equal: + - notExists: path: spec.values - value: null - it: should apply hami defaults when valuesOverride key is omitted set: From 1dc02d44f4ea15253ffd799e85a54ed79ae87fd1 Mon Sep 17 00:00:00 2001 From: Timofei Larkin Date: Mon, 27 Apr 2026 10:27:46 +0300 Subject: [PATCH 19/27] refactor(lineage-controller-webhook): align with cozystack-api shape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Collapse the chart to a single Deployment shape that mirrors cozystack-api: 2 replicas, soft (preferred) nodeAffinity to node-role.kubernetes.io/control-plane via Exists, permissive tolerations, soft podAntiAffinity on hostname, an unconditional PodDisruptionBudget with maxUnavailable: 1, and a Service with spec.trafficDistribution: PreferClose. The same shape works on Talos / kubeadm / k3s and on managed Kubernetes / Cozy-in-Cozy tenant clusters without per-distro overrides — fixing #2417 by making the soft control-plane affinity gracefully fall back to worker scheduling when no control-plane nodes are visible. Drop the DaemonSet path entirely. The previous PR's deployment.enabled toggle, the workload-kind switch in templates/workload.yaml, the fail-guard for localK8sAPIEndpoint+nodeAffinity=[], the conditional PDB, and the values-shape knobs for nodeAffinity and tolerations all go away as a consequence. Override the standard Deployment fields through the usual component-values mechanism if a non-default topology is ever needed. Mark localK8sAPIEndpoint.enabled deprecated and flip the default to false. The flag injects KUBERNETES_SERVICE_HOST=status.hostIP, which is only valid when the pod is actually scheduled on an apiserver- bearing node. With the new soft control-plane affinity, the pod can land off-control-plane and crash-loop. The latency motivation for the flag is real but pending separate webhook performance work; once addressed, the flag can be removed. Revert all changes to packages/core/platform/images/migrations/migrations/20. The earlier ds/...-or-deploy/... fallback was over-engineered: per run-migrations.sh, migration 20 fires only when CURRENT < 20 (i.e. direct upgrades from pre-0.37 to 1.3+), and that path is unsupported anyway. The original ds/... rollout-status line is dead code on every supported install and upgrade path. Add helm-unittest coverage for: the default Deployment shape (replicas, soft nodeAffinity, soft podAntiAffinity, tolerations); no env vars at the default localK8sAPIEndpoint.enabled=false; PDB rendered unconditionally with maxUnavailable: 1, including at replicas=1 (the no-op case); replicas value drives spec.replicas; and that localK8sAPIEndpoint.enabled=true does inject the env vars. The Service test continues to assert trafficDistribution: PreferClose with no internalTrafficPolicy. Add a slim README documenting the topology, parameters, and the deprecation note for localK8sAPIEndpoint. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Timofei Larkin --- .../platform/templates/bundles/system.yaml | 3 - .../lineage-controller-webhook/Makefile | 5 + .../lineage-controller-webhook/README.md | 67 +++++++++++++ .../templates/deployment.yaml | 49 --------- .../templates/pdb.yaml | 4 +- .../templates/service.yaml | 2 +- .../{daemonset.yaml => workload.yaml} | 27 +++-- .../tests/service_test.yaml | 30 ++++++ .../tests/workload_test.yaml | 99 +++++++++++++++++++ .../lineage-controller-webhook/values.yaml | 18 ++-- 10 files changed, 230 insertions(+), 74 deletions(-) create mode 100644 packages/system/lineage-controller-webhook/README.md delete mode 100644 packages/system/lineage-controller-webhook/templates/deployment.yaml rename packages/system/lineage-controller-webhook/templates/{daemonset.yaml => workload.yaml} (67%) create mode 100644 packages/system/lineage-controller-webhook/tests/service_test.yaml create mode 100644 packages/system/lineage-controller-webhook/tests/workload_test.yaml diff --git a/packages/core/platform/templates/bundles/system.yaml b/packages/core/platform/templates/bundles/system.yaml index 43ea7266..6c587c82 100644 --- a/packages/core/platform/templates/bundles/system.yaml +++ b/packages/core/platform/templates/bundles/system.yaml @@ -105,9 +105,6 @@ {{- /* cozystack-api DaemonSet */ -}} {{- $apiValues := dict "cozystackAPI" (dict "nodeSelector" $genericNodeSelector) -}} {{- $_ := set $cozystackEngineComponents "cozystack-api" (dict "values" $apiValues) -}} -{{- /* lineage-controller-webhook DaemonSet */ -}} -{{- $lineageValues := dict "lineageControllerWebhook" (dict "nodeSelector" $genericNodeSelector) -}} -{{- $_ := set $cozystackEngineComponents "lineage-controller-webhook" (dict "values" $lineageValues) -}} {{- end -}} {{- if .Values.authentication.oidc.enabled }} {{include "cozystack.platform.package" (list "cozystack.cozystack-engine" "oidc" $ $cozystackEngineComponents) }} diff --git a/packages/system/lineage-controller-webhook/Makefile b/packages/system/lineage-controller-webhook/Makefile index d5bada31..cdc1cf01 100644 --- a/packages/system/lineage-controller-webhook/Makefile +++ b/packages/system/lineage-controller-webhook/Makefile @@ -4,8 +4,13 @@ NAMESPACE=cozy-system include ../../../hack/common-envs.mk include ../../../hack/package.mk +.PHONY: test + image: image-lineage-controller-webhook +test: + helm unittest . + image-lineage-controller-webhook: docker buildx build -f images/lineage-controller-webhook/Dockerfile ../../.. \ --tag $(REGISTRY)/lineage-controller-webhook:$(call settag,$(TAG)) \ diff --git a/packages/system/lineage-controller-webhook/README.md b/packages/system/lineage-controller-webhook/README.md new file mode 100644 index 00000000..1e43c9c8 --- /dev/null +++ b/packages/system/lineage-controller-webhook/README.md @@ -0,0 +1,67 @@ +# lineage-controller-webhook + +Cozystack system package for the **lineage controller webhook** — a mutating +admission webhook that stamps "lineage" labels onto tenant workloads, linking +each resource back to the Cozystack `Application` that ultimately owns it. + +The webhook intercepts CREATE and UPDATE on `pods`, `secrets`, `services`, +`persistentvolumeclaims`, `ingresses.networking.k8s.io`, and +`workloadmonitors.cozystack.io` outside system namespaces. For each request it +walks the ownership graph upward (Kubernetes `ownerReferences`, then the +`HelmRelease` Flux installed the resource with, then the Cozystack +`Application`-derived CRD that produced the HelmRelease) and writes the +discovered application's group, kind and name as labels +(`apps.cozystack.io/application.{group,kind,name}`) on the incoming object. +Those labels let the aggregated API server, the Cozystack dashboard, and other +lineage-aware consumers reason about which application a resource belongs to. + +The webhook serves TLS on port 9443 with a cert-manager issued certificate, and +runs with `failurePolicy: Fail` — every CREATE/UPDATE on the resources above +is gated on the webhook being reachable. + +## Topology + +The chart ships a single shape, modelled on `cozystack-api`: + +- **Deployment** with two replicas (override via `replicas`). +- **Soft `nodeAffinity`** preferring `node-role.kubernetes.io/control-plane` + (`Exists`, so it matches both Talos's empty value and k3s/kubeadm's `"true"`). + Soft means: the pod lands on a control-plane node when one is reachable, and + on any worker otherwise — no override needed for managed Kubernetes, + Cozy-in-Cozy tenants, or any other cluster where control-plane nodes aren't + visible. +- **Permissive `tolerations`** (`[{operator: Exists}]`) so the pod can land on + tainted control-plane nodes when the soft affinity is satisfiable. +- **Soft `podAntiAffinity`** on `kubernetes.io/hostname` so replicas spread + across nodes when possible (best-effort). +- **`PodDisruptionBudget`** with `maxUnavailable: 1`, unconditional. At + `replicas: 1` it's a useful no-op (allows full drain); at `replicas >= 2` + it caps disruption to one pod. +- **`Service` with `spec.trafficDistribution: PreferClose`** so the apiserver + prefers a webhook endpoint on its own node when one exists, and transparently + falls over to a remote endpoint otherwise. Requires Kubernetes ≥ 1.31; on + older clusters the field is silently ignored and traffic uses default + cluster-wide distribution. + +## Parameters + +All keys live under the top-level `lineageControllerWebhook:` map. + +| Name | Description | Value | +| ----------------------------- | --------------------------------------------------------- | -------------------------------------------------------------------------------- | +| `image` | Container image (digest-pinned) | `ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.3.0@sha256:e898…6fb0` | +| `debug` | Enable `--zap-log-level=debug` instead of `info` | `false` | +| `replicas` | Deployment replica count | `2` | +| `localK8sAPIEndpoint.enabled` | **Deprecated.** See note below. | `false` | + +### `localK8sAPIEndpoint.enabled` (deprecated) + +When enabled, this injects `KUBERNETES_SERVICE_HOST=status.hostIP` and +`KUBERNETES_SERVICE_PORT=6443` so the webhook talks to the kube-apiserver on +its own node. It was originally added to avoid latency on the +webhook-to-apiserver path, but it is only valid when the pod is actually +scheduled on an apiserver-bearing node — which the chart's soft control-plane +affinity no longer guarantees. With this flag enabled and the pod scheduled +off a control-plane node, the controller will crash-loop dialing a non- +apiserver hostIP. Slated for removal once the latency motivation is addressed +in the webhook itself; **leave disabled**. diff --git a/packages/system/lineage-controller-webhook/templates/deployment.yaml b/packages/system/lineage-controller-webhook/templates/deployment.yaml deleted file mode 100644 index 87717fd3..00000000 --- a/packages/system/lineage-controller-webhook/templates/deployment.yaml +++ /dev/null @@ -1,49 +0,0 @@ -{{- if .Values.lineageControllerWebhook.deployment.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: lineage-controller-webhook - labels: - app: lineage-controller-webhook -spec: - replicas: {{ .Values.lineageControllerWebhook.deployment.replicas }} - selector: - matchLabels: - app: lineage-controller-webhook - template: - metadata: - labels: - app: lineage-controller-webhook - spec: - serviceAccountName: lineage-controller-webhook - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - topologyKey: kubernetes.io/hostname - labelSelector: - matchLabels: - app: lineage-controller-webhook - containers: - - name: lineage-controller-webhook - image: "{{ .Values.lineageControllerWebhook.image }}" - args: - {{- if .Values.lineageControllerWebhook.debug }} - - --zap-log-level=debug - {{- else }} - - --zap-log-level=info - {{- end }} - ports: - - name: webhook - containerPort: 9443 - volumeMounts: - - name: webhook-certs - mountPath: /tmp/k8s-webhook-server/serving-certs - readOnly: true - volumes: - - name: webhook-certs - secret: - secretName: lineage-controller-webhook-cert - defaultMode: 0400 -{{- end }} \ No newline at end of file diff --git a/packages/system/lineage-controller-webhook/templates/pdb.yaml b/packages/system/lineage-controller-webhook/templates/pdb.yaml index b0a9d4e4..0786c8ff 100644 --- a/packages/system/lineage-controller-webhook/templates/pdb.yaml +++ b/packages/system/lineage-controller-webhook/templates/pdb.yaml @@ -1,11 +1,9 @@ -{{- if .Values.lineageControllerWebhook.deployment.enabled }} apiVersion: policy/v1 kind: PodDisruptionBudget metadata: name: lineage-controller-webhook spec: - minAvailable: {{ if gt (int .Values.lineageControllerWebhook.deployment.replicas) 1 }}1{{ else }}0{{ end }} + maxUnavailable: 1 selector: matchLabels: app: lineage-controller-webhook -{{- end }} \ No newline at end of file diff --git a/packages/system/lineage-controller-webhook/templates/service.yaml b/packages/system/lineage-controller-webhook/templates/service.yaml index c5df90fb..fbb3ad54 100644 --- a/packages/system/lineage-controller-webhook/templates/service.yaml +++ b/packages/system/lineage-controller-webhook/templates/service.yaml @@ -5,7 +5,7 @@ metadata: labels: app: lineage-controller-webhook spec: - internalTrafficPolicy: Local + trafficDistribution: PreferClose type: ClusterIP ports: - port: 443 diff --git a/packages/system/lineage-controller-webhook/templates/daemonset.yaml b/packages/system/lineage-controller-webhook/templates/workload.yaml similarity index 67% rename from packages/system/lineage-controller-webhook/templates/daemonset.yaml rename to packages/system/lineage-controller-webhook/templates/workload.yaml index 536ea24a..097919c6 100644 --- a/packages/system/lineage-controller-webhook/templates/daemonset.yaml +++ b/packages/system/lineage-controller-webhook/templates/workload.yaml @@ -1,11 +1,11 @@ -{{- if not .Values.lineageControllerWebhook.deployment.enabled }} apiVersion: apps/v1 -kind: DaemonSet +kind: Deployment metadata: name: lineage-controller-webhook labels: app: lineage-controller-webhook spec: + replicas: {{ .Values.lineageControllerWebhook.replicas }} selector: matchLabels: app: lineage-controller-webhook @@ -14,13 +14,25 @@ spec: labels: app: lineage-controller-webhook spec: - {{- with .Values.lineageControllerWebhook.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} + serviceAccountName: lineage-controller-webhook tolerations: - operator: Exists - serviceAccountName: lineage-controller-webhook + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app: lineage-controller-webhook containers: - name: lineage-controller-webhook image: "{{ .Values.lineageControllerWebhook.image }}" @@ -52,4 +64,3 @@ spec: secret: secretName: lineage-controller-webhook-cert defaultMode: 0400 -{{- end }} \ No newline at end of file diff --git a/packages/system/lineage-controller-webhook/tests/service_test.yaml b/packages/system/lineage-controller-webhook/tests/service_test.yaml new file mode 100644 index 00000000..3cdbddd8 --- /dev/null +++ b/packages/system/lineage-controller-webhook/tests/service_test.yaml @@ -0,0 +1,30 @@ +suite: lineage-controller-webhook service +templates: + - templates/service.yaml + +release: + name: lineage-controller-webhook + namespace: cozy-system + +tests: + - it: renders a ClusterIP service on 443 -> 9443 with PreferClose traffic distribution + asserts: + - isKind: + of: Service + - equal: + path: spec.type + value: ClusterIP + - equal: + path: spec.trafficDistribution + value: PreferClose + - notExists: + path: spec.internalTrafficPolicy + - equal: + path: spec.ports[0].port + value: 443 + - equal: + path: spec.ports[0].targetPort + value: 9443 + - equal: + path: spec.selector.app + value: lineage-controller-webhook diff --git a/packages/system/lineage-controller-webhook/tests/workload_test.yaml b/packages/system/lineage-controller-webhook/tests/workload_test.yaml new file mode 100644 index 00000000..bcb24014 --- /dev/null +++ b/packages/system/lineage-controller-webhook/tests/workload_test.yaml @@ -0,0 +1,99 @@ +suite: lineage-controller-webhook workload + PDB +templates: + - templates/workload.yaml + - templates/pdb.yaml + +release: + name: lineage-controller-webhook + namespace: cozy-system + +tests: + # ---- Default Deployment shape ---------------------------------------------- + + - it: defaults render a 2-replica Deployment with soft control-plane nodeAffinity, soft podAntiAffinity, and Exists tolerations + template: templates/workload.yaml + asserts: + - isKind: + of: Deployment + - equal: + path: spec.replicas + value: 2 + - equal: + path: spec.template.spec.tolerations + value: + - operator: Exists + - equal: + path: spec.template.spec.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution + value: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - notExists: + path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution + - equal: + path: spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.topologyKey + value: kubernetes.io/hostname + + - it: defaults inject no localK8sAPIEndpoint env vars (knob is opt-in) + template: templates/workload.yaml + asserts: + - notExists: + path: spec.template.spec.containers[0].env + + # ---- PDB -------------------------------------------------------------------- + + - it: PDB renders unconditionally with maxUnavailable=1 at the default replica count + template: templates/pdb.yaml + asserts: + - isKind: + of: PodDisruptionBudget + - equal: + path: spec.maxUnavailable + value: 1 + - notExists: + path: spec.minAvailable + + - it: PDB still renders at replicas=1 (no-op, but consistent shape) + set: + lineageControllerWebhook.replicas: 1 + template: templates/pdb.yaml + asserts: + - isKind: + of: PodDisruptionBudget + - equal: + path: spec.maxUnavailable + value: 1 + + # ---- Replica override ------------------------------------------------------- + + - it: replicas value drives spec.replicas + set: + lineageControllerWebhook.replicas: 5 + template: templates/workload.yaml + asserts: + - equal: + path: spec.replicas + value: 5 + + # ---- Deprecated localK8sAPIEndpoint knob ----------------------------------- + + - it: localK8sAPIEndpoint.enabled=true injects KUBERNETES_SERVICE_HOST and PORT env vars + set: + lineageControllerWebhook.localK8sAPIEndpoint.enabled: true + template: templates/workload.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: KUBERNETES_SERVICE_HOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + - contains: + path: spec.template.spec.containers[0].env + content: + name: KUBERNETES_SERVICE_PORT + value: "6443" diff --git a/packages/system/lineage-controller-webhook/values.yaml b/packages/system/lineage-controller-webhook/values.yaml index 2b84f4e1..e56d332d 100644 --- a/packages/system/lineage-controller-webhook/values.yaml +++ b/packages/system/lineage-controller-webhook/values.yaml @@ -1,15 +1,13 @@ lineageControllerWebhook: image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.3.0@sha256:e8984709686a5eaf19b89da378d7b8c688ea5607e0783a88d9c9e4ccfca96fb0 debug: false + replicas: 2 + # DEPRECATED. Injects KUBERNETES_SERVICE_HOST=status.hostIP and + # KUBERNETES_SERVICE_PORT=6443 so the webhook talks to the apiserver on its + # own node — only valid when the pod is actually scheduled on an apiserver- + # bearing node. Now that the chart's nodeAffinity to control-plane is soft + # (preferred, not required), the pod can land off-control-plane and crash- + # loop dialing a non-apiserver hostIP. Slated for removal once webhook + # performance work obviates the locality motivation. Leave disabled. localK8sAPIEndpoint: - # incompatable with Deployment mode - enabled: true - # nodeSelector for the DaemonSet - # Talos uses empty value: "node-role.kubernetes.io/control-plane": "" - # Generic k8s (k3s, kubeadm) uses: "node-role.kubernetes.io/control-plane": "true" - nodeSelector: - node-role.kubernetes.io/control-plane: "" - # if enabled, replace DaemonSet by Deployment - deployment: enabled: false - replicas: 1 From 3a8359fa73f5cd96d8363c48d31414d8cadf3447 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 18:59:43 +0300 Subject: [PATCH 20/27] chore(kubernetes): regenerate deepcopy after merge Run `make generate` to pick up the Images type added in main. Signed-off-by: Arsolitt --- .../v1alpha1/kubernetes/zz_generated.deepcopy.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go b/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go index 3f1cb5ce..e021fbaa 100644 --- a/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go +++ b/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go @@ -136,6 +136,7 @@ func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) { } in.Addons.DeepCopyInto(&out.Addons) in.ControlPlane.DeepCopyInto(&out.ControlPlane) + out.Images = in.Images } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigSpec. @@ -277,6 +278,21 @@ func (in *HAMiAddon) DeepCopy() *HAMiAddon { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Images) DeepCopyInto(out *Images) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Images. +func (in *Images) DeepCopy() *Images { + if in == nil { + return nil + } + out := new(Images) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IngressNginxAddon) DeepCopyInto(out *IngressNginxAddon) { *out = *in From 80b86b0c1a39266488b84473724a54b0d3ad7d9c Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 19:02:35 +0300 Subject: [PATCH 21/27] chore(kubernetes): regenerate CRD after merge conflict resolution Per-package `make generate` corrects keysOrder placement and openAPISchema formatting in the kubernetes CRD definition. Signed-off-by: Arsolitt --- api/apps/v1alpha1/kubernetes/types.go | 2 +- packages/system/kubernetes-rd/cozyrds/kubernetes.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/apps/v1alpha1/kubernetes/types.go b/api/apps/v1alpha1/kubernetes/types.go index e0143445..88a3c97e 100644 --- a/api/apps/v1alpha1/kubernetes/types.go +++ b/api/apps/v1alpha1/kubernetes/types.go @@ -6,7 +6,7 @@ package kubernetes import ( resource "k8s.io/apimachinery/pkg/api/resource" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" k8sRuntime "k8s.io/apimachinery/pkg/runtime" ) diff --git a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml index b4ada238..f81eaff0 100644 --- a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml +++ b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml @@ -22,7 +22,7 @@ spec: singular: kubernetes plural: kuberneteses openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"ephemeralStorage":"20Gi","gpus":[],"instanceType":"u1.medium","maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"]}},"additionalProperties":{"type":"object","required":["ephemeralStorage","instanceType","maxReplicas","minReplicas","resources"],"properties":{"ephemeralStorage":{"description":"Ephemeral storage size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string"}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium"},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `.` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","hami","ingressNginx","monitoringAgents","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"hami":{"description":"HAMi GPU virtualization middleware.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable HAMi (requires GPU Operator).","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"large","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"images":{"description":"Optional image overrides for air-gapped or rate-limited registries.","type":"object","default":{},"properties":{"waitForKubeconfig":{"description":"Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.","type":"string","default":""}}}}} + {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"ephemeralStorage":"20Gi","gpus":[],"instanceType":"u1.medium","maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"]}},"additionalProperties":{"type":"object","required":["ephemeralStorage","instanceType","maxReplicas","minReplicas","resources"],"properties":{"ephemeralStorage":{"description":"Ephemeral storage size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string"}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium"},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `.` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","hami","ingressNginx","monitoringAgents","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"hami":{"description":"HAMi GPU virtualization middleware.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable HAMi (requires GPU Operator).","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"large","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"images":{"description":"Optional image overrides for air-gapped or rate-limited registries.","type":"object","default":{},"properties":{"waitForKubeconfig":{"description":"Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.","type":"string","default":""}}}}} release: prefix: kubernetes- labels: @@ -40,7 +40,7 @@ spec: tags: - compute icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODFfMjg0NSkiLz4KPHBhdGggZD0iTTcxLjk5NjggMTlDNzAuMzAzOSAxOS4wMDAyIDY4LjkzMTIgMjAuNTMyMiA2OC45MzE0IDIyLjQyMjFDNjguOTMxNCAyMi40NTExIDY4LjkzNzMgMjIuNDc4OCA2OC45Mzc5IDIyLjUwNzZDNjguOTM1NCAyMi43NjQ0IDY4LjkyMzEgMjMuMDczNyA2OC45MzE0IDIzLjI5NzNDNjguOTcxNyAyNC4zODczIDY5LjIwODIgMjUuMjIxNiA2OS4zNTA2IDI2LjIyNThDNjkuNjA4NCAyOC4zNzUyIDY5LjgyNDUgMzAuMTU2OSA2OS42OTEyIDMxLjgxM0M2OS41NjE1IDMyLjQzNzUgNjkuMTAzNyAzMy4wMDg2IDY4LjY5NTYgMzMuNDA1Nkw2OC42MjM1IDM0LjcwODZDNjYuNzgzOSAzNC44NjE3IDY0LjkzMTkgMzUuMTQyMSA2My4wODIxIDM1LjU2NDFDNTUuMTIyNiAzNy4zNzk4IDQ4LjI2OTUgNDEuNDk5MSA0My4wNTIgNDcuMDYwOUM0Mi43MTM0IDQ2LjgyODggNDIuMTIxMSA0Ni40MDE5IDQxLjk0NSA0Ni4yNzEyQzQxLjM5NzcgNDYuMzQ1NCA0MC44NDQ1IDQ2LjUxNTEgNDAuMTI0MSA0Ni4wOTM1QzM4Ljc1MjIgNDUuMTY1NyAzNy41MDI4IDQzLjg4NTEgMzUuOTkxIDQyLjM0MjRDMzUuMjk4MiA0MS42MDQ0IDM0Ljc5NjYgNDAuOTAxOCAzMy45NzM1IDQwLjE5MDRDMzMuNzg2NiA0MC4wMjg5IDMzLjUwMTQgMzkuODEwNCAzMy4yOTIzIDM5LjY0NDJDMzIuNjQ4OSAzOS4xMjg4IDMxLjg5IDM4Ljg2IDMxLjE1NyAzOC44MzQ4QzMwLjIxNDcgMzguODAyNCAyOS4zMDc1IDM5LjE3MjUgMjguNzEzOCAzOS45MjA2QzI3LjY1ODQgNDEuMjUwNiAyNy45OTYzIDQzLjI4MzMgMjkuNDY3MSA0NC40NjE0QzI5LjQ4MiA0NC40NzM0IDI5LjQ5NzkgNDQuNDgyNyAyOS41MTI5IDQ0LjQ5NDNDMjkuNzE1IDQ0LjY1ODkgMjkuOTYyNSA0NC44Njk4IDMwLjE0ODMgNDUuMDA3NkMzMS4wMjE3IDQ1LjY1NTUgMzEuODE5NSA0NS45ODcyIDMyLjY4OTcgNDYuNTAxNUMzNC41MjMxIDQ3LjYzOTEgMzYuMDQzIDQ4LjU4MjMgMzcuMjQ4NiA0OS43MTk2QzM3LjcxOTQgNTAuMjIzNyAzNy44MDE2IDUxLjExMjIgMzcuODY0MyA1MS40OTY0TDM4Ljg0NjggNTIuMzc4MkMzMy41ODcyIDYwLjMzMDggMzEuMTUzIDcwLjE1MzkgMzIuNTkxNSA4MC4xNjI3TDMxLjMwNzcgODAuNTM3OEMzMC45NjkzIDgwLjk3NjggMzAuNDkxMiA4MS42Njc2IDI5Ljk5MTEgODEuODczOEMyOC40MTM4IDgyLjM3MjkgMjYuNjM4NyA4Mi41NTYyIDI0LjQ5NTYgODIuNzgxOUMyMy40ODk0IDgyLjg2NiAyMi42MjEzIDgyLjgxNTggMjEuNTU0NiA4My4wMTg4QzIxLjMxOTggODMuMDYzNSAyMC45OTI3IDgzLjE0OTEgMjAuNzM1OCA4My4yMDk3QzIwLjcyNjkgODMuMjExNiAyMC43MTg2IDgzLjIxNDIgMjAuNzA5NiA4My4yMTYyQzIwLjY5NTYgODMuMjE5NSAyMC42NzcyIDgzLjIyNjMgMjAuNjYzOCA4My4yMjk0QzE4Ljg1NyA4My42NjggMTcuNjk2MyA4NS4zMzY1IDE4LjA2OTkgODYuOTgwNUMxOC40NDM3IDg4LjYyNDggMjAuMjA4NiA4OS42MjQ4IDIyLjAyNjIgODkuMjMxMkMyMi4wMzkzIDg5LjIyODIgMjIuMDU4NCA4OS4yMjc3IDIyLjA3MiA4OS4yMjQ2QzIyLjA5MjYgODkuMjE5OSAyMi4xMTA2IDg5LjIwOTkgMjIuMTMxIDg5LjIwNDlDMjIuMzg0NCA4OS4xNDkgMjIuNzAxOSA4OS4wODY4IDIyLjkyMzYgODkuMDI3MkMyMy45NzIzIDg4Ljc0NTEgMjQuNzMxOCA4OC4zMzA2IDI1LjY3NDYgODcuOTY3N0MyNy43MDI5IDg3LjIzNjggMjkuMzgyOCA4Ni42MjYyIDMxLjAxOTUgODYuMzg4M0MzMS43MDMgODYuMzM0NSAzMi40MjMyIDg2LjgxMiAzMi43ODE0IDg3LjAxMzRMMzQuMTE3NyA4Ni43ODMxQzM3LjE5MjYgOTYuMzYxMyA0My42MzY2IDEwNC4xMDMgNTEuNzk2MyAxMDguOTYxTDUxLjIzOTYgMTEwLjMwM0M1MS40NDAzIDExMC44MjQgNTEuNjYxNiAxMTEuNTMgNTEuNTEyMSAxMTIuMDQ1QzUwLjkxNzEgMTEzLjU5NSA0OS44OTggMTE1LjIzMSA0OC43Mzc0IDExNy4wNTVDNDguMTc1NSAxMTcuODk4IDQ3LjYwMDQgMTE4LjU1MiA0Ny4wOTM0IDExOS41MTZDNDYuOTcyIDExOS43NDcgNDYuODE3NSAxMjAuMTAyIDQ2LjcwMDQgMTIwLjM0NkM0NS45MTI1IDEyMi4wMzkgNDYuNDkwNCAxMjMuOTkgNDguMDAzOCAxMjQuNzIyQzQ5LjUyNjggMTI1LjQ1OSA1MS40MTcxIDEyNC42ODIgNTIuMjM1MiAxMjIuOTg1QzUyLjIzNjQgMTIyLjk4MiA1Mi4yNDA2IDEyMi45OCA1Mi4yNDE3IDEyMi45NzhDNTIuMjQyNiAxMjIuOTc2IDUyLjI0MDkgMTIyLjk3MyA1Mi4yNDE3IDEyMi45NzFDNTIuMzU4MiAxMjIuNzMxIDUyLjUyMzMgMTIyLjQxNSA1Mi42MjE2IDEyMi4xODhDNTMuMDU2IDEyMS4xODkgNTMuMjAwNSAxMjAuMzMyIDUzLjUwNTkgMTE5LjM2NUM1NC4zMTcgMTE3LjMxOCA1NC43NjI2IDExNS4xNyA1NS44NzkxIDExMy44MzJDNTYuMTg0OSAxMTMuNDY2IDU2LjY4MzMgMTEzLjMyNSA1Ny4yMDAxIDExMy4xODZMNTcuODk0NCAxMTEuOTIyQzY1LjAwOCAxMTQuNjY1IDcyLjk3MDUgMTE1LjQwMiA4MC45MjQ1IDExMy41ODdDODIuNzM5MSAxMTMuMTczIDg0LjQ5MDggMTEyLjYzNyA4Ni4xODQzIDExMS45OTRDODYuMzc5NCAxMTIuMzQyIDg2Ljc0MiAxMTMuMDExIDg2LjgzOTMgMTEzLjE3OUM4Ny4zNjQ0IDExMy4zNTEgODcuOTM3NyAxMTMuNDM5IDg4LjQwNDcgMTE0LjEzM0M4OS4yNDAxIDExNS41NjcgODkuODExNCAxMTcuMjYzIDkwLjUwNzMgMTE5LjMxMkM5MC44MTI4IDEyMC4yNzkgOTAuOTYzOCAxMjEuMTM2IDkxLjM5ODEgMTIyLjEzNkM5MS40OTcxIDEyMi4zNjMgOTEuNjYxNCAxMjIuNjg0IDkxLjc3OCAxMjIuOTI1QzkyLjU5NDQgMTI0LjYyOCA5NC40OTA3IDEyNS40MDcgOTYuMDE1OSAxMjQuNjY5Qzk3LjUyOTIgMTIzLjkzNyA5OC4xMDc3IDEyMS45ODYgOTcuMzE5NCAxMjAuMjkzQzk3LjIwMjMgMTIwLjA0OSA5Ny4wNDEyIDExOS42OTUgOTYuOTE5OCAxMTkuNDY0Qzk2LjQxMjcgMTE4LjQ5OSA5NS44Mzc3IDExNy44NTIgOTUuMjc1OCAxMTcuMDA5Qzk0LjExNTIgMTE1LjE4NSA5My4xNTI2IDExMy42NyA5Mi41NTc1IDExMi4xMkM5Mi4zMDg3IDExMS4zMiA5Mi41OTk1IDExMC44MjMgOTIuNzkzMyAxMTAuMzAzQzkyLjY3NzIgMTEwLjE3IDkyLjQyODggMTA5LjQxNCA5Mi4yODI0IDEwOS4wNTlDMTAwLjc2MiAxMDQuMDI5IDEwNy4wMTcgOTUuOTk4NSAxMDkuOTU1IDg2LjcyMzlDMTEwLjM1MSA4Ni43ODY1IDExMS4wNDEgODYuOTA5MSAxMTEuMjY1IDg2Ljk1NDJDMTExLjcyNiA4Ni42NDg3IDExMi4xNDkgODYuMjUwMSAxMTIuOTgxIDg2LjMxNTlDMTE0LjYxNyA4Ni41NTM3IDExNi4yOTcgODcuMTY0NSAxMTguMzI2IDg3Ljg5NTNDMTE5LjI2OCA4OC4yNTgxIDEyMC4wMjggODguNjc5MyAxMjEuMDc3IDg4Ljk2MTRDMTIxLjI5OCA4OS4wMjEgMTIxLjYxNiA4OS4wNzY2IDEyMS44NjkgODkuMTMyNUMxMjEuODg5IDg5LjEzNzUgMTIxLjkwOCA4OS4xNDc1IDEyMS45MjggODkuMTUyMkMxMjEuOTQyIDg5LjE1NTMgMTIxLjk2MSA4OS4xNTU4IDEyMS45NzQgODkuMTU4OEMxMjMuNzkyIDg5LjU1MiAxMjUuNTU3IDg4LjU1MjYgMTI1LjkzIDg2LjkwODFDMTI2LjMwMyA4NS4yNjQxIDEyNS4xNDMgODMuNTk1MiAxMjMuMzM2IDgzLjE1N0MxMjMuMDc0IDgzLjA5NyAxMjIuNzAxIDgyLjk5NSAxMjIuNDQ2IDgyLjk0NjVDMTIxLjM3OSA4Mi43NDM1IDEyMC41MTEgODIuNzkzNSAxMTkuNTA1IDgyLjcwOTVDMTE3LjM2MSA4Mi40ODM5IDExNS41ODYgODIuMzAwNCAxMTQuMDA5IDgxLjgwMTRDMTEzLjM2NiA4MS41NTA3IDExMi45MDggODAuNzgxOSAxMTIuNjg2IDgwLjQ2NTVMMTExLjQ0OCA4MC4xMDM1QzExMi4wOSA3NS40MzggMTExLjkxNyA3MC41ODI1IDExMC44MDYgNjUuNzI0M0MxMDkuNjg1IDYwLjgyMDggMTA3LjcwNCA1Ni4zMzYxIDEwNS4wNjIgNTIuMzg0OEMxMDUuMzc5IDUyLjA5NDggMTA1Ljk3OSA1MS41NjEyIDEwNi4xNDkgNTEuNDA0M0MxMDYuMTk5IDUwLjg1MTcgMTA2LjE1NiA1MC4yNzIyIDEwNi43MjUgNDkuNjYwM0MxMDcuOTMxIDQ4LjUyMyAxMDkuNDUxIDQ3LjU3OTkgMTExLjI4NCA0Ni40NDIzQzExMi4xNTQgNDUuOTI3OSAxMTIuOTU5IDQ1LjU5NjQgMTEzLjgzMiA0NC45NDg0QzExNC4wMyA0NC44MDE5IDExNC4yOTkgNDQuNTY5OSAxMTQuNTA3IDQ0LjQwMjJDMTE1Ljk3NyA0My4yMjM3IDExNi4zMTYgNDEuMTkxMSAxMTUuMjYgMzkuODYxNEMxMTQuMjA0IDM4LjUzMTcgMTEyLjE1OSAzOC40MDY1IDExMC42ODggMzkuNTg1QzExMC40NzkgMzkuNzUxNiAxMTAuMTk1IDM5Ljk2ODggMTEwLjAwNyA0MC4xMzEyQzEwOS4xODQgNDAuODQyNiAxMDguNjc2IDQxLjU0NTIgMTA3Ljk4MyA0Mi4yODMyQzEwNi40NzEgNDMuODI1OSAxMDUuMjIyIDQ1LjExMyAxMDMuODUgNDYuMDQwOUMxMDMuMjU1IDQ2LjM4ODUgMTAyLjM4NSA0Ni4yNjgyIDEwMS45OSA0Ni4yNDQ5TDEwMC44MjQgNDcuMDgwNkM5NC4xNzUzIDQwLjA3NjMgODUuMTIzNSAzNS41OTgyIDc1LjM3NjYgMzQuNzI4M0M3NS4zNDk0IDM0LjMxNzkgNzUuMzEzNyAzMy41NzYxIDc1LjMwNDYgMzMuMzUyOUM3NC45MDU2IDMyLjk2OTMgNzQuNDIzNSAzMi42NDE4IDc0LjMwMjQgMzEuODEzQzc0LjE2OTEgMzAuMTU2OSA3NC4zOTE3IDI4LjM3NTIgNzQuNjQ5NiAyNi4yMjU4Qzc0Ljc5MTkgMjUuMjIxNiA3NS4wMjg0IDI0LjM4NzMgNzUuMDY4OCAyMy4yOTczQzc1LjA3OCAyMy4wNDk1IDc1LjA2MzIgMjIuNjkgNzUuMDYyMiAyMi40MjIxQzc1LjA2MiAyMC41MzIyIDczLjY4OTggMTguOTk5OCA3MS45OTY4IDE5Wk02OC4xNTg1IDQyLjg4ODZMNjcuMjQ4IDU5LjA0NDdMNjcuMTgyNSA1OS4wNzc2QzY3LjEyMTQgNjAuNTIyOSA2NS45Mzc1IDYxLjY3NyA2NC40ODM5IDYxLjY3N0M2My44ODg0IDYxLjY3NyA2My4zMzg4IDYxLjQ4NDkgNjIuODkyMiA2MS4xNTcxTDYyLjg2NiA2MS4xNzAzTDQ5LjY4MDcgNTEuNzc5NEM1My43MzMxIDQ3Ljc3NTkgNTguOTE2NCA0NC44MTcyIDY0Ljg5IDQzLjQ1NDZDNjUuOTgxMiA0My4yMDU2IDY3LjA3MTkgNDMuMDIwOSA2OC4xNTg1IDQyLjg4ODZaTTc1Ljg0MTcgNDIuODg4NkM4Mi44MTU5IDQzLjc1MDQgODkuMjY1NyA0Ni45MjMyIDk0LjIwODEgNTEuNzg2TDgxLjEwOCA2MS4xMTc2TDgxLjA2MjEgNjEuMDk3OUM3OS44OTk0IDYxLjk1MTIgNzguMjYxMSA2MS43Mzk0IDc3LjM1NDggNjAuNTk3OEM3Ni45ODM1IDYwLjEzMDEgNzYuNzg4NyA1OS41ODAxIDc2Ljc2NTMgNTkuMDI0OUw3Ni43NTIyIDU5LjAxODRMNzUuODQxNyA0Mi44ODg2Wk00NC44OTkxIDU3LjgxNEw1Ni45MzgyIDY4LjYzM0w1Ni45MjUxIDY4LjY5ODhDNTguMDExNyA2OS42NDc5IDU4LjE3MiA3MS4yOTQ5IDU3LjI2NTcgNzIuNDM2OEM1Ni44OTQ0IDcyLjkwNDUgNTYuMzk3NSA3My4yMTgyIDU1Ljg2MzkgNzMuMzY0N0w1NS44NTA4IDczLjQxNzNMNDAuNDE4OCA3Ny44OTIzQzM5LjYzMzQgNzAuNjc2NSA0MS4zMjYxIDYzLjY2MjEgNDQuODk5MSA1Ny44MTRaTTk5LjAwOTQgNTcuODIwNkMxMDAuNzk4IDYwLjczMzYgMTAyLjE1MyA2My45ODcxIDEwMi45NTkgNjcuNTE0M0MxMDMuNzU2IDcwLjk5OTEgMTAzLjk1NiA3NC40Nzc4IDEwMy42MjcgNzcuODM5N0w4OC4xMTY2IDczLjM1MTVMODguMTAzNSA3My4yODU3Qzg2LjcxNDUgNzIuOTA0MyA4NS44NjA5IDcxLjQ4NDggODYuMTg0MyA3MC4wNjExQzg2LjMxNjggNjkuNDc3OCA4Ni42MjQ5IDY4Ljk4NDQgODcuMDQyMyA2OC42MTk4TDg3LjAzNTggNjguNTg2OUw5OS4wMDk0IDU3LjgyMDZaTTY5LjUyNzQgNjkuNDY4OEg3NC40NTk2TDc3LjUyNTEgNzMuMzE4Nkw3Ni40MjQ3IDc4LjEyMjZMNzEuOTk2OCA4MC4yNjE0TDY3LjU1NTggNzguMTE2MUw2Ni40NTU0IDczLjMxMkw2OS41Mjc0IDY5LjQ2ODhaTTg1LjMzOTMgODIuNjQzN0M4NS41NDg5IDgyLjYzMzEgODUuNzU3NiA4Mi42NTIgODUuOTYxNiA4Mi42ODk4TDg1Ljk4NzggODIuNjU2OUwxMDEuOTUgODUuMzY4MkM5OS42MTQyIDkxLjk2MjQgOTUuMTQ0IDk3LjY3NSA4OS4xNzExIDEwMS40OThMODIuOTc0NyA4Ni40NjA2TDgyLjk5NDQgODYuNDM0M0M4Mi40MjUyIDg1LjEwNTUgODIuOTk0OCA4My41NDcyIDg0LjMwNDQgODIuOTEzNUM4NC42Mzk3IDgyLjc1MTMgODQuOTkgODIuNjYxNCA4NS4zMzkzIDgyLjY0MzdaTTU4LjUyOTggODIuNzA5NUM1OS43NDggODIuNzI2NyA2MC44NDA2IDgzLjU3NjEgNjEuMTIzNyA4NC44MjJDNjEuMjU2MiA4NS40MDUyIDYxLjE5MTcgODUuOTgzMSA2MC45NzMgODYuNDkzNUw2MS4wMTg5IDg2LjU1MjdMNTQuODg4IDEwMS40MzlDNDkuMTU1OSA5Ny43NDMyIDQ0LjU5MDQgOTIuMjA5OSA0Mi4xNDgxIDg1LjQyMDhMNTcuOTczMSA4Mi43MjI3TDU3Ljk5OTMgODIuNzU1NkM1OC4xNzYzIDgyLjcyMjkgNTguMzU1OCA4Mi43MDcxIDU4LjUyOTggODIuNzA5NVpNNzEuODk4NiA4OS4yMzEyQzcyLjMyMjkgODkuMjE1NSA3Mi43NTM0IDg5LjMwMyA3My4xNjI3IDg5LjUwMUM3My42OTkyIDg5Ljc2MDYgNzQuMTEzNiA5MC4xNjkyIDc0LjM3NDUgOTAuNjU5Mkg3NC40MzM0TDgyLjIzNDYgMTA0LjgyMUM4MS4yMjIxIDEwNS4xNjIgODAuMTgxMyAxMDUuNDU0IDc5LjExNjcgMTA1LjY5N0M3My4xNTA1IDEwNy4wNTggNjcuMjAzMiAxMDYuNjQ1IDYxLjgxOCAxMDQuODAyTDY5LjU5OTUgOTAuNjY1OEg2OS42MTI2QzcwLjA3OTUgODkuNzg4OCA3MC45NjUgODkuMjY1NiA3MS44OTg2IDg5LjIzMTJaIiBmaWxsPSJ3aGl0ZSIgc3Ryb2tlPSJ3aGl0ZSIgc3Ryb2tlLXdpZHRoPSIwLjI1Ii8+CjxkZWZzPgo8bGluZWFyR3JhZGllbnQgaWQ9InBhaW50MF9saW5lYXJfNjgxXzI4NDUiIHgxPSIxMCIgeTE9IjE1LjUiIHgyPSIxNDQiIHkyPSIxMzEuNSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiPgo8c3RvcCBzdG9wLWNvbG9yPSIjNEQ4N0ZGIi8+CjxzdG9wIG9mZnNldD0iMSIgc3RvcC1jb2xvcj0iIzA1NDdEMCIvPgo8L2xpbmVhckdyYWRpZW50Pgo8L2RlZnM+Cjwvc3ZnPgo= - keysOrder: [["apiVersion"],["appVersion"],["kind"],["metadata"],["metadata","name"],["spec","storageClass"],["spec","nodeGroups"],["spec","nodeGroups","md0"],["spec","nodeGroups","md0","minReplicas"],["spec","nodeGroups","md0","maxReplicas"],["spec","nodeGroups","md0","instanceType"],["spec","nodeGroups","md0","ephemeralStorage"],["spec","nodeGroups","md0","roles"],["spec","nodeGroups","md0","resources"],["spec","nodeGroups","md0","gpus"],["spec","version"],["spec","host"],["spec","addons"],["spec","addons","certManager"],["spec","addons","certManager","enabled"],["spec","addons","certManager","valuesOverride"],["spec","addons","cilium"],["spec","addons","cilium","valuesOverride"],["spec","addons","gatewayAPI"],["spec","addons","gatewayAPI","enabled"],["spec","addons","ingressNginx"],["spec","addons","ingressNginx","enabled"],["spec","addons","ingressNginx","exposeMethod"],["spec","addons","ingressNginx","hosts"],["spec","addons","ingressNginx","valuesOverride"],["spec","addons","gpuOperator"],["spec","addons","gpuOperator","enabled"],["spec","addons","gpuOperator","valuesOverride"],["spec","addons","fluxcd"],["spec","addons","fluxcd","enabled"],["spec","addons","fluxcd","valuesOverride"],["spec","addons","monitoringAgents"],["spec","addons","monitoringAgents","enabled"],["spec","addons","monitoringAgents","valuesOverride"],["spec","addons","verticalPodAutoscaler"],["spec","addons","verticalPodAutoscaler","valuesOverride"],["spec","addons","velero"],["spec","addons","velero","enabled"],["spec","addons","velero","valuesOverride"],["spec","addons","coredns"],["spec","addons","coredns","valuesOverride"],["spec","controlPlane"],["spec","controlPlane","replicas"],["spec","controlPlane","apiServer"],["spec","controlPlane","apiServer","resources"],["spec","controlPlane","apiServer","resourcesPreset"],["spec","controlPlane","controllerManager"],["spec","controlPlane","controllerManager","resources"],["spec","controlPlane","controllerManager","resourcesPreset"],["spec","controlPlane","scheduler"],["spec","controlPlane","scheduler","resources"],["spec","controlPlane","scheduler","resourcesPreset"],["spec","controlPlane","konnectivity"],["spec","controlPlane","konnectivity","server"],["spec","controlPlane","konnectivity","server","resources"],["spec","controlPlane","konnectivity","server","resourcesPreset"],["spec","addons","hami"],["spec","addons","hami","enabled"],["spec","addons","hami","valuesOverride"],["spec","images"],["spec","images","waitForKubeconfig"]] + keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "nodeGroups"], ["spec", "nodeGroups", "md0"], ["spec", "nodeGroups", "md0", "minReplicas"], ["spec", "nodeGroups", "md0", "maxReplicas"], ["spec", "nodeGroups", "md0", "instanceType"], ["spec", "nodeGroups", "md0", "ephemeralStorage"], ["spec", "nodeGroups", "md0", "roles"], ["spec", "nodeGroups", "md0", "resources"], ["spec", "nodeGroups", "md0", "gpus"], ["spec", "version"], ["spec", "host"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "hami"], ["spec", "addons", "hami", "enabled"], ["spec", "addons", "hami", "valuesOverride"], ["spec", "addons", "fluxcd"], ["spec", "addons", "fluxcd", "enabled"], ["spec", "addons", "fluxcd", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"], ["spec", "images"], ["spec", "images", "waitForKubeconfig"]] secrets: exclude: [] include: From 805b3b17cbc4d89206203bc24b45c1412c29a2c8 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 19:11:26 +0300 Subject: [PATCH 22/27] fix(kubernetes): add CI values to GPU Operator HAMi tests Tests need _namespace.etcd from values-ci.yaml after the merge introduced an etcd-namespace guard on the gpu-operator HelmRelease condition. Signed-off-by: Arsolitt --- packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml index 4fa754fb..44528470 100644 --- a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml +++ b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml @@ -1,6 +1,8 @@ suite: GPU Operator HelmRelease HAMi integration tests templates: - templates/helmreleases/gpu-operator.yaml +values: + - values-ci.yaml tests: - it: should disable devicePlugin when hami is enabled set: From ba8c1a0535d2e0462fb8269df0750ea6f57fdd0b Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Wed, 29 Apr 2026 11:13:12 +0300 Subject: [PATCH 23/27] docs(hami): drop HAMi#173 reference from glibc tracking issues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HAMi#173 was closed as "not planned" and only suggests a typo fix (< 2.3.0 → < 2.30); it does not establish the actual 2.34 boundary. HAMi-core#174 (symbol-level cause) and HAMi#1190 (empirical per-glibc behavior) cover the same ground accurately. Signed-off-by: Arsolitt --- packages/system/hami/README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/system/hami/README.md b/packages/system/hami/README.md index d648364d..aa80d48a 100644 --- a/packages/system/hami/README.md +++ b/packages/system/hami/README.md @@ -38,9 +38,8 @@ Most modern ML/AI base images (CUDA 12.x, PyTorch 2.x, TensorFlow 2.x) use Ubunt **Upstream tracking issues:** -- [HAMi-core#174](https://github.com/Project-HAMi/HAMi-core/issues/174) — `_dl_sym` removal breaks HAMi-core on glibc >= 2.34 -- [HAMi#1190](https://github.com/Project-HAMi/HAMi/issues/1190) — degraded isolation across glibc versions -- [HAMi#173](https://github.com/Project-HAMi/HAMi/issues/173) — documentation incorrectly states glibc < 2.30 (actual boundary is 2.34) +- [HAMi-core#174](https://github.com/Project-HAMi/HAMi-core/issues/174) — `_dl_sym` removal in glibc 2.34 breaks HAMi-core's CUDA symbol resolution at the symbol level +- [HAMi#1190](https://github.com/Project-HAMi/HAMi/issues/1190) — maintainer thread confirming the empirical per-glibc-version isolation behavior shown in the table above ### musl libc (Alpine) incompatibility From 9876acec2fafbadeb09741f70e01250171a42a48 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Wed, 29 Apr 2026 11:13:24 +0300 Subject: [PATCH 24/27] docs(hami): clarify gpu-operator devicePlugin override behavior The HAMi-driven default of disabling gpu-operator's device plugin is applied via valuesOverride and can be re-enabled by users for advanced topologies (mixed HAMi / vanilla NVIDIA device plugin pools). Document this explicitly so the README aligns with the existing template merge order. Signed-off-by: Arsolitt --- packages/system/hami/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/system/hami/README.md b/packages/system/hami/README.md index aa80d48a..68669ac7 100644 --- a/packages/system/hami/README.md +++ b/packages/system/hami/README.md @@ -57,7 +57,7 @@ addons: enabled: true ``` -When HAMi is enabled, GPU Operator's built-in device plugin is automatically disabled to avoid conflicts. +When HAMi is enabled, GPU Operator's built-in device plugin is automatically disabled to avoid conflicts. This default is preserved by setting `addons.gpuOperator.valuesOverride.gpu-operator.devicePlugin.enabled: false`; advanced topologies that partition GPU pools (e.g. some nodes use HAMi while others run the standard NVIDIA device plugin via node selectors) can re-enable it explicitly through `valuesOverride`. ### Requesting fractional GPU resources From d5caffcc8de0177b248899b03391c312d20dde54 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Wed, 29 Apr 2026 11:13:42 +0300 Subject: [PATCH 25/27] fix(kubernetes): gate HAMi HelmRelease on _namespace.etcd Match the pattern used by every other tenant-cluster addon HelmRelease (gpu-operator, cilium, fluxcd, ...): only render when the upstream etcd namespace is ready, not just when the addon is enabled. Add values-ci.yaml to the HAMi test suite so the new gate has the fixture it needs. Signed-off-by: Arsolitt --- packages/apps/kubernetes/templates/helmreleases/hami.yaml | 2 +- packages/apps/kubernetes/tests/hami_test.yaml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/apps/kubernetes/templates/helmreleases/hami.yaml b/packages/apps/kubernetes/templates/helmreleases/hami.yaml index 1f7a5358..f1538c7c 100644 --- a/packages/apps/kubernetes/templates/helmreleases/hami.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/hami.yaml @@ -1,4 +1,4 @@ -{{- if .Values.addons.hami.enabled }} +{{- if and .Values.addons.hami.enabled .Values._namespace.etcd }} {{- if not .Values.addons.gpuOperator.enabled }} {{- fail "addons.hami requires addons.gpuOperator to be enabled" }} {{- end }} diff --git a/packages/apps/kubernetes/tests/hami_test.yaml b/packages/apps/kubernetes/tests/hami_test.yaml index bdbe990f..27f7c75b 100644 --- a/packages/apps/kubernetes/tests/hami_test.yaml +++ b/packages/apps/kubernetes/tests/hami_test.yaml @@ -1,6 +1,8 @@ suite: HAMi HelmRelease tests templates: - templates/helmreleases/hami.yaml +values: + - values-ci.yaml tests: - it: should not render when hami is disabled set: From 9b5848ed268bfe3bc04b42bae38cc724c00a3bf0 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Wed, 29 Apr 2026 11:13:55 +0300 Subject: [PATCH 26/27] fix(platform): register HAMi as kubernetes-application component MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The HelmRelease at packages/apps/kubernetes/templates/helmreleases/hami.yaml references chartRef.name 'cozystack-kubernetes-application-kubevirt-kubernetes-hami', but that component was missing from the kubernetes-application PackageSource. The HelmRelease would sit in Stalled: ArtifactNotFound at install time. Add kubernetes-hami next to kubernetes-gpu-operator under variant kubevirt, mirroring the existing pattern for every other tenant-cluster addon. The standalone cozystack.hami PackageSource is retained — same shape as gpu-operator, which is registered both standalone (for bundles/iaas) and as a kubernetes-application component (for tenant-cluster HelmReleases). Signed-off-by: Arsolitt --- packages/core/platform/sources/kubernetes-application.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/core/platform/sources/kubernetes-application.yaml b/packages/core/platform/sources/kubernetes-application.yaml index 9787cb19..088383ad 100644 --- a/packages/core/platform/sources/kubernetes-application.yaml +++ b/packages/core/platform/sources/kubernetes-application.yaml @@ -52,6 +52,8 @@ spec: path: system/cilium - name: kubernetes-gpu-operator path: system/gpu-operator + - name: kubernetes-hami + path: system/hami - name: kubernetes-vertical-pod-autoscaler path: system/vertical-pod-autoscaler - name: kubernetes-prometheus-operator-crds From 5d58d0f340f7157370305cc469317f4fa21944af Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Wed, 29 Apr 2026 11:14:41 +0300 Subject: [PATCH 27/27] chore(hami): document and partially automate vendor patches in Makefile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `update:` recipe now reproduces the top-level vendoring overrides (remove broken hami-dra subchart, clear Chart.yaml dependencies, strip dra/hami-dra/podSecurityPolicy from upstream values.yaml) after `helm pull`, so they no longer silently disappear on the next bump. Template-level patches (DRA guards in scheduler/*, indent fix in device-plugin/monitorservice.yaml) are documented with rationale and commit references — they remain a manual step because automating them would be fragile across upstream template restructures. Signed-off-by: Arsolitt --- packages/system/hami/Makefile | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/packages/system/hami/Makefile b/packages/system/hami/Makefile index b7a169c2..83663a66 100644 --- a/packages/system/hami/Makefile +++ b/packages/system/hami/Makefile @@ -4,8 +4,40 @@ export NAMESPACE=cozy-$(NAME) include ../../../hack/common-envs.mk include ../../../hack/package.mk +# When bumping the HAMi version, run `make update` and then review +# the resulting diff in `charts/hami/`. The recipe below reproduces the +# top-level vendoring overrides automatically: +# +# 1. Removes the broken hami-dra subchart. Upstream's NVIDIA DRA driver +# path requires kubelet DRA support that cozystack does not enable +# and has no upstream fix tracked. See commit 3c5521e. +# 2. Empties Chart.yaml dependencies and drops Chart.lock so Helm does +# not try to re-pull hami-dra at build time. See commit 2734dc0. +# 3. Strips dra/hami-dra/podSecurityPolicy blocks from the upstream +# values.yaml since the corresponding code paths are gone. PSP is +# removed from Kubernetes 1.25+ and is unused by cozystack. +# +# Template-level patches are NOT reproduced automatically: +# +# * Scheduler templates have `{{- if .Values.dra.enabled }}` blocks +# that need to be removed because the dra value is gone (commit +# 2734dc0 stripped them). +# * device-plugin/monitorservice.yaml uses `indent` with leading +# whitespace; it must be rewritten to `nindent` for the labels block +# to render correctly when devicePlugin.service.labels is set +# (commit 3685254). +# +# After `make update`, run `git diff -- charts/hami/templates/` and +# replay these template patches against the new upstream version, then +# verify with `helm unittest`. If upstream restructured the affected +# files, the patches may need to be redesigned rather than reapplied. + update: rm -rf charts helm repo add hami-charts https://project-hami.github.io/HAMi/ helm repo update hami-charts helm pull hami-charts/hami --untar --untardir charts + rm -rf charts/hami/charts/hami-dra + yq --inplace '.dependencies = []' charts/hami/Chart.yaml + rm -f charts/hami/Chart.lock + yq --inplace 'del(.dra) | del(.["hami-dra"]) | del(.podSecurityPolicy)' charts/hami/values.yaml