diff --git a/api/apps/v1alpha1/kubernetes/types.go b/api/apps/v1alpha1/kubernetes/types.go index ea4a2b64..88a3c97e 100644 --- a/api/apps/v1alpha1/kubernetes/types.go +++ b/api/apps/v1alpha1/kubernetes/types.go @@ -69,6 +69,9 @@ type Addons struct { // NVIDIA GPU Operator. // +kubebuilder:default:={} GpuOperator GPUOperatorAddon `json:"gpuOperator"` + // HAMi GPU virtualization middleware. + // +kubebuilder:default:={} + Hami HAMiAddon `json:"hami"` // Ingress-NGINX controller. // +kubebuilder:default:={} IngressNginx IngressNginxAddon `json:"ingressNginx"` @@ -160,6 +163,15 @@ type GatewayAPIAddon struct { Enabled bool `json:"enabled"` } +type HAMiAddon struct { + // Enable HAMi (requires GPU Operator). + // +kubebuilder:default:=false + Enabled bool `json:"enabled"` + // Custom Helm values overrides. + // +kubebuilder:default:={} + ValuesOverride k8sRuntime.RawExtension `json:"valuesOverride"` +} + type Images struct { // Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag. // +kubebuilder:default:="" diff --git a/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go b/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go index 2cbe3ba1..e021fbaa 100644 --- a/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go +++ b/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go @@ -49,6 +49,7 @@ func (in *Addons) DeepCopyInto(out *Addons) { in.Fluxcd.DeepCopyInto(&out.Fluxcd) out.GatewayAPI = in.GatewayAPI in.GpuOperator.DeepCopyInto(&out.GpuOperator) + in.Hami.DeepCopyInto(&out.Hami) in.IngressNginx.DeepCopyInto(&out.IngressNginx) in.MonitoringAgents.DeepCopyInto(&out.MonitoringAgents) in.Velero.DeepCopyInto(&out.Velero) @@ -135,6 +136,7 @@ func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) { } in.Addons.DeepCopyInto(&out.Addons) in.ControlPlane.DeepCopyInto(&out.ControlPlane) + out.Images = in.Images } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigSpec. @@ -260,6 +262,37 @@ func (in *GatewayAPIAddon) DeepCopy() *GatewayAPIAddon { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HAMiAddon) DeepCopyInto(out *HAMiAddon) { + *out = *in + in.ValuesOverride.DeepCopyInto(&out.ValuesOverride) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HAMiAddon. +func (in *HAMiAddon) DeepCopy() *HAMiAddon { + if in == nil { + return nil + } + out := new(HAMiAddon) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Images) DeepCopyInto(out *Images) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Images. +func (in *Images) DeepCopy() *Images { + if in == nil { + return nil + } + out := new(Images) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IngressNginxAddon) DeepCopyInto(out *IngressNginxAddon) { *out = *in diff --git a/packages/apps/kubernetes/README.md b/packages/apps/kubernetes/README.md index ecbbc31b..bed1117a 100644 --- a/packages/apps/kubernetes/README.md +++ b/packages/apps/kubernetes/README.md @@ -128,6 +128,9 @@ See the reference for components utilized in this service: | `addons.gpuOperator` | NVIDIA GPU Operator. | `object` | `{}` | | `addons.gpuOperator.enabled` | Enable GPU Operator. | `bool` | `false` | | `addons.gpuOperator.valuesOverride` | Custom Helm values overrides. | `object` | `{}` | +| `addons.hami` | HAMi GPU virtualization middleware. | `object` | `{}` | +| `addons.hami.enabled` | Enable HAMi (requires GPU Operator). | `bool` | `false` | +| `addons.hami.valuesOverride` | Custom Helm values overrides. | `object` | `{}` | | `addons.fluxcd` | FluxCD GitOps operator. | `object` | `{}` | | `addons.fluxcd.enabled` | Enable FluxCD. | `bool` | `false` | | `addons.fluxcd.valuesOverride` | Custom Helm values overrides. | `object` | `{}` | diff --git a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml index e243bfad..655dc868 100644 --- a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml @@ -1,3 +1,11 @@ +{{- define "cozystack.defaultGpuOperatorValues" -}} +{{- if .Values.addons.hami.enabled }} +gpu-operator: + devicePlugin: + enabled: false +{{- end }} +{{- end }} + {{- if and .Values.addons.gpuOperator.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease @@ -29,9 +37,12 @@ spec: force: true remediation: retries: -1 - {{- with .Values.addons.gpuOperator.valuesOverride }} + {{- $defaults := fromYaml (include "cozystack.defaultGpuOperatorValues" .) }} + {{- $overrides := deepCopy (default (dict) .Values.addons.gpuOperator.valuesOverride) }} + {{- $merged := mergeOverwrite (default (dict) $defaults) $overrides }} + {{- if $merged }} values: - {{- toYaml . | nindent 4 }} + {{- toYaml $merged | nindent 4 }} {{- end }} dependsOn: diff --git a/packages/apps/kubernetes/templates/helmreleases/hami.yaml b/packages/apps/kubernetes/templates/helmreleases/hami.yaml new file mode 100644 index 00000000..f1538c7c --- /dev/null +++ b/packages/apps/kubernetes/templates/helmreleases/hami.yaml @@ -0,0 +1,49 @@ +{{- if and .Values.addons.hami.enabled .Values._namespace.etcd }} +{{- if not .Values.addons.gpuOperator.enabled }} +{{- fail "addons.hami requires addons.gpuOperator to be enabled" }} +{{- end }} +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: {{ .Release.Name }}-hami + labels: + cozystack.io/repository: system + cozystack.io/target-cluster-name: {{ .Release.Name }} + sharding.fluxcd.io/key: tenants +spec: + releaseName: hami + chartRef: + kind: ExternalArtifact + name: cozystack-kubernetes-application-kubevirt-kubernetes-hami + namespace: cozy-system + kubeConfig: + secretRef: + name: {{ .Release.Name }}-admin-kubeconfig + key: super-admin.svc + targetNamespace: cozy-hami + storageNamespace: cozy-hami + interval: 5m + timeout: 10m + install: + createNamespace: true + remediation: + retries: -1 + upgrade: + force: true + remediation: + retries: -1 + {{- with .Values.addons.hami.valuesOverride }} + values: + {{- toYaml . | nindent 4 }} + {{- end }} + + dependsOn: + {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }} + - name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} + {{- end }} + - name: {{ .Release.Name }}-cilium + namespace: {{ .Release.Namespace }} + - name: {{ .Release.Name }}-gpu-operator + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml new file mode 100644 index 00000000..44528470 --- /dev/null +++ b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml @@ -0,0 +1,99 @@ +suite: GPU Operator HelmRelease HAMi integration tests +templates: + - templates/helmreleases/gpu-operator.yaml +values: + - values-ci.yaml +tests: + - it: should disable devicePlugin when hami is enabled + set: + addons: + gpuOperator: + enabled: true + valuesOverride: {} + hami: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.values.gpu-operator.devicePlugin.enabled + value: false + + - it: should not have values when hami is disabled and no overrides + set: + addons: + gpuOperator: + enabled: true + valuesOverride: {} + hami: + enabled: false + valuesOverride: {} + asserts: + - notExists: + path: spec.values + + - it: should apply hami defaults when valuesOverride key is omitted + set: + addons: + gpuOperator: + enabled: true + hami: + enabled: true + asserts: + - equal: + path: spec.values.gpu-operator.devicePlugin.enabled + value: false + + - it: should allow user overrides to merge with hami defaults + set: + addons: + gpuOperator: + enabled: true + valuesOverride: + gpu-operator: + driver: + enabled: false + hami: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.values.gpu-operator.devicePlugin.enabled + value: false + - equal: + path: spec.values.gpu-operator.driver.enabled + value: false + + - it: should let user explicitly override devicePlugin.enabled to true with hami enabled + set: + addons: + gpuOperator: + enabled: true + valuesOverride: + gpu-operator: + devicePlugin: + enabled: true + driver: + enabled: false + hami: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.values.gpu-operator.devicePlugin.enabled + value: true + - equal: + path: spec.values.gpu-operator.driver.enabled + value: false + + - it: should not render when gpuOperator is disabled + set: + addons: + gpuOperator: + enabled: false + valuesOverride: {} + hami: + enabled: false + valuesOverride: {} + asserts: + - hasDocuments: + count: 0 diff --git a/packages/apps/kubernetes/tests/hami_test.yaml b/packages/apps/kubernetes/tests/hami_test.yaml new file mode 100644 index 00000000..27f7c75b --- /dev/null +++ b/packages/apps/kubernetes/tests/hami_test.yaml @@ -0,0 +1,153 @@ +suite: HAMi HelmRelease tests +templates: + - templates/helmreleases/hami.yaml +values: + - values-ci.yaml +tests: + - it: should not render when hami is disabled + set: + addons: + hami: + enabled: false + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - hasDocuments: + count: 0 + + - it: should render HelmRelease when hami is enabled + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - hasDocuments: + count: 1 + - isKind: + of: HelmRelease + + - it: should fail when gpuOperator is not enabled + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: false + valuesOverride: {} + asserts: + - failedTemplate: + errorMessage: "addons.hami requires addons.gpuOperator to be enabled" + + - it: should have correct metadata labels + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: metadata.labels["cozystack.io/repository"] + value: system + - equal: + path: metadata.labels["sharding.fluxcd.io/key"] + value: tenants + + - it: should use ExternalArtifact chartRef + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.chartRef.kind + value: ExternalArtifact + - equal: + path: spec.chartRef.namespace + value: cozy-system + + - it: should target cozy-hami namespace + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.targetNamespace + value: cozy-hami + - equal: + path: spec.storageNamespace + value: cozy-hami + + - it: should depend on gpu-operator and cilium + release: + name: test + namespace: test-ns + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - contains: + path: spec.dependsOn + content: + name: test-cilium + namespace: test-ns + - contains: + path: spec.dependsOn + content: + name: test-gpu-operator + namespace: test-ns + + - it: should not render spec.values when valuesOverride is empty + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - hasDocuments: + count: 1 + - notExists: + path: spec.values + + - it: should pass through valuesOverride + set: + addons: + hami: + enabled: true + valuesOverride: + hami: + devicePlugin: + deviceSplitCount: 5 + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.values.hami.devicePlugin.deviceSplitCount + value: 5 diff --git a/packages/apps/kubernetes/values.schema.json b/packages/apps/kubernetes/values.schema.json index 81e9e2b8..a43deaec 100644 --- a/packages/apps/kubernetes/values.schema.json +++ b/packages/apps/kubernetes/values.schema.json @@ -149,6 +149,7 @@ "fluxcd", "gatewayAPI", "gpuOperator", + "hami", "ingressNginx", "monitoringAgents", "velero", @@ -268,6 +269,28 @@ } } }, + "hami": { + "description": "HAMi GPU virtualization middleware.", + "type": "object", + "default": {}, + "required": [ + "enabled", + "valuesOverride" + ], + "properties": { + "enabled": { + "description": "Enable HAMi (requires GPU Operator).", + "type": "boolean", + "default": false + }, + "valuesOverride": { + "description": "Custom Helm values overrides.", + "type": "object", + "default": {}, + "x-kubernetes-preserve-unknown-fields": true + } + } + }, "ingressNginx": { "description": "Ingress-NGINX controller.", "type": "object", diff --git a/packages/apps/kubernetes/values.yaml b/packages/apps/kubernetes/values.yaml index 68ba07b0..d609476c 100644 --- a/packages/apps/kubernetes/values.yaml +++ b/packages/apps/kubernetes/values.yaml @@ -94,6 +94,10 @@ host: "" ## @field {bool} enabled - Enable FluxCD. ## @field {object} valuesOverride - Custom Helm values overrides. +## @typedef {struct} HAMiAddon - HAMi GPU virtualization middleware. +## @field {bool} enabled - Enable HAMi (requires GPU Operator). +## @field {object} valuesOverride - Custom Helm values overrides. + ## @typedef {struct} MonitoringAgentsAddon - Monitoring agents (Fluent Bit, VMAgents). ## @field {bool} enabled - Enable monitoring agents. ## @field {object} valuesOverride - Custom Helm values overrides. @@ -114,6 +118,7 @@ host: "" ## @field {GatewayAPIAddon} gatewayAPI - Gateway API addon. ## @field {IngressNginxAddon} ingressNginx - Ingress-NGINX controller. ## @field {GPUOperatorAddon} gpuOperator - NVIDIA GPU Operator. +## @field {HAMiAddon} hami - HAMi GPU virtualization middleware. ## @field {FluxCDAddon} fluxcd - FluxCD GitOps operator. ## @field {MonitoringAgentsAddon} monitoringAgents - Monitoring agents. ## @field {VerticalPodAutoscalerAddon} verticalPodAutoscaler - Vertical Pod Autoscaler. @@ -137,6 +142,9 @@ addons: gpuOperator: enabled: false valuesOverride: {} + hami: + enabled: false + valuesOverride: {} fluxcd: enabled: false valuesOverride: {} diff --git a/packages/core/platform/sources/hami.yaml b/packages/core/platform/sources/hami.yaml new file mode 100644 index 00000000..0184e405 --- /dev/null +++ b/packages/core/platform/sources/hami.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: cozystack.io/v1alpha1 +kind: PackageSource +metadata: + name: cozystack.hami +spec: + sourceRef: + kind: OCIRepository + name: cozystack-packages + namespace: cozy-system + path: / + variants: + - name: default + dependsOn: + - cozystack.gpu-operator + components: + - name: hami + path: system/hami + valuesFiles: + - values.yaml + install: + privileged: true + namespace: cozy-hami + releaseName: hami diff --git a/packages/core/platform/sources/kubernetes-application.yaml b/packages/core/platform/sources/kubernetes-application.yaml index 9787cb19..088383ad 100644 --- a/packages/core/platform/sources/kubernetes-application.yaml +++ b/packages/core/platform/sources/kubernetes-application.yaml @@ -52,6 +52,8 @@ spec: path: system/cilium - name: kubernetes-gpu-operator path: system/gpu-operator + - name: kubernetes-hami + path: system/hami - name: kubernetes-vertical-pod-autoscaler path: system/vertical-pod-autoscaler - name: kubernetes-prometheus-operator-crds diff --git a/packages/core/platform/templates/bundles/iaas.yaml b/packages/core/platform/templates/bundles/iaas.yaml index ced0a322..66ec98b2 100644 --- a/packages/core/platform/templates/bundles/iaas.yaml +++ b/packages/core/platform/templates/bundles/iaas.yaml @@ -10,6 +10,7 @@ {{include "cozystack.platform.package.default" (list "cozystack.kubevirt-cdi" $) }} {{include "cozystack.platform.package.optional.default" (list "cozystack.vm-default-images" $) }} {{include "cozystack.platform.package.optional.default" (list "cozystack.gpu-operator" $) }} +{{include "cozystack.platform.package.optional.default" (list "cozystack.hami" $) }} {{include "cozystack.platform.package.default" (list "cozystack.kamaji" $) }} {{include "cozystack.platform.package.default" (list "cozystack.capi-operator" $) }} {{include "cozystack.platform.package.default" (list "cozystack.capi-provider-bootstrap-kubeadm" $) }} diff --git a/packages/system/hami/Chart.yaml b/packages/system/hami/Chart.yaml new file mode 100644 index 00000000..3fcf4c5d --- /dev/null +++ b/packages/system/hami/Chart.yaml @@ -0,0 +1,3 @@ +apiVersion: v2 +name: cozy-hami +version: 0.0.0 # Placeholder, the actual version will be automatically set during the build process diff --git a/packages/system/hami/Makefile b/packages/system/hami/Makefile new file mode 100644 index 00000000..83663a66 --- /dev/null +++ b/packages/system/hami/Makefile @@ -0,0 +1,43 @@ +export NAME=hami +export NAMESPACE=cozy-$(NAME) + +include ../../../hack/common-envs.mk +include ../../../hack/package.mk + +# When bumping the HAMi version, run `make update` and then review +# the resulting diff in `charts/hami/`. The recipe below reproduces the +# top-level vendoring overrides automatically: +# +# 1. Removes the broken hami-dra subchart. Upstream's NVIDIA DRA driver +# path requires kubelet DRA support that cozystack does not enable +# and has no upstream fix tracked. See commit 3c5521e. +# 2. Empties Chart.yaml dependencies and drops Chart.lock so Helm does +# not try to re-pull hami-dra at build time. See commit 2734dc0. +# 3. Strips dra/hami-dra/podSecurityPolicy blocks from the upstream +# values.yaml since the corresponding code paths are gone. PSP is +# removed from Kubernetes 1.25+ and is unused by cozystack. +# +# Template-level patches are NOT reproduced automatically: +# +# * Scheduler templates have `{{- if .Values.dra.enabled }}` blocks +# that need to be removed because the dra value is gone (commit +# 2734dc0 stripped them). +# * device-plugin/monitorservice.yaml uses `indent` with leading +# whitespace; it must be rewritten to `nindent` for the labels block +# to render correctly when devicePlugin.service.labels is set +# (commit 3685254). +# +# After `make update`, run `git diff -- charts/hami/templates/` and +# replay these template patches against the new upstream version, then +# verify with `helm unittest`. If upstream restructured the affected +# files, the patches may need to be redesigned rather than reapplied. + +update: + rm -rf charts + helm repo add hami-charts https://project-hami.github.io/HAMi/ + helm repo update hami-charts + helm pull hami-charts/hami --untar --untardir charts + rm -rf charts/hami/charts/hami-dra + yq --inplace '.dependencies = []' charts/hami/Chart.yaml + rm -f charts/hami/Chart.lock + yq --inplace 'del(.dra) | del(.["hami-dra"]) | del(.podSecurityPolicy)' charts/hami/values.yaml diff --git a/packages/system/hami/README.md b/packages/system/hami/README.md new file mode 100644 index 00000000..68669ac7 --- /dev/null +++ b/packages/system/hami/README.md @@ -0,0 +1,82 @@ +# HAMi — GPU Virtualization Middleware + +[HAMi](https://github.com/Project-HAMi/HAMi) (Heterogeneous AI Computing Virtualization Middleware) is a CNCF Sandbox project that enables fractional GPU sharing in Kubernetes. It allows workloads to request specific amounts of GPU memory and compute cores instead of claiming entire GPUs. + +## Architecture + +HAMi consists of four components: + +- **MutatingWebhook** — intercepts pod creation, injects `schedulerName: hami-scheduler` +- **Scheduler Extender** — extends kube-scheduler with GPU-aware Filter and Bind logic +- **Device Plugin** (DaemonSet) — registers vGPU resources via the Kubernetes Device Plugin API +- **HAMi-core** (`libvgpu.so`) — `LD_PRELOAD` library injected into workload containers, intercepts CUDA API calls to enforce memory and compute isolation + +## Prerequisites + +- GPU Operator must be enabled (`addons.gpuOperator.enabled: true`) +- NVIDIA driver >= 440 on host nodes +- nvidia-container-toolkit configured as the default container runtime +- GPU nodes labeled with `gpu=on` + +## Known Limitations + +### glibc < 2.34 requirement for workload containers + +HAMi-core uses `LD_PRELOAD` to intercept `dlsym()` for CUDA symbol resolution. The fallback code path relies on `_dl_sym`, a private glibc internal symbol that was removed in glibc 2.34 when libdl and libpthread were merged into libc.so. + +**This limitation affects workload containers only**, not the host OS or HAMi's own components. + +| Distribution | glibc | Result | +| --------------- | ----- | -------------------------------------------- | +| Ubuntu 18.04 | 2.27 | Full isolation (memory + compute) | +| Ubuntu 20.04 | 2.31 | Full isolation (memory + compute) | +| Ubuntu 22.04 | 2.35 | Memory isolation works, compute breaks | +| Ubuntu 24.04 | 2.39 | Both memory and compute isolation break | +| Alpine (musl) | N/A | Completely incompatible (`dlvsym` absent) | + +Most modern ML/AI base images (CUDA 12.x, PyTorch 2.x, TensorFlow 2.x) use Ubuntu 22.04+ with glibc >= 2.35, which means compute isolation will not work with these images until the upstream fix is merged. + +**Upstream tracking issues:** + +- [HAMi-core#174](https://github.com/Project-HAMi/HAMi-core/issues/174) — `_dl_sym` removal in glibc 2.34 breaks HAMi-core's CUDA symbol resolution at the symbol level +- [HAMi#1190](https://github.com/Project-HAMi/HAMi/issues/1190) — maintainer thread confirming the empirical per-glibc-version isolation behavior shown in the table above + +### musl libc (Alpine) incompatibility + +HAMi-core is completely incompatible with musl libc. The `dlvsym()` function used by HAMi-core is a glibc extension not available in musl. Only glibc-based container images (Debian, Ubuntu, RHEL, etc.) can use HAMi GPU isolation. + +## Usage + +Enable HAMi in your tenant Kubernetes cluster values: + +```yaml +addons: + gpuOperator: + enabled: true + hami: + enabled: true +``` + +When HAMi is enabled, GPU Operator's built-in device plugin is automatically disabled to avoid conflicts. This default is preserved by setting `addons.gpuOperator.valuesOverride.gpu-operator.devicePlugin.enabled: false`; advanced topologies that partition GPU pools (e.g. some nodes use HAMi while others run the standard NVIDIA device plugin via node selectors) can re-enable it explicitly through `valuesOverride`. + +### Requesting fractional GPU resources + +```yaml +resources: + limits: + nvidia.com/gpu: 1 + nvidia.com/gpumem: 3000 # 3000 MB of GPU memory + nvidia.com/gpucores: 30 # 30% of GPU compute cores +``` + +## Parameters + +Default values shown below are inherited from the upstream HAMi chart and may change with upstream updates. + +| Name | Description | Default | +| --- | --- | --- | +| `hami.devicePlugin.runtimeClassName` | RuntimeClass for device plugin pods | `nvidia` | +| `hami.devicePlugin.deviceSplitCount` | Max virtual GPUs per physical GPU | `10` | +| `hami.devicePlugin.deviceMemoryScaling` | Memory overcommit factor (> 1.0 enables overcommit) | `1` | +| `hami.scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy` | Node packing strategy (`binpack` or `spread`) | `binpack` | +| `hami.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy` | GPU packing strategy (`binpack` or `spread`) | `spread` | diff --git a/packages/system/hami/charts/hami/Chart.yaml b/packages/system/hami/charts/hami/Chart.yaml new file mode 100644 index 00000000..55f32ab6 --- /dev/null +++ b/packages/system/hami/charts/hami/Chart.yaml @@ -0,0 +1,18 @@ +apiVersion: v2 +appVersion: 2.8.1 +dependencies: [] +description: Heterogeneous AI Computing Virtualization Middleware +keywords: +- vgpu +- gpu +kubeVersion: '>= 1.18.0-0' +maintainers: +- email: archlitchi@gmail.com + name: limengxuan +- email: xiaozhang0210@hotmail.com + name: zhangxiao +name: hami +sources: +- https://github.com/Project-HAMi/HAMi +type: application +version: 2.8.1 diff --git a/packages/system/hami/charts/hami/README.md b/packages/system/hami/charts/hami/README.md new file mode 100644 index 00000000..e8217290 --- /dev/null +++ b/packages/system/hami/charts/hami/README.md @@ -0,0 +1,237 @@ +# HAMi Helm Chart Values Documentation + +This document provides detailed descriptions of all configurable values parameters for the HAMi Helm Chart. + +## Global Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `global.imageRegistry` | Global Docker image registry | `""` | +| `global.imagePullSecrets` | Global Docker image pull secrets | `[]` | +| `global.imageTag` | Image tag | `"v2.8.1"` | +| `global.gpuHookPath` | GPU Hook path | `/usr/local` | +| `global.labels` | Global labels | `{}` | +| `global.annotations` | Global annotations | `{}` | +| `global.managedNodeSelectorEnable` | Whether to enable managed node selector | `false` | +| `global.managedNodeSelector.usage` | Managed node selector usage | `"gpu"` | +| `nameOverride` | Name override | `""` | +| `fullnameOverride` | Full name override | `""` | +| `namespaceOverride` | Namespace override | `""` | + +## Resource Name Configuration + +### NVIDIA GPU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `resourceName` | GPU resource name | `"nvidia.com/gpu"` | +| `resourceMem` | GPU memory resource name | `"nvidia.com/gpumem"` | +| `resourceMemPercentage` | GPU memory percentage resource name | `"nvidia.com/gpumem-percentage"` | +| `resourceCores` | GPU core resource name | `"nvidia.com/gpucores"` | +| `resourcePriority` | GPU priority resource name | `"nvidia.com/priority"` | + +### Cambricon MLU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `mluResourceName` | MLU resource name | `"cambricon.com/vmlu"` | +| `mluResourceMem` | MLU memory resource name | `"cambricon.com/mlu.smlu.vmemory"` | +| `mluResourceCores` | MLU core resource name | `"cambricon.com/mlu.smlu.vcore"` | + +### Hygon DCU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `dcuResourceName` | DCU resource name | `"hygon.com/dcunum"` | +| `dcuResourceMem` | DCU memory resource name | `"hygon.com/dcumem"` | +| `dcuResourceCores` | DCU core resource name | `"hygon.com/dcucores"` | + +### Metax GPU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `metaxResourceName` | GPU resource name | `"metax-tech.com/sgpu"` | +| `metaxResourceCore` | GPU core resource name | `"metax-tech.com/vcore"` | +| `metaxResourceMem` | GPU memory resource name | `"metax-tech.com/vmemory"` | +| `metaxsGPUTopologyAware` | GPU topology awareness | `"false"` | + +### Enflame GCU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `enflameResourceNameVGCU` | vGCU resource name | `"enflame.com/vgcu"` | +| `enflameResourceNameVGCUPercentage` | vGCU percentage resource name | `"enflame.com/vgcu-percentage"` | + +### Kunlunxin XPU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `kunlunResourceName` | XPU resource name | `"kunlunxin.com/xpu"` | + +## Scheduler Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `schedulerName` | Scheduler name | `"hami-scheduler"` | +| `scheduler.nodeName` | Define node name, scheduler will schedule to this node | `""` | +| `scheduler.overwriteEnv` | Whether to overwrite environment variables | `"false"` | +| `scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy` | Node scheduler policy | `binpack` | +| `scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy` | GPU scheduler policy | `spread` | +| `scheduler.metricsBindAddress` | Metrics bind address | `":9395"` | +| `scheduler.forceOverwriteDefaultScheduler` | Whether to force overwrite default scheduler | `true` | +| `scheduler.livenessProbe` | Whether to enable liveness probe | `false` | +| `scheduler.leaderElect` | Whether to enable leader election | `true` | +| `scheduler.replicas` | Number of replicas | `1` | + +### Kube Scheduler Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `scheduler.kubeScheduler.enabled` | Whether to run kube-scheduler container in scheduler pod | `true` | +| `scheduler.kubeScheduler.image.registry` | Kube scheduler image registry | `"registry.cn-hangzhou.aliyuncs.com"` | +| `scheduler.kubeScheduler.image.repository` | Kube scheduler image repository | `"google_containers/kube-scheduler"` | +| `scheduler.kubeScheduler.image.tag` | Kube scheduler image tag | `""` | +| `scheduler.kubeScheduler.image.pullPolicy` | Kube scheduler image pull policy | `IfNotPresent` | +| `scheduler.kubeScheduler.image.pullSecrets` | Kube scheduler image pull secrets | `[]` | +| `scheduler.kubeScheduler.extraNewArgs` | Extra new arguments | `["--config=/config/config.yaml", "-v=4"]` | +| `scheduler.kubeScheduler.extraArgs` | Extra arguments | `["--policy-config-file=/config/config.json", "-v=4"]` | + +### Extender Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `scheduler.extender.image.registry` | Scheduler extender image registry | `"docker.io"` | +| `scheduler.extender.image.repository` | Scheduler extender image repository | `"projecthami/hami"` | +| `scheduler.extender.image.tag` | Scheduler extender image tag | `""` | +| `scheduler.extender.image.pullPolicy` | Scheduler extender image pull policy | `IfNotPresent` | +| `scheduler.extender.image.pullSecrets` | Scheduler extender image pull secrets | `[]` | +| `scheduler.extender.extraArgs` | Scheduler extender extra arguments | `["--debug", "-v=4"]` | + +### Admission Webhook Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `scheduler.admissionWebhook.enabled` | Whether to enable admission webhook | `true` | +| `scheduler.admissionWebhook.customURL.enabled` | Whether to enable custom URL | `false` | +| `scheduler.admissionWebhook.customURL.host` | Custom URL host | `127.0.0.1` | +| `scheduler.admissionWebhook.customURL.port` | Custom URL port | `31998` | +| `scheduler.admissionWebhook.customURL.path` | Custom URL path | `/webhook` | +| `scheduler.admissionWebhook.reinvocationPolicy` | Reinvocation policy | `Never` | +| `scheduler.admissionWebhook.failurePolicy` | Failure policy | `Ignore` | + +### TLS Certificate Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `scheduler.certManager.enabled` | Whether to use cert-manager to generate self-signed certificates | `false` | +| `scheduler.patch.enabled` | Whether to use kube-webhook-certgen to generate self-signed certificates | `true` | +| `scheduler.patch.image.registry` | Certgen image registry | `"docker.io"` | +| `scheduler.patch.image.repository` | Certgen image repository | `"jettech/kube-webhook-certgen"` | +| `scheduler.patch.image.tag` | Certgen image tag | `"v1.5.2"` | +| `scheduler.patch.image.pullPolicy` | Certgen image pull policy | `IfNotPresent` | +| `scheduler.patch.imageNew.registry` | New certgen image registry | `"docker.io"` | +| `scheduler.patch.imageNew.repository` | New certgen image repository | `"liangjw/kube-webhook-certgen"` | +| `scheduler.patch.imageNew.tag` | New certgen image tag | `"v1.1.1"` | + +### Scheduler Service Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `scheduler.service.type` | Service type | `NodePort` | +| `scheduler.service.httpPort` | HTTP port | `443` | +| `scheduler.service.schedulerPort` | Scheduler NodePort | `31998` | +| `scheduler.service.monitorPort` | Monitor port | `31993` | +| `scheduler.service.monitorTargetPort` | Monitor target port | `9395` | + +## Device Plugin Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devicePlugin.image.registry` | Device plugin image registry | `"docker.io"` | +| `devicePlugin.image.repository` | Device plugin image repository | `"projecthami/hami"` | +| `devicePlugin.image.tag` | Device plugin image tag | `""` | +| `devicePlugin.image.pullPolicy` | Device plugin image pull policy | `IfNotPresent` | +| `devicePlugin.image.pullSecrets` | Device plugin image pull secrets | `[]` | + +### Monitor Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devicePlugin.monitor.image.registry` | Monitor image registry | `"docker.io"` | +| `devicePlugin.monitor.image.repository` | Monitor image repository | `"projecthami/hami"` | +| `devicePlugin.monitor.image.tag` | Monitor image tag | `""` | +| `devicePlugin.monitor.image.pullPolicy` | Monitor image pull policy | `IfNotPresent` | +| `devicePlugin.monitor.image.pullSecrets` | Monitor image pull secrets | `[]` | +| `devicePlugin.monitor.ctrPath` | Container path | `/usr/local/vgpu/containers` | +| `devicePlugin.monitor.extraArgs` | Monitor extra arguments | `["-v=4"]` | +| `devicePlugin.monitor.extraEnvs` | Monitor extra environments | `{}` | + +### Device Plugin Other Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devicePlugin.deviceSplitCount` | Integer type, default value: 10. Maximum number of tasks assigned to a single GPU device | `10` | +| `devicePlugin.deviceMemoryScaling` | Device memory scaling ratio | `1` | +| `devicePlugin.deviceCoreScaling` | Device core scaling ratio | `1` | +| `devicePlugin.runtimeClassName` | Runtime class name | `""` | +| `devicePlugin.createRuntimeClass` | Whether to create runtime class | `false` | +| `devicePlugin.migStrategy` | String type, "none" means ignore MIG functionality, "mixed" means allocate MIG devices through independent resources | `"none"` | +| `devicePlugin.disablecorelimit` | String type, "true" means disable core limit, "false" means enable core limit | `"false"` | +| `devicePlugin.passDeviceSpecsEnabled` | Whether to enable passing device specs | `false` | +| `devicePlugin.extraArgs` | Device plugin extra arguments | `["-v=4"]` | +| `devicePlugin.nodeConfiguration.config` | Node configuration for device plugin by json | An example of default configuration. | +| `devicePlugin.nodeConfiguration.externalConfigName` | Node configuration for device plugin by external congimap | `""` | +| `devicePlugin.extraEnvs` | Device plugin extra environments | `{}` | + +### Device Plugin Service Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devicePlugin.service.type` | Service type | `NodePort` | +| `devicePlugin.service.httpPort` | HTTP port | `31992` | + +### Device Plugin Deployment Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devicePlugin.pluginPath` | Plugin path | `/var/lib/kubelet/device-plugins` | +| `devicePlugin.libPath` | Library path | `/usr/local/vgpu` | +| `devicePlugin.nvidiaNodeSelector` | NVIDIA node selector | `{"gpu": "on"}` | +| `devicePlugin.updateStrategy.type` | Update strategy type | `RollingUpdate` | +| `devicePlugin.updateStrategy.rollingUpdate.maxUnavailable` | Maximum unavailable count | `1` | + +## Device Configuration + +### AWS Neuron +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.awsneuron.customresources` | Custom resources | `["aws.amazon.com/neuron", "aws.amazon.com/neuroncore"]` | + +### Kunlunxin +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.kunlun.enabled` | Whether to enable | `true` | +| `devices.kunlun.customresources` | Custom resources | `["kunlunxin.com/xpu"]` | + +### Mthreads +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.mthreads.enabled` | Whether to enable | `true` | +| `devices.mthreads.customresources` | Custom resources | `["mthreads.com/vgpu"]` | + +### NVIDIA +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.nvidia.gpuCorePolicy` | GPU core policy | `default` | +| `devices.nvidia.libCudaLogLevel` | CUDA library log level | `1` | + +### Huawei Ascend +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.ascend.enabled` | Whether to enable | `false` | +| `devices.ascend.image` | Image | `""` | +| `devices.ascend.imagePullPolicy` | Image pull policy | `IfNotPresent` | +| `devices.ascend.extraArgs` | Extra arguments | `[]` | +| `devices.ascend.nodeSelector` | Node selector | `{"ascend": "on"}` | +| `devices.ascend.tolerations` | Tolerations | `[]` | +| `devices.ascend.customresources` | Custom resources | `["huawei.com/Ascend910A", "huawei.com/Ascend910A-memory", ...]` | + +### Iluvatar +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.iluvatar.enabled` | Whether to enable | `false` | +| `devices.iluvatar.customresources` | Custom resources | `["iluvatar.ai/BI-V150-vgpu", "iluvatar.ai/BI-V150.vMem","iluvatar.ai/BI-V150.vCore", ...]` | diff --git a/packages/system/hami/charts/hami/templates/NOTES.txt b/packages/system/hami/charts/hami/templates/NOTES.txt new file mode 100644 index 00000000..15bb1218 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/NOTES.txt @@ -0,0 +1,3 @@ +** Please be patient while the chart is being deployed ** +Resource name: {{ .Values.resourceName }} + diff --git a/packages/system/hami/charts/hami/templates/_commons.tpl b/packages/system/hami/charts/hami/templates/_commons.tpl new file mode 100644 index 00000000..b68018e4 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/_commons.tpl @@ -0,0 +1,49 @@ +{{/* +Return the proper image name +{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" $) }} +*/}} +{{- define "common.images.image" -}} +{{- $registryName := .imageRoot.registry -}} +{{- $repositoryName := .imageRoot.repository -}} +{{- $tag := .imageRoot.tag | toString -}} +{{- if .global }} + {{- if .global.imageRegistry }} + {{- $registryName = .global.imageRegistry -}} + {{- end -}} +{{- end -}} +{{- if .tag }} + {{- $tag = .tag | toString -}} +{{- end -}} +{{- if $registryName }} +{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- else -}} +{{- printf "%s:%s" $repositoryName $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead) +{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }} +*/}} +{{- define "common.images.pullSecrets" -}} + {{- $pullSecrets := list }} + + {{- if .global }} + {{- range .global.imagePullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- range .images -}} + {{- range .pullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- if (not (empty $pullSecrets)) }} +imagePullSecrets: + {{- range $pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/_helpers.tpl b/packages/system/hami/charts/hami/templates/_helpers.tpl new file mode 100644 index 00000000..ffcc61da --- /dev/null +++ b/packages/system/hami/charts/hami/templates/_helpers.tpl @@ -0,0 +1,163 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "hami-vgpu.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "hami-vgpu.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "hami-vgpu.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +The app name for Scheduler +*/}} +{{- define "hami-vgpu.scheduler" -}} +{{- printf "%s-scheduler" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +The app name for DevicePlugin +*/}} +{{- define "hami-vgpu.device-plugin" -}} +{{- printf "%s-device-plugin" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* + The app name for MockDevicePlugin + */}} +{{- define "hami-vgpu.mock-device-plugin" -}} +{{- printf "%s-mock-device-plugin" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +The tls secret name for Scheduler +*/}} +{{- define "hami-vgpu.scheduler.tls" -}} +{{- printf "%s-scheduler-tls" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +The webhook name +*/}} +{{- define "hami-vgpu.scheduler.webhook" -}} +{{- printf "%s-webhook" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "hami-vgpu.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "hami-vgpu.labels" -}} +helm.sh/chart: {{ include "hami-vgpu.chart" . }} +{{ include "hami-vgpu.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "hami-vgpu.selectorLabels" -}} +app.kubernetes.io/name: {{ include "hami-vgpu.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + + +{{/* + Resolve the tag for kubeScheduler. +*/}} +{{- define "resolvedKubeSchedulerTag" -}} +{{- if .Values.scheduler.kubeScheduler.image.tag }} +{{- .Values.scheduler.kubeScheduler.image.tag | trim -}} +{{- else }} +{{- include "strippedKubeVersion" . | trim -}} +{{- end }} +{{- end }} + +{{/* + Return the stripped Kubernetes version string by removing extra parts after semantic version number. + v1.31.1+k3s1 -> v1.31.1 + v1.30.8-eks-2d5f260 -> v1.30.8 + v1.31.1 -> v1.31.1 +*/}} +{{- define "strippedKubeVersion" -}} +{{ regexReplaceAll "^(v[0-9]+\\.[0-9]+\\.[0-9]+)(.*)$" .Capabilities.KubeVersion.Version "$1" }} +{{- end -}} + +{{- define "hami.scheduler.kubeScheduler.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.kubeScheduler.image "global" .Values.global "tag" (include "resolvedKubeSchedulerTag" .)) }} +{{- end -}} + +{{- define "hami.scheduler.extender.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.extender.image "global" .Values.global "tag" .Values.global.imageTag) }} +{{- end -}} + +{{- define "hami.devicePlugin.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.devicePlugin.image "global" .Values.global "tag" .Values.global.imageTag) }} +{{- end -}} + +{{- define "hami.mockDevicePlugin.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.mockDevicePlugin.image "global" .Values.global "tag" .Values.mockDevicePlugin.tag) }} +{{- end -}} + +{{- define "hami.devicePlugin.monitor.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.devicePlugin.monitor.image "global" .Values.global "tag" .Values.global.imageTag) }} +{{- end -}} + +{{- define "hami.scheduler.patch.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.patch.image "global" .Values.global) }} +{{- end -}} + +{{- define "hami.scheduler.patch.new.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.patch.imageNew "global" .Values.global) }} +{{- end -}} + +{{- define "hami.scheduler.extender.imagePullSecrets" -}} +{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.extender.image) "global" .Values.global) }} +{{- end -}} + +{{- define "hami.devicePlugin.imagePullSecrets" -}} +{{ include "common.images.pullSecrets" (dict "images" (list .Values.devicePlugin.image) "global" .Values.global) }} +{{- end -}} + +{{- define "hami.scheduler.patch.imagePullSecrets" -}} +{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.patch.image) "global" .Values.global) }} +{{- end -}} + +{{- define "hami.scheduler.patch.new.imagePullSecrets" -}} +{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.patch.imageNew) "global" .Values.global) }} +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml b/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml new file mode 100644 index 00000000..74631e24 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml @@ -0,0 +1,13 @@ +{{- if and .Values.devicePlugin.enabled (not .Values.devicePlugin.nodeConfiguration.externalConfigName) -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "hami-vgpu.device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-device-plugin + {{- include "hami-vgpu.labels" . | nindent 4 }} +data: + config.json: | +{{- .Values.devicePlugin.nodeConfiguration.config | nindent 4 }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/daemonsetmock.yaml b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetmock.yaml new file mode 100644 index 00000000..a0dc4ff3 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetmock.yaml @@ -0,0 +1,55 @@ +{{- if .Values.mockDevicePlugin.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "hami-vgpu.mock-device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +spec: + selector: + matchLabels: + app.kubernetes.io/component: hami-mock-device-plugin + {{- include "hami-vgpu.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" + labels: + app.kubernetes.io/component: hami-mock-device-plugin + {{- include "hami-vgpu.selectorLabels" . | nindent 8 }} + spec: + serviceAccountName: {{ include "hami-vgpu.mock-device-plugin" . }} + tolerations: + - key: CriticalAddonsOnly + operator: Exists + containers: + - image: {{ include "hami.mockDevicePlugin.image" . }} + imagePullPolicy: {{ .Values.mockDevicePlugin.image.pullPolicy }} + name: hami-mock-dp-cntr + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + command: + - ./k8s-device-plugin + - -v=5 + - --device-config-file=/device-config.yaml + volumeMounts: + - name: dp + mountPath: /var/lib/kubelet/device-plugins + - name: sys + mountPath: /sys + - name: device-config + mountPath: /device-config.yaml + subPath: device-config.yaml + volumes: + - name: dp + hostPath: + path: /var/lib/kubelet/device-plugins + - name: sys + hostPath: + path: /sys + - name: device-config + configMap: + name: {{ include "hami-vgpu.scheduler" . }}-device +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml new file mode 100644 index 00000000..1f4c24f3 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml @@ -0,0 +1,262 @@ +{{- if .Values.devicePlugin.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "hami-vgpu.device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-device-plugin + {{- include "hami-vgpu.labels" . | nindent 4 }} + {{- with .Values.global.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.global.annotations }} + annotations: {{ toYaml .Values.global.annotations | nindent 4}} + {{- end }} +spec: + updateStrategy: + {{- with .Values.devicePlugin.updateStrategy }} + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/component: hami-device-plugin + {{- include "hami-vgpu.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app.kubernetes.io/component: hami-device-plugin + hami.io/webhook: ignore + {{- include "hami-vgpu.selectorLabels" . | nindent 8 }} + annotations: + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + checksum/hami-scheduler-newversion-config: {{ include (print $.Template.BasePath "/scheduler/configmapnew.yaml") . | sha256sum }} + {{- else }} + checksum/hami-scheduler-config: {{ include (print $.Template.BasePath "/scheduler/configmap.yaml") . | sha256sum }} + {{- end }} + checksum/hami-scheduler-device-config: {{ include (print $.Template.BasePath "/scheduler/device-configmap.yaml") . | sha256sum }} + {{- if .Values.devicePlugin.podAnnotations }} + {{- toYaml .Values.devicePlugin.podAnnotations | nindent 8 }} + {{- end }} + spec: + {{- if .Values.devicePlugin.runtimeClassName }} + runtimeClassName: {{ .Values.devicePlugin.runtimeClassName }} + {{- end }} + serviceAccountName: {{ include "hami-vgpu.device-plugin" . }} + priorityClassName: system-node-critical + hostPID: true + hostNetwork: true + {{- include "hami.devicePlugin.imagePullSecrets" . | nindent 6 }} + {{- if .Values.devicePlugin.gpuOperatorToolkitReady.enabled }} + initContainers: + - name: toolkit-validation + image: {{ include "hami.devicePlugin.image" . }} + imagePullPolicy: {{ .Values.devicePlugin.image.pullPolicy }} + securityContext: + privileged: true + runAsUser: 0 + command: ["sh", "-c"] + args: + - | + echo "Waiting for NVIDIA Toolkit to be ready..." + until [ -f {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }}/toolkit-ready ]; do + echo "Waiting for {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }}/toolkit-ready..." + sleep 5 + done + echo "NVIDIA Toolkit is ready!" + volumeMounts: + - name: nvidia-validations + mountPath: {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath | quote }} + mountPropagation: HostToContainer + readOnly: true + {{- end }} + containers: + - name: device-plugin + image: {{ include "hami.devicePlugin.image" . }} + imagePullPolicy: {{ .Values.devicePlugin.image.pullPolicy }} + lifecycle: + postStart: + exec: + command: ["/bin/sh","-c", {{ printf "/k8s-vgpu/bin/vgpu-init.sh %s/vgpu/" .Values.global.gpuHookPath | quote }}] + command: + - nvidia-device-plugin + - --config-file=/device-config.yaml + - --mig-strategy={{ .Values.devicePlugin.migStrategy }} + - --disable-core-limit={{ .Values.devicePlugin.disablecorelimit }} + {{- range .Values.devicePlugin.extraArgs }} + - {{ . }} + {{- end }} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: NVIDIA_MIG_MONITOR_DEVICES + value: all + - name: DEVICE_LIST_STRATEGY + value: {{ .Values.devicePlugin.deviceListStrategy }} + - name: HOOK_PATH + value: {{ .Values.global.gpuHookPath }} + {{- if typeIs "bool" .Values.devicePlugin.passDeviceSpecsEnabled }} + - name: PASS_DEVICE_SPECS + value: {{ .Values.devicePlugin.passDeviceSpecsEnabled | quote }} + {{- end }} + {{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }} + - name: NVIDIA_DRIVER_ROOT + value: {{ .Values.devicePlugin.nvidiaDriverRoot }} + {{- end }} + {{- if typeIs "string" .Values.devicePlugin.nvidiaHookPath }} + - name: NVIDIA_CDI_HOOK_PATH + value: {{ .Values.devicePlugin.nvidiaHookPath }} + {{- end }} + {{- if typeIs "bool" .Values.devicePlugin.gdrcopyEnabled }} + - name: GDRCOPY_ENABLED + value: {{ .Values.devicePlugin.gdrcopyEnabled | quote }} + {{- end }} + {{- if typeIs "bool" .Values.devicePlugin.gdsEnabled }} + - name: GDS_ENABLED + value: {{ .Values.devicePlugin.gdsEnabled | quote }} + {{- end }} + {{- if typeIs "bool" .Values.devicePlugin.mofedEnabled }} + - name: MOFED_ENABLED + value: {{ .Values.devicePlugin.mofedEnabled | quote }} + {{- end }} + {{- if eq (.Values.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy | default "spread") "topology-aware" }} + - name: ENABLE_TOPOLOGY_SCORE + value: "true" + {{- end }} + {{- with .Values.devicePlugin.extraEnvs }} + {{- . | toYaml | nindent 12 }} + {{- end }} + securityContext: + privileged: true + allowPrivilegeEscalation: true + capabilities: + drop: ["ALL"] + add: ["SYS_ADMIN"] + resources: + {{- toYaml .Values.devicePlugin.resources | nindent 12 }} + volumeMounts: + - name: device-plugin + mountPath: /var/lib/kubelet/device-plugins + - name: lib + mountPath: {{ printf "%s%s" .Values.global.gpuHookPath "/vgpu" }} + - name: usrbin + mountPath: /usrbin + - name: deviceconfig + mountPath: /config + - name: hosttmp + mountPath: /tmp + - name: device-config + mountPath: /device-config.yaml + subPath: device-config.yaml + - name: cdi-root + mountPath: /var/run/cdi + {{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }} + # We always mount the driver root at /driver-root in the container. + # This is required for CDI detection to work correctly. + - name: driver-root + mountPath: /driver-root + readOnly: true + {{- end }} + - name: vgpu-monitor + image: {{ include "hami.devicePlugin.monitor.image" . }} + imagePullPolicy: {{ .Values.devicePlugin.monitor.image.pullPolicy }} + command: + - "vGPUmonitor" + {{- range .Values.devicePlugin.monitor.extraArgs }} + - {{ . }} + {{- end }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + add: ["SYS_ADMIN"] + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: NVIDIA_VISIBLE_DEVICES + value: "all" + - name: NVIDIA_MIG_MONITOR_DEVICES + value: "all" + - name: HOOK_PATH + value: "{{ .Values.global.gpuHookPath }}/vgpu" + - name: HAMI_RESYNC_INTERVAL + value: {{ .Values.devicePlugin.monitor.resyncInterval | default "5m" | quote }} + {{- with .Values.devicePlugin.monitor.extraEnvs }} + {{- . | toYaml | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.devicePlugin.monitor.resources | nindent 12 }} + volumeMounts: + - name: ctrs + mountPath: {{ .Values.devicePlugin.monitor.ctrPath }} + - name: dockers + mountPath: /run/docker + - name: containerds + mountPath: /run/containerd + - name: sysinfo + mountPath: /sysinfo + - name: hostvar + mountPath: /hostvar + - name: hosttmp + mountPath: /tmp + volumes: + - name: ctrs + hostPath: + path: {{ .Values.devicePlugin.monitor.ctrPath }} + - name: hosttmp + hostPath: + path: /tmp + - name: dockers + hostPath: + path: /run/docker + - name: containerds + hostPath: + path: /run/containerd + - name: device-plugin + hostPath: + path: {{ .Values.devicePlugin.pluginPath }} + - name: lib + hostPath: + path: {{ .Values.devicePlugin.libPath }} + {{- if .Values.devicePlugin.gpuOperatorToolkitReady.enabled }} + - name: nvidia-validations + hostPath: + path: {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }} + type: DirectoryOrCreate + {{- end }} + {{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }} + - name: driver-root + hostPath: + path: {{ .Values.devicePlugin.nvidiaDriverRoot }} + type: Directory + {{- end }} + - name: cdi-root + hostPath: + path: /var/run/cdi + type: DirectoryOrCreate + - name: usrbin + hostPath: + path: /usr/bin + - name: sysinfo + hostPath: + path: /sys + - name: hostvar + hostPath: + path: /var + - name: deviceconfig + configMap: + name: {{ .Values.devicePlugin.nodeConfiguration.externalConfigName | default (include "hami-vgpu.device-plugin" .) }} + - name: device-config + configMap: + name: {{ include "hami-vgpu.scheduler" . }}-device + {{- if .Values.devicePlugin.nvidiaNodeSelector }} + nodeSelector: {{ toYaml .Values.devicePlugin.nvidiaNodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.devicePlugin.tolerations }} + tolerations: {{ toYaml .Values.devicePlugin.tolerations | nindent 8 }} + {{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml new file mode 100644 index 00000000..6ac757f2 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml @@ -0,0 +1,28 @@ +{{- if .Values.devicePlugin.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "hami-vgpu.device-plugin" . }}-monitor +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - create + - watch + - list + - update + - patch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - update + - list + - patch +{{- end -}} + diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml new file mode 100644 index 00000000..2f0a14ba --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml @@ -0,0 +1,17 @@ +{{- if .Values.devicePlugin.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.device-plugin" . }} + labels: + app.kubernetes.io/component: "hami-device-plugin" + {{- include "hami-vgpu.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "hami-vgpu.device-plugin" . }}-monitor +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml new file mode 100644 index 00000000..88f1b214 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml @@ -0,0 +1,29 @@ +{{- if .Values.devicePlugin.enabled -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "hami-vgpu.device-plugin" . }}-monitor + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-device-plugin + {{- include "hami-vgpu.labels" . | nindent 4 }} + {{- with .Values.devicePlugin.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.devicePlugin.service.annotations }} # Use devicePlugin instead of scheduler + annotations: {{ toYaml .Values.devicePlugin.service.annotations | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.devicePlugin.service.type | default "NodePort" }} # Default type is NodePort + ports: + - name: monitorport + port: {{ .Values.devicePlugin.service.httpPort | default 31992 }} # Default HTTP port is 31992 + targetPort: 9394 + {{- if eq (.Values.devicePlugin.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort + nodePort: {{ .Values.devicePlugin.service.httpPort | default 31992 }} + {{- end }} + protocol: TCP + selector: + app.kubernetes.io/component: hami-device-plugin + {{- include "hami-vgpu.selectorLabels" . | nindent 4 }} +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml new file mode 100644 index 00000000..6e3c2a44 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml @@ -0,0 +1,10 @@ +{{- if .Values.devicePlugin.enabled -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "hami-vgpu.device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: "hami-device-plugin" + {{- include "hami-vgpu.labels" . | nindent 4 }} +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml b/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml new file mode 100644 index 00000000..ebcc2c9c --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml @@ -0,0 +1,9 @@ +{{- if and .Values.devicePlugin.enabled .Values.devicePlugin.createRuntimeClass .Values.devicePlugin.runtimeClassName -}} +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: {{ .Values.devicePlugin.runtimeClassName }} + annotations: + helm.sh/hook: pre-install,pre-upgrade +handler: nvidia +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml b/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml new file mode 100644 index 00000000..e6d28721 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml @@ -0,0 +1,31 @@ +{{- if .Values.scheduler.admissionWebhook.enabled -}} +{{- if .Values.scheduler.certManager.enabled }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-serving-cert + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} +spec: + dnsNames: + - {{ include "hami-vgpu.scheduler" . }}.{{ include "hami-vgpu.namespace" . }}.svc + - {{ include "hami-vgpu.scheduler" . }}.{{ include "hami-vgpu.namespace" . }}.svc.cluster.local + issuerRef: + kind: Issuer + name: {{ include "hami-vgpu.scheduler" . }}-selfsigned-issuer + secretName: {{ include "hami-vgpu.scheduler.tls" . }} +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-selfsigned-issuer + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} +spec: + selfSigned: {} +{{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml b/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml new file mode 100644 index 00000000..81c4fddf --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml @@ -0,0 +1,34 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["pods", "configmaps"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: [""] + resources: ["pods/binding"] + verbs: ["create"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "patch", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "get", "list"] + - apiGroups: [""] + resources: ["resourcequotas"] + verbs: ["get", "list", "watch"] +--- +{{- if .Values.mockDevicePlugin.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "hami-vgpu.mock-device-plugin" . }} +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "update", "list", "patch"] +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml new file mode 100644 index 00000000..a81d425c --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml @@ -0,0 +1,47 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-kube + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-scheduler +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-volume + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:volume-scheduler +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "hami-vgpu.scheduler" . }} +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml b/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml new file mode 100644 index 00000000..109ecbce --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml @@ -0,0 +1,142 @@ +{{- if .Values.scheduler.kubeScheduler.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} +data: + config.json: | + { + "kind": "Policy", + "apiVersion": "v1", + "extenders": [ + { + {{- if .Values.scheduler.admissionWebhook.enabled }} + "urlPrefix": "https://127.0.0.1:443", + "enableHttps": true, + "tlsConfig": { + "insecure": true + }, + {{- else }} + "urlPrefix": "http://127.0.0.1:80", + "enableHttps": false, + {{- end }} + "filterVerb": "filter", + "bindVerb": "bind", + "weight": 1, + "nodeCacheCapable": true, + "httpTimeout": 30000000000, + "managedResources": [ + {{- range .Values.devices.amd.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- if .Values.devices.ascend.enabled }} + {{- range .Values.devices.ascend.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- end }} + {{- if .Values.devices.mthreads.enabled }} + {{- range .Values.devices.mthreads.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- end }} + {{- if .Values.devices.enflame.enabled }} + {{- range .Values.devices.enflame.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- end }} + {{- if .Values.devices.kunlun.enabled }} + {{- range .Values.devices.kunlun.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- end }} + {{- range .Values.devices.awsneuron.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- if .Values.devices.iluvatar.enabled }} + {{- range .Values.devices.iluvatar.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- end }} + { + "name": "{{ .Values.resourceName }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.resourceMem }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.resourceCores }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.resourceMemPercentage }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.resourcePriority }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.mluResourceName }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.dcuResourceName }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.dcuResourceMem }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.dcuResourceCores }}", + "ignoredByScheduler": true + }, + { + "name": "metax-tech.com/gpu", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.metaxResourceName }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.metaxResourceCore }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.metaxResourceMem }}", + "ignoredByScheduler": true + } + ], + "ignoreable": false + } + ] + } +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml b/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml new file mode 100644 index 00000000..6f6db097 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml @@ -0,0 +1,102 @@ +{{- if .Values.scheduler.kubeScheduler.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-newversion + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} +data: + config.yaml: | + {{- if gt (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 25}} + apiVersion: kubescheduler.config.k8s.io/v1 + {{- else }} + apiVersion: kubescheduler.config.k8s.io/v1beta2 + {{- end }} + kind: KubeSchedulerConfiguration + leaderElection: + leaderElect: false + profiles: + - schedulerName: {{ .Values.schedulerName }} + extenders: + {{- if .Values.scheduler.admissionWebhook.enabled }} + - urlPrefix: "https://127.0.0.1:443" + enableHTTPS: true + tlsConfig: + insecure: true + {{- else }} + - urlPrefix: "http://127.0.0.1:80" + enableHTTPS: false + {{- end }} + filterVerb: filter + bindVerb: bind + nodeCacheCapable: true + weight: 1 + httpTimeout: 30s + managedResources: + - name: {{ .Values.resourceName }} + ignoredByScheduler: true + - name: {{ .Values.resourceMem }} + ignoredByScheduler: true + - name: {{ .Values.resourceCores }} + ignoredByScheduler: true + - name: {{ .Values.resourceMemPercentage }} + ignoredByScheduler: true + - name: {{ .Values.resourcePriority }} + ignoredByScheduler: true + - name: {{ .Values.mluResourceName }} + ignoredByScheduler: true + - name: {{ .Values.dcuResourceName }} + ignoredByScheduler: true + - name: {{ .Values.dcuResourceMem }} + ignoredByScheduler: true + - name: {{ .Values.dcuResourceCores }} + ignoredByScheduler: true + - name: "metax-tech.com/gpu" + ignoredByScheduler: true + - name: {{ .Values.metaxResourceName }} + ignoredByScheduler: true + - name: {{ .Values.metaxResourceCore }} + ignoredByScheduler: true + - name: {{ .Values.metaxResourceMem }} + ignoredByScheduler: true + {{- if .Values.devices.ascend.enabled }} + {{- range .Values.devices.ascend.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- end }} + {{- if .Values.devices.mthreads.enabled }} + {{- range .Values.devices.mthreads.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- end }} + {{- if .Values.devices.enflame.enabled }} + {{- range .Values.devices.enflame.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- end }} + {{- if .Values.devices.kunlun.enabled }} + {{- range .Values.devices.kunlun.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- end }} + {{- range .Values.devices.awsneuron.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- if .Values.devices.iluvatar.enabled }} + {{- range .Values.devices.iluvatar.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- end }} + {{- range .Values.devices.amd.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml b/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml new file mode 100644 index 00000000..1d89e189 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml @@ -0,0 +1,225 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} + {{- with .Values.global.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.global.annotations }} + annotations: {{ toYaml .Values.global.annotations | nindent 4}} + {{- end }} +spec: + {{- if .Values.scheduler.leaderElect }} + replicas: {{ .Values.scheduler.replicas }} + {{- else }} + replicas: 1 + {{- end }} + selector: + matchLabels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.selectorLabels" . | nindent 8 }} + hami.io/webhook: ignore + annotations: + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + checksum/hami-scheduler-newversion-config: {{ include (print $.Template.BasePath "/scheduler/configmapnew.yaml") . | sha256sum }} + {{- else }} + checksum/hami-scheduler-config: {{ include (print $.Template.BasePath "/scheduler/configmap.yaml") . | sha256sum }} + {{- end }} + checksum/hami-scheduler-device-config: {{ include (print $.Template.BasePath "/scheduler/device-configmap.yaml") . | sha256sum }} + {{- if .Values.scheduler.podAnnotations }} + {{- toYaml .Values.scheduler.podAnnotations | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "hami-vgpu.scheduler" . }} + priorityClassName: system-node-critical + {{- include "hami.scheduler.extender.imagePullSecrets" . | nindent 6 }} + containers: + {{- if .Values.scheduler.kubeScheduler.enabled }} + - name: kube-scheduler + image: {{ include "hami.scheduler.kubeScheduler.image" . }} + imagePullPolicy: {{ .Values.scheduler.kubeScheduler.image.pullPolicy }} + command: + - kube-scheduler + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + {{- range .Values.scheduler.kubeScheduler.extraNewArgs }} + - {{ . }} + {{- end }} + {{- else }} + - --scheduler-name={{ .Values.schedulerName }} + {{- range .Values.scheduler.kubeScheduler.extraArgs }} + - {{ . }} + {{- end }} + {{- end }} + - --leader-elect={{ .Values.scheduler.leaderElect }} + - --leader-elect-resource-name={{ .Values.schedulerName }} + - --leader-elect-resource-namespace={{ include "hami-vgpu.namespace" . }} + resources: + {{- toYaml .Values.scheduler.kubeScheduler.resources | nindent 12 }} + volumeMounts: + - name: scheduler-config + mountPath: /config + {{- end }} + {{- if .Values.scheduler.livenessProbe }} + livenessProbe: + failureThreshold: 8 + httpGet: + path: /healthz + port: 10259 + scheme: HTTPS + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 15 + {{- end }} + - name: vgpu-scheduler-extender + image: {{ include "hami.scheduler.extender.image" . }} + imagePullPolicy: {{ .Values.scheduler.extender.image.pullPolicy }} + env: + {{- if .Values.scheduler.nodeLockExpire }} + - name: HAMI_NODELOCK_EXPIRE + value: "{{ .Values.scheduler.nodeLockExpire }}" + {{- end }} + {{- if .Values.global.managedNodeSelectorEnable }} + {{- range $key, $value := .Values.global.managedNodeSelector }} + - name: NODE_SELECTOR_{{ $key | upper | replace "-" "_" }} + value: "{{ $value }}" + {{- end }} + {{- end }} + command: + - scheduler + {{- if .Values.scheduler.admissionWebhook.enabled }} + - --http_bind=0.0.0.0:443 + - --cert_file=/tls/tls.crt + - --key_file=/tls/tls.key + {{- else }} + - --http_bind=0.0.0.0:80 + {{- end }} + - --scheduler-name={{ .Values.schedulerName }} + - --metrics-bind-address={{ .Values.scheduler.metricsBindAddress }} + - --node-scheduler-policy={{ .Values.scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy }} + - --gpu-scheduler-policy={{ .Values.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy }} + - --force-overwrite-default-scheduler={{ .Values.scheduler.forceOverwriteDefaultScheduler}} + - --device-config-file=/device-config.yaml + - --leader-elect={{ .Values.scheduler.leaderElect }} + - --leader-elect-resource-name={{ .Values.schedulerName }} + - --leader-elect-resource-namespace={{ include "hami-vgpu.namespace" . }} + {{- if .Values.devices.ascend.enabled }} + - --enable-ascend=true + {{- end }} + {{- if .Values.devices.iluvatar.enabled }} + - --enable-iluvatar=true + {{- end }} + {{- if .Values.scheduler.nodeLabelSelector }} + - --node-label-selector={{- $first := true -}} + {{- range $key, $value := .Values.scheduler.nodeLabelSelector -}} + {{- if not $first }},{{ end -}} + {{- $key }}={{ $value -}} + {{- $first = false -}} + {{- end -}} + {{- end }} + {{- range .Values.scheduler.extender.extraArgs }} + - {{ . }} + {{- end }} + ports: + {{- if .Values.scheduler.admissionWebhook.enabled }} + - name: https + containerPort: 443 + protocol: TCP + {{- else }} + - name: http + containerPort: 80 + protocol: TCP + {{- end }} + - name: metrics + containerPort: {{ last (splitList ":" .Values.scheduler.metricsBindAddress) | int }} + protocol: TCP + resources: + {{- toYaml .Values.scheduler.extender.resources | nindent 12 }} + volumeMounts: + - name: device-config + mountPath: /device-config.yaml + subPath: device-config.yaml + {{- if .Values.scheduler.admissionWebhook.enabled }} + - name: tls-config + mountPath: /tls + {{- end }} + {{- if .Values.scheduler.livenessProbe }} + livenessProbe: + httpGet: + path: /healthz + {{- if .Values.scheduler.admissionWebhook.enabled }} + port: https + scheme: HTTPS + {{- else }} + port: http + scheme: HTTP + {{- end }} + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 3 + timeoutSeconds: 5 + {{- end }} + {{- if .Values.scheduler.leaderElect }} + readinessProbe: + httpGet: + path: /readyz + {{- if .Values.scheduler.admissionWebhook.enabled }} + port: https + scheme: HTTPS + {{- else }} + port: http + scheme: HTTP + {{- end }} + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 3 + timeoutSeconds: 5 + {{- end }} + {{- if .Values.scheduler.leaderElect }} + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app.kubernetes.io/component + operator: In + values: + - hami-scheduler + topologyKey: "kubernetes.io/hostname" + {{- end }} + volumes: + {{- if .Values.scheduler.admissionWebhook.enabled }} + - name: tls-config + secret: + secretName: {{ template "hami-vgpu.scheduler.tls" . }} + {{- end }} + {{- if .Values.scheduler.kubeScheduler.enabled }} + - name: scheduler-config + configMap: + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + name: {{ template "hami-vgpu.scheduler" . }}-newversion + {{- else }} + name: {{ template "hami-vgpu.scheduler" . }} + {{- end }} + {{- end }} + - name: device-config + configMap: + name: {{ include "hami-vgpu.scheduler" . }}-device + {{- if .Values.scheduler.nodeSelector }} + nodeSelector: {{ toYaml .Values.scheduler.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.scheduler.tolerations }} + tolerations: {{ toYaml .Values.scheduler.tolerations | nindent 8 }} + {{- end }} + {{- if .Values.scheduler.nodeName }} + nodeName: {{ .Values.scheduler.nodeName }} + {{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml b/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml new file mode 100644 index 00000000..873b813b --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml @@ -0,0 +1,408 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-device + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} +data: + device-config.yaml: |- + {{- if .Files.Glob "files/device-config.yaml" }} + {{- .Files.Get "files/device-config.yaml" | nindent 4}} + {{- else }} + nvidia: + resourceCountName: {{ .Values.resourceName }} + resourceMemoryName: {{ .Values.resourceMem }} + resourceMemoryPercentageName: {{ .Values.resourceMemPercentage }} + resourceCoreName: {{ .Values.resourceCores }} + resourcePriorityName: {{ .Values.resourcePriority }} + overwriteEnv: false + defaultMemory: 0 + defaultCores: 0 + defaultGPUNum: 1 + memoryFactor: 1 + deviceSplitCount: {{ .Values.devicePlugin.deviceSplitCount }} + deviceMemoryScaling: {{ .Values.devicePlugin.deviceMemoryScaling }} + deviceCoreScaling: {{ .Values.devicePlugin.deviceCoreScaling }} + gpuCorePolicy: {{ .Values.devices.nvidia.gpuCorePolicy }} + libCudaLogLevel: {{ .Values.devices.nvidia.libCudaLogLevel }} + runtimeClassName: "{{ .Values.devicePlugin.runtimeClassName }}" + knownMigGeometries: + - models: [ "A30" ] + allowedGeometries: + - + - name: 1g.6gb + core: 25 + memory: 6144 + count: 4 + - + - name: 2g.12gb + core: 50 + memory: 12288 + count: 2 + - + - name: 4g.24gb + core: 100 + memory: 24576 + count: 1 + - models: [ "A100-SXM4-40GB", "A100-40GB-PCIe", "A100-PCIE-40GB"] + allowedGeometries: + - + - name: 1g.5gb + core: 14 + memory: 5120 + count: 7 + - + - name: 1g.5gb + core: 14 + memory: 5120 + count: 1 + - name: 2g.10gb + core: 28 + memory: 10240 + count: 3 + - + - name: 3g.20gb + core: 42 + memory: 20480 + count: 2 + - + - name: 7g.40gb + core: 100 + memory: 40960 + count: 1 + - models: [ "A100-SXM4-80GB", "A100-80GB-PCIe", "A100-PCIE-80GB"] + allowedGeometries: + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 7 + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 1 + - name: 2g.20gb + core: 28 + memory: 20480 + count: 3 + - + - name: 3g.40gb + core: 42 + memory: 40960 + count: 2 + - + - name: 7g.79gb + core: 100 + memory: 80896 + count: 1 + - models: [ "H100-PCIE-80GB", "H100-SXM5-80GB"] + allowedGeometries: + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 7 + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 1 + - name: 2g.20gb + core: 28 + memory: 20480 + count: 3 + - + - name: 3g.40gb + core: 42 + memory: 40960 + count: 2 + - + - name: 7g.80gb + core: 100 + memory: 81920 + count: 1 + - models: [ "H100-PCIE-94GB", "H100-SXM5-94GB"] + allowedGeometries: + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 7 + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 1 + - name: 2g.24gb + core: 28 + memory: 24576 + count: 3 + - + - name: 3g.47gb + core: 42 + memory: 48128 + count: 2 + - + - name: 7g.94gb + core: 100 + memory: 96256 + count: 1 + - models: [ "H20", "H100 on GH200"] + allowedGeometries: + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 7 + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 1 + - name: 2g.24gb + core: 28 + memory: 24576 + count: 3 + - + - name: 3g.48gb + core: 42 + memory: 49152 + count: 2 + - + - name: 7g.96gb + core: 100 + memory: 98304 + count: 1 + - models: [ "H200 NVL", "H200-SXM5"] + allowedGeometries: + - + - name: 1g.18gb + core: 14 + memory: 18432 + count: 7 + - + - name: 1g.18gb + core: 14 + memory: 18432 + count: 1 + - name: 2g.35gb + core: 28 + memory: 35840 + count: 3 + - + - name: 3g.71gb + core: 42 + memory: 72704 + count: 2 + - + - name: 7g.141gb + core: 100 + memory: 144384 + count: 1 + - models: [ "B200" ] + allowedGeometries: + - + - name: 1g.23gb + core: 14 + memory: 23552 + count: 7 + - + - name: 1g.23gb + core: 14 + memory: 23552 + count: 1 + - name: 2g.45gb + core: 28 + memory: 46080 + count: 3 + - + - name: 3g.90gb + core: 42 + memory: 92160 + count: 2 + - + - name: 7g.180gb + core: 100 + memory: 184320 + count: 1 + cambricon: + resourceCountName: {{ .Values.mluResourceName }} + resourceMemoryName: {{ .Values.mluResourceMem }} + resourceCoreName: {{ .Values.mluResourceCores }} + hygon: + resourceCountName: {{ .Values.dcuResourceName }} + resourceMemoryName: {{ .Values.dcuResourceMem }} + resourceCoreName: {{ .Values.dcuResourceCores }} + memoryFactor: 1 + metax: + resourceCountName: "metax-tech.com/gpu" + resourceVCountName: {{ .Values.metaxResourceName }} + resourceVMemoryName: {{ .Values.metaxResourceMem }} + resourceVCoreName: {{ .Values.metaxResourceCore }} + sgpuTopologyAware: {{ .Values.metaxsGPUTopologyAware }} + enflame: + resourceNameGCU: "enflame.com/gcu" + resourceNameVGCU: {{ .Values.enflameResourceNameVGCU }} + resourceNameVGCUPercentage: {{ .Values.enflameResourceNameVGCUPercentage }} + mthreads: + resourceCountName: "mthreads.com/vgpu" + resourceMemoryName: "mthreads.com/sgpu-memory" + resourceCoreName: "mthreads.com/sgpu-core" + iluvatars: + - chipName: MR-V100 + commonWord: MR-V100 + resourceCountName: iluvatar.ai/MR-V100-vgpu + resourceMemoryName: iluvatar.ai/MR-V100.vMem + resourceCoreName: iluvatar.ai/MR-V100.vCore + - chipName: MR-V50 + commonWord: MR-V50 + resourceCountName: iluvatar.ai/MR-V50-vgpu + resourceMemoryName: iluvatar.ai/MR-V50.vMem + resourceCoreName: iluvatar.ai/MR-V50.vCore + - chipName: BI-V150 + commonWord: BI-V150 + resourceCountName: iluvatar.ai/BI-V150-vgpu + resourceMemoryName: iluvatar.ai/BI-V150.vMem + resourceCoreName: iluvatar.ai/BI-V150.vCore + - chipName: BI-V100 + commonWord: BI-V100 + resourceCountName: iluvatar.ai/BI-V100-vgpu + resourceMemoryName: iluvatar.ai/BI-V100.vMem + resourceCoreName: iluvatar.ai/BI-V100.vCore + kunlun: + resourceCountName: {{ .Values.kunlunResourceName }} + resourceVCountName: {{ .Values.kunlunResourceVCountName }} + resourceVMemoryName: {{ .Values.kunlunResourceVMemoryName }} + awsneuron: + resourceCountName: "aws.amazon.com/neuron" + resourceCoreName: "aws.amazon.com/neuroncore" + amd: + resourceCountName: "amd.com/gpu" + vnpus: + - chipName: 910A + commonWord: Ascend910A + resourceName: huawei.com/Ascend910A + resourceMemoryName: huawei.com/Ascend910A-memory + memoryAllocatable: 32768 + memoryCapacity: 32768 + memoryFactor: 1 + aiCore: 30 + templates: + - name: vir02 + memory: 2184 + aiCore: 2 + - name: vir04 + memory: 4369 + aiCore: 4 + - name: vir08 + memory: 8738 + aiCore: 8 + - name: vir16 + memory: 17476 + aiCore: 16 + - chipName: 910B2 + commonWord: Ascend910B2 + resourceName: huawei.com/Ascend910B2 + resourceMemoryName: huawei.com/Ascend910B2-memory + memoryAllocatable: 65536 + memoryCapacity: 65536 + memoryFactor: 1 + aiCore: 24 + aiCPU: 6 + templates: + - name: vir03_1c_8g + memory: 8192 + aiCore: 3 + aiCPU: 1 + - name: vir06_1c_16g + memory: 16384 + aiCore: 6 + aiCPU: 1 + - name: vir12_3c_32g + memory: 32768 + aiCore: 12 + aiCPU: 3 + - chipName: 910B3 + commonWord: Ascend910B3 + resourceName: huawei.com/Ascend910B3 + resourceMemoryName: huawei.com/Ascend910B3-memory + memoryAllocatable: 65536 + memoryCapacity: 65536 + memoryFactor: 1 + aiCore: 20 + aiCPU: 7 + templates: + - name: vir05_1c_16g + memory: 16384 + aiCore: 5 + aiCPU: 1 + - name: vir10_3c_32g + memory: 32768 + aiCore: 10 + aiCPU: 3 + - chipName: 910B4-1 + commonWord: Ascend910B4-1 + resourceName: huawei.com/Ascend910B4-1 + resourceMemoryName: huawei.com/Ascend910B4-1-memory + memoryAllocatable: 65536 + memoryCapacity: 65536 + memoryFactor: 1 + aiCore: 20 + aiCPU: 7 + templates: + # NOTE: Names of vnpu template for 910B4-1 are fixed by Ascend runtime and must not be changed. + # The memory is used for scheduling so the correct values must be set. + # Template vir05_1c_8g actually provides 16GB memory, + - name: vir05_1c_8g + memory: 16384 + aiCore: 5 + aiCPU: 1 + # Template vir10_3c_16g actually provides 32GB memory + - name: vir10_3c_16g + memory: 32768 + aiCore: 10 + aiCPU: 3 + - chipName: 910B4 + commonWord: Ascend910B4 + resourceName: huawei.com/Ascend910B4 + resourceMemoryName: huawei.com/Ascend910B4-memory + memoryAllocatable: 32768 + memoryCapacity: 32768 + memoryFactor: 1 + aiCore: 20 + aiCPU: 7 + templates: + - name: vir05_1c_8g + memory: 8192 + aiCore: 5 + aiCPU: 1 + - name: vir10_3c_16g + memory: 16384 + aiCore: 10 + aiCPU: 3 + - chipName: 310P3 + commonWord: Ascend310P + resourceName: huawei.com/Ascend310P + resourceMemoryName: huawei.com/Ascend310P-memory + memoryAllocatable: 21527 + memoryCapacity: 24576 + memoryFactor: 1 + aiCore: 8 + aiCPU: 7 + templates: + - name: vir01 + memory: 3072 + aiCore: 1 + aiCPU: 1 + - name: vir02 + memory: 6144 + aiCore: 2 + aiCPU: 2 + - name: vir04 + memory: 12288 + aiCore: 4 + aiCPU: 4 + {{ end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml new file mode 100644 index 00000000..77e891cc --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +rules: + - apiGroups: + - admissionregistration.k8s.io + resources: + #- validatingwebhookconfigurations + - mutatingwebhookconfigurations + verbs: + - get + - update +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml new file mode 100644 index 00000000..2b82f926 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "hami-vgpu.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.fullname" . }}-admission + namespace: {{ include "hami-vgpu.namespace" . }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml new file mode 100644 index 00000000..0e36d95f --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml @@ -0,0 +1,68 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission-create + namespace: {{ include "hami-vgpu.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission-create + {{- if .Values.scheduler.patch.podAnnotations }} + annotations: {{ toYaml .Values.scheduler.patch.podAnnotations | nindent 8 }} + {{- end }} + labels: + {{- include "hami-vgpu.labels" . | nindent 8 }} + app.kubernetes.io/component: admission-webhook + hami.io/webhook: ignore + spec: + {{- if .Values.scheduler.patch.priorityClassName }} + priorityClassName: {{ .Values.scheduler.patch.priorityClassName }} + {{- end }} + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + {{- include "hami.scheduler.patch.new.imagePullSecrets" . | nindent 6 }} + {{- else }} + {{- include "hami.scheduler.patch.imagePullSecrets" . | nindent 6 }} + {{- end }} + containers: + - name: create + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + image: {{ include "hami.scheduler.patch.new.image" . }} + imagePullPolicy: {{ .Values.scheduler.patch.imageNew.pullPolicy }} + {{- else }} + image: {{ include "hami.scheduler.patch.image" . }} + imagePullPolicy: {{ .Values.scheduler.patch.image.pullPolicy }} + {{- end }} + args: + - create + - --cert-name=tls.crt + - --key-name=tls.key + {{- if .Values.scheduler.admissionWebhook.customURL.enabled }} + - --host={{ printf "%s.%s.svc,127.0.0.1,%s" (include "hami-vgpu.scheduler" .) (include "hami-vgpu.namespace" .) .Values.scheduler.admissionWebhook.customURL.host}} + {{- else }} + - --host={{ printf "%s.%s.svc,127.0.0.1" (include "hami-vgpu.scheduler" .) (include "hami-vgpu.namespace" .) }} + {{- end }} + - --namespace={{ include "hami-vgpu.namespace" . }} + - --secret-name={{ include "hami-vgpu.scheduler.tls" . }} + restartPolicy: OnFailure + serviceAccountName: {{ include "hami-vgpu.fullname" . }}-admission + {{- if .Values.scheduler.patch.nodeSelector }} + nodeSelector: {{ toYaml .Values.scheduler.patch.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.scheduler.patch.tolerations }} + tolerations: {{ toYaml .Values.scheduler.patch.tolerations | nindent 8 }} + {{- end }} + securityContext: + runAsNonRoot: true + runAsUser: {{ .Values.scheduler.patch.runAsUser }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml new file mode 100644 index 00000000..ce52042c --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml @@ -0,0 +1,63 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission-patch + namespace: {{ include "hami-vgpu.namespace" . }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission-patch + {{- if .Values.scheduler.patch.podAnnotations }} + annotations: {{ toYaml .Values.scheduler.patch.podAnnotations | nindent 8 }} + {{- end }} + labels: + {{- include "hami-vgpu.labels" . | nindent 8 }} + app.kubernetes.io/component: admission-webhook + hami.io/webhook: ignore + spec: + {{- if .Values.scheduler.patch.priorityClassName }} + priorityClassName: {{ .Values.scheduler.patch.priorityClassName }} + {{- end }} + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + {{- include "hami.scheduler.patch.new.imagePullSecrets" . | nindent 6 }} + {{- else }} + {{- include "hami.scheduler.patch.imagePullSecrets" . | nindent 6 }} + {{- end }} + containers: + - name: patch + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + image: {{ include "hami.scheduler.patch.new.image" . }} + imagePullPolicy: {{ .Values.scheduler.patch.imageNew.pullPolicy }} + {{- else }} + image: {{ include "hami.scheduler.patch.image" . }} + imagePullPolicy: {{ .Values.scheduler.patch.image.pullPolicy }} + {{- end }} + args: + - patch + - --webhook-name={{ include "hami-vgpu.scheduler.webhook" . }} + - --namespace={{ include "hami-vgpu.namespace" . }} + - --patch-validating=false + - --secret-name={{ include "hami-vgpu.scheduler.tls" . }} + restartPolicy: OnFailure + serviceAccountName: {{ include "hami-vgpu.fullname" . }}-admission + {{- if .Values.scheduler.patch.nodeSelector }} + nodeSelector: {{ toYaml .Values.scheduler.patch.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.scheduler.patch.tolerations }} + tolerations: {{ toYaml .Values.scheduler.patch.tolerations | nindent 8 }} + {{- end }} + securityContext: + runAsNonRoot: true + runAsUser: {{ .Values.scheduler.patch.runAsUser }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml new file mode 100644 index 00000000..56682f8e --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission + namespace: {{ include "hami-vgpu.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml new file mode 100644 index 00000000..7239b128 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission + namespace: {{ include "hami-vgpu.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "hami-vgpu.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.fullname" . }}-admission + namespace: {{ include "hami-vgpu.namespace" . }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml new file mode 100644 index 00000000..857c6a8d --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission + namespace: {{ include "hami-vgpu.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/role.yaml b/packages/system/hami/charts/hami/templates/scheduler/role.yaml new file mode 100644 index 00000000..5f7d7e44 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/role.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create", "list", "watch", "get", "update", "patch"] diff --git a/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml new file mode 100644 index 00000000..96e175a1 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml @@ -0,0 +1,31 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "hami-vgpu.scheduler" . }} +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +--- +{{- if .Values.mockDevicePlugin.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.mock-device-plugin" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "hami-vgpu.mock-device-plugin" . }} +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.mock-device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/scheduler/service.yaml b/packages/system/hami/charts/hami/templates/scheduler/service.yaml new file mode 100644 index 00000000..d7538fed --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/service.yaml @@ -0,0 +1,34 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} + {{- if .Values.scheduler.service.labels }} + {{ toYaml .Values.scheduler.service.labels | indent 4 }} + {{- end }} + {{- if .Values.scheduler.service.annotations }} + annotations: {{ toYaml .Values.scheduler.service.annotations | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.scheduler.service.type | default "NodePort" }} # Default type is NodePort + ports: + - name: http + port: {{ .Values.scheduler.service.httpPort | default 443 }} # Default HTTP port is 443 + targetPort: {{ .Values.scheduler.service.httpTargetPort | default 443 }} + {{- if eq (.Values.scheduler.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort + nodePort: {{ .Values.scheduler.service.schedulerPort | default 31998 }} + {{- end }} + protocol: TCP + - name: monitor + port: {{ .Values.scheduler.service.monitorPort | default 31993 }} # Default monitoring port is 31993 + targetPort: {{ .Values.scheduler.service.monitorTargetPort | default 9395 }} + {{- if eq (.Values.scheduler.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort + nodePort: {{ .Values.scheduler.service.monitorPort | default 31993 }} + {{- end }} + protocol: TCP + selector: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.selectorLabels" . | nindent 4 }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml b/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml new file mode 100644 index 00000000..0435c003 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +--- +{{- if .Values.mockDevicePlugin.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "hami-vgpu.mock-device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml b/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml new file mode 100644 index 00000000..db9f8029 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml @@ -0,0 +1,57 @@ +{{- if .Values.scheduler.admissionWebhook.enabled -}} +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + {{- if .Values.scheduler.certManager.enabled }} + annotations: + cert-manager.io/inject-ca-from: {{ include "hami-vgpu.namespace" . }}/{{ include "hami-vgpu.scheduler" . }}-serving-cert + {{- end }} + name: {{ include "hami-vgpu.scheduler.webhook" . }} +webhooks: + - admissionReviewVersions: + - v1beta1 + clientConfig: + {{- if .Values.scheduler.admissionWebhook.customURL.enabled }} + url: https://{{ .Values.scheduler.admissionWebhook.customURL.host}}:{{.Values.scheduler.admissionWebhook.customURL.port}}{{.Values.scheduler.admissionWebhook.customURL.path}} + {{- else }} + service: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + path: /webhook + port: {{ .Values.scheduler.service.httpPort }} + {{- end }} + failurePolicy: {{ .Values.scheduler.admissionWebhook.failurePolicy }} + matchPolicy: Equivalent + name: vgpu.hami.io + namespaceSelector: + matchExpressions: + - key: hami.io/webhook + operator: NotIn + values: + - ignore + {{- if .Values.scheduler.admissionWebhook.whitelistNamespaces }} + - key: kubernetes.io/metadata.name + operator: NotIn + values: + {{- toYaml .Values.scheduler.admissionWebhook.whitelistNamespaces | nindent 10 }} + {{- end }} + objectSelector: + matchExpressions: + - key: hami.io/webhook + operator: NotIn + values: + - ignore + reinvocationPolicy: {{ .Values.scheduler.admissionWebhook.reinvocationPolicy }} + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - pods + scope: '*' + sideEffects: None + timeoutSeconds: 10 +{{- end }} diff --git a/packages/system/hami/charts/hami/values.yaml b/packages/system/hami/charts/hami/values.yaml new file mode 100644 index 00000000..01caca0b --- /dev/null +++ b/packages/system/hami/charts/hami/values.yaml @@ -0,0 +1,455 @@ +## @section Global configuration +## @param global.imageRegistry Global Docker image registry +## @param global.imagePullSecrets Global Docker image pull secrets +global: + ## @param global.imageRegistry Global Docker image registry + imageRegistry: "" + ## E.g. + ## imagePullSecrets: + ## - myRegistryKeySecretName + ## @param global.imagePullSecrets Global Docker image pull secrets + imagePullSecrets: [] + imageTag: "v2.8.1" + gpuHookPath: /usr/local + labels: {} + annotations: {} + managedNodeSelectorEnable: false + managedNodeSelector: + usage: "gpu" + +nameOverride: "" +fullnameOverride: "" +namespaceOverride: "" + +#Nvidia GPU Parameters +resourceName: "nvidia.com/gpu" +resourceMem: "nvidia.com/gpumem" +resourceMemPercentage: "nvidia.com/gpumem-percentage" +resourceCores: "nvidia.com/gpucores" +resourcePriority: "nvidia.com/priority" + +#MLU Parameters +mluResourceName: "cambricon.com/vmlu" +mluResourceMem: "cambricon.com/mlu.smlu.vmemory" +mluResourceCores: "cambricon.com/mlu.smlu.vcore" + +#Hygon DCU Parameters +dcuResourceName: "hygon.com/dcunum" +dcuResourceMem: "hygon.com/dcumem" +dcuResourceCores: "hygon.com/dcucores" + +#Metax sGPU Parameters +metaxResourceName: "metax-tech.com/sgpu" +metaxResourceCore: "metax-tech.com/vcore" +metaxResourceMem: "metax-tech.com/vmemory" +metaxsGPUTopologyAware: "false" + +#Enflame VGCU Parameters +enflameResourceNameVGCU: "enflame.com/vgcu" +enflameResourceNameVGCUPercentage: "enflame.com/vgcu-percentage" + +#Kunlun XPU Parameters +kunlunResourceName: "kunlunxin.com/xpu" +kunlunResourceVCountName: "kunlunxin.com/vxpu" +kunlunResourceVMemoryName: "kunlunxin.com/vxpu-memory" + +schedulerName: "hami-scheduler" + +scheduler: + # @param nodeName defines the node name and the nvidia-vgpu-scheduler-scheduler will schedule to the node. + # if we install the nvidia-vgpu-scheduler-scheduler as default scheduler, we need to remove the k8s default + # scheduler pod from the cluster first, we must specify node name to skip the schedule workflow. + nodeName: "" + #nodeLabelSelector: + # "gpu": "on" + overwriteEnv: "false" + defaultSchedulerPolicy: + nodeSchedulerPolicy: binpack + gpuSchedulerPolicy: spread + metricsBindAddress: ":9395" + # If set to false, When Pod.Spec.SchedulerName equals to the const DefaultSchedulerName in k8s.io/api/core/v1 package, webhook will not overwrite it + forceOverwriteDefaultScheduler: true + livenessProbe: false + leaderElect: true + # when leaderElect is true, replicas is available, otherwise replicas is 1. + replicas: 1 + kubeScheduler: + # @param enabled indicate whether to run kube-scheduler container in the scheduler pod, it's true by default. + enabled: true + ## @param image.registry kube scheduler image registry + ## @param image.repository kube scheduler image repository + ## @param image.tag kube scheduler image tag (immutable tags are recommended) + ## @param image.pullPolicy kube scheduler image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + image: + registry: "registry.cn-hangzhou.aliyuncs.com" + repository: "google_containers/kube-scheduler" + tag: "" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + resources: {} + # If you do want to specify resources, uncomment the following lines, adjust them as necessary. + # and remove the curly braces after 'resources:'. +# limits: +# cpu: 1000m +# memory: 1000Mi +# requests: +# cpu: 100m +# memory: 100Mi + extraNewArgs: + - --config=/config/config.yaml + - -v=4 + extraArgs: + - --policy-config-file=/config/config.json + - -v=4 + extender: + ## @param image.registry scheduler extender image registry + ## @param image.repository scheduler extender image repository + ## @param image.tag scheduler extender image tag (immutable tags are recommended) + ## @param image.pullPolicy scheduler extender image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + image: + registry: "docker.io" + repository: "projecthami/hami" + tag: "" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + resources: {} + # If you do want to specify resources, uncomment the following lines, adjust them as necessary, + # and remove the curly braces after 'resources:'. +# limits: +# cpu: 1000m +# memory: 1000Mi +# requests: +# cpu: 100m +# memory: 100Mi + extraArgs: + - --debug + - -v=4 + nodeLockExpire: "5m" + podAnnotations: {} + tolerations: [] + #serviceAccountName: "hami-vgpu-scheduler-sa" + admissionWebhook: + # If set to false, the admission webhook is not installed and any pods that should use HAMi must be + # configured to use the right 'schedulerName' and other device-specific configurations. + enabled: true + customURL: + enabled: false + # must be an endpoint using https. + # should generate host certs here + host: 127.0.0.1 # hostname or ip, can be your node'IP if you want to use https://:/ + port: 31998 + path: /webhook + whitelistNamespaces: + # Specify the namespaces that the webhook will not be applied to. + # - default + # - kube-system + # - istio-system + reinvocationPolicy: Never + failurePolicy: Ignore + ## TLS Certificate Option 1: Use cert-manager to generate self-signed certificate. + ## If enabled, always takes precedence over options 2. + certManager: + enabled: false + ## TLS Certificate Option 2: Use kube-webhook-certgen to generate self-signed certificate. + ## If true and certManager.enabled is false, Helm will automatically create a self-signed cert and secret for you. + patch: + enabled: true + ## @param image.registry scheduler extender image registry + ## @param image.repository scheduler extender image repository + ## @param image.tag scheduler extender image tag (immutable tags are recommended) + ## @param image.pullPolicy scheduler extender image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + image: + registry: "docker.io" + repository: "jettech/kube-webhook-certgen" + tag: "v1.5.2" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## @param image.registry scheduler extender image registry + ## @param image.repository scheduler extender image repository + ## @param image.tag scheduler extender image tag (immutable tags are recommended) + ## @param image.pullPolicy scheduler extender image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + imageNew: + registry: "docker.io" + repository: "liangjw/kube-webhook-certgen" + tag: "v1.1.1" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + priorityClassName: "" + podAnnotations: {} + nodeSelector: {} + tolerations: [] + runAsUser: 2000 + service: + type: NodePort # Default type is NodePort, can be changed to ClusterIP + httpPort: 443 # HTTP port + schedulerPort: 31998 # NodePort for HTTP + monitorPort: 31993 # Monitoring port + monitorTargetPort: 9395 + httpTargetPort: 443 + labels: {} + annotations: {} + +devicePlugin: + enabled: true + gpuOperatorToolkitReady: + enabled: false + hostPath: "/run/nvidia/validations" + ## @param image.registry devicePlugin image registry + ## @param image.repository devicePlugin image repository + ## @param image.tag devicePlugin image tag (immutable tags are recommended) + ## @param image.pullPolicy devicePlugin image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + image: + registry: "docker.io" + repository: "projecthami/hami" + tag: "" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + monitor: + ## @param image.registry monitor image registry + ## @param image.repository monitor image repository + ## @param image.tag monitor image tag (immutable tags are recommended) + ## @param image.pullPolicy monitor image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + image: + registry: "docker.io" + repository: "projecthami/hami" + tag: "" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ctrPath: /usr/local/vgpu/containers + resyncInterval: "5m" + extraArgs: + - -v=4 + extraEnvs: {} + resources: {} + # If you do want to specify resources, uncomment the following lines, adjust them as necessary. + # and remove the curly braces after 'resources:'. +# limits: +# cpu: 1000m +# memory: 1000Mi +# requests: +# cpu: 100m +# memory: 100Mi + + deviceSplitCount: 10 + deviceMemoryScaling: 1 + deviceCoreScaling: 1 + # Node configuration for device plugin, Priority: externalConfigName > config > default config + nodeConfiguration: + # If you want to use a custom config.json, you can set the content here. + # If this is set, it will override the default config.json(An example is as follows). + config: | + { + "nodeconfig": [ + { + "name": "your-node-name", + "operatingmode": "hami-core", + "devicememoryscaling": 1, + "devicesplitcount": 10, + "migstrategy": "none", + "filterdevices": { + "uuid": [], + "index": [] + } + } + ] + } + # If you want to use an existing ConfigMap, you can set the name here. + # If this is set, the chart will not create the ConfigMap and will use the existing one. + externalConfigName: "" + # The runtime class name to be used by the device plugin, and added to the pod.spec.runtimeClassName of applications utilizing NVIDIA GPUs + runtimeClassName: "" + # Whether to create runtime class, name comes from runtimeClassName when it is set + createRuntimeClass: false + migStrategy: "none" + disablecorelimit: "false" + passDeviceSpecsEnabled: false + deviceListStrategy: "envvar" + nvidiaHookPath: null + nvidiaDriverRoot: null + gdrcopyEnabled: null + gdsEnabled: null + mofedEnabled: null + + extraArgs: + - -v=4 + extraEnvs: {} + service: + type: NodePort # Default type is NodePort, can be changed to ClusterIP + httpPort: 31992 + labels: {} + annotations: {} + + pluginPath: /var/lib/kubelet/device-plugins + libPath: /usr/local/vgpu + + podAnnotations: {} + nvidiaNodeSelector: + gpu: "on" + tolerations: [] + # The updateStrategy for DevicePlugin DaemonSet. + # If you want to update the DaemonSet by manual, set type as "OnDelete". + # We recommend use OnDelete update strategy because DevicePlugin pod restart will cause business pod restart, this behavior is destructive. + # Otherwise, you can use RollingUpdate update strategy to rolling update DevicePlugin pod. + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + + resources: {} + # If you do want to specify resources, uncomment the following lines, adjust them as necessary. + # and remove the curly braces after 'resources:'. +# limits: +# cpu: 1000m +# memory: 1000Mi +# requests: +# cpu: 100m +# memory: 100Mi + +mockDevicePlugin: + enabled: false + image: + registry: "docker.io" + repository: "projecthami/mock-device-plugin" + tag: "1.0.1" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] +devices: + amd: + customresources: + - amd.com/gpu + - amd.com/gpu-memory + awsneuron: + customresources: + - aws.amazon.com/neuron + - aws.amazon.com/neuroncore + kunlun: + enabled: true + customresources: + - kunlunxin.com/xpu + - kunlunxin.com/vxpu + - kunlunxin.com/vxpu-memory + enflame: + enabled: true + customresources: + - enflame.com/vgcu + - enflame.com/vgcu-percentage + - enflame.com/gcu + mthreads: + enabled: true + customresources: + - mthreads.com/vgpu + nvidia: + gpuCorePolicy: default + libCudaLogLevel: 1 + ascend: + enabled: false + image: "" + imagePullPolicy: IfNotPresent + extraArgs: [] + nodeSelector: + ascend: "on" + tolerations: [] + customresources: + - huawei.com/Ascend910A + - huawei.com/Ascend910A-memory + - huawei.com/Ascend910B2 + - huawei.com/Ascend910B2-memory + - huawei.com/Ascend910B3 + - huawei.com/Ascend910B3-memory + - huawei.com/Ascend910B4 + - huawei.com/Ascend910B4-memory + - huawei.com/Ascend910B4-1 + - huawei.com/Ascend910B4-1-memory + - huawei.com/Ascend310P + - huawei.com/Ascend310P-memory + iluvatar: + enabled: false + customresources: + - iluvatar.ai/BI-V100-vgpu + - iluvatar.ai/BI-V100.vCore + - iluvatar.ai/BI-V100.vMem + - iluvatar.ai/BI-V150-vgpu + - iluvatar.ai/BI-V150.vCore + - iluvatar.ai/BI-V150.vMem + - iluvatar.ai/MR-V100-vgpu + - iluvatar.ai/MR-V100.vCore + - iluvatar.ai/MR-V100.vMem + - iluvatar.ai/MR-V50-vgpu + - iluvatar.ai/MR-V50.vCore + - iluvatar.ai/MR-V50.vMem diff --git a/packages/system/hami/values.yaml b/packages/system/hami/values.yaml new file mode 100644 index 00000000..76e8d631 --- /dev/null +++ b/packages/system/hami/values.yaml @@ -0,0 +1,17 @@ +hami: + scheduler: + kubeScheduler: + image: + registry: registry.k8s.io + repository: kube-scheduler + devicePlugin: + runtimeClassName: nvidia + updateStrategy: + type: RollingUpdate + # See https://github.com/Project-HAMi/HAMi/blob/v2.8.1/deployments/charts/hami/values.yaml + # Each entry: {"name": "node-name", "operatingmode": "hami-core", "devicesplitcount": N, ...} + nodeConfiguration: + config: | + { + "nodeconfig": [] + } diff --git a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml index 1631c86e..f81eaff0 100644 --- a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml +++ b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml @@ -22,7 +22,7 @@ spec: singular: kubernetes plural: kuberneteses openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"ephemeralStorage":"20Gi","gpus":[],"instanceType":"u1.medium","maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"]}},"additionalProperties":{"type":"object","required":["ephemeralStorage","instanceType","maxReplicas","minReplicas","resources"],"properties":{"ephemeralStorage":{"description":"Ephemeral storage size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string"}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium"},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `.` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","ingressNginx","monitoringAgents","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"large","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"images":{"description":"Optional image overrides for air-gapped or rate-limited registries.","type":"object","default":{},"properties":{"waitForKubeconfig":{"description":"Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.","type":"string","default":""}}}}} + {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"ephemeralStorage":"20Gi","gpus":[],"instanceType":"u1.medium","maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"]}},"additionalProperties":{"type":"object","required":["ephemeralStorage","instanceType","maxReplicas","minReplicas","resources"],"properties":{"ephemeralStorage":{"description":"Ephemeral storage size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string"}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium"},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `.` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","hami","ingressNginx","monitoringAgents","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"hami":{"description":"HAMi GPU virtualization middleware.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable HAMi (requires GPU Operator).","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"large","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"images":{"description":"Optional image overrides for air-gapped or rate-limited registries.","type":"object","default":{},"properties":{"waitForKubeconfig":{"description":"Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.","type":"string","default":""}}}}} release: prefix: kubernetes- labels: @@ -40,7 +40,7 @@ spec: tags: - compute icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODFfMjg0NSkiLz4KPHBhdGggZD0iTTcxLjk5NjggMTlDNzAuMzAzOSAxOS4wMDAyIDY4LjkzMTIgMjAuNTMyMiA2OC45MzE0IDIyLjQyMjFDNjguOTMxNCAyMi40NTExIDY4LjkzNzMgMjIuNDc4OCA2OC45Mzc5IDIyLjUwNzZDNjguOTM1NCAyMi43NjQ0IDY4LjkyMzEgMjMuMDczNyA2OC45MzE0IDIzLjI5NzNDNjguOTcxNyAyNC4zODczIDY5LjIwODIgMjUuMjIxNiA2OS4zNTA2IDI2LjIyNThDNjkuNjA4NCAyOC4zNzUyIDY5LjgyNDUgMzAuMTU2OSA2OS42OTEyIDMxLjgxM0M2OS41NjE1IDMyLjQzNzUgNjkuMTAzNyAzMy4wMDg2IDY4LjY5NTYgMzMuNDA1Nkw2OC42MjM1IDM0LjcwODZDNjYuNzgzOSAzNC44NjE3IDY0LjkzMTkgMzUuMTQyMSA2My4wODIxIDM1LjU2NDFDNTUuMTIyNiAzNy4zNzk4IDQ4LjI2OTUgNDEuNDk5MSA0My4wNTIgNDcuMDYwOUM0Mi43MTM0IDQ2LjgyODggNDIuMTIxMSA0Ni40MDE5IDQxLjk0NSA0Ni4yNzEyQzQxLjM5NzcgNDYuMzQ1NCA0MC44NDQ1IDQ2LjUxNTEgNDAuMTI0MSA0Ni4wOTM1QzM4Ljc1MjIgNDUuMTY1NyAzNy41MDI4IDQzLjg4NTEgMzUuOTkxIDQyLjM0MjRDMzUuMjk4MiA0MS42MDQ0IDM0Ljc5NjYgNDAuOTAxOCAzMy45NzM1IDQwLjE5MDRDMzMuNzg2NiA0MC4wMjg5IDMzLjUwMTQgMzkuODEwNCAzMy4yOTIzIDM5LjY0NDJDMzIuNjQ4OSAzOS4xMjg4IDMxLjg5IDM4Ljg2IDMxLjE1NyAzOC44MzQ4QzMwLjIxNDcgMzguODAyNCAyOS4zMDc1IDM5LjE3MjUgMjguNzEzOCAzOS45MjA2QzI3LjY1ODQgNDEuMjUwNiAyNy45OTYzIDQzLjI4MzMgMjkuNDY3MSA0NC40NjE0QzI5LjQ4MiA0NC40NzM0IDI5LjQ5NzkgNDQuNDgyNyAyOS41MTI5IDQ0LjQ5NDNDMjkuNzE1IDQ0LjY1ODkgMjkuOTYyNSA0NC44Njk4IDMwLjE0ODMgNDUuMDA3NkMzMS4wMjE3IDQ1LjY1NTUgMzEuODE5NSA0NS45ODcyIDMyLjY4OTcgNDYuNTAxNUMzNC41MjMxIDQ3LjYzOTEgMzYuMDQzIDQ4LjU4MjMgMzcuMjQ4NiA0OS43MTk2QzM3LjcxOTQgNTAuMjIzNyAzNy44MDE2IDUxLjExMjIgMzcuODY0MyA1MS40OTY0TDM4Ljg0NjggNTIuMzc4MkMzMy41ODcyIDYwLjMzMDggMzEuMTUzIDcwLjE1MzkgMzIuNTkxNSA4MC4xNjI3TDMxLjMwNzcgODAuNTM3OEMzMC45NjkzIDgwLjk3NjggMzAuNDkxMiA4MS42Njc2IDI5Ljk5MTEgODEuODczOEMyOC40MTM4IDgyLjM3MjkgMjYuNjM4NyA4Mi41NTYyIDI0LjQ5NTYgODIuNzgxOUMyMy40ODk0IDgyLjg2NiAyMi42MjEzIDgyLjgxNTggMjEuNTU0NiA4My4wMTg4QzIxLjMxOTggODMuMDYzNSAyMC45OTI3IDgzLjE0OTEgMjAuNzM1OCA4My4yMDk3QzIwLjcyNjkgODMuMjExNiAyMC43MTg2IDgzLjIxNDIgMjAuNzA5NiA4My4yMTYyQzIwLjY5NTYgODMuMjE5NSAyMC42NzcyIDgzLjIyNjMgMjAuNjYzOCA4My4yMjk0QzE4Ljg1NyA4My42NjggMTcuNjk2MyA4NS4zMzY1IDE4LjA2OTkgODYuOTgwNUMxOC40NDM3IDg4LjYyNDggMjAuMjA4NiA4OS42MjQ4IDIyLjAyNjIgODkuMjMxMkMyMi4wMzkzIDg5LjIyODIgMjIuMDU4NCA4OS4yMjc3IDIyLjA3MiA4OS4yMjQ2QzIyLjA5MjYgODkuMjE5OSAyMi4xMTA2IDg5LjIwOTkgMjIuMTMxIDg5LjIwNDlDMjIuMzg0NCA4OS4xNDkgMjIuNzAxOSA4OS4wODY4IDIyLjkyMzYgODkuMDI3MkMyMy45NzIzIDg4Ljc0NTEgMjQuNzMxOCA4OC4zMzA2IDI1LjY3NDYgODcuOTY3N0MyNy43MDI5IDg3LjIzNjggMjkuMzgyOCA4Ni42MjYyIDMxLjAxOTUgODYuMzg4M0MzMS43MDMgODYuMzM0NSAzMi40MjMyIDg2LjgxMiAzMi43ODE0IDg3LjAxMzRMMzQuMTE3NyA4Ni43ODMxQzM3LjE5MjYgOTYuMzYxMyA0My42MzY2IDEwNC4xMDMgNTEuNzk2MyAxMDguOTYxTDUxLjIzOTYgMTEwLjMwM0M1MS40NDAzIDExMC44MjQgNTEuNjYxNiAxMTEuNTMgNTEuNTEyMSAxMTIuMDQ1QzUwLjkxNzEgMTEzLjU5NSA0OS44OTggMTE1LjIzMSA0OC43Mzc0IDExNy4wNTVDNDguMTc1NSAxMTcuODk4IDQ3LjYwMDQgMTE4LjU1MiA0Ny4wOTM0IDExOS41MTZDNDYuOTcyIDExOS43NDcgNDYuODE3NSAxMjAuMTAyIDQ2LjcwMDQgMTIwLjM0NkM0NS45MTI1IDEyMi4wMzkgNDYuNDkwNCAxMjMuOTkgNDguMDAzOCAxMjQuNzIyQzQ5LjUyNjggMTI1LjQ1OSA1MS40MTcxIDEyNC42ODIgNTIuMjM1MiAxMjIuOTg1QzUyLjIzNjQgMTIyLjk4MiA1Mi4yNDA2IDEyMi45OCA1Mi4yNDE3IDEyMi45NzhDNTIuMjQyNiAxMjIuOTc2IDUyLjI0MDkgMTIyLjk3MyA1Mi4yNDE3IDEyMi45NzFDNTIuMzU4MiAxMjIuNzMxIDUyLjUyMzMgMTIyLjQxNSA1Mi42MjE2IDEyMi4xODhDNTMuMDU2IDEyMS4xODkgNTMuMjAwNSAxMjAuMzMyIDUzLjUwNTkgMTE5LjM2NUM1NC4zMTcgMTE3LjMxOCA1NC43NjI2IDExNS4xNyA1NS44NzkxIDExMy44MzJDNTYuMTg0OSAxMTMuNDY2IDU2LjY4MzMgMTEzLjMyNSA1Ny4yMDAxIDExMy4xODZMNTcuODk0NCAxMTEuOTIyQzY1LjAwOCAxMTQuNjY1IDcyLjk3MDUgMTE1LjQwMiA4MC45MjQ1IDExMy41ODdDODIuNzM5MSAxMTMuMTczIDg0LjQ5MDggMTEyLjYzNyA4Ni4xODQzIDExMS45OTRDODYuMzc5NCAxMTIuMzQyIDg2Ljc0MiAxMTMuMDExIDg2LjgzOTMgMTEzLjE3OUM4Ny4zNjQ0IDExMy4zNTEgODcuOTM3NyAxMTMuNDM5IDg4LjQwNDcgMTE0LjEzM0M4OS4yNDAxIDExNS41NjcgODkuODExNCAxMTcuMjYzIDkwLjUwNzMgMTE5LjMxMkM5MC44MTI4IDEyMC4yNzkgOTAuOTYzOCAxMjEuMTM2IDkxLjM5ODEgMTIyLjEzNkM5MS40OTcxIDEyMi4zNjMgOTEuNjYxNCAxMjIuNjg0IDkxLjc3OCAxMjIuOTI1QzkyLjU5NDQgMTI0LjYyOCA5NC40OTA3IDEyNS40MDcgOTYuMDE1OSAxMjQuNjY5Qzk3LjUyOTIgMTIzLjkzNyA5OC4xMDc3IDEyMS45ODYgOTcuMzE5NCAxMjAuMjkzQzk3LjIwMjMgMTIwLjA0OSA5Ny4wNDEyIDExOS42OTUgOTYuOTE5OCAxMTkuNDY0Qzk2LjQxMjcgMTE4LjQ5OSA5NS44Mzc3IDExNy44NTIgOTUuMjc1OCAxMTcuMDA5Qzk0LjExNTIgMTE1LjE4NSA5My4xNTI2IDExMy42NyA5Mi41NTc1IDExMi4xMkM5Mi4zMDg3IDExMS4zMiA5Mi41OTk1IDExMC44MjMgOTIuNzkzMyAxMTAuMzAzQzkyLjY3NzIgMTEwLjE3IDkyLjQyODggMTA5LjQxNCA5Mi4yODI0IDEwOS4wNTlDMTAwLjc2MiAxMDQuMDI5IDEwNy4wMTcgOTUuOTk4NSAxMDkuOTU1IDg2LjcyMzlDMTEwLjM1MSA4Ni43ODY1IDExMS4wNDEgODYuOTA5MSAxMTEuMjY1IDg2Ljk1NDJDMTExLjcyNiA4Ni42NDg3IDExMi4xNDkgODYuMjUwMSAxMTIuOTgxIDg2LjMxNTlDMTE0LjYxNyA4Ni41NTM3IDExNi4yOTcgODcuMTY0NSAxMTguMzI2IDg3Ljg5NTNDMTE5LjI2OCA4OC4yNTgxIDEyMC4wMjggODguNjc5MyAxMjEuMDc3IDg4Ljk2MTRDMTIxLjI5OCA4OS4wMjEgMTIxLjYxNiA4OS4wNzY2IDEyMS44NjkgODkuMTMyNUMxMjEuODg5IDg5LjEzNzUgMTIxLjkwOCA4OS4xNDc1IDEyMS45MjggODkuMTUyMkMxMjEuOTQyIDg5LjE1NTMgMTIxLjk2MSA4OS4xNTU4IDEyMS45NzQgODkuMTU4OEMxMjMuNzkyIDg5LjU1MiAxMjUuNTU3IDg4LjU1MjYgMTI1LjkzIDg2LjkwODFDMTI2LjMwMyA4NS4yNjQxIDEyNS4xNDMgODMuNTk1MiAxMjMuMzM2IDgzLjE1N0MxMjMuMDc0IDgzLjA5NyAxMjIuNzAxIDgyLjk5NSAxMjIuNDQ2IDgyLjk0NjVDMTIxLjM3OSA4Mi43NDM1IDEyMC41MTEgODIuNzkzNSAxMTkuNTA1IDgyLjcwOTVDMTE3LjM2MSA4Mi40ODM5IDExNS41ODYgODIuMzAwNCAxMTQuMDA5IDgxLjgwMTRDMTEzLjM2NiA4MS41NTA3IDExMi45MDggODAuNzgxOSAxMTIuNjg2IDgwLjQ2NTVMMTExLjQ0OCA4MC4xMDM1QzExMi4wOSA3NS40MzggMTExLjkxNyA3MC41ODI1IDExMC44MDYgNjUuNzI0M0MxMDkuNjg1IDYwLjgyMDggMTA3LjcwNCA1Ni4zMzYxIDEwNS4wNjIgNTIuMzg0OEMxMDUuMzc5IDUyLjA5NDggMTA1Ljk3OSA1MS41NjEyIDEwNi4xNDkgNTEuNDA0M0MxMDYuMTk5IDUwLjg1MTcgMTA2LjE1NiA1MC4yNzIyIDEwNi43MjUgNDkuNjYwM0MxMDcuOTMxIDQ4LjUyMyAxMDkuNDUxIDQ3LjU3OTkgMTExLjI4NCA0Ni40NDIzQzExMi4xNTQgNDUuOTI3OSAxMTIuOTU5IDQ1LjU5NjQgMTEzLjgzMiA0NC45NDg0QzExNC4wMyA0NC44MDE5IDExNC4yOTkgNDQuNTY5OSAxMTQuNTA3IDQ0LjQwMjJDMTE1Ljk3NyA0My4yMjM3IDExNi4zMTYgNDEuMTkxMSAxMTUuMjYgMzkuODYxNEMxMTQuMjA0IDM4LjUzMTcgMTEyLjE1OSAzOC40MDY1IDExMC42ODggMzkuNTg1QzExMC40NzkgMzkuNzUxNiAxMTAuMTk1IDM5Ljk2ODggMTEwLjAwNyA0MC4xMzEyQzEwOS4xODQgNDAuODQyNiAxMDguNjc2IDQxLjU0NTIgMTA3Ljk4MyA0Mi4yODMyQzEwNi40NzEgNDMuODI1OSAxMDUuMjIyIDQ1LjExMyAxMDMuODUgNDYuMDQwOUMxMDMuMjU1IDQ2LjM4ODUgMTAyLjM4NSA0Ni4yNjgyIDEwMS45OSA0Ni4yNDQ5TDEwMC44MjQgNDcuMDgwNkM5NC4xNzUzIDQwLjA3NjMgODUuMTIzNSAzNS41OTgyIDc1LjM3NjYgMzQuNzI4M0M3NS4zNDk0IDM0LjMxNzkgNzUuMzEzNyAzMy41NzYxIDc1LjMwNDYgMzMuMzUyOUM3NC45MDU2IDMyLjk2OTMgNzQuNDIzNSAzMi42NDE4IDc0LjMwMjQgMzEuODEzQzc0LjE2OTEgMzAuMTU2OSA3NC4zOTE3IDI4LjM3NTIgNzQuNjQ5NiAyNi4yMjU4Qzc0Ljc5MTkgMjUuMjIxNiA3NS4wMjg0IDI0LjM4NzMgNzUuMDY4OCAyMy4yOTczQzc1LjA3OCAyMy4wNDk1IDc1LjA2MzIgMjIuNjkgNzUuMDYyMiAyMi40MjIxQzc1LjA2MiAyMC41MzIyIDczLjY4OTggMTguOTk5OCA3MS45OTY4IDE5Wk02OC4xNTg1IDQyLjg4ODZMNjcuMjQ4IDU5LjA0NDdMNjcuMTgyNSA1OS4wNzc2QzY3LjEyMTQgNjAuNTIyOSA2NS45Mzc1IDYxLjY3NyA2NC40ODM5IDYxLjY3N0M2My44ODg0IDYxLjY3NyA2My4zMzg4IDYxLjQ4NDkgNjIuODkyMiA2MS4xNTcxTDYyLjg2NiA2MS4xNzAzTDQ5LjY4MDcgNTEuNzc5NEM1My43MzMxIDQ3Ljc3NTkgNTguOTE2NCA0NC44MTcyIDY0Ljg5IDQzLjQ1NDZDNjUuOTgxMiA0My4yMDU2IDY3LjA3MTkgNDMuMDIwOSA2OC4xNTg1IDQyLjg4ODZaTTc1Ljg0MTcgNDIuODg4NkM4Mi44MTU5IDQzLjc1MDQgODkuMjY1NyA0Ni45MjMyIDk0LjIwODEgNTEuNzg2TDgxLjEwOCA2MS4xMTc2TDgxLjA2MjEgNjEuMDk3OUM3OS44OTk0IDYxLjk1MTIgNzguMjYxMSA2MS43Mzk0IDc3LjM1NDggNjAuNTk3OEM3Ni45ODM1IDYwLjEzMDEgNzYuNzg4NyA1OS41ODAxIDc2Ljc2NTMgNTkuMDI0OUw3Ni43NTIyIDU5LjAxODRMNzUuODQxNyA0Mi44ODg2Wk00NC44OTkxIDU3LjgxNEw1Ni45MzgyIDY4LjYzM0w1Ni45MjUxIDY4LjY5ODhDNTguMDExNyA2OS42NDc5IDU4LjE3MiA3MS4yOTQ5IDU3LjI2NTcgNzIuNDM2OEM1Ni44OTQ0IDcyLjkwNDUgNTYuMzk3NSA3My4yMTgyIDU1Ljg2MzkgNzMuMzY0N0w1NS44NTA4IDczLjQxNzNMNDAuNDE4OCA3Ny44OTIzQzM5LjYzMzQgNzAuNjc2NSA0MS4zMjYxIDYzLjY2MjEgNDQuODk5MSA1Ny44MTRaTTk5LjAwOTQgNTcuODIwNkMxMDAuNzk4IDYwLjczMzYgMTAyLjE1MyA2My45ODcxIDEwMi45NTkgNjcuNTE0M0MxMDMuNzU2IDcwLjk5OTEgMTAzLjk1NiA3NC40Nzc4IDEwMy42MjcgNzcuODM5N0w4OC4xMTY2IDczLjM1MTVMODguMTAzNSA3My4yODU3Qzg2LjcxNDUgNzIuOTA0MyA4NS44NjA5IDcxLjQ4NDggODYuMTg0MyA3MC4wNjExQzg2LjMxNjggNjkuNDc3OCA4Ni42MjQ5IDY4Ljk4NDQgODcuMDQyMyA2OC42MTk4TDg3LjAzNTggNjguNTg2OUw5OS4wMDk0IDU3LjgyMDZaTTY5LjUyNzQgNjkuNDY4OEg3NC40NTk2TDc3LjUyNTEgNzMuMzE4Nkw3Ni40MjQ3IDc4LjEyMjZMNzEuOTk2OCA4MC4yNjE0TDY3LjU1NTggNzguMTE2MUw2Ni40NTU0IDczLjMxMkw2OS41Mjc0IDY5LjQ2ODhaTTg1LjMzOTMgODIuNjQzN0M4NS41NDg5IDgyLjYzMzEgODUuNzU3NiA4Mi42NTIgODUuOTYxNiA4Mi42ODk4TDg1Ljk4NzggODIuNjU2OUwxMDEuOTUgODUuMzY4MkM5OS42MTQyIDkxLjk2MjQgOTUuMTQ0IDk3LjY3NSA4OS4xNzExIDEwMS40OThMODIuOTc0NyA4Ni40NjA2TDgyLjk5NDQgODYuNDM0M0M4Mi40MjUyIDg1LjEwNTUgODIuOTk0OCA4My41NDcyIDg0LjMwNDQgODIuOTEzNUM4NC42Mzk3IDgyLjc1MTMgODQuOTkgODIuNjYxNCA4NS4zMzkzIDgyLjY0MzdaTTU4LjUyOTggODIuNzA5NUM1OS43NDggODIuNzI2NyA2MC44NDA2IDgzLjU3NjEgNjEuMTIzNyA4NC44MjJDNjEuMjU2MiA4NS40MDUyIDYxLjE5MTcgODUuOTgzMSA2MC45NzMgODYuNDkzNUw2MS4wMTg5IDg2LjU1MjdMNTQuODg4IDEwMS40MzlDNDkuMTU1OSA5Ny43NDMyIDQ0LjU5MDQgOTIuMjA5OSA0Mi4xNDgxIDg1LjQyMDhMNTcuOTczMSA4Mi43MjI3TDU3Ljk5OTMgODIuNzU1NkM1OC4xNzYzIDgyLjcyMjkgNTguMzU1OCA4Mi43MDcxIDU4LjUyOTggODIuNzA5NVpNNzEuODk4NiA4OS4yMzEyQzcyLjMyMjkgODkuMjE1NSA3Mi43NTM0IDg5LjMwMyA3My4xNjI3IDg5LjUwMUM3My42OTkyIDg5Ljc2MDYgNzQuMTEzNiA5MC4xNjkyIDc0LjM3NDUgOTAuNjU5Mkg3NC40MzM0TDgyLjIzNDYgMTA0LjgyMUM4MS4yMjIxIDEwNS4xNjIgODAuMTgxMyAxMDUuNDU0IDc5LjExNjcgMTA1LjY5N0M3My4xNTA1IDEwNy4wNTggNjcuMjAzMiAxMDYuNjQ1IDYxLjgxOCAxMDQuODAyTDY5LjU5OTUgOTAuNjY1OEg2OS42MTI2QzcwLjA3OTUgODkuNzg4OCA3MC45NjUgODkuMjY1NiA3MS44OTg2IDg5LjIzMTJaIiBmaWxsPSJ3aGl0ZSIgc3Ryb2tlPSJ3aGl0ZSIgc3Ryb2tlLXdpZHRoPSIwLjI1Ii8+CjxkZWZzPgo8bGluZWFyR3JhZGllbnQgaWQ9InBhaW50MF9saW5lYXJfNjgxXzI4NDUiIHgxPSIxMCIgeTE9IjE1LjUiIHgyPSIxNDQiIHkyPSIxMzEuNSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiPgo8c3RvcCBzdG9wLWNvbG9yPSIjNEQ4N0ZGIi8+CjxzdG9wIG9mZnNldD0iMSIgc3RvcC1jb2xvcj0iIzA1NDdEMCIvPgo8L2xpbmVhckdyYWRpZW50Pgo8L2RlZnM+Cjwvc3ZnPgo= - keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "nodeGroups"], ["spec", "nodeGroups", "md0"], ["spec", "nodeGroups", "md0", "minReplicas"], ["spec", "nodeGroups", "md0", "maxReplicas"], ["spec", "nodeGroups", "md0", "instanceType"], ["spec", "nodeGroups", "md0", "ephemeralStorage"], ["spec", "nodeGroups", "md0", "roles"], ["spec", "nodeGroups", "md0", "resources"], ["spec", "nodeGroups", "md0", "gpus"], ["spec", "version"], ["spec", "host"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "fluxcd"], ["spec", "addons", "fluxcd", "enabled"], ["spec", "addons", "fluxcd", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"], ["spec", "images"], ["spec", "images", "waitForKubeconfig"]] + keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "nodeGroups"], ["spec", "nodeGroups", "md0"], ["spec", "nodeGroups", "md0", "minReplicas"], ["spec", "nodeGroups", "md0", "maxReplicas"], ["spec", "nodeGroups", "md0", "instanceType"], ["spec", "nodeGroups", "md0", "ephemeralStorage"], ["spec", "nodeGroups", "md0", "roles"], ["spec", "nodeGroups", "md0", "resources"], ["spec", "nodeGroups", "md0", "gpus"], ["spec", "version"], ["spec", "host"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "hami"], ["spec", "addons", "hami", "enabled"], ["spec", "addons", "hami", "valuesOverride"], ["spec", "addons", "fluxcd"], ["spec", "addons", "fluxcd", "enabled"], ["spec", "addons", "fluxcd", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"], ["spec", "images"], ["spec", "images", "waitForKubeconfig"]] secrets: exclude: [] include: