diff --git a/api/apps/v1alpha1/kubernetes/types.go b/api/apps/v1alpha1/kubernetes/types.go index ea4a2b64..88a3c97e 100644 --- a/api/apps/v1alpha1/kubernetes/types.go +++ b/api/apps/v1alpha1/kubernetes/types.go @@ -69,6 +69,9 @@ type Addons struct { // NVIDIA GPU Operator. // +kubebuilder:default:={} GpuOperator GPUOperatorAddon `json:"gpuOperator"` + // HAMi GPU virtualization middleware. + // +kubebuilder:default:={} + Hami HAMiAddon `json:"hami"` // Ingress-NGINX controller. // +kubebuilder:default:={} IngressNginx IngressNginxAddon `json:"ingressNginx"` @@ -160,6 +163,15 @@ type GatewayAPIAddon struct { Enabled bool `json:"enabled"` } +type HAMiAddon struct { + // Enable HAMi (requires GPU Operator). + // +kubebuilder:default:=false + Enabled bool `json:"enabled"` + // Custom Helm values overrides. + // +kubebuilder:default:={} + ValuesOverride k8sRuntime.RawExtension `json:"valuesOverride"` +} + type Images struct { // Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag. // +kubebuilder:default:="" diff --git a/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go b/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go index 2cbe3ba1..e021fbaa 100644 --- a/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go +++ b/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go @@ -49,6 +49,7 @@ func (in *Addons) DeepCopyInto(out *Addons) { in.Fluxcd.DeepCopyInto(&out.Fluxcd) out.GatewayAPI = in.GatewayAPI in.GpuOperator.DeepCopyInto(&out.GpuOperator) + in.Hami.DeepCopyInto(&out.Hami) in.IngressNginx.DeepCopyInto(&out.IngressNginx) in.MonitoringAgents.DeepCopyInto(&out.MonitoringAgents) in.Velero.DeepCopyInto(&out.Velero) @@ -135,6 +136,7 @@ func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) { } in.Addons.DeepCopyInto(&out.Addons) in.ControlPlane.DeepCopyInto(&out.ControlPlane) + out.Images = in.Images } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigSpec. @@ -260,6 +262,37 @@ func (in *GatewayAPIAddon) DeepCopy() *GatewayAPIAddon { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HAMiAddon) DeepCopyInto(out *HAMiAddon) { + *out = *in + in.ValuesOverride.DeepCopyInto(&out.ValuesOverride) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HAMiAddon. +func (in *HAMiAddon) DeepCopy() *HAMiAddon { + if in == nil { + return nil + } + out := new(HAMiAddon) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Images) DeepCopyInto(out *Images) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Images. +func (in *Images) DeepCopy() *Images { + if in == nil { + return nil + } + out := new(Images) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IngressNginxAddon) DeepCopyInto(out *IngressNginxAddon) { *out = *in diff --git a/docs/changelogs/v1.3.1.md b/docs/changelogs/v1.3.1.md new file mode 100644 index 00000000..0f149d0d --- /dev/null +++ b/docs/changelogs/v1.3.1.md @@ -0,0 +1,37 @@ + + +# v1.3.1 (2026-04-28) + +Patch release covering a TenantNamespace IDOR fix in the API, a destructive `post-upgrade` hook removed from the etcd chart, kamaji controller stability, a `linstor-csi` bump that fixes live migration on Protocol-A/B DRBD resources, the missing `linstor-gui` build wiring, and a velero RBAC fix that unblocked installs on bundles without Velero. + +## Security + +* **fix(api): prevent IDOR in TenantNamespace Get and Watch handlers**: Two IDOR (Insecure Direct Object Reference) vulnerabilities allowed authenticated users to read TenantNamespace metadata they had no RoleBinding for. The `Get` and `Watch` handlers now go through a new `hasAccessToNamespace()` helper that lists RoleBindings scoped only to the target namespace (orders of magnitude faster than the previous all-cluster scan), returns `NotFound` instead of leaking existence on unauthorized access, and applies the same check on the `Watch` filter path. Includes regression tests for the unauthorized paths. ([**@IvanHunters**](https://github.com/IvanHunters) in #2471, backport #2524) + +## Features + +* **feat(linstor): bump linstor-csi to v1.10.6 with Protocol-C dual-attach fix**: Live migration of KubeVirt VMs on Protocol-A/B (async) DRBD volumes no longer fails with `Protocol C required`. `linstor-csi` v1.10.6 now installs a `Protocol=C` override on the resource-definition during dual-attach and reverts it on detach, so `replicated-async` StorageClasses and other Protocol-A/B resource groups support live migration without manual `drbdadm adjust` intervention. ([**@kvaps**](https://github.com/kvaps) in #2496, backport #2505) + +## Fixes + +* **fix(backups): move velero-configmap Role to velero chart**: The `backupstrategy-controller` (a default package) declared a Role/RoleBinding scoped to the `cozy-velero` namespace for managing `ResourceModifier` ConfigMaps. On bundles where Velero was not enabled, that namespace did not exist and the HelmRelease failed with `namespaces "cozy-velero" not found`, blocking installation. The Role/RoleBinding now lives in the velero chart, so it is created only when velero is actually deployed. ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2459, backport #2467) + +* **fix(etcd): remove destructive post-upgrade cert-regeneration hook**: The etcd chart ran a `post-upgrade` Helm hook on every upgrade that deleted etcd TLS Secrets (`etcd-ca-tls`, `etcd-peer-ca-tls`, `etcd-client-tls`, `etcd-peer-tls`, `etcd-server-tls`) and then deleted etcd pods, forcing cert-manager to re-issue the entire etcd CA chain. On clusters with Kamaji-managed tenant control planes this put every tenant `kube-apiserver` into CrashLoopBackOff until each DataStore was manually re-reconciled. The hook was a one-shot `2.6.0 → 2.6.1` migration that became a permanent footgun once chart versioning moved to `0.0.0+` (always `< 2.6.1` per semver) and after the underlying `rotationPolicy: Always` issue was fixed in `47d81f70`. The hook is now removed entirely. ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2462, backport #2511) + +* **fix(kamaji): increase memory limits and add startup probe**: The kamaji controller frequently entered CrashLoopBackOff due to OOMKills (exit 137) within ~20–25 seconds of startup, with the readiness probe failing while the controller was still finishing initialization. Memory limit raised from 500Mi to 512Mi, request from 100Mi to 256Mi, and a 60-second startup probe (12 attempts × 5s periods) is added so the controller has room to boot before liveness/readiness probes engage. ([**@IvanHunters**](https://github.com/IvanHunters) in #2421, backport #2491) + +## Build + +* **build(linstor): include linstor-gui in root image build target**: The `linstor-gui` package (added in #2382) was never wired into the root `Makefile`'s `build:` target, so CI never built or published the image. `ghcr.io/cozystack/cozystack/linstor-gui` returned `NAME_UNKNOWN` and `values.yaml` stayed pinned to `tag: 2.3.0` without a digest. The missing build line is added so the next CI run publishes the image and the per-package `Makefile` digest-pins `values.yaml` automatically. ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2498, backport #2518) + +## Contributors + +Thanks to everyone who contributed to this patch release: + +* [**@IvanHunters**](https://github.com/IvanHunters) +* [**@kvaps**](https://github.com/kvaps) +* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) + +**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.3.0...v1.3.1 diff --git a/docs/gpu-vgpu.md b/docs/gpu-vgpu.md deleted file mode 100644 index 1c402096..00000000 --- a/docs/gpu-vgpu.md +++ /dev/null @@ -1,172 +0,0 @@ -# GPU Operator: vGPU Support - -This document describes how to configure the GPU Operator package with NVIDIA vGPU support for sharing a single physical GPU across multiple virtual machines using mediated devices. - -## Prerequisites - -- NVIDIA GPU with vGPU support (e.g., NVIDIA L40S, A100, A30, etc.) -- Talos Linux as the host OS -- NVIDIA vGPU Software license (NVIDIA AI Enterprise or vGPU subscription) -- Access to the NVIDIA Licensing Portal ([ui.licensing.nvidia.com](https://ui.licensing.nvidia.com)) - -## Variants - -The GPU Operator package supports two variants: - -- **`default`** — GPU passthrough mode (vfio-pci). The entire GPU is passed through to a single VM. -- **`vgpu`** — vGPU mode. A physical GPU is shared between multiple VMs using NVIDIA mediated devices. - -## Building the vGPU Manager Image - -The vGPU Manager driver is proprietary and must be obtained from NVIDIA. The GPU Operator expects a pre-built driver container image — it does not install the driver from a raw `.run` file at runtime. - -1. Log in to the [NVIDIA Licensing Portal](https://ui.licensing.nvidia.com) -2. Navigate to **Software Downloads** and download the NVIDIA vGPU Software package for your GPU -3. Build the driver container image using NVIDIA's Makefile-based build system: - -```bash -# Clone the NVIDIA driver container repository -git clone https://gitlab.com/nvidia/container-images/driver.git -cd driver - -# Place the downloaded .run file in the appropriate directory -cp NVIDIA-Linux-x86_64-550.90.05-vgpu-kvm.run vgpu/ - -# Build using the provided Makefile -make OS_TAG=ubuntu22.04 \ - VGPU_DRIVER_VERSION=550.90.05 \ - PRIVATE_REGISTRY=registry.example.com/nvidia - -# Push to your private registry -docker push registry.example.com/nvidia/vgpu-manager:550.90.05 -``` - -> **Important:** The build process compiles kernel modules against the host kernel version. Refer to the [NVIDIA GPU Operator vGPU documentation](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/install-gpu-operator-vgpu.html) for the complete build procedure and supported OS/kernel combinations. -> -> Uploading the vGPU driver to a publicly available registry is a violation of the NVIDIA vGPU EULA. - -## Deploying with vGPU Variant - -Create a Package CR with the `vgpu` variant and provide your vGPU Manager image coordinates: - -```yaml -apiVersion: cozystack.io/v1alpha1 -kind: Package -metadata: - name: cozystack.gpu-operator -spec: - variant: vgpu - components: - gpu-operator: - values: - gpu-operator: - vgpuManager: - repository: registry.example.com/nvidia - version: "550.90.05" -``` - -If your registry requires authentication, create an `imagePullSecret` in the `cozy-gpu-operator` namespace and reference it: - -```yaml -gpu-operator: - vgpuManager: - repository: registry.example.com/nvidia - version: "550.90.05" - imagePullSecrets: - - name: nvidia-registry-secret -``` - -## NVIDIA License Server (NLS) Configuration - -vGPU requires a license server. Configure NLS by passing the license server address via a Secret: - -1. Create a Secret with the NLS client configuration in the `cozy-gpu-operator` namespace: - -```yaml -apiVersion: v1 -kind: Secret -metadata: - name: licensing-config - namespace: cozy-gpu-operator -stringData: - gridd.conf: | - ServerAddress=nls.example.com - ServerPort=443 - FeatureType=1 # 1 for vGPU (vPC/vWS), 2 for Virtual Compute Server (vCS) - # ServerPort depends on your NLS deployment (commonly 443 for DLS or 7070 for legacy NLS) -``` - -2. Reference the Secret in the Package values: - -```yaml -gpu-operator: - vgpuManager: - repository: registry.example.com/nvidia - version: "550.90.05" - driver: - licensingConfig: - secretName: licensing-config -``` - -## vGPU Profiles - -Each GPU model supports specific vGPU profiles that determine how the GPU is partitioned. To list available profiles for your GPU, consult the [NVIDIA vGPU User Guide](https://docs.nvidia.com/grid/latest/grid-vgpu-user-guide/). - -Example profiles for NVIDIA L40S: - -| Profile | Frame Buffer | Max Instances | Use Case | -| --- | --- | --- | --- | -| NVIDIA L40S-1Q | 1 GB | 48 | Light 3D/VDI | -| NVIDIA L40S-2Q | 2 GB | 24 | Medium 3D/VDI | -| NVIDIA L40S-4Q | 4 GB | 12 | Heavy 3D/VDI | -| NVIDIA L40S-6Q | 6 GB | 8 | Professional 3D | -| NVIDIA L40S-8Q | 8 GB | 6 | AI/ML inference | -| NVIDIA L40S-12Q | 12 GB | 4 | AI/ML training | -| NVIDIA L40S-24Q | 24 GB | 2 | Large AI workloads | -| NVIDIA L40S-48Q | 48 GB | 1 | Full GPU equivalent | - -Custom vGPU device configuration can be provided via a ConfigMap: - -```yaml -gpu-operator: - vgpuDeviceManager: - enabled: true - config: - name: vgpu-devices-config - default: default -``` - -## KubeVirt Integration - -To use vGPU with KubeVirt VMs, configure `mediatedDeviceTypes` in the KubeVirt CR. This maps vGPU profiles to node selectors: - -```yaml -apiVersion: kubevirt.io/v1 -kind: KubeVirt -metadata: - name: kubevirt -spec: - configuration: - mediatedDevicesConfiguration: - mediatedDeviceTypes: - - nvidia-592 # NVIDIA L40S-24Q - permittedHostDevices: - mediatedDevices: - - mdevNameSelector: NVIDIA L40S-24Q - resourceName: nvidia.com/NVIDIA_L40S-24Q -``` - -Then reference the vGPU resource in a VirtualMachine spec: - -```yaml -apiVersion: kubevirt.io/v1 -kind: VirtualMachine -spec: - template: - spec: - domain: - devices: - gpus: - - name: gpu1 - deviceName: nvidia.com/NVIDIA_L40S-24Q -``` diff --git a/packages/apps/kubernetes/README.md b/packages/apps/kubernetes/README.md index ecbbc31b..bed1117a 100644 --- a/packages/apps/kubernetes/README.md +++ b/packages/apps/kubernetes/README.md @@ -128,6 +128,9 @@ See the reference for components utilized in this service: | `addons.gpuOperator` | NVIDIA GPU Operator. | `object` | `{}` | | `addons.gpuOperator.enabled` | Enable GPU Operator. | `bool` | `false` | | `addons.gpuOperator.valuesOverride` | Custom Helm values overrides. | `object` | `{}` | +| `addons.hami` | HAMi GPU virtualization middleware. | `object` | `{}` | +| `addons.hami.enabled` | Enable HAMi (requires GPU Operator). | `bool` | `false` | +| `addons.hami.valuesOverride` | Custom Helm values overrides. | `object` | `{}` | | `addons.fluxcd` | FluxCD GitOps operator. | `object` | `{}` | | `addons.fluxcd.enabled` | Enable FluxCD. | `bool` | `false` | | `addons.fluxcd.valuesOverride` | Custom Helm values overrides. | `object` | `{}` | diff --git a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml index e243bfad..655dc868 100644 --- a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml @@ -1,3 +1,11 @@ +{{- define "cozystack.defaultGpuOperatorValues" -}} +{{- if .Values.addons.hami.enabled }} +gpu-operator: + devicePlugin: + enabled: false +{{- end }} +{{- end }} + {{- if and .Values.addons.gpuOperator.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease @@ -29,9 +37,12 @@ spec: force: true remediation: retries: -1 - {{- with .Values.addons.gpuOperator.valuesOverride }} + {{- $defaults := fromYaml (include "cozystack.defaultGpuOperatorValues" .) }} + {{- $overrides := deepCopy (default (dict) .Values.addons.gpuOperator.valuesOverride) }} + {{- $merged := mergeOverwrite (default (dict) $defaults) $overrides }} + {{- if $merged }} values: - {{- toYaml . | nindent 4 }} + {{- toYaml $merged | nindent 4 }} {{- end }} dependsOn: diff --git a/packages/apps/kubernetes/templates/helmreleases/hami.yaml b/packages/apps/kubernetes/templates/helmreleases/hami.yaml new file mode 100644 index 00000000..f1538c7c --- /dev/null +++ b/packages/apps/kubernetes/templates/helmreleases/hami.yaml @@ -0,0 +1,49 @@ +{{- if and .Values.addons.hami.enabled .Values._namespace.etcd }} +{{- if not .Values.addons.gpuOperator.enabled }} +{{- fail "addons.hami requires addons.gpuOperator to be enabled" }} +{{- end }} +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: {{ .Release.Name }}-hami + labels: + cozystack.io/repository: system + cozystack.io/target-cluster-name: {{ .Release.Name }} + sharding.fluxcd.io/key: tenants +spec: + releaseName: hami + chartRef: + kind: ExternalArtifact + name: cozystack-kubernetes-application-kubevirt-kubernetes-hami + namespace: cozy-system + kubeConfig: + secretRef: + name: {{ .Release.Name }}-admin-kubeconfig + key: super-admin.svc + targetNamespace: cozy-hami + storageNamespace: cozy-hami + interval: 5m + timeout: 10m + install: + createNamespace: true + remediation: + retries: -1 + upgrade: + force: true + remediation: + retries: -1 + {{- with .Values.addons.hami.valuesOverride }} + values: + {{- toYaml . | nindent 4 }} + {{- end }} + + dependsOn: + {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }} + - name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} + {{- end }} + - name: {{ .Release.Name }}-cilium + namespace: {{ .Release.Namespace }} + - name: {{ .Release.Name }}-gpu-operator + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml new file mode 100644 index 00000000..44528470 --- /dev/null +++ b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml @@ -0,0 +1,99 @@ +suite: GPU Operator HelmRelease HAMi integration tests +templates: + - templates/helmreleases/gpu-operator.yaml +values: + - values-ci.yaml +tests: + - it: should disable devicePlugin when hami is enabled + set: + addons: + gpuOperator: + enabled: true + valuesOverride: {} + hami: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.values.gpu-operator.devicePlugin.enabled + value: false + + - it: should not have values when hami is disabled and no overrides + set: + addons: + gpuOperator: + enabled: true + valuesOverride: {} + hami: + enabled: false + valuesOverride: {} + asserts: + - notExists: + path: spec.values + + - it: should apply hami defaults when valuesOverride key is omitted + set: + addons: + gpuOperator: + enabled: true + hami: + enabled: true + asserts: + - equal: + path: spec.values.gpu-operator.devicePlugin.enabled + value: false + + - it: should allow user overrides to merge with hami defaults + set: + addons: + gpuOperator: + enabled: true + valuesOverride: + gpu-operator: + driver: + enabled: false + hami: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.values.gpu-operator.devicePlugin.enabled + value: false + - equal: + path: spec.values.gpu-operator.driver.enabled + value: false + + - it: should let user explicitly override devicePlugin.enabled to true with hami enabled + set: + addons: + gpuOperator: + enabled: true + valuesOverride: + gpu-operator: + devicePlugin: + enabled: true + driver: + enabled: false + hami: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.values.gpu-operator.devicePlugin.enabled + value: true + - equal: + path: spec.values.gpu-operator.driver.enabled + value: false + + - it: should not render when gpuOperator is disabled + set: + addons: + gpuOperator: + enabled: false + valuesOverride: {} + hami: + enabled: false + valuesOverride: {} + asserts: + - hasDocuments: + count: 0 diff --git a/packages/apps/kubernetes/tests/hami_test.yaml b/packages/apps/kubernetes/tests/hami_test.yaml new file mode 100644 index 00000000..27f7c75b --- /dev/null +++ b/packages/apps/kubernetes/tests/hami_test.yaml @@ -0,0 +1,153 @@ +suite: HAMi HelmRelease tests +templates: + - templates/helmreleases/hami.yaml +values: + - values-ci.yaml +tests: + - it: should not render when hami is disabled + set: + addons: + hami: + enabled: false + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - hasDocuments: + count: 0 + + - it: should render HelmRelease when hami is enabled + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - hasDocuments: + count: 1 + - isKind: + of: HelmRelease + + - it: should fail when gpuOperator is not enabled + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: false + valuesOverride: {} + asserts: + - failedTemplate: + errorMessage: "addons.hami requires addons.gpuOperator to be enabled" + + - it: should have correct metadata labels + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: metadata.labels["cozystack.io/repository"] + value: system + - equal: + path: metadata.labels["sharding.fluxcd.io/key"] + value: tenants + + - it: should use ExternalArtifact chartRef + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.chartRef.kind + value: ExternalArtifact + - equal: + path: spec.chartRef.namespace + value: cozy-system + + - it: should target cozy-hami namespace + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.targetNamespace + value: cozy-hami + - equal: + path: spec.storageNamespace + value: cozy-hami + + - it: should depend on gpu-operator and cilium + release: + name: test + namespace: test-ns + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - contains: + path: spec.dependsOn + content: + name: test-cilium + namespace: test-ns + - contains: + path: spec.dependsOn + content: + name: test-gpu-operator + namespace: test-ns + + - it: should not render spec.values when valuesOverride is empty + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - hasDocuments: + count: 1 + - notExists: + path: spec.values + + - it: should pass through valuesOverride + set: + addons: + hami: + enabled: true + valuesOverride: + hami: + devicePlugin: + deviceSplitCount: 5 + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.values.hami.devicePlugin.deviceSplitCount + value: 5 diff --git a/packages/apps/kubernetes/values.schema.json b/packages/apps/kubernetes/values.schema.json index 81e9e2b8..a43deaec 100644 --- a/packages/apps/kubernetes/values.schema.json +++ b/packages/apps/kubernetes/values.schema.json @@ -149,6 +149,7 @@ "fluxcd", "gatewayAPI", "gpuOperator", + "hami", "ingressNginx", "monitoringAgents", "velero", @@ -268,6 +269,28 @@ } } }, + "hami": { + "description": "HAMi GPU virtualization middleware.", + "type": "object", + "default": {}, + "required": [ + "enabled", + "valuesOverride" + ], + "properties": { + "enabled": { + "description": "Enable HAMi (requires GPU Operator).", + "type": "boolean", + "default": false + }, + "valuesOverride": { + "description": "Custom Helm values overrides.", + "type": "object", + "default": {}, + "x-kubernetes-preserve-unknown-fields": true + } + } + }, "ingressNginx": { "description": "Ingress-NGINX controller.", "type": "object", diff --git a/packages/apps/kubernetes/values.yaml b/packages/apps/kubernetes/values.yaml index 68ba07b0..d609476c 100644 --- a/packages/apps/kubernetes/values.yaml +++ b/packages/apps/kubernetes/values.yaml @@ -94,6 +94,10 @@ host: "" ## @field {bool} enabled - Enable FluxCD. ## @field {object} valuesOverride - Custom Helm values overrides. +## @typedef {struct} HAMiAddon - HAMi GPU virtualization middleware. +## @field {bool} enabled - Enable HAMi (requires GPU Operator). +## @field {object} valuesOverride - Custom Helm values overrides. + ## @typedef {struct} MonitoringAgentsAddon - Monitoring agents (Fluent Bit, VMAgents). ## @field {bool} enabled - Enable monitoring agents. ## @field {object} valuesOverride - Custom Helm values overrides. @@ -114,6 +118,7 @@ host: "" ## @field {GatewayAPIAddon} gatewayAPI - Gateway API addon. ## @field {IngressNginxAddon} ingressNginx - Ingress-NGINX controller. ## @field {GPUOperatorAddon} gpuOperator - NVIDIA GPU Operator. +## @field {HAMiAddon} hami - HAMi GPU virtualization middleware. ## @field {FluxCDAddon} fluxcd - FluxCD GitOps operator. ## @field {MonitoringAgentsAddon} monitoringAgents - Monitoring agents. ## @field {VerticalPodAutoscalerAddon} verticalPodAutoscaler - Vertical Pod Autoscaler. @@ -137,6 +142,9 @@ addons: gpuOperator: enabled: false valuesOverride: {} + hami: + enabled: false + valuesOverride: {} fluxcd: enabled: false valuesOverride: {} diff --git a/packages/core/platform/sources/gpu-operator.yaml b/packages/core/platform/sources/gpu-operator.yaml index 10d36d65..e41cc9b2 100644 --- a/packages/core/platform/sources/gpu-operator.yaml +++ b/packages/core/platform/sources/gpu-operator.yaml @@ -23,16 +23,3 @@ spec: privileged: true namespace: cozy-gpu-operator releaseName: gpu-operator - - name: vgpu - dependsOn: - - cozystack.networking - components: - - name: gpu-operator - path: system/gpu-operator - valuesFiles: - - values.yaml - - values-talos-vgpu.yaml - install: - privileged: true - namespace: cozy-gpu-operator - releaseName: gpu-operator diff --git a/packages/core/platform/sources/hami.yaml b/packages/core/platform/sources/hami.yaml new file mode 100644 index 00000000..0184e405 --- /dev/null +++ b/packages/core/platform/sources/hami.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: cozystack.io/v1alpha1 +kind: PackageSource +metadata: + name: cozystack.hami +spec: + sourceRef: + kind: OCIRepository + name: cozystack-packages + namespace: cozy-system + path: / + variants: + - name: default + dependsOn: + - cozystack.gpu-operator + components: + - name: hami + path: system/hami + valuesFiles: + - values.yaml + install: + privileged: true + namespace: cozy-hami + releaseName: hami diff --git a/packages/core/platform/sources/kubernetes-application.yaml b/packages/core/platform/sources/kubernetes-application.yaml index 9787cb19..088383ad 100644 --- a/packages/core/platform/sources/kubernetes-application.yaml +++ b/packages/core/platform/sources/kubernetes-application.yaml @@ -52,6 +52,8 @@ spec: path: system/cilium - name: kubernetes-gpu-operator path: system/gpu-operator + - name: kubernetes-hami + path: system/hami - name: kubernetes-vertical-pod-autoscaler path: system/vertical-pod-autoscaler - name: kubernetes-prometheus-operator-crds diff --git a/packages/core/platform/templates/bundles/iaas.yaml b/packages/core/platform/templates/bundles/iaas.yaml index ced0a322..66ec98b2 100644 --- a/packages/core/platform/templates/bundles/iaas.yaml +++ b/packages/core/platform/templates/bundles/iaas.yaml @@ -10,6 +10,7 @@ {{include "cozystack.platform.package.default" (list "cozystack.kubevirt-cdi" $) }} {{include "cozystack.platform.package.optional.default" (list "cozystack.vm-default-images" $) }} {{include "cozystack.platform.package.optional.default" (list "cozystack.gpu-operator" $) }} +{{include "cozystack.platform.package.optional.default" (list "cozystack.hami" $) }} {{include "cozystack.platform.package.default" (list "cozystack.kamaji" $) }} {{include "cozystack.platform.package.default" (list "cozystack.capi-operator" $) }} {{include "cozystack.platform.package.default" (list "cozystack.capi-provider-bootstrap-kubeadm" $) }} diff --git a/packages/core/platform/templates/bundles/system.yaml b/packages/core/platform/templates/bundles/system.yaml index 43ea7266..6c587c82 100644 --- a/packages/core/platform/templates/bundles/system.yaml +++ b/packages/core/platform/templates/bundles/system.yaml @@ -105,9 +105,6 @@ {{- /* cozystack-api DaemonSet */ -}} {{- $apiValues := dict "cozystackAPI" (dict "nodeSelector" $genericNodeSelector) -}} {{- $_ := set $cozystackEngineComponents "cozystack-api" (dict "values" $apiValues) -}} -{{- /* lineage-controller-webhook DaemonSet */ -}} -{{- $lineageValues := dict "lineageControllerWebhook" (dict "nodeSelector" $genericNodeSelector) -}} -{{- $_ := set $cozystackEngineComponents "lineage-controller-webhook" (dict "values" $lineageValues) -}} {{- end -}} {{- if .Values.authentication.oidc.enabled }} {{include "cozystack.platform.package" (list "cozystack.cozystack-engine" "oidc" $ $cozystackEngineComponents) }} diff --git a/packages/system/gpu-operator/Makefile b/packages/system/gpu-operator/Makefile index a8b3b43f..49b26e26 100644 --- a/packages/system/gpu-operator/Makefile +++ b/packages/system/gpu-operator/Makefile @@ -8,4 +8,4 @@ update: rm -rf charts helm repo add nvidia https://helm.ngc.nvidia.com/nvidia helm repo update nvidia - helm pull nvidia/gpu-operator --untar --untardir charts --version v26.3.1 + helm pull nvidia/gpu-operator --untar --untardir charts diff --git a/packages/system/gpu-operator/charts/gpu-operator/Chart.lock b/packages/system/gpu-operator/charts/gpu-operator/Chart.lock index 707da9d1..14674306 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/Chart.lock +++ b/packages/system/gpu-operator/charts/gpu-operator/Chart.lock @@ -1,6 +1,6 @@ dependencies: - name: node-feature-discovery - repository: oci://registry.k8s.io/nfd/charts - version: 0.18.3 -digest: sha256:1bce3b80da1e6d47a0befbaa6a2f758aeac6d20382e900f0c301f45020afbebc -generated: "2026-01-23T15:08:56.617442-08:00" + repository: https://kubernetes-sigs.github.io/node-feature-discovery/charts + version: 0.17.2 +digest: sha256:4c55d30d958027ef8997a2976449326de3c90049025c3ebb9bee017cad32cc3f +generated: "2025-02-25T09:08:49.128088-08:00" diff --git a/packages/system/gpu-operator/charts/gpu-operator/Chart.yaml b/packages/system/gpu-operator/charts/gpu-operator/Chart.yaml index 1e1ddabe..f14fc2ad 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/Chart.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/Chart.yaml @@ -1,10 +1,10 @@ apiVersion: v2 -appVersion: v26.3.1 +appVersion: v25.3.0 dependencies: - condition: nfd.enabled name: node-feature-discovery - repository: oci://registry.k8s.io/nfd/charts - version: 0.18.3 + repository: https://kubernetes-sigs.github.io/node-feature-discovery/charts + version: v0.17.2 description: NVIDIA GPU Operator creates/configures/manages GPUs atop Kubernetes home: https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/overview.html icon: https://assets.nvidiagrid.net/ngc/logos/GPUoperator.png @@ -20,4 +20,4 @@ kubeVersion: '>= 1.16.0-0' name: gpu-operator sources: - https://github.com/NVIDIA/gpu-operator -version: v26.3.1 +version: v25.3.0 diff --git a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/Chart.yaml b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/Chart.yaml index 8609178c..f62d84c2 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/Chart.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/Chart.yaml @@ -1,9 +1,8 @@ apiVersion: v2 -appVersion: v0.18.3 +appVersion: v0.17.2 description: 'Detects hardware features available on each node in a Kubernetes cluster, and advertises those features using node labels. ' home: https://github.com/kubernetes-sigs/node-feature-discovery -icon: https://kubernetes-sigs.github.io/node-feature-discovery/v0.18/assets/images/nfd/favicon.svg keywords: - feature-discovery - feature-detection @@ -12,4 +11,4 @@ name: node-feature-discovery sources: - https://github.com/kubernetes-sigs/node-feature-discovery type: application -version: 0.18.3 +version: 0.17.2 diff --git a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/README.md b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/README.md index 2b46e4af..02f7b170 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/README.md +++ b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/README.md @@ -6,5 +6,5 @@ labels. NFD provides flexible configuration and extension points for a wide range of vendor and application specific node labeling needs. See -[NFD documentation](https://kubernetes-sigs.github.io/node-feature-discovery/v0.18/deployment/helm.html) +[NFD documentation](https://kubernetes-sigs.github.io/node-feature-discovery/v0.17/deployment/helm.html) for deployment instructions. diff --git a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/crds/nfd-api-crds.yaml b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/crds/nfd-api-crds.yaml index 69e306c3..9f62da6f 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/crds/nfd-api-crds.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/crds/nfd-api-crds.yaml @@ -204,19 +204,11 @@ spec: - Exists - DoesNotExist - Gt - - Ge - Lt - - Le - GtLt - - GeLe - IsTrue - IsFalse type: string - type: - description: |- - Type defines the value type for specific operators. - The currently supported type is 'version' for Gt,Ge,Lt,Le,GtLt,GeLe operators. - type: string value: description: |- Value is the list of values that the operand evaluates the input @@ -248,19 +240,11 @@ spec: - Exists - DoesNotExist - Gt - - Ge - Lt - - Le - GtLt - - GeLe - IsTrue - IsFalse type: string - type: - description: |- - Type defines the value type for specific operators. - The currently supported type is 'version' for Gt,Ge,Lt,Le,GtLt,GeLe operators. - type: string value: description: |- Value is the list of values that the operand evaluates the input @@ -311,19 +295,11 @@ spec: - Exists - DoesNotExist - Gt - - Ge - Lt - - Le - GtLt - - GeLe - IsTrue - IsFalse type: string - type: - description: |- - Type defines the value type for specific operators. - The currently supported type is 'version' for Gt,Ge,Lt,Le,GtLt,GeLe operators. - type: string value: description: |- Value is the list of values that the operand evaluates the input @@ -355,19 +331,11 @@ spec: - Exists - DoesNotExist - Gt - - Ge - Lt - - Le - GtLt - - GeLe - IsTrue - IsFalse type: string - type: - description: |- - Type defines the value type for specific operators. - The currently supported type is 'version' for Gt,Ge,Lt,Le,GtLt,GeLe operators. - type: string value: description: |- Value is the list of values that the operand evaluates the input @@ -388,19 +356,6 @@ spec: name: description: Name of the rule. type: string - vars: - additionalProperties: - type: string - description: |- - Vars is the variables to store if the rule matches. Variables can be - referenced from other rules enabling more complex rule hierarchies. - type: object - varsTemplate: - description: |- - VarsTemplate specifies a template to expand for dynamically generating - multiple variables. Data (after template expansion) must be keys with an - optional value ([=]) separated by newlines. - type: string required: - name type: object @@ -543,19 +498,11 @@ spec: - Exists - DoesNotExist - Gt - - Ge - Lt - - Le - GtLt - - GeLe - IsTrue - IsFalse type: string - type: - description: |- - Type defines the value type for specific operators. - The currently supported type is 'version' for Gt,Ge,Lt,Le,GtLt,GeLe operators. - type: string value: description: |- Value is the list of values that the operand evaluates the input @@ -587,19 +534,11 @@ spec: - Exists - DoesNotExist - Gt - - Ge - Lt - - Le - GtLt - - GeLe - IsTrue - IsFalse type: string - type: - description: |- - Type defines the value type for specific operators. - The currently supported type is 'version' for Gt,Ge,Lt,Le,GtLt,GeLe operators. - type: string value: description: |- Value is the list of values that the operand evaluates the input @@ -650,19 +589,11 @@ spec: - Exists - DoesNotExist - Gt - - Ge - Lt - - Le - GtLt - - GeLe - IsTrue - IsFalse type: string - type: - description: |- - Type defines the value type for specific operators. - The currently supported type is 'version' for Gt,Ge,Lt,Le,GtLt,GeLe operators. - type: string value: description: |- Value is the list of values that the operand evaluates the input @@ -694,19 +625,11 @@ spec: - Exists - DoesNotExist - Gt - - Ge - Lt - - Le - GtLt - - GeLe - IsTrue - IsFalse type: string - type: - description: |- - Type defines the value type for specific operators. - The currently supported type is 'version' for Gt,Ge,Lt,Le,GtLt,GeLe operators. - type: string value: description: |- Value is the list of values that the operand evaluates the input diff --git a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/_helpers.tpl b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/_helpers.tpl index b540e3bb..928ece78 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/_helpers.tpl +++ b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/_helpers.tpl @@ -105,22 +105,3 @@ Create the name of the service account which nfd-gc will use {{ default "default" .Values.gc.serviceAccount.name }} {{- end -}} {{- end -}} - -{{/* -imagePullSecrets helper - uses local values or falls back to global values -*/}} -{{- define "node-feature-discovery.imagePullSecrets" -}} -{{- $imagePullSecrets := list -}} -{{- if .Values.imagePullSecrets -}} - {{- range .Values.imagePullSecrets -}} - {{- $imagePullSecrets = append $imagePullSecrets . -}} - {{- end -}} -{{- else if and .Values.global .Values.global.imagePullSecrets -}} - {{- range .Values.global.imagePullSecrets -}} - {{- $imagePullSecrets = append $imagePullSecrets . -}} - {{- end -}} -{{- end -}} -{{- if $imagePullSecrets -}} -{{- $imagePullSecrets | toJson }} -{{- end -}} -{{- end -}} diff --git a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/master-pdb.yaml b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/master-pdb.yaml deleted file mode 100644 index 532b0e6d..00000000 --- a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/master-pdb.yaml +++ /dev/null @@ -1,17 +0,0 @@ -{{- if .Values.master.enable }} -{{- if .Values.master.podDisruptionBudget.enable -}} -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - name: {{ include "node-feature-discovery.fullname" . }}-master - namespace: {{ include "node-feature-discovery.namespace" . }} - labels: - {{- include "node-feature-discovery.labels" . | nindent 4 }} -spec: - selector: - matchLabels: - {{- include "node-feature-discovery.selectorLabels" . | nindent 6 }} - role: master -{{- toYaml (omit .Values.master.podDisruptionBudget "enable") | nindent 2 }} -{{- end }} -{{- end }} diff --git a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/master.yaml b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/master.yaml index 666af5c5..da3ca240 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/master.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/master.yaml @@ -29,11 +29,13 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} spec: - dnsPolicy: {{ .Values.master.dnsPolicy }} {{- with .Values.priorityClassName }} priorityClassName: {{ . }} {{- end }} - imagePullSecrets: {{ include "node-feature-discovery.imagePullSecrets" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} serviceAccountName: {{ include "node-feature-discovery.master.serviceAccountName" . }} enableServiceLinks: false securityContext: @@ -46,9 +48,8 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} startupProbe: - httpGet: - path: /healthz - port: http + grpc: + port: {{ .Values.master.healthPort | default "8082" }} {{- with .Values.master.startupProbe.initialDelaySeconds }} initialDelaySeconds: {{ . }} {{- end }} @@ -62,9 +63,8 @@ spec: timeoutSeconds: {{ . }} {{- end }} livenessProbe: - httpGet: - path: /healthz - port: http + grpc: + port: {{ .Values.master.healthPort | default "8082" }} {{- with .Values.master.livenessProbe.initialDelaySeconds }} initialDelaySeconds: {{ . }} {{- end }} @@ -78,9 +78,8 @@ spec: timeoutSeconds: {{ . }} {{- end }} readinessProbe: - httpGet: - path: /healthz - port: http + grpc: + port: {{ .Values.master.healthPort | default "8082" }} {{- with .Values.master.readinessProbe.initialDelaySeconds }} initialDelaySeconds: {{ . }} {{- end }} @@ -97,15 +96,17 @@ spec: successThreshold: {{ . }} {{- end }} ports: - - containerPort: {{ .Values.master.port | default "8080" }} - name: http + - containerPort: {{ .Values.master.metricsPort | default "8081" }} + name: metrics + - containerPort: {{ .Values.master.healthPort | default "8082" }} + name: health env: - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName {{- with .Values.master.extraEnvs }} - {{- toYaml . | nindent 10 }} + {{- toYaml . | nindent 8 }} {{- end}} command: - "nfd-master" @@ -125,6 +126,9 @@ spec: {{- if .Values.master.enableTaints }} - "-enable-taints" {{- end }} + {{- if .Values.master.featureRulesController | kindIs "invalid" | not }} + - "-featurerules-controller={{ .Values.master.featureRulesController }}" + {{- end }} {{- if .Values.master.resyncPeriod }} - "-resync-period={{ .Values.master.resyncPeriod }}" {{- end }} @@ -135,7 +139,8 @@ spec: {{- range $key, $value := .Values.featureGates }} - "-feature-gates={{ $key }}={{ $value }}" {{- end }} - - "-port={{ .Values.master.port | default "8080" }}" + - "-metrics={{ .Values.master.metricsPort | default "8081" }}" + - "-grpc-health={{ .Values.master.healthPort | default "8082" }}" {{- with .Values.master.extraArgs }} {{- toYaml . | nindent 12 }} {{- end }} diff --git a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/nfd-gc-pdb.yaml b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/nfd-gc-pdb.yaml deleted file mode 100644 index 373ed92c..00000000 --- a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/nfd-gc-pdb.yaml +++ /dev/null @@ -1,17 +0,0 @@ -{{- if .Values.gc.enable }} -{{- if .Values.gc.podDisruptionBudget.enable -}} -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - name: {{ include "node-feature-discovery.fullname" . }}-gc - namespace: {{ include "node-feature-discovery.namespace" . }} - labels: - {{- include "node-feature-discovery.labels" . | nindent 4 }} -spec: - selector: - matchLabels: - {{- include "node-feature-discovery.selectorLabels" . | nindent 6 }} - role: gc -{{- toYaml (omit .Values.gc.podDisruptionBudget "enable") | nindent 2 }} -{{- end }} -{{- end }} diff --git a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/nfd-gc.yaml b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/nfd-gc.yaml index c0e8d083..3642aa64 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/nfd-gc.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/nfd-gc.yaml @@ -29,11 +29,14 @@ spec: {{- end }} spec: serviceAccountName: {{ include "node-feature-discovery.gc.serviceAccountName" . }} - dnsPolicy: {{ .Values.gc.dnsPolicy }} + dnsPolicy: ClusterFirstWithHostNet {{- with .Values.priorityClassName }} priorityClassName: {{ . }} {{- end }} - imagePullSecrets: {{ include "node-feature-discovery.imagePullSecrets" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} securityContext: {{- toYaml .Values.gc.podSecurityContext | nindent 8 }} hostNetwork: {{ .Values.gc.hostNetwork }} @@ -41,41 +44,6 @@ spec: - name: gc image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: "{{ .Values.image.pullPolicy }}" - livenessProbe: - httpGet: - path: /healthz - port: http - {{- with .Values.gc.livenessProbe.initialDelaySeconds }} - initialDelaySeconds: {{ . }} - {{- end }} - {{- with .Values.gc.livenessProbe.failureThreshold }} - failureThreshold: {{ . }} - {{- end }} - {{- with .Values.gc.livenessProbe.periodSeconds }} - periodSeconds: {{ . }} - {{- end }} - {{- with .Values.gc.livenessProbe.timeoutSeconds }} - timeoutSeconds: {{ . }} - {{- end }} - readinessProbe: - httpGet: - path: /healthz - port: http - {{- with .Values.gc.readinessProbe.initialDelaySeconds }} - initialDelaySeconds: {{ . }} - {{- end }} - {{- with .Values.gc.readinessProbe.failureThreshold }} - failureThreshold: {{ . }} - {{- end }} - {{- with .Values.gc.readinessProbe.periodSeconds }} - periodSeconds: {{ . }} - {{- end }} - {{- with .Values.gc.readinessProbe.timeoutSeconds }} - timeoutSeconds: {{ . }} - {{- end }} - {{- with .Values.gc.readinessProbe.successThreshold }} - successThreshold: {{ . }} - {{- end }} env: - name: NODE_NAME valueFrom: @@ -102,8 +70,8 @@ spec: readOnlyRootFilesystem: true runAsNonRoot: true ports: - - name: http - containerPort: {{ .Values.gc.port | default "8080"}} + - name: metrics + containerPort: {{ .Values.gc.metricsPort | default "8081"}} {{- with .Values.gc.nodeSelector }} nodeSelector: diff --git a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/post-delete-job.yaml b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/post-delete-job.yaml index ddb1c9db..4364f1aa 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/post-delete-job.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/post-delete-job.yaml @@ -1,4 +1,3 @@ -{{- if .Values.postDeleteCleanup }} apiVersion: v1 kind: ServiceAccount metadata: @@ -67,7 +66,6 @@ spec: role: prune spec: serviceAccountName: {{ include "node-feature-discovery.fullname" . }}-prune - imagePullSecrets: {{ include "node-feature-discovery.imagePullSecrets" . }} containers: - name: nfd-master securityContext: @@ -94,9 +92,3 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.master.resources }} - resources: - {{- toYaml . | nindent 8 }} - {{- end }} -{{- end }} - diff --git a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/prometheus.yaml b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/prometheus.yaml index bed37f4c..3d680e24 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/prometheus.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/prometheus.yaml @@ -14,7 +14,7 @@ spec: - honorLabels: true interval: {{ .Values.prometheus.scrapeInterval }} path: /metrics - port: http + port: metrics scheme: http namespaceSelector: matchNames: diff --git a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/topologyupdater.yaml b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/topologyupdater.yaml index 010e8a0b..9a466f88 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/topologyupdater.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/topologyupdater.yaml @@ -29,11 +29,14 @@ spec: {{- end }} spec: serviceAccountName: {{ include "node-feature-discovery.topologyUpdater.serviceAccountName" . }} - dnsPolicy: {{ .Values.topologyUpdater.dnsPolicy }} + dnsPolicy: ClusterFirstWithHostNet {{- with .Values.priorityClassName }} priorityClassName: {{ . }} {{- end }} - imagePullSecrets: {{ include "node-feature-discovery.imagePullSecrets" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} securityContext: {{- toYaml .Values.topologyUpdater.podSecurityContext | nindent 8 }} hostNetwork: {{ .Values.topologyUpdater.hostNetwork }} @@ -42,9 +45,8 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: "{{ .Values.image.pullPolicy }}" livenessProbe: - httpGet: - path: /healthz - port: http + grpc: + port: {{ .Values.topologyUpdater.healthPort | default "8082" }} {{- with .Values.topologyUpdater.livenessProbe.initialDelaySeconds }} initialDelaySeconds: {{ . }} {{- end }} @@ -58,9 +60,8 @@ spec: timeoutSeconds: {{ . }} {{- end }} readinessProbe: - httpGet: - path: /healthz - port: http + grpc: + port: {{ .Values.topologyUpdater.healthPort | default "8082" }} {{- with .Values.topologyUpdater.readinessProbe.initialDelaySeconds }} initialDelaySeconds: {{ . }} {{- end }} @@ -112,13 +113,16 @@ spec: # Disable kubelet state tracking by giving an empty path - "-kubelet-state-dir=" {{- end }} - - "-port={{ .Values.topologyUpdater.port | default "8080"}}" + - "-metrics={{ .Values.topologyUpdater.metricsPort | default "8081"}}" + - "-grpc-health={{ .Values.topologyUpdater.healthPort | default "8082" }}" {{- with .Values.topologyUpdater.extraArgs }} {{- toYaml . | nindent 10 }} {{- end }} ports: - - containerPort: {{ .Values.topologyUpdater.port | default "8080"}} - name: http + - containerPort: {{ .Values.topologyUpdater.metricsPort | default "8081"}} + name: metrics + - containerPort: {{ .Values.topologyUpdater.healthPort | default "8082" }} + name: health volumeMounts: {{- if .Values.topologyUpdater.kubeletConfigPath | empty | not }} - name: kubelet-config diff --git a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/worker.yaml b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/worker.yaml index 58c3e041..4aadd800 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/worker.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/worker.yaml @@ -13,10 +13,6 @@ metadata: {{- end }} spec: revisionHistoryLimit: {{ .Values.worker.revisionHistoryLimit }} - {{- with .Values.worker.updateStrategy }} - updateStrategy: - {{- toYaml . | nindent 4 }} - {{- end}} selector: matchLabels: {{- include "node-feature-discovery.selectorLabels" . | nindent 6 }} @@ -32,11 +28,14 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} spec: - dnsPolicy: {{ .Values.worker.dnsPolicy }} + dnsPolicy: ClusterFirstWithHostNet {{- with .Values.priorityClassName }} priorityClassName: {{ . }} {{- end }} - imagePullSecrets: {{ include "node-feature-discovery.imagePullSecrets" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} serviceAccountName: {{ include "node-feature-discovery.worker.serviceAccountName" . }} securityContext: {{- toYaml .Values.worker.podSecurityContext | nindent 8 }} @@ -48,9 +47,8 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} livenessProbe: - httpGet: - path: /healthz - port: http + grpc: + port: {{ .Values.worker.healthPort | default "8082" }} {{- with .Values.worker.livenessProbe.initialDelaySeconds }} initialDelaySeconds: {{ . }} {{- end }} @@ -64,9 +62,8 @@ spec: timeoutSeconds: {{ . }} {{- end }} readinessProbe: - httpGet: - path: /healthz - port: http + grpc: + port: {{ .Values.worker.healthPort | default "8082" }} {{- with .Values.worker.readinessProbe.initialDelaySeconds }} initialDelaySeconds: {{ . }} {{- end }} @@ -107,13 +104,16 @@ spec: {{- range $key, $value := .Values.featureGates }} - "-feature-gates={{ $key }}={{ $value }}" {{- end }} - - "-port={{ .Values.worker.port | default "8080"}}" + - "-metrics={{ .Values.worker.metricsPort | default "8081"}}" + - "-grpc-health={{ .Values.worker.healthPort | default "8082" }}" {{- with .Values.worker.extraArgs }} {{- toYaml . | nindent 8 }} {{- end }} ports: - - containerPort: {{ .Values.worker.port | default "8080"}} - name: http + - containerPort: {{ .Values.worker.metricsPort | default "8081"}} + name: metrics + - containerPort: {{ .Values.worker.healthPort | default "8082" }} + name: health volumeMounts: - name: host-boot mountPath: "/host-boot" diff --git a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/values.yaml b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/values.yaml index 86712bd2..18aa7bcb 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/values.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/values.yaml @@ -15,20 +15,17 @@ featureGates: priorityClassName: "" -postDeleteCleanup: true - master: enable: true extraArgs: [] extraEnvs: [] hostNetwork: false - dnsPolicy: ClusterFirstWithHostNet config: ### # noPublish: false + # autoDefaultNs: true # extraLabelNs: ["added.ns.io","added.kubernets.io"] # denyLabelNs: ["denied.ns.io","denied.kubernetes.io"] # enableTaints: false - # informerPageSize: 200 # labelWhiteList: "foo" # resyncPeriod: "2h" # restrictions: @@ -69,12 +66,15 @@ master: # retryPeriod: 2s # nfdApiParallelism: 10 ### - port: 8080 + metricsPort: 8081 + healthPort: 8082 instance: + featureApi: resyncPeriod: denyLabelNs: [] extraLabelNs: [] enableTaints: false + featureRulesController: null nfdApiParallelism: null deploymentAnnotations: {} replicaCount: 1 @@ -120,21 +120,26 @@ master: nodeSelector: {} tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Equal" + value: "" + effect: "NoSchedule" - key: "node-role.kubernetes.io/control-plane" operator: "Equal" value: "" effect: "NoSchedule" - podDisruptionBudget: - enable: false - minAvailable: 1 - unhealthyPodEvictionPolicy: AlwaysAllow - annotations: {} affinity: nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: "node-role.kubernetes.io/master" + operator: In + values: [""] - weight: 1 preference: matchExpressions: @@ -143,14 +148,20 @@ master: values: [""] startupProbe: + grpc: + port: 8082 failureThreshold: 30 # periodSeconds: 10 - livenessProbe: {} + livenessProbe: + grpc: + port: 8082 # failureThreshold: 3 # initialDelaySeconds: 0 # periodSeconds: 10 # timeoutSeconds: 1 readinessProbe: + grpc: + port: 8082 failureThreshold: 10 # initialDelaySeconds: 0 # periodSeconds: 10 @@ -162,7 +173,6 @@ worker: extraArgs: [] extraEnvs: [] hostNetwork: false - dnsPolicy: ClusterFirstWithHostNet config: ### #core: # labelWhiteList: @@ -406,7 +416,8 @@ worker: # matchName: {op: In, value: ["SWAP", "X86", "ARM"]} ### - port: 8080 + metricsPort: 8081 + healthPort: 8082 daemonsetAnnotations: {} podSecurityContext: {} # fsGroup: 2000 @@ -420,11 +431,15 @@ worker: # runAsUser: 1000 livenessProbe: + grpc: + port: 8082 initialDelaySeconds: 10 # failureThreshold: 3 # periodSeconds: 10 # timeoutSeconds: 1 readinessProbe: + grpc: + port: 8082 initialDelaySeconds: 5 failureThreshold: 10 # periodSeconds: 10 @@ -468,12 +483,6 @@ worker: priorityClassName: "" - updateStrategy: {} - # type: RollingUpdate - # rollingUpdate: - # maxSurge: 0 - # maxUnavailable: "10%" - topologyUpdater: config: ### ## key = node name, value = list of resources to be excluded. @@ -490,7 +499,6 @@ topologyUpdater: extraArgs: [] extraEnvs: [] hostNetwork: false - dnsPolicy: ClusterFirstWithHostNet serviceAccount: create: true @@ -503,7 +511,8 @@ topologyUpdater: rbac: create: true - port: 8080 + metricsPort: 8081 + healthPort: 8082 kubeletConfigPath: kubeletPodResourcesSockPath: updateInterval: 60s @@ -517,13 +526,17 @@ topologyUpdater: drop: [ "ALL" ] readOnlyRootFilesystem: true runAsUser: 0 - + livenessProbe: + grpc: + port: 8082 initialDelaySeconds: 10 # failureThreshold: 3 # periodSeconds: 10 # timeoutSeconds: 1 readinessProbe: + grpc: + port: 8082 initialDelaySeconds: 5 failureThreshold: 10 # periodSeconds: 10 @@ -550,7 +563,6 @@ gc: extraEnvs: [] hostNetwork: false replicaCount: 1 - dnsPolicy: ClusterFirstWithHostNet serviceAccount: create: true @@ -563,18 +575,6 @@ gc: podSecurityContext: {} - livenessProbe: - initialDelaySeconds: 10 - # failureThreshold: 3 - # periodSeconds: 10 - # timeoutSeconds: 1 - readinessProbe: - initialDelaySeconds: 5 - # failureThreshold: 3 - # periodSeconds: 10 - # timeoutSeconds: 1 - # successThreshold: 1 - resources: limits: memory: 1Gi @@ -582,7 +582,7 @@ gc: cpu: 10m memory: 128Mi - port: 8080 + metricsPort: 8081 nodeSelector: {} tolerations: [] @@ -590,11 +590,6 @@ gc: deploymentAnnotations: {} affinity: {} - podDisruptionBudget: - enable: false - minAvailable: 1 - unhealthyPodEvictionPolicy: AlwaysAllow - # specify how many old ReplicaSets for the Deployment to retain. revisionHistoryLimit: diff --git a/packages/system/gpu-operator/charts/gpu-operator/crds/nvidia.com_clusterpolicies.yaml b/packages/system/gpu-operator/charts/gpu-operator/crds/nvidia.com_clusterpolicies.yaml index 29737027..5a9f99e1 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/crds/nvidia.com_clusterpolicies.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/crds/nvidia.com_clusterpolicies.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.1 + controller-gen.kubebuilder.io/version: v0.17.2 name: clusterpolicies.nvidia.com spec: group: nvidia.com @@ -82,10 +82,6 @@ spec: - name type: object type: array - hostNetwork: - description: HostNetwork indicates whether the CC Manager pod - uses the host's network namespace. - type: boolean image: description: CC Manager image name pattern: '[a-zA-Z0-9\-]+' @@ -140,21 +136,13 @@ spec: properties: default: default: false - description: 'Deprecated: This field is no longer used. Setting - cdi.enabled=true will configure CDI as the default mechanism - for making GPUs accessible to containers.' + description: Default indicates whether to use CDI as the default + mechanism for providing GPU access to containers. type: boolean enabled: - default: true - description: Enabled indicates whether the Container Device Interface - (CDI) should be used as the mechanism for making GPUs accessible - to containers. - type: boolean - nriPluginEnabled: default: false - description: NRIPluginEnabled indicates whether an NRI Plugin - should be run as a means of injecting CDI devices to gpu management - containers. + description: Enabled indicates whether CDI can be used to make + GPUs accessible to containers. type: boolean type: object daemonsets: @@ -176,239 +164,6 @@ spec: (scope and select) objects. May match selectors of replication controllers and services. type: object - podSecurityContext: - description: 'Optional: Set pod-level security context for all - DaemonSet pods (applies as defaults to all containers)' - properties: - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by the containers in this pod. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - fsGroup: - description: |- - A special supplemental group that applies to all containers in a pod. - Some volume types allow the Kubelet to change the ownership of that volume - to be owned by the pod: - - 1. The owning GID will be the FSGroup - 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) - 3. The permission bits are OR'd with rw-rw---- - - If unset, the Kubelet will not modify the ownership and permissions of any volume. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - fsGroupChangePolicy: - description: |- - fsGroupChangePolicy defines behavior of changing ownership and permission of the volume - before being exposed inside Pod. This field will only apply to - volume types which support fsGroup based ownership(and permissions). - It will have no effect on ephemeral volume types such as: secret, configmaps - and emptydir. - Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. - Note that this field cannot be set when spec.os.name is windows. - type: string - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxChangePolicy: - description: |- - seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. - It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. - Valid values are "MountOption" and "Recursive". - - "Recursive" means relabeling of all files on all Pod volumes by the container runtime. - This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. - - "MountOption" mounts all eligible Pod volumes with `-o context` mount option. - This requires all Pods that share the same volume to use the same SELinux label. - It is not possible to share the same volume among privileged and unprivileged Pods. - Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes - whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their - CSIDriver instance. Other volumes are always re-labelled recursively. - "MountOption" value is allowed only when SELinuxMount feature gate is enabled. - - If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. - If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes - and "Recursive" for all other volumes. - - This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. - - All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. - Note that this field cannot be set when spec.os.name is windows. - type: string - seLinuxOptions: - description: |- - The SELinux context to be applied to all containers. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in SecurityContext. If set in - both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence for that container. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that applies - to the container. - type: string - role: - description: Role is a SELinux role label that applies - to the container. - type: string - type: - description: Type is a SELinux type label that applies - to the container. - type: string - user: - description: User is a SELinux user label that applies - to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by the containers in this pod. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - supplementalGroups: - description: |- - A list of groups applied to the first process run in each container, in - addition to the container's primary GID and fsGroup (if specified). If - the SupplementalGroupsPolicy feature is enabled, the - supplementalGroupsPolicy field determines whether these are in addition - to or instead of any group memberships defined in the container image. - If unspecified, no additional groups are added, though group memberships - defined in the container image may still be used, depending on the - supplementalGroupsPolicy field. - Note that this field cannot be set when spec.os.name is windows. - items: - format: int64 - type: integer - type: array - x-kubernetes-list-type: atomic - supplementalGroupsPolicy: - description: |- - Defines how supplemental groups of the first container processes are calculated. - Valid values are "Merge" and "Strict". If not specified, "Merge" is used. - (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled - and the container runtime must implement support for this feature. - Note that this field cannot be set when spec.os.name is windows. - type: string - sysctls: - description: |- - Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported - sysctls (by the container runtime) might fail to launch. - Note that this field cannot be set when spec.os.name is windows. - items: - description: Sysctl defines a kernel parameter to be set - properties: - name: - description: Name of a property to set - type: string - value: - description: Value of a property to set - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options within a container's SecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the - GMSA credential spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object priorityClassName: type: string rollingUpdate: @@ -438,10 +193,9 @@ spec: operator: description: |- Operator represents a key's relationship to the value. - Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. - Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string tolerationSeconds: description: |- @@ -493,10 +247,6 @@ spec: - name type: object type: array - hostNetwork: - description: HostNetwork indicates whether the DCGM pod uses the - host's network namespace. - type: boolean hostPort: description: 'Deprecated: HostPort represents host port that needs to be bound for DCGM engine (Default: 5555)' @@ -587,27 +337,6 @@ spec: - name type: object type: array - hostNetwork: - description: HostNetwork allows the DCGM-Exporter daemon set to - expose metrics port on the host's network namespace. - type: boolean - hostPID: - description: HostPID allows the DCGM-Exporter daemon set to access - the host's PID namespace - type: boolean - hpcJobMapping: - description: 'Optional: HPC job mapping configuration for NVIDIA - DCGM Exporter' - properties: - directory: - description: |- - Directory path where HPC job mapping files are created by the workload manager - Defaults to /var/lib/dcgm-exporter/job-mapping if not specified - type: string - enabled: - description: Enable HPC job mapping for DCGM Exporter - type: boolean - type: object image: description: NVIDIA DCGM Exporter image name pattern: '[a-zA-Z0-9\-]+' @@ -652,19 +381,6 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object - service: - description: 'Optional: Service configuration for NVIDIA DCGM - Exporter' - properties: - internalTrafficPolicy: - description: InternalTrafficPolicy describes how nodes distribute - service traffic they receive on the ClusterIP. - type: string - type: - description: Type represents the ServiceType which describes - ingress methods for a service - type: string - type: object serviceMonitor: description: 'Optional: ServiceMonitor configuration for NVIDIA DCGM Exporter' @@ -702,7 +418,7 @@ spec: action: default: replace description: |- - action to perform based on the regex matching. + Action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -734,41 +450,41 @@ spec: type: string modulus: description: |- - modulus to take of the hash of the source label values. + Modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: regex defines the regular expression against - which the extracted value is matched. + description: Regular expression against which the extracted + value is matched. type: string replacement: description: |- - replacement value against which a Replace action is performed if the + Replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: separator defines the string between concatenated + description: Separator is the string between concatenated SourceLabels. type: string sourceLabels: description: |- - sourceLabels defines the source labels select values from existing labels. Their content is + The source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name. - For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. - For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. + LabelName is a valid Prometheus label name which may only contain ASCII + letters, numbers, as well as underscores. + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ type: string type: array targetLabel: description: |- - targetLabel defines the label to which the resulting string is written in a replacement. + Label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -823,10 +539,6 @@ spec: - name type: object type: array - hostNetwork: - description: HostNetwork indicates whether the Device Plugin pod - uses the host's network namespace. - type: boolean image: description: NVIDIA Device Plugin image name pattern: '[a-zA-Z0-9\-]+' @@ -919,10 +631,6 @@ spec: - name type: object type: array - hostNetwork: - description: HostNetwork indicates whether the Driver pod uses - the host's network namespace. - type: boolean image: description: NVIDIA Driver image name pattern: '[a-zA-Z0-9\-]+' @@ -958,18 +666,11 @@ spec: licensing' properties: configMapName: - description: 'Deprecated: ConfigMapName has been deprecated - in favour of SecretName. Please use secrets to handle the - licensing server configuration more securely' type: string nlsEnabled: description: NLSEnabled indicates if NVIDIA Licensing System is used for licensing. type: boolean - secretName: - description: SecretName indicates the name of the secret containing - the licensing token - type: string type: object livenessProbe: description: NVIDIA Driver container liveness probe settings @@ -1143,10 +844,6 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object - secretEnv: - description: 'Optional: SecretEnv represents the name of the Kubernetes - Secret with secret environment variables for the NVIDIA Driver' - type: string startupProbe: description: NVIDIA Driver container startup probe settings properties: @@ -1435,10 +1132,6 @@ spec: - name type: object type: array - hostNetwork: - description: HostNetwork indicates whether the GPU Feature Discovery - pod uses the host's network namespace. - type: boolean image: description: GFD image name pattern: '[a-zA-Z0-9\-]+' @@ -1582,10 +1275,6 @@ spec: - name type: object type: array - hostNetwork: - description: HostNetwork indicates whether the Kata Manager pod - uses the host's network namespace. - type: boolean image: description: Kata Manager image name pattern: '[a-zA-Z0-9\-]+' @@ -1634,86 +1323,6 @@ spec: description: Kata Manager image tag type: string type: object - kataSandboxDevicePlugin: - description: KataSandboxDevicePlugin component spec - properties: - args: - description: 'Optional: List of arguments' - items: - type: string - type: array - enabled: - description: Enabled indicates if deployment of NVIDIA component - through operator is enabled - type: boolean - env: - description: 'Optional: List of environment variables' - items: - description: EnvVar represents an environment variable present - in a Container. - properties: - name: - description: Name of the environment variable. - type: string - value: - description: Value of the environment variable. - type: string - required: - - name - type: object - type: array - hostNetwork: - description: HostNetwork indicates whether the Kata Sandbox Device - Plugin pod uses the host's network namespace. - type: boolean - image: - description: NVIDIA component image name - pattern: '[a-zA-Z0-9\-]+' - type: string - imagePullPolicy: - description: Image pull policy - type: string - imagePullSecrets: - description: Image pull secrets - items: - type: string - type: array - repository: - description: NVIDIA component image repository - type: string - resources: - description: 'Optional: Define resources requests and limits for - each pod' - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - version: - description: NVIDIA component image tag - type: string - type: object mig: description: MIG spec properties: @@ -1747,8 +1356,8 @@ spec: - "" type: string name: - description: ConfigMap name. If not specified, MIG configuration - will be dynamically generated from hardware. + default: default-mig-parted-config + description: ConfigMap name type: string type: object enabled: @@ -1779,10 +1388,6 @@ spec: description: ConfigMap name type: string type: object - hostNetwork: - description: HostNetwork indicates whether the MIG Manager pod - uses the host's network namespace. - type: boolean image: description: NVIDIA MIG Manager image name pattern: '[a-zA-Z0-9\-]+' @@ -1859,10 +1464,6 @@ spec: - name type: object type: array - hostNetwork: - description: HostNetwork indicates whether the Node Status Exporter - pod uses the host's network namespace. - type: boolean image: description: Node Status Exporter image name pattern: '[a-zA-Z0-9\-]+' @@ -1923,12 +1524,16 @@ spec: queryable and should be preserved when modifying objects. type: object defaultRuntime: - description: 'Deprecated: DefaultRuntime is no longer used by - the gpu-operator. This is instead, detected at runtime.' + default: docker + description: Runtime defines container runtime type + enum: + - docker + - crio + - containerd type: string initContainer: - description: 'Deprecated: InitContainerSpec describes configuration - for initContainer image used with all components' + description: InitContainerSpec describes configuration for initContainer + image used with all components properties: image: description: Image represents image name @@ -1965,6 +1570,8 @@ spec: image should be used on OpenShift to build and install driver modules type: boolean + required: + - defaultRuntime type: object psa: description: PSA defines spec for PodSecurityAdmission configuration @@ -2012,10 +1619,6 @@ spec: - name type: object type: array - hostNetwork: - description: HostNetwork indicates whether the Sandbox Device - Plugin pod uses the host's network namespace. - type: boolean image: description: NVIDIA Sandbox Device Plugin image name pattern: '[a-zA-Z0-9\-]+' @@ -2083,15 +1686,6 @@ spec: Enabled indicates if the GPU Operator should manage additional operands required for sandbox workloads (i.e. VFIO Manager, vGPU Manager, and additional device plugins) type: boolean - mode: - default: kubevirt - description: |- - Mode indicates the sandbox mode. Accepted values are "kubevirt" - and "kata". The default value is "kubevirt". - enum: - - kubevirt - - kata - type: string type: object toolkit: description: Toolkit component spec @@ -2121,10 +1715,6 @@ spec: - name type: object type: array - hostNetwork: - description: HostNetwork indicates whether the Container Toolkit - pod uses the host's network namespace. - type: boolean image: description: NVIDIA Container Toolkit image name pattern: '[a-zA-Z0-9\-]+' @@ -2241,10 +1831,6 @@ spec: - name type: object type: array - hostNetwork: - description: HostNetwork indicates whether the Validator pod uses - the host's network namespace. - type: boolean image: description: Validator image name pattern: '[a-zA-Z0-9\-]+' @@ -2463,10 +2049,6 @@ spec: - name type: object type: array - hostNetwork: - description: HostNetwork indicates whether the VFIO Manager pod - uses the host's network namespace. - type: boolean image: description: VFIO Manager image name pattern: '[a-zA-Z0-9\-]+' @@ -2555,10 +2137,6 @@ spec: - name type: object type: array - hostNetwork: - description: HostNetwork indicates whether the vGPU Device Manager - pod uses the host's network namespace. - type: boolean image: description: NVIDIA vGPU Device Manager image name pattern: '[a-zA-Z0-9\-]+' @@ -2677,10 +2255,6 @@ spec: - name type: object type: array - hostNetwork: - description: HostNetwork indicates whether the vGPU Manager pod - uses the host's network namespace. - type: boolean image: description: NVIDIA vGPU Manager image name pattern: '[a-zA-Z0-9\-]+' @@ -2693,13 +2267,6 @@ spec: items: type: string type: array - kernelModuleConfig: - description: 'Optional: Kernel module configuration parameters - for the vGPU manager' - properties: - name: - type: string - type: object repository: description: NVIDIA vGPU Manager image repository type: string diff --git a/packages/system/gpu-operator/charts/gpu-operator/crds/nvidia.com_nvidiadrivers.yaml b/packages/system/gpu-operator/charts/gpu-operator/crds/nvidia.com_nvidiadrivers.yaml index 81c23415..97e023bf 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/crds/nvidia.com_nvidiadrivers.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/crds/nvidia.com_nvidiadrivers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.1 + controller-gen.kubebuilder.io/version: v0.17.2 name: nvidiadrivers.nvidia.com spec: group: nvidia.com @@ -192,10 +192,6 @@ spec: description: NVIDIA GPUDirect Storage Driver image tag type: string type: object - hostNetwork: - description: HostNetwork indicates whether the Driver pod uses the - host's network namespace. - type: boolean image: default: nvcr.io/nvidia/driver description: NVIDIA Driver container image name @@ -238,16 +234,11 @@ spec: description: 'Optional: Licensing configuration for NVIDIA vGPU licensing' properties: name: - description: 'Deprecated: ConfigMapName has been deprecated in - favour of SecretName. Please use secrets to handle the licensing - server configuration more securely' type: string nlsEnabled: description: NLSEnabled indicates if NVIDIA Licensing System is used for licensing. type: boolean - secretName: - type: string type: object livenessProbe: description: NVIDIA Driver container liveness probe settings @@ -531,239 +522,6 @@ spec: description: NodeSelector specifies a selector for installation of NVIDIA driver type: object - podSecurityContext: - description: 'Optional: Set pod-level security context for driver - pod' - properties: - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by the containers in this pod. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - fsGroup: - description: |- - A special supplemental group that applies to all containers in a pod. - Some volume types allow the Kubelet to change the ownership of that volume - to be owned by the pod: - - 1. The owning GID will be the FSGroup - 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) - 3. The permission bits are OR'd with rw-rw---- - - If unset, the Kubelet will not modify the ownership and permissions of any volume. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - fsGroupChangePolicy: - description: |- - fsGroupChangePolicy defines behavior of changing ownership and permission of the volume - before being exposed inside Pod. This field will only apply to - volume types which support fsGroup based ownership(and permissions). - It will have no effect on ephemeral volume types such as: secret, configmaps - and emptydir. - Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. - Note that this field cannot be set when spec.os.name is windows. - type: string - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxChangePolicy: - description: |- - seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. - It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. - Valid values are "MountOption" and "Recursive". - - "Recursive" means relabeling of all files on all Pod volumes by the container runtime. - This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. - - "MountOption" mounts all eligible Pod volumes with `-o context` mount option. - This requires all Pods that share the same volume to use the same SELinux label. - It is not possible to share the same volume among privileged and unprivileged Pods. - Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes - whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their - CSIDriver instance. Other volumes are always re-labelled recursively. - "MountOption" value is allowed only when SELinuxMount feature gate is enabled. - - If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. - If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes - and "Recursive" for all other volumes. - - This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. - - All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. - Note that this field cannot be set when spec.os.name is windows. - type: string - seLinuxOptions: - description: |- - The SELinux context to be applied to all containers. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in SecurityContext. If set in - both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence for that container. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that applies to - the container. - type: string - role: - description: Role is a SELinux role label that applies to - the container. - type: string - type: - description: Type is a SELinux type label that applies to - the container. - type: string - user: - description: User is a SELinux user label that applies to - the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by the containers in this pod. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - supplementalGroups: - description: |- - A list of groups applied to the first process run in each container, in - addition to the container's primary GID and fsGroup (if specified). If - the SupplementalGroupsPolicy feature is enabled, the - supplementalGroupsPolicy field determines whether these are in addition - to or instead of any group memberships defined in the container image. - If unspecified, no additional groups are added, though group memberships - defined in the container image may still be used, depending on the - supplementalGroupsPolicy field. - Note that this field cannot be set when spec.os.name is windows. - items: - format: int64 - type: integer - type: array - x-kubernetes-list-type: atomic - supplementalGroupsPolicy: - description: |- - Defines how supplemental groups of the first container processes are calculated. - Valid values are "Merge" and "Strict". If not specified, "Merge" is used. - (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled - and the container runtime must implement support for this feature. - Note that this field cannot be set when spec.os.name is windows. - type: string - sysctls: - description: |- - Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported - sysctls (by the container runtime) might fail to launch. - Note that this field cannot be set when spec.os.name is windows. - items: - description: Sysctl defines a kernel parameter to be set - properties: - name: - description: Name of a property to set - type: string - value: - description: Value of a property to set - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options within a container's SecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA - credential spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object priorityClassName: description: 'Optional: Set priorityClassName' type: string @@ -858,10 +616,6 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object - secretEnv: - description: 'Optional: SecretEnv represents the name of the Kubernetes - Secret with secret environment variables for the NVIDIA Driver' - type: string startupProbe: description: NVIDIA Driver container startup probe settings properties: @@ -921,10 +675,9 @@ spec: operator: description: |- Operator represents a key's relationship to the value. - Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. - Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string tolerationSeconds: description: |- @@ -1045,7 +798,6 @@ spec: - ignored - ready - notReady - - disabled type: string required: - state diff --git a/packages/system/gpu-operator/charts/gpu-operator/templates/cleanup_crd.yaml b/packages/system/gpu-operator/charts/gpu-operator/templates/cleanup_crd.yaml index 0d426f95..670bedc2 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/templates/cleanup_crd.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/templates/cleanup_crd.yaml @@ -35,13 +35,16 @@ spec: image: {{ include "gpu-operator.fullimage" . }} imagePullPolicy: {{ .Values.operator.imagePullPolicy }} command: - - /usr/bin/manage-crds - args: - - delete - - --filepath=/opt/gpu-operator/nvidia.com_clusterpolicies.yaml - - --filepath=/opt/gpu-operator/nvidia.com_nvidiadrivers.yaml - {{- if .Values.nfd.enabled }} - - --filepath=/opt/gpu-operator/nfd-api-crds.yaml - {{- end }} + - /bin/sh + - -c + - > + kubectl delete clusterpolicy cluster-policy; + kubectl delete crd clusterpolicies.nvidia.com; + kubectl delete crd nvidiadrivers.nvidia.com --ignore-not-found=true; + {{- if .Values.nfd.enabled -}} + kubectl delete crd nodefeatures.nfd.k8s-sigs.io --ignore-not-found=true; + kubectl delete crd nodefeaturegroups.nfd.k8s-sigs.io --ignore-not-found=true; + kubectl delete crd nodefeaturerules.nfd.k8s-sigs.io --ignore-not-found=true; + {{- end }} restartPolicy: OnFailure {{- end }} diff --git a/packages/system/gpu-operator/charts/gpu-operator/templates/clusterpolicy.yaml b/packages/system/gpu-operator/charts/gpu-operator/templates/clusterpolicy.yaml index 6e5c6a9c..763716d7 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/templates/clusterpolicy.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/templates/clusterpolicy.yaml @@ -85,65 +85,49 @@ spec: {{- if .Values.validator.args }} args: {{ toYaml .Values.validator.args | nindent 6 }} {{- end }} - {{- if .Values.validator.hostNetwork }} - hostNetwork: {{ .Values.validator.hostNetwork }} - {{- end }} {{- if .Values.validator.plugin }} plugin: {{- if .Values.validator.plugin.env }} env: {{ toYaml .Values.validator.plugin.env | nindent 8 }} - {{- else }} - env: [] {{- end }} {{- end }} {{- if .Values.validator.cuda }} cuda: {{- if .Values.validator.cuda.env }} env: {{ toYaml .Values.validator.cuda.env | nindent 8 }} - {{- else }} - env: [] {{- end }} {{- end }} {{- if .Values.validator.driver }} driver: {{- if .Values.validator.driver.env }} env: {{ toYaml .Values.validator.driver.env | nindent 8 }} - {{- else }} - env: [] {{- end }} {{- end }} {{- if .Values.validator.toolkit }} toolkit: {{- if .Values.validator.toolkit.env }} env: {{ toYaml .Values.validator.toolkit.env | nindent 8 }} - {{- else }} - env: [] {{- end }} {{- end }} {{- if .Values.validator.vfioPCI }} vfioPCI: {{- if .Values.validator.vfioPCI.env }} env: {{ toYaml .Values.validator.vfioPCI.env | nindent 8 }} - {{- else }} - env: [] {{- end }} {{- end }} {{- if .Values.validator.vgpuManager }} vgpuManager: {{- if .Values.validator.vgpuManager.env }} env: {{ toYaml .Values.validator.vgpuManager.env | nindent 8 }} - {{- else }} - env: [] {{- end }} {{- end }} {{- if .Values.validator.vgpuDevices }} vgpuDevices: {{- if .Values.validator.vgpuDevices.env }} env: {{ toYaml .Values.validator.vgpuDevices.env | nindent 8 }} - {{- else }} - env: [] {{- end }} {{- end }} + mig: {{- if .Values.mig.strategy }} strategy: {{ .Values.mig.strategy }} @@ -152,12 +136,7 @@ spec: enabled: {{ .Values.psa.enabled }} cdi: enabled: {{ .Values.cdi.enabled }} - {{- if .Values.cdi.default }} default: {{ .Values.cdi.default }} - {{- end }} - {{- if and (.Values.cdi.enabled) (.Values.cdi.nriPluginEnabled) }} - nriPluginEnabled: {{ .Values.cdi.nriPluginEnabled }} - {{- end }} driver: enabled: {{ .Values.driver.enabled }} useNvidiaDriverCRD: {{ .Values.driver.nvidiaDriverCRD.enabled }} @@ -221,9 +200,6 @@ spec: {{- if .Values.driver.kernelModuleConfig }} kernelModuleConfig: {{ toYaml .Values.driver.kernelModuleConfig | nindent 6 }} {{- end }} - {{- if .Values.driver.secretEnv }} - secretEnv: {{ .Values.driver.secretEnv }} - {{- end }} {{- if .Values.driver.resources }} resources: {{ toYaml .Values.driver.resources | nindent 6 }} {{- end }} @@ -256,9 +232,6 @@ spec: timeoutSeconds: {{ .Values.driver.upgradePolicy.drain.timeoutSeconds }} deleteEmptyDir: {{ .Values.driver.upgradePolicy.drain.deleteEmptyDir | default false}} {{- end }} - {{- if .Values.driver.hostNetwork }} - hostNetwork: {{ .Values.driver.hostNetwork }} - {{- end }} vgpuManager: enabled: {{ .Values.vgpuManager.enabled }} {{- if .Values.vgpuManager.repository }} @@ -285,12 +258,6 @@ spec: {{- if .Values.vgpuManager.args }} args: {{ toYaml .Values.vgpuManager.args | nindent 6 }} {{- end }} - {{- if .Values.vgpuManager.kernelModuleConfig }} - kernelModuleConfig: {{ toYaml .Values.vgpuManager.kernelModuleConfig | nindent 6 }} - {{- end }} - {{- if .Values.vgpuManager.hostNetwork }} - hostNetwork: {{ .Values.vgpuManager.hostNetwork }} - {{- end }} driverManager: {{- if .Values.vgpuManager.driverManager.repository }} repository: {{ .Values.vgpuManager.driverManager.repository }} @@ -309,9 +276,7 @@ spec: {{- end }} kataManager: enabled: {{ .Values.kataManager.enabled }} - {{- if .Values.kataManager.config }} config: {{ toYaml .Values.kataManager.config | nindent 6 }} - {{- end }} {{- if .Values.kataManager.repository }} repository: {{ .Values.kataManager.repository }} {{- end }} @@ -336,9 +301,6 @@ spec: {{- if .Values.kataManager.args }} args: {{ toYaml .Values.kataManager.args | nindent 6 }} {{- end }} - {{- if .Values.kataManager.hostNetwork }} - hostNetwork: {{ .Values.kataManager.hostNetwork }} - {{- end }} vfioManager: enabled: {{ .Values.vfioManager.enabled }} {{- if .Values.vfioManager.repository }} @@ -381,9 +343,6 @@ spec: {{- if .Values.vfioManager.driverManager.env }} env: {{ toYaml .Values.vfioManager.driverManager.env | nindent 8 }} {{- end }} - {{- if .Values.vfioManager.hostNetwork }} - hostNetwork: {{ .Values.vfioManager.hostNetwork }} - {{- end }} vgpuDeviceManager: enabled: {{ .Values.vgpuDeviceManager.enabled }} {{- if .Values.vgpuDeviceManager.repository }} @@ -413,9 +372,6 @@ spec: {{- if .Values.vgpuDeviceManager.config }} config: {{ toYaml .Values.vgpuDeviceManager.config | nindent 6 }} {{- end }} - {{- if .Values.vgpuDeviceManager.hostNetwork }} - hostNetwork: {{ .Values.vgpuDeviceManager.hostNetwork }} - {{- end }} ccManager: enabled: {{ .Values.ccManager.enabled }} defaultMode: {{ .Values.ccManager.defaultMode | quote }} @@ -438,14 +394,11 @@ spec: resources: {{ toYaml .Values.ccManager.resources | nindent 6 }} {{- end }} {{- if .Values.ccManager.env }} - env: {{ toYaml .Values.ccManager.env | nindent 6 }} + env: {{ toYaml .Values.vfioManager.env | nindent 6 }} {{- end }} {{- if .Values.ccManager.args }} args: {{ toYaml .Values.ccManager.args | nindent 6 }} {{- end }} - {{- if .Values.ccManager.hostNetwork }} - hostNetwork: {{ .Values.ccManager.hostNetwork }} - {{- end }} toolkit: enabled: {{ .Values.toolkit.enabled }} {{- if .Values.toolkit.repository }} @@ -472,9 +425,6 @@ spec: {{- if .Values.toolkit.installDir }} installDir: {{ .Values.toolkit.installDir }} {{- end }} - {{- if .Values.toolkit.hostNetwork }} - hostNetwork: {{ .Values.toolkit.hostNetwork }} - {{- end }} devicePlugin: enabled: {{ .Values.devicePlugin.enabled }} {{- if .Values.devicePlugin.repository }} @@ -503,11 +453,8 @@ spec: {{- end }} {{- if .Values.devicePlugin.config.name }} config: - name: {{ .Values.devicePlugin.config.name | quote }} - default: {{ .Values.devicePlugin.config.default | quote }} - {{- end }} - {{- if .Values.devicePlugin.hostNetwork }} - hostNetwork: {{ .Values.devicePlugin.hostNetwork }} + name: {{ .Values.devicePlugin.config.name }} + default: {{ .Values.devicePlugin.config.default }} {{- end }} dcgm: enabled: {{ .Values.dcgm.enabled }} @@ -535,9 +482,6 @@ spec: {{- if .Values.dcgm.args }} args: {{ toYaml .Values.dcgm.args | nindent 6 }} {{- end }} - {{- if .Values.dcgm.hostNetwork }} - hostNetwork: {{ .Values.dcgm.hostNetwork }} - {{- end }} dcgmExporter: enabled: {{ .Values.dcgmExporter.enabled }} {{- if .Values.dcgmExporter.repository }} @@ -571,18 +515,6 @@ spec: {{- if .Values.dcgmExporter.serviceMonitor }} serviceMonitor: {{ toYaml .Values.dcgmExporter.serviceMonitor | nindent 6 }} {{- end }} - {{- if .Values.dcgmExporter.service }} - service: {{ toYaml .Values.dcgmExporter.service | nindent 6 }} - {{- end }} - {{- if .Values.dcgmExporter.hostPID }} - hostPID: {{ .Values.dcgmExporter.hostPID }} - {{- end }} - {{- if .Values.dcgmExporter.hostNetwork }} - hostNetwork: {{ .Values.dcgmExporter.hostNetwork }} - {{- end }} - {{- if .Values.dcgmExporter.hpcJobMapping }} - hpcJobMapping: {{ toYaml .Values.dcgmExporter.hpcJobMapping | nindent 6 }} - {{- end }} gfd: enabled: {{ .Values.gfd.enabled }} {{- if .Values.gfd.repository }} @@ -609,9 +541,6 @@ spec: {{- if .Values.gfd.args }} args: {{ toYaml .Values.gfd.args | nindent 6 }} {{- end }} - {{- if .Values.gfd.hostNetwork }} - hostNetwork: {{ .Values.gfd.hostNetwork }} - {{- end }} migManager: enabled: {{ .Values.migManager.enabled }} {{- if .Values.migManager.repository }} @@ -640,17 +569,12 @@ spec: {{- end }} {{- if .Values.migManager.config }} config: - {{- if .Values.migManager.config.name }} name: {{ .Values.migManager.config.name }} - {{- end }} - default: {{ .Values.migManager.config.default | quote }} + default: {{ .Values.migManager.config.default }} {{- end }} {{- if .Values.migManager.gpuClientsConfig }} gpuClientsConfig: {{ toYaml .Values.migManager.gpuClientsConfig | nindent 6 }} {{- end }} - {{- if .Values.migManager.hostNetwork }} - hostNetwork: {{ .Values.migManager.hostNetwork }} - {{- end }} nodeStatusExporter: enabled: {{ .Values.nodeStatusExporter.enabled }} {{- if .Values.nodeStatusExporter.repository }} @@ -675,10 +599,7 @@ spec: {{- if .Values.nodeStatusExporter.args }} args: {{ toYaml .Values.nodeStatusExporter.args | nindent 6 }} {{- end }} - {{- if .Values.nodeStatusExporter.hostNetwork }} - hostNetwork: {{ .Values.nodeStatusExporter.hostNetwork }} - {{- end }} - {{- if .Values.gds }} + {{- if .Values.gds.enabled }} gds: enabled: {{ .Values.gds.enabled }} {{- if .Values.gds.repository }} @@ -729,9 +650,6 @@ spec: {{- if .Values.sandboxWorkloads.defaultWorkload }} defaultWorkload: {{ .Values.sandboxWorkloads.defaultWorkload }} {{- end }} - {{- if .Values.sandboxWorkloads.mode }} - mode: {{ .Values.sandboxWorkloads.mode | quote }} - {{- end }} sandboxDevicePlugin: {{- if .Values.sandboxDevicePlugin.enabled }} enabled: {{ .Values.sandboxDevicePlugin.enabled }} @@ -760,37 +678,3 @@ spec: {{- if .Values.sandboxDevicePlugin.args }} args: {{ toYaml .Values.sandboxDevicePlugin.args | nindent 6 }} {{- end }} - {{- if .Values.sandboxDevicePlugin.hostNetwork }} - hostNetwork: {{ .Values.sandboxDevicePlugin.hostNetwork }} - {{- end }} - kataSandboxDevicePlugin: - {{- if ne .Values.kataSandboxDevicePlugin.enabled nil }} - enabled: {{ .Values.kataSandboxDevicePlugin.enabled }} - {{- end }} - {{- if .Values.kataSandboxDevicePlugin.repository }} - repository: {{ .Values.kataSandboxDevicePlugin.repository }} - {{- end }} - {{- if .Values.kataSandboxDevicePlugin.image }} - image: {{ .Values.kataSandboxDevicePlugin.image }} - {{- end }} - {{- if .Values.kataSandboxDevicePlugin.version }} - version: {{ .Values.kataSandboxDevicePlugin.version | quote }} - {{- end }} - {{- if .Values.kataSandboxDevicePlugin.imagePullPolicy }} - imagePullPolicy: {{ .Values.kataSandboxDevicePlugin.imagePullPolicy }} - {{- end }} - {{- if .Values.kataSandboxDevicePlugin.imagePullSecrets }} - imagePullSecrets: {{ toYaml .Values.kataSandboxDevicePlugin.imagePullSecrets | nindent 6 }} - {{- end }} - {{- if .Values.kataSandboxDevicePlugin.resources }} - resources: {{ toYaml .Values.kataSandboxDevicePlugin.resources | nindent 6 }} - {{- end }} - {{- if .Values.kataSandboxDevicePlugin.env }} - env: {{ toYaml .Values.kataSandboxDevicePlugin.env | nindent 6 }} - {{- end }} - {{- if .Values.kataSandboxDevicePlugin.args }} - args: {{ toYaml .Values.kataSandboxDevicePlugin.args | nindent 6 }} - {{- end }} - {{- if .Values.kataSandboxDevicePlugin.hostNetwork }} - hostNetwork: {{ .Values.kataSandboxDevicePlugin.hostNetwork }} - {{- end }} diff --git a/packages/system/gpu-operator/charts/gpu-operator/templates/extra-objects.yaml b/packages/system/gpu-operator/charts/gpu-operator/templates/extra-objects.yaml deleted file mode 100644 index fc9a76b8..00000000 --- a/packages/system/gpu-operator/charts/gpu-operator/templates/extra-objects.yaml +++ /dev/null @@ -1,8 +0,0 @@ -{{ range .Values.extraObjects }} ---- -{{ if typeIs "string" . }} - {{- tpl . $ }} -{{- else }} - {{- tpl (toYaml .) $ }} -{{- end }} -{{ end }} diff --git a/packages/system/gpu-operator/charts/gpu-operator/templates/nodefeaturerules.yaml b/packages/system/gpu-operator/charts/gpu-operator/templates/nodefeaturerules.yaml index 4584cffc..6076b3d3 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/templates/nodefeaturerules.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/templates/nodefeaturerules.yaml @@ -2,32 +2,106 @@ apiVersion: nfd.k8s-sigs.io/v1alpha1 kind: NodeFeatureRule metadata: - name: nvidia-kernel-modules + name: nvidia-nfd-nodefeaturerules spec: rules: - - name: kernel-module-gdrdrv + - name: "TDX rule" labels: - nvidia.com/gdrcopy.capable: "true" + tdx.enabled: "true" matchFeatures: - - feature: kernel.loadedmodule + - feature: cpu.security matchExpressions: - gdrdrv: - op: Exists - - name: kernel-module-nvidia_fs + tdx.enabled: {op: IsTrue} + - name: "TDX total keys rule" + extendedResources: + tdx.total_keys: "@cpu.security.tdx.total_keys" + matchFeatures: + - feature: cpu.security + matchExpressions: + tdx.enabled: {op: IsTrue} + - name: "SEV-SNP rule" labels: - nvidia.com/gds.capable: "true" + sev.snp.enabled: "true" matchFeatures: - - feature: kernel.loadedmodule - matchExpressions: - nvidia_fs: - op: Exists - - name: kernel-module-nvidia_peermem + - feature: cpu.security + matchExpressions: + sev.snp.enabled: + op: IsTrue + - name: "SEV-ES rule" labels: - nvidia.com/peermem.capable: "true" + sev.es.enabled: "true" matchFeatures: - - feature: kernel.loadedmodule + - feature: cpu.security + matchExpressions: + sev.es.enabled: + op: IsTrue + - name: SEV system capacities + extendedResources: + sev_asids: '@cpu.security.sev.asids' + sev_es: '@cpu.security.sev.encrypted_state_ids' + matchFeatures: + - feature: cpu.security + matchExpressions: + sev.enabled: + op: Exists + - name: "NVIDIA H100" + labels: + "nvidia.com/gpu.H100": "true" + "nvidia.com/gpu.family": "hopper" + matchFeatures: + - feature: pci.device matchExpressions: - nvidia_peermem: - op: Exists + vendor: {op: In, value: ["10de"]} + device: {op: In, value: ["2339"]} + - name: "NVIDIA H100 PCIe" + labels: + "nvidia.com/gpu.H100.pcie": "true" + "nvidia.com/gpu.family": "hopper" + matchFeatures: + - feature: pci.device + matchExpressions: + vendor: {op: In, value: ["10de"]} + device: {op: In, value: ["2331"]} + - name: "NVIDIA H100 80GB HBM3" + labels: + "nvidia.com/gpu.H100.HBM3": "true" + "nvidia.com/gpu.family": "hopper" + matchFeatures: + - feature: pci.device + matchExpressions: + vendor: {op: In, value: ["10de"]} + device: {op: In, value: ["2330"]} + - name: "NVIDIA H800" + labels: + "nvidia.com/gpu.H800": "true" + "nvidia.com/gpu.family": "hopper" + matchFeatures: + - feature: pci.device + matchExpressions: + vendor: {op: In, value: ["10de"]} + device: {op: In, value: ["2324"]} + - name: "NVIDIA H800 PCIE" + labels: + "nvidia.com/gpu.H800.pcie": "true" + "nvidia.com/gpu.family": "hopper" + matchFeatures: + - feature: pci.device + matchExpressions: + vendor: {op: In, value: ["10de"]} + device: {op: In, value: ["2322"]} + - name: "NVIDIA CC Enabled" + labels: + "nvidia.com/cc.capable": "true" + matchAny: # TDX/SEV + Hopper GPU + - matchFeatures: + - feature: rule.matched + matchExpressions: + nvidia.com/gpu.family: {op: In, value: ["hopper"]} + sev.snp.enabled: {op: IsTrue} + - matchFeatures: + - feature: rule.matched + matchExpressions: + nvidia.com/gpu.family: {op: In, value: ["hopper"]} + tdx.enabled: {op: IsTrue} {{- end }} diff --git a/packages/system/gpu-operator/charts/gpu-operator/templates/nvidiadriver.yaml b/packages/system/gpu-operator/charts/gpu-operator/templates/nvidiadriver.yaml index 1a059a49..cbe56713 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/templates/nvidiadriver.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/templates/nvidiadriver.yaml @@ -1,4 +1,3 @@ -{{- if .Values.driver.enabled }} {{- if and .Values.driver.nvidiaDriverCRD.enabled .Values.driver.nvidiaDriverCRD.deployDefaultCR }} apiVersion: nvidia.com/v1alpha1 kind: NVIDIADriver @@ -49,11 +48,7 @@ spec: certConfig: name: {{ .Values.driver.certConfig.name }} {{- end }} - {{- if .Values.driver.licensingConfig.secretName }} - licensingConfig: - secretName: {{ .Values.driver.licensingConfig.secretName }} - nlsEnabled: {{ .Values.driver.licensingConfig.nlsEnabled | default true }} - {{- else if .Values.driver.licensingConfig.configMapName }} + {{- if .Values.driver.licensingConfig.configMapName }} licensingConfig: name: {{ .Values.driver.licensingConfig.configMapName }} nlsEnabled: {{ .Values.driver.licensingConfig.nlsEnabled | default true }} @@ -66,9 +61,6 @@ spec: kernelModuleConfig: name: {{ .Values.driver.kernelModuleConfig.name }} {{- end }} - {{- if .Values.driver.secretEnv }} - secretEnv: {{ .Values.driver.secretEnv }} - {{- end }} {{- if .Values.driver.resources }} resources: {{ toYaml .Values.driver.resources | nindent 6 }} {{- end }} @@ -78,9 +70,6 @@ spec: {{- if .Values.driver.args }} args: {{ toYaml .Values.driver.args | nindent 6 }} {{- end }} - {{- if .Values.driver.hostNetwork }} - hostNetwork: {{ .Values.driver.hostNetwork }} - {{- end }} {{- if .Values.gds.enabled }} gds: enabled: {{ .Values.gds.enabled }} @@ -128,4 +117,3 @@ spec: {{- end }} {{- end }} {{- end }} -{{- end }} diff --git a/packages/system/gpu-operator/charts/gpu-operator/templates/operator.yaml b/packages/system/gpu-operator/charts/gpu-operator/templates/operator.yaml index c6ec3bf4..6f484826 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/templates/operator.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/templates/operator.yaml @@ -2,7 +2,6 @@ apiVersion: apps/v1 kind: Deployment metadata: name: gpu-operator - namespace: {{ .Release.Namespace }} labels: {{- include "gpu-operator.labels" . | nindent 4 }} app.kubernetes.io/component: "gpu-operator" @@ -59,6 +58,10 @@ spec: fieldPath: metadata.namespace - name: "DRIVER_MANAGER_IMAGE" value: "{{ include "driver-manager.fullimage" . }}" + volumeMounts: + - name: host-os-release + mountPath: "/host-etc/os-release" + readOnly: true livenessProbe: httpGet: path: /healthz @@ -78,6 +81,10 @@ spec: ports: - name: metrics containerPort: 8080 + volumes: + - name: host-os-release + hostPath: + path: "/etc/os-release" {{- with .Values.operator.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/packages/system/gpu-operator/charts/gpu-operator/templates/role.yaml b/packages/system/gpu-operator/charts/gpu-operator/templates/role.yaml index dc4674c5..9e5bcede 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/templates/role.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/templates/role.yaml @@ -2,7 +2,6 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: gpu-operator - namespace: {{ .Release.Namespace }} labels: {{- include "gpu-operator.labels" . | nindent 4 }} app.kubernetes.io/component: "gpu-operator" @@ -83,13 +82,3 @@ rules: - watch - update - delete -- apiGroups: - - "nfd.k8s-sigs.io" - resources: - - "nodefeatures" - verbs: - - get - - list - - watch - - create - - update diff --git a/packages/system/gpu-operator/charts/gpu-operator/templates/rolebinding.yaml b/packages/system/gpu-operator/charts/gpu-operator/templates/rolebinding.yaml index 732d8d52..c915a465 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/templates/rolebinding.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/templates/rolebinding.yaml @@ -2,7 +2,6 @@ kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: gpu-operator - namespace: {{ .Release.Namespace }} labels: {{- include "gpu-operator.labels" . | nindent 4 }} app.kubernetes.io/component: "gpu-operator" diff --git a/packages/system/gpu-operator/charts/gpu-operator/templates/serviceaccount.yaml b/packages/system/gpu-operator/charts/gpu-operator/templates/serviceaccount.yaml index 79a3b642..50555e53 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/templates/serviceaccount.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/templates/serviceaccount.yaml @@ -2,7 +2,6 @@ apiVersion: v1 kind: ServiceAccount metadata: name: gpu-operator - namespace: {{ .Release.Namespace }} labels: {{- include "gpu-operator.labels" . | nindent 4 }} app.kubernetes.io/component: "gpu-operator" diff --git a/packages/system/gpu-operator/charts/gpu-operator/templates/upgrade_crd.yaml b/packages/system/gpu-operator/charts/gpu-operator/templates/upgrade_crd.yaml index e887b3a8..6552558a 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/templates/upgrade_crd.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/templates/upgrade_crd.yaml @@ -4,7 +4,6 @@ apiVersion: v1 kind: ServiceAccount metadata: name: gpu-operator-upgrade-crd-hook-sa - namespace: {{ .Release.Namespace }} annotations: helm.sh/hook: pre-upgrade helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation @@ -84,13 +83,13 @@ spec: image: {{ include "gpu-operator.fullimage" . }} imagePullPolicy: {{ .Values.operator.imagePullPolicy }} command: - - /usr/bin/manage-crds - args: - - apply - - --filepath=/opt/gpu-operator/nvidia.com_clusterpolicies.yaml - - --filepath=/opt/gpu-operator/nvidia.com_nvidiadrivers.yaml + - /bin/sh + - -c + - > + kubectl apply -f /opt/gpu-operator/nvidia.com_clusterpolicies.yaml; + kubectl apply -f /opt/gpu-operator/nvidia.com_nvidiadrivers.yaml; {{- if .Values.nfd.enabled }} - - --filepath=/opt/gpu-operator/nfd-api-crds.yaml + kubectl apply -f /opt/gpu-operator/nfd-api-crds.yaml; {{- end }} restartPolicy: OnFailure {{- end }} diff --git a/packages/system/gpu-operator/charts/gpu-operator/templates/validations.yaml b/packages/system/gpu-operator/charts/gpu-operator/templates/validations.yaml deleted file mode 100644 index 573786d2..00000000 --- a/packages/system/gpu-operator/charts/gpu-operator/templates/validations.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if and (eq .Values.cdi.enabled false) (eq .Values.cdi.nriPluginEnabled true) }} -{{ fail "the NRI Plugin cannot be enabled when CDI is disabled" }} -{{- end}} diff --git a/packages/system/gpu-operator/charts/gpu-operator/values.yaml b/packages/system/gpu-operator/charts/gpu-operator/values.yaml index 6ee3c2f0..2806eac3 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/values.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/values.yaml @@ -13,14 +13,12 @@ psa: enabled: false cdi: - enabled: true - nriPluginEnabled: false + enabled: false + default: false sandboxWorkloads: enabled: false defaultWorkload: "container" - # Sandbox mode: "kubevirt" (default) or "kata". When "kata", the Kata device plugin is deployed on vm-passthrough nodes. - mode: "kubevirt" hostPaths: # rootFS represents the path to the root filesystem of the host. @@ -52,8 +50,8 @@ daemonsets: maxUnavailable: "1" validator: - repository: nvcr.io/nvidia - image: gpu-operator + repository: nvcr.io/nvidia/cloud-native + image: gpu-operator-validator # If version is not specified, then default is to use chart.AppVersion #version: "" imagePullPolicy: IfNotPresent @@ -61,9 +59,10 @@ validator: env: [] args: [] resources: {} - hostNetwork: false plugin: - env: [] + env: + - name: WITH_WORKLOAD + value: "false" operator: repository: nvcr.io/nvidia @@ -80,7 +79,16 @@ operator: # upgrade CRD on chart upgrade, requires --disable-openapi-validation flag # to be passed during helm upgrade. upgradeCRD: true + initContainer: + image: cuda + repository: nvcr.io/nvidia + version: 12.8.1-base-ubi9 + imagePullPolicy: IfNotPresent tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Equal" + value: "" + effect: "NoSchedule" - key: "node-role.kubernetes.io/control-plane" operator: "Equal" value: "" @@ -90,6 +98,12 @@ operator: affinity: nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: "node-role.kubernetes.io/master" + operator: In + values: [""] - weight: 1 preference: matchExpressions: @@ -132,7 +146,7 @@ driver: usePrecompiled: false repository: nvcr.io/nvidia image: driver - version: "580.126.20" + version: "570.124.06" imagePullPolicy: IfNotPresent imagePullSecrets: [] startupProbe: @@ -178,13 +192,25 @@ driver: timeoutSeconds: 300 deleteEmptyDir: false manager: - repository: nvcr.io/nvidia/cloud-native image: k8s-driver-manager + repository: nvcr.io/nvidia/cloud-native # When choosing a different version of k8s-driver-manager, DO NOT downgrade to a version lower than v0.6.4 # to ensure k8s-driver-manager stays compatible with gpu-operator starting from v24.3.0 - version: v0.10.0 + version: v0.8.0 imagePullPolicy: IfNotPresent - env: [] + env: + - name: ENABLE_GPU_POD_EVICTION + value: "true" + - name: ENABLE_AUTO_DRAIN + value: "false" + - name: DRAIN_USE_FORCE + value: "false" + - name: DRAIN_POD_SELECTOR_LABEL + value: "" + - name: DRAIN_TIMEOUT_SECONDS + value: "0s" + - name: DRAIN_DELETE_EMPTYDIR_DATA + value: "false" env: [] resources: {} # Private mirror repository configuration @@ -195,7 +221,7 @@ driver: name: "" # vGPU licensing configuration licensingConfig: - secretName: "" + configMapName: "" nlsEnabled: true # vGPU topology daemon configuration virtualTopology: @@ -203,31 +229,39 @@ driver: # kernel module configuration for NVIDIA driver kernelModuleConfig: name: "" - # Name of Kubernetes Secret which contains secrets to be passed in as environment variables - secretEnv: "" - hostNetwork: false toolkit: enabled: true repository: nvcr.io/nvidia/k8s image: container-toolkit - version: v1.19.0 + version: v1.17.5-ubuntu20.04 imagePullPolicy: IfNotPresent imagePullSecrets: [] env: [] resources: {} installDir: "/usr/local/nvidia" - hostNetwork: false devicePlugin: enabled: true repository: nvcr.io/nvidia image: k8s-device-plugin - version: v0.19.0 + version: v0.17.1 imagePullPolicy: IfNotPresent imagePullSecrets: [] args: [] - env: [] + env: + - name: PASS_DEVICE_SPECS + value: "true" + - name: FAIL_ON_INIT_ERROR + value: "true" + - name: DEVICE_LIST_STRATEGY + value: envvar + - name: DEVICE_ID_STRATEGY + value: uuid + - name: NVIDIA_VISIBLE_DEVICES + value: all + - name: NVIDIA_DRIVER_CAPABILITIES + value: all resources: {} # Plugin configuration # Use "name" to either point to an existing ConfigMap or to create a new one with a list of configurations(i.e with create=true). @@ -262,7 +296,6 @@ devicePlugin: mps: # MPS root path on the host root: "/run/nvidia/mps" - hostNetwork: false # standalone dcgm hostengine dcgm: @@ -270,30 +303,26 @@ dcgm: enabled: false repository: nvcr.io/nvidia/cloud-native image: dcgm - version: 4.5.2-1-ubuntu22.04 + version: 4.1.1-2-ubuntu22.04 imagePullPolicy: IfNotPresent args: [] env: [] resources: {} - hostNetwork: false dcgmExporter: enabled: true repository: nvcr.io/nvidia/k8s image: dcgm-exporter - version: 4.5.1-4.8.0-distroless + version: 4.1.1-4.0.4-ubuntu22.04 imagePullPolicy: IfNotPresent - env: [] + env: + - name: DCGM_EXPORTER_LISTEN + value: ":9400" + - name: DCGM_EXPORTER_KUBERNETES + value: "true" + - name: DCGM_EXPORTER_COLLECTORS + value: "/etc/dcgm-exporter/dcp-metrics-included.csv" resources: {} - hostPID: false - hostNetwork: false - # HPC job mapping configuration for correlating GPU metrics with HPC workload manager jobs - # This is used by HPC workload managers like Slurm to label GPU metrics with job IDs - # hpcJobMapping: - # enabled: true - # directory: /var/lib/dcgm-exporter/job-mapping - service: - internalTrafficPolicy: Cluster serviceMonitor: enabled: false interval: 15s @@ -330,35 +359,31 @@ gfd: enabled: true repository: nvcr.io/nvidia image: k8s-device-plugin - version: v0.19.0 + version: v0.17.1 imagePullPolicy: IfNotPresent imagePullSecrets: [] - env: [] + env: + - name: GFD_SLEEP_INTERVAL + value: 60s + - name: GFD_FAIL_ON_INIT_ERROR + value: "true" resources: {} - hostNetwork: false migManager: enabled: true repository: nvcr.io/nvidia/cloud-native image: k8s-mig-manager - version: v0.14.0 + version: v0.12.1-ubuntu20.04 imagePullPolicy: IfNotPresent imagePullSecrets: [] - env: [] + env: + - name: WITH_REBOOT + value: "false" resources: {} # MIG configuration - # NOTE: MIG manager automatically generates configuration from hardware on each node. - # Only provide a custom config if you need settings that differ from hardware discovery. - # - # To use an existing ConfigMap: - # - Set name="your-configmap-name" with create=false - # - ConfigMap MUST have a key named "config.yaml" - # - # To create a new ConfigMap via Helm: - # - Set create=true, name="your-configmap-name", and provide data below - # - If create=true but data is empty, ConfigMap creation is skipped - # - # Example of creating a custom ConfigMap: + # Use "name" to either point to an existing ConfigMap or to create a new one with a list of configurations(i.e with create=true). + # Use "data" to build an integrated ConfigMap from a set of configurations as + # part of this helm chart. An example of setting "data" might be: # config: # name: custom-mig-parted-configs # create: true @@ -390,32 +415,28 @@ migManager: default: "all-disabled" # Create a ConfigMap (default: false) create: false - # ConfigMap name (either existing or to create with create=true) - # If name is provided, mig-manager will use this config instead of auto-generated one. - # REQUIREMENT: Custom ConfigMaps must contain a key named "config.yaml" + # ConfigMap name (either existing or to create a new one with create=true above) name: "" - # Data section for the ConfigMap (required only if create=true) + # Data section for the ConfigMap to create (i.e only applies when create=true) data: {} gpuClientsConfig: name: "" - hostNetwork: false nodeStatusExporter: enabled: false - repository: nvcr.io/nvidia - image: gpu-operator + repository: nvcr.io/nvidia/cloud-native + image: gpu-operator-validator # If version is not specified, then default is to use chart.AppVersion #version: "" imagePullPolicy: IfNotPresent imagePullSecrets: [] resources: {} - hostNetwork: false gds: enabled: false repository: nvcr.io/nvidia/cloud-native image: nvidia-fs - version: "2.27.3" + version: "2.20.5" imagePullPolicy: IfNotPresent imagePullSecrets: [] env: [] @@ -425,7 +446,7 @@ gdrcopy: enabled: false repository: nvcr.io/nvidia/cloud-native image: gdrdrv - version: "v2.5.2" + version: "v2.4.4" imagePullPolicy: IfNotPresent imagePullSecrets: [] env: [] @@ -441,100 +462,102 @@ vgpuManager: env: [] resources: {} driverManager: - repository: nvcr.io/nvidia/cloud-native image: k8s-driver-manager + repository: nvcr.io/nvidia/cloud-native # When choosing a different version of k8s-driver-manager, DO NOT downgrade to a version lower than v0.6.4 # to ensure k8s-driver-manager stays compatible with gpu-operator starting from v24.3.0 - version: v0.10.0 + version: v0.8.0 imagePullPolicy: IfNotPresent - env: [] - # kernel module configuration for vGPU manager - kernelModuleConfig: - name: "" - hostNetwork: false + env: + - name: ENABLE_GPU_POD_EVICTION + value: "false" + - name: ENABLE_AUTO_DRAIN + value: "false" vgpuDeviceManager: enabled: true repository: nvcr.io/nvidia/cloud-native image: vgpu-device-manager - version: v0.4.2 + version: v0.3.0 imagePullPolicy: IfNotPresent imagePullSecrets: [] env: [] config: name: "" default: "default" - hostNetwork: false vfioManager: enabled: true - repository: nvcr.io/nvidia/cloud-native - image: k8s-driver-manager - version: v0.10.0 + repository: nvcr.io/nvidia + image: cuda + version: 12.8.1-base-ubi9 imagePullPolicy: IfNotPresent imagePullSecrets: [] env: [] resources: {} driverManager: - repository: nvcr.io/nvidia/cloud-native image: k8s-driver-manager + repository: nvcr.io/nvidia/cloud-native # When choosing a different version of k8s-driver-manager, DO NOT downgrade to a version lower than v0.6.4 # to ensure k8s-driver-manager stays compatible with gpu-operator starting from v24.3.0 - version: v0.10.0 + version: v0.8.0 imagePullPolicy: IfNotPresent - env: [] - hostNetwork: false + env: + - name: ENABLE_GPU_POD_EVICTION + value: "false" + - name: ENABLE_AUTO_DRAIN + value: "false" kataManager: enabled: false - config: {} + config: + artifactsDir: "/opt/nvidia-gpu-operator/artifacts/runtimeclasses" + runtimeClasses: + - name: kata-nvidia-gpu + nodeSelector: {} + artifacts: + url: nvcr.io/nvidia/cloud-native/kata-gpu-artifacts:ubuntu22.04-535.54.03 + pullSecret: "" + - name: kata-nvidia-gpu-snp + nodeSelector: + "nvidia.com/cc.capable": "true" + artifacts: + url: nvcr.io/nvidia/cloud-native/kata-gpu-artifacts:ubuntu22.04-535.86.10-snp + pullSecret: "" + repository: nvcr.io/nvidia/cloud-native + image: k8s-kata-manager + version: v0.2.3 imagePullPolicy: IfNotPresent imagePullSecrets: [] env: [] resources: {} - hostNetwork: false sandboxDevicePlugin: enabled: true repository: nvcr.io/nvidia image: kubevirt-gpu-device-plugin - version: v1.5.0 + version: v1.3.1 imagePullPolicy: IfNotPresent imagePullSecrets: [] args: [] env: [] resources: {} - hostNetwork: false - -# Kata sandbox device plugin (used when sandboxWorkloads.mode is "kata"). -kataSandboxDevicePlugin: - enabled: true - repository: nvcr.io/nvidia/cloud-native - image: nvidia-sandbox-device-plugin - version: "v0.0.3" - imagePullPolicy: IfNotPresent - imagePullSecrets: [] - args: [] - env: [] - resources: {} - hostNetwork: false ccManager: - enabled: true - defaultMode: "on" + enabled: false + defaultMode: "off" repository: nvcr.io/nvidia/cloud-native image: k8s-cc-manager - version: v0.4.0 + version: v0.1.1 imagePullPolicy: IfNotPresent imagePullSecrets: [] + env: + - name: CC_CAPABLE_DEVICE_IDS + value: "0x2339,0x2331,0x2330,0x2324,0x2322,0x233d" resources: {} - hostNetwork: false - -# Array of extra K8s manifests to deploy -# Supports use of custom Helm templates -extraObjects: [] node-feature-discovery: + enableNodeFeatureApi: true priorityClassName: system-node-critical gc: enable: true @@ -548,6 +571,10 @@ node-feature-discovery: # disable creation to avoid duplicate serviceaccount creation by master spec below create: false tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Equal" + value: "" + effect: "NoSchedule" - key: "node-role.kubernetes.io/control-plane" operator: "Equal" value: "" diff --git a/packages/system/gpu-operator/values-talos-vgpu.yaml b/packages/system/gpu-operator/values-talos-vgpu.yaml deleted file mode 100644 index 00ed8219..00000000 --- a/packages/system/gpu-operator/values-talos-vgpu.yaml +++ /dev/null @@ -1,16 +0,0 @@ -gpu-operator: - sandboxWorkloads: - enabled: true - driver: - enabled: false - devicePlugin: - enabled: false - vgpuManager: - enabled: true - repository: "" - image: vgpu-manager - version: "" - vgpuDeviceManager: - enabled: true - config: - default: default diff --git a/packages/system/hami/Chart.yaml b/packages/system/hami/Chart.yaml new file mode 100644 index 00000000..3fcf4c5d --- /dev/null +++ b/packages/system/hami/Chart.yaml @@ -0,0 +1,3 @@ +apiVersion: v2 +name: cozy-hami +version: 0.0.0 # Placeholder, the actual version will be automatically set during the build process diff --git a/packages/system/hami/Makefile b/packages/system/hami/Makefile new file mode 100644 index 00000000..83663a66 --- /dev/null +++ b/packages/system/hami/Makefile @@ -0,0 +1,43 @@ +export NAME=hami +export NAMESPACE=cozy-$(NAME) + +include ../../../hack/common-envs.mk +include ../../../hack/package.mk + +# When bumping the HAMi version, run `make update` and then review +# the resulting diff in `charts/hami/`. The recipe below reproduces the +# top-level vendoring overrides automatically: +# +# 1. Removes the broken hami-dra subchart. Upstream's NVIDIA DRA driver +# path requires kubelet DRA support that cozystack does not enable +# and has no upstream fix tracked. See commit 3c5521e. +# 2. Empties Chart.yaml dependencies and drops Chart.lock so Helm does +# not try to re-pull hami-dra at build time. See commit 2734dc0. +# 3. Strips dra/hami-dra/podSecurityPolicy blocks from the upstream +# values.yaml since the corresponding code paths are gone. PSP is +# removed from Kubernetes 1.25+ and is unused by cozystack. +# +# Template-level patches are NOT reproduced automatically: +# +# * Scheduler templates have `{{- if .Values.dra.enabled }}` blocks +# that need to be removed because the dra value is gone (commit +# 2734dc0 stripped them). +# * device-plugin/monitorservice.yaml uses `indent` with leading +# whitespace; it must be rewritten to `nindent` for the labels block +# to render correctly when devicePlugin.service.labels is set +# (commit 3685254). +# +# After `make update`, run `git diff -- charts/hami/templates/` and +# replay these template patches against the new upstream version, then +# verify with `helm unittest`. If upstream restructured the affected +# files, the patches may need to be redesigned rather than reapplied. + +update: + rm -rf charts + helm repo add hami-charts https://project-hami.github.io/HAMi/ + helm repo update hami-charts + helm pull hami-charts/hami --untar --untardir charts + rm -rf charts/hami/charts/hami-dra + yq --inplace '.dependencies = []' charts/hami/Chart.yaml + rm -f charts/hami/Chart.lock + yq --inplace 'del(.dra) | del(.["hami-dra"]) | del(.podSecurityPolicy)' charts/hami/values.yaml diff --git a/packages/system/hami/README.md b/packages/system/hami/README.md new file mode 100644 index 00000000..68669ac7 --- /dev/null +++ b/packages/system/hami/README.md @@ -0,0 +1,82 @@ +# HAMi — GPU Virtualization Middleware + +[HAMi](https://github.com/Project-HAMi/HAMi) (Heterogeneous AI Computing Virtualization Middleware) is a CNCF Sandbox project that enables fractional GPU sharing in Kubernetes. It allows workloads to request specific amounts of GPU memory and compute cores instead of claiming entire GPUs. + +## Architecture + +HAMi consists of four components: + +- **MutatingWebhook** — intercepts pod creation, injects `schedulerName: hami-scheduler` +- **Scheduler Extender** — extends kube-scheduler with GPU-aware Filter and Bind logic +- **Device Plugin** (DaemonSet) — registers vGPU resources via the Kubernetes Device Plugin API +- **HAMi-core** (`libvgpu.so`) — `LD_PRELOAD` library injected into workload containers, intercepts CUDA API calls to enforce memory and compute isolation + +## Prerequisites + +- GPU Operator must be enabled (`addons.gpuOperator.enabled: true`) +- NVIDIA driver >= 440 on host nodes +- nvidia-container-toolkit configured as the default container runtime +- GPU nodes labeled with `gpu=on` + +## Known Limitations + +### glibc < 2.34 requirement for workload containers + +HAMi-core uses `LD_PRELOAD` to intercept `dlsym()` for CUDA symbol resolution. The fallback code path relies on `_dl_sym`, a private glibc internal symbol that was removed in glibc 2.34 when libdl and libpthread were merged into libc.so. + +**This limitation affects workload containers only**, not the host OS or HAMi's own components. + +| Distribution | glibc | Result | +| --------------- | ----- | -------------------------------------------- | +| Ubuntu 18.04 | 2.27 | Full isolation (memory + compute) | +| Ubuntu 20.04 | 2.31 | Full isolation (memory + compute) | +| Ubuntu 22.04 | 2.35 | Memory isolation works, compute breaks | +| Ubuntu 24.04 | 2.39 | Both memory and compute isolation break | +| Alpine (musl) | N/A | Completely incompatible (`dlvsym` absent) | + +Most modern ML/AI base images (CUDA 12.x, PyTorch 2.x, TensorFlow 2.x) use Ubuntu 22.04+ with glibc >= 2.35, which means compute isolation will not work with these images until the upstream fix is merged. + +**Upstream tracking issues:** + +- [HAMi-core#174](https://github.com/Project-HAMi/HAMi-core/issues/174) — `_dl_sym` removal in glibc 2.34 breaks HAMi-core's CUDA symbol resolution at the symbol level +- [HAMi#1190](https://github.com/Project-HAMi/HAMi/issues/1190) — maintainer thread confirming the empirical per-glibc-version isolation behavior shown in the table above + +### musl libc (Alpine) incompatibility + +HAMi-core is completely incompatible with musl libc. The `dlvsym()` function used by HAMi-core is a glibc extension not available in musl. Only glibc-based container images (Debian, Ubuntu, RHEL, etc.) can use HAMi GPU isolation. + +## Usage + +Enable HAMi in your tenant Kubernetes cluster values: + +```yaml +addons: + gpuOperator: + enabled: true + hami: + enabled: true +``` + +When HAMi is enabled, GPU Operator's built-in device plugin is automatically disabled to avoid conflicts. This default is preserved by setting `addons.gpuOperator.valuesOverride.gpu-operator.devicePlugin.enabled: false`; advanced topologies that partition GPU pools (e.g. some nodes use HAMi while others run the standard NVIDIA device plugin via node selectors) can re-enable it explicitly through `valuesOverride`. + +### Requesting fractional GPU resources + +```yaml +resources: + limits: + nvidia.com/gpu: 1 + nvidia.com/gpumem: 3000 # 3000 MB of GPU memory + nvidia.com/gpucores: 30 # 30% of GPU compute cores +``` + +## Parameters + +Default values shown below are inherited from the upstream HAMi chart and may change with upstream updates. + +| Name | Description | Default | +| --- | --- | --- | +| `hami.devicePlugin.runtimeClassName` | RuntimeClass for device plugin pods | `nvidia` | +| `hami.devicePlugin.deviceSplitCount` | Max virtual GPUs per physical GPU | `10` | +| `hami.devicePlugin.deviceMemoryScaling` | Memory overcommit factor (> 1.0 enables overcommit) | `1` | +| `hami.scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy` | Node packing strategy (`binpack` or `spread`) | `binpack` | +| `hami.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy` | GPU packing strategy (`binpack` or `spread`) | `spread` | diff --git a/packages/system/hami/charts/hami/Chart.yaml b/packages/system/hami/charts/hami/Chart.yaml new file mode 100644 index 00000000..55f32ab6 --- /dev/null +++ b/packages/system/hami/charts/hami/Chart.yaml @@ -0,0 +1,18 @@ +apiVersion: v2 +appVersion: 2.8.1 +dependencies: [] +description: Heterogeneous AI Computing Virtualization Middleware +keywords: +- vgpu +- gpu +kubeVersion: '>= 1.18.0-0' +maintainers: +- email: archlitchi@gmail.com + name: limengxuan +- email: xiaozhang0210@hotmail.com + name: zhangxiao +name: hami +sources: +- https://github.com/Project-HAMi/HAMi +type: application +version: 2.8.1 diff --git a/packages/system/hami/charts/hami/README.md b/packages/system/hami/charts/hami/README.md new file mode 100644 index 00000000..e8217290 --- /dev/null +++ b/packages/system/hami/charts/hami/README.md @@ -0,0 +1,237 @@ +# HAMi Helm Chart Values Documentation + +This document provides detailed descriptions of all configurable values parameters for the HAMi Helm Chart. + +## Global Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `global.imageRegistry` | Global Docker image registry | `""` | +| `global.imagePullSecrets` | Global Docker image pull secrets | `[]` | +| `global.imageTag` | Image tag | `"v2.8.1"` | +| `global.gpuHookPath` | GPU Hook path | `/usr/local` | +| `global.labels` | Global labels | `{}` | +| `global.annotations` | Global annotations | `{}` | +| `global.managedNodeSelectorEnable` | Whether to enable managed node selector | `false` | +| `global.managedNodeSelector.usage` | Managed node selector usage | `"gpu"` | +| `nameOverride` | Name override | `""` | +| `fullnameOverride` | Full name override | `""` | +| `namespaceOverride` | Namespace override | `""` | + +## Resource Name Configuration + +### NVIDIA GPU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `resourceName` | GPU resource name | `"nvidia.com/gpu"` | +| `resourceMem` | GPU memory resource name | `"nvidia.com/gpumem"` | +| `resourceMemPercentage` | GPU memory percentage resource name | `"nvidia.com/gpumem-percentage"` | +| `resourceCores` | GPU core resource name | `"nvidia.com/gpucores"` | +| `resourcePriority` | GPU priority resource name | `"nvidia.com/priority"` | + +### Cambricon MLU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `mluResourceName` | MLU resource name | `"cambricon.com/vmlu"` | +| `mluResourceMem` | MLU memory resource name | `"cambricon.com/mlu.smlu.vmemory"` | +| `mluResourceCores` | MLU core resource name | `"cambricon.com/mlu.smlu.vcore"` | + +### Hygon DCU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `dcuResourceName` | DCU resource name | `"hygon.com/dcunum"` | +| `dcuResourceMem` | DCU memory resource name | `"hygon.com/dcumem"` | +| `dcuResourceCores` | DCU core resource name | `"hygon.com/dcucores"` | + +### Metax GPU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `metaxResourceName` | GPU resource name | `"metax-tech.com/sgpu"` | +| `metaxResourceCore` | GPU core resource name | `"metax-tech.com/vcore"` | +| `metaxResourceMem` | GPU memory resource name | `"metax-tech.com/vmemory"` | +| `metaxsGPUTopologyAware` | GPU topology awareness | `"false"` | + +### Enflame GCU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `enflameResourceNameVGCU` | vGCU resource name | `"enflame.com/vgcu"` | +| `enflameResourceNameVGCUPercentage` | vGCU percentage resource name | `"enflame.com/vgcu-percentage"` | + +### Kunlunxin XPU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `kunlunResourceName` | XPU resource name | `"kunlunxin.com/xpu"` | + +## Scheduler Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `schedulerName` | Scheduler name | `"hami-scheduler"` | +| `scheduler.nodeName` | Define node name, scheduler will schedule to this node | `""` | +| `scheduler.overwriteEnv` | Whether to overwrite environment variables | `"false"` | +| `scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy` | Node scheduler policy | `binpack` | +| `scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy` | GPU scheduler policy | `spread` | +| `scheduler.metricsBindAddress` | Metrics bind address | `":9395"` | +| `scheduler.forceOverwriteDefaultScheduler` | Whether to force overwrite default scheduler | `true` | +| `scheduler.livenessProbe` | Whether to enable liveness probe | `false` | +| `scheduler.leaderElect` | Whether to enable leader election | `true` | +| `scheduler.replicas` | Number of replicas | `1` | + +### Kube Scheduler Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `scheduler.kubeScheduler.enabled` | Whether to run kube-scheduler container in scheduler pod | `true` | +| `scheduler.kubeScheduler.image.registry` | Kube scheduler image registry | `"registry.cn-hangzhou.aliyuncs.com"` | +| `scheduler.kubeScheduler.image.repository` | Kube scheduler image repository | `"google_containers/kube-scheduler"` | +| `scheduler.kubeScheduler.image.tag` | Kube scheduler image tag | `""` | +| `scheduler.kubeScheduler.image.pullPolicy` | Kube scheduler image pull policy | `IfNotPresent` | +| `scheduler.kubeScheduler.image.pullSecrets` | Kube scheduler image pull secrets | `[]` | +| `scheduler.kubeScheduler.extraNewArgs` | Extra new arguments | `["--config=/config/config.yaml", "-v=4"]` | +| `scheduler.kubeScheduler.extraArgs` | Extra arguments | `["--policy-config-file=/config/config.json", "-v=4"]` | + +### Extender Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `scheduler.extender.image.registry` | Scheduler extender image registry | `"docker.io"` | +| `scheduler.extender.image.repository` | Scheduler extender image repository | `"projecthami/hami"` | +| `scheduler.extender.image.tag` | Scheduler extender image tag | `""` | +| `scheduler.extender.image.pullPolicy` | Scheduler extender image pull policy | `IfNotPresent` | +| `scheduler.extender.image.pullSecrets` | Scheduler extender image pull secrets | `[]` | +| `scheduler.extender.extraArgs` | Scheduler extender extra arguments | `["--debug", "-v=4"]` | + +### Admission Webhook Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `scheduler.admissionWebhook.enabled` | Whether to enable admission webhook | `true` | +| `scheduler.admissionWebhook.customURL.enabled` | Whether to enable custom URL | `false` | +| `scheduler.admissionWebhook.customURL.host` | Custom URL host | `127.0.0.1` | +| `scheduler.admissionWebhook.customURL.port` | Custom URL port | `31998` | +| `scheduler.admissionWebhook.customURL.path` | Custom URL path | `/webhook` | +| `scheduler.admissionWebhook.reinvocationPolicy` | Reinvocation policy | `Never` | +| `scheduler.admissionWebhook.failurePolicy` | Failure policy | `Ignore` | + +### TLS Certificate Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `scheduler.certManager.enabled` | Whether to use cert-manager to generate self-signed certificates | `false` | +| `scheduler.patch.enabled` | Whether to use kube-webhook-certgen to generate self-signed certificates | `true` | +| `scheduler.patch.image.registry` | Certgen image registry | `"docker.io"` | +| `scheduler.patch.image.repository` | Certgen image repository | `"jettech/kube-webhook-certgen"` | +| `scheduler.patch.image.tag` | Certgen image tag | `"v1.5.2"` | +| `scheduler.patch.image.pullPolicy` | Certgen image pull policy | `IfNotPresent` | +| `scheduler.patch.imageNew.registry` | New certgen image registry | `"docker.io"` | +| `scheduler.patch.imageNew.repository` | New certgen image repository | `"liangjw/kube-webhook-certgen"` | +| `scheduler.patch.imageNew.tag` | New certgen image tag | `"v1.1.1"` | + +### Scheduler Service Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `scheduler.service.type` | Service type | `NodePort` | +| `scheduler.service.httpPort` | HTTP port | `443` | +| `scheduler.service.schedulerPort` | Scheduler NodePort | `31998` | +| `scheduler.service.monitorPort` | Monitor port | `31993` | +| `scheduler.service.monitorTargetPort` | Monitor target port | `9395` | + +## Device Plugin Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devicePlugin.image.registry` | Device plugin image registry | `"docker.io"` | +| `devicePlugin.image.repository` | Device plugin image repository | `"projecthami/hami"` | +| `devicePlugin.image.tag` | Device plugin image tag | `""` | +| `devicePlugin.image.pullPolicy` | Device plugin image pull policy | `IfNotPresent` | +| `devicePlugin.image.pullSecrets` | Device plugin image pull secrets | `[]` | + +### Monitor Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devicePlugin.monitor.image.registry` | Monitor image registry | `"docker.io"` | +| `devicePlugin.monitor.image.repository` | Monitor image repository | `"projecthami/hami"` | +| `devicePlugin.monitor.image.tag` | Monitor image tag | `""` | +| `devicePlugin.monitor.image.pullPolicy` | Monitor image pull policy | `IfNotPresent` | +| `devicePlugin.monitor.image.pullSecrets` | Monitor image pull secrets | `[]` | +| `devicePlugin.monitor.ctrPath` | Container path | `/usr/local/vgpu/containers` | +| `devicePlugin.monitor.extraArgs` | Monitor extra arguments | `["-v=4"]` | +| `devicePlugin.monitor.extraEnvs` | Monitor extra environments | `{}` | + +### Device Plugin Other Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devicePlugin.deviceSplitCount` | Integer type, default value: 10. Maximum number of tasks assigned to a single GPU device | `10` | +| `devicePlugin.deviceMemoryScaling` | Device memory scaling ratio | `1` | +| `devicePlugin.deviceCoreScaling` | Device core scaling ratio | `1` | +| `devicePlugin.runtimeClassName` | Runtime class name | `""` | +| `devicePlugin.createRuntimeClass` | Whether to create runtime class | `false` | +| `devicePlugin.migStrategy` | String type, "none" means ignore MIG functionality, "mixed" means allocate MIG devices through independent resources | `"none"` | +| `devicePlugin.disablecorelimit` | String type, "true" means disable core limit, "false" means enable core limit | `"false"` | +| `devicePlugin.passDeviceSpecsEnabled` | Whether to enable passing device specs | `false` | +| `devicePlugin.extraArgs` | Device plugin extra arguments | `["-v=4"]` | +| `devicePlugin.nodeConfiguration.config` | Node configuration for device plugin by json | An example of default configuration. | +| `devicePlugin.nodeConfiguration.externalConfigName` | Node configuration for device plugin by external congimap | `""` | +| `devicePlugin.extraEnvs` | Device plugin extra environments | `{}` | + +### Device Plugin Service Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devicePlugin.service.type` | Service type | `NodePort` | +| `devicePlugin.service.httpPort` | HTTP port | `31992` | + +### Device Plugin Deployment Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devicePlugin.pluginPath` | Plugin path | `/var/lib/kubelet/device-plugins` | +| `devicePlugin.libPath` | Library path | `/usr/local/vgpu` | +| `devicePlugin.nvidiaNodeSelector` | NVIDIA node selector | `{"gpu": "on"}` | +| `devicePlugin.updateStrategy.type` | Update strategy type | `RollingUpdate` | +| `devicePlugin.updateStrategy.rollingUpdate.maxUnavailable` | Maximum unavailable count | `1` | + +## Device Configuration + +### AWS Neuron +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.awsneuron.customresources` | Custom resources | `["aws.amazon.com/neuron", "aws.amazon.com/neuroncore"]` | + +### Kunlunxin +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.kunlun.enabled` | Whether to enable | `true` | +| `devices.kunlun.customresources` | Custom resources | `["kunlunxin.com/xpu"]` | + +### Mthreads +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.mthreads.enabled` | Whether to enable | `true` | +| `devices.mthreads.customresources` | Custom resources | `["mthreads.com/vgpu"]` | + +### NVIDIA +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.nvidia.gpuCorePolicy` | GPU core policy | `default` | +| `devices.nvidia.libCudaLogLevel` | CUDA library log level | `1` | + +### Huawei Ascend +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.ascend.enabled` | Whether to enable | `false` | +| `devices.ascend.image` | Image | `""` | +| `devices.ascend.imagePullPolicy` | Image pull policy | `IfNotPresent` | +| `devices.ascend.extraArgs` | Extra arguments | `[]` | +| `devices.ascend.nodeSelector` | Node selector | `{"ascend": "on"}` | +| `devices.ascend.tolerations` | Tolerations | `[]` | +| `devices.ascend.customresources` | Custom resources | `["huawei.com/Ascend910A", "huawei.com/Ascend910A-memory", ...]` | + +### Iluvatar +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.iluvatar.enabled` | Whether to enable | `false` | +| `devices.iluvatar.customresources` | Custom resources | `["iluvatar.ai/BI-V150-vgpu", "iluvatar.ai/BI-V150.vMem","iluvatar.ai/BI-V150.vCore", ...]` | diff --git a/packages/system/hami/charts/hami/templates/NOTES.txt b/packages/system/hami/charts/hami/templates/NOTES.txt new file mode 100644 index 00000000..15bb1218 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/NOTES.txt @@ -0,0 +1,3 @@ +** Please be patient while the chart is being deployed ** +Resource name: {{ .Values.resourceName }} + diff --git a/packages/system/hami/charts/hami/templates/_commons.tpl b/packages/system/hami/charts/hami/templates/_commons.tpl new file mode 100644 index 00000000..b68018e4 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/_commons.tpl @@ -0,0 +1,49 @@ +{{/* +Return the proper image name +{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" $) }} +*/}} +{{- define "common.images.image" -}} +{{- $registryName := .imageRoot.registry -}} +{{- $repositoryName := .imageRoot.repository -}} +{{- $tag := .imageRoot.tag | toString -}} +{{- if .global }} + {{- if .global.imageRegistry }} + {{- $registryName = .global.imageRegistry -}} + {{- end -}} +{{- end -}} +{{- if .tag }} + {{- $tag = .tag | toString -}} +{{- end -}} +{{- if $registryName }} +{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- else -}} +{{- printf "%s:%s" $repositoryName $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead) +{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }} +*/}} +{{- define "common.images.pullSecrets" -}} + {{- $pullSecrets := list }} + + {{- if .global }} + {{- range .global.imagePullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- range .images -}} + {{- range .pullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- if (not (empty $pullSecrets)) }} +imagePullSecrets: + {{- range $pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/_helpers.tpl b/packages/system/hami/charts/hami/templates/_helpers.tpl new file mode 100644 index 00000000..ffcc61da --- /dev/null +++ b/packages/system/hami/charts/hami/templates/_helpers.tpl @@ -0,0 +1,163 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "hami-vgpu.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "hami-vgpu.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "hami-vgpu.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +The app name for Scheduler +*/}} +{{- define "hami-vgpu.scheduler" -}} +{{- printf "%s-scheduler" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +The app name for DevicePlugin +*/}} +{{- define "hami-vgpu.device-plugin" -}} +{{- printf "%s-device-plugin" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* + The app name for MockDevicePlugin + */}} +{{- define "hami-vgpu.mock-device-plugin" -}} +{{- printf "%s-mock-device-plugin" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +The tls secret name for Scheduler +*/}} +{{- define "hami-vgpu.scheduler.tls" -}} +{{- printf "%s-scheduler-tls" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +The webhook name +*/}} +{{- define "hami-vgpu.scheduler.webhook" -}} +{{- printf "%s-webhook" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "hami-vgpu.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "hami-vgpu.labels" -}} +helm.sh/chart: {{ include "hami-vgpu.chart" . }} +{{ include "hami-vgpu.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "hami-vgpu.selectorLabels" -}} +app.kubernetes.io/name: {{ include "hami-vgpu.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + + +{{/* + Resolve the tag for kubeScheduler. +*/}} +{{- define "resolvedKubeSchedulerTag" -}} +{{- if .Values.scheduler.kubeScheduler.image.tag }} +{{- .Values.scheduler.kubeScheduler.image.tag | trim -}} +{{- else }} +{{- include "strippedKubeVersion" . | trim -}} +{{- end }} +{{- end }} + +{{/* + Return the stripped Kubernetes version string by removing extra parts after semantic version number. + v1.31.1+k3s1 -> v1.31.1 + v1.30.8-eks-2d5f260 -> v1.30.8 + v1.31.1 -> v1.31.1 +*/}} +{{- define "strippedKubeVersion" -}} +{{ regexReplaceAll "^(v[0-9]+\\.[0-9]+\\.[0-9]+)(.*)$" .Capabilities.KubeVersion.Version "$1" }} +{{- end -}} + +{{- define "hami.scheduler.kubeScheduler.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.kubeScheduler.image "global" .Values.global "tag" (include "resolvedKubeSchedulerTag" .)) }} +{{- end -}} + +{{- define "hami.scheduler.extender.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.extender.image "global" .Values.global "tag" .Values.global.imageTag) }} +{{- end -}} + +{{- define "hami.devicePlugin.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.devicePlugin.image "global" .Values.global "tag" .Values.global.imageTag) }} +{{- end -}} + +{{- define "hami.mockDevicePlugin.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.mockDevicePlugin.image "global" .Values.global "tag" .Values.mockDevicePlugin.tag) }} +{{- end -}} + +{{- define "hami.devicePlugin.monitor.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.devicePlugin.monitor.image "global" .Values.global "tag" .Values.global.imageTag) }} +{{- end -}} + +{{- define "hami.scheduler.patch.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.patch.image "global" .Values.global) }} +{{- end -}} + +{{- define "hami.scheduler.patch.new.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.patch.imageNew "global" .Values.global) }} +{{- end -}} + +{{- define "hami.scheduler.extender.imagePullSecrets" -}} +{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.extender.image) "global" .Values.global) }} +{{- end -}} + +{{- define "hami.devicePlugin.imagePullSecrets" -}} +{{ include "common.images.pullSecrets" (dict "images" (list .Values.devicePlugin.image) "global" .Values.global) }} +{{- end -}} + +{{- define "hami.scheduler.patch.imagePullSecrets" -}} +{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.patch.image) "global" .Values.global) }} +{{- end -}} + +{{- define "hami.scheduler.patch.new.imagePullSecrets" -}} +{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.patch.imageNew) "global" .Values.global) }} +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml b/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml new file mode 100644 index 00000000..74631e24 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml @@ -0,0 +1,13 @@ +{{- if and .Values.devicePlugin.enabled (not .Values.devicePlugin.nodeConfiguration.externalConfigName) -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "hami-vgpu.device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-device-plugin + {{- include "hami-vgpu.labels" . | nindent 4 }} +data: + config.json: | +{{- .Values.devicePlugin.nodeConfiguration.config | nindent 4 }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/daemonsetmock.yaml b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetmock.yaml new file mode 100644 index 00000000..a0dc4ff3 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetmock.yaml @@ -0,0 +1,55 @@ +{{- if .Values.mockDevicePlugin.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "hami-vgpu.mock-device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +spec: + selector: + matchLabels: + app.kubernetes.io/component: hami-mock-device-plugin + {{- include "hami-vgpu.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" + labels: + app.kubernetes.io/component: hami-mock-device-plugin + {{- include "hami-vgpu.selectorLabels" . | nindent 8 }} + spec: + serviceAccountName: {{ include "hami-vgpu.mock-device-plugin" . }} + tolerations: + - key: CriticalAddonsOnly + operator: Exists + containers: + - image: {{ include "hami.mockDevicePlugin.image" . }} + imagePullPolicy: {{ .Values.mockDevicePlugin.image.pullPolicy }} + name: hami-mock-dp-cntr + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + command: + - ./k8s-device-plugin + - -v=5 + - --device-config-file=/device-config.yaml + volumeMounts: + - name: dp + mountPath: /var/lib/kubelet/device-plugins + - name: sys + mountPath: /sys + - name: device-config + mountPath: /device-config.yaml + subPath: device-config.yaml + volumes: + - name: dp + hostPath: + path: /var/lib/kubelet/device-plugins + - name: sys + hostPath: + path: /sys + - name: device-config + configMap: + name: {{ include "hami-vgpu.scheduler" . }}-device +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml new file mode 100644 index 00000000..1f4c24f3 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml @@ -0,0 +1,262 @@ +{{- if .Values.devicePlugin.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "hami-vgpu.device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-device-plugin + {{- include "hami-vgpu.labels" . | nindent 4 }} + {{- with .Values.global.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.global.annotations }} + annotations: {{ toYaml .Values.global.annotations | nindent 4}} + {{- end }} +spec: + updateStrategy: + {{- with .Values.devicePlugin.updateStrategy }} + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/component: hami-device-plugin + {{- include "hami-vgpu.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app.kubernetes.io/component: hami-device-plugin + hami.io/webhook: ignore + {{- include "hami-vgpu.selectorLabels" . | nindent 8 }} + annotations: + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + checksum/hami-scheduler-newversion-config: {{ include (print $.Template.BasePath "/scheduler/configmapnew.yaml") . | sha256sum }} + {{- else }} + checksum/hami-scheduler-config: {{ include (print $.Template.BasePath "/scheduler/configmap.yaml") . | sha256sum }} + {{- end }} + checksum/hami-scheduler-device-config: {{ include (print $.Template.BasePath "/scheduler/device-configmap.yaml") . | sha256sum }} + {{- if .Values.devicePlugin.podAnnotations }} + {{- toYaml .Values.devicePlugin.podAnnotations | nindent 8 }} + {{- end }} + spec: + {{- if .Values.devicePlugin.runtimeClassName }} + runtimeClassName: {{ .Values.devicePlugin.runtimeClassName }} + {{- end }} + serviceAccountName: {{ include "hami-vgpu.device-plugin" . }} + priorityClassName: system-node-critical + hostPID: true + hostNetwork: true + {{- include "hami.devicePlugin.imagePullSecrets" . | nindent 6 }} + {{- if .Values.devicePlugin.gpuOperatorToolkitReady.enabled }} + initContainers: + - name: toolkit-validation + image: {{ include "hami.devicePlugin.image" . }} + imagePullPolicy: {{ .Values.devicePlugin.image.pullPolicy }} + securityContext: + privileged: true + runAsUser: 0 + command: ["sh", "-c"] + args: + - | + echo "Waiting for NVIDIA Toolkit to be ready..." + until [ -f {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }}/toolkit-ready ]; do + echo "Waiting for {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }}/toolkit-ready..." + sleep 5 + done + echo "NVIDIA Toolkit is ready!" + volumeMounts: + - name: nvidia-validations + mountPath: {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath | quote }} + mountPropagation: HostToContainer + readOnly: true + {{- end }} + containers: + - name: device-plugin + image: {{ include "hami.devicePlugin.image" . }} + imagePullPolicy: {{ .Values.devicePlugin.image.pullPolicy }} + lifecycle: + postStart: + exec: + command: ["/bin/sh","-c", {{ printf "/k8s-vgpu/bin/vgpu-init.sh %s/vgpu/" .Values.global.gpuHookPath | quote }}] + command: + - nvidia-device-plugin + - --config-file=/device-config.yaml + - --mig-strategy={{ .Values.devicePlugin.migStrategy }} + - --disable-core-limit={{ .Values.devicePlugin.disablecorelimit }} + {{- range .Values.devicePlugin.extraArgs }} + - {{ . }} + {{- end }} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: NVIDIA_MIG_MONITOR_DEVICES + value: all + - name: DEVICE_LIST_STRATEGY + value: {{ .Values.devicePlugin.deviceListStrategy }} + - name: HOOK_PATH + value: {{ .Values.global.gpuHookPath }} + {{- if typeIs "bool" .Values.devicePlugin.passDeviceSpecsEnabled }} + - name: PASS_DEVICE_SPECS + value: {{ .Values.devicePlugin.passDeviceSpecsEnabled | quote }} + {{- end }} + {{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }} + - name: NVIDIA_DRIVER_ROOT + value: {{ .Values.devicePlugin.nvidiaDriverRoot }} + {{- end }} + {{- if typeIs "string" .Values.devicePlugin.nvidiaHookPath }} + - name: NVIDIA_CDI_HOOK_PATH + value: {{ .Values.devicePlugin.nvidiaHookPath }} + {{- end }} + {{- if typeIs "bool" .Values.devicePlugin.gdrcopyEnabled }} + - name: GDRCOPY_ENABLED + value: {{ .Values.devicePlugin.gdrcopyEnabled | quote }} + {{- end }} + {{- if typeIs "bool" .Values.devicePlugin.gdsEnabled }} + - name: GDS_ENABLED + value: {{ .Values.devicePlugin.gdsEnabled | quote }} + {{- end }} + {{- if typeIs "bool" .Values.devicePlugin.mofedEnabled }} + - name: MOFED_ENABLED + value: {{ .Values.devicePlugin.mofedEnabled | quote }} + {{- end }} + {{- if eq (.Values.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy | default "spread") "topology-aware" }} + - name: ENABLE_TOPOLOGY_SCORE + value: "true" + {{- end }} + {{- with .Values.devicePlugin.extraEnvs }} + {{- . | toYaml | nindent 12 }} + {{- end }} + securityContext: + privileged: true + allowPrivilegeEscalation: true + capabilities: + drop: ["ALL"] + add: ["SYS_ADMIN"] + resources: + {{- toYaml .Values.devicePlugin.resources | nindent 12 }} + volumeMounts: + - name: device-plugin + mountPath: /var/lib/kubelet/device-plugins + - name: lib + mountPath: {{ printf "%s%s" .Values.global.gpuHookPath "/vgpu" }} + - name: usrbin + mountPath: /usrbin + - name: deviceconfig + mountPath: /config + - name: hosttmp + mountPath: /tmp + - name: device-config + mountPath: /device-config.yaml + subPath: device-config.yaml + - name: cdi-root + mountPath: /var/run/cdi + {{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }} + # We always mount the driver root at /driver-root in the container. + # This is required for CDI detection to work correctly. + - name: driver-root + mountPath: /driver-root + readOnly: true + {{- end }} + - name: vgpu-monitor + image: {{ include "hami.devicePlugin.monitor.image" . }} + imagePullPolicy: {{ .Values.devicePlugin.monitor.image.pullPolicy }} + command: + - "vGPUmonitor" + {{- range .Values.devicePlugin.monitor.extraArgs }} + - {{ . }} + {{- end }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + add: ["SYS_ADMIN"] + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: NVIDIA_VISIBLE_DEVICES + value: "all" + - name: NVIDIA_MIG_MONITOR_DEVICES + value: "all" + - name: HOOK_PATH + value: "{{ .Values.global.gpuHookPath }}/vgpu" + - name: HAMI_RESYNC_INTERVAL + value: {{ .Values.devicePlugin.monitor.resyncInterval | default "5m" | quote }} + {{- with .Values.devicePlugin.monitor.extraEnvs }} + {{- . | toYaml | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.devicePlugin.monitor.resources | nindent 12 }} + volumeMounts: + - name: ctrs + mountPath: {{ .Values.devicePlugin.monitor.ctrPath }} + - name: dockers + mountPath: /run/docker + - name: containerds + mountPath: /run/containerd + - name: sysinfo + mountPath: /sysinfo + - name: hostvar + mountPath: /hostvar + - name: hosttmp + mountPath: /tmp + volumes: + - name: ctrs + hostPath: + path: {{ .Values.devicePlugin.monitor.ctrPath }} + - name: hosttmp + hostPath: + path: /tmp + - name: dockers + hostPath: + path: /run/docker + - name: containerds + hostPath: + path: /run/containerd + - name: device-plugin + hostPath: + path: {{ .Values.devicePlugin.pluginPath }} + - name: lib + hostPath: + path: {{ .Values.devicePlugin.libPath }} + {{- if .Values.devicePlugin.gpuOperatorToolkitReady.enabled }} + - name: nvidia-validations + hostPath: + path: {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }} + type: DirectoryOrCreate + {{- end }} + {{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }} + - name: driver-root + hostPath: + path: {{ .Values.devicePlugin.nvidiaDriverRoot }} + type: Directory + {{- end }} + - name: cdi-root + hostPath: + path: /var/run/cdi + type: DirectoryOrCreate + - name: usrbin + hostPath: + path: /usr/bin + - name: sysinfo + hostPath: + path: /sys + - name: hostvar + hostPath: + path: /var + - name: deviceconfig + configMap: + name: {{ .Values.devicePlugin.nodeConfiguration.externalConfigName | default (include "hami-vgpu.device-plugin" .) }} + - name: device-config + configMap: + name: {{ include "hami-vgpu.scheduler" . }}-device + {{- if .Values.devicePlugin.nvidiaNodeSelector }} + nodeSelector: {{ toYaml .Values.devicePlugin.nvidiaNodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.devicePlugin.tolerations }} + tolerations: {{ toYaml .Values.devicePlugin.tolerations | nindent 8 }} + {{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml new file mode 100644 index 00000000..6ac757f2 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml @@ -0,0 +1,28 @@ +{{- if .Values.devicePlugin.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "hami-vgpu.device-plugin" . }}-monitor +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - create + - watch + - list + - update + - patch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - update + - list + - patch +{{- end -}} + diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml new file mode 100644 index 00000000..2f0a14ba --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml @@ -0,0 +1,17 @@ +{{- if .Values.devicePlugin.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.device-plugin" . }} + labels: + app.kubernetes.io/component: "hami-device-plugin" + {{- include "hami-vgpu.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "hami-vgpu.device-plugin" . }}-monitor +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml new file mode 100644 index 00000000..88f1b214 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml @@ -0,0 +1,29 @@ +{{- if .Values.devicePlugin.enabled -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "hami-vgpu.device-plugin" . }}-monitor + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-device-plugin + {{- include "hami-vgpu.labels" . | nindent 4 }} + {{- with .Values.devicePlugin.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.devicePlugin.service.annotations }} # Use devicePlugin instead of scheduler + annotations: {{ toYaml .Values.devicePlugin.service.annotations | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.devicePlugin.service.type | default "NodePort" }} # Default type is NodePort + ports: + - name: monitorport + port: {{ .Values.devicePlugin.service.httpPort | default 31992 }} # Default HTTP port is 31992 + targetPort: 9394 + {{- if eq (.Values.devicePlugin.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort + nodePort: {{ .Values.devicePlugin.service.httpPort | default 31992 }} + {{- end }} + protocol: TCP + selector: + app.kubernetes.io/component: hami-device-plugin + {{- include "hami-vgpu.selectorLabels" . | nindent 4 }} +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml new file mode 100644 index 00000000..6e3c2a44 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml @@ -0,0 +1,10 @@ +{{- if .Values.devicePlugin.enabled -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "hami-vgpu.device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: "hami-device-plugin" + {{- include "hami-vgpu.labels" . | nindent 4 }} +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml b/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml new file mode 100644 index 00000000..ebcc2c9c --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml @@ -0,0 +1,9 @@ +{{- if and .Values.devicePlugin.enabled .Values.devicePlugin.createRuntimeClass .Values.devicePlugin.runtimeClassName -}} +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: {{ .Values.devicePlugin.runtimeClassName }} + annotations: + helm.sh/hook: pre-install,pre-upgrade +handler: nvidia +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml b/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml new file mode 100644 index 00000000..e6d28721 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml @@ -0,0 +1,31 @@ +{{- if .Values.scheduler.admissionWebhook.enabled -}} +{{- if .Values.scheduler.certManager.enabled }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-serving-cert + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} +spec: + dnsNames: + - {{ include "hami-vgpu.scheduler" . }}.{{ include "hami-vgpu.namespace" . }}.svc + - {{ include "hami-vgpu.scheduler" . }}.{{ include "hami-vgpu.namespace" . }}.svc.cluster.local + issuerRef: + kind: Issuer + name: {{ include "hami-vgpu.scheduler" . }}-selfsigned-issuer + secretName: {{ include "hami-vgpu.scheduler.tls" . }} +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-selfsigned-issuer + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} +spec: + selfSigned: {} +{{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml b/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml new file mode 100644 index 00000000..81c4fddf --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml @@ -0,0 +1,34 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["pods", "configmaps"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: [""] + resources: ["pods/binding"] + verbs: ["create"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "patch", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "get", "list"] + - apiGroups: [""] + resources: ["resourcequotas"] + verbs: ["get", "list", "watch"] +--- +{{- if .Values.mockDevicePlugin.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "hami-vgpu.mock-device-plugin" . }} +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "update", "list", "patch"] +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml new file mode 100644 index 00000000..a81d425c --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml @@ -0,0 +1,47 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-kube + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-scheduler +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-volume + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:volume-scheduler +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "hami-vgpu.scheduler" . }} +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml b/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml new file mode 100644 index 00000000..109ecbce --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml @@ -0,0 +1,142 @@ +{{- if .Values.scheduler.kubeScheduler.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} +data: + config.json: | + { + "kind": "Policy", + "apiVersion": "v1", + "extenders": [ + { + {{- if .Values.scheduler.admissionWebhook.enabled }} + "urlPrefix": "https://127.0.0.1:443", + "enableHttps": true, + "tlsConfig": { + "insecure": true + }, + {{- else }} + "urlPrefix": "http://127.0.0.1:80", + "enableHttps": false, + {{- end }} + "filterVerb": "filter", + "bindVerb": "bind", + "weight": 1, + "nodeCacheCapable": true, + "httpTimeout": 30000000000, + "managedResources": [ + {{- range .Values.devices.amd.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- if .Values.devices.ascend.enabled }} + {{- range .Values.devices.ascend.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- end }} + {{- if .Values.devices.mthreads.enabled }} + {{- range .Values.devices.mthreads.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- end }} + {{- if .Values.devices.enflame.enabled }} + {{- range .Values.devices.enflame.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- end }} + {{- if .Values.devices.kunlun.enabled }} + {{- range .Values.devices.kunlun.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- end }} + {{- range .Values.devices.awsneuron.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- if .Values.devices.iluvatar.enabled }} + {{- range .Values.devices.iluvatar.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- end }} + { + "name": "{{ .Values.resourceName }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.resourceMem }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.resourceCores }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.resourceMemPercentage }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.resourcePriority }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.mluResourceName }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.dcuResourceName }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.dcuResourceMem }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.dcuResourceCores }}", + "ignoredByScheduler": true + }, + { + "name": "metax-tech.com/gpu", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.metaxResourceName }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.metaxResourceCore }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.metaxResourceMem }}", + "ignoredByScheduler": true + } + ], + "ignoreable": false + } + ] + } +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml b/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml new file mode 100644 index 00000000..6f6db097 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml @@ -0,0 +1,102 @@ +{{- if .Values.scheduler.kubeScheduler.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-newversion + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} +data: + config.yaml: | + {{- if gt (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 25}} + apiVersion: kubescheduler.config.k8s.io/v1 + {{- else }} + apiVersion: kubescheduler.config.k8s.io/v1beta2 + {{- end }} + kind: KubeSchedulerConfiguration + leaderElection: + leaderElect: false + profiles: + - schedulerName: {{ .Values.schedulerName }} + extenders: + {{- if .Values.scheduler.admissionWebhook.enabled }} + - urlPrefix: "https://127.0.0.1:443" + enableHTTPS: true + tlsConfig: + insecure: true + {{- else }} + - urlPrefix: "http://127.0.0.1:80" + enableHTTPS: false + {{- end }} + filterVerb: filter + bindVerb: bind + nodeCacheCapable: true + weight: 1 + httpTimeout: 30s + managedResources: + - name: {{ .Values.resourceName }} + ignoredByScheduler: true + - name: {{ .Values.resourceMem }} + ignoredByScheduler: true + - name: {{ .Values.resourceCores }} + ignoredByScheduler: true + - name: {{ .Values.resourceMemPercentage }} + ignoredByScheduler: true + - name: {{ .Values.resourcePriority }} + ignoredByScheduler: true + - name: {{ .Values.mluResourceName }} + ignoredByScheduler: true + - name: {{ .Values.dcuResourceName }} + ignoredByScheduler: true + - name: {{ .Values.dcuResourceMem }} + ignoredByScheduler: true + - name: {{ .Values.dcuResourceCores }} + ignoredByScheduler: true + - name: "metax-tech.com/gpu" + ignoredByScheduler: true + - name: {{ .Values.metaxResourceName }} + ignoredByScheduler: true + - name: {{ .Values.metaxResourceCore }} + ignoredByScheduler: true + - name: {{ .Values.metaxResourceMem }} + ignoredByScheduler: true + {{- if .Values.devices.ascend.enabled }} + {{- range .Values.devices.ascend.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- end }} + {{- if .Values.devices.mthreads.enabled }} + {{- range .Values.devices.mthreads.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- end }} + {{- if .Values.devices.enflame.enabled }} + {{- range .Values.devices.enflame.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- end }} + {{- if .Values.devices.kunlun.enabled }} + {{- range .Values.devices.kunlun.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- end }} + {{- range .Values.devices.awsneuron.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- if .Values.devices.iluvatar.enabled }} + {{- range .Values.devices.iluvatar.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- end }} + {{- range .Values.devices.amd.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml b/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml new file mode 100644 index 00000000..1d89e189 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml @@ -0,0 +1,225 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} + {{- with .Values.global.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.global.annotations }} + annotations: {{ toYaml .Values.global.annotations | nindent 4}} + {{- end }} +spec: + {{- if .Values.scheduler.leaderElect }} + replicas: {{ .Values.scheduler.replicas }} + {{- else }} + replicas: 1 + {{- end }} + selector: + matchLabels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.selectorLabels" . | nindent 8 }} + hami.io/webhook: ignore + annotations: + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + checksum/hami-scheduler-newversion-config: {{ include (print $.Template.BasePath "/scheduler/configmapnew.yaml") . | sha256sum }} + {{- else }} + checksum/hami-scheduler-config: {{ include (print $.Template.BasePath "/scheduler/configmap.yaml") . | sha256sum }} + {{- end }} + checksum/hami-scheduler-device-config: {{ include (print $.Template.BasePath "/scheduler/device-configmap.yaml") . | sha256sum }} + {{- if .Values.scheduler.podAnnotations }} + {{- toYaml .Values.scheduler.podAnnotations | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "hami-vgpu.scheduler" . }} + priorityClassName: system-node-critical + {{- include "hami.scheduler.extender.imagePullSecrets" . | nindent 6 }} + containers: + {{- if .Values.scheduler.kubeScheduler.enabled }} + - name: kube-scheduler + image: {{ include "hami.scheduler.kubeScheduler.image" . }} + imagePullPolicy: {{ .Values.scheduler.kubeScheduler.image.pullPolicy }} + command: + - kube-scheduler + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + {{- range .Values.scheduler.kubeScheduler.extraNewArgs }} + - {{ . }} + {{- end }} + {{- else }} + - --scheduler-name={{ .Values.schedulerName }} + {{- range .Values.scheduler.kubeScheduler.extraArgs }} + - {{ . }} + {{- end }} + {{- end }} + - --leader-elect={{ .Values.scheduler.leaderElect }} + - --leader-elect-resource-name={{ .Values.schedulerName }} + - --leader-elect-resource-namespace={{ include "hami-vgpu.namespace" . }} + resources: + {{- toYaml .Values.scheduler.kubeScheduler.resources | nindent 12 }} + volumeMounts: + - name: scheduler-config + mountPath: /config + {{- end }} + {{- if .Values.scheduler.livenessProbe }} + livenessProbe: + failureThreshold: 8 + httpGet: + path: /healthz + port: 10259 + scheme: HTTPS + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 15 + {{- end }} + - name: vgpu-scheduler-extender + image: {{ include "hami.scheduler.extender.image" . }} + imagePullPolicy: {{ .Values.scheduler.extender.image.pullPolicy }} + env: + {{- if .Values.scheduler.nodeLockExpire }} + - name: HAMI_NODELOCK_EXPIRE + value: "{{ .Values.scheduler.nodeLockExpire }}" + {{- end }} + {{- if .Values.global.managedNodeSelectorEnable }} + {{- range $key, $value := .Values.global.managedNodeSelector }} + - name: NODE_SELECTOR_{{ $key | upper | replace "-" "_" }} + value: "{{ $value }}" + {{- end }} + {{- end }} + command: + - scheduler + {{- if .Values.scheduler.admissionWebhook.enabled }} + - --http_bind=0.0.0.0:443 + - --cert_file=/tls/tls.crt + - --key_file=/tls/tls.key + {{- else }} + - --http_bind=0.0.0.0:80 + {{- end }} + - --scheduler-name={{ .Values.schedulerName }} + - --metrics-bind-address={{ .Values.scheduler.metricsBindAddress }} + - --node-scheduler-policy={{ .Values.scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy }} + - --gpu-scheduler-policy={{ .Values.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy }} + - --force-overwrite-default-scheduler={{ .Values.scheduler.forceOverwriteDefaultScheduler}} + - --device-config-file=/device-config.yaml + - --leader-elect={{ .Values.scheduler.leaderElect }} + - --leader-elect-resource-name={{ .Values.schedulerName }} + - --leader-elect-resource-namespace={{ include "hami-vgpu.namespace" . }} + {{- if .Values.devices.ascend.enabled }} + - --enable-ascend=true + {{- end }} + {{- if .Values.devices.iluvatar.enabled }} + - --enable-iluvatar=true + {{- end }} + {{- if .Values.scheduler.nodeLabelSelector }} + - --node-label-selector={{- $first := true -}} + {{- range $key, $value := .Values.scheduler.nodeLabelSelector -}} + {{- if not $first }},{{ end -}} + {{- $key }}={{ $value -}} + {{- $first = false -}} + {{- end -}} + {{- end }} + {{- range .Values.scheduler.extender.extraArgs }} + - {{ . }} + {{- end }} + ports: + {{- if .Values.scheduler.admissionWebhook.enabled }} + - name: https + containerPort: 443 + protocol: TCP + {{- else }} + - name: http + containerPort: 80 + protocol: TCP + {{- end }} + - name: metrics + containerPort: {{ last (splitList ":" .Values.scheduler.metricsBindAddress) | int }} + protocol: TCP + resources: + {{- toYaml .Values.scheduler.extender.resources | nindent 12 }} + volumeMounts: + - name: device-config + mountPath: /device-config.yaml + subPath: device-config.yaml + {{- if .Values.scheduler.admissionWebhook.enabled }} + - name: tls-config + mountPath: /tls + {{- end }} + {{- if .Values.scheduler.livenessProbe }} + livenessProbe: + httpGet: + path: /healthz + {{- if .Values.scheduler.admissionWebhook.enabled }} + port: https + scheme: HTTPS + {{- else }} + port: http + scheme: HTTP + {{- end }} + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 3 + timeoutSeconds: 5 + {{- end }} + {{- if .Values.scheduler.leaderElect }} + readinessProbe: + httpGet: + path: /readyz + {{- if .Values.scheduler.admissionWebhook.enabled }} + port: https + scheme: HTTPS + {{- else }} + port: http + scheme: HTTP + {{- end }} + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 3 + timeoutSeconds: 5 + {{- end }} + {{- if .Values.scheduler.leaderElect }} + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app.kubernetes.io/component + operator: In + values: + - hami-scheduler + topologyKey: "kubernetes.io/hostname" + {{- end }} + volumes: + {{- if .Values.scheduler.admissionWebhook.enabled }} + - name: tls-config + secret: + secretName: {{ template "hami-vgpu.scheduler.tls" . }} + {{- end }} + {{- if .Values.scheduler.kubeScheduler.enabled }} + - name: scheduler-config + configMap: + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + name: {{ template "hami-vgpu.scheduler" . }}-newversion + {{- else }} + name: {{ template "hami-vgpu.scheduler" . }} + {{- end }} + {{- end }} + - name: device-config + configMap: + name: {{ include "hami-vgpu.scheduler" . }}-device + {{- if .Values.scheduler.nodeSelector }} + nodeSelector: {{ toYaml .Values.scheduler.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.scheduler.tolerations }} + tolerations: {{ toYaml .Values.scheduler.tolerations | nindent 8 }} + {{- end }} + {{- if .Values.scheduler.nodeName }} + nodeName: {{ .Values.scheduler.nodeName }} + {{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml b/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml new file mode 100644 index 00000000..873b813b --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml @@ -0,0 +1,408 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-device + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} +data: + device-config.yaml: |- + {{- if .Files.Glob "files/device-config.yaml" }} + {{- .Files.Get "files/device-config.yaml" | nindent 4}} + {{- else }} + nvidia: + resourceCountName: {{ .Values.resourceName }} + resourceMemoryName: {{ .Values.resourceMem }} + resourceMemoryPercentageName: {{ .Values.resourceMemPercentage }} + resourceCoreName: {{ .Values.resourceCores }} + resourcePriorityName: {{ .Values.resourcePriority }} + overwriteEnv: false + defaultMemory: 0 + defaultCores: 0 + defaultGPUNum: 1 + memoryFactor: 1 + deviceSplitCount: {{ .Values.devicePlugin.deviceSplitCount }} + deviceMemoryScaling: {{ .Values.devicePlugin.deviceMemoryScaling }} + deviceCoreScaling: {{ .Values.devicePlugin.deviceCoreScaling }} + gpuCorePolicy: {{ .Values.devices.nvidia.gpuCorePolicy }} + libCudaLogLevel: {{ .Values.devices.nvidia.libCudaLogLevel }} + runtimeClassName: "{{ .Values.devicePlugin.runtimeClassName }}" + knownMigGeometries: + - models: [ "A30" ] + allowedGeometries: + - + - name: 1g.6gb + core: 25 + memory: 6144 + count: 4 + - + - name: 2g.12gb + core: 50 + memory: 12288 + count: 2 + - + - name: 4g.24gb + core: 100 + memory: 24576 + count: 1 + - models: [ "A100-SXM4-40GB", "A100-40GB-PCIe", "A100-PCIE-40GB"] + allowedGeometries: + - + - name: 1g.5gb + core: 14 + memory: 5120 + count: 7 + - + - name: 1g.5gb + core: 14 + memory: 5120 + count: 1 + - name: 2g.10gb + core: 28 + memory: 10240 + count: 3 + - + - name: 3g.20gb + core: 42 + memory: 20480 + count: 2 + - + - name: 7g.40gb + core: 100 + memory: 40960 + count: 1 + - models: [ "A100-SXM4-80GB", "A100-80GB-PCIe", "A100-PCIE-80GB"] + allowedGeometries: + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 7 + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 1 + - name: 2g.20gb + core: 28 + memory: 20480 + count: 3 + - + - name: 3g.40gb + core: 42 + memory: 40960 + count: 2 + - + - name: 7g.79gb + core: 100 + memory: 80896 + count: 1 + - models: [ "H100-PCIE-80GB", "H100-SXM5-80GB"] + allowedGeometries: + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 7 + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 1 + - name: 2g.20gb + core: 28 + memory: 20480 + count: 3 + - + - name: 3g.40gb + core: 42 + memory: 40960 + count: 2 + - + - name: 7g.80gb + core: 100 + memory: 81920 + count: 1 + - models: [ "H100-PCIE-94GB", "H100-SXM5-94GB"] + allowedGeometries: + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 7 + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 1 + - name: 2g.24gb + core: 28 + memory: 24576 + count: 3 + - + - name: 3g.47gb + core: 42 + memory: 48128 + count: 2 + - + - name: 7g.94gb + core: 100 + memory: 96256 + count: 1 + - models: [ "H20", "H100 on GH200"] + allowedGeometries: + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 7 + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 1 + - name: 2g.24gb + core: 28 + memory: 24576 + count: 3 + - + - name: 3g.48gb + core: 42 + memory: 49152 + count: 2 + - + - name: 7g.96gb + core: 100 + memory: 98304 + count: 1 + - models: [ "H200 NVL", "H200-SXM5"] + allowedGeometries: + - + - name: 1g.18gb + core: 14 + memory: 18432 + count: 7 + - + - name: 1g.18gb + core: 14 + memory: 18432 + count: 1 + - name: 2g.35gb + core: 28 + memory: 35840 + count: 3 + - + - name: 3g.71gb + core: 42 + memory: 72704 + count: 2 + - + - name: 7g.141gb + core: 100 + memory: 144384 + count: 1 + - models: [ "B200" ] + allowedGeometries: + - + - name: 1g.23gb + core: 14 + memory: 23552 + count: 7 + - + - name: 1g.23gb + core: 14 + memory: 23552 + count: 1 + - name: 2g.45gb + core: 28 + memory: 46080 + count: 3 + - + - name: 3g.90gb + core: 42 + memory: 92160 + count: 2 + - + - name: 7g.180gb + core: 100 + memory: 184320 + count: 1 + cambricon: + resourceCountName: {{ .Values.mluResourceName }} + resourceMemoryName: {{ .Values.mluResourceMem }} + resourceCoreName: {{ .Values.mluResourceCores }} + hygon: + resourceCountName: {{ .Values.dcuResourceName }} + resourceMemoryName: {{ .Values.dcuResourceMem }} + resourceCoreName: {{ .Values.dcuResourceCores }} + memoryFactor: 1 + metax: + resourceCountName: "metax-tech.com/gpu" + resourceVCountName: {{ .Values.metaxResourceName }} + resourceVMemoryName: {{ .Values.metaxResourceMem }} + resourceVCoreName: {{ .Values.metaxResourceCore }} + sgpuTopologyAware: {{ .Values.metaxsGPUTopologyAware }} + enflame: + resourceNameGCU: "enflame.com/gcu" + resourceNameVGCU: {{ .Values.enflameResourceNameVGCU }} + resourceNameVGCUPercentage: {{ .Values.enflameResourceNameVGCUPercentage }} + mthreads: + resourceCountName: "mthreads.com/vgpu" + resourceMemoryName: "mthreads.com/sgpu-memory" + resourceCoreName: "mthreads.com/sgpu-core" + iluvatars: + - chipName: MR-V100 + commonWord: MR-V100 + resourceCountName: iluvatar.ai/MR-V100-vgpu + resourceMemoryName: iluvatar.ai/MR-V100.vMem + resourceCoreName: iluvatar.ai/MR-V100.vCore + - chipName: MR-V50 + commonWord: MR-V50 + resourceCountName: iluvatar.ai/MR-V50-vgpu + resourceMemoryName: iluvatar.ai/MR-V50.vMem + resourceCoreName: iluvatar.ai/MR-V50.vCore + - chipName: BI-V150 + commonWord: BI-V150 + resourceCountName: iluvatar.ai/BI-V150-vgpu + resourceMemoryName: iluvatar.ai/BI-V150.vMem + resourceCoreName: iluvatar.ai/BI-V150.vCore + - chipName: BI-V100 + commonWord: BI-V100 + resourceCountName: iluvatar.ai/BI-V100-vgpu + resourceMemoryName: iluvatar.ai/BI-V100.vMem + resourceCoreName: iluvatar.ai/BI-V100.vCore + kunlun: + resourceCountName: {{ .Values.kunlunResourceName }} + resourceVCountName: {{ .Values.kunlunResourceVCountName }} + resourceVMemoryName: {{ .Values.kunlunResourceVMemoryName }} + awsneuron: + resourceCountName: "aws.amazon.com/neuron" + resourceCoreName: "aws.amazon.com/neuroncore" + amd: + resourceCountName: "amd.com/gpu" + vnpus: + - chipName: 910A + commonWord: Ascend910A + resourceName: huawei.com/Ascend910A + resourceMemoryName: huawei.com/Ascend910A-memory + memoryAllocatable: 32768 + memoryCapacity: 32768 + memoryFactor: 1 + aiCore: 30 + templates: + - name: vir02 + memory: 2184 + aiCore: 2 + - name: vir04 + memory: 4369 + aiCore: 4 + - name: vir08 + memory: 8738 + aiCore: 8 + - name: vir16 + memory: 17476 + aiCore: 16 + - chipName: 910B2 + commonWord: Ascend910B2 + resourceName: huawei.com/Ascend910B2 + resourceMemoryName: huawei.com/Ascend910B2-memory + memoryAllocatable: 65536 + memoryCapacity: 65536 + memoryFactor: 1 + aiCore: 24 + aiCPU: 6 + templates: + - name: vir03_1c_8g + memory: 8192 + aiCore: 3 + aiCPU: 1 + - name: vir06_1c_16g + memory: 16384 + aiCore: 6 + aiCPU: 1 + - name: vir12_3c_32g + memory: 32768 + aiCore: 12 + aiCPU: 3 + - chipName: 910B3 + commonWord: Ascend910B3 + resourceName: huawei.com/Ascend910B3 + resourceMemoryName: huawei.com/Ascend910B3-memory + memoryAllocatable: 65536 + memoryCapacity: 65536 + memoryFactor: 1 + aiCore: 20 + aiCPU: 7 + templates: + - name: vir05_1c_16g + memory: 16384 + aiCore: 5 + aiCPU: 1 + - name: vir10_3c_32g + memory: 32768 + aiCore: 10 + aiCPU: 3 + - chipName: 910B4-1 + commonWord: Ascend910B4-1 + resourceName: huawei.com/Ascend910B4-1 + resourceMemoryName: huawei.com/Ascend910B4-1-memory + memoryAllocatable: 65536 + memoryCapacity: 65536 + memoryFactor: 1 + aiCore: 20 + aiCPU: 7 + templates: + # NOTE: Names of vnpu template for 910B4-1 are fixed by Ascend runtime and must not be changed. + # The memory is used for scheduling so the correct values must be set. + # Template vir05_1c_8g actually provides 16GB memory, + - name: vir05_1c_8g + memory: 16384 + aiCore: 5 + aiCPU: 1 + # Template vir10_3c_16g actually provides 32GB memory + - name: vir10_3c_16g + memory: 32768 + aiCore: 10 + aiCPU: 3 + - chipName: 910B4 + commonWord: Ascend910B4 + resourceName: huawei.com/Ascend910B4 + resourceMemoryName: huawei.com/Ascend910B4-memory + memoryAllocatable: 32768 + memoryCapacity: 32768 + memoryFactor: 1 + aiCore: 20 + aiCPU: 7 + templates: + - name: vir05_1c_8g + memory: 8192 + aiCore: 5 + aiCPU: 1 + - name: vir10_3c_16g + memory: 16384 + aiCore: 10 + aiCPU: 3 + - chipName: 310P3 + commonWord: Ascend310P + resourceName: huawei.com/Ascend310P + resourceMemoryName: huawei.com/Ascend310P-memory + memoryAllocatable: 21527 + memoryCapacity: 24576 + memoryFactor: 1 + aiCore: 8 + aiCPU: 7 + templates: + - name: vir01 + memory: 3072 + aiCore: 1 + aiCPU: 1 + - name: vir02 + memory: 6144 + aiCore: 2 + aiCPU: 2 + - name: vir04 + memory: 12288 + aiCore: 4 + aiCPU: 4 + {{ end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml new file mode 100644 index 00000000..77e891cc --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +rules: + - apiGroups: + - admissionregistration.k8s.io + resources: + #- validatingwebhookconfigurations + - mutatingwebhookconfigurations + verbs: + - get + - update +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml new file mode 100644 index 00000000..2b82f926 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "hami-vgpu.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.fullname" . }}-admission + namespace: {{ include "hami-vgpu.namespace" . }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml new file mode 100644 index 00000000..0e36d95f --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml @@ -0,0 +1,68 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission-create + namespace: {{ include "hami-vgpu.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission-create + {{- if .Values.scheduler.patch.podAnnotations }} + annotations: {{ toYaml .Values.scheduler.patch.podAnnotations | nindent 8 }} + {{- end }} + labels: + {{- include "hami-vgpu.labels" . | nindent 8 }} + app.kubernetes.io/component: admission-webhook + hami.io/webhook: ignore + spec: + {{- if .Values.scheduler.patch.priorityClassName }} + priorityClassName: {{ .Values.scheduler.patch.priorityClassName }} + {{- end }} + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + {{- include "hami.scheduler.patch.new.imagePullSecrets" . | nindent 6 }} + {{- else }} + {{- include "hami.scheduler.patch.imagePullSecrets" . | nindent 6 }} + {{- end }} + containers: + - name: create + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + image: {{ include "hami.scheduler.patch.new.image" . }} + imagePullPolicy: {{ .Values.scheduler.patch.imageNew.pullPolicy }} + {{- else }} + image: {{ include "hami.scheduler.patch.image" . }} + imagePullPolicy: {{ .Values.scheduler.patch.image.pullPolicy }} + {{- end }} + args: + - create + - --cert-name=tls.crt + - --key-name=tls.key + {{- if .Values.scheduler.admissionWebhook.customURL.enabled }} + - --host={{ printf "%s.%s.svc,127.0.0.1,%s" (include "hami-vgpu.scheduler" .) (include "hami-vgpu.namespace" .) .Values.scheduler.admissionWebhook.customURL.host}} + {{- else }} + - --host={{ printf "%s.%s.svc,127.0.0.1" (include "hami-vgpu.scheduler" .) (include "hami-vgpu.namespace" .) }} + {{- end }} + - --namespace={{ include "hami-vgpu.namespace" . }} + - --secret-name={{ include "hami-vgpu.scheduler.tls" . }} + restartPolicy: OnFailure + serviceAccountName: {{ include "hami-vgpu.fullname" . }}-admission + {{- if .Values.scheduler.patch.nodeSelector }} + nodeSelector: {{ toYaml .Values.scheduler.patch.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.scheduler.patch.tolerations }} + tolerations: {{ toYaml .Values.scheduler.patch.tolerations | nindent 8 }} + {{- end }} + securityContext: + runAsNonRoot: true + runAsUser: {{ .Values.scheduler.patch.runAsUser }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml new file mode 100644 index 00000000..ce52042c --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml @@ -0,0 +1,63 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission-patch + namespace: {{ include "hami-vgpu.namespace" . }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission-patch + {{- if .Values.scheduler.patch.podAnnotations }} + annotations: {{ toYaml .Values.scheduler.patch.podAnnotations | nindent 8 }} + {{- end }} + labels: + {{- include "hami-vgpu.labels" . | nindent 8 }} + app.kubernetes.io/component: admission-webhook + hami.io/webhook: ignore + spec: + {{- if .Values.scheduler.patch.priorityClassName }} + priorityClassName: {{ .Values.scheduler.patch.priorityClassName }} + {{- end }} + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + {{- include "hami.scheduler.patch.new.imagePullSecrets" . | nindent 6 }} + {{- else }} + {{- include "hami.scheduler.patch.imagePullSecrets" . | nindent 6 }} + {{- end }} + containers: + - name: patch + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + image: {{ include "hami.scheduler.patch.new.image" . }} + imagePullPolicy: {{ .Values.scheduler.patch.imageNew.pullPolicy }} + {{- else }} + image: {{ include "hami.scheduler.patch.image" . }} + imagePullPolicy: {{ .Values.scheduler.patch.image.pullPolicy }} + {{- end }} + args: + - patch + - --webhook-name={{ include "hami-vgpu.scheduler.webhook" . }} + - --namespace={{ include "hami-vgpu.namespace" . }} + - --patch-validating=false + - --secret-name={{ include "hami-vgpu.scheduler.tls" . }} + restartPolicy: OnFailure + serviceAccountName: {{ include "hami-vgpu.fullname" . }}-admission + {{- if .Values.scheduler.patch.nodeSelector }} + nodeSelector: {{ toYaml .Values.scheduler.patch.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.scheduler.patch.tolerations }} + tolerations: {{ toYaml .Values.scheduler.patch.tolerations | nindent 8 }} + {{- end }} + securityContext: + runAsNonRoot: true + runAsUser: {{ .Values.scheduler.patch.runAsUser }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml new file mode 100644 index 00000000..56682f8e --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission + namespace: {{ include "hami-vgpu.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml new file mode 100644 index 00000000..7239b128 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission + namespace: {{ include "hami-vgpu.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "hami-vgpu.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.fullname" . }}-admission + namespace: {{ include "hami-vgpu.namespace" . }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml new file mode 100644 index 00000000..857c6a8d --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission + namespace: {{ include "hami-vgpu.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/role.yaml b/packages/system/hami/charts/hami/templates/scheduler/role.yaml new file mode 100644 index 00000000..5f7d7e44 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/role.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create", "list", "watch", "get", "update", "patch"] diff --git a/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml new file mode 100644 index 00000000..96e175a1 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml @@ -0,0 +1,31 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "hami-vgpu.scheduler" . }} +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +--- +{{- if .Values.mockDevicePlugin.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.mock-device-plugin" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "hami-vgpu.mock-device-plugin" . }} +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.mock-device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/scheduler/service.yaml b/packages/system/hami/charts/hami/templates/scheduler/service.yaml new file mode 100644 index 00000000..d7538fed --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/service.yaml @@ -0,0 +1,34 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} + {{- if .Values.scheduler.service.labels }} + {{ toYaml .Values.scheduler.service.labels | indent 4 }} + {{- end }} + {{- if .Values.scheduler.service.annotations }} + annotations: {{ toYaml .Values.scheduler.service.annotations | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.scheduler.service.type | default "NodePort" }} # Default type is NodePort + ports: + - name: http + port: {{ .Values.scheduler.service.httpPort | default 443 }} # Default HTTP port is 443 + targetPort: {{ .Values.scheduler.service.httpTargetPort | default 443 }} + {{- if eq (.Values.scheduler.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort + nodePort: {{ .Values.scheduler.service.schedulerPort | default 31998 }} + {{- end }} + protocol: TCP + - name: monitor + port: {{ .Values.scheduler.service.monitorPort | default 31993 }} # Default monitoring port is 31993 + targetPort: {{ .Values.scheduler.service.monitorTargetPort | default 9395 }} + {{- if eq (.Values.scheduler.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort + nodePort: {{ .Values.scheduler.service.monitorPort | default 31993 }} + {{- end }} + protocol: TCP + selector: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.selectorLabels" . | nindent 4 }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml b/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml new file mode 100644 index 00000000..0435c003 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +--- +{{- if .Values.mockDevicePlugin.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "hami-vgpu.mock-device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml b/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml new file mode 100644 index 00000000..db9f8029 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml @@ -0,0 +1,57 @@ +{{- if .Values.scheduler.admissionWebhook.enabled -}} +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + {{- if .Values.scheduler.certManager.enabled }} + annotations: + cert-manager.io/inject-ca-from: {{ include "hami-vgpu.namespace" . }}/{{ include "hami-vgpu.scheduler" . }}-serving-cert + {{- end }} + name: {{ include "hami-vgpu.scheduler.webhook" . }} +webhooks: + - admissionReviewVersions: + - v1beta1 + clientConfig: + {{- if .Values.scheduler.admissionWebhook.customURL.enabled }} + url: https://{{ .Values.scheduler.admissionWebhook.customURL.host}}:{{.Values.scheduler.admissionWebhook.customURL.port}}{{.Values.scheduler.admissionWebhook.customURL.path}} + {{- else }} + service: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + path: /webhook + port: {{ .Values.scheduler.service.httpPort }} + {{- end }} + failurePolicy: {{ .Values.scheduler.admissionWebhook.failurePolicy }} + matchPolicy: Equivalent + name: vgpu.hami.io + namespaceSelector: + matchExpressions: + - key: hami.io/webhook + operator: NotIn + values: + - ignore + {{- if .Values.scheduler.admissionWebhook.whitelistNamespaces }} + - key: kubernetes.io/metadata.name + operator: NotIn + values: + {{- toYaml .Values.scheduler.admissionWebhook.whitelistNamespaces | nindent 10 }} + {{- end }} + objectSelector: + matchExpressions: + - key: hami.io/webhook + operator: NotIn + values: + - ignore + reinvocationPolicy: {{ .Values.scheduler.admissionWebhook.reinvocationPolicy }} + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - pods + scope: '*' + sideEffects: None + timeoutSeconds: 10 +{{- end }} diff --git a/packages/system/hami/charts/hami/values.yaml b/packages/system/hami/charts/hami/values.yaml new file mode 100644 index 00000000..01caca0b --- /dev/null +++ b/packages/system/hami/charts/hami/values.yaml @@ -0,0 +1,455 @@ +## @section Global configuration +## @param global.imageRegistry Global Docker image registry +## @param global.imagePullSecrets Global Docker image pull secrets +global: + ## @param global.imageRegistry Global Docker image registry + imageRegistry: "" + ## E.g. + ## imagePullSecrets: + ## - myRegistryKeySecretName + ## @param global.imagePullSecrets Global Docker image pull secrets + imagePullSecrets: [] + imageTag: "v2.8.1" + gpuHookPath: /usr/local + labels: {} + annotations: {} + managedNodeSelectorEnable: false + managedNodeSelector: + usage: "gpu" + +nameOverride: "" +fullnameOverride: "" +namespaceOverride: "" + +#Nvidia GPU Parameters +resourceName: "nvidia.com/gpu" +resourceMem: "nvidia.com/gpumem" +resourceMemPercentage: "nvidia.com/gpumem-percentage" +resourceCores: "nvidia.com/gpucores" +resourcePriority: "nvidia.com/priority" + +#MLU Parameters +mluResourceName: "cambricon.com/vmlu" +mluResourceMem: "cambricon.com/mlu.smlu.vmemory" +mluResourceCores: "cambricon.com/mlu.smlu.vcore" + +#Hygon DCU Parameters +dcuResourceName: "hygon.com/dcunum" +dcuResourceMem: "hygon.com/dcumem" +dcuResourceCores: "hygon.com/dcucores" + +#Metax sGPU Parameters +metaxResourceName: "metax-tech.com/sgpu" +metaxResourceCore: "metax-tech.com/vcore" +metaxResourceMem: "metax-tech.com/vmemory" +metaxsGPUTopologyAware: "false" + +#Enflame VGCU Parameters +enflameResourceNameVGCU: "enflame.com/vgcu" +enflameResourceNameVGCUPercentage: "enflame.com/vgcu-percentage" + +#Kunlun XPU Parameters +kunlunResourceName: "kunlunxin.com/xpu" +kunlunResourceVCountName: "kunlunxin.com/vxpu" +kunlunResourceVMemoryName: "kunlunxin.com/vxpu-memory" + +schedulerName: "hami-scheduler" + +scheduler: + # @param nodeName defines the node name and the nvidia-vgpu-scheduler-scheduler will schedule to the node. + # if we install the nvidia-vgpu-scheduler-scheduler as default scheduler, we need to remove the k8s default + # scheduler pod from the cluster first, we must specify node name to skip the schedule workflow. + nodeName: "" + #nodeLabelSelector: + # "gpu": "on" + overwriteEnv: "false" + defaultSchedulerPolicy: + nodeSchedulerPolicy: binpack + gpuSchedulerPolicy: spread + metricsBindAddress: ":9395" + # If set to false, When Pod.Spec.SchedulerName equals to the const DefaultSchedulerName in k8s.io/api/core/v1 package, webhook will not overwrite it + forceOverwriteDefaultScheduler: true + livenessProbe: false + leaderElect: true + # when leaderElect is true, replicas is available, otherwise replicas is 1. + replicas: 1 + kubeScheduler: + # @param enabled indicate whether to run kube-scheduler container in the scheduler pod, it's true by default. + enabled: true + ## @param image.registry kube scheduler image registry + ## @param image.repository kube scheduler image repository + ## @param image.tag kube scheduler image tag (immutable tags are recommended) + ## @param image.pullPolicy kube scheduler image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + image: + registry: "registry.cn-hangzhou.aliyuncs.com" + repository: "google_containers/kube-scheduler" + tag: "" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + resources: {} + # If you do want to specify resources, uncomment the following lines, adjust them as necessary. + # and remove the curly braces after 'resources:'. +# limits: +# cpu: 1000m +# memory: 1000Mi +# requests: +# cpu: 100m +# memory: 100Mi + extraNewArgs: + - --config=/config/config.yaml + - -v=4 + extraArgs: + - --policy-config-file=/config/config.json + - -v=4 + extender: + ## @param image.registry scheduler extender image registry + ## @param image.repository scheduler extender image repository + ## @param image.tag scheduler extender image tag (immutable tags are recommended) + ## @param image.pullPolicy scheduler extender image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + image: + registry: "docker.io" + repository: "projecthami/hami" + tag: "" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + resources: {} + # If you do want to specify resources, uncomment the following lines, adjust them as necessary, + # and remove the curly braces after 'resources:'. +# limits: +# cpu: 1000m +# memory: 1000Mi +# requests: +# cpu: 100m +# memory: 100Mi + extraArgs: + - --debug + - -v=4 + nodeLockExpire: "5m" + podAnnotations: {} + tolerations: [] + #serviceAccountName: "hami-vgpu-scheduler-sa" + admissionWebhook: + # If set to false, the admission webhook is not installed and any pods that should use HAMi must be + # configured to use the right 'schedulerName' and other device-specific configurations. + enabled: true + customURL: + enabled: false + # must be an endpoint using https. + # should generate host certs here + host: 127.0.0.1 # hostname or ip, can be your node'IP if you want to use https://:/ + port: 31998 + path: /webhook + whitelistNamespaces: + # Specify the namespaces that the webhook will not be applied to. + # - default + # - kube-system + # - istio-system + reinvocationPolicy: Never + failurePolicy: Ignore + ## TLS Certificate Option 1: Use cert-manager to generate self-signed certificate. + ## If enabled, always takes precedence over options 2. + certManager: + enabled: false + ## TLS Certificate Option 2: Use kube-webhook-certgen to generate self-signed certificate. + ## If true and certManager.enabled is false, Helm will automatically create a self-signed cert and secret for you. + patch: + enabled: true + ## @param image.registry scheduler extender image registry + ## @param image.repository scheduler extender image repository + ## @param image.tag scheduler extender image tag (immutable tags are recommended) + ## @param image.pullPolicy scheduler extender image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + image: + registry: "docker.io" + repository: "jettech/kube-webhook-certgen" + tag: "v1.5.2" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## @param image.registry scheduler extender image registry + ## @param image.repository scheduler extender image repository + ## @param image.tag scheduler extender image tag (immutable tags are recommended) + ## @param image.pullPolicy scheduler extender image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + imageNew: + registry: "docker.io" + repository: "liangjw/kube-webhook-certgen" + tag: "v1.1.1" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + priorityClassName: "" + podAnnotations: {} + nodeSelector: {} + tolerations: [] + runAsUser: 2000 + service: + type: NodePort # Default type is NodePort, can be changed to ClusterIP + httpPort: 443 # HTTP port + schedulerPort: 31998 # NodePort for HTTP + monitorPort: 31993 # Monitoring port + monitorTargetPort: 9395 + httpTargetPort: 443 + labels: {} + annotations: {} + +devicePlugin: + enabled: true + gpuOperatorToolkitReady: + enabled: false + hostPath: "/run/nvidia/validations" + ## @param image.registry devicePlugin image registry + ## @param image.repository devicePlugin image repository + ## @param image.tag devicePlugin image tag (immutable tags are recommended) + ## @param image.pullPolicy devicePlugin image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + image: + registry: "docker.io" + repository: "projecthami/hami" + tag: "" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + monitor: + ## @param image.registry monitor image registry + ## @param image.repository monitor image repository + ## @param image.tag monitor image tag (immutable tags are recommended) + ## @param image.pullPolicy monitor image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + image: + registry: "docker.io" + repository: "projecthami/hami" + tag: "" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ctrPath: /usr/local/vgpu/containers + resyncInterval: "5m" + extraArgs: + - -v=4 + extraEnvs: {} + resources: {} + # If you do want to specify resources, uncomment the following lines, adjust them as necessary. + # and remove the curly braces after 'resources:'. +# limits: +# cpu: 1000m +# memory: 1000Mi +# requests: +# cpu: 100m +# memory: 100Mi + + deviceSplitCount: 10 + deviceMemoryScaling: 1 + deviceCoreScaling: 1 + # Node configuration for device plugin, Priority: externalConfigName > config > default config + nodeConfiguration: + # If you want to use a custom config.json, you can set the content here. + # If this is set, it will override the default config.json(An example is as follows). + config: | + { + "nodeconfig": [ + { + "name": "your-node-name", + "operatingmode": "hami-core", + "devicememoryscaling": 1, + "devicesplitcount": 10, + "migstrategy": "none", + "filterdevices": { + "uuid": [], + "index": [] + } + } + ] + } + # If you want to use an existing ConfigMap, you can set the name here. + # If this is set, the chart will not create the ConfigMap and will use the existing one. + externalConfigName: "" + # The runtime class name to be used by the device plugin, and added to the pod.spec.runtimeClassName of applications utilizing NVIDIA GPUs + runtimeClassName: "" + # Whether to create runtime class, name comes from runtimeClassName when it is set + createRuntimeClass: false + migStrategy: "none" + disablecorelimit: "false" + passDeviceSpecsEnabled: false + deviceListStrategy: "envvar" + nvidiaHookPath: null + nvidiaDriverRoot: null + gdrcopyEnabled: null + gdsEnabled: null + mofedEnabled: null + + extraArgs: + - -v=4 + extraEnvs: {} + service: + type: NodePort # Default type is NodePort, can be changed to ClusterIP + httpPort: 31992 + labels: {} + annotations: {} + + pluginPath: /var/lib/kubelet/device-plugins + libPath: /usr/local/vgpu + + podAnnotations: {} + nvidiaNodeSelector: + gpu: "on" + tolerations: [] + # The updateStrategy for DevicePlugin DaemonSet. + # If you want to update the DaemonSet by manual, set type as "OnDelete". + # We recommend use OnDelete update strategy because DevicePlugin pod restart will cause business pod restart, this behavior is destructive. + # Otherwise, you can use RollingUpdate update strategy to rolling update DevicePlugin pod. + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + + resources: {} + # If you do want to specify resources, uncomment the following lines, adjust them as necessary. + # and remove the curly braces after 'resources:'. +# limits: +# cpu: 1000m +# memory: 1000Mi +# requests: +# cpu: 100m +# memory: 100Mi + +mockDevicePlugin: + enabled: false + image: + registry: "docker.io" + repository: "projecthami/mock-device-plugin" + tag: "1.0.1" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] +devices: + amd: + customresources: + - amd.com/gpu + - amd.com/gpu-memory + awsneuron: + customresources: + - aws.amazon.com/neuron + - aws.amazon.com/neuroncore + kunlun: + enabled: true + customresources: + - kunlunxin.com/xpu + - kunlunxin.com/vxpu + - kunlunxin.com/vxpu-memory + enflame: + enabled: true + customresources: + - enflame.com/vgcu + - enflame.com/vgcu-percentage + - enflame.com/gcu + mthreads: + enabled: true + customresources: + - mthreads.com/vgpu + nvidia: + gpuCorePolicy: default + libCudaLogLevel: 1 + ascend: + enabled: false + image: "" + imagePullPolicy: IfNotPresent + extraArgs: [] + nodeSelector: + ascend: "on" + tolerations: [] + customresources: + - huawei.com/Ascend910A + - huawei.com/Ascend910A-memory + - huawei.com/Ascend910B2 + - huawei.com/Ascend910B2-memory + - huawei.com/Ascend910B3 + - huawei.com/Ascend910B3-memory + - huawei.com/Ascend910B4 + - huawei.com/Ascend910B4-memory + - huawei.com/Ascend910B4-1 + - huawei.com/Ascend910B4-1-memory + - huawei.com/Ascend310P + - huawei.com/Ascend310P-memory + iluvatar: + enabled: false + customresources: + - iluvatar.ai/BI-V100-vgpu + - iluvatar.ai/BI-V100.vCore + - iluvatar.ai/BI-V100.vMem + - iluvatar.ai/BI-V150-vgpu + - iluvatar.ai/BI-V150.vCore + - iluvatar.ai/BI-V150.vMem + - iluvatar.ai/MR-V100-vgpu + - iluvatar.ai/MR-V100.vCore + - iluvatar.ai/MR-V100.vMem + - iluvatar.ai/MR-V50-vgpu + - iluvatar.ai/MR-V50.vCore + - iluvatar.ai/MR-V50.vMem diff --git a/packages/system/hami/values.yaml b/packages/system/hami/values.yaml new file mode 100644 index 00000000..76e8d631 --- /dev/null +++ b/packages/system/hami/values.yaml @@ -0,0 +1,17 @@ +hami: + scheduler: + kubeScheduler: + image: + registry: registry.k8s.io + repository: kube-scheduler + devicePlugin: + runtimeClassName: nvidia + updateStrategy: + type: RollingUpdate + # See https://github.com/Project-HAMi/HAMi/blob/v2.8.1/deployments/charts/hami/values.yaml + # Each entry: {"name": "node-name", "operatingmode": "hami-core", "devicesplitcount": N, ...} + nodeConfiguration: + config: | + { + "nodeconfig": [] + } diff --git a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml index 1631c86e..f81eaff0 100644 --- a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml +++ b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml @@ -22,7 +22,7 @@ spec: singular: kubernetes plural: kuberneteses openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"ephemeralStorage":"20Gi","gpus":[],"instanceType":"u1.medium","maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"]}},"additionalProperties":{"type":"object","required":["ephemeralStorage","instanceType","maxReplicas","minReplicas","resources"],"properties":{"ephemeralStorage":{"description":"Ephemeral storage size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string"}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium"},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `.` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","ingressNginx","monitoringAgents","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"large","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"images":{"description":"Optional image overrides for air-gapped or rate-limited registries.","type":"object","default":{},"properties":{"waitForKubeconfig":{"description":"Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.","type":"string","default":""}}}}} + {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"ephemeralStorage":"20Gi","gpus":[],"instanceType":"u1.medium","maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"]}},"additionalProperties":{"type":"object","required":["ephemeralStorage","instanceType","maxReplicas","minReplicas","resources"],"properties":{"ephemeralStorage":{"description":"Ephemeral storage size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string"}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium"},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `.` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","hami","ingressNginx","monitoringAgents","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"hami":{"description":"HAMi GPU virtualization middleware.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable HAMi (requires GPU Operator).","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"large","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"images":{"description":"Optional image overrides for air-gapped or rate-limited registries.","type":"object","default":{},"properties":{"waitForKubeconfig":{"description":"Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.","type":"string","default":""}}}}} release: prefix: kubernetes- labels: @@ -40,7 +40,7 @@ spec: tags: - compute icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODFfMjg0NSkiLz4KPHBhdGggZD0iTTcxLjk5NjggMTlDNzAuMzAzOSAxOS4wMDAyIDY4LjkzMTIgMjAuNTMyMiA2OC45MzE0IDIyLjQyMjFDNjguOTMxNCAyMi40NTExIDY4LjkzNzMgMjIuNDc4OCA2OC45Mzc5IDIyLjUwNzZDNjguOTM1NCAyMi43NjQ0IDY4LjkyMzEgMjMuMDczNyA2OC45MzE0IDIzLjI5NzNDNjguOTcxNyAyNC4zODczIDY5LjIwODIgMjUuMjIxNiA2OS4zNTA2IDI2LjIyNThDNjkuNjA4NCAyOC4zNzUyIDY5LjgyNDUgMzAuMTU2OSA2OS42OTEyIDMxLjgxM0M2OS41NjE1IDMyLjQzNzUgNjkuMTAzNyAzMy4wMDg2IDY4LjY5NTYgMzMuNDA1Nkw2OC42MjM1IDM0LjcwODZDNjYuNzgzOSAzNC44NjE3IDY0LjkzMTkgMzUuMTQyMSA2My4wODIxIDM1LjU2NDFDNTUuMTIyNiAzNy4zNzk4IDQ4LjI2OTUgNDEuNDk5MSA0My4wNTIgNDcuMDYwOUM0Mi43MTM0IDQ2LjgyODggNDIuMTIxMSA0Ni40MDE5IDQxLjk0NSA0Ni4yNzEyQzQxLjM5NzcgNDYuMzQ1NCA0MC44NDQ1IDQ2LjUxNTEgNDAuMTI0MSA0Ni4wOTM1QzM4Ljc1MjIgNDUuMTY1NyAzNy41MDI4IDQzLjg4NTEgMzUuOTkxIDQyLjM0MjRDMzUuMjk4MiA0MS42MDQ0IDM0Ljc5NjYgNDAuOTAxOCAzMy45NzM1IDQwLjE5MDRDMzMuNzg2NiA0MC4wMjg5IDMzLjUwMTQgMzkuODEwNCAzMy4yOTIzIDM5LjY0NDJDMzIuNjQ4OSAzOS4xMjg4IDMxLjg5IDM4Ljg2IDMxLjE1NyAzOC44MzQ4QzMwLjIxNDcgMzguODAyNCAyOS4zMDc1IDM5LjE3MjUgMjguNzEzOCAzOS45MjA2QzI3LjY1ODQgNDEuMjUwNiAyNy45OTYzIDQzLjI4MzMgMjkuNDY3MSA0NC40NjE0QzI5LjQ4MiA0NC40NzM0IDI5LjQ5NzkgNDQuNDgyNyAyOS41MTI5IDQ0LjQ5NDNDMjkuNzE1IDQ0LjY1ODkgMjkuOTYyNSA0NC44Njk4IDMwLjE0ODMgNDUuMDA3NkMzMS4wMjE3IDQ1LjY1NTUgMzEuODE5NSA0NS45ODcyIDMyLjY4OTcgNDYuNTAxNUMzNC41MjMxIDQ3LjYzOTEgMzYuMDQzIDQ4LjU4MjMgMzcuMjQ4NiA0OS43MTk2QzM3LjcxOTQgNTAuMjIzNyAzNy44MDE2IDUxLjExMjIgMzcuODY0MyA1MS40OTY0TDM4Ljg0NjggNTIuMzc4MkMzMy41ODcyIDYwLjMzMDggMzEuMTUzIDcwLjE1MzkgMzIuNTkxNSA4MC4xNjI3TDMxLjMwNzcgODAuNTM3OEMzMC45NjkzIDgwLjk3NjggMzAuNDkxMiA4MS42Njc2IDI5Ljk5MTEgODEuODczOEMyOC40MTM4IDgyLjM3MjkgMjYuNjM4NyA4Mi41NTYyIDI0LjQ5NTYgODIuNzgxOUMyMy40ODk0IDgyLjg2NiAyMi42MjEzIDgyLjgxNTggMjEuNTU0NiA4My4wMTg4QzIxLjMxOTggODMuMDYzNSAyMC45OTI3IDgzLjE0OTEgMjAuNzM1OCA4My4yMDk3QzIwLjcyNjkgODMuMjExNiAyMC43MTg2IDgzLjIxNDIgMjAuNzA5NiA4My4yMTYyQzIwLjY5NTYgODMuMjE5NSAyMC42NzcyIDgzLjIyNjMgMjAuNjYzOCA4My4yMjk0QzE4Ljg1NyA4My42NjggMTcuNjk2MyA4NS4zMzY1IDE4LjA2OTkgODYuOTgwNUMxOC40NDM3IDg4LjYyNDggMjAuMjA4NiA4OS42MjQ4IDIyLjAyNjIgODkuMjMxMkMyMi4wMzkzIDg5LjIyODIgMjIuMDU4NCA4OS4yMjc3IDIyLjA3MiA4OS4yMjQ2QzIyLjA5MjYgODkuMjE5OSAyMi4xMTA2IDg5LjIwOTkgMjIuMTMxIDg5LjIwNDlDMjIuMzg0NCA4OS4xNDkgMjIuNzAxOSA4OS4wODY4IDIyLjkyMzYgODkuMDI3MkMyMy45NzIzIDg4Ljc0NTEgMjQuNzMxOCA4OC4zMzA2IDI1LjY3NDYgODcuOTY3N0MyNy43MDI5IDg3LjIzNjggMjkuMzgyOCA4Ni42MjYyIDMxLjAxOTUgODYuMzg4M0MzMS43MDMgODYuMzM0NSAzMi40MjMyIDg2LjgxMiAzMi43ODE0IDg3LjAxMzRMMzQuMTE3NyA4Ni43ODMxQzM3LjE5MjYgOTYuMzYxMyA0My42MzY2IDEwNC4xMDMgNTEuNzk2MyAxMDguOTYxTDUxLjIzOTYgMTEwLjMwM0M1MS40NDAzIDExMC44MjQgNTEuNjYxNiAxMTEuNTMgNTEuNTEyMSAxMTIuMDQ1QzUwLjkxNzEgMTEzLjU5NSA0OS44OTggMTE1LjIzMSA0OC43Mzc0IDExNy4wNTVDNDguMTc1NSAxMTcuODk4IDQ3LjYwMDQgMTE4LjU1MiA0Ny4wOTM0IDExOS41MTZDNDYuOTcyIDExOS43NDcgNDYuODE3NSAxMjAuMTAyIDQ2LjcwMDQgMTIwLjM0NkM0NS45MTI1IDEyMi4wMzkgNDYuNDkwNCAxMjMuOTkgNDguMDAzOCAxMjQuNzIyQzQ5LjUyNjggMTI1LjQ1OSA1MS40MTcxIDEyNC42ODIgNTIuMjM1MiAxMjIuOTg1QzUyLjIzNjQgMTIyLjk4MiA1Mi4yNDA2IDEyMi45OCA1Mi4yNDE3IDEyMi45NzhDNTIuMjQyNiAxMjIuOTc2IDUyLjI0MDkgMTIyLjk3MyA1Mi4yNDE3IDEyMi45NzFDNTIuMzU4MiAxMjIuNzMxIDUyLjUyMzMgMTIyLjQxNSA1Mi42MjE2IDEyMi4xODhDNTMuMDU2IDEyMS4xODkgNTMuMjAwNSAxMjAuMzMyIDUzLjUwNTkgMTE5LjM2NUM1NC4zMTcgMTE3LjMxOCA1NC43NjI2IDExNS4xNyA1NS44NzkxIDExMy44MzJDNTYuMTg0OSAxMTMuNDY2IDU2LjY4MzMgMTEzLjMyNSA1Ny4yMDAxIDExMy4xODZMNTcuODk0NCAxMTEuOTIyQzY1LjAwOCAxMTQuNjY1IDcyLjk3MDUgMTE1LjQwMiA4MC45MjQ1IDExMy41ODdDODIuNzM5MSAxMTMuMTczIDg0LjQ5MDggMTEyLjYzNyA4Ni4xODQzIDExMS45OTRDODYuMzc5NCAxMTIuMzQyIDg2Ljc0MiAxMTMuMDExIDg2LjgzOTMgMTEzLjE3OUM4Ny4zNjQ0IDExMy4zNTEgODcuOTM3NyAxMTMuNDM5IDg4LjQwNDcgMTE0LjEzM0M4OS4yNDAxIDExNS41NjcgODkuODExNCAxMTcuMjYzIDkwLjUwNzMgMTE5LjMxMkM5MC44MTI4IDEyMC4yNzkgOTAuOTYzOCAxMjEuMTM2IDkxLjM5ODEgMTIyLjEzNkM5MS40OTcxIDEyMi4zNjMgOTEuNjYxNCAxMjIuNjg0IDkxLjc3OCAxMjIuOTI1QzkyLjU5NDQgMTI0LjYyOCA5NC40OTA3IDEyNS40MDcgOTYuMDE1OSAxMjQuNjY5Qzk3LjUyOTIgMTIzLjkzNyA5OC4xMDc3IDEyMS45ODYgOTcuMzE5NCAxMjAuMjkzQzk3LjIwMjMgMTIwLjA0OSA5Ny4wNDEyIDExOS42OTUgOTYuOTE5OCAxMTkuNDY0Qzk2LjQxMjcgMTE4LjQ5OSA5NS44Mzc3IDExNy44NTIgOTUuMjc1OCAxMTcuMDA5Qzk0LjExNTIgMTE1LjE4NSA5My4xNTI2IDExMy42NyA5Mi41NTc1IDExMi4xMkM5Mi4zMDg3IDExMS4zMiA5Mi41OTk1IDExMC44MjMgOTIuNzkzMyAxMTAuMzAzQzkyLjY3NzIgMTEwLjE3IDkyLjQyODggMTA5LjQxNCA5Mi4yODI0IDEwOS4wNTlDMTAwLjc2MiAxMDQuMDI5IDEwNy4wMTcgOTUuOTk4NSAxMDkuOTU1IDg2LjcyMzlDMTEwLjM1MSA4Ni43ODY1IDExMS4wNDEgODYuOTA5MSAxMTEuMjY1IDg2Ljk1NDJDMTExLjcyNiA4Ni42NDg3IDExMi4xNDkgODYuMjUwMSAxMTIuOTgxIDg2LjMxNTlDMTE0LjYxNyA4Ni41NTM3IDExNi4yOTcgODcuMTY0NSAxMTguMzI2IDg3Ljg5NTNDMTE5LjI2OCA4OC4yNTgxIDEyMC4wMjggODguNjc5MyAxMjEuMDc3IDg4Ljk2MTRDMTIxLjI5OCA4OS4wMjEgMTIxLjYxNiA4OS4wNzY2IDEyMS44NjkgODkuMTMyNUMxMjEuODg5IDg5LjEzNzUgMTIxLjkwOCA4OS4xNDc1IDEyMS45MjggODkuMTUyMkMxMjEuOTQyIDg5LjE1NTMgMTIxLjk2MSA4OS4xNTU4IDEyMS45NzQgODkuMTU4OEMxMjMuNzkyIDg5LjU1MiAxMjUuNTU3IDg4LjU1MjYgMTI1LjkzIDg2LjkwODFDMTI2LjMwMyA4NS4yNjQxIDEyNS4xNDMgODMuNTk1MiAxMjMuMzM2IDgzLjE1N0MxMjMuMDc0IDgzLjA5NyAxMjIuNzAxIDgyLjk5NSAxMjIuNDQ2IDgyLjk0NjVDMTIxLjM3OSA4Mi43NDM1IDEyMC41MTEgODIuNzkzNSAxMTkuNTA1IDgyLjcwOTVDMTE3LjM2MSA4Mi40ODM5IDExNS41ODYgODIuMzAwNCAxMTQuMDA5IDgxLjgwMTRDMTEzLjM2NiA4MS41NTA3IDExMi45MDggODAuNzgxOSAxMTIuNjg2IDgwLjQ2NTVMMTExLjQ0OCA4MC4xMDM1QzExMi4wOSA3NS40MzggMTExLjkxNyA3MC41ODI1IDExMC44MDYgNjUuNzI0M0MxMDkuNjg1IDYwLjgyMDggMTA3LjcwNCA1Ni4zMzYxIDEwNS4wNjIgNTIuMzg0OEMxMDUuMzc5IDUyLjA5NDggMTA1Ljk3OSA1MS41NjEyIDEwNi4xNDkgNTEuNDA0M0MxMDYuMTk5IDUwLjg1MTcgMTA2LjE1NiA1MC4yNzIyIDEwNi43MjUgNDkuNjYwM0MxMDcuOTMxIDQ4LjUyMyAxMDkuNDUxIDQ3LjU3OTkgMTExLjI4NCA0Ni40NDIzQzExMi4xNTQgNDUuOTI3OSAxMTIuOTU5IDQ1LjU5NjQgMTEzLjgzMiA0NC45NDg0QzExNC4wMyA0NC44MDE5IDExNC4yOTkgNDQuNTY5OSAxMTQuNTA3IDQ0LjQwMjJDMTE1Ljk3NyA0My4yMjM3IDExNi4zMTYgNDEuMTkxMSAxMTUuMjYgMzkuODYxNEMxMTQuMjA0IDM4LjUzMTcgMTEyLjE1OSAzOC40MDY1IDExMC42ODggMzkuNTg1QzExMC40NzkgMzkuNzUxNiAxMTAuMTk1IDM5Ljk2ODggMTEwLjAwNyA0MC4xMzEyQzEwOS4xODQgNDAuODQyNiAxMDguNjc2IDQxLjU0NTIgMTA3Ljk4MyA0Mi4yODMyQzEwNi40NzEgNDMuODI1OSAxMDUuMjIyIDQ1LjExMyAxMDMuODUgNDYuMDQwOUMxMDMuMjU1IDQ2LjM4ODUgMTAyLjM4NSA0Ni4yNjgyIDEwMS45OSA0Ni4yNDQ5TDEwMC44MjQgNDcuMDgwNkM5NC4xNzUzIDQwLjA3NjMgODUuMTIzNSAzNS41OTgyIDc1LjM3NjYgMzQuNzI4M0M3NS4zNDk0IDM0LjMxNzkgNzUuMzEzNyAzMy41NzYxIDc1LjMwNDYgMzMuMzUyOUM3NC45MDU2IDMyLjk2OTMgNzQuNDIzNSAzMi42NDE4IDc0LjMwMjQgMzEuODEzQzc0LjE2OTEgMzAuMTU2OSA3NC4zOTE3IDI4LjM3NTIgNzQuNjQ5NiAyNi4yMjU4Qzc0Ljc5MTkgMjUuMjIxNiA3NS4wMjg0IDI0LjM4NzMgNzUuMDY4OCAyMy4yOTczQzc1LjA3OCAyMy4wNDk1IDc1LjA2MzIgMjIuNjkgNzUuMDYyMiAyMi40MjIxQzc1LjA2MiAyMC41MzIyIDczLjY4OTggMTguOTk5OCA3MS45OTY4IDE5Wk02OC4xNTg1IDQyLjg4ODZMNjcuMjQ4IDU5LjA0NDdMNjcuMTgyNSA1OS4wNzc2QzY3LjEyMTQgNjAuNTIyOSA2NS45Mzc1IDYxLjY3NyA2NC40ODM5IDYxLjY3N0M2My44ODg0IDYxLjY3NyA2My4zMzg4IDYxLjQ4NDkgNjIuODkyMiA2MS4xNTcxTDYyLjg2NiA2MS4xNzAzTDQ5LjY4MDcgNTEuNzc5NEM1My43MzMxIDQ3Ljc3NTkgNTguOTE2NCA0NC44MTcyIDY0Ljg5IDQzLjQ1NDZDNjUuOTgxMiA0My4yMDU2IDY3LjA3MTkgNDMuMDIwOSA2OC4xNTg1IDQyLjg4ODZaTTc1Ljg0MTcgNDIuODg4NkM4Mi44MTU5IDQzLjc1MDQgODkuMjY1NyA0Ni45MjMyIDk0LjIwODEgNTEuNzg2TDgxLjEwOCA2MS4xMTc2TDgxLjA2MjEgNjEuMDk3OUM3OS44OTk0IDYxLjk1MTIgNzguMjYxMSA2MS43Mzk0IDc3LjM1NDggNjAuNTk3OEM3Ni45ODM1IDYwLjEzMDEgNzYuNzg4NyA1OS41ODAxIDc2Ljc2NTMgNTkuMDI0OUw3Ni43NTIyIDU5LjAxODRMNzUuODQxNyA0Mi44ODg2Wk00NC44OTkxIDU3LjgxNEw1Ni45MzgyIDY4LjYzM0w1Ni45MjUxIDY4LjY5ODhDNTguMDExNyA2OS42NDc5IDU4LjE3MiA3MS4yOTQ5IDU3LjI2NTcgNzIuNDM2OEM1Ni44OTQ0IDcyLjkwNDUgNTYuMzk3NSA3My4yMTgyIDU1Ljg2MzkgNzMuMzY0N0w1NS44NTA4IDczLjQxNzNMNDAuNDE4OCA3Ny44OTIzQzM5LjYzMzQgNzAuNjc2NSA0MS4zMjYxIDYzLjY2MjEgNDQuODk5MSA1Ny44MTRaTTk5LjAwOTQgNTcuODIwNkMxMDAuNzk4IDYwLjczMzYgMTAyLjE1MyA2My45ODcxIDEwMi45NTkgNjcuNTE0M0MxMDMuNzU2IDcwLjk5OTEgMTAzLjk1NiA3NC40Nzc4IDEwMy42MjcgNzcuODM5N0w4OC4xMTY2IDczLjM1MTVMODguMTAzNSA3My4yODU3Qzg2LjcxNDUgNzIuOTA0MyA4NS44NjA5IDcxLjQ4NDggODYuMTg0MyA3MC4wNjExQzg2LjMxNjggNjkuNDc3OCA4Ni42MjQ5IDY4Ljk4NDQgODcuMDQyMyA2OC42MTk4TDg3LjAzNTggNjguNTg2OUw5OS4wMDk0IDU3LjgyMDZaTTY5LjUyNzQgNjkuNDY4OEg3NC40NTk2TDc3LjUyNTEgNzMuMzE4Nkw3Ni40MjQ3IDc4LjEyMjZMNzEuOTk2OCA4MC4yNjE0TDY3LjU1NTggNzguMTE2MUw2Ni40NTU0IDczLjMxMkw2OS41Mjc0IDY5LjQ2ODhaTTg1LjMzOTMgODIuNjQzN0M4NS41NDg5IDgyLjYzMzEgODUuNzU3NiA4Mi42NTIgODUuOTYxNiA4Mi42ODk4TDg1Ljk4NzggODIuNjU2OUwxMDEuOTUgODUuMzY4MkM5OS42MTQyIDkxLjk2MjQgOTUuMTQ0IDk3LjY3NSA4OS4xNzExIDEwMS40OThMODIuOTc0NyA4Ni40NjA2TDgyLjk5NDQgODYuNDM0M0M4Mi40MjUyIDg1LjEwNTUgODIuOTk0OCA4My41NDcyIDg0LjMwNDQgODIuOTEzNUM4NC42Mzk3IDgyLjc1MTMgODQuOTkgODIuNjYxNCA4NS4zMzkzIDgyLjY0MzdaTTU4LjUyOTggODIuNzA5NUM1OS43NDggODIuNzI2NyA2MC44NDA2IDgzLjU3NjEgNjEuMTIzNyA4NC44MjJDNjEuMjU2MiA4NS40MDUyIDYxLjE5MTcgODUuOTgzMSA2MC45NzMgODYuNDkzNUw2MS4wMTg5IDg2LjU1MjdMNTQuODg4IDEwMS40MzlDNDkuMTU1OSA5Ny43NDMyIDQ0LjU5MDQgOTIuMjA5OSA0Mi4xNDgxIDg1LjQyMDhMNTcuOTczMSA4Mi43MjI3TDU3Ljk5OTMgODIuNzU1NkM1OC4xNzYzIDgyLjcyMjkgNTguMzU1OCA4Mi43MDcxIDU4LjUyOTggODIuNzA5NVpNNzEuODk4NiA4OS4yMzEyQzcyLjMyMjkgODkuMjE1NSA3Mi43NTM0IDg5LjMwMyA3My4xNjI3IDg5LjUwMUM3My42OTkyIDg5Ljc2MDYgNzQuMTEzNiA5MC4xNjkyIDc0LjM3NDUgOTAuNjU5Mkg3NC40MzM0TDgyLjIzNDYgMTA0LjgyMUM4MS4yMjIxIDEwNS4xNjIgODAuMTgxMyAxMDUuNDU0IDc5LjExNjcgMTA1LjY5N0M3My4xNTA1IDEwNy4wNTggNjcuMjAzMiAxMDYuNjQ1IDYxLjgxOCAxMDQuODAyTDY5LjU5OTUgOTAuNjY1OEg2OS42MTI2QzcwLjA3OTUgODkuNzg4OCA3MC45NjUgODkuMjY1NiA3MS44OTg2IDg5LjIzMTJaIiBmaWxsPSJ3aGl0ZSIgc3Ryb2tlPSJ3aGl0ZSIgc3Ryb2tlLXdpZHRoPSIwLjI1Ii8+CjxkZWZzPgo8bGluZWFyR3JhZGllbnQgaWQ9InBhaW50MF9saW5lYXJfNjgxXzI4NDUiIHgxPSIxMCIgeTE9IjE1LjUiIHgyPSIxNDQiIHkyPSIxMzEuNSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiPgo8c3RvcCBzdG9wLWNvbG9yPSIjNEQ4N0ZGIi8+CjxzdG9wIG9mZnNldD0iMSIgc3RvcC1jb2xvcj0iIzA1NDdEMCIvPgo8L2xpbmVhckdyYWRpZW50Pgo8L2RlZnM+Cjwvc3ZnPgo= - keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "nodeGroups"], ["spec", "nodeGroups", "md0"], ["spec", "nodeGroups", "md0", "minReplicas"], ["spec", "nodeGroups", "md0", "maxReplicas"], ["spec", "nodeGroups", "md0", "instanceType"], ["spec", "nodeGroups", "md0", "ephemeralStorage"], ["spec", "nodeGroups", "md0", "roles"], ["spec", "nodeGroups", "md0", "resources"], ["spec", "nodeGroups", "md0", "gpus"], ["spec", "version"], ["spec", "host"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "fluxcd"], ["spec", "addons", "fluxcd", "enabled"], ["spec", "addons", "fluxcd", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"], ["spec", "images"], ["spec", "images", "waitForKubeconfig"]] + keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "nodeGroups"], ["spec", "nodeGroups", "md0"], ["spec", "nodeGroups", "md0", "minReplicas"], ["spec", "nodeGroups", "md0", "maxReplicas"], ["spec", "nodeGroups", "md0", "instanceType"], ["spec", "nodeGroups", "md0", "ephemeralStorage"], ["spec", "nodeGroups", "md0", "roles"], ["spec", "nodeGroups", "md0", "resources"], ["spec", "nodeGroups", "md0", "gpus"], ["spec", "version"], ["spec", "host"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "hami"], ["spec", "addons", "hami", "enabled"], ["spec", "addons", "hami", "valuesOverride"], ["spec", "addons", "fluxcd"], ["spec", "addons", "fluxcd", "enabled"], ["spec", "addons", "fluxcd", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"], ["spec", "images"], ["spec", "images", "waitForKubeconfig"]] secrets: exclude: [] include: diff --git a/packages/system/lineage-controller-webhook/Makefile b/packages/system/lineage-controller-webhook/Makefile index d5bada31..cdc1cf01 100644 --- a/packages/system/lineage-controller-webhook/Makefile +++ b/packages/system/lineage-controller-webhook/Makefile @@ -4,8 +4,13 @@ NAMESPACE=cozy-system include ../../../hack/common-envs.mk include ../../../hack/package.mk +.PHONY: test + image: image-lineage-controller-webhook +test: + helm unittest . + image-lineage-controller-webhook: docker buildx build -f images/lineage-controller-webhook/Dockerfile ../../.. \ --tag $(REGISTRY)/lineage-controller-webhook:$(call settag,$(TAG)) \ diff --git a/packages/system/lineage-controller-webhook/README.md b/packages/system/lineage-controller-webhook/README.md new file mode 100644 index 00000000..1e43c9c8 --- /dev/null +++ b/packages/system/lineage-controller-webhook/README.md @@ -0,0 +1,67 @@ +# lineage-controller-webhook + +Cozystack system package for the **lineage controller webhook** — a mutating +admission webhook that stamps "lineage" labels onto tenant workloads, linking +each resource back to the Cozystack `Application` that ultimately owns it. + +The webhook intercepts CREATE and UPDATE on `pods`, `secrets`, `services`, +`persistentvolumeclaims`, `ingresses.networking.k8s.io`, and +`workloadmonitors.cozystack.io` outside system namespaces. For each request it +walks the ownership graph upward (Kubernetes `ownerReferences`, then the +`HelmRelease` Flux installed the resource with, then the Cozystack +`Application`-derived CRD that produced the HelmRelease) and writes the +discovered application's group, kind and name as labels +(`apps.cozystack.io/application.{group,kind,name}`) on the incoming object. +Those labels let the aggregated API server, the Cozystack dashboard, and other +lineage-aware consumers reason about which application a resource belongs to. + +The webhook serves TLS on port 9443 with a cert-manager issued certificate, and +runs with `failurePolicy: Fail` — every CREATE/UPDATE on the resources above +is gated on the webhook being reachable. + +## Topology + +The chart ships a single shape, modelled on `cozystack-api`: + +- **Deployment** with two replicas (override via `replicas`). +- **Soft `nodeAffinity`** preferring `node-role.kubernetes.io/control-plane` + (`Exists`, so it matches both Talos's empty value and k3s/kubeadm's `"true"`). + Soft means: the pod lands on a control-plane node when one is reachable, and + on any worker otherwise — no override needed for managed Kubernetes, + Cozy-in-Cozy tenants, or any other cluster where control-plane nodes aren't + visible. +- **Permissive `tolerations`** (`[{operator: Exists}]`) so the pod can land on + tainted control-plane nodes when the soft affinity is satisfiable. +- **Soft `podAntiAffinity`** on `kubernetes.io/hostname` so replicas spread + across nodes when possible (best-effort). +- **`PodDisruptionBudget`** with `maxUnavailable: 1`, unconditional. At + `replicas: 1` it's a useful no-op (allows full drain); at `replicas >= 2` + it caps disruption to one pod. +- **`Service` with `spec.trafficDistribution: PreferClose`** so the apiserver + prefers a webhook endpoint on its own node when one exists, and transparently + falls over to a remote endpoint otherwise. Requires Kubernetes ≥ 1.31; on + older clusters the field is silently ignored and traffic uses default + cluster-wide distribution. + +## Parameters + +All keys live under the top-level `lineageControllerWebhook:` map. + +| Name | Description | Value | +| ----------------------------- | --------------------------------------------------------- | -------------------------------------------------------------------------------- | +| `image` | Container image (digest-pinned) | `ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.3.0@sha256:e898…6fb0` | +| `debug` | Enable `--zap-log-level=debug` instead of `info` | `false` | +| `replicas` | Deployment replica count | `2` | +| `localK8sAPIEndpoint.enabled` | **Deprecated.** See note below. | `false` | + +### `localK8sAPIEndpoint.enabled` (deprecated) + +When enabled, this injects `KUBERNETES_SERVICE_HOST=status.hostIP` and +`KUBERNETES_SERVICE_PORT=6443` so the webhook talks to the kube-apiserver on +its own node. It was originally added to avoid latency on the +webhook-to-apiserver path, but it is only valid when the pod is actually +scheduled on an apiserver-bearing node — which the chart's soft control-plane +affinity no longer guarantees. With this flag enabled and the pod scheduled +off a control-plane node, the controller will crash-loop dialing a non- +apiserver hostIP. Slated for removal once the latency motivation is addressed +in the webhook itself; **leave disabled**. diff --git a/packages/system/lineage-controller-webhook/templates/pdb.yaml b/packages/system/lineage-controller-webhook/templates/pdb.yaml new file mode 100644 index 00000000..0786c8ff --- /dev/null +++ b/packages/system/lineage-controller-webhook/templates/pdb.yaml @@ -0,0 +1,9 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: lineage-controller-webhook +spec: + maxUnavailable: 1 + selector: + matchLabels: + app: lineage-controller-webhook diff --git a/packages/system/lineage-controller-webhook/templates/service.yaml b/packages/system/lineage-controller-webhook/templates/service.yaml index c5df90fb..fbb3ad54 100644 --- a/packages/system/lineage-controller-webhook/templates/service.yaml +++ b/packages/system/lineage-controller-webhook/templates/service.yaml @@ -5,7 +5,7 @@ metadata: labels: app: lineage-controller-webhook spec: - internalTrafficPolicy: Local + trafficDistribution: PreferClose type: ClusterIP ports: - port: 443 diff --git a/packages/system/lineage-controller-webhook/templates/daemonset.yaml b/packages/system/lineage-controller-webhook/templates/workload.yaml similarity index 67% rename from packages/system/lineage-controller-webhook/templates/daemonset.yaml rename to packages/system/lineage-controller-webhook/templates/workload.yaml index 860aee6e..097919c6 100644 --- a/packages/system/lineage-controller-webhook/templates/daemonset.yaml +++ b/packages/system/lineage-controller-webhook/templates/workload.yaml @@ -1,10 +1,11 @@ apiVersion: apps/v1 -kind: DaemonSet +kind: Deployment metadata: name: lineage-controller-webhook labels: app: lineage-controller-webhook spec: + replicas: {{ .Values.lineageControllerWebhook.replicas }} selector: matchLabels: app: lineage-controller-webhook @@ -13,13 +14,25 @@ spec: labels: app: lineage-controller-webhook spec: - {{- with .Values.lineageControllerWebhook.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} + serviceAccountName: lineage-controller-webhook tolerations: - operator: Exists - serviceAccountName: lineage-controller-webhook + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app: lineage-controller-webhook containers: - name: lineage-controller-webhook image: "{{ .Values.lineageControllerWebhook.image }}" diff --git a/packages/system/lineage-controller-webhook/tests/service_test.yaml b/packages/system/lineage-controller-webhook/tests/service_test.yaml new file mode 100644 index 00000000..3cdbddd8 --- /dev/null +++ b/packages/system/lineage-controller-webhook/tests/service_test.yaml @@ -0,0 +1,30 @@ +suite: lineage-controller-webhook service +templates: + - templates/service.yaml + +release: + name: lineage-controller-webhook + namespace: cozy-system + +tests: + - it: renders a ClusterIP service on 443 -> 9443 with PreferClose traffic distribution + asserts: + - isKind: + of: Service + - equal: + path: spec.type + value: ClusterIP + - equal: + path: spec.trafficDistribution + value: PreferClose + - notExists: + path: spec.internalTrafficPolicy + - equal: + path: spec.ports[0].port + value: 443 + - equal: + path: spec.ports[0].targetPort + value: 9443 + - equal: + path: spec.selector.app + value: lineage-controller-webhook diff --git a/packages/system/lineage-controller-webhook/tests/workload_test.yaml b/packages/system/lineage-controller-webhook/tests/workload_test.yaml new file mode 100644 index 00000000..bcb24014 --- /dev/null +++ b/packages/system/lineage-controller-webhook/tests/workload_test.yaml @@ -0,0 +1,99 @@ +suite: lineage-controller-webhook workload + PDB +templates: + - templates/workload.yaml + - templates/pdb.yaml + +release: + name: lineage-controller-webhook + namespace: cozy-system + +tests: + # ---- Default Deployment shape ---------------------------------------------- + + - it: defaults render a 2-replica Deployment with soft control-plane nodeAffinity, soft podAntiAffinity, and Exists tolerations + template: templates/workload.yaml + asserts: + - isKind: + of: Deployment + - equal: + path: spec.replicas + value: 2 + - equal: + path: spec.template.spec.tolerations + value: + - operator: Exists + - equal: + path: spec.template.spec.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution + value: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - notExists: + path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution + - equal: + path: spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.topologyKey + value: kubernetes.io/hostname + + - it: defaults inject no localK8sAPIEndpoint env vars (knob is opt-in) + template: templates/workload.yaml + asserts: + - notExists: + path: spec.template.spec.containers[0].env + + # ---- PDB -------------------------------------------------------------------- + + - it: PDB renders unconditionally with maxUnavailable=1 at the default replica count + template: templates/pdb.yaml + asserts: + - isKind: + of: PodDisruptionBudget + - equal: + path: spec.maxUnavailable + value: 1 + - notExists: + path: spec.minAvailable + + - it: PDB still renders at replicas=1 (no-op, but consistent shape) + set: + lineageControllerWebhook.replicas: 1 + template: templates/pdb.yaml + asserts: + - isKind: + of: PodDisruptionBudget + - equal: + path: spec.maxUnavailable + value: 1 + + # ---- Replica override ------------------------------------------------------- + + - it: replicas value drives spec.replicas + set: + lineageControllerWebhook.replicas: 5 + template: templates/workload.yaml + asserts: + - equal: + path: spec.replicas + value: 5 + + # ---- Deprecated localK8sAPIEndpoint knob ----------------------------------- + + - it: localK8sAPIEndpoint.enabled=true injects KUBERNETES_SERVICE_HOST and PORT env vars + set: + lineageControllerWebhook.localK8sAPIEndpoint.enabled: true + template: templates/workload.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: KUBERNETES_SERVICE_HOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + - contains: + path: spec.template.spec.containers[0].env + content: + name: KUBERNETES_SERVICE_PORT + value: "6443" diff --git a/packages/system/lineage-controller-webhook/values.yaml b/packages/system/lineage-controller-webhook/values.yaml index c5671548..e56d332d 100644 --- a/packages/system/lineage-controller-webhook/values.yaml +++ b/packages/system/lineage-controller-webhook/values.yaml @@ -1,10 +1,13 @@ lineageControllerWebhook: image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.3.0@sha256:e8984709686a5eaf19b89da378d7b8c688ea5607e0783a88d9c9e4ccfca96fb0 debug: false + replicas: 2 + # DEPRECATED. Injects KUBERNETES_SERVICE_HOST=status.hostIP and + # KUBERNETES_SERVICE_PORT=6443 so the webhook talks to the apiserver on its + # own node — only valid when the pod is actually scheduled on an apiserver- + # bearing node. Now that the chart's nodeAffinity to control-plane is soft + # (preferred, not required), the pod can land off-control-plane and crash- + # loop dialing a non-apiserver hostIP. Slated for removal once webhook + # performance work obviates the locality motivation. Leave disabled. localK8sAPIEndpoint: - enabled: true - # nodeSelector for the DaemonSet - # Talos uses empty value: "node-role.kubernetes.io/control-plane": "" - # Generic k8s (k3s, kubeadm) uses: "node-role.kubernetes.io/control-plane": "true" - nodeSelector: - node-role.kubernetes.io/control-plane: "" + enabled: false