Compare commits
5 commits
main
...
feat/updat
| Author | SHA1 | Date | |
|---|---|---|---|
|
5a61a33b85 | ||
|
|
dbe37e7ed4 | ||
|
|
6a45f0e65d | ||
|
|
8c86693408 | ||
|
|
9ee5f5d2f4 |
93 changed files with 1502 additions and 3948 deletions
|
|
@ -69,9 +69,6 @@ type Addons struct {
|
|||
// NVIDIA GPU Operator.
|
||||
// +kubebuilder:default:={}
|
||||
GpuOperator GPUOperatorAddon `json:"gpuOperator"`
|
||||
// HAMi GPU virtualization middleware.
|
||||
// +kubebuilder:default:={}
|
||||
Hami HAMiAddon `json:"hami"`
|
||||
// Ingress-NGINX controller.
|
||||
// +kubebuilder:default:={}
|
||||
IngressNginx IngressNginxAddon `json:"ingressNginx"`
|
||||
|
|
@ -163,15 +160,6 @@ type GatewayAPIAddon struct {
|
|||
Enabled bool `json:"enabled"`
|
||||
}
|
||||
|
||||
type HAMiAddon struct {
|
||||
// Enable HAMi (requires GPU Operator).
|
||||
// +kubebuilder:default:=false
|
||||
Enabled bool `json:"enabled"`
|
||||
// Custom Helm values overrides.
|
||||
// +kubebuilder:default:={}
|
||||
ValuesOverride k8sRuntime.RawExtension `json:"valuesOverride"`
|
||||
}
|
||||
|
||||
type Images struct {
|
||||
// Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.
|
||||
// +kubebuilder:default:=""
|
||||
|
|
|
|||
|
|
@ -49,7 +49,6 @@ func (in *Addons) DeepCopyInto(out *Addons) {
|
|||
in.Fluxcd.DeepCopyInto(&out.Fluxcd)
|
||||
out.GatewayAPI = in.GatewayAPI
|
||||
in.GpuOperator.DeepCopyInto(&out.GpuOperator)
|
||||
in.Hami.DeepCopyInto(&out.Hami)
|
||||
in.IngressNginx.DeepCopyInto(&out.IngressNginx)
|
||||
in.MonitoringAgents.DeepCopyInto(&out.MonitoringAgents)
|
||||
in.Velero.DeepCopyInto(&out.Velero)
|
||||
|
|
@ -136,7 +135,6 @@ func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) {
|
|||
}
|
||||
in.Addons.DeepCopyInto(&out.Addons)
|
||||
in.ControlPlane.DeepCopyInto(&out.ControlPlane)
|
||||
out.Images = in.Images
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigSpec.
|
||||
|
|
@ -262,37 +260,6 @@ func (in *GatewayAPIAddon) DeepCopy() *GatewayAPIAddon {
|
|||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *HAMiAddon) DeepCopyInto(out *HAMiAddon) {
|
||||
*out = *in
|
||||
in.ValuesOverride.DeepCopyInto(&out.ValuesOverride)
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HAMiAddon.
|
||||
func (in *HAMiAddon) DeepCopy() *HAMiAddon {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(HAMiAddon)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *Images) DeepCopyInto(out *Images) {
|
||||
*out = *in
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Images.
|
||||
func (in *Images) DeepCopy() *Images {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(Images)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *IngressNginxAddon) DeepCopyInto(out *IngressNginxAddon) {
|
||||
*out = *in
|
||||
|
|
|
|||
|
|
@ -1,37 +0,0 @@
|
|||
<!--
|
||||
https://github.com/cozystack/cozystack/releases/tag/v1.3.1
|
||||
-->
|
||||
|
||||
# v1.3.1 (2026-04-28)
|
||||
|
||||
Patch release covering a TenantNamespace IDOR fix in the API, a destructive `post-upgrade` hook removed from the etcd chart, kamaji controller stability, a `linstor-csi` bump that fixes live migration on Protocol-A/B DRBD resources, the missing `linstor-gui` build wiring, and a velero RBAC fix that unblocked installs on bundles without Velero.
|
||||
|
||||
## Security
|
||||
|
||||
* **fix(api): prevent IDOR in TenantNamespace Get and Watch handlers**: Two IDOR (Insecure Direct Object Reference) vulnerabilities allowed authenticated users to read TenantNamespace metadata they had no RoleBinding for. The `Get` and `Watch` handlers now go through a new `hasAccessToNamespace()` helper that lists RoleBindings scoped only to the target namespace (orders of magnitude faster than the previous all-cluster scan), returns `NotFound` instead of leaking existence on unauthorized access, and applies the same check on the `Watch` filter path. Includes regression tests for the unauthorized paths. ([**@IvanHunters**](https://github.com/IvanHunters) in #2471, backport #2524)
|
||||
|
||||
## Features
|
||||
|
||||
* **feat(linstor): bump linstor-csi to v1.10.6 with Protocol-C dual-attach fix**: Live migration of KubeVirt VMs on Protocol-A/B (async) DRBD volumes no longer fails with `Protocol C required`. `linstor-csi` v1.10.6 now installs a `Protocol=C` override on the resource-definition during dual-attach and reverts it on detach, so `replicated-async` StorageClasses and other Protocol-A/B resource groups support live migration without manual `drbdadm adjust` intervention. ([**@kvaps**](https://github.com/kvaps) in #2496, backport #2505)
|
||||
|
||||
## Fixes
|
||||
|
||||
* **fix(backups): move velero-configmap Role to velero chart**: The `backupstrategy-controller` (a default package) declared a Role/RoleBinding scoped to the `cozy-velero` namespace for managing `ResourceModifier` ConfigMaps. On bundles where Velero was not enabled, that namespace did not exist and the HelmRelease failed with `namespaces "cozy-velero" not found`, blocking installation. The Role/RoleBinding now lives in the velero chart, so it is created only when velero is actually deployed. ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2459, backport #2467)
|
||||
|
||||
* **fix(etcd): remove destructive post-upgrade cert-regeneration hook**: The etcd chart ran a `post-upgrade` Helm hook on every upgrade that deleted etcd TLS Secrets (`etcd-ca-tls`, `etcd-peer-ca-tls`, `etcd-client-tls`, `etcd-peer-tls`, `etcd-server-tls`) and then deleted etcd pods, forcing cert-manager to re-issue the entire etcd CA chain. On clusters with Kamaji-managed tenant control planes this put every tenant `kube-apiserver` into CrashLoopBackOff until each DataStore was manually re-reconciled. The hook was a one-shot `2.6.0 → 2.6.1` migration that became a permanent footgun once chart versioning moved to `0.0.0+<git-hash>` (always `< 2.6.1` per semver) and after the underlying `rotationPolicy: Always` issue was fixed in `47d81f70`. The hook is now removed entirely. ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2462, backport #2511)
|
||||
|
||||
* **fix(kamaji): increase memory limits and add startup probe**: The kamaji controller frequently entered CrashLoopBackOff due to OOMKills (exit 137) within ~20–25 seconds of startup, with the readiness probe failing while the controller was still finishing initialization. Memory limit raised from 500Mi to 512Mi, request from 100Mi to 256Mi, and a 60-second startup probe (12 attempts × 5s periods) is added so the controller has room to boot before liveness/readiness probes engage. ([**@IvanHunters**](https://github.com/IvanHunters) in #2421, backport #2491)
|
||||
|
||||
## Build
|
||||
|
||||
* **build(linstor): include linstor-gui in root image build target**: The `linstor-gui` package (added in #2382) was never wired into the root `Makefile`'s `build:` target, so CI never built or published the image. `ghcr.io/cozystack/cozystack/linstor-gui` returned `NAME_UNKNOWN` and `values.yaml` stayed pinned to `tag: 2.3.0` without a digest. The missing build line is added so the next CI run publishes the image and the per-package `Makefile` digest-pins `values.yaml` automatically. ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2498, backport #2518)
|
||||
|
||||
## Contributors
|
||||
|
||||
Thanks to everyone who contributed to this patch release:
|
||||
|
||||
* [**@IvanHunters**](https://github.com/IvanHunters)
|
||||
* [**@kvaps**](https://github.com/kvaps)
|
||||
* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil)
|
||||
|
||||
**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.3.0...v1.3.1
|
||||
172
docs/gpu-vgpu.md
Normal file
172
docs/gpu-vgpu.md
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
# GPU Operator: vGPU Support
|
||||
|
||||
This document describes how to configure the GPU Operator package with NVIDIA vGPU support for sharing a single physical GPU across multiple virtual machines using mediated devices.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- NVIDIA GPU with vGPU support (e.g., NVIDIA L40S, A100, A30, etc.)
|
||||
- Talos Linux as the host OS
|
||||
- NVIDIA vGPU Software license (NVIDIA AI Enterprise or vGPU subscription)
|
||||
- Access to the NVIDIA Licensing Portal ([ui.licensing.nvidia.com](https://ui.licensing.nvidia.com))
|
||||
|
||||
## Variants
|
||||
|
||||
The GPU Operator package supports two variants:
|
||||
|
||||
- **`default`** — GPU passthrough mode (vfio-pci). The entire GPU is passed through to a single VM.
|
||||
- **`vgpu`** — vGPU mode. A physical GPU is shared between multiple VMs using NVIDIA mediated devices.
|
||||
|
||||
## Building the vGPU Manager Image
|
||||
|
||||
The vGPU Manager driver is proprietary and must be obtained from NVIDIA. The GPU Operator expects a pre-built driver container image — it does not install the driver from a raw `.run` file at runtime.
|
||||
|
||||
1. Log in to the [NVIDIA Licensing Portal](https://ui.licensing.nvidia.com)
|
||||
2. Navigate to **Software Downloads** and download the NVIDIA vGPU Software package for your GPU
|
||||
3. Build the driver container image using NVIDIA's Makefile-based build system:
|
||||
|
||||
```bash
|
||||
# Clone the NVIDIA driver container repository
|
||||
git clone https://gitlab.com/nvidia/container-images/driver.git
|
||||
cd driver
|
||||
|
||||
# Place the downloaded .run file in the appropriate directory
|
||||
cp NVIDIA-Linux-x86_64-550.90.05-vgpu-kvm.run vgpu/
|
||||
|
||||
# Build using the provided Makefile
|
||||
make OS_TAG=ubuntu22.04 \
|
||||
VGPU_DRIVER_VERSION=550.90.05 \
|
||||
PRIVATE_REGISTRY=registry.example.com/nvidia
|
||||
|
||||
# Push to your private registry
|
||||
docker push registry.example.com/nvidia/vgpu-manager:550.90.05
|
||||
```
|
||||
|
||||
> **Important:** The build process compiles kernel modules against the host kernel version. Refer to the [NVIDIA GPU Operator vGPU documentation](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/install-gpu-operator-vgpu.html) for the complete build procedure and supported OS/kernel combinations.
|
||||
>
|
||||
> Uploading the vGPU driver to a publicly available registry is a violation of the NVIDIA vGPU EULA.
|
||||
|
||||
## Deploying with vGPU Variant
|
||||
|
||||
Create a Package CR with the `vgpu` variant and provide your vGPU Manager image coordinates:
|
||||
|
||||
```yaml
|
||||
apiVersion: cozystack.io/v1alpha1
|
||||
kind: Package
|
||||
metadata:
|
||||
name: cozystack.gpu-operator
|
||||
spec:
|
||||
variant: vgpu
|
||||
components:
|
||||
gpu-operator:
|
||||
values:
|
||||
gpu-operator:
|
||||
vgpuManager:
|
||||
repository: registry.example.com/nvidia
|
||||
version: "550.90.05"
|
||||
```
|
||||
|
||||
If your registry requires authentication, create an `imagePullSecret` in the `cozy-gpu-operator` namespace and reference it:
|
||||
|
||||
```yaml
|
||||
gpu-operator:
|
||||
vgpuManager:
|
||||
repository: registry.example.com/nvidia
|
||||
version: "550.90.05"
|
||||
imagePullSecrets:
|
||||
- name: nvidia-registry-secret
|
||||
```
|
||||
|
||||
## NVIDIA License Server (NLS) Configuration
|
||||
|
||||
vGPU requires a license server. Configure NLS by passing the license server address via a Secret:
|
||||
|
||||
1. Create a Secret with the NLS client configuration in the `cozy-gpu-operator` namespace:
|
||||
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: licensing-config
|
||||
namespace: cozy-gpu-operator
|
||||
stringData:
|
||||
gridd.conf: |
|
||||
ServerAddress=nls.example.com
|
||||
ServerPort=443
|
||||
FeatureType=1 # 1 for vGPU (vPC/vWS), 2 for Virtual Compute Server (vCS)
|
||||
# ServerPort depends on your NLS deployment (commonly 443 for DLS or 7070 for legacy NLS)
|
||||
```
|
||||
|
||||
2. Reference the Secret in the Package values:
|
||||
|
||||
```yaml
|
||||
gpu-operator:
|
||||
vgpuManager:
|
||||
repository: registry.example.com/nvidia
|
||||
version: "550.90.05"
|
||||
driver:
|
||||
licensingConfig:
|
||||
secretName: licensing-config
|
||||
```
|
||||
|
||||
## vGPU Profiles
|
||||
|
||||
Each GPU model supports specific vGPU profiles that determine how the GPU is partitioned. To list available profiles for your GPU, consult the [NVIDIA vGPU User Guide](https://docs.nvidia.com/grid/latest/grid-vgpu-user-guide/).
|
||||
|
||||
Example profiles for NVIDIA L40S:
|
||||
|
||||
| Profile | Frame Buffer | Max Instances | Use Case |
|
||||
| --- | --- | --- | --- |
|
||||
| NVIDIA L40S-1Q | 1 GB | 48 | Light 3D/VDI |
|
||||
| NVIDIA L40S-2Q | 2 GB | 24 | Medium 3D/VDI |
|
||||
| NVIDIA L40S-4Q | 4 GB | 12 | Heavy 3D/VDI |
|
||||
| NVIDIA L40S-6Q | 6 GB | 8 | Professional 3D |
|
||||
| NVIDIA L40S-8Q | 8 GB | 6 | AI/ML inference |
|
||||
| NVIDIA L40S-12Q | 12 GB | 4 | AI/ML training |
|
||||
| NVIDIA L40S-24Q | 24 GB | 2 | Large AI workloads |
|
||||
| NVIDIA L40S-48Q | 48 GB | 1 | Full GPU equivalent |
|
||||
|
||||
Custom vGPU device configuration can be provided via a ConfigMap:
|
||||
|
||||
```yaml
|
||||
gpu-operator:
|
||||
vgpuDeviceManager:
|
||||
enabled: true
|
||||
config:
|
||||
name: vgpu-devices-config
|
||||
default: default
|
||||
```
|
||||
|
||||
## KubeVirt Integration
|
||||
|
||||
To use vGPU with KubeVirt VMs, configure `mediatedDeviceTypes` in the KubeVirt CR. This maps vGPU profiles to node selectors:
|
||||
|
||||
```yaml
|
||||
apiVersion: kubevirt.io/v1
|
||||
kind: KubeVirt
|
||||
metadata:
|
||||
name: kubevirt
|
||||
spec:
|
||||
configuration:
|
||||
mediatedDevicesConfiguration:
|
||||
mediatedDeviceTypes:
|
||||
- nvidia-592 # NVIDIA L40S-24Q
|
||||
permittedHostDevices:
|
||||
mediatedDevices:
|
||||
- mdevNameSelector: NVIDIA L40S-24Q
|
||||
resourceName: nvidia.com/NVIDIA_L40S-24Q
|
||||
```
|
||||
|
||||
Then reference the vGPU resource in a VirtualMachine spec:
|
||||
|
||||
```yaml
|
||||
apiVersion: kubevirt.io/v1
|
||||
kind: VirtualMachine
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
domain:
|
||||
devices:
|
||||
gpus:
|
||||
- name: gpu1
|
||||
deviceName: nvidia.com/NVIDIA_L40S-24Q
|
||||
```
|
||||
|
|
@ -128,9 +128,6 @@ See the reference for components utilized in this service:
|
|||
| `addons.gpuOperator` | NVIDIA GPU Operator. | `object` | `{}` |
|
||||
| `addons.gpuOperator.enabled` | Enable GPU Operator. | `bool` | `false` |
|
||||
| `addons.gpuOperator.valuesOverride` | Custom Helm values overrides. | `object` | `{}` |
|
||||
| `addons.hami` | HAMi GPU virtualization middleware. | `object` | `{}` |
|
||||
| `addons.hami.enabled` | Enable HAMi (requires GPU Operator). | `bool` | `false` |
|
||||
| `addons.hami.valuesOverride` | Custom Helm values overrides. | `object` | `{}` |
|
||||
| `addons.fluxcd` | FluxCD GitOps operator. | `object` | `{}` |
|
||||
| `addons.fluxcd.enabled` | Enable FluxCD. | `bool` | `false` |
|
||||
| `addons.fluxcd.valuesOverride` | Custom Helm values overrides. | `object` | `{}` |
|
||||
|
|
|
|||
|
|
@ -1,11 +1,3 @@
|
|||
{{- define "cozystack.defaultGpuOperatorValues" -}}
|
||||
{{- if .Values.addons.hami.enabled }}
|
||||
gpu-operator:
|
||||
devicePlugin:
|
||||
enabled: false
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{- if and .Values.addons.gpuOperator.enabled .Values._namespace.etcd }}
|
||||
apiVersion: helm.toolkit.fluxcd.io/v2
|
||||
kind: HelmRelease
|
||||
|
|
@ -37,12 +29,9 @@ spec:
|
|||
force: true
|
||||
remediation:
|
||||
retries: -1
|
||||
{{- $defaults := fromYaml (include "cozystack.defaultGpuOperatorValues" .) }}
|
||||
{{- $overrides := deepCopy (default (dict) .Values.addons.gpuOperator.valuesOverride) }}
|
||||
{{- $merged := mergeOverwrite (default (dict) $defaults) $overrides }}
|
||||
{{- if $merged }}
|
||||
{{- with .Values.addons.gpuOperator.valuesOverride }}
|
||||
values:
|
||||
{{- toYaml $merged | nindent 4 }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
|
||||
dependsOn:
|
||||
|
|
|
|||
|
|
@ -1,49 +0,0 @@
|
|||
{{- if and .Values.addons.hami.enabled .Values._namespace.etcd }}
|
||||
{{- if not .Values.addons.gpuOperator.enabled }}
|
||||
{{- fail "addons.hami requires addons.gpuOperator to be enabled" }}
|
||||
{{- end }}
|
||||
apiVersion: helm.toolkit.fluxcd.io/v2
|
||||
kind: HelmRelease
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-hami
|
||||
labels:
|
||||
cozystack.io/repository: system
|
||||
cozystack.io/target-cluster-name: {{ .Release.Name }}
|
||||
sharding.fluxcd.io/key: tenants
|
||||
spec:
|
||||
releaseName: hami
|
||||
chartRef:
|
||||
kind: ExternalArtifact
|
||||
name: cozystack-kubernetes-application-kubevirt-kubernetes-hami
|
||||
namespace: cozy-system
|
||||
kubeConfig:
|
||||
secretRef:
|
||||
name: {{ .Release.Name }}-admin-kubeconfig
|
||||
key: super-admin.svc
|
||||
targetNamespace: cozy-hami
|
||||
storageNamespace: cozy-hami
|
||||
interval: 5m
|
||||
timeout: 10m
|
||||
install:
|
||||
createNamespace: true
|
||||
remediation:
|
||||
retries: -1
|
||||
upgrade:
|
||||
force: true
|
||||
remediation:
|
||||
retries: -1
|
||||
{{- with .Values.addons.hami.valuesOverride }}
|
||||
values:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
|
||||
dependsOn:
|
||||
{{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }}
|
||||
- name: {{ .Release.Name }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
{{- end }}
|
||||
- name: {{ .Release.Name }}-cilium
|
||||
namespace: {{ .Release.Namespace }}
|
||||
- name: {{ .Release.Name }}-gpu-operator
|
||||
namespace: {{ .Release.Namespace }}
|
||||
{{- end }}
|
||||
|
|
@ -1,99 +0,0 @@
|
|||
suite: GPU Operator HelmRelease HAMi integration tests
|
||||
templates:
|
||||
- templates/helmreleases/gpu-operator.yaml
|
||||
values:
|
||||
- values-ci.yaml
|
||||
tests:
|
||||
- it: should disable devicePlugin when hami is enabled
|
||||
set:
|
||||
addons:
|
||||
gpuOperator:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
hami:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.values.gpu-operator.devicePlugin.enabled
|
||||
value: false
|
||||
|
||||
- it: should not have values when hami is disabled and no overrides
|
||||
set:
|
||||
addons:
|
||||
gpuOperator:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
hami:
|
||||
enabled: false
|
||||
valuesOverride: {}
|
||||
asserts:
|
||||
- notExists:
|
||||
path: spec.values
|
||||
|
||||
- it: should apply hami defaults when valuesOverride key is omitted
|
||||
set:
|
||||
addons:
|
||||
gpuOperator:
|
||||
enabled: true
|
||||
hami:
|
||||
enabled: true
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.values.gpu-operator.devicePlugin.enabled
|
||||
value: false
|
||||
|
||||
- it: should allow user overrides to merge with hami defaults
|
||||
set:
|
||||
addons:
|
||||
gpuOperator:
|
||||
enabled: true
|
||||
valuesOverride:
|
||||
gpu-operator:
|
||||
driver:
|
||||
enabled: false
|
||||
hami:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.values.gpu-operator.devicePlugin.enabled
|
||||
value: false
|
||||
- equal:
|
||||
path: spec.values.gpu-operator.driver.enabled
|
||||
value: false
|
||||
|
||||
- it: should let user explicitly override devicePlugin.enabled to true with hami enabled
|
||||
set:
|
||||
addons:
|
||||
gpuOperator:
|
||||
enabled: true
|
||||
valuesOverride:
|
||||
gpu-operator:
|
||||
devicePlugin:
|
||||
enabled: true
|
||||
driver:
|
||||
enabled: false
|
||||
hami:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.values.gpu-operator.devicePlugin.enabled
|
||||
value: true
|
||||
- equal:
|
||||
path: spec.values.gpu-operator.driver.enabled
|
||||
value: false
|
||||
|
||||
- it: should not render when gpuOperator is disabled
|
||||
set:
|
||||
addons:
|
||||
gpuOperator:
|
||||
enabled: false
|
||||
valuesOverride: {}
|
||||
hami:
|
||||
enabled: false
|
||||
valuesOverride: {}
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
|
|
@ -1,153 +0,0 @@
|
|||
suite: HAMi HelmRelease tests
|
||||
templates:
|
||||
- templates/helmreleases/hami.yaml
|
||||
values:
|
||||
- values-ci.yaml
|
||||
tests:
|
||||
- it: should not render when hami is disabled
|
||||
set:
|
||||
addons:
|
||||
hami:
|
||||
enabled: false
|
||||
valuesOverride: {}
|
||||
gpuOperator:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
|
||||
- it: should render HelmRelease when hami is enabled
|
||||
set:
|
||||
addons:
|
||||
hami:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
gpuOperator:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 1
|
||||
- isKind:
|
||||
of: HelmRelease
|
||||
|
||||
- it: should fail when gpuOperator is not enabled
|
||||
set:
|
||||
addons:
|
||||
hami:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
gpuOperator:
|
||||
enabled: false
|
||||
valuesOverride: {}
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: "addons.hami requires addons.gpuOperator to be enabled"
|
||||
|
||||
- it: should have correct metadata labels
|
||||
set:
|
||||
addons:
|
||||
hami:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
gpuOperator:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
asserts:
|
||||
- equal:
|
||||
path: metadata.labels["cozystack.io/repository"]
|
||||
value: system
|
||||
- equal:
|
||||
path: metadata.labels["sharding.fluxcd.io/key"]
|
||||
value: tenants
|
||||
|
||||
- it: should use ExternalArtifact chartRef
|
||||
set:
|
||||
addons:
|
||||
hami:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
gpuOperator:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.chartRef.kind
|
||||
value: ExternalArtifact
|
||||
- equal:
|
||||
path: spec.chartRef.namespace
|
||||
value: cozy-system
|
||||
|
||||
- it: should target cozy-hami namespace
|
||||
set:
|
||||
addons:
|
||||
hami:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
gpuOperator:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.targetNamespace
|
||||
value: cozy-hami
|
||||
- equal:
|
||||
path: spec.storageNamespace
|
||||
value: cozy-hami
|
||||
|
||||
- it: should depend on gpu-operator and cilium
|
||||
release:
|
||||
name: test
|
||||
namespace: test-ns
|
||||
set:
|
||||
addons:
|
||||
hami:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
gpuOperator:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.dependsOn
|
||||
content:
|
||||
name: test-cilium
|
||||
namespace: test-ns
|
||||
- contains:
|
||||
path: spec.dependsOn
|
||||
content:
|
||||
name: test-gpu-operator
|
||||
namespace: test-ns
|
||||
|
||||
- it: should not render spec.values when valuesOverride is empty
|
||||
set:
|
||||
addons:
|
||||
hami:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
gpuOperator:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 1
|
||||
- notExists:
|
||||
path: spec.values
|
||||
|
||||
- it: should pass through valuesOverride
|
||||
set:
|
||||
addons:
|
||||
hami:
|
||||
enabled: true
|
||||
valuesOverride:
|
||||
hami:
|
||||
devicePlugin:
|
||||
deviceSplitCount: 5
|
||||
gpuOperator:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.values.hami.devicePlugin.deviceSplitCount
|
||||
value: 5
|
||||
|
|
@ -149,7 +149,6 @@
|
|||
"fluxcd",
|
||||
"gatewayAPI",
|
||||
"gpuOperator",
|
||||
"hami",
|
||||
"ingressNginx",
|
||||
"monitoringAgents",
|
||||
"velero",
|
||||
|
|
@ -269,28 +268,6 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"hami": {
|
||||
"description": "HAMi GPU virtualization middleware.",
|
||||
"type": "object",
|
||||
"default": {},
|
||||
"required": [
|
||||
"enabled",
|
||||
"valuesOverride"
|
||||
],
|
||||
"properties": {
|
||||
"enabled": {
|
||||
"description": "Enable HAMi (requires GPU Operator).",
|
||||
"type": "boolean",
|
||||
"default": false
|
||||
},
|
||||
"valuesOverride": {
|
||||
"description": "Custom Helm values overrides.",
|
||||
"type": "object",
|
||||
"default": {},
|
||||
"x-kubernetes-preserve-unknown-fields": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"ingressNginx": {
|
||||
"description": "Ingress-NGINX controller.",
|
||||
"type": "object",
|
||||
|
|
|
|||
|
|
@ -94,10 +94,6 @@ host: ""
|
|||
## @field {bool} enabled - Enable FluxCD.
|
||||
## @field {object} valuesOverride - Custom Helm values overrides.
|
||||
|
||||
## @typedef {struct} HAMiAddon - HAMi GPU virtualization middleware.
|
||||
## @field {bool} enabled - Enable HAMi (requires GPU Operator).
|
||||
## @field {object} valuesOverride - Custom Helm values overrides.
|
||||
|
||||
## @typedef {struct} MonitoringAgentsAddon - Monitoring agents (Fluent Bit, VMAgents).
|
||||
## @field {bool} enabled - Enable monitoring agents.
|
||||
## @field {object} valuesOverride - Custom Helm values overrides.
|
||||
|
|
@ -118,7 +114,6 @@ host: ""
|
|||
## @field {GatewayAPIAddon} gatewayAPI - Gateway API addon.
|
||||
## @field {IngressNginxAddon} ingressNginx - Ingress-NGINX controller.
|
||||
## @field {GPUOperatorAddon} gpuOperator - NVIDIA GPU Operator.
|
||||
## @field {HAMiAddon} hami - HAMi GPU virtualization middleware.
|
||||
## @field {FluxCDAddon} fluxcd - FluxCD GitOps operator.
|
||||
## @field {MonitoringAgentsAddon} monitoringAgents - Monitoring agents.
|
||||
## @field {VerticalPodAutoscalerAddon} verticalPodAutoscaler - Vertical Pod Autoscaler.
|
||||
|
|
@ -142,9 +137,6 @@ addons:
|
|||
gpuOperator:
|
||||
enabled: false
|
||||
valuesOverride: {}
|
||||
hami:
|
||||
enabled: false
|
||||
valuesOverride: {}
|
||||
fluxcd:
|
||||
enabled: false
|
||||
valuesOverride: {}
|
||||
|
|
|
|||
|
|
@ -23,3 +23,16 @@ spec:
|
|||
privileged: true
|
||||
namespace: cozy-gpu-operator
|
||||
releaseName: gpu-operator
|
||||
- name: vgpu
|
||||
dependsOn:
|
||||
- cozystack.networking
|
||||
components:
|
||||
- name: gpu-operator
|
||||
path: system/gpu-operator
|
||||
valuesFiles:
|
||||
- values.yaml
|
||||
- values-talos-vgpu.yaml
|
||||
install:
|
||||
privileged: true
|
||||
namespace: cozy-gpu-operator
|
||||
releaseName: gpu-operator
|
||||
|
|
|
|||
|
|
@ -1,24 +0,0 @@
|
|||
---
|
||||
apiVersion: cozystack.io/v1alpha1
|
||||
kind: PackageSource
|
||||
metadata:
|
||||
name: cozystack.hami
|
||||
spec:
|
||||
sourceRef:
|
||||
kind: OCIRepository
|
||||
name: cozystack-packages
|
||||
namespace: cozy-system
|
||||
path: /
|
||||
variants:
|
||||
- name: default
|
||||
dependsOn:
|
||||
- cozystack.gpu-operator
|
||||
components:
|
||||
- name: hami
|
||||
path: system/hami
|
||||
valuesFiles:
|
||||
- values.yaml
|
||||
install:
|
||||
privileged: true
|
||||
namespace: cozy-hami
|
||||
releaseName: hami
|
||||
|
|
@ -52,8 +52,6 @@ spec:
|
|||
path: system/cilium
|
||||
- name: kubernetes-gpu-operator
|
||||
path: system/gpu-operator
|
||||
- name: kubernetes-hami
|
||||
path: system/hami
|
||||
- name: kubernetes-vertical-pod-autoscaler
|
||||
path: system/vertical-pod-autoscaler
|
||||
- name: kubernetes-prometheus-operator-crds
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@
|
|||
{{include "cozystack.platform.package.default" (list "cozystack.kubevirt-cdi" $) }}
|
||||
{{include "cozystack.platform.package.optional.default" (list "cozystack.vm-default-images" $) }}
|
||||
{{include "cozystack.platform.package.optional.default" (list "cozystack.gpu-operator" $) }}
|
||||
{{include "cozystack.platform.package.optional.default" (list "cozystack.hami" $) }}
|
||||
{{include "cozystack.platform.package.default" (list "cozystack.kamaji" $) }}
|
||||
{{include "cozystack.platform.package.default" (list "cozystack.capi-operator" $) }}
|
||||
{{include "cozystack.platform.package.default" (list "cozystack.capi-provider-bootstrap-kubeadm" $) }}
|
||||
|
|
|
|||
|
|
@ -105,6 +105,9 @@
|
|||
{{- /* cozystack-api DaemonSet */ -}}
|
||||
{{- $apiValues := dict "cozystackAPI" (dict "nodeSelector" $genericNodeSelector) -}}
|
||||
{{- $_ := set $cozystackEngineComponents "cozystack-api" (dict "values" $apiValues) -}}
|
||||
{{- /* lineage-controller-webhook DaemonSet */ -}}
|
||||
{{- $lineageValues := dict "lineageControllerWebhook" (dict "nodeSelector" $genericNodeSelector) -}}
|
||||
{{- $_ := set $cozystackEngineComponents "lineage-controller-webhook" (dict "values" $lineageValues) -}}
|
||||
{{- end -}}
|
||||
{{- if .Values.authentication.oidc.enabled }}
|
||||
{{include "cozystack.platform.package" (list "cozystack.cozystack-engine" "oidc" $ $cozystackEngineComponents) }}
|
||||
|
|
|
|||
|
|
@ -8,4 +8,4 @@ update:
|
|||
rm -rf charts
|
||||
helm repo add nvidia https://helm.ngc.nvidia.com/nvidia
|
||||
helm repo update nvidia
|
||||
helm pull nvidia/gpu-operator --untar --untardir charts
|
||||
helm pull nvidia/gpu-operator --untar --untardir charts --version v26.3.1
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
dependencies:
|
||||
- name: node-feature-discovery
|
||||
repository: https://kubernetes-sigs.github.io/node-feature-discovery/charts
|
||||
version: 0.17.2
|
||||
digest: sha256:4c55d30d958027ef8997a2976449326de3c90049025c3ebb9bee017cad32cc3f
|
||||
generated: "2025-02-25T09:08:49.128088-08:00"
|
||||
repository: oci://registry.k8s.io/nfd/charts
|
||||
version: 0.18.3
|
||||
digest: sha256:1bce3b80da1e6d47a0befbaa6a2f758aeac6d20382e900f0c301f45020afbebc
|
||||
generated: "2026-01-23T15:08:56.617442-08:00"
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
apiVersion: v2
|
||||
appVersion: v25.3.0
|
||||
appVersion: v26.3.1
|
||||
dependencies:
|
||||
- condition: nfd.enabled
|
||||
name: node-feature-discovery
|
||||
repository: https://kubernetes-sigs.github.io/node-feature-discovery/charts
|
||||
version: v0.17.2
|
||||
repository: oci://registry.k8s.io/nfd/charts
|
||||
version: 0.18.3
|
||||
description: NVIDIA GPU Operator creates/configures/manages GPUs atop Kubernetes
|
||||
home: https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/overview.html
|
||||
icon: https://assets.nvidiagrid.net/ngc/logos/GPUoperator.png
|
||||
|
|
@ -20,4 +20,4 @@ kubeVersion: '>= 1.16.0-0'
|
|||
name: gpu-operator
|
||||
sources:
|
||||
- https://github.com/NVIDIA/gpu-operator
|
||||
version: v25.3.0
|
||||
version: v26.3.1
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
apiVersion: v2
|
||||
appVersion: v0.17.2
|
||||
appVersion: v0.18.3
|
||||
description: 'Detects hardware features available on each node in a Kubernetes cluster,
|
||||
and advertises those features using node labels. '
|
||||
home: https://github.com/kubernetes-sigs/node-feature-discovery
|
||||
icon: https://kubernetes-sigs.github.io/node-feature-discovery/v0.18/assets/images/nfd/favicon.svg
|
||||
keywords:
|
||||
- feature-discovery
|
||||
- feature-detection
|
||||
|
|
@ -11,4 +12,4 @@ name: node-feature-discovery
|
|||
sources:
|
||||
- https://github.com/kubernetes-sigs/node-feature-discovery
|
||||
type: application
|
||||
version: 0.17.2
|
||||
version: 0.18.3
|
||||
|
|
|
|||
|
|
@ -6,5 +6,5 @@ labels. NFD provides flexible configuration and extension points for a wide
|
|||
range of vendor and application specific node labeling needs.
|
||||
|
||||
See
|
||||
[NFD documentation](https://kubernetes-sigs.github.io/node-feature-discovery/v0.17/deployment/helm.html)
|
||||
[NFD documentation](https://kubernetes-sigs.github.io/node-feature-discovery/v0.18/deployment/helm.html)
|
||||
for deployment instructions.
|
||||
|
|
|
|||
|
|
@ -204,11 +204,19 @@ spec:
|
|||
- Exists
|
||||
- DoesNotExist
|
||||
- Gt
|
||||
- Ge
|
||||
- Lt
|
||||
- Le
|
||||
- GtLt
|
||||
- GeLe
|
||||
- IsTrue
|
||||
- IsFalse
|
||||
type: string
|
||||
type:
|
||||
description: |-
|
||||
Type defines the value type for specific operators.
|
||||
The currently supported type is 'version' for Gt,Ge,Lt,Le,GtLt,GeLe operators.
|
||||
type: string
|
||||
value:
|
||||
description: |-
|
||||
Value is the list of values that the operand evaluates the input
|
||||
|
|
@ -240,11 +248,19 @@ spec:
|
|||
- Exists
|
||||
- DoesNotExist
|
||||
- Gt
|
||||
- Ge
|
||||
- Lt
|
||||
- Le
|
||||
- GtLt
|
||||
- GeLe
|
||||
- IsTrue
|
||||
- IsFalse
|
||||
type: string
|
||||
type:
|
||||
description: |-
|
||||
Type defines the value type for specific operators.
|
||||
The currently supported type is 'version' for Gt,Ge,Lt,Le,GtLt,GeLe operators.
|
||||
type: string
|
||||
value:
|
||||
description: |-
|
||||
Value is the list of values that the operand evaluates the input
|
||||
|
|
@ -295,11 +311,19 @@ spec:
|
|||
- Exists
|
||||
- DoesNotExist
|
||||
- Gt
|
||||
- Ge
|
||||
- Lt
|
||||
- Le
|
||||
- GtLt
|
||||
- GeLe
|
||||
- IsTrue
|
||||
- IsFalse
|
||||
type: string
|
||||
type:
|
||||
description: |-
|
||||
Type defines the value type for specific operators.
|
||||
The currently supported type is 'version' for Gt,Ge,Lt,Le,GtLt,GeLe operators.
|
||||
type: string
|
||||
value:
|
||||
description: |-
|
||||
Value is the list of values that the operand evaluates the input
|
||||
|
|
@ -331,11 +355,19 @@ spec:
|
|||
- Exists
|
||||
- DoesNotExist
|
||||
- Gt
|
||||
- Ge
|
||||
- Lt
|
||||
- Le
|
||||
- GtLt
|
||||
- GeLe
|
||||
- IsTrue
|
||||
- IsFalse
|
||||
type: string
|
||||
type:
|
||||
description: |-
|
||||
Type defines the value type for specific operators.
|
||||
The currently supported type is 'version' for Gt,Ge,Lt,Le,GtLt,GeLe operators.
|
||||
type: string
|
||||
value:
|
||||
description: |-
|
||||
Value is the list of values that the operand evaluates the input
|
||||
|
|
@ -356,6 +388,19 @@ spec:
|
|||
name:
|
||||
description: Name of the rule.
|
||||
type: string
|
||||
vars:
|
||||
additionalProperties:
|
||||
type: string
|
||||
description: |-
|
||||
Vars is the variables to store if the rule matches. Variables can be
|
||||
referenced from other rules enabling more complex rule hierarchies.
|
||||
type: object
|
||||
varsTemplate:
|
||||
description: |-
|
||||
VarsTemplate specifies a template to expand for dynamically generating
|
||||
multiple variables. Data (after template expansion) must be keys with an
|
||||
optional value (<key>[=<value>]) separated by newlines.
|
||||
type: string
|
||||
required:
|
||||
- name
|
||||
type: object
|
||||
|
|
@ -498,11 +543,19 @@ spec:
|
|||
- Exists
|
||||
- DoesNotExist
|
||||
- Gt
|
||||
- Ge
|
||||
- Lt
|
||||
- Le
|
||||
- GtLt
|
||||
- GeLe
|
||||
- IsTrue
|
||||
- IsFalse
|
||||
type: string
|
||||
type:
|
||||
description: |-
|
||||
Type defines the value type for specific operators.
|
||||
The currently supported type is 'version' for Gt,Ge,Lt,Le,GtLt,GeLe operators.
|
||||
type: string
|
||||
value:
|
||||
description: |-
|
||||
Value is the list of values that the operand evaluates the input
|
||||
|
|
@ -534,11 +587,19 @@ spec:
|
|||
- Exists
|
||||
- DoesNotExist
|
||||
- Gt
|
||||
- Ge
|
||||
- Lt
|
||||
- Le
|
||||
- GtLt
|
||||
- GeLe
|
||||
- IsTrue
|
||||
- IsFalse
|
||||
type: string
|
||||
type:
|
||||
description: |-
|
||||
Type defines the value type for specific operators.
|
||||
The currently supported type is 'version' for Gt,Ge,Lt,Le,GtLt,GeLe operators.
|
||||
type: string
|
||||
value:
|
||||
description: |-
|
||||
Value is the list of values that the operand evaluates the input
|
||||
|
|
@ -589,11 +650,19 @@ spec:
|
|||
- Exists
|
||||
- DoesNotExist
|
||||
- Gt
|
||||
- Ge
|
||||
- Lt
|
||||
- Le
|
||||
- GtLt
|
||||
- GeLe
|
||||
- IsTrue
|
||||
- IsFalse
|
||||
type: string
|
||||
type:
|
||||
description: |-
|
||||
Type defines the value type for specific operators.
|
||||
The currently supported type is 'version' for Gt,Ge,Lt,Le,GtLt,GeLe operators.
|
||||
type: string
|
||||
value:
|
||||
description: |-
|
||||
Value is the list of values that the operand evaluates the input
|
||||
|
|
@ -625,11 +694,19 @@ spec:
|
|||
- Exists
|
||||
- DoesNotExist
|
||||
- Gt
|
||||
- Ge
|
||||
- Lt
|
||||
- Le
|
||||
- GtLt
|
||||
- GeLe
|
||||
- IsTrue
|
||||
- IsFalse
|
||||
type: string
|
||||
type:
|
||||
description: |-
|
||||
Type defines the value type for specific operators.
|
||||
The currently supported type is 'version' for Gt,Ge,Lt,Le,GtLt,GeLe operators.
|
||||
type: string
|
||||
value:
|
||||
description: |-
|
||||
Value is the list of values that the operand evaluates the input
|
||||
|
|
|
|||
|
|
@ -105,3 +105,22 @@ Create the name of the service account which nfd-gc will use
|
|||
{{ default "default" .Values.gc.serviceAccount.name }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
imagePullSecrets helper - uses local values or falls back to global values
|
||||
*/}}
|
||||
{{- define "node-feature-discovery.imagePullSecrets" -}}
|
||||
{{- $imagePullSecrets := list -}}
|
||||
{{- if .Values.imagePullSecrets -}}
|
||||
{{- range .Values.imagePullSecrets -}}
|
||||
{{- $imagePullSecrets = append $imagePullSecrets . -}}
|
||||
{{- end -}}
|
||||
{{- else if and .Values.global .Values.global.imagePullSecrets -}}
|
||||
{{- range .Values.global.imagePullSecrets -}}
|
||||
{{- $imagePullSecrets = append $imagePullSecrets . -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- if $imagePullSecrets -}}
|
||||
{{- $imagePullSecrets | toJson }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,17 @@
|
|||
{{- if .Values.master.enable }}
|
||||
{{- if .Values.master.podDisruptionBudget.enable -}}
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: {{ include "node-feature-discovery.fullname" . }}-master
|
||||
namespace: {{ include "node-feature-discovery.namespace" . }}
|
||||
labels:
|
||||
{{- include "node-feature-discovery.labels" . | nindent 4 }}
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "node-feature-discovery.selectorLabels" . | nindent 6 }}
|
||||
role: master
|
||||
{{- toYaml (omit .Values.master.podDisruptionBudget "enable") | nindent 2 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -29,13 +29,11 @@ spec:
|
|||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
dnsPolicy: {{ .Values.master.dnsPolicy }}
|
||||
{{- with .Values.priorityClassName }}
|
||||
priorityClassName: {{ . }}
|
||||
{{- end }}
|
||||
{{- with .Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
imagePullSecrets: {{ include "node-feature-discovery.imagePullSecrets" . }}
|
||||
serviceAccountName: {{ include "node-feature-discovery.master.serviceAccountName" . }}
|
||||
enableServiceLinks: false
|
||||
securityContext:
|
||||
|
|
@ -48,8 +46,9 @@ spec:
|
|||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
startupProbe:
|
||||
grpc:
|
||||
port: {{ .Values.master.healthPort | default "8082" }}
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: http
|
||||
{{- with .Values.master.startupProbe.initialDelaySeconds }}
|
||||
initialDelaySeconds: {{ . }}
|
||||
{{- end }}
|
||||
|
|
@ -63,8 +62,9 @@ spec:
|
|||
timeoutSeconds: {{ . }}
|
||||
{{- end }}
|
||||
livenessProbe:
|
||||
grpc:
|
||||
port: {{ .Values.master.healthPort | default "8082" }}
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: http
|
||||
{{- with .Values.master.livenessProbe.initialDelaySeconds }}
|
||||
initialDelaySeconds: {{ . }}
|
||||
{{- end }}
|
||||
|
|
@ -78,8 +78,9 @@ spec:
|
|||
timeoutSeconds: {{ . }}
|
||||
{{- end }}
|
||||
readinessProbe:
|
||||
grpc:
|
||||
port: {{ .Values.master.healthPort | default "8082" }}
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: http
|
||||
{{- with .Values.master.readinessProbe.initialDelaySeconds }}
|
||||
initialDelaySeconds: {{ . }}
|
||||
{{- end }}
|
||||
|
|
@ -96,17 +97,15 @@ spec:
|
|||
successThreshold: {{ . }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- containerPort: {{ .Values.master.metricsPort | default "8081" }}
|
||||
name: metrics
|
||||
- containerPort: {{ .Values.master.healthPort | default "8082" }}
|
||||
name: health
|
||||
- containerPort: {{ .Values.master.port | default "8080" }}
|
||||
name: http
|
||||
env:
|
||||
- name: NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
{{- with .Values.master.extraEnvs }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- toYaml . | nindent 10 }}
|
||||
{{- end}}
|
||||
command:
|
||||
- "nfd-master"
|
||||
|
|
@ -126,9 +125,6 @@ spec:
|
|||
{{- if .Values.master.enableTaints }}
|
||||
- "-enable-taints"
|
||||
{{- end }}
|
||||
{{- if .Values.master.featureRulesController | kindIs "invalid" | not }}
|
||||
- "-featurerules-controller={{ .Values.master.featureRulesController }}"
|
||||
{{- end }}
|
||||
{{- if .Values.master.resyncPeriod }}
|
||||
- "-resync-period={{ .Values.master.resyncPeriod }}"
|
||||
{{- end }}
|
||||
|
|
@ -139,8 +135,7 @@ spec:
|
|||
{{- range $key, $value := .Values.featureGates }}
|
||||
- "-feature-gates={{ $key }}={{ $value }}"
|
||||
{{- end }}
|
||||
- "-metrics={{ .Values.master.metricsPort | default "8081" }}"
|
||||
- "-grpc-health={{ .Values.master.healthPort | default "8082" }}"
|
||||
- "-port={{ .Values.master.port | default "8080" }}"
|
||||
{{- with .Values.master.extraArgs }}
|
||||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,17 @@
|
|||
{{- if .Values.gc.enable }}
|
||||
{{- if .Values.gc.podDisruptionBudget.enable -}}
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: {{ include "node-feature-discovery.fullname" . }}-gc
|
||||
namespace: {{ include "node-feature-discovery.namespace" . }}
|
||||
labels:
|
||||
{{- include "node-feature-discovery.labels" . | nindent 4 }}
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "node-feature-discovery.selectorLabels" . | nindent 6 }}
|
||||
role: gc
|
||||
{{- toYaml (omit .Values.gc.podDisruptionBudget "enable") | nindent 2 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -29,14 +29,11 @@ spec:
|
|||
{{- end }}
|
||||
spec:
|
||||
serviceAccountName: {{ include "node-feature-discovery.gc.serviceAccountName" . }}
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
dnsPolicy: {{ .Values.gc.dnsPolicy }}
|
||||
{{- with .Values.priorityClassName }}
|
||||
priorityClassName: {{ . }}
|
||||
{{- end }}
|
||||
{{- with .Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
imagePullSecrets: {{ include "node-feature-discovery.imagePullSecrets" . }}
|
||||
securityContext:
|
||||
{{- toYaml .Values.gc.podSecurityContext | nindent 8 }}
|
||||
hostNetwork: {{ .Values.gc.hostNetwork }}
|
||||
|
|
@ -44,6 +41,41 @@ spec:
|
|||
- name: gc
|
||||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
|
||||
imagePullPolicy: "{{ .Values.image.pullPolicy }}"
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: http
|
||||
{{- with .Values.gc.livenessProbe.initialDelaySeconds }}
|
||||
initialDelaySeconds: {{ . }}
|
||||
{{- end }}
|
||||
{{- with .Values.gc.livenessProbe.failureThreshold }}
|
||||
failureThreshold: {{ . }}
|
||||
{{- end }}
|
||||
{{- with .Values.gc.livenessProbe.periodSeconds }}
|
||||
periodSeconds: {{ . }}
|
||||
{{- end }}
|
||||
{{- with .Values.gc.livenessProbe.timeoutSeconds }}
|
||||
timeoutSeconds: {{ . }}
|
||||
{{- end }}
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: http
|
||||
{{- with .Values.gc.readinessProbe.initialDelaySeconds }}
|
||||
initialDelaySeconds: {{ . }}
|
||||
{{- end }}
|
||||
{{- with .Values.gc.readinessProbe.failureThreshold }}
|
||||
failureThreshold: {{ . }}
|
||||
{{- end }}
|
||||
{{- with .Values.gc.readinessProbe.periodSeconds }}
|
||||
periodSeconds: {{ . }}
|
||||
{{- end }}
|
||||
{{- with .Values.gc.readinessProbe.timeoutSeconds }}
|
||||
timeoutSeconds: {{ . }}
|
||||
{{- end }}
|
||||
{{- with .Values.gc.readinessProbe.successThreshold }}
|
||||
successThreshold: {{ . }}
|
||||
{{- end }}
|
||||
env:
|
||||
- name: NODE_NAME
|
||||
valueFrom:
|
||||
|
|
@ -70,8 +102,8 @@ spec:
|
|||
readOnlyRootFilesystem: true
|
||||
runAsNonRoot: true
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: {{ .Values.gc.metricsPort | default "8081"}}
|
||||
- name: http
|
||||
containerPort: {{ .Values.gc.port | default "8080"}}
|
||||
|
||||
{{- with .Values.gc.nodeSelector }}
|
||||
nodeSelector:
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
{{- if .Values.postDeleteCleanup }}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
|
|
@ -66,6 +67,7 @@ spec:
|
|||
role: prune
|
||||
spec:
|
||||
serviceAccountName: {{ include "node-feature-discovery.fullname" . }}-prune
|
||||
imagePullSecrets: {{ include "node-feature-discovery.imagePullSecrets" . }}
|
||||
containers:
|
||||
- name: nfd-master
|
||||
securityContext:
|
||||
|
|
@ -92,3 +94,9 @@ spec:
|
|||
tolerations:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.master.resources }}
|
||||
resources:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ spec:
|
|||
- honorLabels: true
|
||||
interval: {{ .Values.prometheus.scrapeInterval }}
|
||||
path: /metrics
|
||||
port: metrics
|
||||
port: http
|
||||
scheme: http
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
|
|
|
|||
|
|
@ -29,14 +29,11 @@ spec:
|
|||
{{- end }}
|
||||
spec:
|
||||
serviceAccountName: {{ include "node-feature-discovery.topologyUpdater.serviceAccountName" . }}
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
dnsPolicy: {{ .Values.topologyUpdater.dnsPolicy }}
|
||||
{{- with .Values.priorityClassName }}
|
||||
priorityClassName: {{ . }}
|
||||
{{- end }}
|
||||
{{- with .Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
imagePullSecrets: {{ include "node-feature-discovery.imagePullSecrets" . }}
|
||||
securityContext:
|
||||
{{- toYaml .Values.topologyUpdater.podSecurityContext | nindent 8 }}
|
||||
hostNetwork: {{ .Values.topologyUpdater.hostNetwork }}
|
||||
|
|
@ -45,8 +42,9 @@ spec:
|
|||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
|
||||
imagePullPolicy: "{{ .Values.image.pullPolicy }}"
|
||||
livenessProbe:
|
||||
grpc:
|
||||
port: {{ .Values.topologyUpdater.healthPort | default "8082" }}
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: http
|
||||
{{- with .Values.topologyUpdater.livenessProbe.initialDelaySeconds }}
|
||||
initialDelaySeconds: {{ . }}
|
||||
{{- end }}
|
||||
|
|
@ -60,8 +58,9 @@ spec:
|
|||
timeoutSeconds: {{ . }}
|
||||
{{- end }}
|
||||
readinessProbe:
|
||||
grpc:
|
||||
port: {{ .Values.topologyUpdater.healthPort | default "8082" }}
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: http
|
||||
{{- with .Values.topologyUpdater.readinessProbe.initialDelaySeconds }}
|
||||
initialDelaySeconds: {{ . }}
|
||||
{{- end }}
|
||||
|
|
@ -113,16 +112,13 @@ spec:
|
|||
# Disable kubelet state tracking by giving an empty path
|
||||
- "-kubelet-state-dir="
|
||||
{{- end }}
|
||||
- "-metrics={{ .Values.topologyUpdater.metricsPort | default "8081"}}"
|
||||
- "-grpc-health={{ .Values.topologyUpdater.healthPort | default "8082" }}"
|
||||
- "-port={{ .Values.topologyUpdater.port | default "8080"}}"
|
||||
{{- with .Values.topologyUpdater.extraArgs }}
|
||||
{{- toYaml . | nindent 10 }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- containerPort: {{ .Values.topologyUpdater.metricsPort | default "8081"}}
|
||||
name: metrics
|
||||
- containerPort: {{ .Values.topologyUpdater.healthPort | default "8082" }}
|
||||
name: health
|
||||
- containerPort: {{ .Values.topologyUpdater.port | default "8080"}}
|
||||
name: http
|
||||
volumeMounts:
|
||||
{{- if .Values.topologyUpdater.kubeletConfigPath | empty | not }}
|
||||
- name: kubelet-config
|
||||
|
|
|
|||
|
|
@ -13,6 +13,10 @@ metadata:
|
|||
{{- end }}
|
||||
spec:
|
||||
revisionHistoryLimit: {{ .Values.worker.revisionHistoryLimit }}
|
||||
{{- with .Values.worker.updateStrategy }}
|
||||
updateStrategy:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end}}
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "node-feature-discovery.selectorLabels" . | nindent 6 }}
|
||||
|
|
@ -28,14 +32,11 @@ spec:
|
|||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
dnsPolicy: {{ .Values.worker.dnsPolicy }}
|
||||
{{- with .Values.priorityClassName }}
|
||||
priorityClassName: {{ . }}
|
||||
{{- end }}
|
||||
{{- with .Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
imagePullSecrets: {{ include "node-feature-discovery.imagePullSecrets" . }}
|
||||
serviceAccountName: {{ include "node-feature-discovery.worker.serviceAccountName" . }}
|
||||
securityContext:
|
||||
{{- toYaml .Values.worker.podSecurityContext | nindent 8 }}
|
||||
|
|
@ -47,8 +48,9 @@ spec:
|
|||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
livenessProbe:
|
||||
grpc:
|
||||
port: {{ .Values.worker.healthPort | default "8082" }}
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: http
|
||||
{{- with .Values.worker.livenessProbe.initialDelaySeconds }}
|
||||
initialDelaySeconds: {{ . }}
|
||||
{{- end }}
|
||||
|
|
@ -62,8 +64,9 @@ spec:
|
|||
timeoutSeconds: {{ . }}
|
||||
{{- end }}
|
||||
readinessProbe:
|
||||
grpc:
|
||||
port: {{ .Values.worker.healthPort | default "8082" }}
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: http
|
||||
{{- with .Values.worker.readinessProbe.initialDelaySeconds }}
|
||||
initialDelaySeconds: {{ . }}
|
||||
{{- end }}
|
||||
|
|
@ -104,16 +107,13 @@ spec:
|
|||
{{- range $key, $value := .Values.featureGates }}
|
||||
- "-feature-gates={{ $key }}={{ $value }}"
|
||||
{{- end }}
|
||||
- "-metrics={{ .Values.worker.metricsPort | default "8081"}}"
|
||||
- "-grpc-health={{ .Values.worker.healthPort | default "8082" }}"
|
||||
- "-port={{ .Values.worker.port | default "8080"}}"
|
||||
{{- with .Values.worker.extraArgs }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- containerPort: {{ .Values.worker.metricsPort | default "8081"}}
|
||||
name: metrics
|
||||
- containerPort: {{ .Values.worker.healthPort | default "8082" }}
|
||||
name: health
|
||||
- containerPort: {{ .Values.worker.port | default "8080"}}
|
||||
name: http
|
||||
volumeMounts:
|
||||
- name: host-boot
|
||||
mountPath: "/host-boot"
|
||||
|
|
|
|||
|
|
@ -15,17 +15,20 @@ featureGates:
|
|||
|
||||
priorityClassName: ""
|
||||
|
||||
postDeleteCleanup: true
|
||||
|
||||
master:
|
||||
enable: true
|
||||
extraArgs: []
|
||||
extraEnvs: []
|
||||
hostNetwork: false
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
config: ### <NFD-MASTER-CONF-START-DO-NOT-REMOVE>
|
||||
# noPublish: false
|
||||
# autoDefaultNs: true
|
||||
# extraLabelNs: ["added.ns.io","added.kubernets.io"]
|
||||
# denyLabelNs: ["denied.ns.io","denied.kubernetes.io"]
|
||||
# enableTaints: false
|
||||
# informerPageSize: 200
|
||||
# labelWhiteList: "foo"
|
||||
# resyncPeriod: "2h"
|
||||
# restrictions:
|
||||
|
|
@ -66,15 +69,12 @@ master:
|
|||
# retryPeriod: 2s
|
||||
# nfdApiParallelism: 10
|
||||
### <NFD-MASTER-CONF-END-DO-NOT-REMOVE>
|
||||
metricsPort: 8081
|
||||
healthPort: 8082
|
||||
port: 8080
|
||||
instance:
|
||||
featureApi:
|
||||
resyncPeriod:
|
||||
denyLabelNs: []
|
||||
extraLabelNs: []
|
||||
enableTaints: false
|
||||
featureRulesController: null
|
||||
nfdApiParallelism: null
|
||||
deploymentAnnotations: {}
|
||||
replicaCount: 1
|
||||
|
|
@ -120,26 +120,21 @@ master:
|
|||
nodeSelector: {}
|
||||
|
||||
tolerations:
|
||||
- key: "node-role.kubernetes.io/master"
|
||||
operator: "Equal"
|
||||
value: ""
|
||||
effect: "NoSchedule"
|
||||
- key: "node-role.kubernetes.io/control-plane"
|
||||
operator: "Equal"
|
||||
value: ""
|
||||
effect: "NoSchedule"
|
||||
|
||||
podDisruptionBudget:
|
||||
enable: false
|
||||
minAvailable: 1
|
||||
unhealthyPodEvictionPolicy: AlwaysAllow
|
||||
|
||||
annotations: {}
|
||||
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- weight: 1
|
||||
preference:
|
||||
matchExpressions:
|
||||
- key: "node-role.kubernetes.io/master"
|
||||
operator: In
|
||||
values: [""]
|
||||
- weight: 1
|
||||
preference:
|
||||
matchExpressions:
|
||||
|
|
@ -148,20 +143,14 @@ master:
|
|||
values: [""]
|
||||
|
||||
startupProbe:
|
||||
grpc:
|
||||
port: 8082
|
||||
failureThreshold: 30
|
||||
# periodSeconds: 10
|
||||
livenessProbe:
|
||||
grpc:
|
||||
port: 8082
|
||||
livenessProbe: {}
|
||||
# failureThreshold: 3
|
||||
# initialDelaySeconds: 0
|
||||
# periodSeconds: 10
|
||||
# timeoutSeconds: 1
|
||||
readinessProbe:
|
||||
grpc:
|
||||
port: 8082
|
||||
failureThreshold: 10
|
||||
# initialDelaySeconds: 0
|
||||
# periodSeconds: 10
|
||||
|
|
@ -173,6 +162,7 @@ worker:
|
|||
extraArgs: []
|
||||
extraEnvs: []
|
||||
hostNetwork: false
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
config: ### <NFD-WORKER-CONF-START-DO-NOT-REMOVE>
|
||||
#core:
|
||||
# labelWhiteList:
|
||||
|
|
@ -416,8 +406,7 @@ worker:
|
|||
# matchName: {op: In, value: ["SWAP", "X86", "ARM"]}
|
||||
### <NFD-WORKER-CONF-END-DO-NOT-REMOVE>
|
||||
|
||||
metricsPort: 8081
|
||||
healthPort: 8082
|
||||
port: 8080
|
||||
daemonsetAnnotations: {}
|
||||
podSecurityContext: {}
|
||||
# fsGroup: 2000
|
||||
|
|
@ -431,15 +420,11 @@ worker:
|
|||
# runAsUser: 1000
|
||||
|
||||
livenessProbe:
|
||||
grpc:
|
||||
port: 8082
|
||||
initialDelaySeconds: 10
|
||||
# failureThreshold: 3
|
||||
# periodSeconds: 10
|
||||
# timeoutSeconds: 1
|
||||
readinessProbe:
|
||||
grpc:
|
||||
port: 8082
|
||||
initialDelaySeconds: 5
|
||||
failureThreshold: 10
|
||||
# periodSeconds: 10
|
||||
|
|
@ -483,6 +468,12 @@ worker:
|
|||
|
||||
priorityClassName: ""
|
||||
|
||||
updateStrategy: {}
|
||||
# type: RollingUpdate
|
||||
# rollingUpdate:
|
||||
# maxSurge: 0
|
||||
# maxUnavailable: "10%"
|
||||
|
||||
topologyUpdater:
|
||||
config: ### <NFD-TOPOLOGY-UPDATER-CONF-START-DO-NOT-REMOVE>
|
||||
## key = node name, value = list of resources to be excluded.
|
||||
|
|
@ -499,6 +490,7 @@ topologyUpdater:
|
|||
extraArgs: []
|
||||
extraEnvs: []
|
||||
hostNetwork: false
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
|
||||
serviceAccount:
|
||||
create: true
|
||||
|
|
@ -511,8 +503,7 @@ topologyUpdater:
|
|||
rbac:
|
||||
create: true
|
||||
|
||||
metricsPort: 8081
|
||||
healthPort: 8082
|
||||
port: 8080
|
||||
kubeletConfigPath:
|
||||
kubeletPodResourcesSockPath:
|
||||
updateInterval: 60s
|
||||
|
|
@ -526,17 +517,13 @@ topologyUpdater:
|
|||
drop: [ "ALL" ]
|
||||
readOnlyRootFilesystem: true
|
||||
runAsUser: 0
|
||||
|
||||
|
||||
livenessProbe:
|
||||
grpc:
|
||||
port: 8082
|
||||
initialDelaySeconds: 10
|
||||
# failureThreshold: 3
|
||||
# periodSeconds: 10
|
||||
# timeoutSeconds: 1
|
||||
readinessProbe:
|
||||
grpc:
|
||||
port: 8082
|
||||
initialDelaySeconds: 5
|
||||
failureThreshold: 10
|
||||
# periodSeconds: 10
|
||||
|
|
@ -563,6 +550,7 @@ gc:
|
|||
extraEnvs: []
|
||||
hostNetwork: false
|
||||
replicaCount: 1
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
|
||||
serviceAccount:
|
||||
create: true
|
||||
|
|
@ -575,6 +563,18 @@ gc:
|
|||
|
||||
podSecurityContext: {}
|
||||
|
||||
livenessProbe:
|
||||
initialDelaySeconds: 10
|
||||
# failureThreshold: 3
|
||||
# periodSeconds: 10
|
||||
# timeoutSeconds: 1
|
||||
readinessProbe:
|
||||
initialDelaySeconds: 5
|
||||
# failureThreshold: 3
|
||||
# periodSeconds: 10
|
||||
# timeoutSeconds: 1
|
||||
# successThreshold: 1
|
||||
|
||||
resources:
|
||||
limits:
|
||||
memory: 1Gi
|
||||
|
|
@ -582,7 +582,7 @@ gc:
|
|||
cpu: 10m
|
||||
memory: 128Mi
|
||||
|
||||
metricsPort: 8081
|
||||
port: 8080
|
||||
|
||||
nodeSelector: {}
|
||||
tolerations: []
|
||||
|
|
@ -590,6 +590,11 @@ gc:
|
|||
deploymentAnnotations: {}
|
||||
affinity: {}
|
||||
|
||||
podDisruptionBudget:
|
||||
enable: false
|
||||
minAvailable: 1
|
||||
unhealthyPodEvictionPolicy: AlwaysAllow
|
||||
|
||||
# specify how many old ReplicaSets for the Deployment to retain.
|
||||
revisionHistoryLimit:
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
|
|||
kind: CustomResourceDefinition
|
||||
metadata:
|
||||
annotations:
|
||||
controller-gen.kubebuilder.io/version: v0.17.2
|
||||
controller-gen.kubebuilder.io/version: v0.20.1
|
||||
name: clusterpolicies.nvidia.com
|
||||
spec:
|
||||
group: nvidia.com
|
||||
|
|
@ -82,6 +82,10 @@ spec:
|
|||
- name
|
||||
type: object
|
||||
type: array
|
||||
hostNetwork:
|
||||
description: HostNetwork indicates whether the CC Manager pod
|
||||
uses the host's network namespace.
|
||||
type: boolean
|
||||
image:
|
||||
description: CC Manager image name
|
||||
pattern: '[a-zA-Z0-9\-]+'
|
||||
|
|
@ -136,13 +140,21 @@ spec:
|
|||
properties:
|
||||
default:
|
||||
default: false
|
||||
description: Default indicates whether to use CDI as the default
|
||||
mechanism for providing GPU access to containers.
|
||||
description: 'Deprecated: This field is no longer used. Setting
|
||||
cdi.enabled=true will configure CDI as the default mechanism
|
||||
for making GPUs accessible to containers.'
|
||||
type: boolean
|
||||
enabled:
|
||||
default: true
|
||||
description: Enabled indicates whether the Container Device Interface
|
||||
(CDI) should be used as the mechanism for making GPUs accessible
|
||||
to containers.
|
||||
type: boolean
|
||||
nriPluginEnabled:
|
||||
default: false
|
||||
description: Enabled indicates whether CDI can be used to make
|
||||
GPUs accessible to containers.
|
||||
description: NRIPluginEnabled indicates whether an NRI Plugin
|
||||
should be run as a means of injecting CDI devices to gpu management
|
||||
containers.
|
||||
type: boolean
|
||||
type: object
|
||||
daemonsets:
|
||||
|
|
@ -164,6 +176,239 @@ spec:
|
|||
(scope and select) objects. May match selectors of replication controllers
|
||||
and services.
|
||||
type: object
|
||||
podSecurityContext:
|
||||
description: 'Optional: Set pod-level security context for all
|
||||
DaemonSet pods (applies as defaults to all containers)'
|
||||
properties:
|
||||
appArmorProfile:
|
||||
description: |-
|
||||
appArmorProfile is the AppArmor options to use by the containers in this pod.
|
||||
Note that this field cannot be set when spec.os.name is windows.
|
||||
properties:
|
||||
localhostProfile:
|
||||
description: |-
|
||||
localhostProfile indicates a profile loaded on the node that should be used.
|
||||
The profile must be preconfigured on the node to work.
|
||||
Must match the loaded name of the profile.
|
||||
Must be set if and only if type is "Localhost".
|
||||
type: string
|
||||
type:
|
||||
description: |-
|
||||
type indicates which kind of AppArmor profile will be applied.
|
||||
Valid options are:
|
||||
Localhost - a profile pre-loaded on the node.
|
||||
RuntimeDefault - the container runtime's default profile.
|
||||
Unconfined - no AppArmor enforcement.
|
||||
type: string
|
||||
required:
|
||||
- type
|
||||
type: object
|
||||
fsGroup:
|
||||
description: |-
|
||||
A special supplemental group that applies to all containers in a pod.
|
||||
Some volume types allow the Kubelet to change the ownership of that volume
|
||||
to be owned by the pod:
|
||||
|
||||
1. The owning GID will be the FSGroup
|
||||
2. The setgid bit is set (new files created in the volume will be owned by FSGroup)
|
||||
3. The permission bits are OR'd with rw-rw----
|
||||
|
||||
If unset, the Kubelet will not modify the ownership and permissions of any volume.
|
||||
Note that this field cannot be set when spec.os.name is windows.
|
||||
format: int64
|
||||
type: integer
|
||||
fsGroupChangePolicy:
|
||||
description: |-
|
||||
fsGroupChangePolicy defines behavior of changing ownership and permission of the volume
|
||||
before being exposed inside Pod. This field will only apply to
|
||||
volume types which support fsGroup based ownership(and permissions).
|
||||
It will have no effect on ephemeral volume types such as: secret, configmaps
|
||||
and emptydir.
|
||||
Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used.
|
||||
Note that this field cannot be set when spec.os.name is windows.
|
||||
type: string
|
||||
runAsGroup:
|
||||
description: |-
|
||||
The GID to run the entrypoint of the container process.
|
||||
Uses runtime default if unset.
|
||||
May also be set in SecurityContext. If set in both SecurityContext and
|
||||
PodSecurityContext, the value specified in SecurityContext takes precedence
|
||||
for that container.
|
||||
Note that this field cannot be set when spec.os.name is windows.
|
||||
format: int64
|
||||
type: integer
|
||||
runAsNonRoot:
|
||||
description: |-
|
||||
Indicates that the container must run as a non-root user.
|
||||
If true, the Kubelet will validate the image at runtime to ensure that it
|
||||
does not run as UID 0 (root) and fail to start the container if it does.
|
||||
If unset or false, no such validation will be performed.
|
||||
May also be set in SecurityContext. If set in both SecurityContext and
|
||||
PodSecurityContext, the value specified in SecurityContext takes precedence.
|
||||
type: boolean
|
||||
runAsUser:
|
||||
description: |-
|
||||
The UID to run the entrypoint of the container process.
|
||||
Defaults to user specified in image metadata if unspecified.
|
||||
May also be set in SecurityContext. If set in both SecurityContext and
|
||||
PodSecurityContext, the value specified in SecurityContext takes precedence
|
||||
for that container.
|
||||
Note that this field cannot be set when spec.os.name is windows.
|
||||
format: int64
|
||||
type: integer
|
||||
seLinuxChangePolicy:
|
||||
description: |-
|
||||
seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod.
|
||||
It has no effect on nodes that do not support SELinux or to volumes does not support SELinux.
|
||||
Valid values are "MountOption" and "Recursive".
|
||||
|
||||
"Recursive" means relabeling of all files on all Pod volumes by the container runtime.
|
||||
This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node.
|
||||
|
||||
"MountOption" mounts all eligible Pod volumes with `-o context` mount option.
|
||||
This requires all Pods that share the same volume to use the same SELinux label.
|
||||
It is not possible to share the same volume among privileged and unprivileged Pods.
|
||||
Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes
|
||||
whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their
|
||||
CSIDriver instance. Other volumes are always re-labelled recursively.
|
||||
"MountOption" value is allowed only when SELinuxMount feature gate is enabled.
|
||||
|
||||
If not specified and SELinuxMount feature gate is enabled, "MountOption" is used.
|
||||
If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes
|
||||
and "Recursive" for all other volumes.
|
||||
|
||||
This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers.
|
||||
|
||||
All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state.
|
||||
Note that this field cannot be set when spec.os.name is windows.
|
||||
type: string
|
||||
seLinuxOptions:
|
||||
description: |-
|
||||
The SELinux context to be applied to all containers.
|
||||
If unspecified, the container runtime will allocate a random SELinux context for each
|
||||
container. May also be set in SecurityContext. If set in
|
||||
both SecurityContext and PodSecurityContext, the value specified in SecurityContext
|
||||
takes precedence for that container.
|
||||
Note that this field cannot be set when spec.os.name is windows.
|
||||
properties:
|
||||
level:
|
||||
description: Level is SELinux level label that applies
|
||||
to the container.
|
||||
type: string
|
||||
role:
|
||||
description: Role is a SELinux role label that applies
|
||||
to the container.
|
||||
type: string
|
||||
type:
|
||||
description: Type is a SELinux type label that applies
|
||||
to the container.
|
||||
type: string
|
||||
user:
|
||||
description: User is a SELinux user label that applies
|
||||
to the container.
|
||||
type: string
|
||||
type: object
|
||||
seccompProfile:
|
||||
description: |-
|
||||
The seccomp options to use by the containers in this pod.
|
||||
Note that this field cannot be set when spec.os.name is windows.
|
||||
properties:
|
||||
localhostProfile:
|
||||
description: |-
|
||||
localhostProfile indicates a profile defined in a file on the node should be used.
|
||||
The profile must be preconfigured on the node to work.
|
||||
Must be a descending path, relative to the kubelet's configured seccomp profile location.
|
||||
Must be set if type is "Localhost". Must NOT be set for any other type.
|
||||
type: string
|
||||
type:
|
||||
description: |-
|
||||
type indicates which kind of seccomp profile will be applied.
|
||||
Valid options are:
|
||||
|
||||
Localhost - a profile defined in a file on the node should be used.
|
||||
RuntimeDefault - the container runtime default profile should be used.
|
||||
Unconfined - no profile should be applied.
|
||||
type: string
|
||||
required:
|
||||
- type
|
||||
type: object
|
||||
supplementalGroups:
|
||||
description: |-
|
||||
A list of groups applied to the first process run in each container, in
|
||||
addition to the container's primary GID and fsGroup (if specified). If
|
||||
the SupplementalGroupsPolicy feature is enabled, the
|
||||
supplementalGroupsPolicy field determines whether these are in addition
|
||||
to or instead of any group memberships defined in the container image.
|
||||
If unspecified, no additional groups are added, though group memberships
|
||||
defined in the container image may still be used, depending on the
|
||||
supplementalGroupsPolicy field.
|
||||
Note that this field cannot be set when spec.os.name is windows.
|
||||
items:
|
||||
format: int64
|
||||
type: integer
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
supplementalGroupsPolicy:
|
||||
description: |-
|
||||
Defines how supplemental groups of the first container processes are calculated.
|
||||
Valid values are "Merge" and "Strict". If not specified, "Merge" is used.
|
||||
(Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled
|
||||
and the container runtime must implement support for this feature.
|
||||
Note that this field cannot be set when spec.os.name is windows.
|
||||
type: string
|
||||
sysctls:
|
||||
description: |-
|
||||
Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported
|
||||
sysctls (by the container runtime) might fail to launch.
|
||||
Note that this field cannot be set when spec.os.name is windows.
|
||||
items:
|
||||
description: Sysctl defines a kernel parameter to be set
|
||||
properties:
|
||||
name:
|
||||
description: Name of a property to set
|
||||
type: string
|
||||
value:
|
||||
description: Value of a property to set
|
||||
type: string
|
||||
required:
|
||||
- name
|
||||
- value
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
windowsOptions:
|
||||
description: |-
|
||||
The Windows specific settings applied to all containers.
|
||||
If unspecified, the options within a container's SecurityContext will be used.
|
||||
If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.
|
||||
Note that this field cannot be set when spec.os.name is linux.
|
||||
properties:
|
||||
gmsaCredentialSpec:
|
||||
description: |-
|
||||
GMSACredentialSpec is where the GMSA admission webhook
|
||||
(https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the
|
||||
GMSA credential spec named by the GMSACredentialSpecName field.
|
||||
type: string
|
||||
gmsaCredentialSpecName:
|
||||
description: GMSACredentialSpecName is the name of the
|
||||
GMSA credential spec to use.
|
||||
type: string
|
||||
hostProcess:
|
||||
description: |-
|
||||
HostProcess determines if a container should be run as a 'Host Process' container.
|
||||
All of a Pod's containers must have the same effective HostProcess value
|
||||
(it is not allowed to have a mix of HostProcess containers and non-HostProcess containers).
|
||||
In addition, if HostProcess is true then HostNetwork must also be set to true.
|
||||
type: boolean
|
||||
runAsUserName:
|
||||
description: |-
|
||||
The UserName in Windows to run the entrypoint of the container process.
|
||||
Defaults to the user specified in image metadata if unspecified.
|
||||
May also be set in PodSecurityContext. If set in both SecurityContext and
|
||||
PodSecurityContext, the value specified in SecurityContext takes precedence.
|
||||
type: string
|
||||
type: object
|
||||
type: object
|
||||
priorityClassName:
|
||||
type: string
|
||||
rollingUpdate:
|
||||
|
|
@ -193,9 +438,10 @@ spec:
|
|||
operator:
|
||||
description: |-
|
||||
Operator represents a key's relationship to the value.
|
||||
Valid operators are Exists and Equal. Defaults to Equal.
|
||||
Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal.
|
||||
Exists is equivalent to wildcard for value, so that a pod can
|
||||
tolerate all taints of a particular category.
|
||||
Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators).
|
||||
type: string
|
||||
tolerationSeconds:
|
||||
description: |-
|
||||
|
|
@ -247,6 +493,10 @@ spec:
|
|||
- name
|
||||
type: object
|
||||
type: array
|
||||
hostNetwork:
|
||||
description: HostNetwork indicates whether the DCGM pod uses the
|
||||
host's network namespace.
|
||||
type: boolean
|
||||
hostPort:
|
||||
description: 'Deprecated: HostPort represents host port that needs
|
||||
to be bound for DCGM engine (Default: 5555)'
|
||||
|
|
@ -337,6 +587,27 @@ spec:
|
|||
- name
|
||||
type: object
|
||||
type: array
|
||||
hostNetwork:
|
||||
description: HostNetwork allows the DCGM-Exporter daemon set to
|
||||
expose metrics port on the host's network namespace.
|
||||
type: boolean
|
||||
hostPID:
|
||||
description: HostPID allows the DCGM-Exporter daemon set to access
|
||||
the host's PID namespace
|
||||
type: boolean
|
||||
hpcJobMapping:
|
||||
description: 'Optional: HPC job mapping configuration for NVIDIA
|
||||
DCGM Exporter'
|
||||
properties:
|
||||
directory:
|
||||
description: |-
|
||||
Directory path where HPC job mapping files are created by the workload manager
|
||||
Defaults to /var/lib/dcgm-exporter/job-mapping if not specified
|
||||
type: string
|
||||
enabled:
|
||||
description: Enable HPC job mapping for DCGM Exporter
|
||||
type: boolean
|
||||
type: object
|
||||
image:
|
||||
description: NVIDIA DCGM Exporter image name
|
||||
pattern: '[a-zA-Z0-9\-]+'
|
||||
|
|
@ -381,6 +652,19 @@ spec:
|
|||
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
|
||||
type: object
|
||||
type: object
|
||||
service:
|
||||
description: 'Optional: Service configuration for NVIDIA DCGM
|
||||
Exporter'
|
||||
properties:
|
||||
internalTrafficPolicy:
|
||||
description: InternalTrafficPolicy describes how nodes distribute
|
||||
service traffic they receive on the ClusterIP.
|
||||
type: string
|
||||
type:
|
||||
description: Type represents the ServiceType which describes
|
||||
ingress methods for a service
|
||||
type: string
|
||||
type: object
|
||||
serviceMonitor:
|
||||
description: 'Optional: ServiceMonitor configuration for NVIDIA
|
||||
DCGM Exporter'
|
||||
|
|
@ -418,7 +702,7 @@ spec:
|
|||
action:
|
||||
default: replace
|
||||
description: |-
|
||||
Action to perform based on the regex matching.
|
||||
action to perform based on the regex matching.
|
||||
|
||||
`Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0.
|
||||
`DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0.
|
||||
|
|
@ -450,41 +734,41 @@ spec:
|
|||
type: string
|
||||
modulus:
|
||||
description: |-
|
||||
Modulus to take of the hash of the source label values.
|
||||
modulus to take of the hash of the source label values.
|
||||
|
||||
Only applicable when the action is `HashMod`.
|
||||
format: int64
|
||||
type: integer
|
||||
regex:
|
||||
description: Regular expression against which the extracted
|
||||
value is matched.
|
||||
description: regex defines the regular expression against
|
||||
which the extracted value is matched.
|
||||
type: string
|
||||
replacement:
|
||||
description: |-
|
||||
Replacement value against which a Replace action is performed if the
|
||||
replacement value against which a Replace action is performed if the
|
||||
regular expression matches.
|
||||
|
||||
Regex capture groups are available.
|
||||
type: string
|
||||
separator:
|
||||
description: Separator is the string between concatenated
|
||||
description: separator defines the string between concatenated
|
||||
SourceLabels.
|
||||
type: string
|
||||
sourceLabels:
|
||||
description: |-
|
||||
The source labels select values from existing labels. Their content is
|
||||
sourceLabels defines the source labels select values from existing labels. Their content is
|
||||
concatenated using the configured Separator and matched against the
|
||||
configured regular expression.
|
||||
items:
|
||||
description: |-
|
||||
LabelName is a valid Prometheus label name which may only contain ASCII
|
||||
letters, numbers, as well as underscores.
|
||||
pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$
|
||||
LabelName is a valid Prometheus label name.
|
||||
For Prometheus 3.x, a label name is valid if it contains UTF-8 characters.
|
||||
For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores.
|
||||
type: string
|
||||
type: array
|
||||
targetLabel:
|
||||
description: |-
|
||||
Label to which the resulting string is written in a replacement.
|
||||
targetLabel defines the label to which the resulting string is written in a replacement.
|
||||
|
||||
It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`,
|
||||
`KeepEqual` and `DropEqual` actions.
|
||||
|
|
@ -539,6 +823,10 @@ spec:
|
|||
- name
|
||||
type: object
|
||||
type: array
|
||||
hostNetwork:
|
||||
description: HostNetwork indicates whether the Device Plugin pod
|
||||
uses the host's network namespace.
|
||||
type: boolean
|
||||
image:
|
||||
description: NVIDIA Device Plugin image name
|
||||
pattern: '[a-zA-Z0-9\-]+'
|
||||
|
|
@ -631,6 +919,10 @@ spec:
|
|||
- name
|
||||
type: object
|
||||
type: array
|
||||
hostNetwork:
|
||||
description: HostNetwork indicates whether the Driver pod uses
|
||||
the host's network namespace.
|
||||
type: boolean
|
||||
image:
|
||||
description: NVIDIA Driver image name
|
||||
pattern: '[a-zA-Z0-9\-]+'
|
||||
|
|
@ -666,11 +958,18 @@ spec:
|
|||
licensing'
|
||||
properties:
|
||||
configMapName:
|
||||
description: 'Deprecated: ConfigMapName has been deprecated
|
||||
in favour of SecretName. Please use secrets to handle the
|
||||
licensing server configuration more securely'
|
||||
type: string
|
||||
nlsEnabled:
|
||||
description: NLSEnabled indicates if NVIDIA Licensing System
|
||||
is used for licensing.
|
||||
type: boolean
|
||||
secretName:
|
||||
description: SecretName indicates the name of the secret containing
|
||||
the licensing token
|
||||
type: string
|
||||
type: object
|
||||
livenessProbe:
|
||||
description: NVIDIA Driver container liveness probe settings
|
||||
|
|
@ -844,6 +1143,10 @@ spec:
|
|||
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
|
||||
type: object
|
||||
type: object
|
||||
secretEnv:
|
||||
description: 'Optional: SecretEnv represents the name of the Kubernetes
|
||||
Secret with secret environment variables for the NVIDIA Driver'
|
||||
type: string
|
||||
startupProbe:
|
||||
description: NVIDIA Driver container startup probe settings
|
||||
properties:
|
||||
|
|
@ -1132,6 +1435,10 @@ spec:
|
|||
- name
|
||||
type: object
|
||||
type: array
|
||||
hostNetwork:
|
||||
description: HostNetwork indicates whether the GPU Feature Discovery
|
||||
pod uses the host's network namespace.
|
||||
type: boolean
|
||||
image:
|
||||
description: GFD image name
|
||||
pattern: '[a-zA-Z0-9\-]+'
|
||||
|
|
@ -1275,6 +1582,10 @@ spec:
|
|||
- name
|
||||
type: object
|
||||
type: array
|
||||
hostNetwork:
|
||||
description: HostNetwork indicates whether the Kata Manager pod
|
||||
uses the host's network namespace.
|
||||
type: boolean
|
||||
image:
|
||||
description: Kata Manager image name
|
||||
pattern: '[a-zA-Z0-9\-]+'
|
||||
|
|
@ -1323,6 +1634,86 @@ spec:
|
|||
description: Kata Manager image tag
|
||||
type: string
|
||||
type: object
|
||||
kataSandboxDevicePlugin:
|
||||
description: KataSandboxDevicePlugin component spec
|
||||
properties:
|
||||
args:
|
||||
description: 'Optional: List of arguments'
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
enabled:
|
||||
description: Enabled indicates if deployment of NVIDIA component
|
||||
through operator is enabled
|
||||
type: boolean
|
||||
env:
|
||||
description: 'Optional: List of environment variables'
|
||||
items:
|
||||
description: EnvVar represents an environment variable present
|
||||
in a Container.
|
||||
properties:
|
||||
name:
|
||||
description: Name of the environment variable.
|
||||
type: string
|
||||
value:
|
||||
description: Value of the environment variable.
|
||||
type: string
|
||||
required:
|
||||
- name
|
||||
type: object
|
||||
type: array
|
||||
hostNetwork:
|
||||
description: HostNetwork indicates whether the Kata Sandbox Device
|
||||
Plugin pod uses the host's network namespace.
|
||||
type: boolean
|
||||
image:
|
||||
description: NVIDIA component image name
|
||||
pattern: '[a-zA-Z0-9\-]+'
|
||||
type: string
|
||||
imagePullPolicy:
|
||||
description: Image pull policy
|
||||
type: string
|
||||
imagePullSecrets:
|
||||
description: Image pull secrets
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
repository:
|
||||
description: NVIDIA component image repository
|
||||
type: string
|
||||
resources:
|
||||
description: 'Optional: Define resources requests and limits for
|
||||
each pod'
|
||||
properties:
|
||||
limits:
|
||||
additionalProperties:
|
||||
anyOf:
|
||||
- type: integer
|
||||
- type: string
|
||||
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
|
||||
x-kubernetes-int-or-string: true
|
||||
description: |-
|
||||
Limits describes the maximum amount of compute resources allowed.
|
||||
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
|
||||
type: object
|
||||
requests:
|
||||
additionalProperties:
|
||||
anyOf:
|
||||
- type: integer
|
||||
- type: string
|
||||
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
|
||||
x-kubernetes-int-or-string: true
|
||||
description: |-
|
||||
Requests describes the minimum amount of compute resources required.
|
||||
If Requests is omitted for a container, it defaults to Limits if that is explicitly specified,
|
||||
otherwise to an implementation-defined value. Requests cannot exceed Limits.
|
||||
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
|
||||
type: object
|
||||
type: object
|
||||
version:
|
||||
description: NVIDIA component image tag
|
||||
type: string
|
||||
type: object
|
||||
mig:
|
||||
description: MIG spec
|
||||
properties:
|
||||
|
|
@ -1356,8 +1747,8 @@ spec:
|
|||
- ""
|
||||
type: string
|
||||
name:
|
||||
default: default-mig-parted-config
|
||||
description: ConfigMap name
|
||||
description: ConfigMap name. If not specified, MIG configuration
|
||||
will be dynamically generated from hardware.
|
||||
type: string
|
||||
type: object
|
||||
enabled:
|
||||
|
|
@ -1388,6 +1779,10 @@ spec:
|
|||
description: ConfigMap name
|
||||
type: string
|
||||
type: object
|
||||
hostNetwork:
|
||||
description: HostNetwork indicates whether the MIG Manager pod
|
||||
uses the host's network namespace.
|
||||
type: boolean
|
||||
image:
|
||||
description: NVIDIA MIG Manager image name
|
||||
pattern: '[a-zA-Z0-9\-]+'
|
||||
|
|
@ -1464,6 +1859,10 @@ spec:
|
|||
- name
|
||||
type: object
|
||||
type: array
|
||||
hostNetwork:
|
||||
description: HostNetwork indicates whether the Node Status Exporter
|
||||
pod uses the host's network namespace.
|
||||
type: boolean
|
||||
image:
|
||||
description: Node Status Exporter image name
|
||||
pattern: '[a-zA-Z0-9\-]+'
|
||||
|
|
@ -1524,16 +1923,12 @@ spec:
|
|||
queryable and should be preserved when modifying objects.
|
||||
type: object
|
||||
defaultRuntime:
|
||||
default: docker
|
||||
description: Runtime defines container runtime type
|
||||
enum:
|
||||
- docker
|
||||
- crio
|
||||
- containerd
|
||||
description: 'Deprecated: DefaultRuntime is no longer used by
|
||||
the gpu-operator. This is instead, detected at runtime.'
|
||||
type: string
|
||||
initContainer:
|
||||
description: InitContainerSpec describes configuration for initContainer
|
||||
image used with all components
|
||||
description: 'Deprecated: InitContainerSpec describes configuration
|
||||
for initContainer image used with all components'
|
||||
properties:
|
||||
image:
|
||||
description: Image represents image name
|
||||
|
|
@ -1570,8 +1965,6 @@ spec:
|
|||
image should be used on OpenShift to build and install driver
|
||||
modules
|
||||
type: boolean
|
||||
required:
|
||||
- defaultRuntime
|
||||
type: object
|
||||
psa:
|
||||
description: PSA defines spec for PodSecurityAdmission configuration
|
||||
|
|
@ -1619,6 +2012,10 @@ spec:
|
|||
- name
|
||||
type: object
|
||||
type: array
|
||||
hostNetwork:
|
||||
description: HostNetwork indicates whether the Sandbox Device
|
||||
Plugin pod uses the host's network namespace.
|
||||
type: boolean
|
||||
image:
|
||||
description: NVIDIA Sandbox Device Plugin image name
|
||||
pattern: '[a-zA-Z0-9\-]+'
|
||||
|
|
@ -1686,6 +2083,15 @@ spec:
|
|||
Enabled indicates if the GPU Operator should manage additional operands required
|
||||
for sandbox workloads (i.e. VFIO Manager, vGPU Manager, and additional device plugins)
|
||||
type: boolean
|
||||
mode:
|
||||
default: kubevirt
|
||||
description: |-
|
||||
Mode indicates the sandbox mode. Accepted values are "kubevirt"
|
||||
and "kata". The default value is "kubevirt".
|
||||
enum:
|
||||
- kubevirt
|
||||
- kata
|
||||
type: string
|
||||
type: object
|
||||
toolkit:
|
||||
description: Toolkit component spec
|
||||
|
|
@ -1715,6 +2121,10 @@ spec:
|
|||
- name
|
||||
type: object
|
||||
type: array
|
||||
hostNetwork:
|
||||
description: HostNetwork indicates whether the Container Toolkit
|
||||
pod uses the host's network namespace.
|
||||
type: boolean
|
||||
image:
|
||||
description: NVIDIA Container Toolkit image name
|
||||
pattern: '[a-zA-Z0-9\-]+'
|
||||
|
|
@ -1831,6 +2241,10 @@ spec:
|
|||
- name
|
||||
type: object
|
||||
type: array
|
||||
hostNetwork:
|
||||
description: HostNetwork indicates whether the Validator pod uses
|
||||
the host's network namespace.
|
||||
type: boolean
|
||||
image:
|
||||
description: Validator image name
|
||||
pattern: '[a-zA-Z0-9\-]+'
|
||||
|
|
@ -2049,6 +2463,10 @@ spec:
|
|||
- name
|
||||
type: object
|
||||
type: array
|
||||
hostNetwork:
|
||||
description: HostNetwork indicates whether the VFIO Manager pod
|
||||
uses the host's network namespace.
|
||||
type: boolean
|
||||
image:
|
||||
description: VFIO Manager image name
|
||||
pattern: '[a-zA-Z0-9\-]+'
|
||||
|
|
@ -2137,6 +2555,10 @@ spec:
|
|||
- name
|
||||
type: object
|
||||
type: array
|
||||
hostNetwork:
|
||||
description: HostNetwork indicates whether the vGPU Device Manager
|
||||
pod uses the host's network namespace.
|
||||
type: boolean
|
||||
image:
|
||||
description: NVIDIA vGPU Device Manager image name
|
||||
pattern: '[a-zA-Z0-9\-]+'
|
||||
|
|
@ -2255,6 +2677,10 @@ spec:
|
|||
- name
|
||||
type: object
|
||||
type: array
|
||||
hostNetwork:
|
||||
description: HostNetwork indicates whether the vGPU Manager pod
|
||||
uses the host's network namespace.
|
||||
type: boolean
|
||||
image:
|
||||
description: NVIDIA vGPU Manager image name
|
||||
pattern: '[a-zA-Z0-9\-]+'
|
||||
|
|
@ -2267,6 +2693,13 @@ spec:
|
|||
items:
|
||||
type: string
|
||||
type: array
|
||||
kernelModuleConfig:
|
||||
description: 'Optional: Kernel module configuration parameters
|
||||
for the vGPU manager'
|
||||
properties:
|
||||
name:
|
||||
type: string
|
||||
type: object
|
||||
repository:
|
||||
description: NVIDIA vGPU Manager image repository
|
||||
type: string
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
|
|||
kind: CustomResourceDefinition
|
||||
metadata:
|
||||
annotations:
|
||||
controller-gen.kubebuilder.io/version: v0.17.2
|
||||
controller-gen.kubebuilder.io/version: v0.20.1
|
||||
name: nvidiadrivers.nvidia.com
|
||||
spec:
|
||||
group: nvidia.com
|
||||
|
|
@ -192,6 +192,10 @@ spec:
|
|||
description: NVIDIA GPUDirect Storage Driver image tag
|
||||
type: string
|
||||
type: object
|
||||
hostNetwork:
|
||||
description: HostNetwork indicates whether the Driver pod uses the
|
||||
host's network namespace.
|
||||
type: boolean
|
||||
image:
|
||||
default: nvcr.io/nvidia/driver
|
||||
description: NVIDIA Driver container image name
|
||||
|
|
@ -234,11 +238,16 @@ spec:
|
|||
description: 'Optional: Licensing configuration for NVIDIA vGPU licensing'
|
||||
properties:
|
||||
name:
|
||||
description: 'Deprecated: ConfigMapName has been deprecated in
|
||||
favour of SecretName. Please use secrets to handle the licensing
|
||||
server configuration more securely'
|
||||
type: string
|
||||
nlsEnabled:
|
||||
description: NLSEnabled indicates if NVIDIA Licensing System is
|
||||
used for licensing.
|
||||
type: boolean
|
||||
secretName:
|
||||
type: string
|
||||
type: object
|
||||
livenessProbe:
|
||||
description: NVIDIA Driver container liveness probe settings
|
||||
|
|
@ -522,6 +531,239 @@ spec:
|
|||
description: NodeSelector specifies a selector for installation of
|
||||
NVIDIA driver
|
||||
type: object
|
||||
podSecurityContext:
|
||||
description: 'Optional: Set pod-level security context for driver
|
||||
pod'
|
||||
properties:
|
||||
appArmorProfile:
|
||||
description: |-
|
||||
appArmorProfile is the AppArmor options to use by the containers in this pod.
|
||||
Note that this field cannot be set when spec.os.name is windows.
|
||||
properties:
|
||||
localhostProfile:
|
||||
description: |-
|
||||
localhostProfile indicates a profile loaded on the node that should be used.
|
||||
The profile must be preconfigured on the node to work.
|
||||
Must match the loaded name of the profile.
|
||||
Must be set if and only if type is "Localhost".
|
||||
type: string
|
||||
type:
|
||||
description: |-
|
||||
type indicates which kind of AppArmor profile will be applied.
|
||||
Valid options are:
|
||||
Localhost - a profile pre-loaded on the node.
|
||||
RuntimeDefault - the container runtime's default profile.
|
||||
Unconfined - no AppArmor enforcement.
|
||||
type: string
|
||||
required:
|
||||
- type
|
||||
type: object
|
||||
fsGroup:
|
||||
description: |-
|
||||
A special supplemental group that applies to all containers in a pod.
|
||||
Some volume types allow the Kubelet to change the ownership of that volume
|
||||
to be owned by the pod:
|
||||
|
||||
1. The owning GID will be the FSGroup
|
||||
2. The setgid bit is set (new files created in the volume will be owned by FSGroup)
|
||||
3. The permission bits are OR'd with rw-rw----
|
||||
|
||||
If unset, the Kubelet will not modify the ownership and permissions of any volume.
|
||||
Note that this field cannot be set when spec.os.name is windows.
|
||||
format: int64
|
||||
type: integer
|
||||
fsGroupChangePolicy:
|
||||
description: |-
|
||||
fsGroupChangePolicy defines behavior of changing ownership and permission of the volume
|
||||
before being exposed inside Pod. This field will only apply to
|
||||
volume types which support fsGroup based ownership(and permissions).
|
||||
It will have no effect on ephemeral volume types such as: secret, configmaps
|
||||
and emptydir.
|
||||
Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used.
|
||||
Note that this field cannot be set when spec.os.name is windows.
|
||||
type: string
|
||||
runAsGroup:
|
||||
description: |-
|
||||
The GID to run the entrypoint of the container process.
|
||||
Uses runtime default if unset.
|
||||
May also be set in SecurityContext. If set in both SecurityContext and
|
||||
PodSecurityContext, the value specified in SecurityContext takes precedence
|
||||
for that container.
|
||||
Note that this field cannot be set when spec.os.name is windows.
|
||||
format: int64
|
||||
type: integer
|
||||
runAsNonRoot:
|
||||
description: |-
|
||||
Indicates that the container must run as a non-root user.
|
||||
If true, the Kubelet will validate the image at runtime to ensure that it
|
||||
does not run as UID 0 (root) and fail to start the container if it does.
|
||||
If unset or false, no such validation will be performed.
|
||||
May also be set in SecurityContext. If set in both SecurityContext and
|
||||
PodSecurityContext, the value specified in SecurityContext takes precedence.
|
||||
type: boolean
|
||||
runAsUser:
|
||||
description: |-
|
||||
The UID to run the entrypoint of the container process.
|
||||
Defaults to user specified in image metadata if unspecified.
|
||||
May also be set in SecurityContext. If set in both SecurityContext and
|
||||
PodSecurityContext, the value specified in SecurityContext takes precedence
|
||||
for that container.
|
||||
Note that this field cannot be set when spec.os.name is windows.
|
||||
format: int64
|
||||
type: integer
|
||||
seLinuxChangePolicy:
|
||||
description: |-
|
||||
seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod.
|
||||
It has no effect on nodes that do not support SELinux or to volumes does not support SELinux.
|
||||
Valid values are "MountOption" and "Recursive".
|
||||
|
||||
"Recursive" means relabeling of all files on all Pod volumes by the container runtime.
|
||||
This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node.
|
||||
|
||||
"MountOption" mounts all eligible Pod volumes with `-o context` mount option.
|
||||
This requires all Pods that share the same volume to use the same SELinux label.
|
||||
It is not possible to share the same volume among privileged and unprivileged Pods.
|
||||
Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes
|
||||
whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their
|
||||
CSIDriver instance. Other volumes are always re-labelled recursively.
|
||||
"MountOption" value is allowed only when SELinuxMount feature gate is enabled.
|
||||
|
||||
If not specified and SELinuxMount feature gate is enabled, "MountOption" is used.
|
||||
If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes
|
||||
and "Recursive" for all other volumes.
|
||||
|
||||
This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers.
|
||||
|
||||
All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state.
|
||||
Note that this field cannot be set when spec.os.name is windows.
|
||||
type: string
|
||||
seLinuxOptions:
|
||||
description: |-
|
||||
The SELinux context to be applied to all containers.
|
||||
If unspecified, the container runtime will allocate a random SELinux context for each
|
||||
container. May also be set in SecurityContext. If set in
|
||||
both SecurityContext and PodSecurityContext, the value specified in SecurityContext
|
||||
takes precedence for that container.
|
||||
Note that this field cannot be set when spec.os.name is windows.
|
||||
properties:
|
||||
level:
|
||||
description: Level is SELinux level label that applies to
|
||||
the container.
|
||||
type: string
|
||||
role:
|
||||
description: Role is a SELinux role label that applies to
|
||||
the container.
|
||||
type: string
|
||||
type:
|
||||
description: Type is a SELinux type label that applies to
|
||||
the container.
|
||||
type: string
|
||||
user:
|
||||
description: User is a SELinux user label that applies to
|
||||
the container.
|
||||
type: string
|
||||
type: object
|
||||
seccompProfile:
|
||||
description: |-
|
||||
The seccomp options to use by the containers in this pod.
|
||||
Note that this field cannot be set when spec.os.name is windows.
|
||||
properties:
|
||||
localhostProfile:
|
||||
description: |-
|
||||
localhostProfile indicates a profile defined in a file on the node should be used.
|
||||
The profile must be preconfigured on the node to work.
|
||||
Must be a descending path, relative to the kubelet's configured seccomp profile location.
|
||||
Must be set if type is "Localhost". Must NOT be set for any other type.
|
||||
type: string
|
||||
type:
|
||||
description: |-
|
||||
type indicates which kind of seccomp profile will be applied.
|
||||
Valid options are:
|
||||
|
||||
Localhost - a profile defined in a file on the node should be used.
|
||||
RuntimeDefault - the container runtime default profile should be used.
|
||||
Unconfined - no profile should be applied.
|
||||
type: string
|
||||
required:
|
||||
- type
|
||||
type: object
|
||||
supplementalGroups:
|
||||
description: |-
|
||||
A list of groups applied to the first process run in each container, in
|
||||
addition to the container's primary GID and fsGroup (if specified). If
|
||||
the SupplementalGroupsPolicy feature is enabled, the
|
||||
supplementalGroupsPolicy field determines whether these are in addition
|
||||
to or instead of any group memberships defined in the container image.
|
||||
If unspecified, no additional groups are added, though group memberships
|
||||
defined in the container image may still be used, depending on the
|
||||
supplementalGroupsPolicy field.
|
||||
Note that this field cannot be set when spec.os.name is windows.
|
||||
items:
|
||||
format: int64
|
||||
type: integer
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
supplementalGroupsPolicy:
|
||||
description: |-
|
||||
Defines how supplemental groups of the first container processes are calculated.
|
||||
Valid values are "Merge" and "Strict". If not specified, "Merge" is used.
|
||||
(Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled
|
||||
and the container runtime must implement support for this feature.
|
||||
Note that this field cannot be set when spec.os.name is windows.
|
||||
type: string
|
||||
sysctls:
|
||||
description: |-
|
||||
Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported
|
||||
sysctls (by the container runtime) might fail to launch.
|
||||
Note that this field cannot be set when spec.os.name is windows.
|
||||
items:
|
||||
description: Sysctl defines a kernel parameter to be set
|
||||
properties:
|
||||
name:
|
||||
description: Name of a property to set
|
||||
type: string
|
||||
value:
|
||||
description: Value of a property to set
|
||||
type: string
|
||||
required:
|
||||
- name
|
||||
- value
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
windowsOptions:
|
||||
description: |-
|
||||
The Windows specific settings applied to all containers.
|
||||
If unspecified, the options within a container's SecurityContext will be used.
|
||||
If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.
|
||||
Note that this field cannot be set when spec.os.name is linux.
|
||||
properties:
|
||||
gmsaCredentialSpec:
|
||||
description: |-
|
||||
GMSACredentialSpec is where the GMSA admission webhook
|
||||
(https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the
|
||||
GMSA credential spec named by the GMSACredentialSpecName field.
|
||||
type: string
|
||||
gmsaCredentialSpecName:
|
||||
description: GMSACredentialSpecName is the name of the GMSA
|
||||
credential spec to use.
|
||||
type: string
|
||||
hostProcess:
|
||||
description: |-
|
||||
HostProcess determines if a container should be run as a 'Host Process' container.
|
||||
All of a Pod's containers must have the same effective HostProcess value
|
||||
(it is not allowed to have a mix of HostProcess containers and non-HostProcess containers).
|
||||
In addition, if HostProcess is true then HostNetwork must also be set to true.
|
||||
type: boolean
|
||||
runAsUserName:
|
||||
description: |-
|
||||
The UserName in Windows to run the entrypoint of the container process.
|
||||
Defaults to the user specified in image metadata if unspecified.
|
||||
May also be set in PodSecurityContext. If set in both SecurityContext and
|
||||
PodSecurityContext, the value specified in SecurityContext takes precedence.
|
||||
type: string
|
||||
type: object
|
||||
type: object
|
||||
priorityClassName:
|
||||
description: 'Optional: Set priorityClassName'
|
||||
type: string
|
||||
|
|
@ -616,6 +858,10 @@ spec:
|
|||
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
|
||||
type: object
|
||||
type: object
|
||||
secretEnv:
|
||||
description: 'Optional: SecretEnv represents the name of the Kubernetes
|
||||
Secret with secret environment variables for the NVIDIA Driver'
|
||||
type: string
|
||||
startupProbe:
|
||||
description: NVIDIA Driver container startup probe settings
|
||||
properties:
|
||||
|
|
@ -675,9 +921,10 @@ spec:
|
|||
operator:
|
||||
description: |-
|
||||
Operator represents a key's relationship to the value.
|
||||
Valid operators are Exists and Equal. Defaults to Equal.
|
||||
Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal.
|
||||
Exists is equivalent to wildcard for value, so that a pod can
|
||||
tolerate all taints of a particular category.
|
||||
Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators).
|
||||
type: string
|
||||
tolerationSeconds:
|
||||
description: |-
|
||||
|
|
@ -798,6 +1045,7 @@ spec:
|
|||
- ignored
|
||||
- ready
|
||||
- notReady
|
||||
- disabled
|
||||
type: string
|
||||
required:
|
||||
- state
|
||||
|
|
|
|||
|
|
@ -35,16 +35,13 @@ spec:
|
|||
image: {{ include "gpu-operator.fullimage" . }}
|
||||
imagePullPolicy: {{ .Values.operator.imagePullPolicy }}
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- >
|
||||
kubectl delete clusterpolicy cluster-policy;
|
||||
kubectl delete crd clusterpolicies.nvidia.com;
|
||||
kubectl delete crd nvidiadrivers.nvidia.com --ignore-not-found=true;
|
||||
{{- if .Values.nfd.enabled -}}
|
||||
kubectl delete crd nodefeatures.nfd.k8s-sigs.io --ignore-not-found=true;
|
||||
kubectl delete crd nodefeaturegroups.nfd.k8s-sigs.io --ignore-not-found=true;
|
||||
kubectl delete crd nodefeaturerules.nfd.k8s-sigs.io --ignore-not-found=true;
|
||||
{{- end }}
|
||||
- /usr/bin/manage-crds
|
||||
args:
|
||||
- delete
|
||||
- --filepath=/opt/gpu-operator/nvidia.com_clusterpolicies.yaml
|
||||
- --filepath=/opt/gpu-operator/nvidia.com_nvidiadrivers.yaml
|
||||
{{- if .Values.nfd.enabled }}
|
||||
- --filepath=/opt/gpu-operator/nfd-api-crds.yaml
|
||||
{{- end }}
|
||||
restartPolicy: OnFailure
|
||||
{{- end }}
|
||||
|
|
|
|||
|
|
@ -85,49 +85,65 @@ spec:
|
|||
{{- if .Values.validator.args }}
|
||||
args: {{ toYaml .Values.validator.args | nindent 6 }}
|
||||
{{- end }}
|
||||
{{- if .Values.validator.hostNetwork }}
|
||||
hostNetwork: {{ .Values.validator.hostNetwork }}
|
||||
{{- end }}
|
||||
{{- if .Values.validator.plugin }}
|
||||
plugin:
|
||||
{{- if .Values.validator.plugin.env }}
|
||||
env: {{ toYaml .Values.validator.plugin.env | nindent 8 }}
|
||||
{{- else }}
|
||||
env: []
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.validator.cuda }}
|
||||
cuda:
|
||||
{{- if .Values.validator.cuda.env }}
|
||||
env: {{ toYaml .Values.validator.cuda.env | nindent 8 }}
|
||||
{{- else }}
|
||||
env: []
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.validator.driver }}
|
||||
driver:
|
||||
{{- if .Values.validator.driver.env }}
|
||||
env: {{ toYaml .Values.validator.driver.env | nindent 8 }}
|
||||
{{- else }}
|
||||
env: []
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.validator.toolkit }}
|
||||
toolkit:
|
||||
{{- if .Values.validator.toolkit.env }}
|
||||
env: {{ toYaml .Values.validator.toolkit.env | nindent 8 }}
|
||||
{{- else }}
|
||||
env: []
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.validator.vfioPCI }}
|
||||
vfioPCI:
|
||||
{{- if .Values.validator.vfioPCI.env }}
|
||||
env: {{ toYaml .Values.validator.vfioPCI.env | nindent 8 }}
|
||||
{{- else }}
|
||||
env: []
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.validator.vgpuManager }}
|
||||
vgpuManager:
|
||||
{{- if .Values.validator.vgpuManager.env }}
|
||||
env: {{ toYaml .Values.validator.vgpuManager.env | nindent 8 }}
|
||||
{{- else }}
|
||||
env: []
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.validator.vgpuDevices }}
|
||||
vgpuDevices:
|
||||
{{- if .Values.validator.vgpuDevices.env }}
|
||||
env: {{ toYaml .Values.validator.vgpuDevices.env | nindent 8 }}
|
||||
{{- else }}
|
||||
env: []
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
mig:
|
||||
{{- if .Values.mig.strategy }}
|
||||
strategy: {{ .Values.mig.strategy }}
|
||||
|
|
@ -136,7 +152,12 @@ spec:
|
|||
enabled: {{ .Values.psa.enabled }}
|
||||
cdi:
|
||||
enabled: {{ .Values.cdi.enabled }}
|
||||
{{- if .Values.cdi.default }}
|
||||
default: {{ .Values.cdi.default }}
|
||||
{{- end }}
|
||||
{{- if and (.Values.cdi.enabled) (.Values.cdi.nriPluginEnabled) }}
|
||||
nriPluginEnabled: {{ .Values.cdi.nriPluginEnabled }}
|
||||
{{- end }}
|
||||
driver:
|
||||
enabled: {{ .Values.driver.enabled }}
|
||||
useNvidiaDriverCRD: {{ .Values.driver.nvidiaDriverCRD.enabled }}
|
||||
|
|
@ -200,6 +221,9 @@ spec:
|
|||
{{- if .Values.driver.kernelModuleConfig }}
|
||||
kernelModuleConfig: {{ toYaml .Values.driver.kernelModuleConfig | nindent 6 }}
|
||||
{{- end }}
|
||||
{{- if .Values.driver.secretEnv }}
|
||||
secretEnv: {{ .Values.driver.secretEnv }}
|
||||
{{- end }}
|
||||
{{- if .Values.driver.resources }}
|
||||
resources: {{ toYaml .Values.driver.resources | nindent 6 }}
|
||||
{{- end }}
|
||||
|
|
@ -232,6 +256,9 @@ spec:
|
|||
timeoutSeconds: {{ .Values.driver.upgradePolicy.drain.timeoutSeconds }}
|
||||
deleteEmptyDir: {{ .Values.driver.upgradePolicy.drain.deleteEmptyDir | default false}}
|
||||
{{- end }}
|
||||
{{- if .Values.driver.hostNetwork }}
|
||||
hostNetwork: {{ .Values.driver.hostNetwork }}
|
||||
{{- end }}
|
||||
vgpuManager:
|
||||
enabled: {{ .Values.vgpuManager.enabled }}
|
||||
{{- if .Values.vgpuManager.repository }}
|
||||
|
|
@ -258,6 +285,12 @@ spec:
|
|||
{{- if .Values.vgpuManager.args }}
|
||||
args: {{ toYaml .Values.vgpuManager.args | nindent 6 }}
|
||||
{{- end }}
|
||||
{{- if .Values.vgpuManager.kernelModuleConfig }}
|
||||
kernelModuleConfig: {{ toYaml .Values.vgpuManager.kernelModuleConfig | nindent 6 }}
|
||||
{{- end }}
|
||||
{{- if .Values.vgpuManager.hostNetwork }}
|
||||
hostNetwork: {{ .Values.vgpuManager.hostNetwork }}
|
||||
{{- end }}
|
||||
driverManager:
|
||||
{{- if .Values.vgpuManager.driverManager.repository }}
|
||||
repository: {{ .Values.vgpuManager.driverManager.repository }}
|
||||
|
|
@ -276,7 +309,9 @@ spec:
|
|||
{{- end }}
|
||||
kataManager:
|
||||
enabled: {{ .Values.kataManager.enabled }}
|
||||
{{- if .Values.kataManager.config }}
|
||||
config: {{ toYaml .Values.kataManager.config | nindent 6 }}
|
||||
{{- end }}
|
||||
{{- if .Values.kataManager.repository }}
|
||||
repository: {{ .Values.kataManager.repository }}
|
||||
{{- end }}
|
||||
|
|
@ -301,6 +336,9 @@ spec:
|
|||
{{- if .Values.kataManager.args }}
|
||||
args: {{ toYaml .Values.kataManager.args | nindent 6 }}
|
||||
{{- end }}
|
||||
{{- if .Values.kataManager.hostNetwork }}
|
||||
hostNetwork: {{ .Values.kataManager.hostNetwork }}
|
||||
{{- end }}
|
||||
vfioManager:
|
||||
enabled: {{ .Values.vfioManager.enabled }}
|
||||
{{- if .Values.vfioManager.repository }}
|
||||
|
|
@ -343,6 +381,9 @@ spec:
|
|||
{{- if .Values.vfioManager.driverManager.env }}
|
||||
env: {{ toYaml .Values.vfioManager.driverManager.env | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.vfioManager.hostNetwork }}
|
||||
hostNetwork: {{ .Values.vfioManager.hostNetwork }}
|
||||
{{- end }}
|
||||
vgpuDeviceManager:
|
||||
enabled: {{ .Values.vgpuDeviceManager.enabled }}
|
||||
{{- if .Values.vgpuDeviceManager.repository }}
|
||||
|
|
@ -372,6 +413,9 @@ spec:
|
|||
{{- if .Values.vgpuDeviceManager.config }}
|
||||
config: {{ toYaml .Values.vgpuDeviceManager.config | nindent 6 }}
|
||||
{{- end }}
|
||||
{{- if .Values.vgpuDeviceManager.hostNetwork }}
|
||||
hostNetwork: {{ .Values.vgpuDeviceManager.hostNetwork }}
|
||||
{{- end }}
|
||||
ccManager:
|
||||
enabled: {{ .Values.ccManager.enabled }}
|
||||
defaultMode: {{ .Values.ccManager.defaultMode | quote }}
|
||||
|
|
@ -394,11 +438,14 @@ spec:
|
|||
resources: {{ toYaml .Values.ccManager.resources | nindent 6 }}
|
||||
{{- end }}
|
||||
{{- if .Values.ccManager.env }}
|
||||
env: {{ toYaml .Values.vfioManager.env | nindent 6 }}
|
||||
env: {{ toYaml .Values.ccManager.env | nindent 6 }}
|
||||
{{- end }}
|
||||
{{- if .Values.ccManager.args }}
|
||||
args: {{ toYaml .Values.ccManager.args | nindent 6 }}
|
||||
{{- end }}
|
||||
{{- if .Values.ccManager.hostNetwork }}
|
||||
hostNetwork: {{ .Values.ccManager.hostNetwork }}
|
||||
{{- end }}
|
||||
toolkit:
|
||||
enabled: {{ .Values.toolkit.enabled }}
|
||||
{{- if .Values.toolkit.repository }}
|
||||
|
|
@ -425,6 +472,9 @@ spec:
|
|||
{{- if .Values.toolkit.installDir }}
|
||||
installDir: {{ .Values.toolkit.installDir }}
|
||||
{{- end }}
|
||||
{{- if .Values.toolkit.hostNetwork }}
|
||||
hostNetwork: {{ .Values.toolkit.hostNetwork }}
|
||||
{{- end }}
|
||||
devicePlugin:
|
||||
enabled: {{ .Values.devicePlugin.enabled }}
|
||||
{{- if .Values.devicePlugin.repository }}
|
||||
|
|
@ -453,8 +503,11 @@ spec:
|
|||
{{- end }}
|
||||
{{- if .Values.devicePlugin.config.name }}
|
||||
config:
|
||||
name: {{ .Values.devicePlugin.config.name }}
|
||||
default: {{ .Values.devicePlugin.config.default }}
|
||||
name: {{ .Values.devicePlugin.config.name | quote }}
|
||||
default: {{ .Values.devicePlugin.config.default | quote }}
|
||||
{{- end }}
|
||||
{{- if .Values.devicePlugin.hostNetwork }}
|
||||
hostNetwork: {{ .Values.devicePlugin.hostNetwork }}
|
||||
{{- end }}
|
||||
dcgm:
|
||||
enabled: {{ .Values.dcgm.enabled }}
|
||||
|
|
@ -482,6 +535,9 @@ spec:
|
|||
{{- if .Values.dcgm.args }}
|
||||
args: {{ toYaml .Values.dcgm.args | nindent 6 }}
|
||||
{{- end }}
|
||||
{{- if .Values.dcgm.hostNetwork }}
|
||||
hostNetwork: {{ .Values.dcgm.hostNetwork }}
|
||||
{{- end }}
|
||||
dcgmExporter:
|
||||
enabled: {{ .Values.dcgmExporter.enabled }}
|
||||
{{- if .Values.dcgmExporter.repository }}
|
||||
|
|
@ -515,6 +571,18 @@ spec:
|
|||
{{- if .Values.dcgmExporter.serviceMonitor }}
|
||||
serviceMonitor: {{ toYaml .Values.dcgmExporter.serviceMonitor | nindent 6 }}
|
||||
{{- end }}
|
||||
{{- if .Values.dcgmExporter.service }}
|
||||
service: {{ toYaml .Values.dcgmExporter.service | nindent 6 }}
|
||||
{{- end }}
|
||||
{{- if .Values.dcgmExporter.hostPID }}
|
||||
hostPID: {{ .Values.dcgmExporter.hostPID }}
|
||||
{{- end }}
|
||||
{{- if .Values.dcgmExporter.hostNetwork }}
|
||||
hostNetwork: {{ .Values.dcgmExporter.hostNetwork }}
|
||||
{{- end }}
|
||||
{{- if .Values.dcgmExporter.hpcJobMapping }}
|
||||
hpcJobMapping: {{ toYaml .Values.dcgmExporter.hpcJobMapping | nindent 6 }}
|
||||
{{- end }}
|
||||
gfd:
|
||||
enabled: {{ .Values.gfd.enabled }}
|
||||
{{- if .Values.gfd.repository }}
|
||||
|
|
@ -541,6 +609,9 @@ spec:
|
|||
{{- if .Values.gfd.args }}
|
||||
args: {{ toYaml .Values.gfd.args | nindent 6 }}
|
||||
{{- end }}
|
||||
{{- if .Values.gfd.hostNetwork }}
|
||||
hostNetwork: {{ .Values.gfd.hostNetwork }}
|
||||
{{- end }}
|
||||
migManager:
|
||||
enabled: {{ .Values.migManager.enabled }}
|
||||
{{- if .Values.migManager.repository }}
|
||||
|
|
@ -569,12 +640,17 @@ spec:
|
|||
{{- end }}
|
||||
{{- if .Values.migManager.config }}
|
||||
config:
|
||||
{{- if .Values.migManager.config.name }}
|
||||
name: {{ .Values.migManager.config.name }}
|
||||
default: {{ .Values.migManager.config.default }}
|
||||
{{- end }}
|
||||
default: {{ .Values.migManager.config.default | quote }}
|
||||
{{- end }}
|
||||
{{- if .Values.migManager.gpuClientsConfig }}
|
||||
gpuClientsConfig: {{ toYaml .Values.migManager.gpuClientsConfig | nindent 6 }}
|
||||
{{- end }}
|
||||
{{- if .Values.migManager.hostNetwork }}
|
||||
hostNetwork: {{ .Values.migManager.hostNetwork }}
|
||||
{{- end }}
|
||||
nodeStatusExporter:
|
||||
enabled: {{ .Values.nodeStatusExporter.enabled }}
|
||||
{{- if .Values.nodeStatusExporter.repository }}
|
||||
|
|
@ -599,7 +675,10 @@ spec:
|
|||
{{- if .Values.nodeStatusExporter.args }}
|
||||
args: {{ toYaml .Values.nodeStatusExporter.args | nindent 6 }}
|
||||
{{- end }}
|
||||
{{- if .Values.gds.enabled }}
|
||||
{{- if .Values.nodeStatusExporter.hostNetwork }}
|
||||
hostNetwork: {{ .Values.nodeStatusExporter.hostNetwork }}
|
||||
{{- end }}
|
||||
{{- if .Values.gds }}
|
||||
gds:
|
||||
enabled: {{ .Values.gds.enabled }}
|
||||
{{- if .Values.gds.repository }}
|
||||
|
|
@ -650,6 +729,9 @@ spec:
|
|||
{{- if .Values.sandboxWorkloads.defaultWorkload }}
|
||||
defaultWorkload: {{ .Values.sandboxWorkloads.defaultWorkload }}
|
||||
{{- end }}
|
||||
{{- if .Values.sandboxWorkloads.mode }}
|
||||
mode: {{ .Values.sandboxWorkloads.mode | quote }}
|
||||
{{- end }}
|
||||
sandboxDevicePlugin:
|
||||
{{- if .Values.sandboxDevicePlugin.enabled }}
|
||||
enabled: {{ .Values.sandboxDevicePlugin.enabled }}
|
||||
|
|
@ -678,3 +760,37 @@ spec:
|
|||
{{- if .Values.sandboxDevicePlugin.args }}
|
||||
args: {{ toYaml .Values.sandboxDevicePlugin.args | nindent 6 }}
|
||||
{{- end }}
|
||||
{{- if .Values.sandboxDevicePlugin.hostNetwork }}
|
||||
hostNetwork: {{ .Values.sandboxDevicePlugin.hostNetwork }}
|
||||
{{- end }}
|
||||
kataSandboxDevicePlugin:
|
||||
{{- if ne .Values.kataSandboxDevicePlugin.enabled nil }}
|
||||
enabled: {{ .Values.kataSandboxDevicePlugin.enabled }}
|
||||
{{- end }}
|
||||
{{- if .Values.kataSandboxDevicePlugin.repository }}
|
||||
repository: {{ .Values.kataSandboxDevicePlugin.repository }}
|
||||
{{- end }}
|
||||
{{- if .Values.kataSandboxDevicePlugin.image }}
|
||||
image: {{ .Values.kataSandboxDevicePlugin.image }}
|
||||
{{- end }}
|
||||
{{- if .Values.kataSandboxDevicePlugin.version }}
|
||||
version: {{ .Values.kataSandboxDevicePlugin.version | quote }}
|
||||
{{- end }}
|
||||
{{- if .Values.kataSandboxDevicePlugin.imagePullPolicy }}
|
||||
imagePullPolicy: {{ .Values.kataSandboxDevicePlugin.imagePullPolicy }}
|
||||
{{- end }}
|
||||
{{- if .Values.kataSandboxDevicePlugin.imagePullSecrets }}
|
||||
imagePullSecrets: {{ toYaml .Values.kataSandboxDevicePlugin.imagePullSecrets | nindent 6 }}
|
||||
{{- end }}
|
||||
{{- if .Values.kataSandboxDevicePlugin.resources }}
|
||||
resources: {{ toYaml .Values.kataSandboxDevicePlugin.resources | nindent 6 }}
|
||||
{{- end }}
|
||||
{{- if .Values.kataSandboxDevicePlugin.env }}
|
||||
env: {{ toYaml .Values.kataSandboxDevicePlugin.env | nindent 6 }}
|
||||
{{- end }}
|
||||
{{- if .Values.kataSandboxDevicePlugin.args }}
|
||||
args: {{ toYaml .Values.kataSandboxDevicePlugin.args | nindent 6 }}
|
||||
{{- end }}
|
||||
{{- if .Values.kataSandboxDevicePlugin.hostNetwork }}
|
||||
hostNetwork: {{ .Values.kataSandboxDevicePlugin.hostNetwork }}
|
||||
{{- end }}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,8 @@
|
|||
{{ range .Values.extraObjects }}
|
||||
---
|
||||
{{ if typeIs "string" . }}
|
||||
{{- tpl . $ }}
|
||||
{{- else }}
|
||||
{{- tpl (toYaml .) $ }}
|
||||
{{- end }}
|
||||
{{ end }}
|
||||
|
|
@ -2,106 +2,32 @@
|
|||
apiVersion: nfd.k8s-sigs.io/v1alpha1
|
||||
kind: NodeFeatureRule
|
||||
metadata:
|
||||
name: nvidia-nfd-nodefeaturerules
|
||||
name: nvidia-kernel-modules
|
||||
spec:
|
||||
rules:
|
||||
- name: "TDX rule"
|
||||
- name: kernel-module-gdrdrv
|
||||
labels:
|
||||
tdx.enabled: "true"
|
||||
nvidia.com/gdrcopy.capable: "true"
|
||||
matchFeatures:
|
||||
- feature: cpu.security
|
||||
- feature: kernel.loadedmodule
|
||||
matchExpressions:
|
||||
tdx.enabled: {op: IsTrue}
|
||||
- name: "TDX total keys rule"
|
||||
extendedResources:
|
||||
tdx.total_keys: "@cpu.security.tdx.total_keys"
|
||||
gdrdrv:
|
||||
op: Exists
|
||||
- name: kernel-module-nvidia_fs
|
||||
labels:
|
||||
nvidia.com/gds.capable: "true"
|
||||
matchFeatures:
|
||||
- feature: cpu.security
|
||||
- feature: kernel.loadedmodule
|
||||
matchExpressions:
|
||||
tdx.enabled: {op: IsTrue}
|
||||
- name: "SEV-SNP rule"
|
||||
nvidia_fs:
|
||||
op: Exists
|
||||
- name: kernel-module-nvidia_peermem
|
||||
labels:
|
||||
sev.snp.enabled: "true"
|
||||
nvidia.com/peermem.capable: "true"
|
||||
matchFeatures:
|
||||
- feature: cpu.security
|
||||
matchExpressions:
|
||||
sev.snp.enabled:
|
||||
op: IsTrue
|
||||
- name: "SEV-ES rule"
|
||||
labels:
|
||||
sev.es.enabled: "true"
|
||||
matchFeatures:
|
||||
- feature: cpu.security
|
||||
matchExpressions:
|
||||
sev.es.enabled:
|
||||
op: IsTrue
|
||||
- name: SEV system capacities
|
||||
extendedResources:
|
||||
sev_asids: '@cpu.security.sev.asids'
|
||||
sev_es: '@cpu.security.sev.encrypted_state_ids'
|
||||
matchFeatures:
|
||||
- feature: cpu.security
|
||||
matchExpressions:
|
||||
sev.enabled:
|
||||
op: Exists
|
||||
- name: "NVIDIA H100"
|
||||
labels:
|
||||
"nvidia.com/gpu.H100": "true"
|
||||
"nvidia.com/gpu.family": "hopper"
|
||||
matchFeatures:
|
||||
- feature: pci.device
|
||||
- feature: kernel.loadedmodule
|
||||
matchExpressions:
|
||||
vendor: {op: In, value: ["10de"]}
|
||||
device: {op: In, value: ["2339"]}
|
||||
- name: "NVIDIA H100 PCIe"
|
||||
labels:
|
||||
"nvidia.com/gpu.H100.pcie": "true"
|
||||
"nvidia.com/gpu.family": "hopper"
|
||||
matchFeatures:
|
||||
- feature: pci.device
|
||||
matchExpressions:
|
||||
vendor: {op: In, value: ["10de"]}
|
||||
device: {op: In, value: ["2331"]}
|
||||
- name: "NVIDIA H100 80GB HBM3"
|
||||
labels:
|
||||
"nvidia.com/gpu.H100.HBM3": "true"
|
||||
"nvidia.com/gpu.family": "hopper"
|
||||
matchFeatures:
|
||||
- feature: pci.device
|
||||
matchExpressions:
|
||||
vendor: {op: In, value: ["10de"]}
|
||||
device: {op: In, value: ["2330"]}
|
||||
- name: "NVIDIA H800"
|
||||
labels:
|
||||
"nvidia.com/gpu.H800": "true"
|
||||
"nvidia.com/gpu.family": "hopper"
|
||||
matchFeatures:
|
||||
- feature: pci.device
|
||||
matchExpressions:
|
||||
vendor: {op: In, value: ["10de"]}
|
||||
device: {op: In, value: ["2324"]}
|
||||
- name: "NVIDIA H800 PCIE"
|
||||
labels:
|
||||
"nvidia.com/gpu.H800.pcie": "true"
|
||||
"nvidia.com/gpu.family": "hopper"
|
||||
matchFeatures:
|
||||
- feature: pci.device
|
||||
matchExpressions:
|
||||
vendor: {op: In, value: ["10de"]}
|
||||
device: {op: In, value: ["2322"]}
|
||||
- name: "NVIDIA CC Enabled"
|
||||
labels:
|
||||
"nvidia.com/cc.capable": "true"
|
||||
matchAny: # TDX/SEV + Hopper GPU
|
||||
- matchFeatures:
|
||||
- feature: rule.matched
|
||||
matchExpressions:
|
||||
nvidia.com/gpu.family: {op: In, value: ["hopper"]}
|
||||
sev.snp.enabled: {op: IsTrue}
|
||||
- matchFeatures:
|
||||
- feature: rule.matched
|
||||
matchExpressions:
|
||||
nvidia.com/gpu.family: {op: In, value: ["hopper"]}
|
||||
tdx.enabled: {op: IsTrue}
|
||||
nvidia_peermem:
|
||||
op: Exists
|
||||
{{- end }}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
{{- if .Values.driver.enabled }}
|
||||
{{- if and .Values.driver.nvidiaDriverCRD.enabled .Values.driver.nvidiaDriverCRD.deployDefaultCR }}
|
||||
apiVersion: nvidia.com/v1alpha1
|
||||
kind: NVIDIADriver
|
||||
|
|
@ -48,7 +49,11 @@ spec:
|
|||
certConfig:
|
||||
name: {{ .Values.driver.certConfig.name }}
|
||||
{{- end }}
|
||||
{{- if .Values.driver.licensingConfig.configMapName }}
|
||||
{{- if .Values.driver.licensingConfig.secretName }}
|
||||
licensingConfig:
|
||||
secretName: {{ .Values.driver.licensingConfig.secretName }}
|
||||
nlsEnabled: {{ .Values.driver.licensingConfig.nlsEnabled | default true }}
|
||||
{{- else if .Values.driver.licensingConfig.configMapName }}
|
||||
licensingConfig:
|
||||
name: {{ .Values.driver.licensingConfig.configMapName }}
|
||||
nlsEnabled: {{ .Values.driver.licensingConfig.nlsEnabled | default true }}
|
||||
|
|
@ -61,6 +66,9 @@ spec:
|
|||
kernelModuleConfig:
|
||||
name: {{ .Values.driver.kernelModuleConfig.name }}
|
||||
{{- end }}
|
||||
{{- if .Values.driver.secretEnv }}
|
||||
secretEnv: {{ .Values.driver.secretEnv }}
|
||||
{{- end }}
|
||||
{{- if .Values.driver.resources }}
|
||||
resources: {{ toYaml .Values.driver.resources | nindent 6 }}
|
||||
{{- end }}
|
||||
|
|
@ -70,6 +78,9 @@ spec:
|
|||
{{- if .Values.driver.args }}
|
||||
args: {{ toYaml .Values.driver.args | nindent 6 }}
|
||||
{{- end }}
|
||||
{{- if .Values.driver.hostNetwork }}
|
||||
hostNetwork: {{ .Values.driver.hostNetwork }}
|
||||
{{- end }}
|
||||
{{- if .Values.gds.enabled }}
|
||||
gds:
|
||||
enabled: {{ .Values.gds.enabled }}
|
||||
|
|
@ -117,3 +128,4 @@ spec:
|
|||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ apiVersion: apps/v1
|
|||
kind: Deployment
|
||||
metadata:
|
||||
name: gpu-operator
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "gpu-operator.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: "gpu-operator"
|
||||
|
|
@ -58,10 +59,6 @@ spec:
|
|||
fieldPath: metadata.namespace
|
||||
- name: "DRIVER_MANAGER_IMAGE"
|
||||
value: "{{ include "driver-manager.fullimage" . }}"
|
||||
volumeMounts:
|
||||
- name: host-os-release
|
||||
mountPath: "/host-etc/os-release"
|
||||
readOnly: true
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
|
|
@ -81,10 +78,6 @@ spec:
|
|||
ports:
|
||||
- name: metrics
|
||||
containerPort: 8080
|
||||
volumes:
|
||||
- name: host-os-release
|
||||
hostPath:
|
||||
path: "/etc/os-release"
|
||||
{{- with .Values.operator.nodeSelector }}
|
||||
nodeSelector:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ apiVersion: rbac.authorization.k8s.io/v1
|
|||
kind: Role
|
||||
metadata:
|
||||
name: gpu-operator
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "gpu-operator.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: "gpu-operator"
|
||||
|
|
@ -82,3 +83,13 @@ rules:
|
|||
- watch
|
||||
- update
|
||||
- delete
|
||||
- apiGroups:
|
||||
- "nfd.k8s-sigs.io"
|
||||
resources:
|
||||
- "nodefeatures"
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- create
|
||||
- update
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ kind: RoleBinding
|
|||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: gpu-operator
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "gpu-operator.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: "gpu-operator"
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ apiVersion: v1
|
|||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: gpu-operator
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "gpu-operator.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: "gpu-operator"
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ apiVersion: v1
|
|||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: gpu-operator-upgrade-crd-hook-sa
|
||||
namespace: {{ .Release.Namespace }}
|
||||
annotations:
|
||||
helm.sh/hook: pre-upgrade
|
||||
helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation
|
||||
|
|
@ -83,13 +84,13 @@ spec:
|
|||
image: {{ include "gpu-operator.fullimage" . }}
|
||||
imagePullPolicy: {{ .Values.operator.imagePullPolicy }}
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- >
|
||||
kubectl apply -f /opt/gpu-operator/nvidia.com_clusterpolicies.yaml;
|
||||
kubectl apply -f /opt/gpu-operator/nvidia.com_nvidiadrivers.yaml;
|
||||
- /usr/bin/manage-crds
|
||||
args:
|
||||
- apply
|
||||
- --filepath=/opt/gpu-operator/nvidia.com_clusterpolicies.yaml
|
||||
- --filepath=/opt/gpu-operator/nvidia.com_nvidiadrivers.yaml
|
||||
{{- if .Values.nfd.enabled }}
|
||||
kubectl apply -f /opt/gpu-operator/nfd-api-crds.yaml;
|
||||
- --filepath=/opt/gpu-operator/nfd-api-crds.yaml
|
||||
{{- end }}
|
||||
restartPolicy: OnFailure
|
||||
{{- end }}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
{{- if and (eq .Values.cdi.enabled false) (eq .Values.cdi.nriPluginEnabled true) }}
|
||||
{{ fail "the NRI Plugin cannot be enabled when CDI is disabled" }}
|
||||
{{- end}}
|
||||
|
|
@ -13,12 +13,14 @@ psa:
|
|||
enabled: false
|
||||
|
||||
cdi:
|
||||
enabled: false
|
||||
default: false
|
||||
enabled: true
|
||||
nriPluginEnabled: false
|
||||
|
||||
sandboxWorkloads:
|
||||
enabled: false
|
||||
defaultWorkload: "container"
|
||||
# Sandbox mode: "kubevirt" (default) or "kata". When "kata", the Kata device plugin is deployed on vm-passthrough nodes.
|
||||
mode: "kubevirt"
|
||||
|
||||
hostPaths:
|
||||
# rootFS represents the path to the root filesystem of the host.
|
||||
|
|
@ -50,8 +52,8 @@ daemonsets:
|
|||
maxUnavailable: "1"
|
||||
|
||||
validator:
|
||||
repository: nvcr.io/nvidia/cloud-native
|
||||
image: gpu-operator-validator
|
||||
repository: nvcr.io/nvidia
|
||||
image: gpu-operator
|
||||
# If version is not specified, then default is to use chart.AppVersion
|
||||
#version: ""
|
||||
imagePullPolicy: IfNotPresent
|
||||
|
|
@ -59,10 +61,9 @@ validator:
|
|||
env: []
|
||||
args: []
|
||||
resources: {}
|
||||
hostNetwork: false
|
||||
plugin:
|
||||
env:
|
||||
- name: WITH_WORKLOAD
|
||||
value: "false"
|
||||
env: []
|
||||
|
||||
operator:
|
||||
repository: nvcr.io/nvidia
|
||||
|
|
@ -79,16 +80,7 @@ operator:
|
|||
# upgrade CRD on chart upgrade, requires --disable-openapi-validation flag
|
||||
# to be passed during helm upgrade.
|
||||
upgradeCRD: true
|
||||
initContainer:
|
||||
image: cuda
|
||||
repository: nvcr.io/nvidia
|
||||
version: 12.8.1-base-ubi9
|
||||
imagePullPolicy: IfNotPresent
|
||||
tolerations:
|
||||
- key: "node-role.kubernetes.io/master"
|
||||
operator: "Equal"
|
||||
value: ""
|
||||
effect: "NoSchedule"
|
||||
- key: "node-role.kubernetes.io/control-plane"
|
||||
operator: "Equal"
|
||||
value: ""
|
||||
|
|
@ -98,12 +90,6 @@ operator:
|
|||
affinity:
|
||||
nodeAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- weight: 1
|
||||
preference:
|
||||
matchExpressions:
|
||||
- key: "node-role.kubernetes.io/master"
|
||||
operator: In
|
||||
values: [""]
|
||||
- weight: 1
|
||||
preference:
|
||||
matchExpressions:
|
||||
|
|
@ -146,7 +132,7 @@ driver:
|
|||
usePrecompiled: false
|
||||
repository: nvcr.io/nvidia
|
||||
image: driver
|
||||
version: "570.124.06"
|
||||
version: "580.126.20"
|
||||
imagePullPolicy: IfNotPresent
|
||||
imagePullSecrets: []
|
||||
startupProbe:
|
||||
|
|
@ -192,25 +178,13 @@ driver:
|
|||
timeoutSeconds: 300
|
||||
deleteEmptyDir: false
|
||||
manager:
|
||||
image: k8s-driver-manager
|
||||
repository: nvcr.io/nvidia/cloud-native
|
||||
image: k8s-driver-manager
|
||||
# When choosing a different version of k8s-driver-manager, DO NOT downgrade to a version lower than v0.6.4
|
||||
# to ensure k8s-driver-manager stays compatible with gpu-operator starting from v24.3.0
|
||||
version: v0.8.0
|
||||
version: v0.10.0
|
||||
imagePullPolicy: IfNotPresent
|
||||
env:
|
||||
- name: ENABLE_GPU_POD_EVICTION
|
||||
value: "true"
|
||||
- name: ENABLE_AUTO_DRAIN
|
||||
value: "false"
|
||||
- name: DRAIN_USE_FORCE
|
||||
value: "false"
|
||||
- name: DRAIN_POD_SELECTOR_LABEL
|
||||
value: ""
|
||||
- name: DRAIN_TIMEOUT_SECONDS
|
||||
value: "0s"
|
||||
- name: DRAIN_DELETE_EMPTYDIR_DATA
|
||||
value: "false"
|
||||
env: []
|
||||
env: []
|
||||
resources: {}
|
||||
# Private mirror repository configuration
|
||||
|
|
@ -221,7 +195,7 @@ driver:
|
|||
name: ""
|
||||
# vGPU licensing configuration
|
||||
licensingConfig:
|
||||
configMapName: ""
|
||||
secretName: ""
|
||||
nlsEnabled: true
|
||||
# vGPU topology daemon configuration
|
||||
virtualTopology:
|
||||
|
|
@ -229,39 +203,31 @@ driver:
|
|||
# kernel module configuration for NVIDIA driver
|
||||
kernelModuleConfig:
|
||||
name: ""
|
||||
# Name of Kubernetes Secret which contains secrets to be passed in as environment variables
|
||||
secretEnv: ""
|
||||
hostNetwork: false
|
||||
|
||||
toolkit:
|
||||
enabled: true
|
||||
repository: nvcr.io/nvidia/k8s
|
||||
image: container-toolkit
|
||||
version: v1.17.5-ubuntu20.04
|
||||
version: v1.19.0
|
||||
imagePullPolicy: IfNotPresent
|
||||
imagePullSecrets: []
|
||||
env: []
|
||||
resources: {}
|
||||
installDir: "/usr/local/nvidia"
|
||||
hostNetwork: false
|
||||
|
||||
devicePlugin:
|
||||
enabled: true
|
||||
repository: nvcr.io/nvidia
|
||||
image: k8s-device-plugin
|
||||
version: v0.17.1
|
||||
version: v0.19.0
|
||||
imagePullPolicy: IfNotPresent
|
||||
imagePullSecrets: []
|
||||
args: []
|
||||
env:
|
||||
- name: PASS_DEVICE_SPECS
|
||||
value: "true"
|
||||
- name: FAIL_ON_INIT_ERROR
|
||||
value: "true"
|
||||
- name: DEVICE_LIST_STRATEGY
|
||||
value: envvar
|
||||
- name: DEVICE_ID_STRATEGY
|
||||
value: uuid
|
||||
- name: NVIDIA_VISIBLE_DEVICES
|
||||
value: all
|
||||
- name: NVIDIA_DRIVER_CAPABILITIES
|
||||
value: all
|
||||
env: []
|
||||
resources: {}
|
||||
# Plugin configuration
|
||||
# Use "name" to either point to an existing ConfigMap or to create a new one with a list of configurations(i.e with create=true).
|
||||
|
|
@ -296,6 +262,7 @@ devicePlugin:
|
|||
mps:
|
||||
# MPS root path on the host
|
||||
root: "/run/nvidia/mps"
|
||||
hostNetwork: false
|
||||
|
||||
# standalone dcgm hostengine
|
||||
dcgm:
|
||||
|
|
@ -303,26 +270,30 @@ dcgm:
|
|||
enabled: false
|
||||
repository: nvcr.io/nvidia/cloud-native
|
||||
image: dcgm
|
||||
version: 4.1.1-2-ubuntu22.04
|
||||
version: 4.5.2-1-ubuntu22.04
|
||||
imagePullPolicy: IfNotPresent
|
||||
args: []
|
||||
env: []
|
||||
resources: {}
|
||||
hostNetwork: false
|
||||
|
||||
dcgmExporter:
|
||||
enabled: true
|
||||
repository: nvcr.io/nvidia/k8s
|
||||
image: dcgm-exporter
|
||||
version: 4.1.1-4.0.4-ubuntu22.04
|
||||
version: 4.5.1-4.8.0-distroless
|
||||
imagePullPolicy: IfNotPresent
|
||||
env:
|
||||
- name: DCGM_EXPORTER_LISTEN
|
||||
value: ":9400"
|
||||
- name: DCGM_EXPORTER_KUBERNETES
|
||||
value: "true"
|
||||
- name: DCGM_EXPORTER_COLLECTORS
|
||||
value: "/etc/dcgm-exporter/dcp-metrics-included.csv"
|
||||
env: []
|
||||
resources: {}
|
||||
hostPID: false
|
||||
hostNetwork: false
|
||||
# HPC job mapping configuration for correlating GPU metrics with HPC workload manager jobs
|
||||
# This is used by HPC workload managers like Slurm to label GPU metrics with job IDs
|
||||
# hpcJobMapping:
|
||||
# enabled: true
|
||||
# directory: /var/lib/dcgm-exporter/job-mapping
|
||||
service:
|
||||
internalTrafficPolicy: Cluster
|
||||
serviceMonitor:
|
||||
enabled: false
|
||||
interval: 15s
|
||||
|
|
@ -359,31 +330,35 @@ gfd:
|
|||
enabled: true
|
||||
repository: nvcr.io/nvidia
|
||||
image: k8s-device-plugin
|
||||
version: v0.17.1
|
||||
version: v0.19.0
|
||||
imagePullPolicy: IfNotPresent
|
||||
imagePullSecrets: []
|
||||
env:
|
||||
- name: GFD_SLEEP_INTERVAL
|
||||
value: 60s
|
||||
- name: GFD_FAIL_ON_INIT_ERROR
|
||||
value: "true"
|
||||
env: []
|
||||
resources: {}
|
||||
hostNetwork: false
|
||||
|
||||
migManager:
|
||||
enabled: true
|
||||
repository: nvcr.io/nvidia/cloud-native
|
||||
image: k8s-mig-manager
|
||||
version: v0.12.1-ubuntu20.04
|
||||
version: v0.14.0
|
||||
imagePullPolicy: IfNotPresent
|
||||
imagePullSecrets: []
|
||||
env:
|
||||
- name: WITH_REBOOT
|
||||
value: "false"
|
||||
env: []
|
||||
resources: {}
|
||||
# MIG configuration
|
||||
# Use "name" to either point to an existing ConfigMap or to create a new one with a list of configurations(i.e with create=true).
|
||||
# Use "data" to build an integrated ConfigMap from a set of configurations as
|
||||
# part of this helm chart. An example of setting "data" might be:
|
||||
# NOTE: MIG manager automatically generates configuration from hardware on each node.
|
||||
# Only provide a custom config if you need settings that differ from hardware discovery.
|
||||
#
|
||||
# To use an existing ConfigMap:
|
||||
# - Set name="your-configmap-name" with create=false
|
||||
# - ConfigMap MUST have a key named "config.yaml"
|
||||
#
|
||||
# To create a new ConfigMap via Helm:
|
||||
# - Set create=true, name="your-configmap-name", and provide data below
|
||||
# - If create=true but data is empty, ConfigMap creation is skipped
|
||||
#
|
||||
# Example of creating a custom ConfigMap:
|
||||
# config:
|
||||
# name: custom-mig-parted-configs
|
||||
# create: true
|
||||
|
|
@ -415,28 +390,32 @@ migManager:
|
|||
default: "all-disabled"
|
||||
# Create a ConfigMap (default: false)
|
||||
create: false
|
||||
# ConfigMap name (either existing or to create a new one with create=true above)
|
||||
# ConfigMap name (either existing or to create with create=true)
|
||||
# If name is provided, mig-manager will use this config instead of auto-generated one.
|
||||
# REQUIREMENT: Custom ConfigMaps must contain a key named "config.yaml"
|
||||
name: ""
|
||||
# Data section for the ConfigMap to create (i.e only applies when create=true)
|
||||
# Data section for the ConfigMap (required only if create=true)
|
||||
data: {}
|
||||
gpuClientsConfig:
|
||||
name: ""
|
||||
hostNetwork: false
|
||||
|
||||
nodeStatusExporter:
|
||||
enabled: false
|
||||
repository: nvcr.io/nvidia/cloud-native
|
||||
image: gpu-operator-validator
|
||||
repository: nvcr.io/nvidia
|
||||
image: gpu-operator
|
||||
# If version is not specified, then default is to use chart.AppVersion
|
||||
#version: ""
|
||||
imagePullPolicy: IfNotPresent
|
||||
imagePullSecrets: []
|
||||
resources: {}
|
||||
hostNetwork: false
|
||||
|
||||
gds:
|
||||
enabled: false
|
||||
repository: nvcr.io/nvidia/cloud-native
|
||||
image: nvidia-fs
|
||||
version: "2.20.5"
|
||||
version: "2.27.3"
|
||||
imagePullPolicy: IfNotPresent
|
||||
imagePullSecrets: []
|
||||
env: []
|
||||
|
|
@ -446,7 +425,7 @@ gdrcopy:
|
|||
enabled: false
|
||||
repository: nvcr.io/nvidia/cloud-native
|
||||
image: gdrdrv
|
||||
version: "v2.4.4"
|
||||
version: "v2.5.2"
|
||||
imagePullPolicy: IfNotPresent
|
||||
imagePullSecrets: []
|
||||
env: []
|
||||
|
|
@ -462,102 +441,100 @@ vgpuManager:
|
|||
env: []
|
||||
resources: {}
|
||||
driverManager:
|
||||
image: k8s-driver-manager
|
||||
repository: nvcr.io/nvidia/cloud-native
|
||||
image: k8s-driver-manager
|
||||
# When choosing a different version of k8s-driver-manager, DO NOT downgrade to a version lower than v0.6.4
|
||||
# to ensure k8s-driver-manager stays compatible with gpu-operator starting from v24.3.0
|
||||
version: v0.8.0
|
||||
version: v0.10.0
|
||||
imagePullPolicy: IfNotPresent
|
||||
env:
|
||||
- name: ENABLE_GPU_POD_EVICTION
|
||||
value: "false"
|
||||
- name: ENABLE_AUTO_DRAIN
|
||||
value: "false"
|
||||
env: []
|
||||
# kernel module configuration for vGPU manager
|
||||
kernelModuleConfig:
|
||||
name: ""
|
||||
hostNetwork: false
|
||||
|
||||
vgpuDeviceManager:
|
||||
enabled: true
|
||||
repository: nvcr.io/nvidia/cloud-native
|
||||
image: vgpu-device-manager
|
||||
version: v0.3.0
|
||||
version: v0.4.2
|
||||
imagePullPolicy: IfNotPresent
|
||||
imagePullSecrets: []
|
||||
env: []
|
||||
config:
|
||||
name: ""
|
||||
default: "default"
|
||||
hostNetwork: false
|
||||
|
||||
vfioManager:
|
||||
enabled: true
|
||||
repository: nvcr.io/nvidia
|
||||
image: cuda
|
||||
version: 12.8.1-base-ubi9
|
||||
repository: nvcr.io/nvidia/cloud-native
|
||||
image: k8s-driver-manager
|
||||
version: v0.10.0
|
||||
imagePullPolicy: IfNotPresent
|
||||
imagePullSecrets: []
|
||||
env: []
|
||||
resources: {}
|
||||
driverManager:
|
||||
image: k8s-driver-manager
|
||||
repository: nvcr.io/nvidia/cloud-native
|
||||
image: k8s-driver-manager
|
||||
# When choosing a different version of k8s-driver-manager, DO NOT downgrade to a version lower than v0.6.4
|
||||
# to ensure k8s-driver-manager stays compatible with gpu-operator starting from v24.3.0
|
||||
version: v0.8.0
|
||||
version: v0.10.0
|
||||
imagePullPolicy: IfNotPresent
|
||||
env:
|
||||
- name: ENABLE_GPU_POD_EVICTION
|
||||
value: "false"
|
||||
- name: ENABLE_AUTO_DRAIN
|
||||
value: "false"
|
||||
env: []
|
||||
hostNetwork: false
|
||||
|
||||
kataManager:
|
||||
enabled: false
|
||||
config:
|
||||
artifactsDir: "/opt/nvidia-gpu-operator/artifacts/runtimeclasses"
|
||||
runtimeClasses:
|
||||
- name: kata-nvidia-gpu
|
||||
nodeSelector: {}
|
||||
artifacts:
|
||||
url: nvcr.io/nvidia/cloud-native/kata-gpu-artifacts:ubuntu22.04-535.54.03
|
||||
pullSecret: ""
|
||||
- name: kata-nvidia-gpu-snp
|
||||
nodeSelector:
|
||||
"nvidia.com/cc.capable": "true"
|
||||
artifacts:
|
||||
url: nvcr.io/nvidia/cloud-native/kata-gpu-artifacts:ubuntu22.04-535.86.10-snp
|
||||
pullSecret: ""
|
||||
repository: nvcr.io/nvidia/cloud-native
|
||||
image: k8s-kata-manager
|
||||
version: v0.2.3
|
||||
config: {}
|
||||
imagePullPolicy: IfNotPresent
|
||||
imagePullSecrets: []
|
||||
env: []
|
||||
resources: {}
|
||||
hostNetwork: false
|
||||
|
||||
sandboxDevicePlugin:
|
||||
enabled: true
|
||||
repository: nvcr.io/nvidia
|
||||
image: kubevirt-gpu-device-plugin
|
||||
version: v1.3.1
|
||||
version: v1.5.0
|
||||
imagePullPolicy: IfNotPresent
|
||||
imagePullSecrets: []
|
||||
args: []
|
||||
env: []
|
||||
resources: {}
|
||||
hostNetwork: false
|
||||
|
||||
ccManager:
|
||||
enabled: false
|
||||
defaultMode: "off"
|
||||
# Kata sandbox device plugin (used when sandboxWorkloads.mode is "kata").
|
||||
kataSandboxDevicePlugin:
|
||||
enabled: true
|
||||
repository: nvcr.io/nvidia/cloud-native
|
||||
image: k8s-cc-manager
|
||||
version: v0.1.1
|
||||
image: nvidia-sandbox-device-plugin
|
||||
version: "v0.0.3"
|
||||
imagePullPolicy: IfNotPresent
|
||||
imagePullSecrets: []
|
||||
env:
|
||||
- name: CC_CAPABLE_DEVICE_IDS
|
||||
value: "0x2339,0x2331,0x2330,0x2324,0x2322,0x233d"
|
||||
args: []
|
||||
env: []
|
||||
resources: {}
|
||||
hostNetwork: false
|
||||
|
||||
ccManager:
|
||||
enabled: true
|
||||
defaultMode: "on"
|
||||
repository: nvcr.io/nvidia/cloud-native
|
||||
image: k8s-cc-manager
|
||||
version: v0.4.0
|
||||
imagePullPolicy: IfNotPresent
|
||||
imagePullSecrets: []
|
||||
resources: {}
|
||||
hostNetwork: false
|
||||
|
||||
# Array of extra K8s manifests to deploy
|
||||
# Supports use of custom Helm templates
|
||||
extraObjects: []
|
||||
|
||||
node-feature-discovery:
|
||||
enableNodeFeatureApi: true
|
||||
priorityClassName: system-node-critical
|
||||
gc:
|
||||
enable: true
|
||||
|
|
@ -571,10 +548,6 @@ node-feature-discovery:
|
|||
# disable creation to avoid duplicate serviceaccount creation by master spec below
|
||||
create: false
|
||||
tolerations:
|
||||
- key: "node-role.kubernetes.io/master"
|
||||
operator: "Equal"
|
||||
value: ""
|
||||
effect: "NoSchedule"
|
||||
- key: "node-role.kubernetes.io/control-plane"
|
||||
operator: "Equal"
|
||||
value: ""
|
||||
|
|
|
|||
16
packages/system/gpu-operator/values-talos-vgpu.yaml
Normal file
16
packages/system/gpu-operator/values-talos-vgpu.yaml
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
gpu-operator:
|
||||
sandboxWorkloads:
|
||||
enabled: true
|
||||
driver:
|
||||
enabled: false
|
||||
devicePlugin:
|
||||
enabled: false
|
||||
vgpuManager:
|
||||
enabled: true
|
||||
repository: ""
|
||||
image: vgpu-manager
|
||||
version: ""
|
||||
vgpuDeviceManager:
|
||||
enabled: true
|
||||
config:
|
||||
default: default
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
apiVersion: v2
|
||||
name: cozy-hami
|
||||
version: 0.0.0 # Placeholder, the actual version will be automatically set during the build process
|
||||
|
|
@ -1,43 +0,0 @@
|
|||
export NAME=hami
|
||||
export NAMESPACE=cozy-$(NAME)
|
||||
|
||||
include ../../../hack/common-envs.mk
|
||||
include ../../../hack/package.mk
|
||||
|
||||
# When bumping the HAMi version, run `make update` and then review
|
||||
# the resulting diff in `charts/hami/`. The recipe below reproduces the
|
||||
# top-level vendoring overrides automatically:
|
||||
#
|
||||
# 1. Removes the broken hami-dra subchart. Upstream's NVIDIA DRA driver
|
||||
# path requires kubelet DRA support that cozystack does not enable
|
||||
# and has no upstream fix tracked. See commit 3c5521e.
|
||||
# 2. Empties Chart.yaml dependencies and drops Chart.lock so Helm does
|
||||
# not try to re-pull hami-dra at build time. See commit 2734dc0.
|
||||
# 3. Strips dra/hami-dra/podSecurityPolicy blocks from the upstream
|
||||
# values.yaml since the corresponding code paths are gone. PSP is
|
||||
# removed from Kubernetes 1.25+ and is unused by cozystack.
|
||||
#
|
||||
# Template-level patches are NOT reproduced automatically:
|
||||
#
|
||||
# * Scheduler templates have `{{- if .Values.dra.enabled }}` blocks
|
||||
# that need to be removed because the dra value is gone (commit
|
||||
# 2734dc0 stripped them).
|
||||
# * device-plugin/monitorservice.yaml uses `indent` with leading
|
||||
# whitespace; it must be rewritten to `nindent` for the labels block
|
||||
# to render correctly when devicePlugin.service.labels is set
|
||||
# (commit 3685254).
|
||||
#
|
||||
# After `make update`, run `git diff -- charts/hami/templates/` and
|
||||
# replay these template patches against the new upstream version, then
|
||||
# verify with `helm unittest`. If upstream restructured the affected
|
||||
# files, the patches may need to be redesigned rather than reapplied.
|
||||
|
||||
update:
|
||||
rm -rf charts
|
||||
helm repo add hami-charts https://project-hami.github.io/HAMi/
|
||||
helm repo update hami-charts
|
||||
helm pull hami-charts/hami --untar --untardir charts
|
||||
rm -rf charts/hami/charts/hami-dra
|
||||
yq --inplace '.dependencies = []' charts/hami/Chart.yaml
|
||||
rm -f charts/hami/Chart.lock
|
||||
yq --inplace 'del(.dra) | del(.["hami-dra"]) | del(.podSecurityPolicy)' charts/hami/values.yaml
|
||||
|
|
@ -1,82 +0,0 @@
|
|||
# HAMi — GPU Virtualization Middleware
|
||||
|
||||
[HAMi](https://github.com/Project-HAMi/HAMi) (Heterogeneous AI Computing Virtualization Middleware) is a CNCF Sandbox project that enables fractional GPU sharing in Kubernetes. It allows workloads to request specific amounts of GPU memory and compute cores instead of claiming entire GPUs.
|
||||
|
||||
## Architecture
|
||||
|
||||
HAMi consists of four components:
|
||||
|
||||
- **MutatingWebhook** — intercepts pod creation, injects `schedulerName: hami-scheduler`
|
||||
- **Scheduler Extender** — extends kube-scheduler with GPU-aware Filter and Bind logic
|
||||
- **Device Plugin** (DaemonSet) — registers vGPU resources via the Kubernetes Device Plugin API
|
||||
- **HAMi-core** (`libvgpu.so`) — `LD_PRELOAD` library injected into workload containers, intercepts CUDA API calls to enforce memory and compute isolation
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- GPU Operator must be enabled (`addons.gpuOperator.enabled: true`)
|
||||
- NVIDIA driver >= 440 on host nodes
|
||||
- nvidia-container-toolkit configured as the default container runtime
|
||||
- GPU nodes labeled with `gpu=on`
|
||||
|
||||
## Known Limitations
|
||||
|
||||
### glibc < 2.34 requirement for workload containers
|
||||
|
||||
HAMi-core uses `LD_PRELOAD` to intercept `dlsym()` for CUDA symbol resolution. The fallback code path relies on `_dl_sym`, a private glibc internal symbol that was removed in glibc 2.34 when libdl and libpthread were merged into libc.so.
|
||||
|
||||
**This limitation affects workload containers only**, not the host OS or HAMi's own components.
|
||||
|
||||
| Distribution | glibc | Result |
|
||||
| --------------- | ----- | -------------------------------------------- |
|
||||
| Ubuntu 18.04 | 2.27 | Full isolation (memory + compute) |
|
||||
| Ubuntu 20.04 | 2.31 | Full isolation (memory + compute) |
|
||||
| Ubuntu 22.04 | 2.35 | Memory isolation works, compute breaks |
|
||||
| Ubuntu 24.04 | 2.39 | Both memory and compute isolation break |
|
||||
| Alpine (musl) | N/A | Completely incompatible (`dlvsym` absent) |
|
||||
|
||||
Most modern ML/AI base images (CUDA 12.x, PyTorch 2.x, TensorFlow 2.x) use Ubuntu 22.04+ with glibc >= 2.35, which means compute isolation will not work with these images until the upstream fix is merged.
|
||||
|
||||
**Upstream tracking issues:**
|
||||
|
||||
- [HAMi-core#174](https://github.com/Project-HAMi/HAMi-core/issues/174) — `_dl_sym` removal in glibc 2.34 breaks HAMi-core's CUDA symbol resolution at the symbol level
|
||||
- [HAMi#1190](https://github.com/Project-HAMi/HAMi/issues/1190) — maintainer thread confirming the empirical per-glibc-version isolation behavior shown in the table above
|
||||
|
||||
### musl libc (Alpine) incompatibility
|
||||
|
||||
HAMi-core is completely incompatible with musl libc. The `dlvsym()` function used by HAMi-core is a glibc extension not available in musl. Only glibc-based container images (Debian, Ubuntu, RHEL, etc.) can use HAMi GPU isolation.
|
||||
|
||||
## Usage
|
||||
|
||||
Enable HAMi in your tenant Kubernetes cluster values:
|
||||
|
||||
```yaml
|
||||
addons:
|
||||
gpuOperator:
|
||||
enabled: true
|
||||
hami:
|
||||
enabled: true
|
||||
```
|
||||
|
||||
When HAMi is enabled, GPU Operator's built-in device plugin is automatically disabled to avoid conflicts. This default is preserved by setting `addons.gpuOperator.valuesOverride.gpu-operator.devicePlugin.enabled: false`; advanced topologies that partition GPU pools (e.g. some nodes use HAMi while others run the standard NVIDIA device plugin via node selectors) can re-enable it explicitly through `valuesOverride`.
|
||||
|
||||
### Requesting fractional GPU resources
|
||||
|
||||
```yaml
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 1
|
||||
nvidia.com/gpumem: 3000 # 3000 MB of GPU memory
|
||||
nvidia.com/gpucores: 30 # 30% of GPU compute cores
|
||||
```
|
||||
|
||||
## Parameters
|
||||
|
||||
Default values shown below are inherited from the upstream HAMi chart and may change with upstream updates.
|
||||
|
||||
| Name | Description | Default |
|
||||
| --- | --- | --- |
|
||||
| `hami.devicePlugin.runtimeClassName` | RuntimeClass for device plugin pods | `nvidia` |
|
||||
| `hami.devicePlugin.deviceSplitCount` | Max virtual GPUs per physical GPU | `10` |
|
||||
| `hami.devicePlugin.deviceMemoryScaling` | Memory overcommit factor (> 1.0 enables overcommit) | `1` |
|
||||
| `hami.scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy` | Node packing strategy (`binpack` or `spread`) | `binpack` |
|
||||
| `hami.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy` | GPU packing strategy (`binpack` or `spread`) | `spread` |
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
apiVersion: v2
|
||||
appVersion: 2.8.1
|
||||
dependencies: []
|
||||
description: Heterogeneous AI Computing Virtualization Middleware
|
||||
keywords:
|
||||
- vgpu
|
||||
- gpu
|
||||
kubeVersion: '>= 1.18.0-0'
|
||||
maintainers:
|
||||
- email: archlitchi@gmail.com
|
||||
name: limengxuan
|
||||
- email: xiaozhang0210@hotmail.com
|
||||
name: zhangxiao
|
||||
name: hami
|
||||
sources:
|
||||
- https://github.com/Project-HAMi/HAMi
|
||||
type: application
|
||||
version: 2.8.1
|
||||
|
|
@ -1,237 +0,0 @@
|
|||
# HAMi Helm Chart Values Documentation
|
||||
|
||||
This document provides detailed descriptions of all configurable values parameters for the HAMi Helm Chart.
|
||||
|
||||
## Global Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `global.imageRegistry` | Global Docker image registry | `""` |
|
||||
| `global.imagePullSecrets` | Global Docker image pull secrets | `[]` |
|
||||
| `global.imageTag` | Image tag | `"v2.8.1"` |
|
||||
| `global.gpuHookPath` | GPU Hook path | `/usr/local` |
|
||||
| `global.labels` | Global labels | `{}` |
|
||||
| `global.annotations` | Global annotations | `{}` |
|
||||
| `global.managedNodeSelectorEnable` | Whether to enable managed node selector | `false` |
|
||||
| `global.managedNodeSelector.usage` | Managed node selector usage | `"gpu"` |
|
||||
| `nameOverride` | Name override | `""` |
|
||||
| `fullnameOverride` | Full name override | `""` |
|
||||
| `namespaceOverride` | Namespace override | `""` |
|
||||
|
||||
## Resource Name Configuration
|
||||
|
||||
### NVIDIA GPU Resources
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `resourceName` | GPU resource name | `"nvidia.com/gpu"` |
|
||||
| `resourceMem` | GPU memory resource name | `"nvidia.com/gpumem"` |
|
||||
| `resourceMemPercentage` | GPU memory percentage resource name | `"nvidia.com/gpumem-percentage"` |
|
||||
| `resourceCores` | GPU core resource name | `"nvidia.com/gpucores"` |
|
||||
| `resourcePriority` | GPU priority resource name | `"nvidia.com/priority"` |
|
||||
|
||||
### Cambricon MLU Resources
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `mluResourceName` | MLU resource name | `"cambricon.com/vmlu"` |
|
||||
| `mluResourceMem` | MLU memory resource name | `"cambricon.com/mlu.smlu.vmemory"` |
|
||||
| `mluResourceCores` | MLU core resource name | `"cambricon.com/mlu.smlu.vcore"` |
|
||||
|
||||
### Hygon DCU Resources
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `dcuResourceName` | DCU resource name | `"hygon.com/dcunum"` |
|
||||
| `dcuResourceMem` | DCU memory resource name | `"hygon.com/dcumem"` |
|
||||
| `dcuResourceCores` | DCU core resource name | `"hygon.com/dcucores"` |
|
||||
|
||||
### Metax GPU Resources
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `metaxResourceName` | GPU resource name | `"metax-tech.com/sgpu"` |
|
||||
| `metaxResourceCore` | GPU core resource name | `"metax-tech.com/vcore"` |
|
||||
| `metaxResourceMem` | GPU memory resource name | `"metax-tech.com/vmemory"` |
|
||||
| `metaxsGPUTopologyAware` | GPU topology awareness | `"false"` |
|
||||
|
||||
### Enflame GCU Resources
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `enflameResourceNameVGCU` | vGCU resource name | `"enflame.com/vgcu"` |
|
||||
| `enflameResourceNameVGCUPercentage` | vGCU percentage resource name | `"enflame.com/vgcu-percentage"` |
|
||||
|
||||
### Kunlunxin XPU Resources
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `kunlunResourceName` | XPU resource name | `"kunlunxin.com/xpu"` |
|
||||
|
||||
## Scheduler Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `schedulerName` | Scheduler name | `"hami-scheduler"` |
|
||||
| `scheduler.nodeName` | Define node name, scheduler will schedule to this node | `""` |
|
||||
| `scheduler.overwriteEnv` | Whether to overwrite environment variables | `"false"` |
|
||||
| `scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy` | Node scheduler policy | `binpack` |
|
||||
| `scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy` | GPU scheduler policy | `spread` |
|
||||
| `scheduler.metricsBindAddress` | Metrics bind address | `":9395"` |
|
||||
| `scheduler.forceOverwriteDefaultScheduler` | Whether to force overwrite default scheduler | `true` |
|
||||
| `scheduler.livenessProbe` | Whether to enable liveness probe | `false` |
|
||||
| `scheduler.leaderElect` | Whether to enable leader election | `true` |
|
||||
| `scheduler.replicas` | Number of replicas | `1` |
|
||||
|
||||
### Kube Scheduler Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `scheduler.kubeScheduler.enabled` | Whether to run kube-scheduler container in scheduler pod | `true` |
|
||||
| `scheduler.kubeScheduler.image.registry` | Kube scheduler image registry | `"registry.cn-hangzhou.aliyuncs.com"` |
|
||||
| `scheduler.kubeScheduler.image.repository` | Kube scheduler image repository | `"google_containers/kube-scheduler"` |
|
||||
| `scheduler.kubeScheduler.image.tag` | Kube scheduler image tag | `""` |
|
||||
| `scheduler.kubeScheduler.image.pullPolicy` | Kube scheduler image pull policy | `IfNotPresent` |
|
||||
| `scheduler.kubeScheduler.image.pullSecrets` | Kube scheduler image pull secrets | `[]` |
|
||||
| `scheduler.kubeScheduler.extraNewArgs` | Extra new arguments | `["--config=/config/config.yaml", "-v=4"]` |
|
||||
| `scheduler.kubeScheduler.extraArgs` | Extra arguments | `["--policy-config-file=/config/config.json", "-v=4"]` |
|
||||
|
||||
### Extender Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `scheduler.extender.image.registry` | Scheduler extender image registry | `"docker.io"` |
|
||||
| `scheduler.extender.image.repository` | Scheduler extender image repository | `"projecthami/hami"` |
|
||||
| `scheduler.extender.image.tag` | Scheduler extender image tag | `""` |
|
||||
| `scheduler.extender.image.pullPolicy` | Scheduler extender image pull policy | `IfNotPresent` |
|
||||
| `scheduler.extender.image.pullSecrets` | Scheduler extender image pull secrets | `[]` |
|
||||
| `scheduler.extender.extraArgs` | Scheduler extender extra arguments | `["--debug", "-v=4"]` |
|
||||
|
||||
### Admission Webhook Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `scheduler.admissionWebhook.enabled` | Whether to enable admission webhook | `true` |
|
||||
| `scheduler.admissionWebhook.customURL.enabled` | Whether to enable custom URL | `false` |
|
||||
| `scheduler.admissionWebhook.customURL.host` | Custom URL host | `127.0.0.1` |
|
||||
| `scheduler.admissionWebhook.customURL.port` | Custom URL port | `31998` |
|
||||
| `scheduler.admissionWebhook.customURL.path` | Custom URL path | `/webhook` |
|
||||
| `scheduler.admissionWebhook.reinvocationPolicy` | Reinvocation policy | `Never` |
|
||||
| `scheduler.admissionWebhook.failurePolicy` | Failure policy | `Ignore` |
|
||||
|
||||
### TLS Certificate Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `scheduler.certManager.enabled` | Whether to use cert-manager to generate self-signed certificates | `false` |
|
||||
| `scheduler.patch.enabled` | Whether to use kube-webhook-certgen to generate self-signed certificates | `true` |
|
||||
| `scheduler.patch.image.registry` | Certgen image registry | `"docker.io"` |
|
||||
| `scheduler.patch.image.repository` | Certgen image repository | `"jettech/kube-webhook-certgen"` |
|
||||
| `scheduler.patch.image.tag` | Certgen image tag | `"v1.5.2"` |
|
||||
| `scheduler.patch.image.pullPolicy` | Certgen image pull policy | `IfNotPresent` |
|
||||
| `scheduler.patch.imageNew.registry` | New certgen image registry | `"docker.io"` |
|
||||
| `scheduler.patch.imageNew.repository` | New certgen image repository | `"liangjw/kube-webhook-certgen"` |
|
||||
| `scheduler.patch.imageNew.tag` | New certgen image tag | `"v1.1.1"` |
|
||||
|
||||
### Scheduler Service Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `scheduler.service.type` | Service type | `NodePort` |
|
||||
| `scheduler.service.httpPort` | HTTP port | `443` |
|
||||
| `scheduler.service.schedulerPort` | Scheduler NodePort | `31998` |
|
||||
| `scheduler.service.monitorPort` | Monitor port | `31993` |
|
||||
| `scheduler.service.monitorTargetPort` | Monitor target port | `9395` |
|
||||
|
||||
## Device Plugin Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `devicePlugin.image.registry` | Device plugin image registry | `"docker.io"` |
|
||||
| `devicePlugin.image.repository` | Device plugin image repository | `"projecthami/hami"` |
|
||||
| `devicePlugin.image.tag` | Device plugin image tag | `""` |
|
||||
| `devicePlugin.image.pullPolicy` | Device plugin image pull policy | `IfNotPresent` |
|
||||
| `devicePlugin.image.pullSecrets` | Device plugin image pull secrets | `[]` |
|
||||
|
||||
### Monitor Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `devicePlugin.monitor.image.registry` | Monitor image registry | `"docker.io"` |
|
||||
| `devicePlugin.monitor.image.repository` | Monitor image repository | `"projecthami/hami"` |
|
||||
| `devicePlugin.monitor.image.tag` | Monitor image tag | `""` |
|
||||
| `devicePlugin.monitor.image.pullPolicy` | Monitor image pull policy | `IfNotPresent` |
|
||||
| `devicePlugin.monitor.image.pullSecrets` | Monitor image pull secrets | `[]` |
|
||||
| `devicePlugin.monitor.ctrPath` | Container path | `/usr/local/vgpu/containers` |
|
||||
| `devicePlugin.monitor.extraArgs` | Monitor extra arguments | `["-v=4"]` |
|
||||
| `devicePlugin.monitor.extraEnvs` | Monitor extra environments | `{}` |
|
||||
|
||||
### Device Plugin Other Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `devicePlugin.deviceSplitCount` | Integer type, default value: 10. Maximum number of tasks assigned to a single GPU device | `10` |
|
||||
| `devicePlugin.deviceMemoryScaling` | Device memory scaling ratio | `1` |
|
||||
| `devicePlugin.deviceCoreScaling` | Device core scaling ratio | `1` |
|
||||
| `devicePlugin.runtimeClassName` | Runtime class name | `""` |
|
||||
| `devicePlugin.createRuntimeClass` | Whether to create runtime class | `false` |
|
||||
| `devicePlugin.migStrategy` | String type, "none" means ignore MIG functionality, "mixed" means allocate MIG devices through independent resources | `"none"` |
|
||||
| `devicePlugin.disablecorelimit` | String type, "true" means disable core limit, "false" means enable core limit | `"false"` |
|
||||
| `devicePlugin.passDeviceSpecsEnabled` | Whether to enable passing device specs | `false` |
|
||||
| `devicePlugin.extraArgs` | Device plugin extra arguments | `["-v=4"]` |
|
||||
| `devicePlugin.nodeConfiguration.config` | Node configuration for device plugin by json | An example of default configuration. |
|
||||
| `devicePlugin.nodeConfiguration.externalConfigName` | Node configuration for device plugin by external congimap | `""` |
|
||||
| `devicePlugin.extraEnvs` | Device plugin extra environments | `{}` |
|
||||
|
||||
### Device Plugin Service Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `devicePlugin.service.type` | Service type | `NodePort` |
|
||||
| `devicePlugin.service.httpPort` | HTTP port | `31992` |
|
||||
|
||||
### Device Plugin Deployment Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `devicePlugin.pluginPath` | Plugin path | `/var/lib/kubelet/device-plugins` |
|
||||
| `devicePlugin.libPath` | Library path | `/usr/local/vgpu` |
|
||||
| `devicePlugin.nvidiaNodeSelector` | NVIDIA node selector | `{"gpu": "on"}` |
|
||||
| `devicePlugin.updateStrategy.type` | Update strategy type | `RollingUpdate` |
|
||||
| `devicePlugin.updateStrategy.rollingUpdate.maxUnavailable` | Maximum unavailable count | `1` |
|
||||
|
||||
## Device Configuration
|
||||
|
||||
### AWS Neuron
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `devices.awsneuron.customresources` | Custom resources | `["aws.amazon.com/neuron", "aws.amazon.com/neuroncore"]` |
|
||||
|
||||
### Kunlunxin
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `devices.kunlun.enabled` | Whether to enable | `true` |
|
||||
| `devices.kunlun.customresources` | Custom resources | `["kunlunxin.com/xpu"]` |
|
||||
|
||||
### Mthreads
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `devices.mthreads.enabled` | Whether to enable | `true` |
|
||||
| `devices.mthreads.customresources` | Custom resources | `["mthreads.com/vgpu"]` |
|
||||
|
||||
### NVIDIA
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `devices.nvidia.gpuCorePolicy` | GPU core policy | `default` |
|
||||
| `devices.nvidia.libCudaLogLevel` | CUDA library log level | `1` |
|
||||
|
||||
### Huawei Ascend
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `devices.ascend.enabled` | Whether to enable | `false` |
|
||||
| `devices.ascend.image` | Image | `""` |
|
||||
| `devices.ascend.imagePullPolicy` | Image pull policy | `IfNotPresent` |
|
||||
| `devices.ascend.extraArgs` | Extra arguments | `[]` |
|
||||
| `devices.ascend.nodeSelector` | Node selector | `{"ascend": "on"}` |
|
||||
| `devices.ascend.tolerations` | Tolerations | `[]` |
|
||||
| `devices.ascend.customresources` | Custom resources | `["huawei.com/Ascend910A", "huawei.com/Ascend910A-memory", ...]` |
|
||||
|
||||
### Iluvatar
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `devices.iluvatar.enabled` | Whether to enable | `false` |
|
||||
| `devices.iluvatar.customresources` | Custom resources | `["iluvatar.ai/BI-V150-vgpu", "iluvatar.ai/BI-V150.vMem","iluvatar.ai/BI-V150.vCore", ...]` |
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
** Please be patient while the chart is being deployed **
|
||||
Resource name: {{ .Values.resourceName }}
|
||||
|
||||
|
|
@ -1,49 +0,0 @@
|
|||
{{/*
|
||||
Return the proper image name
|
||||
{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" $) }}
|
||||
*/}}
|
||||
{{- define "common.images.image" -}}
|
||||
{{- $registryName := .imageRoot.registry -}}
|
||||
{{- $repositoryName := .imageRoot.repository -}}
|
||||
{{- $tag := .imageRoot.tag | toString -}}
|
||||
{{- if .global }}
|
||||
{{- if .global.imageRegistry }}
|
||||
{{- $registryName = .global.imageRegistry -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- if .tag }}
|
||||
{{- $tag = .tag | toString -}}
|
||||
{{- end -}}
|
||||
{{- if $registryName }}
|
||||
{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}}
|
||||
{{- else -}}
|
||||
{{- printf "%s:%s" $repositoryName $tag -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead)
|
||||
{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }}
|
||||
*/}}
|
||||
{{- define "common.images.pullSecrets" -}}
|
||||
{{- $pullSecrets := list }}
|
||||
|
||||
{{- if .global }}
|
||||
{{- range .global.imagePullSecrets -}}
|
||||
{{- $pullSecrets = append $pullSecrets . -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- range .images -}}
|
||||
{{- range .pullSecrets -}}
|
||||
{{- $pullSecrets = append $pullSecrets . -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- if (not (empty $pullSecrets)) }}
|
||||
imagePullSecrets:
|
||||
{{- range $pullSecrets }}
|
||||
- name: {{ . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
|
|
@ -1,163 +0,0 @@
|
|||
{{/*
|
||||
Expand the name of the chart.
|
||||
*/}}
|
||||
{{- define "hami-vgpu.name" -}}
|
||||
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create a default fully qualified app name.
|
||||
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
|
||||
If release name contains chart name it will be used as a full name.
|
||||
*/}}
|
||||
{{- define "hami-vgpu.fullname" -}}
|
||||
{{- if .Values.fullnameOverride -}}
|
||||
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
|
||||
{{- else }}
|
||||
{{- $name := default .Chart.Name .Values.nameOverride -}}
|
||||
{{- if contains $name .Release.Name }}
|
||||
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
|
||||
{{- else -}}
|
||||
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Allow the release namespace to be overridden for multi-namespace deployments in combined charts
|
||||
*/}}
|
||||
{{- define "hami-vgpu.namespace" -}}
|
||||
{{- if .Values.namespaceOverride -}}
|
||||
{{- .Values.namespaceOverride -}}
|
||||
{{- else -}}
|
||||
{{- .Release.Namespace -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
The app name for Scheduler
|
||||
*/}}
|
||||
{{- define "hami-vgpu.scheduler" -}}
|
||||
{{- printf "%s-scheduler" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
The app name for DevicePlugin
|
||||
*/}}
|
||||
{{- define "hami-vgpu.device-plugin" -}}
|
||||
{{- printf "%s-device-plugin" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
The app name for MockDevicePlugin
|
||||
*/}}
|
||||
{{- define "hami-vgpu.mock-device-plugin" -}}
|
||||
{{- printf "%s-mock-device-plugin" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
The tls secret name for Scheduler
|
||||
*/}}
|
||||
{{- define "hami-vgpu.scheduler.tls" -}}
|
||||
{{- printf "%s-scheduler-tls" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
The webhook name
|
||||
*/}}
|
||||
{{- define "hami-vgpu.scheduler.webhook" -}}
|
||||
{{- printf "%s-webhook" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create chart name and version as used by the chart label.
|
||||
*/}}
|
||||
{{- define "hami-vgpu.chart" -}}
|
||||
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Common labels
|
||||
*/}}
|
||||
{{- define "hami-vgpu.labels" -}}
|
||||
helm.sh/chart: {{ include "hami-vgpu.chart" . }}
|
||||
{{ include "hami-vgpu.selectorLabels" . }}
|
||||
{{- if .Chart.AppVersion }}
|
||||
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
|
||||
{{- end }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Selector labels
|
||||
*/}}
|
||||
{{- define "hami-vgpu.selectorLabels" -}}
|
||||
app.kubernetes.io/name: {{ include "hami-vgpu.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
{{- end }}
|
||||
|
||||
|
||||
{{/*
|
||||
Resolve the tag for kubeScheduler.
|
||||
*/}}
|
||||
{{- define "resolvedKubeSchedulerTag" -}}
|
||||
{{- if .Values.scheduler.kubeScheduler.image.tag }}
|
||||
{{- .Values.scheduler.kubeScheduler.image.tag | trim -}}
|
||||
{{- else }}
|
||||
{{- include "strippedKubeVersion" . | trim -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Return the stripped Kubernetes version string by removing extra parts after semantic version number.
|
||||
v1.31.1+k3s1 -> v1.31.1
|
||||
v1.30.8-eks-2d5f260 -> v1.30.8
|
||||
v1.31.1 -> v1.31.1
|
||||
*/}}
|
||||
{{- define "strippedKubeVersion" -}}
|
||||
{{ regexReplaceAll "^(v[0-9]+\\.[0-9]+\\.[0-9]+)(.*)$" .Capabilities.KubeVersion.Version "$1" }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.scheduler.kubeScheduler.image" -}}
|
||||
{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.kubeScheduler.image "global" .Values.global "tag" (include "resolvedKubeSchedulerTag" .)) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.scheduler.extender.image" -}}
|
||||
{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.extender.image "global" .Values.global "tag" .Values.global.imageTag) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.devicePlugin.image" -}}
|
||||
{{ include "common.images.image" (dict "imageRoot" .Values.devicePlugin.image "global" .Values.global "tag" .Values.global.imageTag) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.mockDevicePlugin.image" -}}
|
||||
{{ include "common.images.image" (dict "imageRoot" .Values.mockDevicePlugin.image "global" .Values.global "tag" .Values.mockDevicePlugin.tag) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.devicePlugin.monitor.image" -}}
|
||||
{{ include "common.images.image" (dict "imageRoot" .Values.devicePlugin.monitor.image "global" .Values.global "tag" .Values.global.imageTag) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.scheduler.patch.image" -}}
|
||||
{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.patch.image "global" .Values.global) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.scheduler.patch.new.image" -}}
|
||||
{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.patch.imageNew "global" .Values.global) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.scheduler.extender.imagePullSecrets" -}}
|
||||
{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.extender.image) "global" .Values.global) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.devicePlugin.imagePullSecrets" -}}
|
||||
{{ include "common.images.pullSecrets" (dict "images" (list .Values.devicePlugin.image) "global" .Values.global) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.scheduler.patch.imagePullSecrets" -}}
|
||||
{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.patch.image) "global" .Values.global) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.scheduler.patch.new.imagePullSecrets" -}}
|
||||
{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.patch.imageNew) "global" .Values.global) }}
|
||||
{{- end -}}
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
{{- if and .Values.devicePlugin.enabled (not .Values.devicePlugin.nodeConfiguration.externalConfigName) -}}
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.device-plugin" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-device-plugin
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
data:
|
||||
config.json: |
|
||||
{{- .Values.devicePlugin.nodeConfiguration.config | nindent 4 }}
|
||||
{{- end }}
|
||||
|
|
@ -1,55 +0,0 @@
|
|||
{{- if .Values.mockDevicePlugin.enabled }}
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.mock-device-plugin" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/component: hami-mock-device-plugin
|
||||
{{- include "hami-vgpu.selectorLabels" . | nindent 6 }}
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
scheduler.alpha.kubernetes.io/critical-pod: ""
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-mock-device-plugin
|
||||
{{- include "hami-vgpu.selectorLabels" . | nindent 8 }}
|
||||
spec:
|
||||
serviceAccountName: {{ include "hami-vgpu.mock-device-plugin" . }}
|
||||
tolerations:
|
||||
- key: CriticalAddonsOnly
|
||||
operator: Exists
|
||||
containers:
|
||||
- image: {{ include "hami.mockDevicePlugin.image" . }}
|
||||
imagePullPolicy: {{ .Values.mockDevicePlugin.image.pullPolicy }}
|
||||
name: hami-mock-dp-cntr
|
||||
env:
|
||||
- name: NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
command:
|
||||
- ./k8s-device-plugin
|
||||
- -v=5
|
||||
- --device-config-file=/device-config.yaml
|
||||
volumeMounts:
|
||||
- name: dp
|
||||
mountPath: /var/lib/kubelet/device-plugins
|
||||
- name: sys
|
||||
mountPath: /sys
|
||||
- name: device-config
|
||||
mountPath: /device-config.yaml
|
||||
subPath: device-config.yaml
|
||||
volumes:
|
||||
- name: dp
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/device-plugins
|
||||
- name: sys
|
||||
hostPath:
|
||||
path: /sys
|
||||
- name: device-config
|
||||
configMap:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}-device
|
||||
{{- end -}}
|
||||
|
|
@ -1,262 +0,0 @@
|
|||
{{- if .Values.devicePlugin.enabled }}
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.device-plugin" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-device-plugin
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
{{- with .Values.global.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.annotations }}
|
||||
annotations: {{ toYaml .Values.global.annotations | nindent 4}}
|
||||
{{- end }}
|
||||
spec:
|
||||
updateStrategy:
|
||||
{{- with .Values.devicePlugin.updateStrategy }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/component: hami-device-plugin
|
||||
{{- include "hami-vgpu.selectorLabels" . | nindent 6 }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-device-plugin
|
||||
hami.io/webhook: ignore
|
||||
{{- include "hami-vgpu.selectorLabels" . | nindent 8 }}
|
||||
annotations:
|
||||
{{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
|
||||
checksum/hami-scheduler-newversion-config: {{ include (print $.Template.BasePath "/scheduler/configmapnew.yaml") . | sha256sum }}
|
||||
{{- else }}
|
||||
checksum/hami-scheduler-config: {{ include (print $.Template.BasePath "/scheduler/configmap.yaml") . | sha256sum }}
|
||||
{{- end }}
|
||||
checksum/hami-scheduler-device-config: {{ include (print $.Template.BasePath "/scheduler/device-configmap.yaml") . | sha256sum }}
|
||||
{{- if .Values.devicePlugin.podAnnotations }}
|
||||
{{- toYaml .Values.devicePlugin.podAnnotations | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if .Values.devicePlugin.runtimeClassName }}
|
||||
runtimeClassName: {{ .Values.devicePlugin.runtimeClassName }}
|
||||
{{- end }}
|
||||
serviceAccountName: {{ include "hami-vgpu.device-plugin" . }}
|
||||
priorityClassName: system-node-critical
|
||||
hostPID: true
|
||||
hostNetwork: true
|
||||
{{- include "hami.devicePlugin.imagePullSecrets" . | nindent 6 }}
|
||||
{{- if .Values.devicePlugin.gpuOperatorToolkitReady.enabled }}
|
||||
initContainers:
|
||||
- name: toolkit-validation
|
||||
image: {{ include "hami.devicePlugin.image" . }}
|
||||
imagePullPolicy: {{ .Values.devicePlugin.image.pullPolicy }}
|
||||
securityContext:
|
||||
privileged: true
|
||||
runAsUser: 0
|
||||
command: ["sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
echo "Waiting for NVIDIA Toolkit to be ready..."
|
||||
until [ -f {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }}/toolkit-ready ]; do
|
||||
echo "Waiting for {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }}/toolkit-ready..."
|
||||
sleep 5
|
||||
done
|
||||
echo "NVIDIA Toolkit is ready!"
|
||||
volumeMounts:
|
||||
- name: nvidia-validations
|
||||
mountPath: {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath | quote }}
|
||||
mountPropagation: HostToContainer
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: device-plugin
|
||||
image: {{ include "hami.devicePlugin.image" . }}
|
||||
imagePullPolicy: {{ .Values.devicePlugin.image.pullPolicy }}
|
||||
lifecycle:
|
||||
postStart:
|
||||
exec:
|
||||
command: ["/bin/sh","-c", {{ printf "/k8s-vgpu/bin/vgpu-init.sh %s/vgpu/" .Values.global.gpuHookPath | quote }}]
|
||||
command:
|
||||
- nvidia-device-plugin
|
||||
- --config-file=/device-config.yaml
|
||||
- --mig-strategy={{ .Values.devicePlugin.migStrategy }}
|
||||
- --disable-core-limit={{ .Values.devicePlugin.disablecorelimit }}
|
||||
{{- range .Values.devicePlugin.extraArgs }}
|
||||
- {{ . }}
|
||||
{{- end }}
|
||||
env:
|
||||
- name: NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
- name: NVIDIA_MIG_MONITOR_DEVICES
|
||||
value: all
|
||||
- name: DEVICE_LIST_STRATEGY
|
||||
value: {{ .Values.devicePlugin.deviceListStrategy }}
|
||||
- name: HOOK_PATH
|
||||
value: {{ .Values.global.gpuHookPath }}
|
||||
{{- if typeIs "bool" .Values.devicePlugin.passDeviceSpecsEnabled }}
|
||||
- name: PASS_DEVICE_SPECS
|
||||
value: {{ .Values.devicePlugin.passDeviceSpecsEnabled | quote }}
|
||||
{{- end }}
|
||||
{{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }}
|
||||
- name: NVIDIA_DRIVER_ROOT
|
||||
value: {{ .Values.devicePlugin.nvidiaDriverRoot }}
|
||||
{{- end }}
|
||||
{{- if typeIs "string" .Values.devicePlugin.nvidiaHookPath }}
|
||||
- name: NVIDIA_CDI_HOOK_PATH
|
||||
value: {{ .Values.devicePlugin.nvidiaHookPath }}
|
||||
{{- end }}
|
||||
{{- if typeIs "bool" .Values.devicePlugin.gdrcopyEnabled }}
|
||||
- name: GDRCOPY_ENABLED
|
||||
value: {{ .Values.devicePlugin.gdrcopyEnabled | quote }}
|
||||
{{- end }}
|
||||
{{- if typeIs "bool" .Values.devicePlugin.gdsEnabled }}
|
||||
- name: GDS_ENABLED
|
||||
value: {{ .Values.devicePlugin.gdsEnabled | quote }}
|
||||
{{- end }}
|
||||
{{- if typeIs "bool" .Values.devicePlugin.mofedEnabled }}
|
||||
- name: MOFED_ENABLED
|
||||
value: {{ .Values.devicePlugin.mofedEnabled | quote }}
|
||||
{{- end }}
|
||||
{{- if eq (.Values.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy | default "spread") "topology-aware" }}
|
||||
- name: ENABLE_TOPOLOGY_SCORE
|
||||
value: "true"
|
||||
{{- end }}
|
||||
{{- with .Values.devicePlugin.extraEnvs }}
|
||||
{{- . | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
securityContext:
|
||||
privileged: true
|
||||
allowPrivilegeEscalation: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
add: ["SYS_ADMIN"]
|
||||
resources:
|
||||
{{- toYaml .Values.devicePlugin.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
- name: device-plugin
|
||||
mountPath: /var/lib/kubelet/device-plugins
|
||||
- name: lib
|
||||
mountPath: {{ printf "%s%s" .Values.global.gpuHookPath "/vgpu" }}
|
||||
- name: usrbin
|
||||
mountPath: /usrbin
|
||||
- name: deviceconfig
|
||||
mountPath: /config
|
||||
- name: hosttmp
|
||||
mountPath: /tmp
|
||||
- name: device-config
|
||||
mountPath: /device-config.yaml
|
||||
subPath: device-config.yaml
|
||||
- name: cdi-root
|
||||
mountPath: /var/run/cdi
|
||||
{{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }}
|
||||
# We always mount the driver root at /driver-root in the container.
|
||||
# This is required for CDI detection to work correctly.
|
||||
- name: driver-root
|
||||
mountPath: /driver-root
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
- name: vgpu-monitor
|
||||
image: {{ include "hami.devicePlugin.monitor.image" . }}
|
||||
imagePullPolicy: {{ .Values.devicePlugin.monitor.image.pullPolicy }}
|
||||
command:
|
||||
- "vGPUmonitor"
|
||||
{{- range .Values.devicePlugin.monitor.extraArgs }}
|
||||
- {{ . }}
|
||||
{{- end }}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
add: ["SYS_ADMIN"]
|
||||
env:
|
||||
- name: NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
- name: NVIDIA_VISIBLE_DEVICES
|
||||
value: "all"
|
||||
- name: NVIDIA_MIG_MONITOR_DEVICES
|
||||
value: "all"
|
||||
- name: HOOK_PATH
|
||||
value: "{{ .Values.global.gpuHookPath }}/vgpu"
|
||||
- name: HAMI_RESYNC_INTERVAL
|
||||
value: {{ .Values.devicePlugin.monitor.resyncInterval | default "5m" | quote }}
|
||||
{{- with .Values.devicePlugin.monitor.extraEnvs }}
|
||||
{{- . | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
resources:
|
||||
{{- toYaml .Values.devicePlugin.monitor.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
- name: ctrs
|
||||
mountPath: {{ .Values.devicePlugin.monitor.ctrPath }}
|
||||
- name: dockers
|
||||
mountPath: /run/docker
|
||||
- name: containerds
|
||||
mountPath: /run/containerd
|
||||
- name: sysinfo
|
||||
mountPath: /sysinfo
|
||||
- name: hostvar
|
||||
mountPath: /hostvar
|
||||
- name: hosttmp
|
||||
mountPath: /tmp
|
||||
volumes:
|
||||
- name: ctrs
|
||||
hostPath:
|
||||
path: {{ .Values.devicePlugin.monitor.ctrPath }}
|
||||
- name: hosttmp
|
||||
hostPath:
|
||||
path: /tmp
|
||||
- name: dockers
|
||||
hostPath:
|
||||
path: /run/docker
|
||||
- name: containerds
|
||||
hostPath:
|
||||
path: /run/containerd
|
||||
- name: device-plugin
|
||||
hostPath:
|
||||
path: {{ .Values.devicePlugin.pluginPath }}
|
||||
- name: lib
|
||||
hostPath:
|
||||
path: {{ .Values.devicePlugin.libPath }}
|
||||
{{- if .Values.devicePlugin.gpuOperatorToolkitReady.enabled }}
|
||||
- name: nvidia-validations
|
||||
hostPath:
|
||||
path: {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }}
|
||||
type: DirectoryOrCreate
|
||||
{{- end }}
|
||||
{{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }}
|
||||
- name: driver-root
|
||||
hostPath:
|
||||
path: {{ .Values.devicePlugin.nvidiaDriverRoot }}
|
||||
type: Directory
|
||||
{{- end }}
|
||||
- name: cdi-root
|
||||
hostPath:
|
||||
path: /var/run/cdi
|
||||
type: DirectoryOrCreate
|
||||
- name: usrbin
|
||||
hostPath:
|
||||
path: /usr/bin
|
||||
- name: sysinfo
|
||||
hostPath:
|
||||
path: /sys
|
||||
- name: hostvar
|
||||
hostPath:
|
||||
path: /var
|
||||
- name: deviceconfig
|
||||
configMap:
|
||||
name: {{ .Values.devicePlugin.nodeConfiguration.externalConfigName | default (include "hami-vgpu.device-plugin" .) }}
|
||||
- name: device-config
|
||||
configMap:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}-device
|
||||
{{- if .Values.devicePlugin.nvidiaNodeSelector }}
|
||||
nodeSelector: {{ toYaml .Values.devicePlugin.nvidiaNodeSelector | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.devicePlugin.tolerations }}
|
||||
tolerations: {{ toYaml .Values.devicePlugin.tolerations | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -1,28 +0,0 @@
|
|||
{{- if .Values.devicePlugin.enabled -}}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.device-plugin" . }}-monitor
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- pods
|
||||
verbs:
|
||||
- get
|
||||
- create
|
||||
- watch
|
||||
- list
|
||||
- update
|
||||
- patch
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- nodes
|
||||
verbs:
|
||||
- get
|
||||
- update
|
||||
- list
|
||||
- patch
|
||||
{{- end -}}
|
||||
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
{{- if .Values.devicePlugin.enabled -}}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.device-plugin" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: "hami-device-plugin"
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: {{ include "hami-vgpu.device-plugin" . }}-monitor
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "hami-vgpu.device-plugin" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
{{- end -}}
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
{{- if .Values.devicePlugin.enabled -}}
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.device-plugin" . }}-monitor
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-device-plugin
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
{{- with .Values.devicePlugin.service.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- if .Values.devicePlugin.service.annotations }} # Use devicePlugin instead of scheduler
|
||||
annotations: {{ toYaml .Values.devicePlugin.service.annotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
type: {{ .Values.devicePlugin.service.type | default "NodePort" }} # Default type is NodePort
|
||||
ports:
|
||||
- name: monitorport
|
||||
port: {{ .Values.devicePlugin.service.httpPort | default 31992 }} # Default HTTP port is 31992
|
||||
targetPort: 9394
|
||||
{{- if eq (.Values.devicePlugin.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort
|
||||
nodePort: {{ .Values.devicePlugin.service.httpPort | default 31992 }}
|
||||
{{- end }}
|
||||
protocol: TCP
|
||||
selector:
|
||||
app.kubernetes.io/component: hami-device-plugin
|
||||
{{- include "hami-vgpu.selectorLabels" . | nindent 4 }}
|
||||
{{- end -}}
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
{{- if .Values.devicePlugin.enabled -}}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.device-plugin" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: "hami-device-plugin"
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
{{- end -}}
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
{{- if and .Values.devicePlugin.enabled .Values.devicePlugin.createRuntimeClass .Values.devicePlugin.runtimeClassName -}}
|
||||
apiVersion: node.k8s.io/v1
|
||||
kind: RuntimeClass
|
||||
metadata:
|
||||
name: {{ .Values.devicePlugin.runtimeClassName }}
|
||||
annotations:
|
||||
helm.sh/hook: pre-install,pre-upgrade
|
||||
handler: nvidia
|
||||
{{- end -}}
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
{{- if .Values.scheduler.admissionWebhook.enabled -}}
|
||||
{{- if .Values.scheduler.certManager.enabled }}
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}-serving-cert
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-scheduler
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
spec:
|
||||
dnsNames:
|
||||
- {{ include "hami-vgpu.scheduler" . }}.{{ include "hami-vgpu.namespace" . }}.svc
|
||||
- {{ include "hami-vgpu.scheduler" . }}.{{ include "hami-vgpu.namespace" . }}.svc.cluster.local
|
||||
issuerRef:
|
||||
kind: Issuer
|
||||
name: {{ include "hami-vgpu.scheduler" . }}-selfsigned-issuer
|
||||
secretName: {{ include "hami-vgpu.scheduler.tls" . }}
|
||||
---
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Issuer
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}-selfsigned-issuer
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-scheduler
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
spec:
|
||||
selfSigned: {}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: "hami-scheduler"
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["pods", "configmaps"]
|
||||
verbs: ["get", "list", "watch", "patch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["pods/binding"]
|
||||
verbs: ["create"]
|
||||
- apiGroups: [""]
|
||||
resources: ["nodes"]
|
||||
verbs: ["get", "list", "patch", "watch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["events"]
|
||||
verbs: ["create", "get", "list"]
|
||||
- apiGroups: [""]
|
||||
resources: ["resourcequotas"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
---
|
||||
{{- if .Values.mockDevicePlugin.enabled }}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.mock-device-plugin" . }}
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["nodes"]
|
||||
verbs: ["get", "update", "list", "patch"]
|
||||
{{- end -}}
|
||||
|
|
@ -1,47 +0,0 @@
|
|||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}-kube
|
||||
labels:
|
||||
app.kubernetes.io/component: "hami-scheduler"
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: system:kube-scheduler
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}-volume
|
||||
labels:
|
||||
app.kubernetes.io/component: "hami-scheduler"
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: system:volume-scheduler
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: "hami-scheduler"
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
|
|
@ -1,142 +0,0 @@
|
|||
{{- if .Values.scheduler.kubeScheduler.enabled -}}
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-scheduler
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
data:
|
||||
config.json: |
|
||||
{
|
||||
"kind": "Policy",
|
||||
"apiVersion": "v1",
|
||||
"extenders": [
|
||||
{
|
||||
{{- if .Values.scheduler.admissionWebhook.enabled }}
|
||||
"urlPrefix": "https://127.0.0.1:443",
|
||||
"enableHttps": true,
|
||||
"tlsConfig": {
|
||||
"insecure": true
|
||||
},
|
||||
{{- else }}
|
||||
"urlPrefix": "http://127.0.0.1:80",
|
||||
"enableHttps": false,
|
||||
{{- end }}
|
||||
"filterVerb": "filter",
|
||||
"bindVerb": "bind",
|
||||
"weight": 1,
|
||||
"nodeCacheCapable": true,
|
||||
"httpTimeout": 30000000000,
|
||||
"managedResources": [
|
||||
{{- range .Values.devices.amd.customresources }}
|
||||
{
|
||||
"name": "{{ . }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{{- end }}
|
||||
{{- if .Values.devices.ascend.enabled }}
|
||||
{{- range .Values.devices.ascend.customresources }}
|
||||
{
|
||||
"name": "{{ . }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.devices.mthreads.enabled }}
|
||||
{{- range .Values.devices.mthreads.customresources }}
|
||||
{
|
||||
"name": "{{ . }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.devices.enflame.enabled }}
|
||||
{{- range .Values.devices.enflame.customresources }}
|
||||
{
|
||||
"name": "{{ . }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.devices.kunlun.enabled }}
|
||||
{{- range .Values.devices.kunlun.customresources }}
|
||||
{
|
||||
"name": "{{ . }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- range .Values.devices.awsneuron.customresources }}
|
||||
{
|
||||
"name": "{{ . }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{{- end }}
|
||||
{{- if .Values.devices.iluvatar.enabled }}
|
||||
{{- range .Values.devices.iluvatar.customresources }}
|
||||
{
|
||||
"name": "{{ . }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{
|
||||
"name": "{{ .Values.resourceName }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "{{ .Values.resourceMem }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "{{ .Values.resourceCores }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "{{ .Values.resourceMemPercentage }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "{{ .Values.resourcePriority }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "{{ .Values.mluResourceName }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "{{ .Values.dcuResourceName }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "{{ .Values.dcuResourceMem }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "{{ .Values.dcuResourceCores }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "metax-tech.com/gpu",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "{{ .Values.metaxResourceName }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "{{ .Values.metaxResourceCore }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "{{ .Values.metaxResourceMem }}",
|
||||
"ignoredByScheduler": true
|
||||
}
|
||||
],
|
||||
"ignoreable": false
|
||||
}
|
||||
]
|
||||
}
|
||||
{{- end }}
|
||||
|
|
@ -1,102 +0,0 @@
|
|||
{{- if .Values.scheduler.kubeScheduler.enabled -}}
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}-newversion
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-scheduler
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
data:
|
||||
config.yaml: |
|
||||
{{- if gt (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 25}}
|
||||
apiVersion: kubescheduler.config.k8s.io/v1
|
||||
{{- else }}
|
||||
apiVersion: kubescheduler.config.k8s.io/v1beta2
|
||||
{{- end }}
|
||||
kind: KubeSchedulerConfiguration
|
||||
leaderElection:
|
||||
leaderElect: false
|
||||
profiles:
|
||||
- schedulerName: {{ .Values.schedulerName }}
|
||||
extenders:
|
||||
{{- if .Values.scheduler.admissionWebhook.enabled }}
|
||||
- urlPrefix: "https://127.0.0.1:443"
|
||||
enableHTTPS: true
|
||||
tlsConfig:
|
||||
insecure: true
|
||||
{{- else }}
|
||||
- urlPrefix: "http://127.0.0.1:80"
|
||||
enableHTTPS: false
|
||||
{{- end }}
|
||||
filterVerb: filter
|
||||
bindVerb: bind
|
||||
nodeCacheCapable: true
|
||||
weight: 1
|
||||
httpTimeout: 30s
|
||||
managedResources:
|
||||
- name: {{ .Values.resourceName }}
|
||||
ignoredByScheduler: true
|
||||
- name: {{ .Values.resourceMem }}
|
||||
ignoredByScheduler: true
|
||||
- name: {{ .Values.resourceCores }}
|
||||
ignoredByScheduler: true
|
||||
- name: {{ .Values.resourceMemPercentage }}
|
||||
ignoredByScheduler: true
|
||||
- name: {{ .Values.resourcePriority }}
|
||||
ignoredByScheduler: true
|
||||
- name: {{ .Values.mluResourceName }}
|
||||
ignoredByScheduler: true
|
||||
- name: {{ .Values.dcuResourceName }}
|
||||
ignoredByScheduler: true
|
||||
- name: {{ .Values.dcuResourceMem }}
|
||||
ignoredByScheduler: true
|
||||
- name: {{ .Values.dcuResourceCores }}
|
||||
ignoredByScheduler: true
|
||||
- name: "metax-tech.com/gpu"
|
||||
ignoredByScheduler: true
|
||||
- name: {{ .Values.metaxResourceName }}
|
||||
ignoredByScheduler: true
|
||||
- name: {{ .Values.metaxResourceCore }}
|
||||
ignoredByScheduler: true
|
||||
- name: {{ .Values.metaxResourceMem }}
|
||||
ignoredByScheduler: true
|
||||
{{- if .Values.devices.ascend.enabled }}
|
||||
{{- range .Values.devices.ascend.customresources }}
|
||||
- name: {{ . }}
|
||||
ignoredByScheduler: true
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.devices.mthreads.enabled }}
|
||||
{{- range .Values.devices.mthreads.customresources }}
|
||||
- name: {{ . }}
|
||||
ignoredByScheduler: true
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.devices.enflame.enabled }}
|
||||
{{- range .Values.devices.enflame.customresources }}
|
||||
- name: {{ . }}
|
||||
ignoredByScheduler: true
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.devices.kunlun.enabled }}
|
||||
{{- range .Values.devices.kunlun.customresources }}
|
||||
- name: {{ . }}
|
||||
ignoredByScheduler: true
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- range .Values.devices.awsneuron.customresources }}
|
||||
- name: {{ . }}
|
||||
ignoredByScheduler: true
|
||||
{{- end }}
|
||||
{{- if .Values.devices.iluvatar.enabled }}
|
||||
{{- range .Values.devices.iluvatar.customresources }}
|
||||
- name: {{ . }}
|
||||
ignoredByScheduler: true
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- range .Values.devices.amd.customresources }}
|
||||
- name: {{ . }}
|
||||
ignoredByScheduler: true
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -1,225 +0,0 @@
|
|||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-scheduler
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
{{- with .Values.global.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.annotations }}
|
||||
annotations: {{ toYaml .Values.global.annotations | nindent 4}}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if .Values.scheduler.leaderElect }}
|
||||
replicas: {{ .Values.scheduler.replicas }}
|
||||
{{- else }}
|
||||
replicas: 1
|
||||
{{- end }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/component: hami-scheduler
|
||||
{{- include "hami-vgpu.selectorLabels" . | nindent 6 }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-scheduler
|
||||
{{- include "hami-vgpu.selectorLabels" . | nindent 8 }}
|
||||
hami.io/webhook: ignore
|
||||
annotations:
|
||||
{{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
|
||||
checksum/hami-scheduler-newversion-config: {{ include (print $.Template.BasePath "/scheduler/configmapnew.yaml") . | sha256sum }}
|
||||
{{- else }}
|
||||
checksum/hami-scheduler-config: {{ include (print $.Template.BasePath "/scheduler/configmap.yaml") . | sha256sum }}
|
||||
{{- end }}
|
||||
checksum/hami-scheduler-device-config: {{ include (print $.Template.BasePath "/scheduler/device-configmap.yaml") . | sha256sum }}
|
||||
{{- if .Values.scheduler.podAnnotations }}
|
||||
{{- toYaml .Values.scheduler.podAnnotations | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
serviceAccountName: {{ include "hami-vgpu.scheduler" . }}
|
||||
priorityClassName: system-node-critical
|
||||
{{- include "hami.scheduler.extender.imagePullSecrets" . | nindent 6 }}
|
||||
containers:
|
||||
{{- if .Values.scheduler.kubeScheduler.enabled }}
|
||||
- name: kube-scheduler
|
||||
image: {{ include "hami.scheduler.kubeScheduler.image" . }}
|
||||
imagePullPolicy: {{ .Values.scheduler.kubeScheduler.image.pullPolicy }}
|
||||
command:
|
||||
- kube-scheduler
|
||||
{{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
|
||||
{{- range .Values.scheduler.kubeScheduler.extraNewArgs }}
|
||||
- {{ . }}
|
||||
{{- end }}
|
||||
{{- else }}
|
||||
- --scheduler-name={{ .Values.schedulerName }}
|
||||
{{- range .Values.scheduler.kubeScheduler.extraArgs }}
|
||||
- {{ . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
- --leader-elect={{ .Values.scheduler.leaderElect }}
|
||||
- --leader-elect-resource-name={{ .Values.schedulerName }}
|
||||
- --leader-elect-resource-namespace={{ include "hami-vgpu.namespace" . }}
|
||||
resources:
|
||||
{{- toYaml .Values.scheduler.kubeScheduler.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
- name: scheduler-config
|
||||
mountPath: /config
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.livenessProbe }}
|
||||
livenessProbe:
|
||||
failureThreshold: 8
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: 10259
|
||||
scheme: HTTPS
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
successThreshold: 1
|
||||
timeoutSeconds: 15
|
||||
{{- end }}
|
||||
- name: vgpu-scheduler-extender
|
||||
image: {{ include "hami.scheduler.extender.image" . }}
|
||||
imagePullPolicy: {{ .Values.scheduler.extender.image.pullPolicy }}
|
||||
env:
|
||||
{{- if .Values.scheduler.nodeLockExpire }}
|
||||
- name: HAMI_NODELOCK_EXPIRE
|
||||
value: "{{ .Values.scheduler.nodeLockExpire }}"
|
||||
{{- end }}
|
||||
{{- if .Values.global.managedNodeSelectorEnable }}
|
||||
{{- range $key, $value := .Values.global.managedNodeSelector }}
|
||||
- name: NODE_SELECTOR_{{ $key | upper | replace "-" "_" }}
|
||||
value: "{{ $value }}"
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
command:
|
||||
- scheduler
|
||||
{{- if .Values.scheduler.admissionWebhook.enabled }}
|
||||
- --http_bind=0.0.0.0:443
|
||||
- --cert_file=/tls/tls.crt
|
||||
- --key_file=/tls/tls.key
|
||||
{{- else }}
|
||||
- --http_bind=0.0.0.0:80
|
||||
{{- end }}
|
||||
- --scheduler-name={{ .Values.schedulerName }}
|
||||
- --metrics-bind-address={{ .Values.scheduler.metricsBindAddress }}
|
||||
- --node-scheduler-policy={{ .Values.scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy }}
|
||||
- --gpu-scheduler-policy={{ .Values.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy }}
|
||||
- --force-overwrite-default-scheduler={{ .Values.scheduler.forceOverwriteDefaultScheduler}}
|
||||
- --device-config-file=/device-config.yaml
|
||||
- --leader-elect={{ .Values.scheduler.leaderElect }}
|
||||
- --leader-elect-resource-name={{ .Values.schedulerName }}
|
||||
- --leader-elect-resource-namespace={{ include "hami-vgpu.namespace" . }}
|
||||
{{- if .Values.devices.ascend.enabled }}
|
||||
- --enable-ascend=true
|
||||
{{- end }}
|
||||
{{- if .Values.devices.iluvatar.enabled }}
|
||||
- --enable-iluvatar=true
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.nodeLabelSelector }}
|
||||
- --node-label-selector={{- $first := true -}}
|
||||
{{- range $key, $value := .Values.scheduler.nodeLabelSelector -}}
|
||||
{{- if not $first }},{{ end -}}
|
||||
{{- $key }}={{ $value -}}
|
||||
{{- $first = false -}}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- range .Values.scheduler.extender.extraArgs }}
|
||||
- {{ . }}
|
||||
{{- end }}
|
||||
ports:
|
||||
{{- if .Values.scheduler.admissionWebhook.enabled }}
|
||||
- name: https
|
||||
containerPort: 443
|
||||
protocol: TCP
|
||||
{{- else }}
|
||||
- name: http
|
||||
containerPort: 80
|
||||
protocol: TCP
|
||||
{{- end }}
|
||||
- name: metrics
|
||||
containerPort: {{ last (splitList ":" .Values.scheduler.metricsBindAddress) | int }}
|
||||
protocol: TCP
|
||||
resources:
|
||||
{{- toYaml .Values.scheduler.extender.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
- name: device-config
|
||||
mountPath: /device-config.yaml
|
||||
subPath: device-config.yaml
|
||||
{{- if .Values.scheduler.admissionWebhook.enabled }}
|
||||
- name: tls-config
|
||||
mountPath: /tls
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.livenessProbe }}
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
{{- if .Values.scheduler.admissionWebhook.enabled }}
|
||||
port: https
|
||||
scheme: HTTPS
|
||||
{{- else }}
|
||||
port: http
|
||||
scheme: HTTP
|
||||
{{- end }}
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
failureThreshold: 3
|
||||
timeoutSeconds: 5
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.leaderElect }}
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /readyz
|
||||
{{- if .Values.scheduler.admissionWebhook.enabled }}
|
||||
port: https
|
||||
scheme: HTTPS
|
||||
{{- else }}
|
||||
port: http
|
||||
scheme: HTTP
|
||||
{{- end }}
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
failureThreshold: 3
|
||||
timeoutSeconds: 5
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.leaderElect }}
|
||||
affinity:
|
||||
podAntiAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
- labelSelector:
|
||||
matchExpressions:
|
||||
- key: app.kubernetes.io/component
|
||||
operator: In
|
||||
values:
|
||||
- hami-scheduler
|
||||
topologyKey: "kubernetes.io/hostname"
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- if .Values.scheduler.admissionWebhook.enabled }}
|
||||
- name: tls-config
|
||||
secret:
|
||||
secretName: {{ template "hami-vgpu.scheduler.tls" . }}
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.kubeScheduler.enabled }}
|
||||
- name: scheduler-config
|
||||
configMap:
|
||||
{{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
|
||||
name: {{ template "hami-vgpu.scheduler" . }}-newversion
|
||||
{{- else }}
|
||||
name: {{ template "hami-vgpu.scheduler" . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
- name: device-config
|
||||
configMap:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}-device
|
||||
{{- if .Values.scheduler.nodeSelector }}
|
||||
nodeSelector: {{ toYaml .Values.scheduler.nodeSelector | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.tolerations }}
|
||||
tolerations: {{ toYaml .Values.scheduler.tolerations | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.nodeName }}
|
||||
nodeName: {{ .Values.scheduler.nodeName }}
|
||||
{{- end }}
|
||||
|
|
@ -1,408 +0,0 @@
|
|||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}-device
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-scheduler
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
data:
|
||||
device-config.yaml: |-
|
||||
{{- if .Files.Glob "files/device-config.yaml" }}
|
||||
{{- .Files.Get "files/device-config.yaml" | nindent 4}}
|
||||
{{- else }}
|
||||
nvidia:
|
||||
resourceCountName: {{ .Values.resourceName }}
|
||||
resourceMemoryName: {{ .Values.resourceMem }}
|
||||
resourceMemoryPercentageName: {{ .Values.resourceMemPercentage }}
|
||||
resourceCoreName: {{ .Values.resourceCores }}
|
||||
resourcePriorityName: {{ .Values.resourcePriority }}
|
||||
overwriteEnv: false
|
||||
defaultMemory: 0
|
||||
defaultCores: 0
|
||||
defaultGPUNum: 1
|
||||
memoryFactor: 1
|
||||
deviceSplitCount: {{ .Values.devicePlugin.deviceSplitCount }}
|
||||
deviceMemoryScaling: {{ .Values.devicePlugin.deviceMemoryScaling }}
|
||||
deviceCoreScaling: {{ .Values.devicePlugin.deviceCoreScaling }}
|
||||
gpuCorePolicy: {{ .Values.devices.nvidia.gpuCorePolicy }}
|
||||
libCudaLogLevel: {{ .Values.devices.nvidia.libCudaLogLevel }}
|
||||
runtimeClassName: "{{ .Values.devicePlugin.runtimeClassName }}"
|
||||
knownMigGeometries:
|
||||
- models: [ "A30" ]
|
||||
allowedGeometries:
|
||||
-
|
||||
- name: 1g.6gb
|
||||
core: 25
|
||||
memory: 6144
|
||||
count: 4
|
||||
-
|
||||
- name: 2g.12gb
|
||||
core: 50
|
||||
memory: 12288
|
||||
count: 2
|
||||
-
|
||||
- name: 4g.24gb
|
||||
core: 100
|
||||
memory: 24576
|
||||
count: 1
|
||||
- models: [ "A100-SXM4-40GB", "A100-40GB-PCIe", "A100-PCIE-40GB"]
|
||||
allowedGeometries:
|
||||
-
|
||||
- name: 1g.5gb
|
||||
core: 14
|
||||
memory: 5120
|
||||
count: 7
|
||||
-
|
||||
- name: 1g.5gb
|
||||
core: 14
|
||||
memory: 5120
|
||||
count: 1
|
||||
- name: 2g.10gb
|
||||
core: 28
|
||||
memory: 10240
|
||||
count: 3
|
||||
-
|
||||
- name: 3g.20gb
|
||||
core: 42
|
||||
memory: 20480
|
||||
count: 2
|
||||
-
|
||||
- name: 7g.40gb
|
||||
core: 100
|
||||
memory: 40960
|
||||
count: 1
|
||||
- models: [ "A100-SXM4-80GB", "A100-80GB-PCIe", "A100-PCIE-80GB"]
|
||||
allowedGeometries:
|
||||
-
|
||||
- name: 1g.10gb
|
||||
core: 14
|
||||
memory: 10240
|
||||
count: 7
|
||||
-
|
||||
- name: 1g.10gb
|
||||
core: 14
|
||||
memory: 10240
|
||||
count: 1
|
||||
- name: 2g.20gb
|
||||
core: 28
|
||||
memory: 20480
|
||||
count: 3
|
||||
-
|
||||
- name: 3g.40gb
|
||||
core: 42
|
||||
memory: 40960
|
||||
count: 2
|
||||
-
|
||||
- name: 7g.79gb
|
||||
core: 100
|
||||
memory: 80896
|
||||
count: 1
|
||||
- models: [ "H100-PCIE-80GB", "H100-SXM5-80GB"]
|
||||
allowedGeometries:
|
||||
-
|
||||
- name: 1g.10gb
|
||||
core: 14
|
||||
memory: 10240
|
||||
count: 7
|
||||
-
|
||||
- name: 1g.10gb
|
||||
core: 14
|
||||
memory: 10240
|
||||
count: 1
|
||||
- name: 2g.20gb
|
||||
core: 28
|
||||
memory: 20480
|
||||
count: 3
|
||||
-
|
||||
- name: 3g.40gb
|
||||
core: 42
|
||||
memory: 40960
|
||||
count: 2
|
||||
-
|
||||
- name: 7g.80gb
|
||||
core: 100
|
||||
memory: 81920
|
||||
count: 1
|
||||
- models: [ "H100-PCIE-94GB", "H100-SXM5-94GB"]
|
||||
allowedGeometries:
|
||||
-
|
||||
- name: 1g.12gb
|
||||
core: 14
|
||||
memory: 12288
|
||||
count: 7
|
||||
-
|
||||
- name: 1g.12gb
|
||||
core: 14
|
||||
memory: 12288
|
||||
count: 1
|
||||
- name: 2g.24gb
|
||||
core: 28
|
||||
memory: 24576
|
||||
count: 3
|
||||
-
|
||||
- name: 3g.47gb
|
||||
core: 42
|
||||
memory: 48128
|
||||
count: 2
|
||||
-
|
||||
- name: 7g.94gb
|
||||
core: 100
|
||||
memory: 96256
|
||||
count: 1
|
||||
- models: [ "H20", "H100 on GH200"]
|
||||
allowedGeometries:
|
||||
-
|
||||
- name: 1g.12gb
|
||||
core: 14
|
||||
memory: 12288
|
||||
count: 7
|
||||
-
|
||||
- name: 1g.12gb
|
||||
core: 14
|
||||
memory: 12288
|
||||
count: 1
|
||||
- name: 2g.24gb
|
||||
core: 28
|
||||
memory: 24576
|
||||
count: 3
|
||||
-
|
||||
- name: 3g.48gb
|
||||
core: 42
|
||||
memory: 49152
|
||||
count: 2
|
||||
-
|
||||
- name: 7g.96gb
|
||||
core: 100
|
||||
memory: 98304
|
||||
count: 1
|
||||
- models: [ "H200 NVL", "H200-SXM5"]
|
||||
allowedGeometries:
|
||||
-
|
||||
- name: 1g.18gb
|
||||
core: 14
|
||||
memory: 18432
|
||||
count: 7
|
||||
-
|
||||
- name: 1g.18gb
|
||||
core: 14
|
||||
memory: 18432
|
||||
count: 1
|
||||
- name: 2g.35gb
|
||||
core: 28
|
||||
memory: 35840
|
||||
count: 3
|
||||
-
|
||||
- name: 3g.71gb
|
||||
core: 42
|
||||
memory: 72704
|
||||
count: 2
|
||||
-
|
||||
- name: 7g.141gb
|
||||
core: 100
|
||||
memory: 144384
|
||||
count: 1
|
||||
- models: [ "B200" ]
|
||||
allowedGeometries:
|
||||
-
|
||||
- name: 1g.23gb
|
||||
core: 14
|
||||
memory: 23552
|
||||
count: 7
|
||||
-
|
||||
- name: 1g.23gb
|
||||
core: 14
|
||||
memory: 23552
|
||||
count: 1
|
||||
- name: 2g.45gb
|
||||
core: 28
|
||||
memory: 46080
|
||||
count: 3
|
||||
-
|
||||
- name: 3g.90gb
|
||||
core: 42
|
||||
memory: 92160
|
||||
count: 2
|
||||
-
|
||||
- name: 7g.180gb
|
||||
core: 100
|
||||
memory: 184320
|
||||
count: 1
|
||||
cambricon:
|
||||
resourceCountName: {{ .Values.mluResourceName }}
|
||||
resourceMemoryName: {{ .Values.mluResourceMem }}
|
||||
resourceCoreName: {{ .Values.mluResourceCores }}
|
||||
hygon:
|
||||
resourceCountName: {{ .Values.dcuResourceName }}
|
||||
resourceMemoryName: {{ .Values.dcuResourceMem }}
|
||||
resourceCoreName: {{ .Values.dcuResourceCores }}
|
||||
memoryFactor: 1
|
||||
metax:
|
||||
resourceCountName: "metax-tech.com/gpu"
|
||||
resourceVCountName: {{ .Values.metaxResourceName }}
|
||||
resourceVMemoryName: {{ .Values.metaxResourceMem }}
|
||||
resourceVCoreName: {{ .Values.metaxResourceCore }}
|
||||
sgpuTopologyAware: {{ .Values.metaxsGPUTopologyAware }}
|
||||
enflame:
|
||||
resourceNameGCU: "enflame.com/gcu"
|
||||
resourceNameVGCU: {{ .Values.enflameResourceNameVGCU }}
|
||||
resourceNameVGCUPercentage: {{ .Values.enflameResourceNameVGCUPercentage }}
|
||||
mthreads:
|
||||
resourceCountName: "mthreads.com/vgpu"
|
||||
resourceMemoryName: "mthreads.com/sgpu-memory"
|
||||
resourceCoreName: "mthreads.com/sgpu-core"
|
||||
iluvatars:
|
||||
- chipName: MR-V100
|
||||
commonWord: MR-V100
|
||||
resourceCountName: iluvatar.ai/MR-V100-vgpu
|
||||
resourceMemoryName: iluvatar.ai/MR-V100.vMem
|
||||
resourceCoreName: iluvatar.ai/MR-V100.vCore
|
||||
- chipName: MR-V50
|
||||
commonWord: MR-V50
|
||||
resourceCountName: iluvatar.ai/MR-V50-vgpu
|
||||
resourceMemoryName: iluvatar.ai/MR-V50.vMem
|
||||
resourceCoreName: iluvatar.ai/MR-V50.vCore
|
||||
- chipName: BI-V150
|
||||
commonWord: BI-V150
|
||||
resourceCountName: iluvatar.ai/BI-V150-vgpu
|
||||
resourceMemoryName: iluvatar.ai/BI-V150.vMem
|
||||
resourceCoreName: iluvatar.ai/BI-V150.vCore
|
||||
- chipName: BI-V100
|
||||
commonWord: BI-V100
|
||||
resourceCountName: iluvatar.ai/BI-V100-vgpu
|
||||
resourceMemoryName: iluvatar.ai/BI-V100.vMem
|
||||
resourceCoreName: iluvatar.ai/BI-V100.vCore
|
||||
kunlun:
|
||||
resourceCountName: {{ .Values.kunlunResourceName }}
|
||||
resourceVCountName: {{ .Values.kunlunResourceVCountName }}
|
||||
resourceVMemoryName: {{ .Values.kunlunResourceVMemoryName }}
|
||||
awsneuron:
|
||||
resourceCountName: "aws.amazon.com/neuron"
|
||||
resourceCoreName: "aws.amazon.com/neuroncore"
|
||||
amd:
|
||||
resourceCountName: "amd.com/gpu"
|
||||
vnpus:
|
||||
- chipName: 910A
|
||||
commonWord: Ascend910A
|
||||
resourceName: huawei.com/Ascend910A
|
||||
resourceMemoryName: huawei.com/Ascend910A-memory
|
||||
memoryAllocatable: 32768
|
||||
memoryCapacity: 32768
|
||||
memoryFactor: 1
|
||||
aiCore: 30
|
||||
templates:
|
||||
- name: vir02
|
||||
memory: 2184
|
||||
aiCore: 2
|
||||
- name: vir04
|
||||
memory: 4369
|
||||
aiCore: 4
|
||||
- name: vir08
|
||||
memory: 8738
|
||||
aiCore: 8
|
||||
- name: vir16
|
||||
memory: 17476
|
||||
aiCore: 16
|
||||
- chipName: 910B2
|
||||
commonWord: Ascend910B2
|
||||
resourceName: huawei.com/Ascend910B2
|
||||
resourceMemoryName: huawei.com/Ascend910B2-memory
|
||||
memoryAllocatable: 65536
|
||||
memoryCapacity: 65536
|
||||
memoryFactor: 1
|
||||
aiCore: 24
|
||||
aiCPU: 6
|
||||
templates:
|
||||
- name: vir03_1c_8g
|
||||
memory: 8192
|
||||
aiCore: 3
|
||||
aiCPU: 1
|
||||
- name: vir06_1c_16g
|
||||
memory: 16384
|
||||
aiCore: 6
|
||||
aiCPU: 1
|
||||
- name: vir12_3c_32g
|
||||
memory: 32768
|
||||
aiCore: 12
|
||||
aiCPU: 3
|
||||
- chipName: 910B3
|
||||
commonWord: Ascend910B3
|
||||
resourceName: huawei.com/Ascend910B3
|
||||
resourceMemoryName: huawei.com/Ascend910B3-memory
|
||||
memoryAllocatable: 65536
|
||||
memoryCapacity: 65536
|
||||
memoryFactor: 1
|
||||
aiCore: 20
|
||||
aiCPU: 7
|
||||
templates:
|
||||
- name: vir05_1c_16g
|
||||
memory: 16384
|
||||
aiCore: 5
|
||||
aiCPU: 1
|
||||
- name: vir10_3c_32g
|
||||
memory: 32768
|
||||
aiCore: 10
|
||||
aiCPU: 3
|
||||
- chipName: 910B4-1
|
||||
commonWord: Ascend910B4-1
|
||||
resourceName: huawei.com/Ascend910B4-1
|
||||
resourceMemoryName: huawei.com/Ascend910B4-1-memory
|
||||
memoryAllocatable: 65536
|
||||
memoryCapacity: 65536
|
||||
memoryFactor: 1
|
||||
aiCore: 20
|
||||
aiCPU: 7
|
||||
templates:
|
||||
# NOTE: Names of vnpu template for 910B4-1 are fixed by Ascend runtime and must not be changed.
|
||||
# The memory is used for scheduling so the correct values must be set.
|
||||
# Template vir05_1c_8g actually provides 16GB memory,
|
||||
- name: vir05_1c_8g
|
||||
memory: 16384
|
||||
aiCore: 5
|
||||
aiCPU: 1
|
||||
# Template vir10_3c_16g actually provides 32GB memory
|
||||
- name: vir10_3c_16g
|
||||
memory: 32768
|
||||
aiCore: 10
|
||||
aiCPU: 3
|
||||
- chipName: 910B4
|
||||
commonWord: Ascend910B4
|
||||
resourceName: huawei.com/Ascend910B4
|
||||
resourceMemoryName: huawei.com/Ascend910B4-memory
|
||||
memoryAllocatable: 32768
|
||||
memoryCapacity: 32768
|
||||
memoryFactor: 1
|
||||
aiCore: 20
|
||||
aiCPU: 7
|
||||
templates:
|
||||
- name: vir05_1c_8g
|
||||
memory: 8192
|
||||
aiCore: 5
|
||||
aiCPU: 1
|
||||
- name: vir10_3c_16g
|
||||
memory: 16384
|
||||
aiCore: 10
|
||||
aiCPU: 3
|
||||
- chipName: 310P3
|
||||
commonWord: Ascend310P
|
||||
resourceName: huawei.com/Ascend310P
|
||||
resourceMemoryName: huawei.com/Ascend310P-memory
|
||||
memoryAllocatable: 21527
|
||||
memoryCapacity: 24576
|
||||
memoryFactor: 1
|
||||
aiCore: 8
|
||||
aiCPU: 7
|
||||
templates:
|
||||
- name: vir01
|
||||
memory: 3072
|
||||
aiCore: 1
|
||||
aiCPU: 1
|
||||
- name: vir02
|
||||
memory: 6144
|
||||
aiCore: 2
|
||||
aiCPU: 2
|
||||
- name: vir04
|
||||
memory: 12288
|
||||
aiCore: 4
|
||||
aiCPU: 4
|
||||
{{ end }}
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
annotations:
|
||||
"helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
labels:
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: admission-webhook
|
||||
rules:
|
||||
- apiGroups:
|
||||
- admissionregistration.k8s.io
|
||||
resources:
|
||||
#- validatingwebhookconfigurations
|
||||
- mutatingwebhookconfigurations
|
||||
verbs:
|
||||
- get
|
||||
- update
|
||||
{{- end }}
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
annotations:
|
||||
"helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
labels:
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: admission-webhook
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
{{- end }}
|
||||
|
|
@ -1,68 +0,0 @@
|
|||
{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}}
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission-create
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
annotations:
|
||||
"helm.sh/hook": pre-install,pre-upgrade
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
labels:
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: admission-webhook
|
||||
spec:
|
||||
{{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }}
|
||||
# Alpha feature since k8s 1.12
|
||||
ttlSecondsAfterFinished: 0
|
||||
{{- end }}
|
||||
template:
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission-create
|
||||
{{- if .Values.scheduler.patch.podAnnotations }}
|
||||
annotations: {{ toYaml .Values.scheduler.patch.podAnnotations | nindent 8 }}
|
||||
{{- end }}
|
||||
labels:
|
||||
{{- include "hami-vgpu.labels" . | nindent 8 }}
|
||||
app.kubernetes.io/component: admission-webhook
|
||||
hami.io/webhook: ignore
|
||||
spec:
|
||||
{{- if .Values.scheduler.patch.priorityClassName }}
|
||||
priorityClassName: {{ .Values.scheduler.patch.priorityClassName }}
|
||||
{{- end }}
|
||||
{{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
|
||||
{{- include "hami.scheduler.patch.new.imagePullSecrets" . | nindent 6 }}
|
||||
{{- else }}
|
||||
{{- include "hami.scheduler.patch.imagePullSecrets" . | nindent 6 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: create
|
||||
{{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
|
||||
image: {{ include "hami.scheduler.patch.new.image" . }}
|
||||
imagePullPolicy: {{ .Values.scheduler.patch.imageNew.pullPolicy }}
|
||||
{{- else }}
|
||||
image: {{ include "hami.scheduler.patch.image" . }}
|
||||
imagePullPolicy: {{ .Values.scheduler.patch.image.pullPolicy }}
|
||||
{{- end }}
|
||||
args:
|
||||
- create
|
||||
- --cert-name=tls.crt
|
||||
- --key-name=tls.key
|
||||
{{- if .Values.scheduler.admissionWebhook.customURL.enabled }}
|
||||
- --host={{ printf "%s.%s.svc,127.0.0.1,%s" (include "hami-vgpu.scheduler" .) (include "hami-vgpu.namespace" .) .Values.scheduler.admissionWebhook.customURL.host}}
|
||||
{{- else }}
|
||||
- --host={{ printf "%s.%s.svc,127.0.0.1" (include "hami-vgpu.scheduler" .) (include "hami-vgpu.namespace" .) }}
|
||||
{{- end }}
|
||||
- --namespace={{ include "hami-vgpu.namespace" . }}
|
||||
- --secret-name={{ include "hami-vgpu.scheduler.tls" . }}
|
||||
restartPolicy: OnFailure
|
||||
serviceAccountName: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
{{- if .Values.scheduler.patch.nodeSelector }}
|
||||
nodeSelector: {{ toYaml .Values.scheduler.patch.nodeSelector | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.patch.tolerations }}
|
||||
tolerations: {{ toYaml .Values.scheduler.patch.tolerations | nindent 8 }}
|
||||
{{- end }}
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: {{ .Values.scheduler.patch.runAsUser }}
|
||||
{{- end }}
|
||||
|
|
@ -1,63 +0,0 @@
|
|||
{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}}
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission-patch
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
annotations:
|
||||
"helm.sh/hook": post-install,post-upgrade
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
labels:
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: admission-webhook
|
||||
spec:
|
||||
{{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }}
|
||||
# Alpha feature since k8s 1.12
|
||||
ttlSecondsAfterFinished: 0
|
||||
{{- end }}
|
||||
template:
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission-patch
|
||||
{{- if .Values.scheduler.patch.podAnnotations }}
|
||||
annotations: {{ toYaml .Values.scheduler.patch.podAnnotations | nindent 8 }}
|
||||
{{- end }}
|
||||
labels:
|
||||
{{- include "hami-vgpu.labels" . | nindent 8 }}
|
||||
app.kubernetes.io/component: admission-webhook
|
||||
hami.io/webhook: ignore
|
||||
spec:
|
||||
{{- if .Values.scheduler.patch.priorityClassName }}
|
||||
priorityClassName: {{ .Values.scheduler.patch.priorityClassName }}
|
||||
{{- end }}
|
||||
{{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
|
||||
{{- include "hami.scheduler.patch.new.imagePullSecrets" . | nindent 6 }}
|
||||
{{- else }}
|
||||
{{- include "hami.scheduler.patch.imagePullSecrets" . | nindent 6 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: patch
|
||||
{{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
|
||||
image: {{ include "hami.scheduler.patch.new.image" . }}
|
||||
imagePullPolicy: {{ .Values.scheduler.patch.imageNew.pullPolicy }}
|
||||
{{- else }}
|
||||
image: {{ include "hami.scheduler.patch.image" . }}
|
||||
imagePullPolicy: {{ .Values.scheduler.patch.image.pullPolicy }}
|
||||
{{- end }}
|
||||
args:
|
||||
- patch
|
||||
- --webhook-name={{ include "hami-vgpu.scheduler.webhook" . }}
|
||||
- --namespace={{ include "hami-vgpu.namespace" . }}
|
||||
- --patch-validating=false
|
||||
- --secret-name={{ include "hami-vgpu.scheduler.tls" . }}
|
||||
restartPolicy: OnFailure
|
||||
serviceAccountName: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
{{- if .Values.scheduler.patch.nodeSelector }}
|
||||
nodeSelector: {{ toYaml .Values.scheduler.patch.nodeSelector | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.patch.tolerations }}
|
||||
tolerations: {{ toYaml .Values.scheduler.patch.tolerations | nindent 8 }}
|
||||
{{- end }}
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: {{ .Values.scheduler.patch.runAsUser }}
|
||||
{{- end }}
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
annotations:
|
||||
"helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
labels:
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: admission-webhook
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- secrets
|
||||
verbs:
|
||||
- get
|
||||
- create
|
||||
{{- end }}
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
annotations:
|
||||
"helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
labels:
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: admission-webhook
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
{{- end }}
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
annotations:
|
||||
"helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
labels:
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: admission-webhook
|
||||
{{- end }}
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: "hami-scheduler"
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
rules:
|
||||
- apiGroups: ["coordination.k8s.io"]
|
||||
resources: ["leases"]
|
||||
verbs: ["create", "list", "watch", "get", "update", "patch"]
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: "hami-scheduler"
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
---
|
||||
{{- if .Values.mockDevicePlugin.enabled }}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.mock-device-plugin" . }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: {{ include "hami-vgpu.mock-device-plugin" . }}
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "hami-vgpu.mock-device-plugin" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
{{- end -}}
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-scheduler
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
{{- if .Values.scheduler.service.labels }}
|
||||
{{ toYaml .Values.scheduler.service.labels | indent 4 }}
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.service.annotations }}
|
||||
annotations: {{ toYaml .Values.scheduler.service.annotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
type: {{ .Values.scheduler.service.type | default "NodePort" }} # Default type is NodePort
|
||||
ports:
|
||||
- name: http
|
||||
port: {{ .Values.scheduler.service.httpPort | default 443 }} # Default HTTP port is 443
|
||||
targetPort: {{ .Values.scheduler.service.httpTargetPort | default 443 }}
|
||||
{{- if eq (.Values.scheduler.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort
|
||||
nodePort: {{ .Values.scheduler.service.schedulerPort | default 31998 }}
|
||||
{{- end }}
|
||||
protocol: TCP
|
||||
- name: monitor
|
||||
port: {{ .Values.scheduler.service.monitorPort | default 31993 }} # Default monitoring port is 31993
|
||||
targetPort: {{ .Values.scheduler.service.monitorTargetPort | default 9395 }}
|
||||
{{- if eq (.Values.scheduler.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort
|
||||
nodePort: {{ .Values.scheduler.service.monitorPort | default 31993 }}
|
||||
{{- end }}
|
||||
protocol: TCP
|
||||
selector:
|
||||
app.kubernetes.io/component: hami-scheduler
|
||||
{{- include "hami-vgpu.selectorLabels" . | nindent 4 }}
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: "hami-scheduler"
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
---
|
||||
{{- if .Values.mockDevicePlugin.enabled }}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.mock-device-plugin" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
{{- end -}}
|
||||
|
|
@ -1,57 +0,0 @@
|
|||
{{- if .Values.scheduler.admissionWebhook.enabled -}}
|
||||
apiVersion: admissionregistration.k8s.io/v1
|
||||
kind: MutatingWebhookConfiguration
|
||||
metadata:
|
||||
{{- if .Values.scheduler.certManager.enabled }}
|
||||
annotations:
|
||||
cert-manager.io/inject-ca-from: {{ include "hami-vgpu.namespace" . }}/{{ include "hami-vgpu.scheduler" . }}-serving-cert
|
||||
{{- end }}
|
||||
name: {{ include "hami-vgpu.scheduler.webhook" . }}
|
||||
webhooks:
|
||||
- admissionReviewVersions:
|
||||
- v1beta1
|
||||
clientConfig:
|
||||
{{- if .Values.scheduler.admissionWebhook.customURL.enabled }}
|
||||
url: https://{{ .Values.scheduler.admissionWebhook.customURL.host}}:{{.Values.scheduler.admissionWebhook.customURL.port}}{{.Values.scheduler.admissionWebhook.customURL.path}}
|
||||
{{- else }}
|
||||
service:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
path: /webhook
|
||||
port: {{ .Values.scheduler.service.httpPort }}
|
||||
{{- end }}
|
||||
failurePolicy: {{ .Values.scheduler.admissionWebhook.failurePolicy }}
|
||||
matchPolicy: Equivalent
|
||||
name: vgpu.hami.io
|
||||
namespaceSelector:
|
||||
matchExpressions:
|
||||
- key: hami.io/webhook
|
||||
operator: NotIn
|
||||
values:
|
||||
- ignore
|
||||
{{- if .Values.scheduler.admissionWebhook.whitelistNamespaces }}
|
||||
- key: kubernetes.io/metadata.name
|
||||
operator: NotIn
|
||||
values:
|
||||
{{- toYaml .Values.scheduler.admissionWebhook.whitelistNamespaces | nindent 10 }}
|
||||
{{- end }}
|
||||
objectSelector:
|
||||
matchExpressions:
|
||||
- key: hami.io/webhook
|
||||
operator: NotIn
|
||||
values:
|
||||
- ignore
|
||||
reinvocationPolicy: {{ .Values.scheduler.admissionWebhook.reinvocationPolicy }}
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
apiVersions:
|
||||
- v1
|
||||
operations:
|
||||
- CREATE
|
||||
resources:
|
||||
- pods
|
||||
scope: '*'
|
||||
sideEffects: None
|
||||
timeoutSeconds: 10
|
||||
{{- end }}
|
||||
|
|
@ -1,455 +0,0 @@
|
|||
## @section Global configuration
|
||||
## @param global.imageRegistry Global Docker image registry
|
||||
## @param global.imagePullSecrets Global Docker image pull secrets
|
||||
global:
|
||||
## @param global.imageRegistry Global Docker image registry
|
||||
imageRegistry: ""
|
||||
## E.g.
|
||||
## imagePullSecrets:
|
||||
## - myRegistryKeySecretName
|
||||
## @param global.imagePullSecrets Global Docker image pull secrets
|
||||
imagePullSecrets: []
|
||||
imageTag: "v2.8.1"
|
||||
gpuHookPath: /usr/local
|
||||
labels: {}
|
||||
annotations: {}
|
||||
managedNodeSelectorEnable: false
|
||||
managedNodeSelector:
|
||||
usage: "gpu"
|
||||
|
||||
nameOverride: ""
|
||||
fullnameOverride: ""
|
||||
namespaceOverride: ""
|
||||
|
||||
#Nvidia GPU Parameters
|
||||
resourceName: "nvidia.com/gpu"
|
||||
resourceMem: "nvidia.com/gpumem"
|
||||
resourceMemPercentage: "nvidia.com/gpumem-percentage"
|
||||
resourceCores: "nvidia.com/gpucores"
|
||||
resourcePriority: "nvidia.com/priority"
|
||||
|
||||
#MLU Parameters
|
||||
mluResourceName: "cambricon.com/vmlu"
|
||||
mluResourceMem: "cambricon.com/mlu.smlu.vmemory"
|
||||
mluResourceCores: "cambricon.com/mlu.smlu.vcore"
|
||||
|
||||
#Hygon DCU Parameters
|
||||
dcuResourceName: "hygon.com/dcunum"
|
||||
dcuResourceMem: "hygon.com/dcumem"
|
||||
dcuResourceCores: "hygon.com/dcucores"
|
||||
|
||||
#Metax sGPU Parameters
|
||||
metaxResourceName: "metax-tech.com/sgpu"
|
||||
metaxResourceCore: "metax-tech.com/vcore"
|
||||
metaxResourceMem: "metax-tech.com/vmemory"
|
||||
metaxsGPUTopologyAware: "false"
|
||||
|
||||
#Enflame VGCU Parameters
|
||||
enflameResourceNameVGCU: "enflame.com/vgcu"
|
||||
enflameResourceNameVGCUPercentage: "enflame.com/vgcu-percentage"
|
||||
|
||||
#Kunlun XPU Parameters
|
||||
kunlunResourceName: "kunlunxin.com/xpu"
|
||||
kunlunResourceVCountName: "kunlunxin.com/vxpu"
|
||||
kunlunResourceVMemoryName: "kunlunxin.com/vxpu-memory"
|
||||
|
||||
schedulerName: "hami-scheduler"
|
||||
|
||||
scheduler:
|
||||
# @param nodeName defines the node name and the nvidia-vgpu-scheduler-scheduler will schedule to the node.
|
||||
# if we install the nvidia-vgpu-scheduler-scheduler as default scheduler, we need to remove the k8s default
|
||||
# scheduler pod from the cluster first, we must specify node name to skip the schedule workflow.
|
||||
nodeName: ""
|
||||
#nodeLabelSelector:
|
||||
# "gpu": "on"
|
||||
overwriteEnv: "false"
|
||||
defaultSchedulerPolicy:
|
||||
nodeSchedulerPolicy: binpack
|
||||
gpuSchedulerPolicy: spread
|
||||
metricsBindAddress: ":9395"
|
||||
# If set to false, When Pod.Spec.SchedulerName equals to the const DefaultSchedulerName in k8s.io/api/core/v1 package, webhook will not overwrite it
|
||||
forceOverwriteDefaultScheduler: true
|
||||
livenessProbe: false
|
||||
leaderElect: true
|
||||
# when leaderElect is true, replicas is available, otherwise replicas is 1.
|
||||
replicas: 1
|
||||
kubeScheduler:
|
||||
# @param enabled indicate whether to run kube-scheduler container in the scheduler pod, it's true by default.
|
||||
enabled: true
|
||||
## @param image.registry kube scheduler image registry
|
||||
## @param image.repository kube scheduler image repository
|
||||
## @param image.tag kube scheduler image tag (immutable tags are recommended)
|
||||
## @param image.pullPolicy kube scheduler image pull policy
|
||||
## @param image.pullSecrets Specify docker-registry secret names as an array
|
||||
image:
|
||||
registry: "registry.cn-hangzhou.aliyuncs.com"
|
||||
repository: "google_containers/kube-scheduler"
|
||||
tag: ""
|
||||
## Specify a imagePullPolicy
|
||||
## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
|
||||
## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
|
||||
##
|
||||
pullPolicy: IfNotPresent
|
||||
## Optionally specify an array of imagePullSecrets.
|
||||
## Secrets must be manually created in the namespace.
|
||||
## Example:
|
||||
## pullSecrets:
|
||||
## - myRegistryKeySecretName
|
||||
##
|
||||
pullSecrets: []
|
||||
resources: {}
|
||||
# If you do want to specify resources, uncomment the following lines, adjust them as necessary.
|
||||
# and remove the curly braces after 'resources:'.
|
||||
# limits:
|
||||
# cpu: 1000m
|
||||
# memory: 1000Mi
|
||||
# requests:
|
||||
# cpu: 100m
|
||||
# memory: 100Mi
|
||||
extraNewArgs:
|
||||
- --config=/config/config.yaml
|
||||
- -v=4
|
||||
extraArgs:
|
||||
- --policy-config-file=/config/config.json
|
||||
- -v=4
|
||||
extender:
|
||||
## @param image.registry scheduler extender image registry
|
||||
## @param image.repository scheduler extender image repository
|
||||
## @param image.tag scheduler extender image tag (immutable tags are recommended)
|
||||
## @param image.pullPolicy scheduler extender image pull policy
|
||||
## @param image.pullSecrets Specify docker-registry secret names as an array
|
||||
image:
|
||||
registry: "docker.io"
|
||||
repository: "projecthami/hami"
|
||||
tag: ""
|
||||
## Specify a imagePullPolicy
|
||||
## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
|
||||
## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
|
||||
##
|
||||
pullPolicy: IfNotPresent
|
||||
## Optionally specify an array of imagePullSecrets.
|
||||
## Secrets must be manually created in the namespace.
|
||||
## Example:
|
||||
## pullSecrets:
|
||||
## - myRegistryKeySecretName
|
||||
##
|
||||
pullSecrets: []
|
||||
resources: {}
|
||||
# If you do want to specify resources, uncomment the following lines, adjust them as necessary,
|
||||
# and remove the curly braces after 'resources:'.
|
||||
# limits:
|
||||
# cpu: 1000m
|
||||
# memory: 1000Mi
|
||||
# requests:
|
||||
# cpu: 100m
|
||||
# memory: 100Mi
|
||||
extraArgs:
|
||||
- --debug
|
||||
- -v=4
|
||||
nodeLockExpire: "5m"
|
||||
podAnnotations: {}
|
||||
tolerations: []
|
||||
#serviceAccountName: "hami-vgpu-scheduler-sa"
|
||||
admissionWebhook:
|
||||
# If set to false, the admission webhook is not installed and any pods that should use HAMi must be
|
||||
# configured to use the right 'schedulerName' and other device-specific configurations.
|
||||
enabled: true
|
||||
customURL:
|
||||
enabled: false
|
||||
# must be an endpoint using https.
|
||||
# should generate host certs here
|
||||
host: 127.0.0.1 # hostname or ip, can be your node'IP if you want to use https://<nodeIP>:<schedulerPort>/<path>
|
||||
port: 31998
|
||||
path: /webhook
|
||||
whitelistNamespaces:
|
||||
# Specify the namespaces that the webhook will not be applied to.
|
||||
# - default
|
||||
# - kube-system
|
||||
# - istio-system
|
||||
reinvocationPolicy: Never
|
||||
failurePolicy: Ignore
|
||||
## TLS Certificate Option 1: Use cert-manager to generate self-signed certificate.
|
||||
## If enabled, always takes precedence over options 2.
|
||||
certManager:
|
||||
enabled: false
|
||||
## TLS Certificate Option 2: Use kube-webhook-certgen to generate self-signed certificate.
|
||||
## If true and certManager.enabled is false, Helm will automatically create a self-signed cert and secret for you.
|
||||
patch:
|
||||
enabled: true
|
||||
## @param image.registry scheduler extender image registry
|
||||
## @param image.repository scheduler extender image repository
|
||||
## @param image.tag scheduler extender image tag (immutable tags are recommended)
|
||||
## @param image.pullPolicy scheduler extender image pull policy
|
||||
## @param image.pullSecrets Specify docker-registry secret names as an array
|
||||
image:
|
||||
registry: "docker.io"
|
||||
repository: "jettech/kube-webhook-certgen"
|
||||
tag: "v1.5.2"
|
||||
## Specify a imagePullPolicy
|
||||
## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
|
||||
## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
|
||||
##
|
||||
pullPolicy: IfNotPresent
|
||||
## Optionally specify an array of imagePullSecrets.
|
||||
## Secrets must be manually created in the namespace.
|
||||
## Example:
|
||||
## pullSecrets:
|
||||
## - myRegistryKeySecretName
|
||||
##
|
||||
pullSecrets: []
|
||||
## @param image.registry scheduler extender image registry
|
||||
## @param image.repository scheduler extender image repository
|
||||
## @param image.tag scheduler extender image tag (immutable tags are recommended)
|
||||
## @param image.pullPolicy scheduler extender image pull policy
|
||||
## @param image.pullSecrets Specify docker-registry secret names as an array
|
||||
imageNew:
|
||||
registry: "docker.io"
|
||||
repository: "liangjw/kube-webhook-certgen"
|
||||
tag: "v1.1.1"
|
||||
## Specify a imagePullPolicy
|
||||
## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
|
||||
## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
|
||||
##
|
||||
pullPolicy: IfNotPresent
|
||||
## Optionally specify an array of imagePullSecrets.
|
||||
## Secrets must be manually created in the namespace.
|
||||
## Example:
|
||||
## pullSecrets:
|
||||
## - myRegistryKeySecretName
|
||||
##
|
||||
pullSecrets: []
|
||||
priorityClassName: ""
|
||||
podAnnotations: {}
|
||||
nodeSelector: {}
|
||||
tolerations: []
|
||||
runAsUser: 2000
|
||||
service:
|
||||
type: NodePort # Default type is NodePort, can be changed to ClusterIP
|
||||
httpPort: 443 # HTTP port
|
||||
schedulerPort: 31998 # NodePort for HTTP
|
||||
monitorPort: 31993 # Monitoring port
|
||||
monitorTargetPort: 9395
|
||||
httpTargetPort: 443
|
||||
labels: {}
|
||||
annotations: {}
|
||||
|
||||
devicePlugin:
|
||||
enabled: true
|
||||
gpuOperatorToolkitReady:
|
||||
enabled: false
|
||||
hostPath: "/run/nvidia/validations"
|
||||
## @param image.registry devicePlugin image registry
|
||||
## @param image.repository devicePlugin image repository
|
||||
## @param image.tag devicePlugin image tag (immutable tags are recommended)
|
||||
## @param image.pullPolicy devicePlugin image pull policy
|
||||
## @param image.pullSecrets Specify docker-registry secret names as an array
|
||||
image:
|
||||
registry: "docker.io"
|
||||
repository: "projecthami/hami"
|
||||
tag: ""
|
||||
## Specify a imagePullPolicy
|
||||
## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
|
||||
## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
|
||||
##
|
||||
pullPolicy: IfNotPresent
|
||||
## Optionally specify an array of imagePullSecrets.
|
||||
## Secrets must be manually created in the namespace.
|
||||
## Example:
|
||||
## pullSecrets:
|
||||
## - myRegistryKeySecretName
|
||||
##
|
||||
pullSecrets: []
|
||||
monitor:
|
||||
## @param image.registry monitor image registry
|
||||
## @param image.repository monitor image repository
|
||||
## @param image.tag monitor image tag (immutable tags are recommended)
|
||||
## @param image.pullPolicy monitor image pull policy
|
||||
## @param image.pullSecrets Specify docker-registry secret names as an array
|
||||
image:
|
||||
registry: "docker.io"
|
||||
repository: "projecthami/hami"
|
||||
tag: ""
|
||||
## Specify a imagePullPolicy
|
||||
## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
|
||||
## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
|
||||
##
|
||||
pullPolicy: IfNotPresent
|
||||
## Optionally specify an array of imagePullSecrets.
|
||||
## Secrets must be manually created in the namespace.
|
||||
## Example:
|
||||
## pullSecrets:
|
||||
## - myRegistryKeySecretName
|
||||
##
|
||||
pullSecrets: []
|
||||
ctrPath: /usr/local/vgpu/containers
|
||||
resyncInterval: "5m"
|
||||
extraArgs:
|
||||
- -v=4
|
||||
extraEnvs: {}
|
||||
resources: {}
|
||||
# If you do want to specify resources, uncomment the following lines, adjust them as necessary.
|
||||
# and remove the curly braces after 'resources:'.
|
||||
# limits:
|
||||
# cpu: 1000m
|
||||
# memory: 1000Mi
|
||||
# requests:
|
||||
# cpu: 100m
|
||||
# memory: 100Mi
|
||||
|
||||
deviceSplitCount: 10
|
||||
deviceMemoryScaling: 1
|
||||
deviceCoreScaling: 1
|
||||
# Node configuration for device plugin, Priority: externalConfigName > config > default config
|
||||
nodeConfiguration:
|
||||
# If you want to use a custom config.json, you can set the content here.
|
||||
# If this is set, it will override the default config.json(An example is as follows).
|
||||
config: |
|
||||
{
|
||||
"nodeconfig": [
|
||||
{
|
||||
"name": "your-node-name",
|
||||
"operatingmode": "hami-core",
|
||||
"devicememoryscaling": 1,
|
||||
"devicesplitcount": 10,
|
||||
"migstrategy": "none",
|
||||
"filterdevices": {
|
||||
"uuid": [],
|
||||
"index": []
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
# If you want to use an existing ConfigMap, you can set the name here.
|
||||
# If this is set, the chart will not create the ConfigMap and will use the existing one.
|
||||
externalConfigName: ""
|
||||
# The runtime class name to be used by the device plugin, and added to the pod.spec.runtimeClassName of applications utilizing NVIDIA GPUs
|
||||
runtimeClassName: ""
|
||||
# Whether to create runtime class, name comes from runtimeClassName when it is set
|
||||
createRuntimeClass: false
|
||||
migStrategy: "none"
|
||||
disablecorelimit: "false"
|
||||
passDeviceSpecsEnabled: false
|
||||
deviceListStrategy: "envvar"
|
||||
nvidiaHookPath: null
|
||||
nvidiaDriverRoot: null
|
||||
gdrcopyEnabled: null
|
||||
gdsEnabled: null
|
||||
mofedEnabled: null
|
||||
|
||||
extraArgs:
|
||||
- -v=4
|
||||
extraEnvs: {}
|
||||
service:
|
||||
type: NodePort # Default type is NodePort, can be changed to ClusterIP
|
||||
httpPort: 31992
|
||||
labels: {}
|
||||
annotations: {}
|
||||
|
||||
pluginPath: /var/lib/kubelet/device-plugins
|
||||
libPath: /usr/local/vgpu
|
||||
|
||||
podAnnotations: {}
|
||||
nvidiaNodeSelector:
|
||||
gpu: "on"
|
||||
tolerations: []
|
||||
# The updateStrategy for DevicePlugin DaemonSet.
|
||||
# If you want to update the DaemonSet by manual, set type as "OnDelete".
|
||||
# We recommend use OnDelete update strategy because DevicePlugin pod restart will cause business pod restart, this behavior is destructive.
|
||||
# Otherwise, you can use RollingUpdate update strategy to rolling update DevicePlugin pod.
|
||||
updateStrategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxUnavailable: 1
|
||||
|
||||
resources: {}
|
||||
# If you do want to specify resources, uncomment the following lines, adjust them as necessary.
|
||||
# and remove the curly braces after 'resources:'.
|
||||
# limits:
|
||||
# cpu: 1000m
|
||||
# memory: 1000Mi
|
||||
# requests:
|
||||
# cpu: 100m
|
||||
# memory: 100Mi
|
||||
|
||||
mockDevicePlugin:
|
||||
enabled: false
|
||||
image:
|
||||
registry: "docker.io"
|
||||
repository: "projecthami/mock-device-plugin"
|
||||
tag: "1.0.1"
|
||||
## Specify a imagePullPolicy
|
||||
## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
|
||||
## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
|
||||
##
|
||||
pullPolicy: IfNotPresent
|
||||
## Optionally specify an array of imagePullSecrets.
|
||||
## Secrets must be manually created in the namespace.
|
||||
## Example:
|
||||
## pullSecrets:
|
||||
## - myRegistryKeySecretName
|
||||
##
|
||||
pullSecrets: []
|
||||
devices:
|
||||
amd:
|
||||
customresources:
|
||||
- amd.com/gpu
|
||||
- amd.com/gpu-memory
|
||||
awsneuron:
|
||||
customresources:
|
||||
- aws.amazon.com/neuron
|
||||
- aws.amazon.com/neuroncore
|
||||
kunlun:
|
||||
enabled: true
|
||||
customresources:
|
||||
- kunlunxin.com/xpu
|
||||
- kunlunxin.com/vxpu
|
||||
- kunlunxin.com/vxpu-memory
|
||||
enflame:
|
||||
enabled: true
|
||||
customresources:
|
||||
- enflame.com/vgcu
|
||||
- enflame.com/vgcu-percentage
|
||||
- enflame.com/gcu
|
||||
mthreads:
|
||||
enabled: true
|
||||
customresources:
|
||||
- mthreads.com/vgpu
|
||||
nvidia:
|
||||
gpuCorePolicy: default
|
||||
libCudaLogLevel: 1
|
||||
ascend:
|
||||
enabled: false
|
||||
image: ""
|
||||
imagePullPolicy: IfNotPresent
|
||||
extraArgs: []
|
||||
nodeSelector:
|
||||
ascend: "on"
|
||||
tolerations: []
|
||||
customresources:
|
||||
- huawei.com/Ascend910A
|
||||
- huawei.com/Ascend910A-memory
|
||||
- huawei.com/Ascend910B2
|
||||
- huawei.com/Ascend910B2-memory
|
||||
- huawei.com/Ascend910B3
|
||||
- huawei.com/Ascend910B3-memory
|
||||
- huawei.com/Ascend910B4
|
||||
- huawei.com/Ascend910B4-memory
|
||||
- huawei.com/Ascend910B4-1
|
||||
- huawei.com/Ascend910B4-1-memory
|
||||
- huawei.com/Ascend310P
|
||||
- huawei.com/Ascend310P-memory
|
||||
iluvatar:
|
||||
enabled: false
|
||||
customresources:
|
||||
- iluvatar.ai/BI-V100-vgpu
|
||||
- iluvatar.ai/BI-V100.vCore
|
||||
- iluvatar.ai/BI-V100.vMem
|
||||
- iluvatar.ai/BI-V150-vgpu
|
||||
- iluvatar.ai/BI-V150.vCore
|
||||
- iluvatar.ai/BI-V150.vMem
|
||||
- iluvatar.ai/MR-V100-vgpu
|
||||
- iluvatar.ai/MR-V100.vCore
|
||||
- iluvatar.ai/MR-V100.vMem
|
||||
- iluvatar.ai/MR-V50-vgpu
|
||||
- iluvatar.ai/MR-V50.vCore
|
||||
- iluvatar.ai/MR-V50.vMem
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
hami:
|
||||
scheduler:
|
||||
kubeScheduler:
|
||||
image:
|
||||
registry: registry.k8s.io
|
||||
repository: kube-scheduler
|
||||
devicePlugin:
|
||||
runtimeClassName: nvidia
|
||||
updateStrategy:
|
||||
type: RollingUpdate
|
||||
# See https://github.com/Project-HAMi/HAMi/blob/v2.8.1/deployments/charts/hami/values.yaml
|
||||
# Each entry: {"name": "node-name", "operatingmode": "hami-core", "devicesplitcount": N, ...}
|
||||
nodeConfiguration:
|
||||
config: |
|
||||
{
|
||||
"nodeconfig": []
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
|
|
@ -4,13 +4,8 @@ NAMESPACE=cozy-system
|
|||
include ../../../hack/common-envs.mk
|
||||
include ../../../hack/package.mk
|
||||
|
||||
.PHONY: test
|
||||
|
||||
image: image-lineage-controller-webhook
|
||||
|
||||
test:
|
||||
helm unittest .
|
||||
|
||||
image-lineage-controller-webhook:
|
||||
docker buildx build -f images/lineage-controller-webhook/Dockerfile ../../.. \
|
||||
--tag $(REGISTRY)/lineage-controller-webhook:$(call settag,$(TAG)) \
|
||||
|
|
|
|||
|
|
@ -1,67 +0,0 @@
|
|||
# lineage-controller-webhook
|
||||
|
||||
Cozystack system package for the **lineage controller webhook** — a mutating
|
||||
admission webhook that stamps "lineage" labels onto tenant workloads, linking
|
||||
each resource back to the Cozystack `Application` that ultimately owns it.
|
||||
|
||||
The webhook intercepts CREATE and UPDATE on `pods`, `secrets`, `services`,
|
||||
`persistentvolumeclaims`, `ingresses.networking.k8s.io`, and
|
||||
`workloadmonitors.cozystack.io` outside system namespaces. For each request it
|
||||
walks the ownership graph upward (Kubernetes `ownerReferences`, then the
|
||||
`HelmRelease` Flux installed the resource with, then the Cozystack
|
||||
`Application`-derived CRD that produced the HelmRelease) and writes the
|
||||
discovered application's group, kind and name as labels
|
||||
(`apps.cozystack.io/application.{group,kind,name}`) on the incoming object.
|
||||
Those labels let the aggregated API server, the Cozystack dashboard, and other
|
||||
lineage-aware consumers reason about which application a resource belongs to.
|
||||
|
||||
The webhook serves TLS on port 9443 with a cert-manager issued certificate, and
|
||||
runs with `failurePolicy: Fail` — every CREATE/UPDATE on the resources above
|
||||
is gated on the webhook being reachable.
|
||||
|
||||
## Topology
|
||||
|
||||
The chart ships a single shape, modelled on `cozystack-api`:
|
||||
|
||||
- **Deployment** with two replicas (override via `replicas`).
|
||||
- **Soft `nodeAffinity`** preferring `node-role.kubernetes.io/control-plane`
|
||||
(`Exists`, so it matches both Talos's empty value and k3s/kubeadm's `"true"`).
|
||||
Soft means: the pod lands on a control-plane node when one is reachable, and
|
||||
on any worker otherwise — no override needed for managed Kubernetes,
|
||||
Cozy-in-Cozy tenants, or any other cluster where control-plane nodes aren't
|
||||
visible.
|
||||
- **Permissive `tolerations`** (`[{operator: Exists}]`) so the pod can land on
|
||||
tainted control-plane nodes when the soft affinity is satisfiable.
|
||||
- **Soft `podAntiAffinity`** on `kubernetes.io/hostname` so replicas spread
|
||||
across nodes when possible (best-effort).
|
||||
- **`PodDisruptionBudget`** with `maxUnavailable: 1`, unconditional. At
|
||||
`replicas: 1` it's a useful no-op (allows full drain); at `replicas >= 2`
|
||||
it caps disruption to one pod.
|
||||
- **`Service` with `spec.trafficDistribution: PreferClose`** so the apiserver
|
||||
prefers a webhook endpoint on its own node when one exists, and transparently
|
||||
falls over to a remote endpoint otherwise. Requires Kubernetes ≥ 1.31; on
|
||||
older clusters the field is silently ignored and traffic uses default
|
||||
cluster-wide distribution.
|
||||
|
||||
## Parameters
|
||||
|
||||
All keys live under the top-level `lineageControllerWebhook:` map.
|
||||
|
||||
| Name | Description | Value |
|
||||
| ----------------------------- | --------------------------------------------------------- | -------------------------------------------------------------------------------- |
|
||||
| `image` | Container image (digest-pinned) | `ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.3.0@sha256:e898…6fb0` |
|
||||
| `debug` | Enable `--zap-log-level=debug` instead of `info` | `false` |
|
||||
| `replicas` | Deployment replica count | `2` |
|
||||
| `localK8sAPIEndpoint.enabled` | **Deprecated.** See note below. | `false` |
|
||||
|
||||
### `localK8sAPIEndpoint.enabled` (deprecated)
|
||||
|
||||
When enabled, this injects `KUBERNETES_SERVICE_HOST=status.hostIP` and
|
||||
`KUBERNETES_SERVICE_PORT=6443` so the webhook talks to the kube-apiserver on
|
||||
its own node. It was originally added to avoid latency on the
|
||||
webhook-to-apiserver path, but it is only valid when the pod is actually
|
||||
scheduled on an apiserver-bearing node — which the chart's soft control-plane
|
||||
affinity no longer guarantees. With this flag enabled and the pod scheduled
|
||||
off a control-plane node, the controller will crash-loop dialing a non-
|
||||
apiserver hostIP. Slated for removal once the latency motivation is addressed
|
||||
in the webhook itself; **leave disabled**.
|
||||
|
|
@ -1,11 +1,10 @@
|
|||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: lineage-controller-webhook
|
||||
labels:
|
||||
app: lineage-controller-webhook
|
||||
spec:
|
||||
replicas: {{ .Values.lineageControllerWebhook.replicas }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app: lineage-controller-webhook
|
||||
|
|
@ -14,25 +13,13 @@ spec:
|
|||
labels:
|
||||
app: lineage-controller-webhook
|
||||
spec:
|
||||
serviceAccountName: lineage-controller-webhook
|
||||
{{- with .Values.lineageControllerWebhook.nodeSelector }}
|
||||
nodeSelector:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
tolerations:
|
||||
- operator: Exists
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- weight: 100
|
||||
preference:
|
||||
matchExpressions:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
podAntiAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- weight: 100
|
||||
podAffinityTerm:
|
||||
topologyKey: kubernetes.io/hostname
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
app: lineage-controller-webhook
|
||||
serviceAccountName: lineage-controller-webhook
|
||||
containers:
|
||||
- name: lineage-controller-webhook
|
||||
image: "{{ .Values.lineageControllerWebhook.image }}"
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: lineage-controller-webhook
|
||||
spec:
|
||||
maxUnavailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: lineage-controller-webhook
|
||||
|
|
@ -5,7 +5,7 @@ metadata:
|
|||
labels:
|
||||
app: lineage-controller-webhook
|
||||
spec:
|
||||
trafficDistribution: PreferClose
|
||||
internalTrafficPolicy: Local
|
||||
type: ClusterIP
|
||||
ports:
|
||||
- port: 443
|
||||
|
|
|
|||
|
|
@ -1,30 +0,0 @@
|
|||
suite: lineage-controller-webhook service
|
||||
templates:
|
||||
- templates/service.yaml
|
||||
|
||||
release:
|
||||
name: lineage-controller-webhook
|
||||
namespace: cozy-system
|
||||
|
||||
tests:
|
||||
- it: renders a ClusterIP service on 443 -> 9443 with PreferClose traffic distribution
|
||||
asserts:
|
||||
- isKind:
|
||||
of: Service
|
||||
- equal:
|
||||
path: spec.type
|
||||
value: ClusterIP
|
||||
- equal:
|
||||
path: spec.trafficDistribution
|
||||
value: PreferClose
|
||||
- notExists:
|
||||
path: spec.internalTrafficPolicy
|
||||
- equal:
|
||||
path: spec.ports[0].port
|
||||
value: 443
|
||||
- equal:
|
||||
path: spec.ports[0].targetPort
|
||||
value: 9443
|
||||
- equal:
|
||||
path: spec.selector.app
|
||||
value: lineage-controller-webhook
|
||||
|
|
@ -1,99 +0,0 @@
|
|||
suite: lineage-controller-webhook workload + PDB
|
||||
templates:
|
||||
- templates/workload.yaml
|
||||
- templates/pdb.yaml
|
||||
|
||||
release:
|
||||
name: lineage-controller-webhook
|
||||
namespace: cozy-system
|
||||
|
||||
tests:
|
||||
# ---- Default Deployment shape ----------------------------------------------
|
||||
|
||||
- it: defaults render a 2-replica Deployment with soft control-plane nodeAffinity, soft podAntiAffinity, and Exists tolerations
|
||||
template: templates/workload.yaml
|
||||
asserts:
|
||||
- isKind:
|
||||
of: Deployment
|
||||
- equal:
|
||||
path: spec.replicas
|
||||
value: 2
|
||||
- equal:
|
||||
path: spec.template.spec.tolerations
|
||||
value:
|
||||
- operator: Exists
|
||||
- equal:
|
||||
path: spec.template.spec.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution
|
||||
value:
|
||||
- weight: 100
|
||||
preference:
|
||||
matchExpressions:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
- notExists:
|
||||
path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution
|
||||
- equal:
|
||||
path: spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.topologyKey
|
||||
value: kubernetes.io/hostname
|
||||
|
||||
- it: defaults inject no localK8sAPIEndpoint env vars (knob is opt-in)
|
||||
template: templates/workload.yaml
|
||||
asserts:
|
||||
- notExists:
|
||||
path: spec.template.spec.containers[0].env
|
||||
|
||||
# ---- PDB --------------------------------------------------------------------
|
||||
|
||||
- it: PDB renders unconditionally with maxUnavailable=1 at the default replica count
|
||||
template: templates/pdb.yaml
|
||||
asserts:
|
||||
- isKind:
|
||||
of: PodDisruptionBudget
|
||||
- equal:
|
||||
path: spec.maxUnavailable
|
||||
value: 1
|
||||
- notExists:
|
||||
path: spec.minAvailable
|
||||
|
||||
- it: PDB still renders at replicas=1 (no-op, but consistent shape)
|
||||
set:
|
||||
lineageControllerWebhook.replicas: 1
|
||||
template: templates/pdb.yaml
|
||||
asserts:
|
||||
- isKind:
|
||||
of: PodDisruptionBudget
|
||||
- equal:
|
||||
path: spec.maxUnavailable
|
||||
value: 1
|
||||
|
||||
# ---- Replica override -------------------------------------------------------
|
||||
|
||||
- it: replicas value drives spec.replicas
|
||||
set:
|
||||
lineageControllerWebhook.replicas: 5
|
||||
template: templates/workload.yaml
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.replicas
|
||||
value: 5
|
||||
|
||||
# ---- Deprecated localK8sAPIEndpoint knob -----------------------------------
|
||||
|
||||
- it: localK8sAPIEndpoint.enabled=true injects KUBERNETES_SERVICE_HOST and PORT env vars
|
||||
set:
|
||||
lineageControllerWebhook.localK8sAPIEndpoint.enabled: true
|
||||
template: templates/workload.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: KUBERNETES_SERVICE_HOST
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
apiVersion: v1
|
||||
fieldPath: status.hostIP
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: KUBERNETES_SERVICE_PORT
|
||||
value: "6443"
|
||||
|
|
@ -1,13 +1,10 @@
|
|||
lineageControllerWebhook:
|
||||
image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.3.0@sha256:e8984709686a5eaf19b89da378d7b8c688ea5607e0783a88d9c9e4ccfca96fb0
|
||||
debug: false
|
||||
replicas: 2
|
||||
# DEPRECATED. Injects KUBERNETES_SERVICE_HOST=status.hostIP and
|
||||
# KUBERNETES_SERVICE_PORT=6443 so the webhook talks to the apiserver on its
|
||||
# own node — only valid when the pod is actually scheduled on an apiserver-
|
||||
# bearing node. Now that the chart's nodeAffinity to control-plane is soft
|
||||
# (preferred, not required), the pod can land off-control-plane and crash-
|
||||
# loop dialing a non-apiserver hostIP. Slated for removal once webhook
|
||||
# performance work obviates the locality motivation. Leave disabled.
|
||||
localK8sAPIEndpoint:
|
||||
enabled: false
|
||||
enabled: true
|
||||
# nodeSelector for the DaemonSet
|
||||
# Talos uses empty value: "node-role.kubernetes.io/control-plane": ""
|
||||
# Generic k8s (k3s, kubeadm) uses: "node-role.kubernetes.io/control-plane": "true"
|
||||
nodeSelector:
|
||||
node-role.kubernetes.io/control-plane: ""
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue