From 9ee5f5d2f4f319f90c4ae1cb58d46f1844c17500 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 2 Apr 2026 14:58:45 +0300 Subject: [PATCH 1/5] feat(gpu-operator): update NVIDIA GPU Operator from v25.3.0 to v26.3.0 Update vendored upstream chart to v26.3.0. Key changes: - CDI enabled by default - Driver version 580.126.20 - NFD subchart updated to 0.18.3 - Component versions bumped (toolkit v1.19.0, device-plugin v0.19.0, dcgm-exporter 4.5.2) - Pin chart version in Makefile for reproducibility Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- packages/system/gpu-operator/Makefile | 2 +- .../charts/gpu-operator/Chart.lock | 8 +- .../charts/gpu-operator/Chart.yaml | 8 +- .../charts/node-feature-discovery/Chart.yaml | 5 +- .../charts/node-feature-discovery/README.md | 2 +- .../crds/nfd-api-crds.yaml | 77 ++++ .../templates/_helpers.tpl | 19 + .../templates/master-pdb.yaml | 17 + .../templates/master.yaml | 35 +- .../templates/nfd-gc-pdb.yaml | 17 + .../templates/nfd-gc.yaml | 46 +- .../templates/post-delete-job.yaml | 8 + .../templates/prometheus.yaml | 2 +- .../templates/topologyupdater.yaml | 26 +- .../templates/worker.yaml | 30 +- .../charts/node-feature-discovery/values.yaml | 77 ++-- .../crds/nvidia.com_clusterpolicies.yaml | 431 ++++++++++++++++-- .../crds/nvidia.com_nvidiadrivers.yaml | 248 +++++++++- .../gpu-operator/templates/cleanup_crd.yaml | 19 +- .../gpu-operator/templates/clusterpolicy.yaml | 83 +++- .../gpu-operator/templates/extra-objects.yaml | 8 + .../templates/nodefeaturerules.yaml | 108 +---- .../gpu-operator/templates/nvidiadriver.yaml | 11 +- .../gpu-operator/templates/operator.yaml | 9 +- .../charts/gpu-operator/templates/role.yaml | 11 + .../gpu-operator/templates/rolebinding.yaml | 1 + .../templates/serviceaccount.yaml | 1 + .../gpu-operator/templates/upgrade_crd.yaml | 13 +- .../gpu-operator/templates/validations.yaml | 3 + .../charts/gpu-operator/values.yaml | 213 ++++----- 30 files changed, 1152 insertions(+), 386 deletions(-) create mode 100644 packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/master-pdb.yaml create mode 100644 packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/nfd-gc-pdb.yaml create mode 100644 packages/system/gpu-operator/charts/gpu-operator/templates/extra-objects.yaml create mode 100644 packages/system/gpu-operator/charts/gpu-operator/templates/validations.yaml diff --git a/packages/system/gpu-operator/Makefile b/packages/system/gpu-operator/Makefile index 49b26e26..11ca047e 100644 --- a/packages/system/gpu-operator/Makefile +++ b/packages/system/gpu-operator/Makefile @@ -8,4 +8,4 @@ update: rm -rf charts helm repo add nvidia https://helm.ngc.nvidia.com/nvidia helm repo update nvidia - helm pull nvidia/gpu-operator --untar --untardir charts + helm pull nvidia/gpu-operator --untar --untardir charts --version v26.3.0 diff --git a/packages/system/gpu-operator/charts/gpu-operator/Chart.lock b/packages/system/gpu-operator/charts/gpu-operator/Chart.lock index 14674306..707da9d1 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/Chart.lock +++ b/packages/system/gpu-operator/charts/gpu-operator/Chart.lock @@ -1,6 +1,6 @@ dependencies: - name: node-feature-discovery - repository: https://kubernetes-sigs.github.io/node-feature-discovery/charts - version: 0.17.2 -digest: sha256:4c55d30d958027ef8997a2976449326de3c90049025c3ebb9bee017cad32cc3f -generated: "2025-02-25T09:08:49.128088-08:00" + repository: oci://registry.k8s.io/nfd/charts + version: 0.18.3 +digest: sha256:1bce3b80da1e6d47a0befbaa6a2f758aeac6d20382e900f0c301f45020afbebc +generated: "2026-01-23T15:08:56.617442-08:00" diff --git a/packages/system/gpu-operator/charts/gpu-operator/Chart.yaml b/packages/system/gpu-operator/charts/gpu-operator/Chart.yaml index f14fc2ad..c2138a54 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/Chart.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/Chart.yaml @@ -1,10 +1,10 @@ apiVersion: v2 -appVersion: v25.3.0 +appVersion: v26.3.0 dependencies: - condition: nfd.enabled name: node-feature-discovery - repository: https://kubernetes-sigs.github.io/node-feature-discovery/charts - version: v0.17.2 + repository: oci://registry.k8s.io/nfd/charts + version: 0.18.3 description: NVIDIA GPU Operator creates/configures/manages GPUs atop Kubernetes home: https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/overview.html icon: https://assets.nvidiagrid.net/ngc/logos/GPUoperator.png @@ -20,4 +20,4 @@ kubeVersion: '>= 1.16.0-0' name: gpu-operator sources: - https://github.com/NVIDIA/gpu-operator -version: v25.3.0 +version: v26.3.0 diff --git a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/Chart.yaml b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/Chart.yaml index f62d84c2..8609178c 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/Chart.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/Chart.yaml @@ -1,8 +1,9 @@ apiVersion: v2 -appVersion: v0.17.2 +appVersion: v0.18.3 description: 'Detects hardware features available on each node in a Kubernetes cluster, and advertises those features using node labels. ' home: https://github.com/kubernetes-sigs/node-feature-discovery +icon: https://kubernetes-sigs.github.io/node-feature-discovery/v0.18/assets/images/nfd/favicon.svg keywords: - feature-discovery - feature-detection @@ -11,4 +12,4 @@ name: node-feature-discovery sources: - https://github.com/kubernetes-sigs/node-feature-discovery type: application -version: 0.17.2 +version: 0.18.3 diff --git a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/README.md b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/README.md index 02f7b170..2b46e4af 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/README.md +++ b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/README.md @@ -6,5 +6,5 @@ labels. NFD provides flexible configuration and extension points for a wide range of vendor and application specific node labeling needs. See -[NFD documentation](https://kubernetes-sigs.github.io/node-feature-discovery/v0.17/deployment/helm.html) +[NFD documentation](https://kubernetes-sigs.github.io/node-feature-discovery/v0.18/deployment/helm.html) for deployment instructions. diff --git a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/crds/nfd-api-crds.yaml b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/crds/nfd-api-crds.yaml index 9f62da6f..69e306c3 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/crds/nfd-api-crds.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/crds/nfd-api-crds.yaml @@ -204,11 +204,19 @@ spec: - Exists - DoesNotExist - Gt + - Ge - Lt + - Le - GtLt + - GeLe - IsTrue - IsFalse type: string + type: + description: |- + Type defines the value type for specific operators. + The currently supported type is 'version' for Gt,Ge,Lt,Le,GtLt,GeLe operators. + type: string value: description: |- Value is the list of values that the operand evaluates the input @@ -240,11 +248,19 @@ spec: - Exists - DoesNotExist - Gt + - Ge - Lt + - Le - GtLt + - GeLe - IsTrue - IsFalse type: string + type: + description: |- + Type defines the value type for specific operators. + The currently supported type is 'version' for Gt,Ge,Lt,Le,GtLt,GeLe operators. + type: string value: description: |- Value is the list of values that the operand evaluates the input @@ -295,11 +311,19 @@ spec: - Exists - DoesNotExist - Gt + - Ge - Lt + - Le - GtLt + - GeLe - IsTrue - IsFalse type: string + type: + description: |- + Type defines the value type for specific operators. + The currently supported type is 'version' for Gt,Ge,Lt,Le,GtLt,GeLe operators. + type: string value: description: |- Value is the list of values that the operand evaluates the input @@ -331,11 +355,19 @@ spec: - Exists - DoesNotExist - Gt + - Ge - Lt + - Le - GtLt + - GeLe - IsTrue - IsFalse type: string + type: + description: |- + Type defines the value type for specific operators. + The currently supported type is 'version' for Gt,Ge,Lt,Le,GtLt,GeLe operators. + type: string value: description: |- Value is the list of values that the operand evaluates the input @@ -356,6 +388,19 @@ spec: name: description: Name of the rule. type: string + vars: + additionalProperties: + type: string + description: |- + Vars is the variables to store if the rule matches. Variables can be + referenced from other rules enabling more complex rule hierarchies. + type: object + varsTemplate: + description: |- + VarsTemplate specifies a template to expand for dynamically generating + multiple variables. Data (after template expansion) must be keys with an + optional value ([=]) separated by newlines. + type: string required: - name type: object @@ -498,11 +543,19 @@ spec: - Exists - DoesNotExist - Gt + - Ge - Lt + - Le - GtLt + - GeLe - IsTrue - IsFalse type: string + type: + description: |- + Type defines the value type for specific operators. + The currently supported type is 'version' for Gt,Ge,Lt,Le,GtLt,GeLe operators. + type: string value: description: |- Value is the list of values that the operand evaluates the input @@ -534,11 +587,19 @@ spec: - Exists - DoesNotExist - Gt + - Ge - Lt + - Le - GtLt + - GeLe - IsTrue - IsFalse type: string + type: + description: |- + Type defines the value type for specific operators. + The currently supported type is 'version' for Gt,Ge,Lt,Le,GtLt,GeLe operators. + type: string value: description: |- Value is the list of values that the operand evaluates the input @@ -589,11 +650,19 @@ spec: - Exists - DoesNotExist - Gt + - Ge - Lt + - Le - GtLt + - GeLe - IsTrue - IsFalse type: string + type: + description: |- + Type defines the value type for specific operators. + The currently supported type is 'version' for Gt,Ge,Lt,Le,GtLt,GeLe operators. + type: string value: description: |- Value is the list of values that the operand evaluates the input @@ -625,11 +694,19 @@ spec: - Exists - DoesNotExist - Gt + - Ge - Lt + - Le - GtLt + - GeLe - IsTrue - IsFalse type: string + type: + description: |- + Type defines the value type for specific operators. + The currently supported type is 'version' for Gt,Ge,Lt,Le,GtLt,GeLe operators. + type: string value: description: |- Value is the list of values that the operand evaluates the input diff --git a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/_helpers.tpl b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/_helpers.tpl index 928ece78..b540e3bb 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/_helpers.tpl +++ b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/_helpers.tpl @@ -105,3 +105,22 @@ Create the name of the service account which nfd-gc will use {{ default "default" .Values.gc.serviceAccount.name }} {{- end -}} {{- end -}} + +{{/* +imagePullSecrets helper - uses local values or falls back to global values +*/}} +{{- define "node-feature-discovery.imagePullSecrets" -}} +{{- $imagePullSecrets := list -}} +{{- if .Values.imagePullSecrets -}} + {{- range .Values.imagePullSecrets -}} + {{- $imagePullSecrets = append $imagePullSecrets . -}} + {{- end -}} +{{- else if and .Values.global .Values.global.imagePullSecrets -}} + {{- range .Values.global.imagePullSecrets -}} + {{- $imagePullSecrets = append $imagePullSecrets . -}} + {{- end -}} +{{- end -}} +{{- if $imagePullSecrets -}} +{{- $imagePullSecrets | toJson }} +{{- end -}} +{{- end -}} diff --git a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/master-pdb.yaml b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/master-pdb.yaml new file mode 100644 index 00000000..532b0e6d --- /dev/null +++ b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/master-pdb.yaml @@ -0,0 +1,17 @@ +{{- if .Values.master.enable }} +{{- if .Values.master.podDisruptionBudget.enable -}} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-master + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "node-feature-discovery.selectorLabels" . | nindent 6 }} + role: master +{{- toYaml (omit .Values.master.podDisruptionBudget "enable") | nindent 2 }} +{{- end }} +{{- end }} diff --git a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/master.yaml b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/master.yaml index da3ca240..666af5c5 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/master.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/master.yaml @@ -29,13 +29,11 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} spec: + dnsPolicy: {{ .Values.master.dnsPolicy }} {{- with .Values.priorityClassName }} priorityClassName: {{ . }} {{- end }} - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} + imagePullSecrets: {{ include "node-feature-discovery.imagePullSecrets" . }} serviceAccountName: {{ include "node-feature-discovery.master.serviceAccountName" . }} enableServiceLinks: false securityContext: @@ -48,8 +46,9 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} startupProbe: - grpc: - port: {{ .Values.master.healthPort | default "8082" }} + httpGet: + path: /healthz + port: http {{- with .Values.master.startupProbe.initialDelaySeconds }} initialDelaySeconds: {{ . }} {{- end }} @@ -63,8 +62,9 @@ spec: timeoutSeconds: {{ . }} {{- end }} livenessProbe: - grpc: - port: {{ .Values.master.healthPort | default "8082" }} + httpGet: + path: /healthz + port: http {{- with .Values.master.livenessProbe.initialDelaySeconds }} initialDelaySeconds: {{ . }} {{- end }} @@ -78,8 +78,9 @@ spec: timeoutSeconds: {{ . }} {{- end }} readinessProbe: - grpc: - port: {{ .Values.master.healthPort | default "8082" }} + httpGet: + path: /healthz + port: http {{- with .Values.master.readinessProbe.initialDelaySeconds }} initialDelaySeconds: {{ . }} {{- end }} @@ -96,17 +97,15 @@ spec: successThreshold: {{ . }} {{- end }} ports: - - containerPort: {{ .Values.master.metricsPort | default "8081" }} - name: metrics - - containerPort: {{ .Values.master.healthPort | default "8082" }} - name: health + - containerPort: {{ .Values.master.port | default "8080" }} + name: http env: - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName {{- with .Values.master.extraEnvs }} - {{- toYaml . | nindent 8 }} + {{- toYaml . | nindent 10 }} {{- end}} command: - "nfd-master" @@ -126,9 +125,6 @@ spec: {{- if .Values.master.enableTaints }} - "-enable-taints" {{- end }} - {{- if .Values.master.featureRulesController | kindIs "invalid" | not }} - - "-featurerules-controller={{ .Values.master.featureRulesController }}" - {{- end }} {{- if .Values.master.resyncPeriod }} - "-resync-period={{ .Values.master.resyncPeriod }}" {{- end }} @@ -139,8 +135,7 @@ spec: {{- range $key, $value := .Values.featureGates }} - "-feature-gates={{ $key }}={{ $value }}" {{- end }} - - "-metrics={{ .Values.master.metricsPort | default "8081" }}" - - "-grpc-health={{ .Values.master.healthPort | default "8082" }}" + - "-port={{ .Values.master.port | default "8080" }}" {{- with .Values.master.extraArgs }} {{- toYaml . | nindent 12 }} {{- end }} diff --git a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/nfd-gc-pdb.yaml b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/nfd-gc-pdb.yaml new file mode 100644 index 00000000..373ed92c --- /dev/null +++ b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/nfd-gc-pdb.yaml @@ -0,0 +1,17 @@ +{{- if .Values.gc.enable }} +{{- if .Values.gc.podDisruptionBudget.enable -}} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-gc + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "node-feature-discovery.selectorLabels" . | nindent 6 }} + role: gc +{{- toYaml (omit .Values.gc.podDisruptionBudget "enable") | nindent 2 }} +{{- end }} +{{- end }} diff --git a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/nfd-gc.yaml b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/nfd-gc.yaml index 3642aa64..c0e8d083 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/nfd-gc.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/nfd-gc.yaml @@ -29,14 +29,11 @@ spec: {{- end }} spec: serviceAccountName: {{ include "node-feature-discovery.gc.serviceAccountName" . }} - dnsPolicy: ClusterFirstWithHostNet + dnsPolicy: {{ .Values.gc.dnsPolicy }} {{- with .Values.priorityClassName }} priorityClassName: {{ . }} {{- end }} - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} + imagePullSecrets: {{ include "node-feature-discovery.imagePullSecrets" . }} securityContext: {{- toYaml .Values.gc.podSecurityContext | nindent 8 }} hostNetwork: {{ .Values.gc.hostNetwork }} @@ -44,6 +41,41 @@ spec: - name: gc image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: "{{ .Values.image.pullPolicy }}" + livenessProbe: + httpGet: + path: /healthz + port: http + {{- with .Values.gc.livenessProbe.initialDelaySeconds }} + initialDelaySeconds: {{ . }} + {{- end }} + {{- with .Values.gc.livenessProbe.failureThreshold }} + failureThreshold: {{ . }} + {{- end }} + {{- with .Values.gc.livenessProbe.periodSeconds }} + periodSeconds: {{ . }} + {{- end }} + {{- with .Values.gc.livenessProbe.timeoutSeconds }} + timeoutSeconds: {{ . }} + {{- end }} + readinessProbe: + httpGet: + path: /healthz + port: http + {{- with .Values.gc.readinessProbe.initialDelaySeconds }} + initialDelaySeconds: {{ . }} + {{- end }} + {{- with .Values.gc.readinessProbe.failureThreshold }} + failureThreshold: {{ . }} + {{- end }} + {{- with .Values.gc.readinessProbe.periodSeconds }} + periodSeconds: {{ . }} + {{- end }} + {{- with .Values.gc.readinessProbe.timeoutSeconds }} + timeoutSeconds: {{ . }} + {{- end }} + {{- with .Values.gc.readinessProbe.successThreshold }} + successThreshold: {{ . }} + {{- end }} env: - name: NODE_NAME valueFrom: @@ -70,8 +102,8 @@ spec: readOnlyRootFilesystem: true runAsNonRoot: true ports: - - name: metrics - containerPort: {{ .Values.gc.metricsPort | default "8081"}} + - name: http + containerPort: {{ .Values.gc.port | default "8080"}} {{- with .Values.gc.nodeSelector }} nodeSelector: diff --git a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/post-delete-job.yaml b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/post-delete-job.yaml index 4364f1aa..ddb1c9db 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/post-delete-job.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/post-delete-job.yaml @@ -1,3 +1,4 @@ +{{- if .Values.postDeleteCleanup }} apiVersion: v1 kind: ServiceAccount metadata: @@ -66,6 +67,7 @@ spec: role: prune spec: serviceAccountName: {{ include "node-feature-discovery.fullname" . }}-prune + imagePullSecrets: {{ include "node-feature-discovery.imagePullSecrets" . }} containers: - name: nfd-master securityContext: @@ -92,3 +94,9 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} + {{- with .Values.master.resources }} + resources: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} + diff --git a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/prometheus.yaml b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/prometheus.yaml index 3d680e24..bed37f4c 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/prometheus.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/prometheus.yaml @@ -14,7 +14,7 @@ spec: - honorLabels: true interval: {{ .Values.prometheus.scrapeInterval }} path: /metrics - port: metrics + port: http scheme: http namespaceSelector: matchNames: diff --git a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/topologyupdater.yaml b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/topologyupdater.yaml index 9a466f88..010e8a0b 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/topologyupdater.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/topologyupdater.yaml @@ -29,14 +29,11 @@ spec: {{- end }} spec: serviceAccountName: {{ include "node-feature-discovery.topologyUpdater.serviceAccountName" . }} - dnsPolicy: ClusterFirstWithHostNet + dnsPolicy: {{ .Values.topologyUpdater.dnsPolicy }} {{- with .Values.priorityClassName }} priorityClassName: {{ . }} {{- end }} - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} + imagePullSecrets: {{ include "node-feature-discovery.imagePullSecrets" . }} securityContext: {{- toYaml .Values.topologyUpdater.podSecurityContext | nindent 8 }} hostNetwork: {{ .Values.topologyUpdater.hostNetwork }} @@ -45,8 +42,9 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: "{{ .Values.image.pullPolicy }}" livenessProbe: - grpc: - port: {{ .Values.topologyUpdater.healthPort | default "8082" }} + httpGet: + path: /healthz + port: http {{- with .Values.topologyUpdater.livenessProbe.initialDelaySeconds }} initialDelaySeconds: {{ . }} {{- end }} @@ -60,8 +58,9 @@ spec: timeoutSeconds: {{ . }} {{- end }} readinessProbe: - grpc: - port: {{ .Values.topologyUpdater.healthPort | default "8082" }} + httpGet: + path: /healthz + port: http {{- with .Values.topologyUpdater.readinessProbe.initialDelaySeconds }} initialDelaySeconds: {{ . }} {{- end }} @@ -113,16 +112,13 @@ spec: # Disable kubelet state tracking by giving an empty path - "-kubelet-state-dir=" {{- end }} - - "-metrics={{ .Values.topologyUpdater.metricsPort | default "8081"}}" - - "-grpc-health={{ .Values.topologyUpdater.healthPort | default "8082" }}" + - "-port={{ .Values.topologyUpdater.port | default "8080"}}" {{- with .Values.topologyUpdater.extraArgs }} {{- toYaml . | nindent 10 }} {{- end }} ports: - - containerPort: {{ .Values.topologyUpdater.metricsPort | default "8081"}} - name: metrics - - containerPort: {{ .Values.topologyUpdater.healthPort | default "8082" }} - name: health + - containerPort: {{ .Values.topologyUpdater.port | default "8080"}} + name: http volumeMounts: {{- if .Values.topologyUpdater.kubeletConfigPath | empty | not }} - name: kubelet-config diff --git a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/worker.yaml b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/worker.yaml index 4aadd800..58c3e041 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/worker.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/templates/worker.yaml @@ -13,6 +13,10 @@ metadata: {{- end }} spec: revisionHistoryLimit: {{ .Values.worker.revisionHistoryLimit }} + {{- with .Values.worker.updateStrategy }} + updateStrategy: + {{- toYaml . | nindent 4 }} + {{- end}} selector: matchLabels: {{- include "node-feature-discovery.selectorLabels" . | nindent 6 }} @@ -28,14 +32,11 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} spec: - dnsPolicy: ClusterFirstWithHostNet + dnsPolicy: {{ .Values.worker.dnsPolicy }} {{- with .Values.priorityClassName }} priorityClassName: {{ . }} {{- end }} - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} + imagePullSecrets: {{ include "node-feature-discovery.imagePullSecrets" . }} serviceAccountName: {{ include "node-feature-discovery.worker.serviceAccountName" . }} securityContext: {{- toYaml .Values.worker.podSecurityContext | nindent 8 }} @@ -47,8 +48,9 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} livenessProbe: - grpc: - port: {{ .Values.worker.healthPort | default "8082" }} + httpGet: + path: /healthz + port: http {{- with .Values.worker.livenessProbe.initialDelaySeconds }} initialDelaySeconds: {{ . }} {{- end }} @@ -62,8 +64,9 @@ spec: timeoutSeconds: {{ . }} {{- end }} readinessProbe: - grpc: - port: {{ .Values.worker.healthPort | default "8082" }} + httpGet: + path: /healthz + port: http {{- with .Values.worker.readinessProbe.initialDelaySeconds }} initialDelaySeconds: {{ . }} {{- end }} @@ -104,16 +107,13 @@ spec: {{- range $key, $value := .Values.featureGates }} - "-feature-gates={{ $key }}={{ $value }}" {{- end }} - - "-metrics={{ .Values.worker.metricsPort | default "8081"}}" - - "-grpc-health={{ .Values.worker.healthPort | default "8082" }}" + - "-port={{ .Values.worker.port | default "8080"}}" {{- with .Values.worker.extraArgs }} {{- toYaml . | nindent 8 }} {{- end }} ports: - - containerPort: {{ .Values.worker.metricsPort | default "8081"}} - name: metrics - - containerPort: {{ .Values.worker.healthPort | default "8082" }} - name: health + - containerPort: {{ .Values.worker.port | default "8080"}} + name: http volumeMounts: - name: host-boot mountPath: "/host-boot" diff --git a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/values.yaml b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/values.yaml index 18aa7bcb..86712bd2 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/values.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/charts/node-feature-discovery/values.yaml @@ -15,17 +15,20 @@ featureGates: priorityClassName: "" +postDeleteCleanup: true + master: enable: true extraArgs: [] extraEnvs: [] hostNetwork: false + dnsPolicy: ClusterFirstWithHostNet config: ### # noPublish: false - # autoDefaultNs: true # extraLabelNs: ["added.ns.io","added.kubernets.io"] # denyLabelNs: ["denied.ns.io","denied.kubernetes.io"] # enableTaints: false + # informerPageSize: 200 # labelWhiteList: "foo" # resyncPeriod: "2h" # restrictions: @@ -66,15 +69,12 @@ master: # retryPeriod: 2s # nfdApiParallelism: 10 ### - metricsPort: 8081 - healthPort: 8082 + port: 8080 instance: - featureApi: resyncPeriod: denyLabelNs: [] extraLabelNs: [] enableTaints: false - featureRulesController: null nfdApiParallelism: null deploymentAnnotations: {} replicaCount: 1 @@ -120,26 +120,21 @@ master: nodeSelector: {} tolerations: - - key: "node-role.kubernetes.io/master" - operator: "Equal" - value: "" - effect: "NoSchedule" - key: "node-role.kubernetes.io/control-plane" operator: "Equal" value: "" effect: "NoSchedule" + podDisruptionBudget: + enable: false + minAvailable: 1 + unhealthyPodEvictionPolicy: AlwaysAllow + annotations: {} affinity: nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: - - weight: 1 - preference: - matchExpressions: - - key: "node-role.kubernetes.io/master" - operator: In - values: [""] - weight: 1 preference: matchExpressions: @@ -148,20 +143,14 @@ master: values: [""] startupProbe: - grpc: - port: 8082 failureThreshold: 30 # periodSeconds: 10 - livenessProbe: - grpc: - port: 8082 + livenessProbe: {} # failureThreshold: 3 # initialDelaySeconds: 0 # periodSeconds: 10 # timeoutSeconds: 1 readinessProbe: - grpc: - port: 8082 failureThreshold: 10 # initialDelaySeconds: 0 # periodSeconds: 10 @@ -173,6 +162,7 @@ worker: extraArgs: [] extraEnvs: [] hostNetwork: false + dnsPolicy: ClusterFirstWithHostNet config: ### #core: # labelWhiteList: @@ -416,8 +406,7 @@ worker: # matchName: {op: In, value: ["SWAP", "X86", "ARM"]} ### - metricsPort: 8081 - healthPort: 8082 + port: 8080 daemonsetAnnotations: {} podSecurityContext: {} # fsGroup: 2000 @@ -431,15 +420,11 @@ worker: # runAsUser: 1000 livenessProbe: - grpc: - port: 8082 initialDelaySeconds: 10 # failureThreshold: 3 # periodSeconds: 10 # timeoutSeconds: 1 readinessProbe: - grpc: - port: 8082 initialDelaySeconds: 5 failureThreshold: 10 # periodSeconds: 10 @@ -483,6 +468,12 @@ worker: priorityClassName: "" + updateStrategy: {} + # type: RollingUpdate + # rollingUpdate: + # maxSurge: 0 + # maxUnavailable: "10%" + topologyUpdater: config: ### ## key = node name, value = list of resources to be excluded. @@ -499,6 +490,7 @@ topologyUpdater: extraArgs: [] extraEnvs: [] hostNetwork: false + dnsPolicy: ClusterFirstWithHostNet serviceAccount: create: true @@ -511,8 +503,7 @@ topologyUpdater: rbac: create: true - metricsPort: 8081 - healthPort: 8082 + port: 8080 kubeletConfigPath: kubeletPodResourcesSockPath: updateInterval: 60s @@ -526,17 +517,13 @@ topologyUpdater: drop: [ "ALL" ] readOnlyRootFilesystem: true runAsUser: 0 - + livenessProbe: - grpc: - port: 8082 initialDelaySeconds: 10 # failureThreshold: 3 # periodSeconds: 10 # timeoutSeconds: 1 readinessProbe: - grpc: - port: 8082 initialDelaySeconds: 5 failureThreshold: 10 # periodSeconds: 10 @@ -563,6 +550,7 @@ gc: extraEnvs: [] hostNetwork: false replicaCount: 1 + dnsPolicy: ClusterFirstWithHostNet serviceAccount: create: true @@ -575,6 +563,18 @@ gc: podSecurityContext: {} + livenessProbe: + initialDelaySeconds: 10 + # failureThreshold: 3 + # periodSeconds: 10 + # timeoutSeconds: 1 + readinessProbe: + initialDelaySeconds: 5 + # failureThreshold: 3 + # periodSeconds: 10 + # timeoutSeconds: 1 + # successThreshold: 1 + resources: limits: memory: 1Gi @@ -582,7 +582,7 @@ gc: cpu: 10m memory: 128Mi - metricsPort: 8081 + port: 8080 nodeSelector: {} tolerations: [] @@ -590,6 +590,11 @@ gc: deploymentAnnotations: {} affinity: {} + podDisruptionBudget: + enable: false + minAvailable: 1 + unhealthyPodEvictionPolicy: AlwaysAllow + # specify how many old ReplicaSets for the Deployment to retain. revisionHistoryLimit: diff --git a/packages/system/gpu-operator/charts/gpu-operator/crds/nvidia.com_clusterpolicies.yaml b/packages/system/gpu-operator/charts/gpu-operator/crds/nvidia.com_clusterpolicies.yaml index 5a9f99e1..030581fc 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/crds/nvidia.com_clusterpolicies.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/crds/nvidia.com_clusterpolicies.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.17.2 + controller-gen.kubebuilder.io/version: v0.20.1 name: clusterpolicies.nvidia.com spec: group: nvidia.com @@ -136,13 +136,21 @@ spec: properties: default: default: false - description: Default indicates whether to use CDI as the default - mechanism for providing GPU access to containers. + description: 'Deprecated: This field is no longer used. Setting + cdi.enabled=true will configure CDI as the default mechanism + for making GPUs accessible to containers.' type: boolean enabled: + default: true + description: Enabled indicates whether the Container Device Interface + (CDI) should be used as the mechanism for making GPUs accessible + to containers. + type: boolean + nriPluginEnabled: default: false - description: Enabled indicates whether CDI can be used to make - GPUs accessible to containers. + description: NRIPluginEnabled indicates whether an NRI Plugin + should be run as a means of injecting CDI devices to gpu management + containers. type: boolean type: object daemonsets: @@ -164,6 +172,239 @@ spec: (scope and select) objects. May match selectors of replication controllers and services. type: object + podSecurityContext: + description: 'Optional: Set pod-level security context for all + DaemonSet pods (applies as defaults to all containers)' + properties: + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxChangePolicy: + description: |- + seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. + It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. + Valid values are "MountOption" and "Recursive". + + "Recursive" means relabeling of all files on all Pod volumes by the container runtime. + This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. + + "MountOption" mounts all eligible Pod volumes with `-o context` mount option. + This requires all Pods that share the same volume to use the same SELinux label. + It is not possible to share the same volume among privileged and unprivileged Pods. + Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes + whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their + CSIDriver instance. Other volumes are always re-labelled recursively. + "MountOption" value is allowed only when SELinuxMount feature gate is enabled. + + If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. + If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes + and "Recursive" for all other volumes. + + This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. + + All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. + Note that this field cannot be set when spec.os.name is windows. + type: string + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object priorityClassName: type: string rollingUpdate: @@ -193,9 +434,10 @@ spec: operator: description: |- Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string tolerationSeconds: description: |- @@ -337,6 +579,27 @@ spec: - name type: object type: array + hostNetwork: + description: HostNetwork allows the DCGM-Exporter daemon set to + expose metrics port on the host's network namespace. + type: boolean + hostPID: + description: HostPID allows the DCGM-Exporter daemon set to access + the host's PID namespace + type: boolean + hpcJobMapping: + description: 'Optional: HPC job mapping configuration for NVIDIA + DCGM Exporter' + properties: + directory: + description: |- + Directory path where HPC job mapping files are created by the workload manager + Defaults to /var/lib/dcgm-exporter/job-mapping if not specified + type: string + enabled: + description: Enable HPC job mapping for DCGM Exporter + type: boolean + type: object image: description: NVIDIA DCGM Exporter image name pattern: '[a-zA-Z0-9\-]+' @@ -381,6 +644,19 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + service: + description: 'Optional: Service configuration for NVIDIA DCGM + Exporter' + properties: + internalTrafficPolicy: + description: InternalTrafficPolicy describes how nodes distribute + service traffic they receive on the ClusterIP. + type: string + type: + description: Type represents the ServiceType which describes + ingress methods for a service + type: string + type: object serviceMonitor: description: 'Optional: ServiceMonitor configuration for NVIDIA DCGM Exporter' @@ -418,7 +694,7 @@ spec: action: default: replace description: |- - Action to perform based on the regex matching. + action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -450,41 +726,41 @@ spec: type: string modulus: description: |- - Modulus to take of the hash of the source label values. + modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: Regular expression against which the extracted - value is matched. + description: regex defines the regular expression against + which the extracted value is matched. type: string replacement: description: |- - Replacement value against which a Replace action is performed if the + replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: Separator is the string between concatenated + description: separator defines the string between concatenated SourceLabels. type: string sourceLabels: description: |- - The source labels select values from existing labels. Their content is + sourceLabels defines the source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + LabelName is a valid Prometheus label name. + For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. + For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. type: string type: array targetLabel: description: |- - Label to which the resulting string is written in a replacement. + targetLabel defines the label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -666,11 +942,18 @@ spec: licensing' properties: configMapName: + description: 'Deprecated: ConfigMapName has been deprecated + in favour of SecretName. Please use secrets to handle the + licensing server configuration more securely' type: string nlsEnabled: description: NLSEnabled indicates if NVIDIA Licensing System is used for licensing. type: boolean + secretName: + description: SecretName indicates the name of the secret containing + the licensing token + type: string type: object livenessProbe: description: NVIDIA Driver container liveness probe settings @@ -844,6 +1127,10 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + secretEnv: + description: 'Optional: SecretEnv represents the name of the Kubernetes + Secret with secret environment variables for the NVIDIA Driver' + type: string startupProbe: description: NVIDIA Driver container startup probe settings properties: @@ -1323,6 +1610,82 @@ spec: description: Kata Manager image tag type: string type: object + kataSandboxDevicePlugin: + description: KataSandboxDevicePlugin component spec + properties: + args: + description: 'Optional: List of arguments' + items: + type: string + type: array + enabled: + description: Enabled indicates if deployment of NVIDIA component + through operator is enabled + type: boolean + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + image: + description: NVIDIA component image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + repository: + description: NVIDIA component image repository + type: string + resources: + description: 'Optional: Define resources requests and limits for + each pod' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + version: + description: NVIDIA component image tag + type: string + type: object mig: description: MIG spec properties: @@ -1356,8 +1719,8 @@ spec: - "" type: string name: - default: default-mig-parted-config - description: ConfigMap name + description: ConfigMap name. If not specified, MIG configuration + will be dynamically generated from hardware. type: string type: object enabled: @@ -1524,16 +1887,12 @@ spec: queryable and should be preserved when modifying objects. type: object defaultRuntime: - default: docker - description: Runtime defines container runtime type - enum: - - docker - - crio - - containerd + description: 'Deprecated: DefaultRuntime is no longer used by + the gpu-operator. This is instead, detected at runtime.' type: string initContainer: - description: InitContainerSpec describes configuration for initContainer - image used with all components + description: 'Deprecated: InitContainerSpec describes configuration + for initContainer image used with all components' properties: image: description: Image represents image name @@ -1570,8 +1929,6 @@ spec: image should be used on OpenShift to build and install driver modules type: boolean - required: - - defaultRuntime type: object psa: description: PSA defines spec for PodSecurityAdmission configuration @@ -1686,6 +2043,15 @@ spec: Enabled indicates if the GPU Operator should manage additional operands required for sandbox workloads (i.e. VFIO Manager, vGPU Manager, and additional device plugins) type: boolean + mode: + default: kubevirt + description: |- + Mode indicates the sandbox mode. Accepted values are "kubevirt" + and "kata". The default value is "kubevirt". + enum: + - kubevirt + - kata + type: string type: object toolkit: description: Toolkit component spec @@ -2267,6 +2633,13 @@ spec: items: type: string type: array + kernelModuleConfig: + description: 'Optional: Kernel module configuration parameters + for the vGPU manager' + properties: + name: + type: string + type: object repository: description: NVIDIA vGPU Manager image repository type: string diff --git a/packages/system/gpu-operator/charts/gpu-operator/crds/nvidia.com_nvidiadrivers.yaml b/packages/system/gpu-operator/charts/gpu-operator/crds/nvidia.com_nvidiadrivers.yaml index 97e023bf..20697b9b 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/crds/nvidia.com_nvidiadrivers.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/crds/nvidia.com_nvidiadrivers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.17.2 + controller-gen.kubebuilder.io/version: v0.20.1 name: nvidiadrivers.nvidia.com spec: group: nvidia.com @@ -234,11 +234,16 @@ spec: description: 'Optional: Licensing configuration for NVIDIA vGPU licensing' properties: name: + description: 'Deprecated: ConfigMapName has been deprecated in + favour of SecretName. Please use secrets to handle the licensing + server configuration more securely' type: string nlsEnabled: description: NLSEnabled indicates if NVIDIA Licensing System is used for licensing. type: boolean + secretName: + type: string type: object livenessProbe: description: NVIDIA Driver container liveness probe settings @@ -522,6 +527,239 @@ spec: description: NodeSelector specifies a selector for installation of NVIDIA driver type: object + podSecurityContext: + description: 'Optional: Set pod-level security context for driver + pod' + properties: + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxChangePolicy: + description: |- + seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. + It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. + Valid values are "MountOption" and "Recursive". + + "Recursive" means relabeling of all files on all Pod volumes by the container runtime. + This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. + + "MountOption" mounts all eligible Pod volumes with `-o context` mount option. + This requires all Pods that share the same volume to use the same SELinux label. + It is not possible to share the same volume among privileged and unprivileged Pods. + Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes + whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their + CSIDriver instance. Other volumes are always re-labelled recursively. + "MountOption" value is allowed only when SELinuxMount feature gate is enabled. + + If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. + If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes + and "Recursive" for all other volumes. + + This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. + + All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. + Note that this field cannot be set when spec.os.name is windows. + type: string + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies to + the container. + type: string + role: + description: Role is a SELinux role label that applies to + the container. + type: string + type: + description: Type is a SELinux type label that applies to + the container. + type: string + user: + description: User is a SELinux user label that applies to + the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA + credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object priorityClassName: description: 'Optional: Set priorityClassName' type: string @@ -616,6 +854,10 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + secretEnv: + description: 'Optional: SecretEnv represents the name of the Kubernetes + Secret with secret environment variables for the NVIDIA Driver' + type: string startupProbe: description: NVIDIA Driver container startup probe settings properties: @@ -675,9 +917,10 @@ spec: operator: description: |- Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string tolerationSeconds: description: |- @@ -798,6 +1041,7 @@ spec: - ignored - ready - notReady + - disabled type: string required: - state diff --git a/packages/system/gpu-operator/charts/gpu-operator/templates/cleanup_crd.yaml b/packages/system/gpu-operator/charts/gpu-operator/templates/cleanup_crd.yaml index 670bedc2..0d426f95 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/templates/cleanup_crd.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/templates/cleanup_crd.yaml @@ -35,16 +35,13 @@ spec: image: {{ include "gpu-operator.fullimage" . }} imagePullPolicy: {{ .Values.operator.imagePullPolicy }} command: - - /bin/sh - - -c - - > - kubectl delete clusterpolicy cluster-policy; - kubectl delete crd clusterpolicies.nvidia.com; - kubectl delete crd nvidiadrivers.nvidia.com --ignore-not-found=true; - {{- if .Values.nfd.enabled -}} - kubectl delete crd nodefeatures.nfd.k8s-sigs.io --ignore-not-found=true; - kubectl delete crd nodefeaturegroups.nfd.k8s-sigs.io --ignore-not-found=true; - kubectl delete crd nodefeaturerules.nfd.k8s-sigs.io --ignore-not-found=true; - {{- end }} + - /usr/bin/manage-crds + args: + - delete + - --filepath=/opt/gpu-operator/nvidia.com_clusterpolicies.yaml + - --filepath=/opt/gpu-operator/nvidia.com_nvidiadrivers.yaml + {{- if .Values.nfd.enabled }} + - --filepath=/opt/gpu-operator/nfd-api-crds.yaml + {{- end }} restartPolicy: OnFailure {{- end }} diff --git a/packages/system/gpu-operator/charts/gpu-operator/templates/clusterpolicy.yaml b/packages/system/gpu-operator/charts/gpu-operator/templates/clusterpolicy.yaml index 763716d7..978b5dc8 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/templates/clusterpolicy.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/templates/clusterpolicy.yaml @@ -89,45 +89,58 @@ spec: plugin: {{- if .Values.validator.plugin.env }} env: {{ toYaml .Values.validator.plugin.env | nindent 8 }} + {{- else }} + env: [] {{- end }} {{- end }} {{- if .Values.validator.cuda }} cuda: {{- if .Values.validator.cuda.env }} env: {{ toYaml .Values.validator.cuda.env | nindent 8 }} + {{- else }} + env: [] {{- end }} {{- end }} {{- if .Values.validator.driver }} driver: {{- if .Values.validator.driver.env }} env: {{ toYaml .Values.validator.driver.env | nindent 8 }} + {{- else }} + env: [] {{- end }} {{- end }} {{- if .Values.validator.toolkit }} toolkit: {{- if .Values.validator.toolkit.env }} env: {{ toYaml .Values.validator.toolkit.env | nindent 8 }} + {{- else }} + env: [] {{- end }} {{- end }} {{- if .Values.validator.vfioPCI }} vfioPCI: {{- if .Values.validator.vfioPCI.env }} env: {{ toYaml .Values.validator.vfioPCI.env | nindent 8 }} + {{- else }} + env: [] {{- end }} {{- end }} {{- if .Values.validator.vgpuManager }} vgpuManager: {{- if .Values.validator.vgpuManager.env }} env: {{ toYaml .Values.validator.vgpuManager.env | nindent 8 }} + {{- else }} + env: [] {{- end }} {{- end }} {{- if .Values.validator.vgpuDevices }} vgpuDevices: {{- if .Values.validator.vgpuDevices.env }} env: {{ toYaml .Values.validator.vgpuDevices.env | nindent 8 }} + {{- else }} + env: [] {{- end }} {{- end }} - mig: {{- if .Values.mig.strategy }} strategy: {{ .Values.mig.strategy }} @@ -136,7 +149,12 @@ spec: enabled: {{ .Values.psa.enabled }} cdi: enabled: {{ .Values.cdi.enabled }} + {{- if .Values.cdi.default }} default: {{ .Values.cdi.default }} + {{- end }} + {{- if and (.Values.cdi.enabled) (.Values.cdi.nriPluginEnabled) }} + nriPluginEnabled: {{ .Values.cdi.nriPluginEnabled }} + {{- end }} driver: enabled: {{ .Values.driver.enabled }} useNvidiaDriverCRD: {{ .Values.driver.nvidiaDriverCRD.enabled }} @@ -200,6 +218,9 @@ spec: {{- if .Values.driver.kernelModuleConfig }} kernelModuleConfig: {{ toYaml .Values.driver.kernelModuleConfig | nindent 6 }} {{- end }} + {{- if .Values.driver.secretEnv }} + secretEnv: {{ .Values.driver.secretEnv }} + {{- end }} {{- if .Values.driver.resources }} resources: {{ toYaml .Values.driver.resources | nindent 6 }} {{- end }} @@ -258,6 +279,9 @@ spec: {{- if .Values.vgpuManager.args }} args: {{ toYaml .Values.vgpuManager.args | nindent 6 }} {{- end }} + {{- if .Values.vgpuManager.kernelModuleConfig }} + kernelModuleConfig: {{ toYaml .Values.vgpuManager.kernelModuleConfig | nindent 6 }} + {{- end }} driverManager: {{- if .Values.vgpuManager.driverManager.repository }} repository: {{ .Values.vgpuManager.driverManager.repository }} @@ -276,7 +300,9 @@ spec: {{- end }} kataManager: enabled: {{ .Values.kataManager.enabled }} + {{- if .Values.kataManager.config }} config: {{ toYaml .Values.kataManager.config | nindent 6 }} + {{- end }} {{- if .Values.kataManager.repository }} repository: {{ .Values.kataManager.repository }} {{- end }} @@ -394,7 +420,7 @@ spec: resources: {{ toYaml .Values.ccManager.resources | nindent 6 }} {{- end }} {{- if .Values.ccManager.env }} - env: {{ toYaml .Values.vfioManager.env | nindent 6 }} + env: {{ toYaml .Values.ccManager.env | nindent 6 }} {{- end }} {{- if .Values.ccManager.args }} args: {{ toYaml .Values.ccManager.args | nindent 6 }} @@ -453,8 +479,8 @@ spec: {{- end }} {{- if .Values.devicePlugin.config.name }} config: - name: {{ .Values.devicePlugin.config.name }} - default: {{ .Values.devicePlugin.config.default }} + name: {{ .Values.devicePlugin.config.name | quote }} + default: {{ .Values.devicePlugin.config.default | quote }} {{- end }} dcgm: enabled: {{ .Values.dcgm.enabled }} @@ -515,6 +541,18 @@ spec: {{- if .Values.dcgmExporter.serviceMonitor }} serviceMonitor: {{ toYaml .Values.dcgmExporter.serviceMonitor | nindent 6 }} {{- end }} + {{- if .Values.dcgmExporter.service }} + service: {{ toYaml .Values.dcgmExporter.service | nindent 6 }} + {{- end }} + {{- if .Values.dcgmExporter.hostPID }} + hostPID: {{ .Values.dcgmExporter.hostPID }} + {{- end }} + {{- if .Values.dcgmExporter.hostNetwork }} + hostNetwork: {{ .Values.dcgmExporter.hostNetwork }} + {{- end }} + {{- if .Values.dcgmExporter.hpcJobMapping }} + hpcJobMapping: {{ toYaml .Values.dcgmExporter.hpcJobMapping | nindent 6 }} + {{- end }} gfd: enabled: {{ .Values.gfd.enabled }} {{- if .Values.gfd.repository }} @@ -569,8 +607,10 @@ spec: {{- end }} {{- if .Values.migManager.config }} config: + {{- if .Values.migManager.config.name }} name: {{ .Values.migManager.config.name }} - default: {{ .Values.migManager.config.default }} + {{- end }} + default: {{ .Values.migManager.config.default | quote }} {{- end }} {{- if .Values.migManager.gpuClientsConfig }} gpuClientsConfig: {{ toYaml .Values.migManager.gpuClientsConfig | nindent 6 }} @@ -599,7 +639,7 @@ spec: {{- if .Values.nodeStatusExporter.args }} args: {{ toYaml .Values.nodeStatusExporter.args | nindent 6 }} {{- end }} - {{- if .Values.gds.enabled }} + {{- if .Values.gds }} gds: enabled: {{ .Values.gds.enabled }} {{- if .Values.gds.repository }} @@ -650,6 +690,9 @@ spec: {{- if .Values.sandboxWorkloads.defaultWorkload }} defaultWorkload: {{ .Values.sandboxWorkloads.defaultWorkload }} {{- end }} + {{- if .Values.sandboxWorkloads.mode }} + mode: {{ .Values.sandboxWorkloads.mode | quote }} + {{- end }} sandboxDevicePlugin: {{- if .Values.sandboxDevicePlugin.enabled }} enabled: {{ .Values.sandboxDevicePlugin.enabled }} @@ -678,3 +721,31 @@ spec: {{- if .Values.sandboxDevicePlugin.args }} args: {{ toYaml .Values.sandboxDevicePlugin.args | nindent 6 }} {{- end }} + kataSandboxDevicePlugin: + {{- if ne .Values.kataSandboxDevicePlugin.enabled nil }} + enabled: {{ .Values.kataSandboxDevicePlugin.enabled }} + {{- end }} + {{- if .Values.kataSandboxDevicePlugin.repository }} + repository: {{ .Values.kataSandboxDevicePlugin.repository }} + {{- end }} + {{- if .Values.kataSandboxDevicePlugin.image }} + image: {{ .Values.kataSandboxDevicePlugin.image }} + {{- end }} + {{- if .Values.kataSandboxDevicePlugin.version }} + version: {{ .Values.kataSandboxDevicePlugin.version | quote }} + {{- end }} + {{- if .Values.kataSandboxDevicePlugin.imagePullPolicy }} + imagePullPolicy: {{ .Values.kataSandboxDevicePlugin.imagePullPolicy }} + {{- end }} + {{- if .Values.kataSandboxDevicePlugin.imagePullSecrets }} + imagePullSecrets: {{ toYaml .Values.kataSandboxDevicePlugin.imagePullSecrets | nindent 6 }} + {{- end }} + {{- if .Values.kataSandboxDevicePlugin.resources }} + resources: {{ toYaml .Values.kataSandboxDevicePlugin.resources | nindent 6 }} + {{- end }} + {{- if .Values.kataSandboxDevicePlugin.env }} + env: {{ toYaml .Values.kataSandboxDevicePlugin.env | nindent 6 }} + {{- end }} + {{- if .Values.kataSandboxDevicePlugin.args }} + args: {{ toYaml .Values.kataSandboxDevicePlugin.args | nindent 6 }} + {{- end }} diff --git a/packages/system/gpu-operator/charts/gpu-operator/templates/extra-objects.yaml b/packages/system/gpu-operator/charts/gpu-operator/templates/extra-objects.yaml new file mode 100644 index 00000000..fc9a76b8 --- /dev/null +++ b/packages/system/gpu-operator/charts/gpu-operator/templates/extra-objects.yaml @@ -0,0 +1,8 @@ +{{ range .Values.extraObjects }} +--- +{{ if typeIs "string" . }} + {{- tpl . $ }} +{{- else }} + {{- tpl (toYaml .) $ }} +{{- end }} +{{ end }} diff --git a/packages/system/gpu-operator/charts/gpu-operator/templates/nodefeaturerules.yaml b/packages/system/gpu-operator/charts/gpu-operator/templates/nodefeaturerules.yaml index 6076b3d3..4584cffc 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/templates/nodefeaturerules.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/templates/nodefeaturerules.yaml @@ -2,106 +2,32 @@ apiVersion: nfd.k8s-sigs.io/v1alpha1 kind: NodeFeatureRule metadata: - name: nvidia-nfd-nodefeaturerules + name: nvidia-kernel-modules spec: rules: - - name: "TDX rule" + - name: kernel-module-gdrdrv labels: - tdx.enabled: "true" + nvidia.com/gdrcopy.capable: "true" matchFeatures: - - feature: cpu.security + - feature: kernel.loadedmodule matchExpressions: - tdx.enabled: {op: IsTrue} - - name: "TDX total keys rule" - extendedResources: - tdx.total_keys: "@cpu.security.tdx.total_keys" + gdrdrv: + op: Exists + - name: kernel-module-nvidia_fs + labels: + nvidia.com/gds.capable: "true" matchFeatures: - - feature: cpu.security + - feature: kernel.loadedmodule matchExpressions: - tdx.enabled: {op: IsTrue} - - name: "SEV-SNP rule" + nvidia_fs: + op: Exists + - name: kernel-module-nvidia_peermem labels: - sev.snp.enabled: "true" + nvidia.com/peermem.capable: "true" matchFeatures: - - feature: cpu.security - matchExpressions: - sev.snp.enabled: - op: IsTrue - - name: "SEV-ES rule" - labels: - sev.es.enabled: "true" - matchFeatures: - - feature: cpu.security - matchExpressions: - sev.es.enabled: - op: IsTrue - - name: SEV system capacities - extendedResources: - sev_asids: '@cpu.security.sev.asids' - sev_es: '@cpu.security.sev.encrypted_state_ids' - matchFeatures: - - feature: cpu.security - matchExpressions: - sev.enabled: - op: Exists - - name: "NVIDIA H100" - labels: - "nvidia.com/gpu.H100": "true" - "nvidia.com/gpu.family": "hopper" - matchFeatures: - - feature: pci.device + - feature: kernel.loadedmodule matchExpressions: - vendor: {op: In, value: ["10de"]} - device: {op: In, value: ["2339"]} - - name: "NVIDIA H100 PCIe" - labels: - "nvidia.com/gpu.H100.pcie": "true" - "nvidia.com/gpu.family": "hopper" - matchFeatures: - - feature: pci.device - matchExpressions: - vendor: {op: In, value: ["10de"]} - device: {op: In, value: ["2331"]} - - name: "NVIDIA H100 80GB HBM3" - labels: - "nvidia.com/gpu.H100.HBM3": "true" - "nvidia.com/gpu.family": "hopper" - matchFeatures: - - feature: pci.device - matchExpressions: - vendor: {op: In, value: ["10de"]} - device: {op: In, value: ["2330"]} - - name: "NVIDIA H800" - labels: - "nvidia.com/gpu.H800": "true" - "nvidia.com/gpu.family": "hopper" - matchFeatures: - - feature: pci.device - matchExpressions: - vendor: {op: In, value: ["10de"]} - device: {op: In, value: ["2324"]} - - name: "NVIDIA H800 PCIE" - labels: - "nvidia.com/gpu.H800.pcie": "true" - "nvidia.com/gpu.family": "hopper" - matchFeatures: - - feature: pci.device - matchExpressions: - vendor: {op: In, value: ["10de"]} - device: {op: In, value: ["2322"]} - - name: "NVIDIA CC Enabled" - labels: - "nvidia.com/cc.capable": "true" - matchAny: # TDX/SEV + Hopper GPU - - matchFeatures: - - feature: rule.matched - matchExpressions: - nvidia.com/gpu.family: {op: In, value: ["hopper"]} - sev.snp.enabled: {op: IsTrue} - - matchFeatures: - - feature: rule.matched - matchExpressions: - nvidia.com/gpu.family: {op: In, value: ["hopper"]} - tdx.enabled: {op: IsTrue} + nvidia_peermem: + op: Exists {{- end }} diff --git a/packages/system/gpu-operator/charts/gpu-operator/templates/nvidiadriver.yaml b/packages/system/gpu-operator/charts/gpu-operator/templates/nvidiadriver.yaml index cbe56713..60c36575 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/templates/nvidiadriver.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/templates/nvidiadriver.yaml @@ -1,3 +1,4 @@ +{{- if .Values.driver.enabled }} {{- if and .Values.driver.nvidiaDriverCRD.enabled .Values.driver.nvidiaDriverCRD.deployDefaultCR }} apiVersion: nvidia.com/v1alpha1 kind: NVIDIADriver @@ -48,7 +49,11 @@ spec: certConfig: name: {{ .Values.driver.certConfig.name }} {{- end }} - {{- if .Values.driver.licensingConfig.configMapName }} + {{- if .Values.driver.licensingConfig.secretName }} + licensingConfig: + secretName: {{ .Values.driver.licensingConfig.secretName }} + nlsEnabled: {{ .Values.driver.licensingConfig.nlsEnabled | default true }} + {{- else if .Values.driver.licensingConfig.configMapName }} licensingConfig: name: {{ .Values.driver.licensingConfig.configMapName }} nlsEnabled: {{ .Values.driver.licensingConfig.nlsEnabled | default true }} @@ -61,6 +66,9 @@ spec: kernelModuleConfig: name: {{ .Values.driver.kernelModuleConfig.name }} {{- end }} + {{- if .Values.driver.secretEnv }} + secretEnv: {{ .Values.driver.secretEnv }} + {{- end }} {{- if .Values.driver.resources }} resources: {{ toYaml .Values.driver.resources | nindent 6 }} {{- end }} @@ -117,3 +125,4 @@ spec: {{- end }} {{- end }} {{- end }} +{{- end }} diff --git a/packages/system/gpu-operator/charts/gpu-operator/templates/operator.yaml b/packages/system/gpu-operator/charts/gpu-operator/templates/operator.yaml index 6f484826..c6ec3bf4 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/templates/operator.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/templates/operator.yaml @@ -2,6 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: gpu-operator + namespace: {{ .Release.Namespace }} labels: {{- include "gpu-operator.labels" . | nindent 4 }} app.kubernetes.io/component: "gpu-operator" @@ -58,10 +59,6 @@ spec: fieldPath: metadata.namespace - name: "DRIVER_MANAGER_IMAGE" value: "{{ include "driver-manager.fullimage" . }}" - volumeMounts: - - name: host-os-release - mountPath: "/host-etc/os-release" - readOnly: true livenessProbe: httpGet: path: /healthz @@ -81,10 +78,6 @@ spec: ports: - name: metrics containerPort: 8080 - volumes: - - name: host-os-release - hostPath: - path: "/etc/os-release" {{- with .Values.operator.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/packages/system/gpu-operator/charts/gpu-operator/templates/role.yaml b/packages/system/gpu-operator/charts/gpu-operator/templates/role.yaml index 9e5bcede..dc4674c5 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/templates/role.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/templates/role.yaml @@ -2,6 +2,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: gpu-operator + namespace: {{ .Release.Namespace }} labels: {{- include "gpu-operator.labels" . | nindent 4 }} app.kubernetes.io/component: "gpu-operator" @@ -82,3 +83,13 @@ rules: - watch - update - delete +- apiGroups: + - "nfd.k8s-sigs.io" + resources: + - "nodefeatures" + verbs: + - get + - list + - watch + - create + - update diff --git a/packages/system/gpu-operator/charts/gpu-operator/templates/rolebinding.yaml b/packages/system/gpu-operator/charts/gpu-operator/templates/rolebinding.yaml index c915a465..732d8d52 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/templates/rolebinding.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/templates/rolebinding.yaml @@ -2,6 +2,7 @@ kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: gpu-operator + namespace: {{ .Release.Namespace }} labels: {{- include "gpu-operator.labels" . | nindent 4 }} app.kubernetes.io/component: "gpu-operator" diff --git a/packages/system/gpu-operator/charts/gpu-operator/templates/serviceaccount.yaml b/packages/system/gpu-operator/charts/gpu-operator/templates/serviceaccount.yaml index 50555e53..79a3b642 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/templates/serviceaccount.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/templates/serviceaccount.yaml @@ -2,6 +2,7 @@ apiVersion: v1 kind: ServiceAccount metadata: name: gpu-operator + namespace: {{ .Release.Namespace }} labels: {{- include "gpu-operator.labels" . | nindent 4 }} app.kubernetes.io/component: "gpu-operator" diff --git a/packages/system/gpu-operator/charts/gpu-operator/templates/upgrade_crd.yaml b/packages/system/gpu-operator/charts/gpu-operator/templates/upgrade_crd.yaml index 6552558a..e887b3a8 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/templates/upgrade_crd.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/templates/upgrade_crd.yaml @@ -4,6 +4,7 @@ apiVersion: v1 kind: ServiceAccount metadata: name: gpu-operator-upgrade-crd-hook-sa + namespace: {{ .Release.Namespace }} annotations: helm.sh/hook: pre-upgrade helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation @@ -83,13 +84,13 @@ spec: image: {{ include "gpu-operator.fullimage" . }} imagePullPolicy: {{ .Values.operator.imagePullPolicy }} command: - - /bin/sh - - -c - - > - kubectl apply -f /opt/gpu-operator/nvidia.com_clusterpolicies.yaml; - kubectl apply -f /opt/gpu-operator/nvidia.com_nvidiadrivers.yaml; + - /usr/bin/manage-crds + args: + - apply + - --filepath=/opt/gpu-operator/nvidia.com_clusterpolicies.yaml + - --filepath=/opt/gpu-operator/nvidia.com_nvidiadrivers.yaml {{- if .Values.nfd.enabled }} - kubectl apply -f /opt/gpu-operator/nfd-api-crds.yaml; + - --filepath=/opt/gpu-operator/nfd-api-crds.yaml {{- end }} restartPolicy: OnFailure {{- end }} diff --git a/packages/system/gpu-operator/charts/gpu-operator/templates/validations.yaml b/packages/system/gpu-operator/charts/gpu-operator/templates/validations.yaml new file mode 100644 index 00000000..573786d2 --- /dev/null +++ b/packages/system/gpu-operator/charts/gpu-operator/templates/validations.yaml @@ -0,0 +1,3 @@ +{{- if and (eq .Values.cdi.enabled false) (eq .Values.cdi.nriPluginEnabled true) }} +{{ fail "the NRI Plugin cannot be enabled when CDI is disabled" }} +{{- end}} diff --git a/packages/system/gpu-operator/charts/gpu-operator/values.yaml b/packages/system/gpu-operator/charts/gpu-operator/values.yaml index 2806eac3..222b0e38 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/values.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/values.yaml @@ -13,12 +13,14 @@ psa: enabled: false cdi: - enabled: false - default: false + enabled: true + nriPluginEnabled: false sandboxWorkloads: enabled: false defaultWorkload: "container" + # Sandbox mode: "kubevirt" (default) or "kata". When "kata", the Kata device plugin is deployed on vm-passthrough nodes. + mode: "kubevirt" hostPaths: # rootFS represents the path to the root filesystem of the host. @@ -50,8 +52,8 @@ daemonsets: maxUnavailable: "1" validator: - repository: nvcr.io/nvidia/cloud-native - image: gpu-operator-validator + repository: nvcr.io/nvidia + image: gpu-operator # If version is not specified, then default is to use chart.AppVersion #version: "" imagePullPolicy: IfNotPresent @@ -60,9 +62,7 @@ validator: args: [] resources: {} plugin: - env: - - name: WITH_WORKLOAD - value: "false" + env: [] operator: repository: nvcr.io/nvidia @@ -79,16 +79,7 @@ operator: # upgrade CRD on chart upgrade, requires --disable-openapi-validation flag # to be passed during helm upgrade. upgradeCRD: true - initContainer: - image: cuda - repository: nvcr.io/nvidia - version: 12.8.1-base-ubi9 - imagePullPolicy: IfNotPresent tolerations: - - key: "node-role.kubernetes.io/master" - operator: "Equal" - value: "" - effect: "NoSchedule" - key: "node-role.kubernetes.io/control-plane" operator: "Equal" value: "" @@ -98,12 +89,6 @@ operator: affinity: nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: - - weight: 1 - preference: - matchExpressions: - - key: "node-role.kubernetes.io/master" - operator: In - values: [""] - weight: 1 preference: matchExpressions: @@ -146,7 +131,7 @@ driver: usePrecompiled: false repository: nvcr.io/nvidia image: driver - version: "570.124.06" + version: "580.126.20" imagePullPolicy: IfNotPresent imagePullSecrets: [] startupProbe: @@ -192,25 +177,13 @@ driver: timeoutSeconds: 300 deleteEmptyDir: false manager: - image: k8s-driver-manager repository: nvcr.io/nvidia/cloud-native + image: k8s-driver-manager # When choosing a different version of k8s-driver-manager, DO NOT downgrade to a version lower than v0.6.4 # to ensure k8s-driver-manager stays compatible with gpu-operator starting from v24.3.0 - version: v0.8.0 + version: v0.10.0 imagePullPolicy: IfNotPresent - env: - - name: ENABLE_GPU_POD_EVICTION - value: "true" - - name: ENABLE_AUTO_DRAIN - value: "false" - - name: DRAIN_USE_FORCE - value: "false" - - name: DRAIN_POD_SELECTOR_LABEL - value: "" - - name: DRAIN_TIMEOUT_SECONDS - value: "0s" - - name: DRAIN_DELETE_EMPTYDIR_DATA - value: "false" + env: [] env: [] resources: {} # Private mirror repository configuration @@ -221,7 +194,7 @@ driver: name: "" # vGPU licensing configuration licensingConfig: - configMapName: "" + secretName: "" nlsEnabled: true # vGPU topology daemon configuration virtualTopology: @@ -229,12 +202,14 @@ driver: # kernel module configuration for NVIDIA driver kernelModuleConfig: name: "" + # Name of Kubernetes Secret which contains secrets to be passed in as environment variables + secretEnv: "" toolkit: enabled: true repository: nvcr.io/nvidia/k8s image: container-toolkit - version: v1.17.5-ubuntu20.04 + version: v1.19.0 imagePullPolicy: IfNotPresent imagePullSecrets: [] env: [] @@ -245,23 +220,11 @@ devicePlugin: enabled: true repository: nvcr.io/nvidia image: k8s-device-plugin - version: v0.17.1 + version: v0.19.0 imagePullPolicy: IfNotPresent imagePullSecrets: [] args: [] - env: - - name: PASS_DEVICE_SPECS - value: "true" - - name: FAIL_ON_INIT_ERROR - value: "true" - - name: DEVICE_LIST_STRATEGY - value: envvar - - name: DEVICE_ID_STRATEGY - value: uuid - - name: NVIDIA_VISIBLE_DEVICES - value: all - - name: NVIDIA_DRIVER_CAPABILITIES - value: all + env: [] resources: {} # Plugin configuration # Use "name" to either point to an existing ConfigMap or to create a new one with a list of configurations(i.e with create=true). @@ -303,7 +266,7 @@ dcgm: enabled: false repository: nvcr.io/nvidia/cloud-native image: dcgm - version: 4.1.1-2-ubuntu22.04 + version: 4.5.2-1-ubuntu22.04 imagePullPolicy: IfNotPresent args: [] env: [] @@ -313,16 +276,19 @@ dcgmExporter: enabled: true repository: nvcr.io/nvidia/k8s image: dcgm-exporter - version: 4.1.1-4.0.4-ubuntu22.04 + version: 4.5.1-4.8.0-distroless imagePullPolicy: IfNotPresent - env: - - name: DCGM_EXPORTER_LISTEN - value: ":9400" - - name: DCGM_EXPORTER_KUBERNETES - value: "true" - - name: DCGM_EXPORTER_COLLECTORS - value: "/etc/dcgm-exporter/dcp-metrics-included.csv" + env: [] resources: {} + hostPID: false + hostNetwork: false + # HPC job mapping configuration for correlating GPU metrics with HPC workload manager jobs + # This is used by HPC workload managers like Slurm to label GPU metrics with job IDs + # hpcJobMapping: + # enabled: true + # directory: /var/lib/dcgm-exporter/job-mapping + service: + internalTrafficPolicy: Cluster serviceMonitor: enabled: false interval: 15s @@ -359,31 +325,34 @@ gfd: enabled: true repository: nvcr.io/nvidia image: k8s-device-plugin - version: v0.17.1 + version: v0.19.0 imagePullPolicy: IfNotPresent imagePullSecrets: [] - env: - - name: GFD_SLEEP_INTERVAL - value: 60s - - name: GFD_FAIL_ON_INIT_ERROR - value: "true" + env: [] resources: {} migManager: enabled: true repository: nvcr.io/nvidia/cloud-native image: k8s-mig-manager - version: v0.12.1-ubuntu20.04 + version: v0.14.0 imagePullPolicy: IfNotPresent imagePullSecrets: [] - env: - - name: WITH_REBOOT - value: "false" + env: [] resources: {} # MIG configuration - # Use "name" to either point to an existing ConfigMap or to create a new one with a list of configurations(i.e with create=true). - # Use "data" to build an integrated ConfigMap from a set of configurations as - # part of this helm chart. An example of setting "data" might be: + # NOTE: MIG manager automatically generates configuration from hardware on each node. + # Only provide a custom config if you need settings that differ from hardware discovery. + # + # To use an existing ConfigMap: + # - Set name="your-configmap-name" with create=false + # - ConfigMap MUST have a key named "config.yaml" + # + # To create a new ConfigMap via Helm: + # - Set create=true, name="your-configmap-name", and provide data below + # - If create=true but data is empty, ConfigMap creation is skipped + # + # Example of creating a custom ConfigMap: # config: # name: custom-mig-parted-configs # create: true @@ -415,17 +384,19 @@ migManager: default: "all-disabled" # Create a ConfigMap (default: false) create: false - # ConfigMap name (either existing or to create a new one with create=true above) + # ConfigMap name (either existing or to create with create=true) + # If name is provided, mig-manager will use this config instead of auto-generated one. + # REQUIREMENT: Custom ConfigMaps must contain a key named "config.yaml" name: "" - # Data section for the ConfigMap to create (i.e only applies when create=true) + # Data section for the ConfigMap (required only if create=true) data: {} gpuClientsConfig: name: "" nodeStatusExporter: enabled: false - repository: nvcr.io/nvidia/cloud-native - image: gpu-operator-validator + repository: nvcr.io/nvidia + image: gpu-operator # If version is not specified, then default is to use chart.AppVersion #version: "" imagePullPolicy: IfNotPresent @@ -436,7 +407,7 @@ gds: enabled: false repository: nvcr.io/nvidia/cloud-native image: nvidia-fs - version: "2.20.5" + version: "2.27.3" imagePullPolicy: IfNotPresent imagePullSecrets: [] env: [] @@ -446,7 +417,7 @@ gdrcopy: enabled: false repository: nvcr.io/nvidia/cloud-native image: gdrdrv - version: "v2.4.4" + version: "v2.5.1" imagePullPolicy: IfNotPresent imagePullSecrets: [] env: [] @@ -462,23 +433,22 @@ vgpuManager: env: [] resources: {} driverManager: - image: k8s-driver-manager repository: nvcr.io/nvidia/cloud-native + image: k8s-driver-manager # When choosing a different version of k8s-driver-manager, DO NOT downgrade to a version lower than v0.6.4 # to ensure k8s-driver-manager stays compatible with gpu-operator starting from v24.3.0 - version: v0.8.0 + version: v0.10.0 imagePullPolicy: IfNotPresent - env: - - name: ENABLE_GPU_POD_EVICTION - value: "false" - - name: ENABLE_AUTO_DRAIN - value: "false" + env: [] + # kernel module configuration for vGPU manager + kernelModuleConfig: + name: "" vgpuDeviceManager: enabled: true repository: nvcr.io/nvidia/cloud-native image: vgpu-device-manager - version: v0.3.0 + version: v0.4.2 imagePullPolicy: IfNotPresent imagePullSecrets: [] env: [] @@ -488,45 +458,25 @@ vgpuDeviceManager: vfioManager: enabled: true - repository: nvcr.io/nvidia - image: cuda - version: 12.8.1-base-ubi9 + repository: nvcr.io/nvidia/cloud-native + image: k8s-driver-manager + version: v0.10.0 imagePullPolicy: IfNotPresent imagePullSecrets: [] env: [] resources: {} driverManager: - image: k8s-driver-manager repository: nvcr.io/nvidia/cloud-native + image: k8s-driver-manager # When choosing a different version of k8s-driver-manager, DO NOT downgrade to a version lower than v0.6.4 # to ensure k8s-driver-manager stays compatible with gpu-operator starting from v24.3.0 - version: v0.8.0 + version: v0.10.0 imagePullPolicy: IfNotPresent - env: - - name: ENABLE_GPU_POD_EVICTION - value: "false" - - name: ENABLE_AUTO_DRAIN - value: "false" + env: [] kataManager: enabled: false - config: - artifactsDir: "/opt/nvidia-gpu-operator/artifacts/runtimeclasses" - runtimeClasses: - - name: kata-nvidia-gpu - nodeSelector: {} - artifacts: - url: nvcr.io/nvidia/cloud-native/kata-gpu-artifacts:ubuntu22.04-535.54.03 - pullSecret: "" - - name: kata-nvidia-gpu-snp - nodeSelector: - "nvidia.com/cc.capable": "true" - artifacts: - url: nvcr.io/nvidia/cloud-native/kata-gpu-artifacts:ubuntu22.04-535.86.10-snp - pullSecret: "" - repository: nvcr.io/nvidia/cloud-native - image: k8s-kata-manager - version: v0.2.3 + config: {} imagePullPolicy: IfNotPresent imagePullSecrets: [] env: [] @@ -536,7 +486,19 @@ sandboxDevicePlugin: enabled: true repository: nvcr.io/nvidia image: kubevirt-gpu-device-plugin - version: v1.3.1 + version: v1.5.0 + imagePullPolicy: IfNotPresent + imagePullSecrets: [] + args: [] + env: [] + resources: {} + +# Kata sandbox device plugin (used when sandboxWorkloads.mode is "kata"). +kataSandboxDevicePlugin: + enabled: true + repository: nvcr.io/nvidia/cloud-native + image: nvidia-sandbox-device-plugin + version: "v0.0.2" imagePullPolicy: IfNotPresent imagePullSecrets: [] args: [] @@ -544,11 +506,11 @@ sandboxDevicePlugin: resources: {} ccManager: - enabled: false - defaultMode: "off" + enabled: true + defaultMode: "on" repository: nvcr.io/nvidia/cloud-native image: k8s-cc-manager - version: v0.1.1 + version: v0.3.0 imagePullPolicy: IfNotPresent imagePullSecrets: [] env: @@ -556,8 +518,11 @@ ccManager: value: "0x2339,0x2331,0x2330,0x2324,0x2322,0x233d" resources: {} +# Array of extra K8s manifests to deploy +# Supports use of custom Helm templates +extraObjects: [] + node-feature-discovery: - enableNodeFeatureApi: true priorityClassName: system-node-critical gc: enable: true @@ -571,10 +536,6 @@ node-feature-discovery: # disable creation to avoid duplicate serviceaccount creation by master spec below create: false tolerations: - - key: "node-role.kubernetes.io/master" - operator: "Equal" - value: "" - effect: "NoSchedule" - key: "node-role.kubernetes.io/control-plane" operator: "Equal" value: "" From 8c866934081d7ec6824f058f075f8c2e3230e195 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 2 Apr 2026 15:09:52 +0300 Subject: [PATCH 2/5] feat(gpu-operator): add vGPU variant for mediated device support Add vGPU mode as a new variant alongside the existing passthrough (default) variant. This enables sharing a single physical GPU across multiple VMs using NVIDIA mediated devices. Changes: - Add values-talos-vgpu.yaml with vGPU Manager and vGPU Device Manager enabled - Add 'vgpu' variant to PackageSource - Add documentation for vGPU setup (driver image, NLS licensing, profiles, KubeVirt) Users select the vgpu variant via Package CR and provide their proprietary vGPU Manager image coordinates through component values. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- docs/gpu-vgpu.md | 167 ++++++++++++++++++ .../core/platform/sources/gpu-operator.yaml | 13 ++ .../gpu-operator/values-talos-vgpu.yaml | 16 ++ 3 files changed, 196 insertions(+) create mode 100644 docs/gpu-vgpu.md create mode 100644 packages/system/gpu-operator/values-talos-vgpu.yaml diff --git a/docs/gpu-vgpu.md b/docs/gpu-vgpu.md new file mode 100644 index 00000000..77b3e80d --- /dev/null +++ b/docs/gpu-vgpu.md @@ -0,0 +1,167 @@ +# GPU Operator: vGPU Support + +This document describes how to configure the GPU Operator package with NVIDIA vGPU support for sharing a single physical GPU across multiple virtual machines using mediated devices. + +## Prerequisites + +- NVIDIA GPU with vGPU support (e.g., NVIDIA L40S, A100, A30, etc.) +- Talos Linux as the host OS +- NVIDIA vGPU Software license (NVIDIA AI Enterprise or vGPU subscription) +- Access to the NVIDIA Licensing Portal ([ui.licensing.nvidia.com](https://ui.licensing.nvidia.com)) + +## Variants + +The GPU Operator package supports two variants: + +- **`default`** — GPU passthrough mode (vfio-pci). The entire GPU is passed through to a single VM. +- **`vgpu`** — vGPU mode. A physical GPU is shared between multiple VMs using NVIDIA mediated devices. + +## Building the vGPU Manager Image + +The vGPU Manager driver is proprietary and must be obtained from NVIDIA: + +1. Log in to the [NVIDIA Licensing Portal](https://ui.licensing.nvidia.com) +2. Download the NVIDIA vGPU Software package for your GPU and target Linux version +3. Build a container image containing the vGPU Manager driver: + +```bash +# Example Containerfile +FROM ubuntu:22.04 +ARG DRIVER_VERSION +COPY NVIDIA-Linux-x86_64-${DRIVER_VERSION}-vgpu-kvm.run /opt/ +RUN chmod +x /opt/NVIDIA-Linux-x86_64-${DRIVER_VERSION}-vgpu-kvm.run +``` + +4. Push the image to your private registry: + +```bash +docker build --build-arg DRIVER_VERSION=550.90.05 --tag registry.example.com/nvidia/vgpu-manager:550.90.05 . +docker push registry.example.com/nvidia/vgpu-manager:550.90.05 +``` + +> Refer to the [NVIDIA GPU Operator documentation](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/install-gpu-operator-vgpu.html) for detailed instructions on building the vGPU Manager image. + +## Deploying with vGPU Variant + +Create a Package CR with the `vgpu` variant and provide your vGPU Manager image coordinates: + +```yaml +apiVersion: cozystack.io/v1alpha1 +kind: Package +metadata: + name: cozystack.gpu-operator +spec: + variant: vgpu + components: + gpu-operator: + values: + gpu-operator: + vgpuManager: + repository: registry.example.com/nvidia + version: "550.90.05" +``` + +If your registry requires authentication, create an `imagePullSecret` in the `cozy-gpu-operator` namespace and reference it: + +```yaml +gpu-operator: + vgpuManager: + repository: registry.example.com/nvidia + version: "550.90.05" + imagePullSecrets: + - name: nvidia-registry-secret +``` + +## NVIDIA License Server (NLS) Configuration + +vGPU requires a license server. Configure NLS by passing the license server address via a ConfigMap: + +1. Create a ConfigMap with the NLS client configuration in the `cozy-gpu-operator` namespace: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: licensing-config + namespace: cozy-gpu-operator +data: + gridd.conf: | + ServerAddress=nls.example.com + ServerPort=443 + FeatureType=1 +``` + +2. Reference the ConfigMap in the Package values: + +```yaml +gpu-operator: + vgpuManager: + repository: registry.example.com/nvidia + version: "550.90.05" + driver: + licensingConfig: + configMapName: licensing-config +``` + +## vGPU Profiles + +Each GPU model supports specific vGPU profiles that determine how the GPU is partitioned. To list available profiles for your GPU, consult the [NVIDIA vGPU User Guide](https://docs.nvidia.com/grid/latest/grid-vgpu-user-guide/). + +Example profiles for NVIDIA L40S: + +| Profile | Frame Buffer | Max Instances | Use Case | +| --- | --- | --- | --- | +| NVIDIA L40S-1Q | 1 GB | 48 | Light 3D/VDI | +| NVIDIA L40S-2Q | 2 GB | 24 | Medium 3D/VDI | +| NVIDIA L40S-4Q | 4 GB | 12 | Heavy 3D/VDI | +| NVIDIA L40S-6Q | 6 GB | 8 | Professional 3D | +| NVIDIA L40S-8Q | 8 GB | 6 | AI/ML inference | +| NVIDIA L40S-12Q | 12 GB | 4 | AI/ML training | +| NVIDIA L40S-24Q | 24 GB | 2 | Large AI workloads | +| NVIDIA L40S-48Q | 48 GB | 1 | Full GPU equivalent | + +Custom vGPU device configuration can be provided via a ConfigMap: + +```yaml +gpu-operator: + vgpuDeviceManager: + enabled: true + config: + name: vgpu-devices-config + default: default +``` + +## KubeVirt Integration + +To use vGPU with KubeVirt VMs, configure `mediatedDeviceTypes` in the KubeVirt CR. This maps vGPU profiles to node selectors: + +```yaml +apiVersion: kubevirt.io/v1 +kind: KubeVirt +metadata: + name: kubevirt +spec: + configuration: + mediatedDevicesConfiguration: + mediatedDeviceTypes: + - nvidia-592 # NVIDIA L40S-24Q + permittedHostDevices: + mediatedDevices: + - mdevNameSelector: NVIDIA L40S-24Q + resourceName: nvidia.com/NVIDIA_L40S-24Q +``` + +Then reference the vGPU resource in a VirtualMachine spec: + +```yaml +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +spec: + template: + spec: + domain: + devices: + gpus: + - name: gpu1 + deviceName: nvidia.com/NVIDIA_L40S-24Q +``` diff --git a/packages/core/platform/sources/gpu-operator.yaml b/packages/core/platform/sources/gpu-operator.yaml index e41cc9b2..10d36d65 100644 --- a/packages/core/platform/sources/gpu-operator.yaml +++ b/packages/core/platform/sources/gpu-operator.yaml @@ -23,3 +23,16 @@ spec: privileged: true namespace: cozy-gpu-operator releaseName: gpu-operator + - name: vgpu + dependsOn: + - cozystack.networking + components: + - name: gpu-operator + path: system/gpu-operator + valuesFiles: + - values.yaml + - values-talos-vgpu.yaml + install: + privileged: true + namespace: cozy-gpu-operator + releaseName: gpu-operator diff --git a/packages/system/gpu-operator/values-talos-vgpu.yaml b/packages/system/gpu-operator/values-talos-vgpu.yaml new file mode 100644 index 00000000..00ed8219 --- /dev/null +++ b/packages/system/gpu-operator/values-talos-vgpu.yaml @@ -0,0 +1,16 @@ +gpu-operator: + sandboxWorkloads: + enabled: true + driver: + enabled: false + devicePlugin: + enabled: false + vgpuManager: + enabled: true + repository: "" + image: vgpu-manager + version: "" + vgpuDeviceManager: + enabled: true + config: + default: default From 6a45f0e65dbbced5d5bf1afba4f1729488e70d8d Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 2 Apr 2026 15:47:33 +0300 Subject: [PATCH 3/5] docs(gpu-vgpu): fix driver container build instructions and NLS config Replace simplified Containerfile with NVIDIA's Makefile-based build system from gitlab.com/nvidia/container-images/driver. The GPU Operator expects pre-compiled kernel modules, not a raw .run file. Add EULA notice about public redistribution of vGPU driver images. Add note about NLS ServerPort being deployment-dependent. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- docs/gpu-vgpu.md | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/docs/gpu-vgpu.md b/docs/gpu-vgpu.md index 77b3e80d..5935c96e 100644 --- a/docs/gpu-vgpu.md +++ b/docs/gpu-vgpu.md @@ -18,28 +18,32 @@ The GPU Operator package supports two variants: ## Building the vGPU Manager Image -The vGPU Manager driver is proprietary and must be obtained from NVIDIA: +The vGPU Manager driver is proprietary and must be obtained from NVIDIA. The GPU Operator expects a pre-built driver container image — it does not install the driver from a raw `.run` file at runtime. 1. Log in to the [NVIDIA Licensing Portal](https://ui.licensing.nvidia.com) -2. Download the NVIDIA vGPU Software package for your GPU and target Linux version -3. Build a container image containing the vGPU Manager driver: +2. Navigate to **Software Downloads** and download the NVIDIA vGPU Software package for your GPU +3. Build the driver container image using NVIDIA's Makefile-based build system: ```bash -# Example Containerfile -FROM ubuntu:22.04 -ARG DRIVER_VERSION -COPY NVIDIA-Linux-x86_64-${DRIVER_VERSION}-vgpu-kvm.run /opt/ -RUN chmod +x /opt/NVIDIA-Linux-x86_64-${DRIVER_VERSION}-vgpu-kvm.run -``` +# Clone the NVIDIA driver container repository +git clone https://gitlab.com/nvidia/container-images/driver.git +cd driver -4. Push the image to your private registry: +# Place the downloaded .run file in the appropriate directory +cp NVIDIA-Linux-x86_64-550.90.05-vgpu-kvm.run vgpu/ -```bash -docker build --build-arg DRIVER_VERSION=550.90.05 --tag registry.example.com/nvidia/vgpu-manager:550.90.05 . +# Build using the provided Makefile +make OS_TAG=ubuntu22.04 \ + VGPU_DRIVER_VERSION=550.90.05 \ + PRIVATE_REGISTRY=registry.example.com/nvidia + +# Push to your private registry docker push registry.example.com/nvidia/vgpu-manager:550.90.05 ``` -> Refer to the [NVIDIA GPU Operator documentation](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/install-gpu-operator-vgpu.html) for detailed instructions on building the vGPU Manager image. +> **Important:** The build process compiles kernel modules against the host kernel version. Refer to the [NVIDIA GPU Operator vGPU documentation](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/install-gpu-operator-vgpu.html) for the complete build procedure and supported OS/kernel combinations. +> +> Uploading the vGPU driver to a publicly available registry is a violation of the NVIDIA vGPU EULA. ## Deploying with vGPU Variant @@ -89,6 +93,7 @@ data: ServerAddress=nls.example.com ServerPort=443 FeatureType=1 + # ServerPort depends on your NLS deployment (commonly 443 for DLS or 7070 for legacy NLS) ``` 2. Reference the ConfigMap in the Package values: From dbe37e7ed48797275622c988d5c8a558f2804faf Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 2 Apr 2026 15:57:11 +0300 Subject: [PATCH 4/5] fix(gpu-operator): use Secret instead of ConfigMap for licensing config ConfigMap-based licensing configuration (configMapName) is deprecated in GPU Operator v26.x. Switch to Secret-based configuration (secretName) as recommended by the updated CRD schema. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- docs/gpu-vgpu.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/gpu-vgpu.md b/docs/gpu-vgpu.md index 5935c96e..1c402096 100644 --- a/docs/gpu-vgpu.md +++ b/docs/gpu-vgpu.md @@ -78,25 +78,25 @@ gpu-operator: ## NVIDIA License Server (NLS) Configuration -vGPU requires a license server. Configure NLS by passing the license server address via a ConfigMap: +vGPU requires a license server. Configure NLS by passing the license server address via a Secret: -1. Create a ConfigMap with the NLS client configuration in the `cozy-gpu-operator` namespace: +1. Create a Secret with the NLS client configuration in the `cozy-gpu-operator` namespace: ```yaml apiVersion: v1 -kind: ConfigMap +kind: Secret metadata: name: licensing-config namespace: cozy-gpu-operator -data: +stringData: gridd.conf: | ServerAddress=nls.example.com ServerPort=443 - FeatureType=1 + FeatureType=1 # 1 for vGPU (vPC/vWS), 2 for Virtual Compute Server (vCS) # ServerPort depends on your NLS deployment (commonly 443 for DLS or 7070 for legacy NLS) ``` -2. Reference the ConfigMap in the Package values: +2. Reference the Secret in the Package values: ```yaml gpu-operator: @@ -105,7 +105,7 @@ gpu-operator: version: "550.90.05" driver: licensingConfig: - configMapName: licensing-config + secretName: licensing-config ``` ## vGPU Profiles From 5a61a33b85096ad4f8d439eb04c2d93c64765eae Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Tue, 28 Apr 2026 23:04:39 +0300 Subject: [PATCH 5/5] chore(gpu-operator): bump to v26.3.1 Pull latest patch release: GDRCopy v2.5.2, Kata Sandbox Device Plugin v0.0.3, multi-arch digest fix for KubeVirt GPU Device Plugin (no longer amd64-only), and OS release name detection fix. NFD remains v0.18.3. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- packages/system/gpu-operator/Makefile | 2 +- .../charts/gpu-operator/Chart.yaml | 4 +- .../crds/nvidia.com_clusterpolicies.yaml | 60 +++++++++++++++++++ .../crds/nvidia.com_nvidiadrivers.yaml | 4 ++ .../gpu-operator/templates/clusterpolicy.yaml | 45 ++++++++++++++ .../gpu-operator/templates/nvidiadriver.yaml | 3 + .../charts/gpu-operator/values.yaml | 24 ++++++-- 7 files changed, 133 insertions(+), 9 deletions(-) diff --git a/packages/system/gpu-operator/Makefile b/packages/system/gpu-operator/Makefile index 11ca047e..a8b3b43f 100644 --- a/packages/system/gpu-operator/Makefile +++ b/packages/system/gpu-operator/Makefile @@ -8,4 +8,4 @@ update: rm -rf charts helm repo add nvidia https://helm.ngc.nvidia.com/nvidia helm repo update nvidia - helm pull nvidia/gpu-operator --untar --untardir charts --version v26.3.0 + helm pull nvidia/gpu-operator --untar --untardir charts --version v26.3.1 diff --git a/packages/system/gpu-operator/charts/gpu-operator/Chart.yaml b/packages/system/gpu-operator/charts/gpu-operator/Chart.yaml index c2138a54..1e1ddabe 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/Chart.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: v26.3.0 +appVersion: v26.3.1 dependencies: - condition: nfd.enabled name: node-feature-discovery @@ -20,4 +20,4 @@ kubeVersion: '>= 1.16.0-0' name: gpu-operator sources: - https://github.com/NVIDIA/gpu-operator -version: v26.3.0 +version: v26.3.1 diff --git a/packages/system/gpu-operator/charts/gpu-operator/crds/nvidia.com_clusterpolicies.yaml b/packages/system/gpu-operator/charts/gpu-operator/crds/nvidia.com_clusterpolicies.yaml index 030581fc..29737027 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/crds/nvidia.com_clusterpolicies.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/crds/nvidia.com_clusterpolicies.yaml @@ -82,6 +82,10 @@ spec: - name type: object type: array + hostNetwork: + description: HostNetwork indicates whether the CC Manager pod + uses the host's network namespace. + type: boolean image: description: CC Manager image name pattern: '[a-zA-Z0-9\-]+' @@ -489,6 +493,10 @@ spec: - name type: object type: array + hostNetwork: + description: HostNetwork indicates whether the DCGM pod uses the + host's network namespace. + type: boolean hostPort: description: 'Deprecated: HostPort represents host port that needs to be bound for DCGM engine (Default: 5555)' @@ -815,6 +823,10 @@ spec: - name type: object type: array + hostNetwork: + description: HostNetwork indicates whether the Device Plugin pod + uses the host's network namespace. + type: boolean image: description: NVIDIA Device Plugin image name pattern: '[a-zA-Z0-9\-]+' @@ -907,6 +919,10 @@ spec: - name type: object type: array + hostNetwork: + description: HostNetwork indicates whether the Driver pod uses + the host's network namespace. + type: boolean image: description: NVIDIA Driver image name pattern: '[a-zA-Z0-9\-]+' @@ -1419,6 +1435,10 @@ spec: - name type: object type: array + hostNetwork: + description: HostNetwork indicates whether the GPU Feature Discovery + pod uses the host's network namespace. + type: boolean image: description: GFD image name pattern: '[a-zA-Z0-9\-]+' @@ -1562,6 +1582,10 @@ spec: - name type: object type: array + hostNetwork: + description: HostNetwork indicates whether the Kata Manager pod + uses the host's network namespace. + type: boolean image: description: Kata Manager image name pattern: '[a-zA-Z0-9\-]+' @@ -1638,6 +1662,10 @@ spec: - name type: object type: array + hostNetwork: + description: HostNetwork indicates whether the Kata Sandbox Device + Plugin pod uses the host's network namespace. + type: boolean image: description: NVIDIA component image name pattern: '[a-zA-Z0-9\-]+' @@ -1751,6 +1779,10 @@ spec: description: ConfigMap name type: string type: object + hostNetwork: + description: HostNetwork indicates whether the MIG Manager pod + uses the host's network namespace. + type: boolean image: description: NVIDIA MIG Manager image name pattern: '[a-zA-Z0-9\-]+' @@ -1827,6 +1859,10 @@ spec: - name type: object type: array + hostNetwork: + description: HostNetwork indicates whether the Node Status Exporter + pod uses the host's network namespace. + type: boolean image: description: Node Status Exporter image name pattern: '[a-zA-Z0-9\-]+' @@ -1976,6 +2012,10 @@ spec: - name type: object type: array + hostNetwork: + description: HostNetwork indicates whether the Sandbox Device + Plugin pod uses the host's network namespace. + type: boolean image: description: NVIDIA Sandbox Device Plugin image name pattern: '[a-zA-Z0-9\-]+' @@ -2081,6 +2121,10 @@ spec: - name type: object type: array + hostNetwork: + description: HostNetwork indicates whether the Container Toolkit + pod uses the host's network namespace. + type: boolean image: description: NVIDIA Container Toolkit image name pattern: '[a-zA-Z0-9\-]+' @@ -2197,6 +2241,10 @@ spec: - name type: object type: array + hostNetwork: + description: HostNetwork indicates whether the Validator pod uses + the host's network namespace. + type: boolean image: description: Validator image name pattern: '[a-zA-Z0-9\-]+' @@ -2415,6 +2463,10 @@ spec: - name type: object type: array + hostNetwork: + description: HostNetwork indicates whether the VFIO Manager pod + uses the host's network namespace. + type: boolean image: description: VFIO Manager image name pattern: '[a-zA-Z0-9\-]+' @@ -2503,6 +2555,10 @@ spec: - name type: object type: array + hostNetwork: + description: HostNetwork indicates whether the vGPU Device Manager + pod uses the host's network namespace. + type: boolean image: description: NVIDIA vGPU Device Manager image name pattern: '[a-zA-Z0-9\-]+' @@ -2621,6 +2677,10 @@ spec: - name type: object type: array + hostNetwork: + description: HostNetwork indicates whether the vGPU Manager pod + uses the host's network namespace. + type: boolean image: description: NVIDIA vGPU Manager image name pattern: '[a-zA-Z0-9\-]+' diff --git a/packages/system/gpu-operator/charts/gpu-operator/crds/nvidia.com_nvidiadrivers.yaml b/packages/system/gpu-operator/charts/gpu-operator/crds/nvidia.com_nvidiadrivers.yaml index 20697b9b..81c23415 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/crds/nvidia.com_nvidiadrivers.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/crds/nvidia.com_nvidiadrivers.yaml @@ -192,6 +192,10 @@ spec: description: NVIDIA GPUDirect Storage Driver image tag type: string type: object + hostNetwork: + description: HostNetwork indicates whether the Driver pod uses the + host's network namespace. + type: boolean image: default: nvcr.io/nvidia/driver description: NVIDIA Driver container image name diff --git a/packages/system/gpu-operator/charts/gpu-operator/templates/clusterpolicy.yaml b/packages/system/gpu-operator/charts/gpu-operator/templates/clusterpolicy.yaml index 978b5dc8..6e5c6a9c 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/templates/clusterpolicy.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/templates/clusterpolicy.yaml @@ -85,6 +85,9 @@ spec: {{- if .Values.validator.args }} args: {{ toYaml .Values.validator.args | nindent 6 }} {{- end }} + {{- if .Values.validator.hostNetwork }} + hostNetwork: {{ .Values.validator.hostNetwork }} + {{- end }} {{- if .Values.validator.plugin }} plugin: {{- if .Values.validator.plugin.env }} @@ -253,6 +256,9 @@ spec: timeoutSeconds: {{ .Values.driver.upgradePolicy.drain.timeoutSeconds }} deleteEmptyDir: {{ .Values.driver.upgradePolicy.drain.deleteEmptyDir | default false}} {{- end }} + {{- if .Values.driver.hostNetwork }} + hostNetwork: {{ .Values.driver.hostNetwork }} + {{- end }} vgpuManager: enabled: {{ .Values.vgpuManager.enabled }} {{- if .Values.vgpuManager.repository }} @@ -282,6 +288,9 @@ spec: {{- if .Values.vgpuManager.kernelModuleConfig }} kernelModuleConfig: {{ toYaml .Values.vgpuManager.kernelModuleConfig | nindent 6 }} {{- end }} + {{- if .Values.vgpuManager.hostNetwork }} + hostNetwork: {{ .Values.vgpuManager.hostNetwork }} + {{- end }} driverManager: {{- if .Values.vgpuManager.driverManager.repository }} repository: {{ .Values.vgpuManager.driverManager.repository }} @@ -327,6 +336,9 @@ spec: {{- if .Values.kataManager.args }} args: {{ toYaml .Values.kataManager.args | nindent 6 }} {{- end }} + {{- if .Values.kataManager.hostNetwork }} + hostNetwork: {{ .Values.kataManager.hostNetwork }} + {{- end }} vfioManager: enabled: {{ .Values.vfioManager.enabled }} {{- if .Values.vfioManager.repository }} @@ -369,6 +381,9 @@ spec: {{- if .Values.vfioManager.driverManager.env }} env: {{ toYaml .Values.vfioManager.driverManager.env | nindent 8 }} {{- end }} + {{- if .Values.vfioManager.hostNetwork }} + hostNetwork: {{ .Values.vfioManager.hostNetwork }} + {{- end }} vgpuDeviceManager: enabled: {{ .Values.vgpuDeviceManager.enabled }} {{- if .Values.vgpuDeviceManager.repository }} @@ -398,6 +413,9 @@ spec: {{- if .Values.vgpuDeviceManager.config }} config: {{ toYaml .Values.vgpuDeviceManager.config | nindent 6 }} {{- end }} + {{- if .Values.vgpuDeviceManager.hostNetwork }} + hostNetwork: {{ .Values.vgpuDeviceManager.hostNetwork }} + {{- end }} ccManager: enabled: {{ .Values.ccManager.enabled }} defaultMode: {{ .Values.ccManager.defaultMode | quote }} @@ -425,6 +443,9 @@ spec: {{- if .Values.ccManager.args }} args: {{ toYaml .Values.ccManager.args | nindent 6 }} {{- end }} + {{- if .Values.ccManager.hostNetwork }} + hostNetwork: {{ .Values.ccManager.hostNetwork }} + {{- end }} toolkit: enabled: {{ .Values.toolkit.enabled }} {{- if .Values.toolkit.repository }} @@ -451,6 +472,9 @@ spec: {{- if .Values.toolkit.installDir }} installDir: {{ .Values.toolkit.installDir }} {{- end }} + {{- if .Values.toolkit.hostNetwork }} + hostNetwork: {{ .Values.toolkit.hostNetwork }} + {{- end }} devicePlugin: enabled: {{ .Values.devicePlugin.enabled }} {{- if .Values.devicePlugin.repository }} @@ -482,6 +506,9 @@ spec: name: {{ .Values.devicePlugin.config.name | quote }} default: {{ .Values.devicePlugin.config.default | quote }} {{- end }} + {{- if .Values.devicePlugin.hostNetwork }} + hostNetwork: {{ .Values.devicePlugin.hostNetwork }} + {{- end }} dcgm: enabled: {{ .Values.dcgm.enabled }} {{- if .Values.dcgm.repository }} @@ -508,6 +535,9 @@ spec: {{- if .Values.dcgm.args }} args: {{ toYaml .Values.dcgm.args | nindent 6 }} {{- end }} + {{- if .Values.dcgm.hostNetwork }} + hostNetwork: {{ .Values.dcgm.hostNetwork }} + {{- end }} dcgmExporter: enabled: {{ .Values.dcgmExporter.enabled }} {{- if .Values.dcgmExporter.repository }} @@ -579,6 +609,9 @@ spec: {{- if .Values.gfd.args }} args: {{ toYaml .Values.gfd.args | nindent 6 }} {{- end }} + {{- if .Values.gfd.hostNetwork }} + hostNetwork: {{ .Values.gfd.hostNetwork }} + {{- end }} migManager: enabled: {{ .Values.migManager.enabled }} {{- if .Values.migManager.repository }} @@ -615,6 +648,9 @@ spec: {{- if .Values.migManager.gpuClientsConfig }} gpuClientsConfig: {{ toYaml .Values.migManager.gpuClientsConfig | nindent 6 }} {{- end }} + {{- if .Values.migManager.hostNetwork }} + hostNetwork: {{ .Values.migManager.hostNetwork }} + {{- end }} nodeStatusExporter: enabled: {{ .Values.nodeStatusExporter.enabled }} {{- if .Values.nodeStatusExporter.repository }} @@ -639,6 +675,9 @@ spec: {{- if .Values.nodeStatusExporter.args }} args: {{ toYaml .Values.nodeStatusExporter.args | nindent 6 }} {{- end }} + {{- if .Values.nodeStatusExporter.hostNetwork }} + hostNetwork: {{ .Values.nodeStatusExporter.hostNetwork }} + {{- end }} {{- if .Values.gds }} gds: enabled: {{ .Values.gds.enabled }} @@ -721,6 +760,9 @@ spec: {{- if .Values.sandboxDevicePlugin.args }} args: {{ toYaml .Values.sandboxDevicePlugin.args | nindent 6 }} {{- end }} + {{- if .Values.sandboxDevicePlugin.hostNetwork }} + hostNetwork: {{ .Values.sandboxDevicePlugin.hostNetwork }} + {{- end }} kataSandboxDevicePlugin: {{- if ne .Values.kataSandboxDevicePlugin.enabled nil }} enabled: {{ .Values.kataSandboxDevicePlugin.enabled }} @@ -749,3 +791,6 @@ spec: {{- if .Values.kataSandboxDevicePlugin.args }} args: {{ toYaml .Values.kataSandboxDevicePlugin.args | nindent 6 }} {{- end }} + {{- if .Values.kataSandboxDevicePlugin.hostNetwork }} + hostNetwork: {{ .Values.kataSandboxDevicePlugin.hostNetwork }} + {{- end }} diff --git a/packages/system/gpu-operator/charts/gpu-operator/templates/nvidiadriver.yaml b/packages/system/gpu-operator/charts/gpu-operator/templates/nvidiadriver.yaml index 60c36575..1a059a49 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/templates/nvidiadriver.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/templates/nvidiadriver.yaml @@ -78,6 +78,9 @@ spec: {{- if .Values.driver.args }} args: {{ toYaml .Values.driver.args | nindent 6 }} {{- end }} + {{- if .Values.driver.hostNetwork }} + hostNetwork: {{ .Values.driver.hostNetwork }} + {{- end }} {{- if .Values.gds.enabled }} gds: enabled: {{ .Values.gds.enabled }} diff --git a/packages/system/gpu-operator/charts/gpu-operator/values.yaml b/packages/system/gpu-operator/charts/gpu-operator/values.yaml index 222b0e38..6ee3c2f0 100644 --- a/packages/system/gpu-operator/charts/gpu-operator/values.yaml +++ b/packages/system/gpu-operator/charts/gpu-operator/values.yaml @@ -61,6 +61,7 @@ validator: env: [] args: [] resources: {} + hostNetwork: false plugin: env: [] @@ -204,6 +205,7 @@ driver: name: "" # Name of Kubernetes Secret which contains secrets to be passed in as environment variables secretEnv: "" + hostNetwork: false toolkit: enabled: true @@ -215,6 +217,7 @@ toolkit: env: [] resources: {} installDir: "/usr/local/nvidia" + hostNetwork: false devicePlugin: enabled: true @@ -259,6 +262,7 @@ devicePlugin: mps: # MPS root path on the host root: "/run/nvidia/mps" + hostNetwork: false # standalone dcgm hostengine dcgm: @@ -271,6 +275,7 @@ dcgm: args: [] env: [] resources: {} + hostNetwork: false dcgmExporter: enabled: true @@ -330,6 +335,7 @@ gfd: imagePullSecrets: [] env: [] resources: {} + hostNetwork: false migManager: enabled: true @@ -392,6 +398,7 @@ migManager: data: {} gpuClientsConfig: name: "" + hostNetwork: false nodeStatusExporter: enabled: false @@ -402,6 +409,7 @@ nodeStatusExporter: imagePullPolicy: IfNotPresent imagePullSecrets: [] resources: {} + hostNetwork: false gds: enabled: false @@ -417,7 +425,7 @@ gdrcopy: enabled: false repository: nvcr.io/nvidia/cloud-native image: gdrdrv - version: "v2.5.1" + version: "v2.5.2" imagePullPolicy: IfNotPresent imagePullSecrets: [] env: [] @@ -443,6 +451,7 @@ vgpuManager: # kernel module configuration for vGPU manager kernelModuleConfig: name: "" + hostNetwork: false vgpuDeviceManager: enabled: true @@ -455,6 +464,7 @@ vgpuDeviceManager: config: name: "" default: "default" + hostNetwork: false vfioManager: enabled: true @@ -473,6 +483,7 @@ vfioManager: version: v0.10.0 imagePullPolicy: IfNotPresent env: [] + hostNetwork: false kataManager: enabled: false @@ -481,6 +492,7 @@ kataManager: imagePullSecrets: [] env: [] resources: {} + hostNetwork: false sandboxDevicePlugin: enabled: true @@ -492,31 +504,31 @@ sandboxDevicePlugin: args: [] env: [] resources: {} + hostNetwork: false # Kata sandbox device plugin (used when sandboxWorkloads.mode is "kata"). kataSandboxDevicePlugin: enabled: true repository: nvcr.io/nvidia/cloud-native image: nvidia-sandbox-device-plugin - version: "v0.0.2" + version: "v0.0.3" imagePullPolicy: IfNotPresent imagePullSecrets: [] args: [] env: [] resources: {} + hostNetwork: false ccManager: enabled: true defaultMode: "on" repository: nvcr.io/nvidia/cloud-native image: k8s-cc-manager - version: v0.3.0 + version: v0.4.0 imagePullPolicy: IfNotPresent imagePullSecrets: [] - env: - - name: CC_CAPABLE_DEVICE_IDS - value: "0x2339,0x2331,0x2330,0x2324,0x2322,0x233d" resources: {} + hostNetwork: false # Array of extra K8s manifests to deploy # Supports use of custom Helm templates