feat(hami): add HAMi GPU virtualization system chart
Add HAMi v2.8.1 (CNCF Sandbox) as a new system package for fractional GPU sharing in tenant Kubernetes clusters. The chart enables workloads to request specific amounts of GPU memory and compute cores instead of claiming entire GPUs. Vendored upstream chart includes device plugin, scheduler extender, mutating webhook, and DRA subchart. Wrapper values configure nvidia RuntimeClass and clear hardcoded node config from upstream defaults. Assisted-By: Claude <noreply@anthropic.com> Signed-off-by: Arsolitt <arsolitt@gmail.com>
This commit is contained in:
parent
907bdba397
commit
51c2ec0ad4
61 changed files with 4198 additions and 0 deletions
3
packages/system/hami/Chart.yaml
Normal file
3
packages/system/hami/Chart.yaml
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
apiVersion: v2
|
||||
name: cozy-hami
|
||||
version: 0.0.0 # Placeholder, the actual version will be automatically set during the build process
|
||||
11
packages/system/hami/Makefile
Normal file
11
packages/system/hami/Makefile
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
export NAME=hami
|
||||
export NAMESPACE=cozy-$(NAME)
|
||||
|
||||
include ../../../hack/common-envs.mk
|
||||
include ../../../hack/package.mk
|
||||
|
||||
update:
|
||||
rm -rf charts
|
||||
helm repo add hami-charts https://project-hami.github.io/HAMi/
|
||||
helm repo update hami-charts
|
||||
helm pull hami-charts/hami --untar --untardir charts
|
||||
6
packages/system/hami/charts/hami/Chart.lock
Normal file
6
packages/system/hami/charts/hami/Chart.lock
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
dependencies:
|
||||
- name: hami-dra
|
||||
repository: https://project-hami.github.io/HAMi-DRA/
|
||||
version: 0.1.0
|
||||
digest: sha256:374551539570bcee82c1c4dea93eac59e6f410b6d5967d188efd630e203d43bb
|
||||
generated: "2026-04-17T04:06:47.689117678Z"
|
||||
22
packages/system/hami/charts/hami/Chart.yaml
Normal file
22
packages/system/hami/charts/hami/Chart.yaml
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
apiVersion: v2
|
||||
appVersion: 2.8.1
|
||||
dependencies:
|
||||
- condition: dra.enabled
|
||||
name: hami-dra
|
||||
repository: https://project-hami.github.io/HAMi-DRA/
|
||||
version: 0.1.0
|
||||
description: Heterogeneous AI Computing Virtualization Middleware
|
||||
keywords:
|
||||
- vgpu
|
||||
- gpu
|
||||
kubeVersion: '>= 1.18.0-0'
|
||||
maintainers:
|
||||
- email: archlitchi@gmail.com
|
||||
name: limengxuan
|
||||
- email: xiaozhang0210@hotmail.com
|
||||
name: zhangxiao
|
||||
name: hami
|
||||
sources:
|
||||
- https://github.com/Project-HAMi/HAMi
|
||||
type: application
|
||||
version: 2.8.1
|
||||
237
packages/system/hami/charts/hami/README.md
Normal file
237
packages/system/hami/charts/hami/README.md
Normal file
|
|
@ -0,0 +1,237 @@
|
|||
# HAMi Helm Chart Values Documentation
|
||||
|
||||
This document provides detailed descriptions of all configurable values parameters for the HAMi Helm Chart.
|
||||
|
||||
## Global Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `global.imageRegistry` | Global Docker image registry | `""` |
|
||||
| `global.imagePullSecrets` | Global Docker image pull secrets | `[]` |
|
||||
| `global.imageTag` | Image tag | `"v2.8.1"` |
|
||||
| `global.gpuHookPath` | GPU Hook path | `/usr/local` |
|
||||
| `global.labels` | Global labels | `{}` |
|
||||
| `global.annotations` | Global annotations | `{}` |
|
||||
| `global.managedNodeSelectorEnable` | Whether to enable managed node selector | `false` |
|
||||
| `global.managedNodeSelector.usage` | Managed node selector usage | `"gpu"` |
|
||||
| `nameOverride` | Name override | `""` |
|
||||
| `fullnameOverride` | Full name override | `""` |
|
||||
| `namespaceOverride` | Namespace override | `""` |
|
||||
|
||||
## Resource Name Configuration
|
||||
|
||||
### NVIDIA GPU Resources
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `resourceName` | GPU resource name | `"nvidia.com/gpu"` |
|
||||
| `resourceMem` | GPU memory resource name | `"nvidia.com/gpumem"` |
|
||||
| `resourceMemPercentage` | GPU memory percentage resource name | `"nvidia.com/gpumem-percentage"` |
|
||||
| `resourceCores` | GPU core resource name | `"nvidia.com/gpucores"` |
|
||||
| `resourcePriority` | GPU priority resource name | `"nvidia.com/priority"` |
|
||||
|
||||
### Cambricon MLU Resources
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `mluResourceName` | MLU resource name | `"cambricon.com/vmlu"` |
|
||||
| `mluResourceMem` | MLU memory resource name | `"cambricon.com/mlu.smlu.vmemory"` |
|
||||
| `mluResourceCores` | MLU core resource name | `"cambricon.com/mlu.smlu.vcore"` |
|
||||
|
||||
### Hygon DCU Resources
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `dcuResourceName` | DCU resource name | `"hygon.com/dcunum"` |
|
||||
| `dcuResourceMem` | DCU memory resource name | `"hygon.com/dcumem"` |
|
||||
| `dcuResourceCores` | DCU core resource name | `"hygon.com/dcucores"` |
|
||||
|
||||
### Metax GPU Resources
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `metaxResourceName` | GPU resource name | `"metax-tech.com/sgpu"` |
|
||||
| `metaxResourceCore` | GPU core resource name | `"metax-tech.com/vcore"` |
|
||||
| `metaxResourceMem` | GPU memory resource name | `"metax-tech.com/vmemory"` |
|
||||
| `metaxsGPUTopologyAware` | GPU topology awareness | `"false"` |
|
||||
|
||||
### Enflame GCU Resources
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `enflameResourceNameVGCU` | vGCU resource name | `"enflame.com/vgcu"` |
|
||||
| `enflameResourceNameVGCUPercentage` | vGCU percentage resource name | `"enflame.com/vgcu-percentage"` |
|
||||
|
||||
### Kunlunxin XPU Resources
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `kunlunResourceName` | XPU resource name | `"kunlunxin.com/xpu"` |
|
||||
|
||||
## Scheduler Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `schedulerName` | Scheduler name | `"hami-scheduler"` |
|
||||
| `scheduler.nodeName` | Define node name, scheduler will schedule to this node | `""` |
|
||||
| `scheduler.overwriteEnv` | Whether to overwrite environment variables | `"false"` |
|
||||
| `scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy` | Node scheduler policy | `binpack` |
|
||||
| `scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy` | GPU scheduler policy | `spread` |
|
||||
| `scheduler.metricsBindAddress` | Metrics bind address | `":9395"` |
|
||||
| `scheduler.forceOverwriteDefaultScheduler` | Whether to force overwrite default scheduler | `true` |
|
||||
| `scheduler.livenessProbe` | Whether to enable liveness probe | `false` |
|
||||
| `scheduler.leaderElect` | Whether to enable leader election | `true` |
|
||||
| `scheduler.replicas` | Number of replicas | `1` |
|
||||
|
||||
### Kube Scheduler Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `scheduler.kubeScheduler.enabled` | Whether to run kube-scheduler container in scheduler pod | `true` |
|
||||
| `scheduler.kubeScheduler.image.registry` | Kube scheduler image registry | `"registry.cn-hangzhou.aliyuncs.com"` |
|
||||
| `scheduler.kubeScheduler.image.repository` | Kube scheduler image repository | `"google_containers/kube-scheduler"` |
|
||||
| `scheduler.kubeScheduler.image.tag` | Kube scheduler image tag | `""` |
|
||||
| `scheduler.kubeScheduler.image.pullPolicy` | Kube scheduler image pull policy | `IfNotPresent` |
|
||||
| `scheduler.kubeScheduler.image.pullSecrets` | Kube scheduler image pull secrets | `[]` |
|
||||
| `scheduler.kubeScheduler.extraNewArgs` | Extra new arguments | `["--config=/config/config.yaml", "-v=4"]` |
|
||||
| `scheduler.kubeScheduler.extraArgs` | Extra arguments | `["--policy-config-file=/config/config.json", "-v=4"]` |
|
||||
|
||||
### Extender Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `scheduler.extender.image.registry` | Scheduler extender image registry | `"docker.io"` |
|
||||
| `scheduler.extender.image.repository` | Scheduler extender image repository | `"projecthami/hami"` |
|
||||
| `scheduler.extender.image.tag` | Scheduler extender image tag | `""` |
|
||||
| `scheduler.extender.image.pullPolicy` | Scheduler extender image pull policy | `IfNotPresent` |
|
||||
| `scheduler.extender.image.pullSecrets` | Scheduler extender image pull secrets | `[]` |
|
||||
| `scheduler.extender.extraArgs` | Scheduler extender extra arguments | `["--debug", "-v=4"]` |
|
||||
|
||||
### Admission Webhook Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `scheduler.admissionWebhook.enabled` | Whether to enable admission webhook | `true` |
|
||||
| `scheduler.admissionWebhook.customURL.enabled` | Whether to enable custom URL | `false` |
|
||||
| `scheduler.admissionWebhook.customURL.host` | Custom URL host | `127.0.0.1` |
|
||||
| `scheduler.admissionWebhook.customURL.port` | Custom URL port | `31998` |
|
||||
| `scheduler.admissionWebhook.customURL.path` | Custom URL path | `/webhook` |
|
||||
| `scheduler.admissionWebhook.reinvocationPolicy` | Reinvocation policy | `Never` |
|
||||
| `scheduler.admissionWebhook.failurePolicy` | Failure policy | `Ignore` |
|
||||
|
||||
### TLS Certificate Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `scheduler.certManager.enabled` | Whether to use cert-manager to generate self-signed certificates | `false` |
|
||||
| `scheduler.patch.enabled` | Whether to use kube-webhook-certgen to generate self-signed certificates | `true` |
|
||||
| `scheduler.patch.image.registry` | Certgen image registry | `"docker.io"` |
|
||||
| `scheduler.patch.image.repository` | Certgen image repository | `"jettech/kube-webhook-certgen"` |
|
||||
| `scheduler.patch.image.tag` | Certgen image tag | `"v1.5.2"` |
|
||||
| `scheduler.patch.image.pullPolicy` | Certgen image pull policy | `IfNotPresent` |
|
||||
| `scheduler.patch.imageNew.registry` | New certgen image registry | `"docker.io"` |
|
||||
| `scheduler.patch.imageNew.repository` | New certgen image repository | `"liangjw/kube-webhook-certgen"` |
|
||||
| `scheduler.patch.imageNew.tag` | New certgen image tag | `"v1.1.1"` |
|
||||
|
||||
### Scheduler Service Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `scheduler.service.type` | Service type | `NodePort` |
|
||||
| `scheduler.service.httpPort` | HTTP port | `443` |
|
||||
| `scheduler.service.schedulerPort` | Scheduler NodePort | `31998` |
|
||||
| `scheduler.service.monitorPort` | Monitor port | `31993` |
|
||||
| `scheduler.service.monitorTargetPort` | Monitor target port | `9395` |
|
||||
|
||||
## Device Plugin Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `devicePlugin.image.registry` | Device plugin image registry | `"docker.io"` |
|
||||
| `devicePlugin.image.repository` | Device plugin image repository | `"projecthami/hami"` |
|
||||
| `devicePlugin.image.tag` | Device plugin image tag | `""` |
|
||||
| `devicePlugin.image.pullPolicy` | Device plugin image pull policy | `IfNotPresent` |
|
||||
| `devicePlugin.image.pullSecrets` | Device plugin image pull secrets | `[]` |
|
||||
|
||||
### Monitor Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `devicePlugin.monitor.image.registry` | Monitor image registry | `"docker.io"` |
|
||||
| `devicePlugin.monitor.image.repository` | Monitor image repository | `"projecthami/hami"` |
|
||||
| `devicePlugin.monitor.image.tag` | Monitor image tag | `""` |
|
||||
| `devicePlugin.monitor.image.pullPolicy` | Monitor image pull policy | `IfNotPresent` |
|
||||
| `devicePlugin.monitor.image.pullSecrets` | Monitor image pull secrets | `[]` |
|
||||
| `devicePlugin.monitor.ctrPath` | Container path | `/usr/local/vgpu/containers` |
|
||||
| `devicePlugin.monitor.extraArgs` | Monitor extra arguments | `["-v=4"]` |
|
||||
| `devicePlugin.monitor.extraEnvs` | Monitor extra environments | `{}` |
|
||||
|
||||
### Device Plugin Other Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `devicePlugin.deviceSplitCount` | Integer type, default value: 10. Maximum number of tasks assigned to a single GPU device | `10` |
|
||||
| `devicePlugin.deviceMemoryScaling` | Device memory scaling ratio | `1` |
|
||||
| `devicePlugin.deviceCoreScaling` | Device core scaling ratio | `1` |
|
||||
| `devicePlugin.runtimeClassName` | Runtime class name | `""` |
|
||||
| `devicePlugin.createRuntimeClass` | Whether to create runtime class | `false` |
|
||||
| `devicePlugin.migStrategy` | String type, "none" means ignore MIG functionality, "mixed" means allocate MIG devices through independent resources | `"none"` |
|
||||
| `devicePlugin.disablecorelimit` | String type, "true" means disable core limit, "false" means enable core limit | `"false"` |
|
||||
| `devicePlugin.passDeviceSpecsEnabled` | Whether to enable passing device specs | `false` |
|
||||
| `devicePlugin.extraArgs` | Device plugin extra arguments | `["-v=4"]` |
|
||||
| `devicePlugin.nodeConfiguration.config` | Node configuration for device plugin by json | An example of default configuration. |
|
||||
| `devicePlugin.nodeConfiguration.externalConfigName` | Node configuration for device plugin by external congimap | `""` |
|
||||
| `devicePlugin.extraEnvs` | Device plugin extra environments | `{}` |
|
||||
|
||||
### Device Plugin Service Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `devicePlugin.service.type` | Service type | `NodePort` |
|
||||
| `devicePlugin.service.httpPort` | HTTP port | `31992` |
|
||||
|
||||
### Device Plugin Deployment Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `devicePlugin.pluginPath` | Plugin path | `/var/lib/kubelet/device-plugins` |
|
||||
| `devicePlugin.libPath` | Library path | `/usr/local/vgpu` |
|
||||
| `devicePlugin.nvidiaNodeSelector` | NVIDIA node selector | `{"gpu": "on"}` |
|
||||
| `devicePlugin.updateStrategy.type` | Update strategy type | `RollingUpdate` |
|
||||
| `devicePlugin.updateStrategy.rollingUpdate.maxUnavailable` | Maximum unavailable count | `1` |
|
||||
|
||||
## Device Configuration
|
||||
|
||||
### AWS Neuron
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `devices.awsneuron.customresources` | Custom resources | `["aws.amazon.com/neuron", "aws.amazon.com/neuroncore"]` |
|
||||
|
||||
### Kunlunxin
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `devices.kunlun.enabled` | Whether to enable | `true` |
|
||||
| `devices.kunlun.customresources` | Custom resources | `["kunlunxin.com/xpu"]` |
|
||||
|
||||
### Mthreads
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `devices.mthreads.enabled` | Whether to enable | `true` |
|
||||
| `devices.mthreads.customresources` | Custom resources | `["mthreads.com/vgpu"]` |
|
||||
|
||||
### NVIDIA
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `devices.nvidia.gpuCorePolicy` | GPU core policy | `default` |
|
||||
| `devices.nvidia.libCudaLogLevel` | CUDA library log level | `1` |
|
||||
|
||||
### Huawei Ascend
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `devices.ascend.enabled` | Whether to enable | `false` |
|
||||
| `devices.ascend.image` | Image | `""` |
|
||||
| `devices.ascend.imagePullPolicy` | Image pull policy | `IfNotPresent` |
|
||||
| `devices.ascend.extraArgs` | Extra arguments | `[]` |
|
||||
| `devices.ascend.nodeSelector` | Node selector | `{"ascend": "on"}` |
|
||||
| `devices.ascend.tolerations` | Tolerations | `[]` |
|
||||
| `devices.ascend.customresources` | Custom resources | `["huawei.com/Ascend910A", "huawei.com/Ascend910A-memory", ...]` |
|
||||
|
||||
### Iluvatar
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `devices.iluvatar.enabled` | Whether to enable | `false` |
|
||||
| `devices.iluvatar.customresources` | Custom resources | `["iluvatar.ai/BI-V150-vgpu", "iluvatar.ai/BI-V150.vMem","iluvatar.ai/BI-V150.vCore", ...]` |
|
||||
24
packages/system/hami/charts/hami/charts/hami-dra/.helmignore
Normal file
24
packages/system/hami/charts/hami/charts/hami-dra/.helmignore
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
# Patterns to ignore when building packages.
|
||||
# This supports shell glob matching, relative path matching, and
|
||||
# negation (prefixed with !). Only one pattern per line.
|
||||
.DS_Store
|
||||
# Common VCS dirs
|
||||
.git/
|
||||
.gitignore
|
||||
.bzr/
|
||||
.bzrignore
|
||||
.hg/
|
||||
.hgignore
|
||||
.svn/
|
||||
# Common backup files
|
||||
*.swp
|
||||
*.bak
|
||||
*.tmp
|
||||
*.orig
|
||||
*~
|
||||
# Various IDEs
|
||||
.project
|
||||
.idea/
|
||||
*.tmproj
|
||||
.vscode/
|
||||
|
||||
18
packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml
Normal file
18
packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
apiVersion: v2
|
||||
appVersion: 0.1.0
|
||||
description: A Helm chart for HAMi DRA
|
||||
home: https://github.com/Project-HAMi/HAMi-DRA
|
||||
keywords:
|
||||
- webhook
|
||||
- kubernetes
|
||||
- admission-controller
|
||||
- hami
|
||||
- dra
|
||||
maintainers:
|
||||
- name: hami-dra
|
||||
url: https://github.com/Project-HAMi/HAMi-DRA
|
||||
name: hami-dra
|
||||
sources:
|
||||
- https://github.com/Project-HAMi/HAMi-DRA
|
||||
type: application
|
||||
version: 0.1.0
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
{{/* vim: set filetype=mustache: */}}
|
||||
|
||||
{{/*
|
||||
Return the proper image name
|
||||
{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" $) }}
|
||||
*/}}
|
||||
{{- define "common.images.image" -}}
|
||||
{{- $registryName := .imageRoot.registry -}}
|
||||
{{- $repositoryName := .imageRoot.repository -}}
|
||||
{{- $tag := .imageRoot.tag | toString -}}
|
||||
{{- if .global }}
|
||||
{{- if .global.imageRegistry }}
|
||||
{{- $registryName = .global.imageRegistry -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- if .tag }}
|
||||
{{- if .tag.imageTag }}
|
||||
{{- $tag = .tag.imageTag -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- if $registryName }}
|
||||
{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}}
|
||||
{{- else -}}
|
||||
{{- printf "%s:%s" $repositoryName $tag -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead)
|
||||
{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }}
|
||||
*/}}
|
||||
{{- define "common.images.pullSecrets" -}}
|
||||
{{- $pullSecrets := list }}
|
||||
|
||||
{{- if .global }}
|
||||
{{- range .global.imagePullSecrets -}}
|
||||
{{- $pullSecrets = append $pullSecrets . -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- range .images -}}
|
||||
{{- range .pullSecrets -}}
|
||||
{{- $pullSecrets = append $pullSecrets . -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- if (not (empty $pullSecrets)) }}
|
||||
imagePullSecrets:
|
||||
{{- range $pullSecrets }}
|
||||
- name: {{ . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Renders a value that contains template.
|
||||
Usage:
|
||||
{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $) }}
|
||||
*/}}
|
||||
{{- define "common.tplvalues.render" -}}
|
||||
{{- if typeIs "string" .value }}
|
||||
{{- tpl .value .context }}
|
||||
{{- else }}
|
||||
{{- tpl (.value | toYaml) .context }}
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create a default fully qualified app name.
|
||||
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
|
||||
If release name contains chart name it will be used as a full name.
|
||||
*/}}
|
||||
{{- define "common.names.fullname" -}}
|
||||
{{- if .Values.fullnameOverride -}}
|
||||
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
|
||||
{{- else -}}
|
||||
{{- $name := default .Chart.Name .Values.nameOverride -}}
|
||||
{{- if contains $name .Release.Name -}}
|
||||
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
|
||||
{{- else -}}
|
||||
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
{{- define "hami.dra.webhook.fullname" -}}
|
||||
{{- printf "%s-%s" (include "common.names.fullname" .) "webhook" | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end }}
|
||||
|
||||
{{- define "hami.dra.webhook.image" -}}
|
||||
{{ include "common.images.image" (dict "imageRoot" .Values.webhook.image "global" .Values.global) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.dra.webhook.imagePullSecrets" -}}
|
||||
{{ include "common.images.pullSecrets" (dict "images" (list .Values.webhook.image) "global" .Values.global) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.dra.driver.nvidia.image" -}}
|
||||
{{ include "common.images.image" (dict "imageRoot" .Values.drivers.nvidia.image "global" .Values.global) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.dra.driver.nvidia.imagePullSecrets" -}}
|
||||
{{ include "common.images.pullSecrets" (dict "images" (list .Values.drivers.nvidia.image) "global" .Values.global) }}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Common labels
|
||||
*/}}
|
||||
{{- define "hami-dra-webhook.labels" -}}
|
||||
helm.sh/chart: {{ .Chart.Name }}
|
||||
{{ include "hami-dra-webhook.selectorLabels" . }}
|
||||
{{- if .Chart.AppVersion }}
|
||||
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
|
||||
{{- end }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Selector labels
|
||||
*/}}
|
||||
{{- define "hami-dra-webhook.selectorLabels" -}}
|
||||
app.kubernetes.io/name: {{ .Release.Name }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: webhook
|
||||
{{- end }}
|
||||
|
||||
{{- define "hami.dra.monitor.fullname" -}}
|
||||
{{- printf "%s-%s" (include "common.names.fullname" .) "monitor" | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end }}
|
||||
|
||||
{{- define "hami.dra.monitor.image" -}}
|
||||
{{ include "common.images.image" (dict "imageRoot" .Values.monitor.image "global" .Values.global) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.dra.monitor.imagePullSecrets" -}}
|
||||
{{ include "common.images.pullSecrets" (dict "images" (list .Values.monitor.image) "global" .Values.global) }}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Common labels for monitor
|
||||
*/}}
|
||||
{{- define "hami-dra-monitor.labels" -}}
|
||||
helm.sh/chart: {{ .Chart.Name }}
|
||||
{{ include "hami-dra-monitor.selectorLabels" . }}
|
||||
{{- if .Chart.AppVersion }}
|
||||
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
|
||||
{{- end }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Selector labels for monitor
|
||||
*/}}
|
||||
{{- define "hami-dra-monitor.selectorLabels" -}}
|
||||
app.kubernetes.io/name: {{ .Release.Name }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: monitor
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
{{- if .Values.drivers.nvidia.enabled }}
|
||||
apiVersion: resource.k8s.io/v1
|
||||
kind: DeviceClass
|
||||
metadata:
|
||||
name: hami-core-gpu.project-hami.io
|
||||
spec:
|
||||
selectors:
|
||||
- cel:
|
||||
expression: |-
|
||||
device.driver == "hami-core-gpu.project-hami.io" && device.attributes["hami-core-gpu.project-hami.io"].type == "hami-gpu"
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,145 @@
|
|||
{{- if .Values.drivers.nvidia.enabled }}
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: hami-dra-driver-kubelet-plugin
|
||||
namespace: {{ .Release.Namespace }}
|
||||
spec:
|
||||
revisionHistoryLimit: 10
|
||||
selector:
|
||||
matchLabels:
|
||||
hami-dra-driver-component: kubelet-plugin
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/instance: hami-dra-driver
|
||||
app.kubernetes.io/name: hami-dra-driver
|
||||
hami-dra-driver-component: kubelet-plugin
|
||||
spec:
|
||||
{{- include "hami.dra.driver.nvidia.imagePullSecrets" . | nindent 6 }}
|
||||
initContainers:
|
||||
- name: init-container
|
||||
image: {{ include "hami.dra.driver.nvidia.image" . }}
|
||||
securityContext:
|
||||
privileged: true
|
||||
command: [bash, /usr/bin/kubelet-plugin-prestart.sh]
|
||||
env:
|
||||
- name: NVIDIA_DRIVER_ROOT
|
||||
value: {{ if .Values.drivers.nvidia.containerDriver }}/run/nvidia/driver{{ else }}/{{ end }}
|
||||
- name: NVIDIA_VISIBLE_DEVICES
|
||||
value: void
|
||||
- name: KUBELET_REGISTRAR_DIRECTORY_PATH
|
||||
value: /var/lib/kubelet/plugins_registry
|
||||
- name: KUBELET_PLUGINS_DIRECTORY_PATH
|
||||
value: /var/lib/kubelet/plugins
|
||||
volumeMounts:
|
||||
- name: driver-root-parent
|
||||
mountPath: /driver-root-parent
|
||||
readOnly: true
|
||||
containers:
|
||||
- args:
|
||||
- |-
|
||||
# Conditionally mask the params file to prevent this container from
|
||||
# recreating any missing GPU device nodes. This is necessary, for
|
||||
# example, when running under nvkind to limit the set GPUs governed
|
||||
# by the plugin even though it has cgroup access to all of them.
|
||||
if [ "${MASK_NVIDIA_DRIVER_PARAMS}" = "true" ]; then
|
||||
cp /proc/driver/nvidia/params root/gpu-params
|
||||
sed -i 's/^ModifyDeviceFiles: 1$/ModifyDeviceFiles: 0/' root/gpu-params
|
||||
mount --bind root/gpu-params /proc/driver/nvidia/params
|
||||
fi
|
||||
hami-kubelet-plugin -v 6
|
||||
command:
|
||||
- bash
|
||||
- -c
|
||||
env:
|
||||
- name: MASK_NVIDIA_DRIVER_PARAMS
|
||||
value: ""
|
||||
- name: NVIDIA_DRIVER_ROOT
|
||||
value: {{ if .Values.drivers.nvidia.containerDriver }}/run/nvidia/driver{{ else }}/{{ end }}
|
||||
- name: NVIDIA_VISIBLE_DEVICES
|
||||
value: void
|
||||
- name: CDI_ROOT
|
||||
value: /var/run/cdi
|
||||
- name: NVIDIA_MIG_CONFIG_DEVICES
|
||||
value: all
|
||||
- name: NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
apiVersion: v1
|
||||
fieldPath: spec.nodeName
|
||||
- name: NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
apiVersion: v1
|
||||
fieldPath: metadata.namespace
|
||||
- name: KUBELET_REGISTRAR_DIRECTORY_PATH
|
||||
value: /var/lib/kubelet/plugins_registry
|
||||
- name: KUBELET_PLUGINS_DIRECTORY_PATH
|
||||
value: /var/lib/kubelet/plugins
|
||||
- name: IMAGE_NAME
|
||||
value: {{ include "hami.dra.driver.nvidia.image" . }}
|
||||
lifecycle:
|
||||
postStart:
|
||||
exec:
|
||||
command: ["/bin/bash", "-c", "/usr/bin/vgpu-init.sh /usr/local/vgpu/"]
|
||||
image: {{ include "hami.dra.driver.nvidia.image" . }}
|
||||
imagePullPolicy: {{ .Values.drivers.nvidia.image.pullPolicy | default "IfNotPresent" }}
|
||||
name: gpus
|
||||
securityContext:
|
||||
privileged: true
|
||||
terminationMessagePath: /dev/termination-log
|
||||
terminationMessagePolicy: File
|
||||
volumeMounts:
|
||||
- mountPath: /var/lib/kubelet/plugins_registry
|
||||
name: plugins-registry
|
||||
- mountPath: /var/lib/kubelet/plugins
|
||||
mountPropagation: Bidirectional
|
||||
name: plugins
|
||||
- mountPath: /var/run/cdi
|
||||
name: cdi
|
||||
- mountPath: /driver-root
|
||||
mountPropagation: HostToContainer
|
||||
name: driver-root
|
||||
readOnly: true
|
||||
- mountPath: /usr/local/vgpu
|
||||
name: host-vgpu
|
||||
- mountPath: /tmp
|
||||
name: host-tmp
|
||||
dnsPolicy: ClusterFirst
|
||||
serviceAccount: hami-dra-driver-service-account
|
||||
serviceAccountName: hami-dra-driver-service-account
|
||||
volumes:
|
||||
- hostPath:
|
||||
path: /var/lib/kubelet/plugins_registry
|
||||
type: ""
|
||||
name: plugins-registry
|
||||
- hostPath:
|
||||
path: /var/lib/kubelet/plugins
|
||||
type: ""
|
||||
name: plugins
|
||||
- hostPath:
|
||||
path: /var/run/cdi
|
||||
type: ""
|
||||
name: cdi
|
||||
- hostPath:
|
||||
path: {{if .Values.drivers.nvidia.containerDriver }}/run/nvidia{{ else }}/{{ end }}
|
||||
type: DirectoryOrCreate
|
||||
name: driver-root-parent
|
||||
- hostPath:
|
||||
path: {{if .Values.drivers.nvidia.containerDriver }}/run/nvidia/driver{{ else }}/{{ end }}
|
||||
type: DirectoryOrCreate
|
||||
name: driver-root
|
||||
- hostPath:
|
||||
path: /dev
|
||||
type: ""
|
||||
name: host-dev
|
||||
- hostPath:
|
||||
path: /usr/local/vgpu
|
||||
type: DirectoryOrCreate
|
||||
name: host-vgpu
|
||||
- hostPath:
|
||||
path: /tmp
|
||||
type: ""
|
||||
name: host-tmp
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,110 @@
|
|||
{{- if .Values.drivers.nvidia.enabled }}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: hami-dra-driver-service-account
|
||||
namespace: {{ .Release.Namespace }}
|
||||
---
|
||||
apiVersion: v1
|
||||
items:
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: hami-dra-driver-role
|
||||
namespace: {{ .Release.Namespace }}
|
||||
rules:
|
||||
- apiGroups:
|
||||
- apps
|
||||
resources:
|
||||
- daemonsets
|
||||
- deployments
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- create
|
||||
- update
|
||||
- patch
|
||||
- delete
|
||||
kind: List
|
||||
metadata:
|
||||
resourceVersion: ""
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: hami-dra-driver-role-binding
|
||||
namespace: {{ .Release.Namespace }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: hami-dra-driver-role
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: hami-dra-driver-service-account
|
||||
namespace: {{ .Release.Namespace }}
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: hami-dra-driver-clusterrole
|
||||
rules:
|
||||
- apiGroups:
|
||||
- resource.k8s.io
|
||||
resources:
|
||||
- resourceclaims
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- apiGroups:
|
||||
- resource.k8s.io
|
||||
resources:
|
||||
- resourceclaimtemplates
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
- apiGroups:
|
||||
- resource.k8s.io
|
||||
resources:
|
||||
- resourceslices
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
- apiGroups:
|
||||
- resource.k8s.io
|
||||
resources:
|
||||
- resourceclaims/status
|
||||
verbs:
|
||||
- update
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- nodes
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- update
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: hami-dra-driver-clusterrole-binding-hami-dra-driver
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: hami-dra-driver-clusterrole
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: hami-dra-driver-service-account
|
||||
namespace: {{ .Release.Namespace }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
{{- if .Values.monitor.enabled }}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: {{ include "hami.dra.monitor.fullname" . }}
|
||||
rules:
|
||||
- apiGroups: ["resource.k8s.io"]
|
||||
resources: ["resourceslices", "resourceclaims"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
{{- end }}
|
||||
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
{{- if .Values.monitor.enabled }}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: {{ include "hami.dra.monitor.fullname" . }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: {{ include "hami.dra.monitor.fullname" . }}
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "hami.dra.monitor.fullname" . }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
{{- end }}
|
||||
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
{{- if .Values.monitor.enabled }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "hami.dra.monitor.fullname" . }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ include "hami.dra.monitor.fullname" . }}
|
||||
{{- include "hami-dra-monitor.labels" . | nindent 4 }}
|
||||
spec:
|
||||
replicas: {{ .Values.monitor.replicas | default 1 }}
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "hami-dra-monitor.selectorLabels" . | nindent 6 }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "hami-dra-monitor.selectorLabels" . | nindent 8 }}
|
||||
spec:
|
||||
{{- include "hami.dra.monitor.imagePullSecrets" . | nindent 6 }}
|
||||
serviceAccountName: {{ include "hami.dra.monitor.fullname" . }}
|
||||
containers:
|
||||
- name: monitor
|
||||
image: {{ include "hami.dra.monitor.image" . }}
|
||||
imagePullPolicy: {{ .Values.monitor.image.pullPolicy | default "IfNotPresent" }}
|
||||
command:
|
||||
- /bin/monitor
|
||||
- --v={{ .Values.monitor.logLevel | default 2 }}
|
||||
- --metrics-bind-address={{ .Values.monitor.metricsBindAddress | default ":8080" }}
|
||||
- --health-probe-bind-address={{ .Values.monitor.healthProbeBindAddress | default ":8000" }}
|
||||
- --kube-api-qps={{ .Values.monitor.kubeAPIQPS | default 40.0 }}
|
||||
- --kube-api-burst={{ .Values.monitor.kubeAPIBurst | default 60 }}
|
||||
- --collect-interval={{ .Values.monitor.collectInterval | default "30s" }}
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
name: metrics
|
||||
protocol: TCP
|
||||
- containerPort: 8000
|
||||
name: health
|
||||
protocol: TCP
|
||||
{{- if .Values.monitor.resources }}
|
||||
resources:
|
||||
{{- toYaml .Values.monitor.resources | nindent 12 }}
|
||||
{{- end }}
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: 8000
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 3
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /readyz
|
||||
port: 8000
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 3
|
||||
{{- end }}
|
||||
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
{{- if .Values.monitor.enabled }}
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "hami.dra.monitor.fullname" . }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "hami-dra-monitor.selectorLabels" . | nindent 4 }}
|
||||
spec:
|
||||
type: {{ .Values.monitor.service.type | default "ClusterIP" }}
|
||||
selector:
|
||||
{{- include "hami-dra-monitor.selectorLabels" . | nindent 4 }}
|
||||
ports:
|
||||
- port: 8080
|
||||
targetPort: 8080
|
||||
name: metrics
|
||||
protocol: TCP
|
||||
{{- if and (eq (.Values.monitor.service.type | default "ClusterIP") "NodePort") .Values.monitor.service.nodePort.metrics }}
|
||||
nodePort: {{ .Values.monitor.service.nodePort.metrics }}
|
||||
{{- end }}
|
||||
- port: 8000
|
||||
targetPort: 8000
|
||||
name: health
|
||||
protocol: TCP
|
||||
{{- end }}
|
||||
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
{{- if .Values.monitor.enabled }}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "hami.dra.monitor.fullname" . }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
{{- end }}
|
||||
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
{{- if .Values.certs.certManager.enabled }}
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-dra-webhook-serving-cert
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "hami-dra-webhook.labels" . | nindent 4 }}
|
||||
spec:
|
||||
dnsNames:
|
||||
- {{ .Release.Name }}-dra-webhook.{{ .Release.Namespace }}.svc
|
||||
- {{ .Release.Name }}-dra-webhook.{{ .Release.Namespace }}.svc.cluster.local
|
||||
issuerRef:
|
||||
kind: Issuer
|
||||
name: {{ .Release.Name }}-dra-webhook-selfsigned-issuer
|
||||
secretName: {{ .Release.Name }}-dra-webhook-tls
|
||||
privateKey:
|
||||
rotationPolicy: {{ .Values.certs.certManager.privateKeyRotationPolicy | default "Never" }}
|
||||
---
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Issuer
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-dra-webhook-selfsigned-issuer
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-dra-webhook
|
||||
spec:
|
||||
selfSigned: {}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
{{- if eq .Values.certs.mode "custom" }}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: {{ include "hami-dra-webhook.tlsSecretName" . }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
type: kubernetes.io/tls
|
||||
data:
|
||||
tls.crt: |
|
||||
{{- .Values.certs.custom.crt | b64enc | indent 4 }}
|
||||
tls.key: |
|
||||
{{- .Values.certs.custom.key | b64enc | indent 4 }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,394 @@
|
|||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "hami.dra.webhook.fullname" . }}-device-config
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-scheduler
|
||||
{{- include "hami-dra-webhook.selectorLabels" . | nindent 4 }}
|
||||
data:
|
||||
device-config.yaml: |-
|
||||
{{- if .Files.Glob "files/device-config.yaml" }}
|
||||
{{- .Files.Get "files/device-config.yaml" | nindent 4}}
|
||||
{{- else }}
|
||||
nvidia:
|
||||
resourceCountName: {{ .Values.resourceName }}
|
||||
resourceMemoryName: {{ .Values.resourceMem }}
|
||||
resourceMemoryPercentageName: {{ .Values.resourceMemPercentage }}
|
||||
resourceCoreName: {{ .Values.resourceCores }}
|
||||
resourcePriorityName: {{ .Values.resourcePriority }}
|
||||
overwriteEnv: false
|
||||
defaultMemory: 0
|
||||
defaultCores: 0
|
||||
defaultGPUNum: 1
|
||||
knownMigGeometries:
|
||||
- models: [ "A30" ]
|
||||
allowedGeometries:
|
||||
-
|
||||
- name: 1g.6gb
|
||||
core: 25
|
||||
memory: 6144
|
||||
count: 4
|
||||
-
|
||||
- name: 2g.12gb
|
||||
core: 50
|
||||
memory: 12288
|
||||
count: 2
|
||||
-
|
||||
- name: 4g.24gb
|
||||
core: 100
|
||||
memory: 24576
|
||||
count: 1
|
||||
- models: [ "A100-SXM4-40GB", "A100-40GB-PCIe", "A100-PCIE-40GB"]
|
||||
allowedGeometries:
|
||||
-
|
||||
- name: 1g.5gb
|
||||
core: 14
|
||||
memory: 5120
|
||||
count: 7
|
||||
-
|
||||
- name: 1g.5gb
|
||||
core: 14
|
||||
memory: 5120
|
||||
count: 1
|
||||
- name: 2g.10gb
|
||||
core: 28
|
||||
memory: 10240
|
||||
count: 3
|
||||
-
|
||||
- name: 3g.20gb
|
||||
core: 42
|
||||
memory: 20480
|
||||
count: 2
|
||||
-
|
||||
- name: 7g.40gb
|
||||
core: 100
|
||||
memory: 40960
|
||||
count: 1
|
||||
- models: [ "A100-SXM4-80GB", "A100-80GB-PCIe", "A100-PCIE-80GB"]
|
||||
allowedGeometries:
|
||||
-
|
||||
- name: 1g.10gb
|
||||
core: 14
|
||||
memory: 10240
|
||||
count: 7
|
||||
-
|
||||
- name: 1g.10gb
|
||||
core: 14
|
||||
memory: 10240
|
||||
count: 1
|
||||
- name: 2g.20gb
|
||||
core: 28
|
||||
memory: 20480
|
||||
count: 3
|
||||
-
|
||||
- name: 3g.40gb
|
||||
core: 42
|
||||
memory: 40960
|
||||
count: 2
|
||||
-
|
||||
- name: 7g.79gb
|
||||
core: 100
|
||||
memory: 80896
|
||||
count: 1
|
||||
- models: [ "H100-PCIE-80GB", "H100-SXM5-80GB"]
|
||||
allowedGeometries:
|
||||
-
|
||||
- name: 1g.10gb
|
||||
core: 14
|
||||
memory: 10240
|
||||
count: 7
|
||||
-
|
||||
- name: 1g.10gb
|
||||
core: 14
|
||||
memory: 10240
|
||||
count: 1
|
||||
- name: 2g.20gb
|
||||
core: 28
|
||||
memory: 20480
|
||||
count: 3
|
||||
-
|
||||
- name: 3g.40gb
|
||||
core: 42
|
||||
memory: 40960
|
||||
count: 2
|
||||
-
|
||||
- name: 7g.80gb
|
||||
core: 100
|
||||
memory: 81920
|
||||
count: 1
|
||||
- models: [ "H100-PCIE-94GB", "H100-SXM5-94GB"]
|
||||
allowedGeometries:
|
||||
-
|
||||
- name: 1g.12gb
|
||||
core: 14
|
||||
memory: 12288
|
||||
count: 7
|
||||
-
|
||||
- name: 1g.12gb
|
||||
core: 14
|
||||
memory: 12288
|
||||
count: 1
|
||||
- name: 2g.24gb
|
||||
core: 28
|
||||
memory: 24576
|
||||
count: 3
|
||||
-
|
||||
- name: 3g.47gb
|
||||
core: 42
|
||||
memory: 48128
|
||||
count: 2
|
||||
-
|
||||
- name: 7g.94gb
|
||||
core: 100
|
||||
memory: 96256
|
||||
count: 1
|
||||
- models: [ "H20", "H100 on GH200"]
|
||||
allowedGeometries:
|
||||
-
|
||||
- name: 1g.12gb
|
||||
core: 14
|
||||
memory: 12288
|
||||
count: 7
|
||||
-
|
||||
- name: 1g.12gb
|
||||
core: 14
|
||||
memory: 12288
|
||||
count: 1
|
||||
- name: 2g.24gb
|
||||
core: 28
|
||||
memory: 24576
|
||||
count: 3
|
||||
-
|
||||
- name: 3g.48gb
|
||||
core: 42
|
||||
memory: 49152
|
||||
count: 2
|
||||
-
|
||||
- name: 7g.96gb
|
||||
core: 100
|
||||
memory: 98304
|
||||
count: 1
|
||||
- models: [ "H200 NVL", "H200-SXM5"]
|
||||
allowedGeometries:
|
||||
-
|
||||
- name: 1g.18gb
|
||||
core: 14
|
||||
memory: 18432
|
||||
count: 7
|
||||
-
|
||||
- name: 1g.18gb
|
||||
core: 14
|
||||
memory: 18432
|
||||
count: 1
|
||||
- name: 2g.35gb
|
||||
core: 28
|
||||
memory: 35840
|
||||
count: 3
|
||||
-
|
||||
- name: 3g.71gb
|
||||
core: 42
|
||||
memory: 72704
|
||||
count: 2
|
||||
-
|
||||
- name: 7g.141gb
|
||||
core: 100
|
||||
memory: 144384
|
||||
count: 1
|
||||
- models: [ "B200" ]
|
||||
allowedGeometries:
|
||||
-
|
||||
- name: 1g.23gb
|
||||
core: 14
|
||||
memory: 23552
|
||||
count: 7
|
||||
-
|
||||
- name: 1g.23gb
|
||||
core: 14
|
||||
memory: 23552
|
||||
count: 1
|
||||
- name: 2g.45gb
|
||||
core: 28
|
||||
memory: 46080
|
||||
count: 3
|
||||
-
|
||||
- name: 3g.90gb
|
||||
core: 42
|
||||
memory: 92160
|
||||
count: 2
|
||||
-
|
||||
- name: 7g.180gb
|
||||
core: 100
|
||||
memory: 184320
|
||||
count: 1
|
||||
cambricon:
|
||||
resourceCountName: {{ .Values.mluResourceName }}
|
||||
resourceMemoryName: {{ .Values.mluResourceMem }}
|
||||
resourceCoreName: {{ .Values.mluResourceCores }}
|
||||
hygon:
|
||||
resourceCountName: {{ .Values.dcuResourceName }}
|
||||
resourceMemoryName: {{ .Values.dcuResourceMem }}
|
||||
resourceCoreName: {{ .Values.dcuResourceCores }}
|
||||
metax:
|
||||
resourceCountName: "metax-tech.com/gpu"
|
||||
resourceVCountName: {{ .Values.metaxResourceName }}
|
||||
resourceVMemoryName: {{ .Values.metaxResourceMem }}
|
||||
resourceVCoreName: {{ .Values.metaxResourceCore }}
|
||||
sgpuTopologyAware: {{ .Values.metaxsGPUTopologyAware }}
|
||||
enflame:
|
||||
resourceNameGCU: "enflame.com/gcu"
|
||||
resourceNameVGCU: {{ .Values.enflameResourceNameVGCU }}
|
||||
resourceNameVGCUPercentage: {{ .Values.enflameResourceNameVGCUPercentage }}
|
||||
mthreads:
|
||||
resourceCountName: "mthreads.com/vgpu"
|
||||
resourceMemoryName: "mthreads.com/sgpu-memory"
|
||||
resourceCoreName: "mthreads.com/sgpu-core"
|
||||
iluvatars:
|
||||
- chipName: MR-V100
|
||||
commonWord: MR-V100
|
||||
resourceCountName: iluvatar.ai/MR-V100-vgpu
|
||||
resourceMemoryName: iluvatar.ai/MR-V100.vMem
|
||||
resourceCoreName: iluvatar.ai/MR-V100.vCore
|
||||
- chipName: MR-V50
|
||||
commonWord: MR-V50
|
||||
resourceCountName: iluvatar.ai/MR-V50-vgpu
|
||||
resourceMemoryName: iluvatar.ai/MR-V50.vMem
|
||||
resourceCoreName: iluvatar.ai/MR-V50.vCore
|
||||
- chipName: BI-V150
|
||||
commonWord: BI-V150
|
||||
resourceCountName: iluvatar.ai/BI-V150-vgpu
|
||||
resourceMemoryName: iluvatar.ai/BI-V150.vMem
|
||||
resourceCoreName: iluvatar.ai/BI-V150.vCore
|
||||
- chipName: BI-V100
|
||||
commonWord: BI-V100
|
||||
resourceCountName: iluvatar.ai/BI-V100-vgpu
|
||||
resourceMemoryName: iluvatar.ai/BI-V100.vMem
|
||||
resourceCoreName: iluvatar.ai/BI-V100.vCore
|
||||
kunlun:
|
||||
resourceCountName: {{ .Values.kunlunResourceName }}
|
||||
resourceVCountName: {{ .Values.kunlunResourceVCountName }}
|
||||
resourceVMemoryName: {{ .Values.kunlunResourceVMemoryName }}
|
||||
awsneuron:
|
||||
resourceCountName: "aws.amazon.com/neuron"
|
||||
resourceCoreName: "aws.amazon.com/neuroncore"
|
||||
amd:
|
||||
resourceCountName: "amd.com/gpu"
|
||||
vnpus:
|
||||
- chipName: 910A
|
||||
commonWord: Ascend910A
|
||||
resourceName: huawei.com/Ascend910A
|
||||
resourceMemoryName: huawei.com/Ascend910A-memory
|
||||
memoryAllocatable: 32768
|
||||
memoryCapacity: 32768
|
||||
aiCore: 30
|
||||
templates:
|
||||
- name: vir02
|
||||
memory: 2184
|
||||
aiCore: 2
|
||||
- name: vir04
|
||||
memory: 4369
|
||||
aiCore: 4
|
||||
- name: vir08
|
||||
memory: 8738
|
||||
aiCore: 8
|
||||
- name: vir16
|
||||
memory: 17476
|
||||
aiCore: 16
|
||||
- chipName: 910B2
|
||||
commonWord: Ascend910B2
|
||||
resourceName: huawei.com/Ascend910B2
|
||||
resourceMemoryName: huawei.com/Ascend910B2-memory
|
||||
memoryAllocatable: 65536
|
||||
memoryCapacity: 65536
|
||||
aiCore: 24
|
||||
aiCPU: 6
|
||||
templates:
|
||||
- name: vir03_1c_8g
|
||||
memory: 8192
|
||||
aiCore: 3
|
||||
aiCPU: 1
|
||||
- name: vir06_1c_16g
|
||||
memory: 16384
|
||||
aiCore: 6
|
||||
aiCPU: 1
|
||||
- name: vir12_3c_32g
|
||||
memory: 32768
|
||||
aiCore: 12
|
||||
aiCPU: 3
|
||||
- chipName: 910B3
|
||||
commonWord: Ascend910B3
|
||||
resourceName: huawei.com/Ascend910B3
|
||||
resourceMemoryName: huawei.com/Ascend910B3-memory
|
||||
memoryAllocatable: 65536
|
||||
memoryCapacity: 65536
|
||||
aiCore: 20
|
||||
aiCPU: 7
|
||||
templates:
|
||||
- name: vir05_1c_16g
|
||||
memory: 16384
|
||||
aiCore: 5
|
||||
aiCPU: 1
|
||||
- name: vir10_3c_32g
|
||||
memory: 32768
|
||||
aiCore: 10
|
||||
aiCPU: 3
|
||||
- chipName: 910B4-1
|
||||
commonWord: Ascend910B4-1
|
||||
resourceName: huawei.com/Ascend910B4-1
|
||||
resourceMemoryName: huawei.com/Ascend910B4-1-memory
|
||||
memoryAllocatable: 65536
|
||||
memoryCapacity: 65536
|
||||
aiCore: 20
|
||||
aiCPU: 7
|
||||
templates:
|
||||
# NOTE: Names of vnpu template for 910B4-1 are fixed by Ascend runtime and must not be changed.
|
||||
# The memory is used for scheduling so the correct values must be set.
|
||||
# Template vir05_1c_8g actually provides 16GB memory,
|
||||
- name: vir05_1c_8g
|
||||
memory: 16384
|
||||
aiCore: 5
|
||||
aiCPU: 1
|
||||
# Template vir10_3c_16g actually provides 32GB memory
|
||||
- name: vir10_3c_16g
|
||||
memory: 32768
|
||||
aiCore: 10
|
||||
aiCPU: 3
|
||||
- chipName: 910B4
|
||||
commonWord: Ascend910B4
|
||||
resourceName: huawei.com/Ascend910B4
|
||||
resourceMemoryName: huawei.com/Ascend910B4-memory
|
||||
memoryAllocatable: 32768
|
||||
memoryCapacity: 32768
|
||||
aiCore: 20
|
||||
aiCPU: 7
|
||||
templates:
|
||||
- name: vir05_1c_8g
|
||||
memory: 8192
|
||||
aiCore: 5
|
||||
aiCPU: 1
|
||||
- name: vir10_3c_16g
|
||||
memory: 16384
|
||||
aiCore: 10
|
||||
aiCPU: 3
|
||||
- chipName: 310P3
|
||||
commonWord: Ascend310P
|
||||
resourceName: huawei.com/Ascend310P
|
||||
resourceMemoryName: huawei.com/Ascend310P-memory
|
||||
memoryAllocatable: 21527
|
||||
memoryCapacity: 24576
|
||||
aiCore: 8
|
||||
aiCPU: 7
|
||||
templates:
|
||||
- name: vir01
|
||||
memory: 3072
|
||||
aiCore: 1
|
||||
aiCPU: 1
|
||||
- name: vir02
|
||||
memory: 6144
|
||||
aiCore: 2
|
||||
aiCPU: 2
|
||||
- name: vir04
|
||||
memory: 12288
|
||||
aiCore: 4
|
||||
aiCPU: 4
|
||||
{{ end }}
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
{{- if .Values.webhook.config.mutating.enabled }}
|
||||
apiVersion: admissionregistration.k8s.io/v1
|
||||
kind: MutatingWebhookConfiguration
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-mutatingwebhookconfiguration
|
||||
annotations:
|
||||
cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ .Release.Name }}-dra-webhook-serving-cert
|
||||
webhooks:
|
||||
- name: mutate.hami.io
|
||||
admissionReviewVersions: ["v1"]
|
||||
clientConfig:
|
||||
service:
|
||||
name: {{ .Release.Name }}-dra-webhook
|
||||
namespace: {{ .Release.Namespace }}
|
||||
path: /mutate
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
apiVersions:
|
||||
- v1
|
||||
operations:
|
||||
- CREATE
|
||||
resources:
|
||||
- pods
|
||||
scope: '*'
|
||||
sideEffects: None
|
||||
timeoutSeconds: 10
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
{{- if .Values.webhook.config.validating.enabled }}
|
||||
apiVersion: admissionregistration.k8s.io/v1
|
||||
kind: ValidatingWebhookConfiguration
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-validatingwebhookconfiguration
|
||||
annotations:
|
||||
cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ .Release.Name }}-dra-webhook-serving-cert
|
||||
webhooks:
|
||||
- name: validate.hami.io
|
||||
admissionReviewVersions: ["v1"]
|
||||
clientConfig:
|
||||
service:
|
||||
name: {{ .Release.Name }}-dra-webhook
|
||||
namespace: {{ .Release.Namespace }}
|
||||
path: /validate
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
apiVersions:
|
||||
- v1
|
||||
operations:
|
||||
- DELETE
|
||||
resources:
|
||||
- pods
|
||||
scope: '*'
|
||||
sideEffects: None
|
||||
timeoutSeconds: 10
|
||||
failurePolicy: Ignore
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: {{ include "hami.dra.webhook.fullname" . }}
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["pods"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["admissionregistration.k8s.io"]
|
||||
resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["resource.k8s.io"]
|
||||
resources: ["resourceclaims"]
|
||||
verbs: ["*"]
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: {{ include "hami.dra.webhook.fullname" . }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: {{ include "hami.dra.webhook.fullname" . }}
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "hami.dra.webhook.fullname" . }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "hami.dra.webhook.fullname" . }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ include "hami.dra.webhook.fullname" . }}
|
||||
{{- include "hami-dra-webhook.labels" . | nindent 4 }}
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "hami-dra-webhook.selectorLabels" . | nindent 6 }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "hami-dra-webhook.selectorLabels" . | nindent 8 }}
|
||||
spec:
|
||||
{{- include "hami.dra.webhook.imagePullSecrets" . | nindent 6 }}
|
||||
serviceAccountName: {{ include "hami.dra.webhook.fullname" . }}
|
||||
containers:
|
||||
- name: webhook
|
||||
image: {{ include "hami.dra.webhook.image" . }}
|
||||
imagePullPolicy: {{ .Values.webhook.image.pullPolicy | default "IfNotPresent" }}
|
||||
command:
|
||||
- /bin/webhook
|
||||
- --v=5
|
||||
- --bind-address=0.0.0.0
|
||||
- --secure-port=8443
|
||||
- --cert-dir=/tls
|
||||
- --tls-cert-file-name=tls.crt
|
||||
- --tls-private-key-file-name=tls.key
|
||||
- --metrics-bind-address=:8080
|
||||
- --health-probe-bind-address=:8000
|
||||
- --device-config-file=/device-config.yaml
|
||||
ports:
|
||||
- containerPort: 8443
|
||||
name: webhook
|
||||
protocol: TCP
|
||||
- containerPort: 8080
|
||||
name: metrics
|
||||
protocol: TCP
|
||||
- containerPort: 8000
|
||||
name: health
|
||||
protocol: TCP
|
||||
volumeMounts:
|
||||
- name: device-config
|
||||
mountPath: /device-config.yaml
|
||||
subPath: device-config.yaml
|
||||
- name: tls-config
|
||||
mountPath: /tls
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: device-config
|
||||
configMap:
|
||||
name: {{ include "hami.dra.webhook.fullname" . }}-device-config
|
||||
- name: tls-config
|
||||
secret:
|
||||
secretName: {{ if .Values.certs.certManager.enabled }}{{ .Release.Name }}-dra-webhook-tls{{ else }}{{ .Release.Name }}-tls{{ end }}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-dra-webhook
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "hami-dra-webhook.selectorLabels" . | nindent 4 }}
|
||||
spec:
|
||||
type: "ClusterIP"
|
||||
selector:
|
||||
{{- include "hami-dra-webhook.selectorLabels" . | nindent 4 }}
|
||||
ports:
|
||||
- port: 443
|
||||
targetPort: 8443
|
||||
name: webhook
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "hami.dra.webhook.fullname" . }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
166
packages/system/hami/charts/hami/charts/hami-dra/values.yaml
Normal file
166
packages/system/hami/charts/hami/charts/hami-dra/values.yaml
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
## @section Global configuration
|
||||
## @param global.imageRegistry Global Docker image registry
|
||||
## @param global.imagePullSecrets Global Docker image pull secrets
|
||||
global:
|
||||
## @param global.imageRegistry Global Docker image registry
|
||||
imageRegistry: ""
|
||||
## E.g.
|
||||
## imagePullSecrets:
|
||||
## - myRegistryKeySecretName
|
||||
## @param global.imagePullSecrets Global Docker image pull secrets
|
||||
imagePullSecrets: []
|
||||
|
||||
#Nvidia GPU Parameters
|
||||
resourceName: "nvidia.com/gpu"
|
||||
resourceMem: "nvidia.com/gpumem"
|
||||
resourceMemPercentage: "nvidia.com/gpumem-percentage"
|
||||
resourceCores: "nvidia.com/gpucores"
|
||||
resourcePriority: "nvidia.com/priority"
|
||||
|
||||
#MLU Parameters
|
||||
mluResourceName: "cambricon.com/vmlu"
|
||||
mluResourceMem: "cambricon.com/mlu.smlu.vmemory"
|
||||
mluResourceCores: "cambricon.com/mlu.smlu.vcore"
|
||||
|
||||
#Hygon DCU Parameters
|
||||
dcuResourceName: "hygon.com/dcunum"
|
||||
dcuResourceMem: "hygon.com/dcumem"
|
||||
dcuResourceCores: "hygon.com/dcucores"
|
||||
|
||||
#Metax sGPU Parameters
|
||||
metaxResourceName: "metax-tech.com/sgpu"
|
||||
metaxResourceCore: "metax-tech.com/vcore"
|
||||
metaxResourceMem: "metax-tech.com/vmemory"
|
||||
metaxsGPUTopologyAware: "false"
|
||||
|
||||
#Enflame VGCU Parameters
|
||||
enflameResourceNameVGCU: "enflame.com/vgcu"
|
||||
enflameResourceNameVGCUPercentage: "enflame.com/vgcu-percentage"
|
||||
|
||||
#Kunlun XPU Parameters
|
||||
kunlunResourceName: "kunlunxin.com/xpu"
|
||||
kunlunResourceVCountName: "kunlunxin.com/vxpu"
|
||||
kunlunResourceVMemoryName: "kunlunxin.com/vxpu-memory"
|
||||
|
||||
# Webhook deployment configuration
|
||||
webhook:
|
||||
image:
|
||||
registry: ghcr.io
|
||||
repository: project-hami/hami-dra-webhook
|
||||
tag: "v0.1.0"
|
||||
## Specify a imagePullPolicy
|
||||
## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
|
||||
## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
|
||||
##
|
||||
pullPolicy: Always
|
||||
## Optionally specify an array of imagePullSecrets.
|
||||
## Secrets must be manually created in the namespace.
|
||||
## Example:
|
||||
## pullSecrets:
|
||||
## - myRegistryKeySecretName
|
||||
##
|
||||
pullSecrets: []
|
||||
## @param resources controllerManager resource requests and limits
|
||||
resources:
|
||||
# If you do want to specify resources, uncomment the following
|
||||
# lines, adjust them as necessary, and remove the curly braces after 'resources:'.
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
## @param nodeSelector controllerManager node labels for pod assignment
|
||||
args:
|
||||
webhookName: "" # Default: {{ .Release.Name }}-hami-dra-webhook
|
||||
secretName: "" # Default: {{ .Release.Name }}-hami-dra-webhook-tls
|
||||
# Webhook configuration
|
||||
config:
|
||||
mutating:
|
||||
enabled: true
|
||||
validating:
|
||||
enabled: true
|
||||
|
||||
# Monitor deployment configuration
|
||||
monitor:
|
||||
enabled: true
|
||||
replicas: 1
|
||||
image:
|
||||
registry: ghcr.io
|
||||
repository: project-hami/hami-dra-monitor
|
||||
tag: "v0.1.0"
|
||||
## Specify a imagePullPolicy
|
||||
## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
|
||||
## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
|
||||
##
|
||||
pullPolicy: Always
|
||||
## Optionally specify an array of imagePullSecrets.
|
||||
## Secrets must be manually created in the namespace.
|
||||
## Example:
|
||||
## pullSecrets:
|
||||
## - myRegistryKeySecretName
|
||||
##
|
||||
pullSecrets: []
|
||||
## @param resources monitor resource requests and limits
|
||||
resources:
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
## Monitor configuration
|
||||
logLevel: 2
|
||||
metricsBindAddress: ":8080"
|
||||
healthProbeBindAddress: ":8000"
|
||||
kubeAPIQPS: 40.0
|
||||
kubeAPIBurst: 60
|
||||
collectInterval: "30s"
|
||||
## Monitor service configuration
|
||||
service:
|
||||
## Service type: ClusterIP, NodePort, or LoadBalancer
|
||||
## Defaults to ClusterIP
|
||||
type: ClusterIP
|
||||
## NodePort configuration (only used when type is NodePort)
|
||||
nodePort:
|
||||
## Metrics port (NodePort). If not specified, Kubernetes will assign a random port
|
||||
metrics: ""
|
||||
|
||||
# Certificate configuration
|
||||
certs:
|
||||
# Custom certificate (used when mode is "custom")
|
||||
custom:
|
||||
crt: ""
|
||||
key: ""
|
||||
# Cert-manager configuration (used when mode is "cert-manager")
|
||||
certManager:
|
||||
enabled: true
|
||||
# Private key rotation policy: "Never" or "Always"
|
||||
# In cert-manager >= v1.18.0, the default changed from "Never" to "Always"
|
||||
# Setting to "Never" avoids frequent certificate rotation for webhooks
|
||||
privateKeyRotationPolicy: "Never"
|
||||
issuer:
|
||||
clusterIssuerName: "" # Required when type is "clusterIssuer"
|
||||
|
||||
# DRA Drivers configuration
|
||||
drivers:
|
||||
nvidia:
|
||||
enabled: true
|
||||
# If you are using gpu driver on host, you need to set this to false
|
||||
containerDriver: true
|
||||
image:
|
||||
registry: ghcr.io
|
||||
repository: projecthami/k8s-dra-driver
|
||||
tag: "v0.0.1-dev"
|
||||
## Specify a imagePullPolicy
|
||||
## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
|
||||
## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
|
||||
##
|
||||
pullPolicy: IfNotPresent
|
||||
## Optionally specify an array of imagePullSecrets.
|
||||
## Secrets must be manually created in the namespace.
|
||||
## Example:
|
||||
## pullSecrets:
|
||||
## - myRegistryKeySecretName
|
||||
##
|
||||
pullSecrets: []
|
||||
3
packages/system/hami/charts/hami/templates/NOTES.txt
Normal file
3
packages/system/hami/charts/hami/templates/NOTES.txt
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
** Please be patient while the chart is being deployed **
|
||||
Resource name: {{ .Values.resourceName }}
|
||||
|
||||
49
packages/system/hami/charts/hami/templates/_commons.tpl
Normal file
49
packages/system/hami/charts/hami/templates/_commons.tpl
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
{{/*
|
||||
Return the proper image name
|
||||
{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" $) }}
|
||||
*/}}
|
||||
{{- define "common.images.image" -}}
|
||||
{{- $registryName := .imageRoot.registry -}}
|
||||
{{- $repositoryName := .imageRoot.repository -}}
|
||||
{{- $tag := .imageRoot.tag | toString -}}
|
||||
{{- if .global }}
|
||||
{{- if .global.imageRegistry }}
|
||||
{{- $registryName = .global.imageRegistry -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- if .tag }}
|
||||
{{- $tag = .tag | toString -}}
|
||||
{{- end -}}
|
||||
{{- if $registryName }}
|
||||
{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}}
|
||||
{{- else -}}
|
||||
{{- printf "%s:%s" $repositoryName $tag -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead)
|
||||
{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }}
|
||||
*/}}
|
||||
{{- define "common.images.pullSecrets" -}}
|
||||
{{- $pullSecrets := list }}
|
||||
|
||||
{{- if .global }}
|
||||
{{- range .global.imagePullSecrets -}}
|
||||
{{- $pullSecrets = append $pullSecrets . -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- range .images -}}
|
||||
{{- range .pullSecrets -}}
|
||||
{{- $pullSecrets = append $pullSecrets . -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- if (not (empty $pullSecrets)) }}
|
||||
imagePullSecrets:
|
||||
{{- range $pullSecrets }}
|
||||
- name: {{ . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
163
packages/system/hami/charts/hami/templates/_helpers.tpl
Normal file
163
packages/system/hami/charts/hami/templates/_helpers.tpl
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
{{/*
|
||||
Expand the name of the chart.
|
||||
*/}}
|
||||
{{- define "hami-vgpu.name" -}}
|
||||
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create a default fully qualified app name.
|
||||
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
|
||||
If release name contains chart name it will be used as a full name.
|
||||
*/}}
|
||||
{{- define "hami-vgpu.fullname" -}}
|
||||
{{- if .Values.fullnameOverride -}}
|
||||
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
|
||||
{{- else }}
|
||||
{{- $name := default .Chart.Name .Values.nameOverride -}}
|
||||
{{- if contains $name .Release.Name }}
|
||||
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
|
||||
{{- else -}}
|
||||
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Allow the release namespace to be overridden for multi-namespace deployments in combined charts
|
||||
*/}}
|
||||
{{- define "hami-vgpu.namespace" -}}
|
||||
{{- if .Values.namespaceOverride -}}
|
||||
{{- .Values.namespaceOverride -}}
|
||||
{{- else -}}
|
||||
{{- .Release.Namespace -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
The app name for Scheduler
|
||||
*/}}
|
||||
{{- define "hami-vgpu.scheduler" -}}
|
||||
{{- printf "%s-scheduler" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
The app name for DevicePlugin
|
||||
*/}}
|
||||
{{- define "hami-vgpu.device-plugin" -}}
|
||||
{{- printf "%s-device-plugin" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
The app name for MockDevicePlugin
|
||||
*/}}
|
||||
{{- define "hami-vgpu.mock-device-plugin" -}}
|
||||
{{- printf "%s-mock-device-plugin" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
The tls secret name for Scheduler
|
||||
*/}}
|
||||
{{- define "hami-vgpu.scheduler.tls" -}}
|
||||
{{- printf "%s-scheduler-tls" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
The webhook name
|
||||
*/}}
|
||||
{{- define "hami-vgpu.scheduler.webhook" -}}
|
||||
{{- printf "%s-webhook" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create chart name and version as used by the chart label.
|
||||
*/}}
|
||||
{{- define "hami-vgpu.chart" -}}
|
||||
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Common labels
|
||||
*/}}
|
||||
{{- define "hami-vgpu.labels" -}}
|
||||
helm.sh/chart: {{ include "hami-vgpu.chart" . }}
|
||||
{{ include "hami-vgpu.selectorLabels" . }}
|
||||
{{- if .Chart.AppVersion }}
|
||||
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
|
||||
{{- end }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Selector labels
|
||||
*/}}
|
||||
{{- define "hami-vgpu.selectorLabels" -}}
|
||||
app.kubernetes.io/name: {{ include "hami-vgpu.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
{{- end }}
|
||||
|
||||
|
||||
{{/*
|
||||
Resolve the tag for kubeScheduler.
|
||||
*/}}
|
||||
{{- define "resolvedKubeSchedulerTag" -}}
|
||||
{{- if .Values.scheduler.kubeScheduler.image.tag }}
|
||||
{{- .Values.scheduler.kubeScheduler.image.tag | trim -}}
|
||||
{{- else }}
|
||||
{{- include "strippedKubeVersion" . | trim -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Return the stripped Kubernetes version string by removing extra parts after semantic version number.
|
||||
v1.31.1+k3s1 -> v1.31.1
|
||||
v1.30.8-eks-2d5f260 -> v1.30.8
|
||||
v1.31.1 -> v1.31.1
|
||||
*/}}
|
||||
{{- define "strippedKubeVersion" -}}
|
||||
{{ regexReplaceAll "^(v[0-9]+\\.[0-9]+\\.[0-9]+)(.*)$" .Capabilities.KubeVersion.Version "$1" }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.scheduler.kubeScheduler.image" -}}
|
||||
{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.kubeScheduler.image "global" .Values.global "tag" (include "resolvedKubeSchedulerTag" .)) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.scheduler.extender.image" -}}
|
||||
{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.extender.image "global" .Values.global "tag" .Values.global.imageTag) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.devicePlugin.image" -}}
|
||||
{{ include "common.images.image" (dict "imageRoot" .Values.devicePlugin.image "global" .Values.global "tag" .Values.global.imageTag) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.mockDevicePlugin.image" -}}
|
||||
{{ include "common.images.image" (dict "imageRoot" .Values.mockDevicePlugin.image "global" .Values.global "tag" .Values.mockDevicePlugin.tag) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.devicePlugin.monitor.image" -}}
|
||||
{{ include "common.images.image" (dict "imageRoot" .Values.devicePlugin.monitor.image "global" .Values.global "tag" .Values.global.imageTag) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.scheduler.patch.image" -}}
|
||||
{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.patch.image "global" .Values.global) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.scheduler.patch.new.image" -}}
|
||||
{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.patch.imageNew "global" .Values.global) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.scheduler.extender.imagePullSecrets" -}}
|
||||
{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.extender.image) "global" .Values.global) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.devicePlugin.imagePullSecrets" -}}
|
||||
{{ include "common.images.pullSecrets" (dict "images" (list .Values.devicePlugin.image) "global" .Values.global) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.scheduler.patch.imagePullSecrets" -}}
|
||||
{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.patch.image) "global" .Values.global) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.scheduler.patch.new.imagePullSecrets" -}}
|
||||
{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.patch.imageNew) "global" .Values.global) }}
|
||||
{{- end -}}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
{{- if and .Values.devicePlugin.enabled (not .Values.devicePlugin.nodeConfiguration.externalConfigName) (not .Values.dra.enabled) -}}
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.device-plugin" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-device-plugin
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
data:
|
||||
config.json: |
|
||||
{{- .Values.devicePlugin.nodeConfiguration.config | nindent 4 }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
{{- if .Values.mockDevicePlugin.enabled }}
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.mock-device-plugin" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/component: hami-mock-device-plugin
|
||||
{{- include "hami-vgpu.selectorLabels" . | nindent 6 }}
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
scheduler.alpha.kubernetes.io/critical-pod: ""
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-mock-device-plugin
|
||||
{{- include "hami-vgpu.selectorLabels" . | nindent 8 }}
|
||||
spec:
|
||||
serviceAccountName: {{ include "hami-vgpu.mock-device-plugin" . }}
|
||||
tolerations:
|
||||
- key: CriticalAddonsOnly
|
||||
operator: Exists
|
||||
containers:
|
||||
- image: {{ include "hami.mockDevicePlugin.image" . }}
|
||||
imagePullPolicy: {{ .Values.mockDevicePlugin.image.pullPolicy }}
|
||||
name: hami-mock-dp-cntr
|
||||
env:
|
||||
- name: NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
command:
|
||||
- ./k8s-device-plugin
|
||||
- -v=5
|
||||
- --device-config-file=/device-config.yaml
|
||||
volumeMounts:
|
||||
- name: dp
|
||||
mountPath: /var/lib/kubelet/device-plugins
|
||||
- name: sys
|
||||
mountPath: /sys
|
||||
- name: device-config
|
||||
mountPath: /device-config.yaml
|
||||
subPath: device-config.yaml
|
||||
volumes:
|
||||
- name: dp
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/device-plugins
|
||||
- name: sys
|
||||
hostPath:
|
||||
path: /sys
|
||||
- name: device-config
|
||||
configMap:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}-device
|
||||
{{- end -}}
|
||||
|
|
@ -0,0 +1,262 @@
|
|||
{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) }}
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.device-plugin" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-device-plugin
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
{{- with .Values.global.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.annotations }}
|
||||
annotations: {{ toYaml .Values.global.annotations | nindent 4}}
|
||||
{{- end }}
|
||||
spec:
|
||||
updateStrategy:
|
||||
{{- with .Values.devicePlugin.updateStrategy }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/component: hami-device-plugin
|
||||
{{- include "hami-vgpu.selectorLabels" . | nindent 6 }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-device-plugin
|
||||
hami.io/webhook: ignore
|
||||
{{- include "hami-vgpu.selectorLabels" . | nindent 8 }}
|
||||
annotations:
|
||||
{{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
|
||||
checksum/hami-scheduler-newversion-config: {{ include (print $.Template.BasePath "/scheduler/configmapnew.yaml") . | sha256sum }}
|
||||
{{- else }}
|
||||
checksum/hami-scheduler-config: {{ include (print $.Template.BasePath "/scheduler/configmap.yaml") . | sha256sum }}
|
||||
{{- end }}
|
||||
checksum/hami-scheduler-device-config: {{ include (print $.Template.BasePath "/scheduler/device-configmap.yaml") . | sha256sum }}
|
||||
{{- if .Values.devicePlugin.podAnnotations }}
|
||||
{{- toYaml .Values.devicePlugin.podAnnotations | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if .Values.devicePlugin.runtimeClassName }}
|
||||
runtimeClassName: {{ .Values.devicePlugin.runtimeClassName }}
|
||||
{{- end }}
|
||||
serviceAccountName: {{ include "hami-vgpu.device-plugin" . }}
|
||||
priorityClassName: system-node-critical
|
||||
hostPID: true
|
||||
hostNetwork: true
|
||||
{{- include "hami.devicePlugin.imagePullSecrets" . | nindent 6 }}
|
||||
{{- if .Values.devicePlugin.gpuOperatorToolkitReady.enabled }}
|
||||
initContainers:
|
||||
- name: toolkit-validation
|
||||
image: {{ include "hami.devicePlugin.image" . }}
|
||||
imagePullPolicy: {{ .Values.devicePlugin.image.pullPolicy }}
|
||||
securityContext:
|
||||
privileged: true
|
||||
runAsUser: 0
|
||||
command: ["sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
echo "Waiting for NVIDIA Toolkit to be ready..."
|
||||
until [ -f {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }}/toolkit-ready ]; do
|
||||
echo "Waiting for {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }}/toolkit-ready..."
|
||||
sleep 5
|
||||
done
|
||||
echo "NVIDIA Toolkit is ready!"
|
||||
volumeMounts:
|
||||
- name: nvidia-validations
|
||||
mountPath: {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath | quote }}
|
||||
mountPropagation: HostToContainer
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: device-plugin
|
||||
image: {{ include "hami.devicePlugin.image" . }}
|
||||
imagePullPolicy: {{ .Values.devicePlugin.image.pullPolicy }}
|
||||
lifecycle:
|
||||
postStart:
|
||||
exec:
|
||||
command: ["/bin/sh","-c", {{ printf "/k8s-vgpu/bin/vgpu-init.sh %s/vgpu/" .Values.global.gpuHookPath | quote }}]
|
||||
command:
|
||||
- nvidia-device-plugin
|
||||
- --config-file=/device-config.yaml
|
||||
- --mig-strategy={{ .Values.devicePlugin.migStrategy }}
|
||||
- --disable-core-limit={{ .Values.devicePlugin.disablecorelimit }}
|
||||
{{- range .Values.devicePlugin.extraArgs }}
|
||||
- {{ . }}
|
||||
{{- end }}
|
||||
env:
|
||||
- name: NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
- name: NVIDIA_MIG_MONITOR_DEVICES
|
||||
value: all
|
||||
- name: DEVICE_LIST_STRATEGY
|
||||
value: {{ .Values.devicePlugin.deviceListStrategy }}
|
||||
- name: HOOK_PATH
|
||||
value: {{ .Values.global.gpuHookPath }}
|
||||
{{- if typeIs "bool" .Values.devicePlugin.passDeviceSpecsEnabled }}
|
||||
- name: PASS_DEVICE_SPECS
|
||||
value: {{ .Values.devicePlugin.passDeviceSpecsEnabled | quote }}
|
||||
{{- end }}
|
||||
{{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }}
|
||||
- name: NVIDIA_DRIVER_ROOT
|
||||
value: {{ .Values.devicePlugin.nvidiaDriverRoot }}
|
||||
{{- end }}
|
||||
{{- if typeIs "string" .Values.devicePlugin.nvidiaHookPath }}
|
||||
- name: NVIDIA_CDI_HOOK_PATH
|
||||
value: {{ .Values.devicePlugin.nvidiaHookPath }}
|
||||
{{- end }}
|
||||
{{- if typeIs "bool" .Values.devicePlugin.gdrcopyEnabled }}
|
||||
- name: GDRCOPY_ENABLED
|
||||
value: {{ .Values.devicePlugin.gdrcopyEnabled | quote }}
|
||||
{{- end }}
|
||||
{{- if typeIs "bool" .Values.devicePlugin.gdsEnabled }}
|
||||
- name: GDS_ENABLED
|
||||
value: {{ .Values.devicePlugin.gdsEnabled | quote }}
|
||||
{{- end }}
|
||||
{{- if typeIs "bool" .Values.devicePlugin.mofedEnabled }}
|
||||
- name: MOFED_ENABLED
|
||||
value: {{ .Values.devicePlugin.mofedEnabled | quote }}
|
||||
{{- end }}
|
||||
{{- if eq (.Values.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy | default "spread") "topology-aware" }}
|
||||
- name: ENABLE_TOPOLOGY_SCORE
|
||||
value: "true"
|
||||
{{- end }}
|
||||
{{- with .Values.devicePlugin.extraEnvs }}
|
||||
{{- . | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
securityContext:
|
||||
privileged: true
|
||||
allowPrivilegeEscalation: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
add: ["SYS_ADMIN"]
|
||||
resources:
|
||||
{{- toYaml .Values.devicePlugin.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
- name: device-plugin
|
||||
mountPath: /var/lib/kubelet/device-plugins
|
||||
- name: lib
|
||||
mountPath: {{ printf "%s%s" .Values.global.gpuHookPath "/vgpu" }}
|
||||
- name: usrbin
|
||||
mountPath: /usrbin
|
||||
- name: deviceconfig
|
||||
mountPath: /config
|
||||
- name: hosttmp
|
||||
mountPath: /tmp
|
||||
- name: device-config
|
||||
mountPath: /device-config.yaml
|
||||
subPath: device-config.yaml
|
||||
- name: cdi-root
|
||||
mountPath: /var/run/cdi
|
||||
{{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }}
|
||||
# We always mount the driver root at /driver-root in the container.
|
||||
# This is required for CDI detection to work correctly.
|
||||
- name: driver-root
|
||||
mountPath: /driver-root
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
- name: vgpu-monitor
|
||||
image: {{ include "hami.devicePlugin.monitor.image" . }}
|
||||
imagePullPolicy: {{ .Values.devicePlugin.monitor.image.pullPolicy }}
|
||||
command:
|
||||
- "vGPUmonitor"
|
||||
{{- range .Values.devicePlugin.monitor.extraArgs }}
|
||||
- {{ . }}
|
||||
{{- end }}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
add: ["SYS_ADMIN"]
|
||||
env:
|
||||
- name: NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
- name: NVIDIA_VISIBLE_DEVICES
|
||||
value: "all"
|
||||
- name: NVIDIA_MIG_MONITOR_DEVICES
|
||||
value: "all"
|
||||
- name: HOOK_PATH
|
||||
value: "{{ .Values.global.gpuHookPath }}/vgpu"
|
||||
- name: HAMI_RESYNC_INTERVAL
|
||||
value: {{ .Values.devicePlugin.monitor.resyncInterval | default "5m" | quote }}
|
||||
{{- with .Values.devicePlugin.monitor.extraEnvs }}
|
||||
{{- . | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
resources:
|
||||
{{- toYaml .Values.devicePlugin.monitor.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
- name: ctrs
|
||||
mountPath: {{ .Values.devicePlugin.monitor.ctrPath }}
|
||||
- name: dockers
|
||||
mountPath: /run/docker
|
||||
- name: containerds
|
||||
mountPath: /run/containerd
|
||||
- name: sysinfo
|
||||
mountPath: /sysinfo
|
||||
- name: hostvar
|
||||
mountPath: /hostvar
|
||||
- name: hosttmp
|
||||
mountPath: /tmp
|
||||
volumes:
|
||||
- name: ctrs
|
||||
hostPath:
|
||||
path: {{ .Values.devicePlugin.monitor.ctrPath }}
|
||||
- name: hosttmp
|
||||
hostPath:
|
||||
path: /tmp
|
||||
- name: dockers
|
||||
hostPath:
|
||||
path: /run/docker
|
||||
- name: containerds
|
||||
hostPath:
|
||||
path: /run/containerd
|
||||
- name: device-plugin
|
||||
hostPath:
|
||||
path: {{ .Values.devicePlugin.pluginPath }}
|
||||
- name: lib
|
||||
hostPath:
|
||||
path: {{ .Values.devicePlugin.libPath }}
|
||||
{{- if .Values.devicePlugin.gpuOperatorToolkitReady.enabled }}
|
||||
- name: nvidia-validations
|
||||
hostPath:
|
||||
path: {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }}
|
||||
type: DirectoryOrCreate
|
||||
{{- end }}
|
||||
{{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }}
|
||||
- name: driver-root
|
||||
hostPath:
|
||||
path: {{ .Values.devicePlugin.nvidiaDriverRoot }}
|
||||
type: Directory
|
||||
{{- end }}
|
||||
- name: cdi-root
|
||||
hostPath:
|
||||
path: /var/run/cdi
|
||||
type: DirectoryOrCreate
|
||||
- name: usrbin
|
||||
hostPath:
|
||||
path: /usr/bin
|
||||
- name: sysinfo
|
||||
hostPath:
|
||||
path: /sys
|
||||
- name: hostvar
|
||||
hostPath:
|
||||
path: /var
|
||||
- name: deviceconfig
|
||||
configMap:
|
||||
name: {{ .Values.devicePlugin.nodeConfiguration.externalConfigName | default (include "hami-vgpu.device-plugin" .) }}
|
||||
- name: device-config
|
||||
configMap:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}-device
|
||||
{{- if .Values.devicePlugin.nvidiaNodeSelector }}
|
||||
nodeSelector: {{ toYaml .Values.devicePlugin.nvidiaNodeSelector | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.devicePlugin.tolerations }}
|
||||
tolerations: {{ toYaml .Values.devicePlugin.tolerations | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.device-plugin" . }}-monitor
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- pods
|
||||
verbs:
|
||||
- get
|
||||
- create
|
||||
- watch
|
||||
- list
|
||||
- update
|
||||
- patch
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- nodes
|
||||
verbs:
|
||||
- get
|
||||
- update
|
||||
- list
|
||||
- patch
|
||||
{{- end -}}
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.device-plugin" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: "hami-device-plugin"
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: {{ include "hami-vgpu.device-plugin" . }}-monitor
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "hami-vgpu.device-plugin" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
{{- end -}}
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}}
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.device-plugin" . }}-monitor
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-device-plugin
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
{{- if .Values.devicePlugin.service.labels }} # Use devicePlugin instead of scheduler
|
||||
{{ toYaml .Values.devicePlugin.service.labels | indent 4 }}
|
||||
{{- end }}
|
||||
{{- if .Values.devicePlugin.service.annotations }} # Use devicePlugin instead of scheduler
|
||||
annotations: {{ toYaml .Values.devicePlugin.service.annotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
type: {{ .Values.devicePlugin.service.type | default "NodePort" }} # Default type is NodePort
|
||||
ports:
|
||||
- name: monitorport
|
||||
port: {{ .Values.devicePlugin.service.httpPort | default 31992 }} # Default HTTP port is 31992
|
||||
targetPort: 9394
|
||||
{{- if eq (.Values.devicePlugin.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort
|
||||
nodePort: {{ .Values.devicePlugin.service.httpPort | default 31992 }}
|
||||
{{- end }}
|
||||
protocol: TCP
|
||||
selector:
|
||||
app.kubernetes.io/component: hami-device-plugin
|
||||
{{- include "hami-vgpu.selectorLabels" . | nindent 4 }}
|
||||
{{- end -}}
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.device-plugin" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: "hami-device-plugin"
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
{{- end -}}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}}
|
||||
{{- if and .Values.devicePlugin.createRuntimeClass .Values.devicePlugin.runtimeClassName }}
|
||||
apiVersion: node.k8s.io/v1
|
||||
kind: RuntimeClass
|
||||
metadata:
|
||||
name: {{ .Values.devicePlugin.runtimeClassName }}
|
||||
annotations:
|
||||
helm.sh/hook: pre-install,pre-upgrade
|
||||
handler: nvidia
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}}
|
||||
{{- if .Values.scheduler.certManager.enabled }}
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}-serving-cert
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-scheduler
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
spec:
|
||||
dnsNames:
|
||||
- {{ include "hami-vgpu.scheduler" . }}.{{ include "hami-vgpu.namespace" . }}.svc
|
||||
- {{ include "hami-vgpu.scheduler" . }}.{{ include "hami-vgpu.namespace" . }}.svc.cluster.local
|
||||
issuerRef:
|
||||
kind: Issuer
|
||||
name: {{ include "hami-vgpu.scheduler" . }}-selfsigned-issuer
|
||||
secretName: {{ include "hami-vgpu.scheduler.tls" . }}
|
||||
---
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Issuer
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}-selfsigned-issuer
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-scheduler
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
spec:
|
||||
selfSigned: {}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
{{- if not .Values.dra.enabled -}}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: "hami-scheduler"
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["pods", "configmaps"]
|
||||
verbs: ["get", "list", "watch", "patch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["pods/binding"]
|
||||
verbs: ["create"]
|
||||
- apiGroups: [""]
|
||||
resources: ["nodes"]
|
||||
verbs: ["get", "list", "patch", "watch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["events"]
|
||||
verbs: ["create", "get", "list"]
|
||||
- apiGroups: [""]
|
||||
resources: ["resourcequotas"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
---
|
||||
{{- if .Values.mockDevicePlugin.enabled }}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.mock-device-plugin" . }}
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["nodes"]
|
||||
verbs: ["get", "update", "list", "patch"]
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
{{- if not .Values.dra.enabled -}}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}-kube
|
||||
labels:
|
||||
app.kubernetes.io/component: "hami-scheduler"
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: system:kube-scheduler
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}-volume
|
||||
labels:
|
||||
app.kubernetes.io/component: "hami-scheduler"
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: system:volume-scheduler
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: "hami-scheduler"
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,142 @@
|
|||
{{- if and .Values.scheduler.kubeScheduler.enabled (not .Values.dra.enabled) -}}
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-scheduler
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
data:
|
||||
config.json: |
|
||||
{
|
||||
"kind": "Policy",
|
||||
"apiVersion": "v1",
|
||||
"extenders": [
|
||||
{
|
||||
{{- if .Values.scheduler.admissionWebhook.enabled }}
|
||||
"urlPrefix": "https://127.0.0.1:443",
|
||||
"enableHttps": true,
|
||||
"tlsConfig": {
|
||||
"insecure": true
|
||||
},
|
||||
{{- else }}
|
||||
"urlPrefix": "http://127.0.0.1:80",
|
||||
"enableHttps": false,
|
||||
{{- end }}
|
||||
"filterVerb": "filter",
|
||||
"bindVerb": "bind",
|
||||
"weight": 1,
|
||||
"nodeCacheCapable": true,
|
||||
"httpTimeout": 30000000000,
|
||||
"managedResources": [
|
||||
{{- range .Values.devices.amd.customresources }}
|
||||
{
|
||||
"name": "{{ . }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{{- end }}
|
||||
{{- if .Values.devices.ascend.enabled }}
|
||||
{{- range .Values.devices.ascend.customresources }}
|
||||
{
|
||||
"name": "{{ . }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.devices.mthreads.enabled }}
|
||||
{{- range .Values.devices.mthreads.customresources }}
|
||||
{
|
||||
"name": "{{ . }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.devices.enflame.enabled }}
|
||||
{{- range .Values.devices.enflame.customresources }}
|
||||
{
|
||||
"name": "{{ . }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.devices.kunlun.enabled }}
|
||||
{{- range .Values.devices.kunlun.customresources }}
|
||||
{
|
||||
"name": "{{ . }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- range .Values.devices.awsneuron.customresources }}
|
||||
{
|
||||
"name": "{{ . }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{{- end }}
|
||||
{{- if .Values.devices.iluvatar.enabled }}
|
||||
{{- range .Values.devices.iluvatar.customresources }}
|
||||
{
|
||||
"name": "{{ . }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{
|
||||
"name": "{{ .Values.resourceName }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "{{ .Values.resourceMem }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "{{ .Values.resourceCores }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "{{ .Values.resourceMemPercentage }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "{{ .Values.resourcePriority }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "{{ .Values.mluResourceName }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "{{ .Values.dcuResourceName }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "{{ .Values.dcuResourceMem }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "{{ .Values.dcuResourceCores }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "metax-tech.com/gpu",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "{{ .Values.metaxResourceName }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "{{ .Values.metaxResourceCore }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "{{ .Values.metaxResourceMem }}",
|
||||
"ignoredByScheduler": true
|
||||
}
|
||||
],
|
||||
"ignoreable": false
|
||||
}
|
||||
]
|
||||
}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,102 @@
|
|||
{{- if and .Values.scheduler.kubeScheduler.enabled (not .Values.dra.enabled) -}}
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}-newversion
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-scheduler
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
data:
|
||||
config.yaml: |
|
||||
{{- if gt (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 25}}
|
||||
apiVersion: kubescheduler.config.k8s.io/v1
|
||||
{{- else }}
|
||||
apiVersion: kubescheduler.config.k8s.io/v1beta2
|
||||
{{- end }}
|
||||
kind: KubeSchedulerConfiguration
|
||||
leaderElection:
|
||||
leaderElect: false
|
||||
profiles:
|
||||
- schedulerName: {{ .Values.schedulerName }}
|
||||
extenders:
|
||||
{{- if .Values.scheduler.admissionWebhook.enabled }}
|
||||
- urlPrefix: "https://127.0.0.1:443"
|
||||
enableHTTPS: true
|
||||
tlsConfig:
|
||||
insecure: true
|
||||
{{- else }}
|
||||
- urlPrefix: "http://127.0.0.1:80"
|
||||
enableHTTPS: false
|
||||
{{- end }}
|
||||
filterVerb: filter
|
||||
bindVerb: bind
|
||||
nodeCacheCapable: true
|
||||
weight: 1
|
||||
httpTimeout: 30s
|
||||
managedResources:
|
||||
- name: {{ .Values.resourceName }}
|
||||
ignoredByScheduler: true
|
||||
- name: {{ .Values.resourceMem }}
|
||||
ignoredByScheduler: true
|
||||
- name: {{ .Values.resourceCores }}
|
||||
ignoredByScheduler: true
|
||||
- name: {{ .Values.resourceMemPercentage }}
|
||||
ignoredByScheduler: true
|
||||
- name: {{ .Values.resourcePriority }}
|
||||
ignoredByScheduler: true
|
||||
- name: {{ .Values.mluResourceName }}
|
||||
ignoredByScheduler: true
|
||||
- name: {{ .Values.dcuResourceName }}
|
||||
ignoredByScheduler: true
|
||||
- name: {{ .Values.dcuResourceMem }}
|
||||
ignoredByScheduler: true
|
||||
- name: {{ .Values.dcuResourceCores }}
|
||||
ignoredByScheduler: true
|
||||
- name: "metax-tech.com/gpu"
|
||||
ignoredByScheduler: true
|
||||
- name: {{ .Values.metaxResourceName }}
|
||||
ignoredByScheduler: true
|
||||
- name: {{ .Values.metaxResourceCore }}
|
||||
ignoredByScheduler: true
|
||||
- name: {{ .Values.metaxResourceMem }}
|
||||
ignoredByScheduler: true
|
||||
{{- if .Values.devices.ascend.enabled }}
|
||||
{{- range .Values.devices.ascend.customresources }}
|
||||
- name: {{ . }}
|
||||
ignoredByScheduler: true
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.devices.mthreads.enabled }}
|
||||
{{- range .Values.devices.mthreads.customresources }}
|
||||
- name: {{ . }}
|
||||
ignoredByScheduler: true
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.devices.enflame.enabled }}
|
||||
{{- range .Values.devices.enflame.customresources }}
|
||||
- name: {{ . }}
|
||||
ignoredByScheduler: true
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.devices.kunlun.enabled }}
|
||||
{{- range .Values.devices.kunlun.customresources }}
|
||||
- name: {{ . }}
|
||||
ignoredByScheduler: true
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- range .Values.devices.awsneuron.customresources }}
|
||||
- name: {{ . }}
|
||||
ignoredByScheduler: true
|
||||
{{- end }}
|
||||
{{- if .Values.devices.iluvatar.enabled }}
|
||||
{{- range .Values.devices.iluvatar.customresources }}
|
||||
- name: {{ . }}
|
||||
ignoredByScheduler: true
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- range .Values.devices.amd.customresources }}
|
||||
- name: {{ . }}
|
||||
ignoredByScheduler: true
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,227 @@
|
|||
{{- if not .Values.dra.enabled -}}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-scheduler
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
{{- with .Values.global.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.annotations }}
|
||||
annotations: {{ toYaml .Values.global.annotations | nindent 4}}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if .Values.scheduler.leaderElect }}
|
||||
replicas: {{ .Values.scheduler.replicas }}
|
||||
{{- else }}
|
||||
replicas: 1
|
||||
{{- end }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/component: hami-scheduler
|
||||
{{- include "hami-vgpu.selectorLabels" . | nindent 6 }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-scheduler
|
||||
{{- include "hami-vgpu.selectorLabels" . | nindent 8 }}
|
||||
hami.io/webhook: ignore
|
||||
annotations:
|
||||
{{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
|
||||
checksum/hami-scheduler-newversion-config: {{ include (print $.Template.BasePath "/scheduler/configmapnew.yaml") . | sha256sum }}
|
||||
{{- else }}
|
||||
checksum/hami-scheduler-config: {{ include (print $.Template.BasePath "/scheduler/configmap.yaml") . | sha256sum }}
|
||||
{{- end }}
|
||||
checksum/hami-scheduler-device-config: {{ include (print $.Template.BasePath "/scheduler/device-configmap.yaml") . | sha256sum }}
|
||||
{{- if .Values.scheduler.podAnnotations }}
|
||||
{{- toYaml .Values.scheduler.podAnnotations | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
serviceAccountName: {{ include "hami-vgpu.scheduler" . }}
|
||||
priorityClassName: system-node-critical
|
||||
{{- include "hami.scheduler.extender.imagePullSecrets" . | nindent 6 }}
|
||||
containers:
|
||||
{{- if .Values.scheduler.kubeScheduler.enabled }}
|
||||
- name: kube-scheduler
|
||||
image: {{ include "hami.scheduler.kubeScheduler.image" . }}
|
||||
imagePullPolicy: {{ .Values.scheduler.kubeScheduler.image.pullPolicy }}
|
||||
command:
|
||||
- kube-scheduler
|
||||
{{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
|
||||
{{- range .Values.scheduler.kubeScheduler.extraNewArgs }}
|
||||
- {{ . }}
|
||||
{{- end }}
|
||||
{{- else }}
|
||||
- --scheduler-name={{ .Values.schedulerName }}
|
||||
{{- range .Values.scheduler.kubeScheduler.extraArgs }}
|
||||
- {{ . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
- --leader-elect={{ .Values.scheduler.leaderElect }}
|
||||
- --leader-elect-resource-name={{ .Values.schedulerName }}
|
||||
- --leader-elect-resource-namespace={{ include "hami-vgpu.namespace" . }}
|
||||
resources:
|
||||
{{- toYaml .Values.scheduler.kubeScheduler.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
- name: scheduler-config
|
||||
mountPath: /config
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.livenessProbe }}
|
||||
livenessProbe:
|
||||
failureThreshold: 8
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: 10259
|
||||
scheme: HTTPS
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
successThreshold: 1
|
||||
timeoutSeconds: 15
|
||||
{{- end }}
|
||||
- name: vgpu-scheduler-extender
|
||||
image: {{ include "hami.scheduler.extender.image" . }}
|
||||
imagePullPolicy: {{ .Values.scheduler.extender.image.pullPolicy }}
|
||||
env:
|
||||
{{- if .Values.scheduler.nodeLockExpire }}
|
||||
- name: HAMI_NODELOCK_EXPIRE
|
||||
value: "{{ .Values.scheduler.nodeLockExpire }}"
|
||||
{{- end }}
|
||||
{{- if .Values.global.managedNodeSelectorEnable }}
|
||||
{{- range $key, $value := .Values.global.managedNodeSelector }}
|
||||
- name: NODE_SELECTOR_{{ $key | upper | replace "-" "_" }}
|
||||
value: "{{ $value }}"
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
command:
|
||||
- scheduler
|
||||
{{- if .Values.scheduler.admissionWebhook.enabled }}
|
||||
- --http_bind=0.0.0.0:443
|
||||
- --cert_file=/tls/tls.crt
|
||||
- --key_file=/tls/tls.key
|
||||
{{- else }}
|
||||
- --http_bind=0.0.0.0:80
|
||||
{{- end }}
|
||||
- --scheduler-name={{ .Values.schedulerName }}
|
||||
- --metrics-bind-address={{ .Values.scheduler.metricsBindAddress }}
|
||||
- --node-scheduler-policy={{ .Values.scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy }}
|
||||
- --gpu-scheduler-policy={{ .Values.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy }}
|
||||
- --force-overwrite-default-scheduler={{ .Values.scheduler.forceOverwriteDefaultScheduler}}
|
||||
- --device-config-file=/device-config.yaml
|
||||
- --leader-elect={{ .Values.scheduler.leaderElect }}
|
||||
- --leader-elect-resource-name={{ .Values.schedulerName }}
|
||||
- --leader-elect-resource-namespace={{ include "hami-vgpu.namespace" . }}
|
||||
{{- if .Values.devices.ascend.enabled }}
|
||||
- --enable-ascend=true
|
||||
{{- end }}
|
||||
{{- if .Values.devices.iluvatar.enabled }}
|
||||
- --enable-iluvatar=true
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.nodeLabelSelector }}
|
||||
- --node-label-selector={{- $first := true -}}
|
||||
{{- range $key, $value := .Values.scheduler.nodeLabelSelector -}}
|
||||
{{- if not $first }},{{ end -}}
|
||||
{{- $key }}={{ $value -}}
|
||||
{{- $first = false -}}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- range .Values.scheduler.extender.extraArgs }}
|
||||
- {{ . }}
|
||||
{{- end }}
|
||||
ports:
|
||||
{{- if .Values.scheduler.admissionWebhook.enabled }}
|
||||
- name: https
|
||||
containerPort: 443
|
||||
protocol: TCP
|
||||
{{- else }}
|
||||
- name: http
|
||||
containerPort: 80
|
||||
protocol: TCP
|
||||
{{- end }}
|
||||
- name: metrics
|
||||
containerPort: {{ last (splitList ":" .Values.scheduler.metricsBindAddress) | int }}
|
||||
protocol: TCP
|
||||
resources:
|
||||
{{- toYaml .Values.scheduler.extender.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
- name: device-config
|
||||
mountPath: /device-config.yaml
|
||||
subPath: device-config.yaml
|
||||
{{- if .Values.scheduler.admissionWebhook.enabled }}
|
||||
- name: tls-config
|
||||
mountPath: /tls
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.livenessProbe }}
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
{{- if .Values.scheduler.admissionWebhook.enabled }}
|
||||
port: https
|
||||
scheme: HTTPS
|
||||
{{- else }}
|
||||
port: http
|
||||
scheme: HTTP
|
||||
{{- end }}
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
failureThreshold: 3
|
||||
timeoutSeconds: 5
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.leaderElect }}
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /readyz
|
||||
{{- if .Values.scheduler.admissionWebhook.enabled }}
|
||||
port: https
|
||||
scheme: HTTPS
|
||||
{{- else }}
|
||||
port: http
|
||||
scheme: HTTP
|
||||
{{- end }}
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
failureThreshold: 3
|
||||
timeoutSeconds: 5
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.leaderElect }}
|
||||
affinity:
|
||||
podAntiAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
- labelSelector:
|
||||
matchExpressions:
|
||||
- key: app.kubernetes.io/component
|
||||
operator: In
|
||||
values:
|
||||
- hami-scheduler
|
||||
topologyKey: "kubernetes.io/hostname"
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- if .Values.scheduler.admissionWebhook.enabled }}
|
||||
- name: tls-config
|
||||
secret:
|
||||
secretName: {{ template "hami-vgpu.scheduler.tls" . }}
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.kubeScheduler.enabled }}
|
||||
- name: scheduler-config
|
||||
configMap:
|
||||
{{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
|
||||
name: {{ template "hami-vgpu.scheduler" . }}-newversion
|
||||
{{- else }}
|
||||
name: {{ template "hami-vgpu.scheduler" . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
- name: device-config
|
||||
configMap:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}-device
|
||||
{{- if .Values.scheduler.nodeSelector }}
|
||||
nodeSelector: {{ toYaml .Values.scheduler.nodeSelector | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.tolerations }}
|
||||
tolerations: {{ toYaml .Values.scheduler.tolerations | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.nodeName }}
|
||||
nodeName: {{ .Values.scheduler.nodeName }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,410 @@
|
|||
{{- if not .Values.dra.enabled -}}
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}-device
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-scheduler
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
data:
|
||||
device-config.yaml: |-
|
||||
{{- if .Files.Glob "files/device-config.yaml" }}
|
||||
{{- .Files.Get "files/device-config.yaml" | nindent 4}}
|
||||
{{- else }}
|
||||
nvidia:
|
||||
resourceCountName: {{ .Values.resourceName }}
|
||||
resourceMemoryName: {{ .Values.resourceMem }}
|
||||
resourceMemoryPercentageName: {{ .Values.resourceMemPercentage }}
|
||||
resourceCoreName: {{ .Values.resourceCores }}
|
||||
resourcePriorityName: {{ .Values.resourcePriority }}
|
||||
overwriteEnv: false
|
||||
defaultMemory: 0
|
||||
defaultCores: 0
|
||||
defaultGPUNum: 1
|
||||
memoryFactor: 1
|
||||
deviceSplitCount: {{ .Values.devicePlugin.deviceSplitCount }}
|
||||
deviceMemoryScaling: {{ .Values.devicePlugin.deviceMemoryScaling }}
|
||||
deviceCoreScaling: {{ .Values.devicePlugin.deviceCoreScaling }}
|
||||
gpuCorePolicy: {{ .Values.devices.nvidia.gpuCorePolicy }}
|
||||
libCudaLogLevel: {{ .Values.devices.nvidia.libCudaLogLevel }}
|
||||
runtimeClassName: "{{ .Values.devicePlugin.runtimeClassName }}"
|
||||
knownMigGeometries:
|
||||
- models: [ "A30" ]
|
||||
allowedGeometries:
|
||||
-
|
||||
- name: 1g.6gb
|
||||
core: 25
|
||||
memory: 6144
|
||||
count: 4
|
||||
-
|
||||
- name: 2g.12gb
|
||||
core: 50
|
||||
memory: 12288
|
||||
count: 2
|
||||
-
|
||||
- name: 4g.24gb
|
||||
core: 100
|
||||
memory: 24576
|
||||
count: 1
|
||||
- models: [ "A100-SXM4-40GB", "A100-40GB-PCIe", "A100-PCIE-40GB"]
|
||||
allowedGeometries:
|
||||
-
|
||||
- name: 1g.5gb
|
||||
core: 14
|
||||
memory: 5120
|
||||
count: 7
|
||||
-
|
||||
- name: 1g.5gb
|
||||
core: 14
|
||||
memory: 5120
|
||||
count: 1
|
||||
- name: 2g.10gb
|
||||
core: 28
|
||||
memory: 10240
|
||||
count: 3
|
||||
-
|
||||
- name: 3g.20gb
|
||||
core: 42
|
||||
memory: 20480
|
||||
count: 2
|
||||
-
|
||||
- name: 7g.40gb
|
||||
core: 100
|
||||
memory: 40960
|
||||
count: 1
|
||||
- models: [ "A100-SXM4-80GB", "A100-80GB-PCIe", "A100-PCIE-80GB"]
|
||||
allowedGeometries:
|
||||
-
|
||||
- name: 1g.10gb
|
||||
core: 14
|
||||
memory: 10240
|
||||
count: 7
|
||||
-
|
||||
- name: 1g.10gb
|
||||
core: 14
|
||||
memory: 10240
|
||||
count: 1
|
||||
- name: 2g.20gb
|
||||
core: 28
|
||||
memory: 20480
|
||||
count: 3
|
||||
-
|
||||
- name: 3g.40gb
|
||||
core: 42
|
||||
memory: 40960
|
||||
count: 2
|
||||
-
|
||||
- name: 7g.79gb
|
||||
core: 100
|
||||
memory: 80896
|
||||
count: 1
|
||||
- models: [ "H100-PCIE-80GB", "H100-SXM5-80GB"]
|
||||
allowedGeometries:
|
||||
-
|
||||
- name: 1g.10gb
|
||||
core: 14
|
||||
memory: 10240
|
||||
count: 7
|
||||
-
|
||||
- name: 1g.10gb
|
||||
core: 14
|
||||
memory: 10240
|
||||
count: 1
|
||||
- name: 2g.20gb
|
||||
core: 28
|
||||
memory: 20480
|
||||
count: 3
|
||||
-
|
||||
- name: 3g.40gb
|
||||
core: 42
|
||||
memory: 40960
|
||||
count: 2
|
||||
-
|
||||
- name: 7g.80gb
|
||||
core: 100
|
||||
memory: 81920
|
||||
count: 1
|
||||
- models: [ "H100-PCIE-94GB", "H100-SXM5-94GB"]
|
||||
allowedGeometries:
|
||||
-
|
||||
- name: 1g.12gb
|
||||
core: 14
|
||||
memory: 12288
|
||||
count: 7
|
||||
-
|
||||
- name: 1g.12gb
|
||||
core: 14
|
||||
memory: 12288
|
||||
count: 1
|
||||
- name: 2g.24gb
|
||||
core: 28
|
||||
memory: 24576
|
||||
count: 3
|
||||
-
|
||||
- name: 3g.47gb
|
||||
core: 42
|
||||
memory: 48128
|
||||
count: 2
|
||||
-
|
||||
- name: 7g.94gb
|
||||
core: 100
|
||||
memory: 96256
|
||||
count: 1
|
||||
- models: [ "H20", "H100 on GH200"]
|
||||
allowedGeometries:
|
||||
-
|
||||
- name: 1g.12gb
|
||||
core: 14
|
||||
memory: 12288
|
||||
count: 7
|
||||
-
|
||||
- name: 1g.12gb
|
||||
core: 14
|
||||
memory: 12288
|
||||
count: 1
|
||||
- name: 2g.24gb
|
||||
core: 28
|
||||
memory: 24576
|
||||
count: 3
|
||||
-
|
||||
- name: 3g.48gb
|
||||
core: 42
|
||||
memory: 49152
|
||||
count: 2
|
||||
-
|
||||
- name: 7g.96gb
|
||||
core: 100
|
||||
memory: 98304
|
||||
count: 1
|
||||
- models: [ "H200 NVL", "H200-SXM5"]
|
||||
allowedGeometries:
|
||||
-
|
||||
- name: 1g.18gb
|
||||
core: 14
|
||||
memory: 18432
|
||||
count: 7
|
||||
-
|
||||
- name: 1g.18gb
|
||||
core: 14
|
||||
memory: 18432
|
||||
count: 1
|
||||
- name: 2g.35gb
|
||||
core: 28
|
||||
memory: 35840
|
||||
count: 3
|
||||
-
|
||||
- name: 3g.71gb
|
||||
core: 42
|
||||
memory: 72704
|
||||
count: 2
|
||||
-
|
||||
- name: 7g.141gb
|
||||
core: 100
|
||||
memory: 144384
|
||||
count: 1
|
||||
- models: [ "B200" ]
|
||||
allowedGeometries:
|
||||
-
|
||||
- name: 1g.23gb
|
||||
core: 14
|
||||
memory: 23552
|
||||
count: 7
|
||||
-
|
||||
- name: 1g.23gb
|
||||
core: 14
|
||||
memory: 23552
|
||||
count: 1
|
||||
- name: 2g.45gb
|
||||
core: 28
|
||||
memory: 46080
|
||||
count: 3
|
||||
-
|
||||
- name: 3g.90gb
|
||||
core: 42
|
||||
memory: 92160
|
||||
count: 2
|
||||
-
|
||||
- name: 7g.180gb
|
||||
core: 100
|
||||
memory: 184320
|
||||
count: 1
|
||||
cambricon:
|
||||
resourceCountName: {{ .Values.mluResourceName }}
|
||||
resourceMemoryName: {{ .Values.mluResourceMem }}
|
||||
resourceCoreName: {{ .Values.mluResourceCores }}
|
||||
hygon:
|
||||
resourceCountName: {{ .Values.dcuResourceName }}
|
||||
resourceMemoryName: {{ .Values.dcuResourceMem }}
|
||||
resourceCoreName: {{ .Values.dcuResourceCores }}
|
||||
memoryFactor: 1
|
||||
metax:
|
||||
resourceCountName: "metax-tech.com/gpu"
|
||||
resourceVCountName: {{ .Values.metaxResourceName }}
|
||||
resourceVMemoryName: {{ .Values.metaxResourceMem }}
|
||||
resourceVCoreName: {{ .Values.metaxResourceCore }}
|
||||
sgpuTopologyAware: {{ .Values.metaxsGPUTopologyAware }}
|
||||
enflame:
|
||||
resourceNameGCU: "enflame.com/gcu"
|
||||
resourceNameVGCU: {{ .Values.enflameResourceNameVGCU }}
|
||||
resourceNameVGCUPercentage: {{ .Values.enflameResourceNameVGCUPercentage }}
|
||||
mthreads:
|
||||
resourceCountName: "mthreads.com/vgpu"
|
||||
resourceMemoryName: "mthreads.com/sgpu-memory"
|
||||
resourceCoreName: "mthreads.com/sgpu-core"
|
||||
iluvatars:
|
||||
- chipName: MR-V100
|
||||
commonWord: MR-V100
|
||||
resourceCountName: iluvatar.ai/MR-V100-vgpu
|
||||
resourceMemoryName: iluvatar.ai/MR-V100.vMem
|
||||
resourceCoreName: iluvatar.ai/MR-V100.vCore
|
||||
- chipName: MR-V50
|
||||
commonWord: MR-V50
|
||||
resourceCountName: iluvatar.ai/MR-V50-vgpu
|
||||
resourceMemoryName: iluvatar.ai/MR-V50.vMem
|
||||
resourceCoreName: iluvatar.ai/MR-V50.vCore
|
||||
- chipName: BI-V150
|
||||
commonWord: BI-V150
|
||||
resourceCountName: iluvatar.ai/BI-V150-vgpu
|
||||
resourceMemoryName: iluvatar.ai/BI-V150.vMem
|
||||
resourceCoreName: iluvatar.ai/BI-V150.vCore
|
||||
- chipName: BI-V100
|
||||
commonWord: BI-V100
|
||||
resourceCountName: iluvatar.ai/BI-V100-vgpu
|
||||
resourceMemoryName: iluvatar.ai/BI-V100.vMem
|
||||
resourceCoreName: iluvatar.ai/BI-V100.vCore
|
||||
kunlun:
|
||||
resourceCountName: {{ .Values.kunlunResourceName }}
|
||||
resourceVCountName: {{ .Values.kunlunResourceVCountName }}
|
||||
resourceVMemoryName: {{ .Values.kunlunResourceVMemoryName }}
|
||||
awsneuron:
|
||||
resourceCountName: "aws.amazon.com/neuron"
|
||||
resourceCoreName: "aws.amazon.com/neuroncore"
|
||||
amd:
|
||||
resourceCountName: "amd.com/gpu"
|
||||
vnpus:
|
||||
- chipName: 910A
|
||||
commonWord: Ascend910A
|
||||
resourceName: huawei.com/Ascend910A
|
||||
resourceMemoryName: huawei.com/Ascend910A-memory
|
||||
memoryAllocatable: 32768
|
||||
memoryCapacity: 32768
|
||||
memoryFactor: 1
|
||||
aiCore: 30
|
||||
templates:
|
||||
- name: vir02
|
||||
memory: 2184
|
||||
aiCore: 2
|
||||
- name: vir04
|
||||
memory: 4369
|
||||
aiCore: 4
|
||||
- name: vir08
|
||||
memory: 8738
|
||||
aiCore: 8
|
||||
- name: vir16
|
||||
memory: 17476
|
||||
aiCore: 16
|
||||
- chipName: 910B2
|
||||
commonWord: Ascend910B2
|
||||
resourceName: huawei.com/Ascend910B2
|
||||
resourceMemoryName: huawei.com/Ascend910B2-memory
|
||||
memoryAllocatable: 65536
|
||||
memoryCapacity: 65536
|
||||
memoryFactor: 1
|
||||
aiCore: 24
|
||||
aiCPU: 6
|
||||
templates:
|
||||
- name: vir03_1c_8g
|
||||
memory: 8192
|
||||
aiCore: 3
|
||||
aiCPU: 1
|
||||
- name: vir06_1c_16g
|
||||
memory: 16384
|
||||
aiCore: 6
|
||||
aiCPU: 1
|
||||
- name: vir12_3c_32g
|
||||
memory: 32768
|
||||
aiCore: 12
|
||||
aiCPU: 3
|
||||
- chipName: 910B3
|
||||
commonWord: Ascend910B3
|
||||
resourceName: huawei.com/Ascend910B3
|
||||
resourceMemoryName: huawei.com/Ascend910B3-memory
|
||||
memoryAllocatable: 65536
|
||||
memoryCapacity: 65536
|
||||
memoryFactor: 1
|
||||
aiCore: 20
|
||||
aiCPU: 7
|
||||
templates:
|
||||
- name: vir05_1c_16g
|
||||
memory: 16384
|
||||
aiCore: 5
|
||||
aiCPU: 1
|
||||
- name: vir10_3c_32g
|
||||
memory: 32768
|
||||
aiCore: 10
|
||||
aiCPU: 3
|
||||
- chipName: 910B4-1
|
||||
commonWord: Ascend910B4-1
|
||||
resourceName: huawei.com/Ascend910B4-1
|
||||
resourceMemoryName: huawei.com/Ascend910B4-1-memory
|
||||
memoryAllocatable: 65536
|
||||
memoryCapacity: 65536
|
||||
memoryFactor: 1
|
||||
aiCore: 20
|
||||
aiCPU: 7
|
||||
templates:
|
||||
# NOTE: Names of vnpu template for 910B4-1 are fixed by Ascend runtime and must not be changed.
|
||||
# The memory is used for scheduling so the correct values must be set.
|
||||
# Template vir05_1c_8g actually provides 16GB memory,
|
||||
- name: vir05_1c_8g
|
||||
memory: 16384
|
||||
aiCore: 5
|
||||
aiCPU: 1
|
||||
# Template vir10_3c_16g actually provides 32GB memory
|
||||
- name: vir10_3c_16g
|
||||
memory: 32768
|
||||
aiCore: 10
|
||||
aiCPU: 3
|
||||
- chipName: 910B4
|
||||
commonWord: Ascend910B4
|
||||
resourceName: huawei.com/Ascend910B4
|
||||
resourceMemoryName: huawei.com/Ascend910B4-memory
|
||||
memoryAllocatable: 32768
|
||||
memoryCapacity: 32768
|
||||
memoryFactor: 1
|
||||
aiCore: 20
|
||||
aiCPU: 7
|
||||
templates:
|
||||
- name: vir05_1c_8g
|
||||
memory: 8192
|
||||
aiCore: 5
|
||||
aiCPU: 1
|
||||
- name: vir10_3c_16g
|
||||
memory: 16384
|
||||
aiCore: 10
|
||||
aiCPU: 3
|
||||
- chipName: 310P3
|
||||
commonWord: Ascend310P
|
||||
resourceName: huawei.com/Ascend310P
|
||||
resourceMemoryName: huawei.com/Ascend310P-memory
|
||||
memoryAllocatable: 21527
|
||||
memoryCapacity: 24576
|
||||
memoryFactor: 1
|
||||
aiCore: 8
|
||||
aiCPU: 7
|
||||
templates:
|
||||
- name: vir01
|
||||
memory: 3072
|
||||
aiCore: 1
|
||||
aiCPU: 1
|
||||
- name: vir02
|
||||
memory: 6144
|
||||
aiCore: 2
|
||||
aiCPU: 2
|
||||
- name: vir04
|
||||
memory: 12288
|
||||
aiCore: 4
|
||||
aiCPU: 4
|
||||
{{ end }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}}
|
||||
{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
annotations:
|
||||
"helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
labels:
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: admission-webhook
|
||||
rules:
|
||||
- apiGroups:
|
||||
- admissionregistration.k8s.io
|
||||
resources:
|
||||
#- validatingwebhookconfigurations
|
||||
- mutatingwebhookconfigurations
|
||||
verbs:
|
||||
- get
|
||||
- update
|
||||
{{- if .Values.podSecurityPolicy.enabled }}
|
||||
- apiGroups: ['extensions']
|
||||
resources: ['podsecuritypolicies']
|
||||
verbs: ['use']
|
||||
resourceNames:
|
||||
- {{ include "hami-vgpu.fullname" . }}-admission
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}}
|
||||
{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
annotations:
|
||||
"helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
labels:
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: admission-webhook
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,70 @@
|
|||
{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}}
|
||||
{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission-create
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
annotations:
|
||||
"helm.sh/hook": pre-install,pre-upgrade
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
labels:
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: admission-webhook
|
||||
spec:
|
||||
{{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }}
|
||||
# Alpha feature since k8s 1.12
|
||||
ttlSecondsAfterFinished: 0
|
||||
{{- end }}
|
||||
template:
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission-create
|
||||
{{- if .Values.scheduler.patch.podAnnotations }}
|
||||
annotations: {{ toYaml .Values.scheduler.patch.podAnnotations | nindent 8 }}
|
||||
{{- end }}
|
||||
labels:
|
||||
{{- include "hami-vgpu.labels" . | nindent 8 }}
|
||||
app.kubernetes.io/component: admission-webhook
|
||||
hami.io/webhook: ignore
|
||||
spec:
|
||||
{{- if .Values.scheduler.patch.priorityClassName }}
|
||||
priorityClassName: {{ .Values.scheduler.patch.priorityClassName }}
|
||||
{{- end }}
|
||||
{{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
|
||||
{{- include "hami.scheduler.patch.new.imagePullSecrets" . | nindent 6 }}
|
||||
{{- else }}
|
||||
{{- include "hami.scheduler.patch.imagePullSecrets" . | nindent 6 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: create
|
||||
{{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
|
||||
image: {{ include "hami.scheduler.patch.new.image" . }}
|
||||
imagePullPolicy: {{ .Values.scheduler.patch.imageNew.pullPolicy }}
|
||||
{{- else }}
|
||||
image: {{ include "hami.scheduler.patch.image" . }}
|
||||
imagePullPolicy: {{ .Values.scheduler.patch.image.pullPolicy }}
|
||||
{{- end }}
|
||||
args:
|
||||
- create
|
||||
- --cert-name=tls.crt
|
||||
- --key-name=tls.key
|
||||
{{- if .Values.scheduler.admissionWebhook.customURL.enabled }}
|
||||
- --host={{ printf "%s.%s.svc,127.0.0.1,%s" (include "hami-vgpu.scheduler" .) (include "hami-vgpu.namespace" .) .Values.scheduler.admissionWebhook.customURL.host}}
|
||||
{{- else }}
|
||||
- --host={{ printf "%s.%s.svc,127.0.0.1" (include "hami-vgpu.scheduler" .) (include "hami-vgpu.namespace" .) }}
|
||||
{{- end }}
|
||||
- --namespace={{ include "hami-vgpu.namespace" . }}
|
||||
- --secret-name={{ include "hami-vgpu.scheduler.tls" . }}
|
||||
restartPolicy: OnFailure
|
||||
serviceAccountName: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
{{- if .Values.scheduler.patch.nodeSelector }}
|
||||
nodeSelector: {{ toYaml .Values.scheduler.patch.nodeSelector | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.patch.tolerations }}
|
||||
tolerations: {{ toYaml .Values.scheduler.patch.tolerations | nindent 8 }}
|
||||
{{- end }}
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: {{ .Values.scheduler.patch.runAsUser }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}}
|
||||
{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission-patch
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
annotations:
|
||||
"helm.sh/hook": post-install,post-upgrade
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
labels:
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: admission-webhook
|
||||
spec:
|
||||
{{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }}
|
||||
# Alpha feature since k8s 1.12
|
||||
ttlSecondsAfterFinished: 0
|
||||
{{- end }}
|
||||
template:
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission-patch
|
||||
{{- if .Values.scheduler.patch.podAnnotations }}
|
||||
annotations: {{ toYaml .Values.scheduler.patch.podAnnotations | nindent 8 }}
|
||||
{{- end }}
|
||||
labels:
|
||||
{{- include "hami-vgpu.labels" . | nindent 8 }}
|
||||
app.kubernetes.io/component: admission-webhook
|
||||
hami.io/webhook: ignore
|
||||
spec:
|
||||
{{- if .Values.scheduler.patch.priorityClassName }}
|
||||
priorityClassName: {{ .Values.scheduler.patch.priorityClassName }}
|
||||
{{- end }}
|
||||
{{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
|
||||
{{- include "hami.scheduler.patch.new.imagePullSecrets" . | nindent 6 }}
|
||||
{{- else }}
|
||||
{{- include "hami.scheduler.patch.imagePullSecrets" . | nindent 6 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: patch
|
||||
{{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
|
||||
image: {{ include "hami.scheduler.patch.new.image" . }}
|
||||
imagePullPolicy: {{ .Values.scheduler.patch.imageNew.pullPolicy }}
|
||||
{{- else }}
|
||||
image: {{ include "hami.scheduler.patch.image" . }}
|
||||
imagePullPolicy: {{ .Values.scheduler.patch.image.pullPolicy }}
|
||||
{{- end }}
|
||||
args:
|
||||
- patch
|
||||
- --webhook-name={{ include "hami-vgpu.scheduler.webhook" . }}
|
||||
- --namespace={{ include "hami-vgpu.namespace" . }}
|
||||
- --patch-validating=false
|
||||
- --secret-name={{ include "hami-vgpu.scheduler.tls" . }}
|
||||
restartPolicy: OnFailure
|
||||
serviceAccountName: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
{{- if .Values.scheduler.patch.nodeSelector }}
|
||||
nodeSelector: {{ toYaml .Values.scheduler.patch.nodeSelector | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.patch.tolerations }}
|
||||
tolerations: {{ toYaml .Values.scheduler.patch.tolerations | nindent 8 }}
|
||||
{{- end }}
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: {{ .Values.scheduler.patch.runAsUser }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}}
|
||||
{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
|
||||
{{- if .Values.podSecurityPolicy.enabled }}
|
||||
apiVersion: policy/v1beta1
|
||||
kind: PodSecurityPolicy
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
annotations:
|
||||
"helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
labels:
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: admission-webhook
|
||||
spec:
|
||||
allowPrivilegeEscalation: false
|
||||
fsGroup:
|
||||
ranges:
|
||||
- max: 65535
|
||||
min: 1
|
||||
rule: MustRunAs
|
||||
requiredDropCapabilities:
|
||||
- ALL
|
||||
runAsUser:
|
||||
rule: MustRunAsNonRoot
|
||||
seLinux:
|
||||
rule: RunAsAny
|
||||
supplementalGroups:
|
||||
ranges:
|
||||
- max: 65535
|
||||
min: 1
|
||||
rule: MustRunAs
|
||||
volumes:
|
||||
- configMap
|
||||
- emptyDir
|
||||
- projected
|
||||
- secret
|
||||
- downwardAPI
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}}
|
||||
{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
annotations:
|
||||
"helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
labels:
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: admission-webhook
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- secrets
|
||||
verbs:
|
||||
- get
|
||||
- create
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}}
|
||||
{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
annotations:
|
||||
"helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
labels:
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: admission-webhook
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) }}
|
||||
{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
annotations:
|
||||
"helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
labels:
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: admission-webhook
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
{{- if not .Values.dra.enabled -}}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: "hami-scheduler"
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
rules:
|
||||
- apiGroups: ["coordination.k8s.io"]
|
||||
resources: ["leases"]
|
||||
verbs: ["create", "list", "watch", "get", "update", "patch"]
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
{{- if not .Values.dra.enabled -}}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: "hami-scheduler"
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
---
|
||||
{{- if .Values.mockDevicePlugin.enabled }}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.mock-device-plugin" . }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: {{ include "hami-vgpu.mock-device-plugin" . }}
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "hami-vgpu.mock-device-plugin" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
{{- if not .Values.dra.enabled }}
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-scheduler
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
{{- if .Values.scheduler.service.labels }}
|
||||
{{ toYaml .Values.scheduler.service.labels | indent 4 }}
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.service.annotations }}
|
||||
annotations: {{ toYaml .Values.scheduler.service.annotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
type: {{ .Values.scheduler.service.type | default "NodePort" }} # Default type is NodePort
|
||||
ports:
|
||||
- name: http
|
||||
port: {{ .Values.scheduler.service.httpPort | default 443 }} # Default HTTP port is 443
|
||||
targetPort: {{ .Values.scheduler.service.httpTargetPort | default 443 }}
|
||||
{{- if eq (.Values.scheduler.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort
|
||||
nodePort: {{ .Values.scheduler.service.schedulerPort | default 31998 }}
|
||||
{{- end }}
|
||||
protocol: TCP
|
||||
- name: monitor
|
||||
port: {{ .Values.scheduler.service.monitorPort | default 31993 }} # Default monitoring port is 31993
|
||||
targetPort: {{ .Values.scheduler.service.monitorTargetPort | default 9395 }}
|
||||
{{- if eq (.Values.scheduler.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort
|
||||
nodePort: {{ .Values.scheduler.service.monitorPort | default 31993 }}
|
||||
{{- end }}
|
||||
protocol: TCP
|
||||
selector:
|
||||
app.kubernetes.io/component: hami-scheduler
|
||||
{{- include "hami-vgpu.selectorLabels" . | nindent 4 }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
{{- if not .Values.dra.enabled -}}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: "hami-scheduler"
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
---
|
||||
{{- if .Values.mockDevicePlugin.enabled }}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.mock-device-plugin" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}}
|
||||
apiVersion: admissionregistration.k8s.io/v1
|
||||
kind: MutatingWebhookConfiguration
|
||||
metadata:
|
||||
{{- if .Values.scheduler.certManager.enabled }}
|
||||
annotations:
|
||||
cert-manager.io/inject-ca-from: {{ include "hami-vgpu.namespace" . }}/{{ include "hami-vgpu.scheduler" . }}-serving-cert
|
||||
{{- end }}
|
||||
name: {{ include "hami-vgpu.scheduler.webhook" . }}
|
||||
webhooks:
|
||||
- admissionReviewVersions:
|
||||
- v1beta1
|
||||
clientConfig:
|
||||
{{- if .Values.scheduler.admissionWebhook.customURL.enabled }}
|
||||
url: https://{{ .Values.scheduler.admissionWebhook.customURL.host}}:{{.Values.scheduler.admissionWebhook.customURL.port}}{{.Values.scheduler.admissionWebhook.customURL.path}}
|
||||
{{- else }}
|
||||
service:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
path: /webhook
|
||||
port: {{ .Values.scheduler.service.httpPort }}
|
||||
{{- end }}
|
||||
failurePolicy: {{ .Values.scheduler.admissionWebhook.failurePolicy }}
|
||||
matchPolicy: Equivalent
|
||||
name: vgpu.hami.io
|
||||
namespaceSelector:
|
||||
matchExpressions:
|
||||
- key: hami.io/webhook
|
||||
operator: NotIn
|
||||
values:
|
||||
- ignore
|
||||
{{- if .Values.scheduler.admissionWebhook.whitelistNamespaces }}
|
||||
- key: kubernetes.io/metadata.name
|
||||
operator: NotIn
|
||||
values:
|
||||
{{- toYaml .Values.scheduler.admissionWebhook.whitelistNamespaces | nindent 10 }}
|
||||
{{- end }}
|
||||
objectSelector:
|
||||
matchExpressions:
|
||||
- key: hami.io/webhook
|
||||
operator: NotIn
|
||||
values:
|
||||
- ignore
|
||||
reinvocationPolicy: {{ .Values.scheduler.admissionWebhook.reinvocationPolicy }}
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
apiVersions:
|
||||
- v1
|
||||
operations:
|
||||
- CREATE
|
||||
resources:
|
||||
- pods
|
||||
scope: '*'
|
||||
sideEffects: None
|
||||
timeoutSeconds: 10
|
||||
{{- end }}
|
||||
476
packages/system/hami/charts/hami/values.yaml
Normal file
476
packages/system/hami/charts/hami/values.yaml
Normal file
|
|
@ -0,0 +1,476 @@
|
|||
## @section Global configuration
|
||||
## @param global.imageRegistry Global Docker image registry
|
||||
## @param global.imagePullSecrets Global Docker image pull secrets
|
||||
global:
|
||||
## @param global.imageRegistry Global Docker image registry
|
||||
imageRegistry: ""
|
||||
## E.g.
|
||||
## imagePullSecrets:
|
||||
## - myRegistryKeySecretName
|
||||
## @param global.imagePullSecrets Global Docker image pull secrets
|
||||
imagePullSecrets: []
|
||||
imageTag: "v2.8.1"
|
||||
gpuHookPath: /usr/local
|
||||
labels: {}
|
||||
annotations: {}
|
||||
managedNodeSelectorEnable: false
|
||||
managedNodeSelector:
|
||||
usage: "gpu"
|
||||
|
||||
nameOverride: ""
|
||||
fullnameOverride: ""
|
||||
namespaceOverride: ""
|
||||
|
||||
#Nvidia GPU Parameters
|
||||
resourceName: "nvidia.com/gpu"
|
||||
resourceMem: "nvidia.com/gpumem"
|
||||
resourceMemPercentage: "nvidia.com/gpumem-percentage"
|
||||
resourceCores: "nvidia.com/gpucores"
|
||||
resourcePriority: "nvidia.com/priority"
|
||||
|
||||
#MLU Parameters
|
||||
mluResourceName: "cambricon.com/vmlu"
|
||||
mluResourceMem: "cambricon.com/mlu.smlu.vmemory"
|
||||
mluResourceCores: "cambricon.com/mlu.smlu.vcore"
|
||||
|
||||
#Hygon DCU Parameters
|
||||
dcuResourceName: "hygon.com/dcunum"
|
||||
dcuResourceMem: "hygon.com/dcumem"
|
||||
dcuResourceCores: "hygon.com/dcucores"
|
||||
|
||||
#Metax sGPU Parameters
|
||||
metaxResourceName: "metax-tech.com/sgpu"
|
||||
metaxResourceCore: "metax-tech.com/vcore"
|
||||
metaxResourceMem: "metax-tech.com/vmemory"
|
||||
metaxsGPUTopologyAware: "false"
|
||||
|
||||
#Enflame VGCU Parameters
|
||||
enflameResourceNameVGCU: "enflame.com/vgcu"
|
||||
enflameResourceNameVGCUPercentage: "enflame.com/vgcu-percentage"
|
||||
|
||||
#Kunlun XPU Parameters
|
||||
kunlunResourceName: "kunlunxin.com/xpu"
|
||||
kunlunResourceVCountName: "kunlunxin.com/vxpu"
|
||||
kunlunResourceVMemoryName: "kunlunxin.com/vxpu-memory"
|
||||
|
||||
schedulerName: "hami-scheduler"
|
||||
|
||||
podSecurityPolicy:
|
||||
enabled: false
|
||||
|
||||
scheduler:
|
||||
# @param nodeName defines the node name and the nvidia-vgpu-scheduler-scheduler will schedule to the node.
|
||||
# if we install the nvidia-vgpu-scheduler-scheduler as default scheduler, we need to remove the k8s default
|
||||
# scheduler pod from the cluster first, we must specify node name to skip the schedule workflow.
|
||||
nodeName: ""
|
||||
#nodeLabelSelector:
|
||||
# "gpu": "on"
|
||||
overwriteEnv: "false"
|
||||
defaultSchedulerPolicy:
|
||||
nodeSchedulerPolicy: binpack
|
||||
gpuSchedulerPolicy: spread
|
||||
metricsBindAddress: ":9395"
|
||||
# If set to false, When Pod.Spec.SchedulerName equals to the const DefaultSchedulerName in k8s.io/api/core/v1 package, webhook will not overwrite it
|
||||
forceOverwriteDefaultScheduler: true
|
||||
livenessProbe: false
|
||||
leaderElect: true
|
||||
# when leaderElect is true, replicas is available, otherwise replicas is 1.
|
||||
replicas: 1
|
||||
kubeScheduler:
|
||||
# @param enabled indicate whether to run kube-scheduler container in the scheduler pod, it's true by default.
|
||||
enabled: true
|
||||
## @param image.registry kube scheduler image registry
|
||||
## @param image.repository kube scheduler image repository
|
||||
## @param image.tag kube scheduler image tag (immutable tags are recommended)
|
||||
## @param image.pullPolicy kube scheduler image pull policy
|
||||
## @param image.pullSecrets Specify docker-registry secret names as an array
|
||||
image:
|
||||
registry: "registry.cn-hangzhou.aliyuncs.com"
|
||||
repository: "google_containers/kube-scheduler"
|
||||
tag: ""
|
||||
## Specify a imagePullPolicy
|
||||
## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
|
||||
## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
|
||||
##
|
||||
pullPolicy: IfNotPresent
|
||||
## Optionally specify an array of imagePullSecrets.
|
||||
## Secrets must be manually created in the namespace.
|
||||
## Example:
|
||||
## pullSecrets:
|
||||
## - myRegistryKeySecretName
|
||||
##
|
||||
pullSecrets: []
|
||||
resources: {}
|
||||
# If you do want to specify resources, uncomment the following lines, adjust them as necessary.
|
||||
# and remove the curly braces after 'resources:'.
|
||||
# limits:
|
||||
# cpu: 1000m
|
||||
# memory: 1000Mi
|
||||
# requests:
|
||||
# cpu: 100m
|
||||
# memory: 100Mi
|
||||
extraNewArgs:
|
||||
- --config=/config/config.yaml
|
||||
- -v=4
|
||||
extraArgs:
|
||||
- --policy-config-file=/config/config.json
|
||||
- -v=4
|
||||
extender:
|
||||
## @param image.registry scheduler extender image registry
|
||||
## @param image.repository scheduler extender image repository
|
||||
## @param image.tag scheduler extender image tag (immutable tags are recommended)
|
||||
## @param image.pullPolicy scheduler extender image pull policy
|
||||
## @param image.pullSecrets Specify docker-registry secret names as an array
|
||||
image:
|
||||
registry: "docker.io"
|
||||
repository: "projecthami/hami"
|
||||
tag: ""
|
||||
## Specify a imagePullPolicy
|
||||
## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
|
||||
## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
|
||||
##
|
||||
pullPolicy: IfNotPresent
|
||||
## Optionally specify an array of imagePullSecrets.
|
||||
## Secrets must be manually created in the namespace.
|
||||
## Example:
|
||||
## pullSecrets:
|
||||
## - myRegistryKeySecretName
|
||||
##
|
||||
pullSecrets: []
|
||||
resources: {}
|
||||
# If you do want to specify resources, uncomment the following lines, adjust them as necessary,
|
||||
# and remove the curly braces after 'resources:'.
|
||||
# limits:
|
||||
# cpu: 1000m
|
||||
# memory: 1000Mi
|
||||
# requests:
|
||||
# cpu: 100m
|
||||
# memory: 100Mi
|
||||
extraArgs:
|
||||
- --debug
|
||||
- -v=4
|
||||
nodeLockExpire: "5m"
|
||||
podAnnotations: {}
|
||||
tolerations: []
|
||||
#serviceAccountName: "hami-vgpu-scheduler-sa"
|
||||
admissionWebhook:
|
||||
# If set to false, the admission webhook is not installed and any pods that should use HAMi must be
|
||||
# configured to use the right 'schedulerName' and other device-specific configurations.
|
||||
enabled: true
|
||||
customURL:
|
||||
enabled: false
|
||||
# must be an endpoint using https.
|
||||
# should generate host certs here
|
||||
host: 127.0.0.1 # hostname or ip, can be your node'IP if you want to use https://<nodeIP>:<schedulerPort>/<path>
|
||||
port: 31998
|
||||
path: /webhook
|
||||
whitelistNamespaces:
|
||||
# Specify the namespaces that the webhook will not be applied to.
|
||||
# - default
|
||||
# - kube-system
|
||||
# - istio-system
|
||||
reinvocationPolicy: Never
|
||||
failurePolicy: Ignore
|
||||
## TLS Certificate Option 1: Use cert-manager to generate self-signed certificate.
|
||||
## If enabled, always takes precedence over options 2.
|
||||
certManager:
|
||||
enabled: false
|
||||
## TLS Certificate Option 2: Use kube-webhook-certgen to generate self-signed certificate.
|
||||
## If true and certManager.enabled is false, Helm will automatically create a self-signed cert and secret for you.
|
||||
patch:
|
||||
enabled: true
|
||||
## @param image.registry scheduler extender image registry
|
||||
## @param image.repository scheduler extender image repository
|
||||
## @param image.tag scheduler extender image tag (immutable tags are recommended)
|
||||
## @param image.pullPolicy scheduler extender image pull policy
|
||||
## @param image.pullSecrets Specify docker-registry secret names as an array
|
||||
image:
|
||||
registry: "docker.io"
|
||||
repository: "jettech/kube-webhook-certgen"
|
||||
tag: "v1.5.2"
|
||||
## Specify a imagePullPolicy
|
||||
## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
|
||||
## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
|
||||
##
|
||||
pullPolicy: IfNotPresent
|
||||
## Optionally specify an array of imagePullSecrets.
|
||||
## Secrets must be manually created in the namespace.
|
||||
## Example:
|
||||
## pullSecrets:
|
||||
## - myRegistryKeySecretName
|
||||
##
|
||||
pullSecrets: []
|
||||
## @param image.registry scheduler extender image registry
|
||||
## @param image.repository scheduler extender image repository
|
||||
## @param image.tag scheduler extender image tag (immutable tags are recommended)
|
||||
## @param image.pullPolicy scheduler extender image pull policy
|
||||
## @param image.pullSecrets Specify docker-registry secret names as an array
|
||||
imageNew:
|
||||
registry: "docker.io"
|
||||
repository: "liangjw/kube-webhook-certgen"
|
||||
tag: "v1.1.1"
|
||||
## Specify a imagePullPolicy
|
||||
## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
|
||||
## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
|
||||
##
|
||||
pullPolicy: IfNotPresent
|
||||
## Optionally specify an array of imagePullSecrets.
|
||||
## Secrets must be manually created in the namespace.
|
||||
## Example:
|
||||
## pullSecrets:
|
||||
## - myRegistryKeySecretName
|
||||
##
|
||||
pullSecrets: []
|
||||
priorityClassName: ""
|
||||
podAnnotations: {}
|
||||
nodeSelector: {}
|
||||
tolerations: []
|
||||
runAsUser: 2000
|
||||
service:
|
||||
type: NodePort # Default type is NodePort, can be changed to ClusterIP
|
||||
httpPort: 443 # HTTP port
|
||||
schedulerPort: 31998 # NodePort for HTTP
|
||||
monitorPort: 31993 # Monitoring port
|
||||
monitorTargetPort: 9395
|
||||
httpTargetPort: 443
|
||||
labels: {}
|
||||
annotations: {}
|
||||
|
||||
devicePlugin:
|
||||
enabled: true
|
||||
gpuOperatorToolkitReady:
|
||||
enabled: false
|
||||
hostPath: "/run/nvidia/validations"
|
||||
## @param image.registry devicePlugin image registry
|
||||
## @param image.repository devicePlugin image repository
|
||||
## @param image.tag devicePlugin image tag (immutable tags are recommended)
|
||||
## @param image.pullPolicy devicePlugin image pull policy
|
||||
## @param image.pullSecrets Specify docker-registry secret names as an array
|
||||
image:
|
||||
registry: "docker.io"
|
||||
repository: "projecthami/hami"
|
||||
tag: ""
|
||||
## Specify a imagePullPolicy
|
||||
## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
|
||||
## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
|
||||
##
|
||||
pullPolicy: IfNotPresent
|
||||
## Optionally specify an array of imagePullSecrets.
|
||||
## Secrets must be manually created in the namespace.
|
||||
## Example:
|
||||
## pullSecrets:
|
||||
## - myRegistryKeySecretName
|
||||
##
|
||||
pullSecrets: []
|
||||
monitor:
|
||||
## @param image.registry monitor image registry
|
||||
## @param image.repository monitor image repository
|
||||
## @param image.tag monitor image tag (immutable tags are recommended)
|
||||
## @param image.pullPolicy monitor image pull policy
|
||||
## @param image.pullSecrets Specify docker-registry secret names as an array
|
||||
image:
|
||||
registry: "docker.io"
|
||||
repository: "projecthami/hami"
|
||||
tag: ""
|
||||
## Specify a imagePullPolicy
|
||||
## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
|
||||
## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
|
||||
##
|
||||
pullPolicy: IfNotPresent
|
||||
## Optionally specify an array of imagePullSecrets.
|
||||
## Secrets must be manually created in the namespace.
|
||||
## Example:
|
||||
## pullSecrets:
|
||||
## - myRegistryKeySecretName
|
||||
##
|
||||
pullSecrets: []
|
||||
ctrPath: /usr/local/vgpu/containers
|
||||
resyncInterval: "5m"
|
||||
extraArgs:
|
||||
- -v=4
|
||||
extraEnvs: {}
|
||||
resources: {}
|
||||
# If you do want to specify resources, uncomment the following lines, adjust them as necessary.
|
||||
# and remove the curly braces after 'resources:'.
|
||||
# limits:
|
||||
# cpu: 1000m
|
||||
# memory: 1000Mi
|
||||
# requests:
|
||||
# cpu: 100m
|
||||
# memory: 100Mi
|
||||
|
||||
deviceSplitCount: 10
|
||||
deviceMemoryScaling: 1
|
||||
deviceCoreScaling: 1
|
||||
# Node configuration for device plugin, Priority: externalConfigName > config > default config
|
||||
nodeConfiguration:
|
||||
# If you want to use a custom config.json, you can set the content here.
|
||||
# If this is set, it will override the default config.json(An example is as follows).
|
||||
config: |
|
||||
{
|
||||
"nodeconfig": [
|
||||
{
|
||||
"name": "your-node-name",
|
||||
"operatingmode": "hami-core",
|
||||
"devicememoryscaling": 1,
|
||||
"devicesplitcount": 10,
|
||||
"migstrategy": "none",
|
||||
"filterdevices": {
|
||||
"uuid": [],
|
||||
"index": []
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
# If you want to use an existing ConfigMap, you can set the name here.
|
||||
# If this is set, the chart will not create the ConfigMap and will use the existing one.
|
||||
externalConfigName: ""
|
||||
# The runtime class name to be used by the device plugin, and added to the pod.spec.runtimeClassName of applications utilizing NVIDIA GPUs
|
||||
runtimeClassName: ""
|
||||
# Whether to create runtime class, name comes from runtimeClassName when it is set
|
||||
createRuntimeClass: false
|
||||
migStrategy: "none"
|
||||
disablecorelimit: "false"
|
||||
passDeviceSpecsEnabled: false
|
||||
deviceListStrategy: "envvar"
|
||||
nvidiaHookPath: null
|
||||
nvidiaDriverRoot: null
|
||||
gdrcopyEnabled: null
|
||||
gdsEnabled: null
|
||||
mofedEnabled: null
|
||||
|
||||
extraArgs:
|
||||
- -v=4
|
||||
extraEnvs: {}
|
||||
service:
|
||||
type: NodePort # Default type is NodePort, can be changed to ClusterIP
|
||||
httpPort: 31992
|
||||
labels: {}
|
||||
annotations: {}
|
||||
|
||||
pluginPath: /var/lib/kubelet/device-plugins
|
||||
libPath: /usr/local/vgpu
|
||||
|
||||
podAnnotations: {}
|
||||
nvidiaNodeSelector:
|
||||
gpu: "on"
|
||||
tolerations: []
|
||||
# The updateStrategy for DevicePlugin DaemonSet.
|
||||
# If you want to update the DaemonSet by manual, set type as "OnDelete".
|
||||
# We recommend use OnDelete update strategy because DevicePlugin pod restart will cause business pod restart, this behavior is destructive.
|
||||
# Otherwise, you can use RollingUpdate update strategy to rolling update DevicePlugin pod.
|
||||
updateStrategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxUnavailable: 1
|
||||
|
||||
resources: {}
|
||||
# If you do want to specify resources, uncomment the following lines, adjust them as necessary.
|
||||
# and remove the curly braces after 'resources:'.
|
||||
# limits:
|
||||
# cpu: 1000m
|
||||
# memory: 1000Mi
|
||||
# requests:
|
||||
# cpu: 100m
|
||||
# memory: 100Mi
|
||||
|
||||
mockDevicePlugin:
|
||||
enabled: false
|
||||
image:
|
||||
registry: "docker.io"
|
||||
repository: "projecthami/mock-device-plugin"
|
||||
tag: "1.0.1"
|
||||
## Specify a imagePullPolicy
|
||||
## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
|
||||
## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
|
||||
##
|
||||
pullPolicy: IfNotPresent
|
||||
## Optionally specify an array of imagePullSecrets.
|
||||
## Secrets must be manually created in the namespace.
|
||||
## Example:
|
||||
## pullSecrets:
|
||||
## - myRegistryKeySecretName
|
||||
##
|
||||
pullSecrets: []
|
||||
# If this option is enabled, the DRA will be installed and the scheduler extender will not be installed.
|
||||
dra:
|
||||
enabled: false
|
||||
|
||||
# Configuration for the hami-dra subchart, which includes DRA drivers.
|
||||
# This is part of config for DRA, please refer to the DRA documentation for more details.
|
||||
hami-dra:
|
||||
monitor:
|
||||
enabled: true
|
||||
drivers:
|
||||
nvidia:
|
||||
enabled: true
|
||||
# If you are using gpu driver on host, you need to set this to false
|
||||
containerDriver: true
|
||||
image:
|
||||
repository: projecthami/k8s-dra-driver
|
||||
tag: "v0.0.1-dev"
|
||||
|
||||
devices:
|
||||
amd:
|
||||
customresources:
|
||||
- amd.com/gpu
|
||||
- amd.com/gpu-memory
|
||||
awsneuron:
|
||||
customresources:
|
||||
- aws.amazon.com/neuron
|
||||
- aws.amazon.com/neuroncore
|
||||
kunlun:
|
||||
enabled: true
|
||||
customresources:
|
||||
- kunlunxin.com/xpu
|
||||
- kunlunxin.com/vxpu
|
||||
- kunlunxin.com/vxpu-memory
|
||||
enflame:
|
||||
enabled: true
|
||||
customresources:
|
||||
- enflame.com/vgcu
|
||||
- enflame.com/vgcu-percentage
|
||||
- enflame.com/gcu
|
||||
mthreads:
|
||||
enabled: true
|
||||
customresources:
|
||||
- mthreads.com/vgpu
|
||||
nvidia:
|
||||
gpuCorePolicy: default
|
||||
libCudaLogLevel: 1
|
||||
ascend:
|
||||
enabled: false
|
||||
image: ""
|
||||
imagePullPolicy: IfNotPresent
|
||||
extraArgs: []
|
||||
nodeSelector:
|
||||
ascend: "on"
|
||||
tolerations: []
|
||||
customresources:
|
||||
- huawei.com/Ascend910A
|
||||
- huawei.com/Ascend910A-memory
|
||||
- huawei.com/Ascend910B2
|
||||
- huawei.com/Ascend910B2-memory
|
||||
- huawei.com/Ascend910B3
|
||||
- huawei.com/Ascend910B3-memory
|
||||
- huawei.com/Ascend910B4
|
||||
- huawei.com/Ascend910B4-memory
|
||||
- huawei.com/Ascend910B4-1
|
||||
- huawei.com/Ascend910B4-1-memory
|
||||
- huawei.com/Ascend310P
|
||||
- huawei.com/Ascend310P-memory
|
||||
iluvatar:
|
||||
enabled: false
|
||||
customresources:
|
||||
- iluvatar.ai/BI-V100-vgpu
|
||||
- iluvatar.ai/BI-V100.vCore
|
||||
- iluvatar.ai/BI-V100.vMem
|
||||
- iluvatar.ai/BI-V150-vgpu
|
||||
- iluvatar.ai/BI-V150.vCore
|
||||
- iluvatar.ai/BI-V150.vMem
|
||||
- iluvatar.ai/MR-V100-vgpu
|
||||
- iluvatar.ai/MR-V100.vCore
|
||||
- iluvatar.ai/MR-V100.vMem
|
||||
- iluvatar.ai/MR-V50-vgpu
|
||||
- iluvatar.ai/MR-V50.vCore
|
||||
- iluvatar.ai/MR-V50.vMem
|
||||
8
packages/system/hami/values.yaml
Normal file
8
packages/system/hami/values.yaml
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
hami:
|
||||
devicePlugin:
|
||||
runtimeClassName: nvidia
|
||||
nodeConfiguration:
|
||||
config: |
|
||||
{
|
||||
"nodeconfig": []
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue