Compare commits
1 commit
main
...
fix/tags-w
| Author | SHA1 | Date | |
|---|---|---|---|
|
2420d8949b |
62 changed files with 51 additions and 3580 deletions
12
.github/workflows/tags.yaml
vendored
12
.github/workflows/tags.yaml
vendored
|
|
@ -286,10 +286,16 @@ jobs:
|
|||
core.setOutput('version', version);
|
||||
core.setOutput('tag', ref);
|
||||
|
||||
- name: Checkout main branch
|
||||
# Check out the tag commit, not main. Patch releases are cut from
|
||||
# release-X.Y branches, so HEAD-on-main contains commits that are
|
||||
# NOT in the release tag. Running the AI against the tag commit
|
||||
# makes `git log v<prev>..HEAD` correspond to the actual release
|
||||
# contents. The Create changelog branch step below resets the file
|
||||
# onto a fresh branch from origin/main for the PR.
|
||||
- name: Checkout release tag
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: main
|
||||
ref: ${{ steps.tag.outputs.tag }}
|
||||
fetch-depth: 0
|
||||
fetch-tags: true
|
||||
token: ${{ steps.app-token.outputs.token }}
|
||||
|
|
@ -325,7 +331,7 @@ jobs:
|
|||
VERSION: ${{ steps.tag.outputs.version }}
|
||||
run: |
|
||||
copilot \
|
||||
--prompt "Generate the release changelog for tag v${VERSION}. Follow the instructions in @docs/agents/changelog.md exactly, including the 'Scope and boundaries' section at the top. Your deliverable is the single file docs/changelogs/v${VERSION}.md — write it and exit; this workflow handles branching, committing, pushing, and opening the PR." \
|
||||
--prompt "Generate the release changelog for tag v${VERSION}. The repository is checked out at the v${VERSION} tag commit, so HEAD == the release commit. Use v${VERSION} (or HEAD) as the upper bound when comparing against the previous version — never assume HEAD is on main, because patch releases are cut from release-X.Y branches. Follow the instructions in @docs/agents/changelog.md exactly, including the 'Scope and boundaries' section at the top. Your deliverable is the single file docs/changelogs/v${VERSION}.md — write it and exit; this workflow handles branching, committing, pushing, and opening the PR." \
|
||||
--allow-all-tools --allow-all-paths < /dev/null
|
||||
|
||||
- name: Create changelog branch and commit
|
||||
|
|
|
|||
|
|
@ -69,9 +69,6 @@ type Addons struct {
|
|||
// NVIDIA GPU Operator.
|
||||
// +kubebuilder:default:={}
|
||||
GpuOperator GPUOperatorAddon `json:"gpuOperator"`
|
||||
// HAMi GPU virtualization middleware.
|
||||
// +kubebuilder:default:={}
|
||||
Hami HAMiAddon `json:"hami"`
|
||||
// Ingress-NGINX controller.
|
||||
// +kubebuilder:default:={}
|
||||
IngressNginx IngressNginxAddon `json:"ingressNginx"`
|
||||
|
|
@ -163,15 +160,6 @@ type GatewayAPIAddon struct {
|
|||
Enabled bool `json:"enabled"`
|
||||
}
|
||||
|
||||
type HAMiAddon struct {
|
||||
// Enable HAMi (requires GPU Operator).
|
||||
// +kubebuilder:default:=false
|
||||
Enabled bool `json:"enabled"`
|
||||
// Custom Helm values overrides.
|
||||
// +kubebuilder:default:={}
|
||||
ValuesOverride k8sRuntime.RawExtension `json:"valuesOverride"`
|
||||
}
|
||||
|
||||
type Images struct {
|
||||
// Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.
|
||||
// +kubebuilder:default:=""
|
||||
|
|
|
|||
|
|
@ -49,7 +49,6 @@ func (in *Addons) DeepCopyInto(out *Addons) {
|
|||
in.Fluxcd.DeepCopyInto(&out.Fluxcd)
|
||||
out.GatewayAPI = in.GatewayAPI
|
||||
in.GpuOperator.DeepCopyInto(&out.GpuOperator)
|
||||
in.Hami.DeepCopyInto(&out.Hami)
|
||||
in.IngressNginx.DeepCopyInto(&out.IngressNginx)
|
||||
in.MonitoringAgents.DeepCopyInto(&out.MonitoringAgents)
|
||||
in.Velero.DeepCopyInto(&out.Velero)
|
||||
|
|
@ -136,7 +135,6 @@ func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) {
|
|||
}
|
||||
in.Addons.DeepCopyInto(&out.Addons)
|
||||
in.ControlPlane.DeepCopyInto(&out.ControlPlane)
|
||||
out.Images = in.Images
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigSpec.
|
||||
|
|
@ -262,37 +260,6 @@ func (in *GatewayAPIAddon) DeepCopy() *GatewayAPIAddon {
|
|||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *HAMiAddon) DeepCopyInto(out *HAMiAddon) {
|
||||
*out = *in
|
||||
in.ValuesOverride.DeepCopyInto(&out.ValuesOverride)
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HAMiAddon.
|
||||
func (in *HAMiAddon) DeepCopy() *HAMiAddon {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(HAMiAddon)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *Images) DeepCopyInto(out *Images) {
|
||||
*out = *in
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Images.
|
||||
func (in *Images) DeepCopy() *Images {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(Images)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *IngressNginxAddon) DeepCopyInto(out *IngressNginxAddon) {
|
||||
*out = *in
|
||||
|
|
|
|||
|
|
@ -103,40 +103,42 @@ git branch --show-current
|
|||
|
||||
**Important**: Determine if you're generating a changelog for a **minor release** (vX.Y.0) or a **patch release** (vX.Y.Z where Z > 0).
|
||||
|
||||
**⚠️ CRITICAL — use the tag, not HEAD:** Always compare the previous version against the **tag for the release you are generating** (e.g. `v1.3.1`), never against `HEAD`. Patch releases are cut from `release-X.Y` branches, so when CI checks out the tag and runs `git log v<prev>..HEAD` you might get the right answer — but if the tag is missing locally, or if a developer runs this on `main`, `HEAD` will contain commits merged to main that are NOT in the release. Using the tag explicitly removes this footgun.
|
||||
|
||||
**For patch releases (vX.Y.Z where Z > 0):**
|
||||
Get the list of commits starting from the previous patch version to HEAD:
|
||||
Get the list of commits between the previous patch tag and the new tag:
|
||||
|
||||
**⚠️ CRITICAL: Do NOT use --first-parent flag! It will skip merge commits including backports!**
|
||||
|
||||
```bash
|
||||
# Get all commits including merge commits (backports)
|
||||
git log <previous_version>..HEAD --pretty=format:"%h - %s (%an, %ar)"
|
||||
# Get all commits including merge commits (backports), bounded by the new tag
|
||||
git log <previous_version>..v<new_version> --pretty=format:"%h - %s (%an, %ar)"
|
||||
```
|
||||
|
||||
For example, if generating changelog for `v0.37.2`:
|
||||
```bash
|
||||
git log v0.37.1..HEAD --pretty=format:"%h - %s (%an, %ar)"
|
||||
git log v0.37.1..v0.37.2 --pretty=format:"%h - %s (%an, %ar)"
|
||||
```
|
||||
|
||||
**⚠️ IMPORTANT: Check for backports:**
|
||||
- Look for commits with "[Backport release-X.Y]" in the commit message
|
||||
- For backport PRs, find the original PR number mentioned in the backport commit message or PR description
|
||||
- Use the original PR author (not the backport PR author) when creating changelog entries
|
||||
- Include both the original PR number and backport PR number in the changelog entry (e.g., `#1606, #1609`)
|
||||
- **MUST** combine the original and backport into a **single entry** with both PR numbers (e.g. `#1606, backport #1609`). Do NOT list the original and the backport as two separate entries — that produces duplicates in the changelog.
|
||||
|
||||
**For minor releases (vX.Y.0):**
|
||||
Minor releases must include **all changes** from patch releases of the previous minor version. Get commits from the previous minor release:
|
||||
Minor releases must include **all changes** from patch releases of the previous minor version. Get commits from the previous minor release tag up to the new tag:
|
||||
|
||||
**⚠️ CRITICAL: Do NOT use --first-parent flag! It will skip merge commits including backports!**
|
||||
|
||||
```bash
|
||||
# For v0.38.0, get all commits since v0.37.0 (including all patch releases v0.37.1, v0.37.2, etc.)
|
||||
git log v<previous_minor_version>..HEAD --pretty=format:"%h - %s (%an, %ar)"
|
||||
git log v<previous_minor_version>..v<new_version> --pretty=format:"%h - %s (%an, %ar)"
|
||||
```
|
||||
|
||||
For example, if generating changelog for `v0.38.0`:
|
||||
```bash
|
||||
git log v0.37.0..HEAD --pretty=format:"%h - %s (%an, %ar)"
|
||||
git log v0.37.0..v0.38.0 --pretty=format:"%h - %s (%an, %ar)"
|
||||
```
|
||||
|
||||
This will include all commits from v0.37.1, v0.37.2, v0.37.3, etc., up to v0.38.0.
|
||||
|
|
@ -279,8 +281,9 @@ git log <previous_version>..<new_version> --format="%s%n%b" | grep -oE '#[0-9]+'
|
|||
- Backport PRs have format: `[Backport release-X.Y] <original title> (#BACKPORT_PR_NUMBER)`
|
||||
- The backport commit message or PR description usually mentions the original PR number
|
||||
- For backport entries in changelog, use the original PR author (not the backport PR author)
|
||||
- Include both original and backport PR numbers in the changelog entry (e.g., `#1606, #1609`)
|
||||
- **MUST** combine the original and backport PR into a **single changelog entry** with both PR numbers (e.g. `#1606, backport #1609`). NEVER list the original and the backport as two separate entries — that produces visible duplicates and was the primary defect that motivated this rule.
|
||||
- To find original PR from backport: Check the backport PR description or commit message for "Backport of #ORIGINAL_PR"
|
||||
- If a release range contains a backport but its original PR is **not** in the same range (because the original was merged earlier and already shipped in a previous release), drop the entry entirely — it has already been changelog'd.
|
||||
|
||||
**For each PR number, get the author:**
|
||||
|
||||
|
|
@ -437,6 +440,8 @@ Create a new changelog file in the format matching previous versions:
|
|||
|
||||
3. **Entry format:**
|
||||
- Use the format: `* **Brief description**: detailed description ([**@username**](https://github.com/username) in #PR_NUMBER)`
|
||||
- **⚠️ The brief description and the detailed description MUST NOT be the same string.** The brief description is typically the conventional-commit subject (e.g. `fix(api): drop legacy field`); the detailed description is a one-or-two-sentence explanation of what the change means for users — what is fixed, what is now possible, what behavior changed. If you find yourself writing `* **fix(foo): X**: fix(foo): X (...)` you have not done the second half of the work — go read the PR body or commit diff and write the user-facing description.
|
||||
- **⚠️ Do NOT invent entries.** Every entry must correspond to a commit in the `git log v<previous>..v<new>` range. If you can't find a commit for an entry you wrote, drop the entry. PR numbers from outside the release range (e.g. PRs from years ago, or PRs that were merged after the tag commit) MUST NOT appear in the changelog.
|
||||
- **CRITICAL - Get authorship correctly**:
|
||||
- **ALWAYS use PR author, not commit author**: Extract PR number from commit message, then use `gh pr view` to get the PR author. The commit author (especially for squash/merge commits) is usually the person who merged the PR (or GitHub bot), NOT the person who wrote the code.
|
||||
```bash
|
||||
|
|
@ -494,20 +499,17 @@ Create a new changelog file in the format matching previous versions:
|
|||
Since you've already generated the changelog with all PR authors correctly identified, simply extract GitHub usernames from the changelog entries:
|
||||
|
||||
```bash
|
||||
# Extract all GitHub usernames from the current release changelog
|
||||
# This method is simpler and more reliable than extracting from git history
|
||||
|
||||
# For patch releases: extract from the current changelog file
|
||||
grep -oE '\[@[a-zA-Z0-9_-]+\]' docs/changelogs/v<version>.md | \
|
||||
sed 's/\[@/@/' | sed 's/\]//' | \
|
||||
sort -u
|
||||
|
||||
# For minor releases: extract from the current changelog file
|
||||
grep -oE '\[@[a-zA-Z0-9_-]+\]' docs/changelogs/v<version>.md | \
|
||||
sed 's/\[@/@/' | sed 's/\]//' | \
|
||||
# Extract all GitHub usernames from the current release changelog,
|
||||
# filtering out CI/bot accounts that should not appear as human contributors.
|
||||
# This method is simpler and more reliable than extracting from git history.
|
||||
grep -oE '\[\*\*@[a-zA-Z0-9_/-]+\*\*\]' docs/changelogs/v<version>.md | \
|
||||
sed -E 's|\[\*\*@||; s|\*\*\]||' | \
|
||||
grep -viE '^(app/|.*\[bot\]$|cozystack-ci$|github-actions$|dependabot$|renovate$)' | \
|
||||
sort -u
|
||||
```
|
||||
|
||||
**⚠️ MANDATORY: Exclude bot/CI accounts from the human Contributors list.** Accounts whose login starts with `app/`, ends with `[bot]`, or matches `cozystack-ci`, `github-actions`, `dependabot`, or `renovate` represent automation, not human contributions, and must NOT appear under "Contributors". They may legitimately appear in **entry attribution** (e.g. a Renovate PR is attributed to `@app/renovate`) — but never in the contributors list.
|
||||
|
||||
**Get all previous contributors (to identify new ones):**
|
||||
```bash
|
||||
# Extract GitHub usernames from all previous changelogs
|
||||
|
|
|
|||
|
|
@ -1,37 +0,0 @@
|
|||
<!--
|
||||
https://github.com/cozystack/cozystack/releases/tag/v1.3.1
|
||||
-->
|
||||
|
||||
# v1.3.1 (2026-04-28)
|
||||
|
||||
Patch release covering a TenantNamespace IDOR fix in the API, a destructive `post-upgrade` hook removed from the etcd chart, kamaji controller stability, a `linstor-csi` bump that fixes live migration on Protocol-A/B DRBD resources, the missing `linstor-gui` build wiring, and a velero RBAC fix that unblocked installs on bundles without Velero.
|
||||
|
||||
## Security
|
||||
|
||||
* **fix(api): prevent IDOR in TenantNamespace Get and Watch handlers**: Two IDOR (Insecure Direct Object Reference) vulnerabilities allowed authenticated users to read TenantNamespace metadata they had no RoleBinding for. The `Get` and `Watch` handlers now go through a new `hasAccessToNamespace()` helper that lists RoleBindings scoped only to the target namespace (orders of magnitude faster than the previous all-cluster scan), returns `NotFound` instead of leaking existence on unauthorized access, and applies the same check on the `Watch` filter path. Includes regression tests for the unauthorized paths. ([**@IvanHunters**](https://github.com/IvanHunters) in #2471, backport #2524)
|
||||
|
||||
## Features
|
||||
|
||||
* **feat(linstor): bump linstor-csi to v1.10.6 with Protocol-C dual-attach fix**: Live migration of KubeVirt VMs on Protocol-A/B (async) DRBD volumes no longer fails with `Protocol C required`. `linstor-csi` v1.10.6 now installs a `Protocol=C` override on the resource-definition during dual-attach and reverts it on detach, so `replicated-async` StorageClasses and other Protocol-A/B resource groups support live migration without manual `drbdadm adjust` intervention. ([**@kvaps**](https://github.com/kvaps) in #2496, backport #2505)
|
||||
|
||||
## Fixes
|
||||
|
||||
* **fix(backups): move velero-configmap Role to velero chart**: The `backupstrategy-controller` (a default package) declared a Role/RoleBinding scoped to the `cozy-velero` namespace for managing `ResourceModifier` ConfigMaps. On bundles where Velero was not enabled, that namespace did not exist and the HelmRelease failed with `namespaces "cozy-velero" not found`, blocking installation. The Role/RoleBinding now lives in the velero chart, so it is created only when velero is actually deployed. ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2459, backport #2467)
|
||||
|
||||
* **fix(etcd): remove destructive post-upgrade cert-regeneration hook**: The etcd chart ran a `post-upgrade` Helm hook on every upgrade that deleted etcd TLS Secrets (`etcd-ca-tls`, `etcd-peer-ca-tls`, `etcd-client-tls`, `etcd-peer-tls`, `etcd-server-tls`) and then deleted etcd pods, forcing cert-manager to re-issue the entire etcd CA chain. On clusters with Kamaji-managed tenant control planes this put every tenant `kube-apiserver` into CrashLoopBackOff until each DataStore was manually re-reconciled. The hook was a one-shot `2.6.0 → 2.6.1` migration that became a permanent footgun once chart versioning moved to `0.0.0+<git-hash>` (always `< 2.6.1` per semver) and after the underlying `rotationPolicy: Always` issue was fixed in `47d81f70`. The hook is now removed entirely. ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2462, backport #2511)
|
||||
|
||||
* **fix(kamaji): increase memory limits and add startup probe**: The kamaji controller frequently entered CrashLoopBackOff due to OOMKills (exit 137) within ~20–25 seconds of startup, with the readiness probe failing while the controller was still finishing initialization. Memory limit raised from 500Mi to 512Mi, request from 100Mi to 256Mi, and a 60-second startup probe (12 attempts × 5s periods) is added so the controller has room to boot before liveness/readiness probes engage. ([**@IvanHunters**](https://github.com/IvanHunters) in #2421, backport #2491)
|
||||
|
||||
## Build
|
||||
|
||||
* **build(linstor): include linstor-gui in root image build target**: The `linstor-gui` package (added in #2382) was never wired into the root `Makefile`'s `build:` target, so CI never built or published the image. `ghcr.io/cozystack/cozystack/linstor-gui` returned `NAME_UNKNOWN` and `values.yaml` stayed pinned to `tag: 2.3.0` without a digest. The missing build line is added so the next CI run publishes the image and the per-package `Makefile` digest-pins `values.yaml` automatically. ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2498, backport #2518)
|
||||
|
||||
## Contributors
|
||||
|
||||
Thanks to everyone who contributed to this patch release:
|
||||
|
||||
* [**@IvanHunters**](https://github.com/IvanHunters)
|
||||
* [**@kvaps**](https://github.com/kvaps)
|
||||
* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil)
|
||||
|
||||
**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.3.0...v1.3.1
|
||||
|
|
@ -128,9 +128,6 @@ See the reference for components utilized in this service:
|
|||
| `addons.gpuOperator` | NVIDIA GPU Operator. | `object` | `{}` |
|
||||
| `addons.gpuOperator.enabled` | Enable GPU Operator. | `bool` | `false` |
|
||||
| `addons.gpuOperator.valuesOverride` | Custom Helm values overrides. | `object` | `{}` |
|
||||
| `addons.hami` | HAMi GPU virtualization middleware. | `object` | `{}` |
|
||||
| `addons.hami.enabled` | Enable HAMi (requires GPU Operator). | `bool` | `false` |
|
||||
| `addons.hami.valuesOverride` | Custom Helm values overrides. | `object` | `{}` |
|
||||
| `addons.fluxcd` | FluxCD GitOps operator. | `object` | `{}` |
|
||||
| `addons.fluxcd.enabled` | Enable FluxCD. | `bool` | `false` |
|
||||
| `addons.fluxcd.valuesOverride` | Custom Helm values overrides. | `object` | `{}` |
|
||||
|
|
|
|||
|
|
@ -1,11 +1,3 @@
|
|||
{{- define "cozystack.defaultGpuOperatorValues" -}}
|
||||
{{- if .Values.addons.hami.enabled }}
|
||||
gpu-operator:
|
||||
devicePlugin:
|
||||
enabled: false
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{- if and .Values.addons.gpuOperator.enabled .Values._namespace.etcd }}
|
||||
apiVersion: helm.toolkit.fluxcd.io/v2
|
||||
kind: HelmRelease
|
||||
|
|
@ -37,12 +29,9 @@ spec:
|
|||
force: true
|
||||
remediation:
|
||||
retries: -1
|
||||
{{- $defaults := fromYaml (include "cozystack.defaultGpuOperatorValues" .) }}
|
||||
{{- $overrides := deepCopy (default (dict) .Values.addons.gpuOperator.valuesOverride) }}
|
||||
{{- $merged := mergeOverwrite (default (dict) $defaults) $overrides }}
|
||||
{{- if $merged }}
|
||||
{{- with .Values.addons.gpuOperator.valuesOverride }}
|
||||
values:
|
||||
{{- toYaml $merged | nindent 4 }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
|
||||
dependsOn:
|
||||
|
|
|
|||
|
|
@ -1,49 +0,0 @@
|
|||
{{- if and .Values.addons.hami.enabled .Values._namespace.etcd }}
|
||||
{{- if not .Values.addons.gpuOperator.enabled }}
|
||||
{{- fail "addons.hami requires addons.gpuOperator to be enabled" }}
|
||||
{{- end }}
|
||||
apiVersion: helm.toolkit.fluxcd.io/v2
|
||||
kind: HelmRelease
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-hami
|
||||
labels:
|
||||
cozystack.io/repository: system
|
||||
cozystack.io/target-cluster-name: {{ .Release.Name }}
|
||||
sharding.fluxcd.io/key: tenants
|
||||
spec:
|
||||
releaseName: hami
|
||||
chartRef:
|
||||
kind: ExternalArtifact
|
||||
name: cozystack-kubernetes-application-kubevirt-kubernetes-hami
|
||||
namespace: cozy-system
|
||||
kubeConfig:
|
||||
secretRef:
|
||||
name: {{ .Release.Name }}-admin-kubeconfig
|
||||
key: super-admin.svc
|
||||
targetNamespace: cozy-hami
|
||||
storageNamespace: cozy-hami
|
||||
interval: 5m
|
||||
timeout: 10m
|
||||
install:
|
||||
createNamespace: true
|
||||
remediation:
|
||||
retries: -1
|
||||
upgrade:
|
||||
force: true
|
||||
remediation:
|
||||
retries: -1
|
||||
{{- with .Values.addons.hami.valuesOverride }}
|
||||
values:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
|
||||
dependsOn:
|
||||
{{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }}
|
||||
- name: {{ .Release.Name }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
{{- end }}
|
||||
- name: {{ .Release.Name }}-cilium
|
||||
namespace: {{ .Release.Namespace }}
|
||||
- name: {{ .Release.Name }}-gpu-operator
|
||||
namespace: {{ .Release.Namespace }}
|
||||
{{- end }}
|
||||
|
|
@ -1,99 +0,0 @@
|
|||
suite: GPU Operator HelmRelease HAMi integration tests
|
||||
templates:
|
||||
- templates/helmreleases/gpu-operator.yaml
|
||||
values:
|
||||
- values-ci.yaml
|
||||
tests:
|
||||
- it: should disable devicePlugin when hami is enabled
|
||||
set:
|
||||
addons:
|
||||
gpuOperator:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
hami:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.values.gpu-operator.devicePlugin.enabled
|
||||
value: false
|
||||
|
||||
- it: should not have values when hami is disabled and no overrides
|
||||
set:
|
||||
addons:
|
||||
gpuOperator:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
hami:
|
||||
enabled: false
|
||||
valuesOverride: {}
|
||||
asserts:
|
||||
- notExists:
|
||||
path: spec.values
|
||||
|
||||
- it: should apply hami defaults when valuesOverride key is omitted
|
||||
set:
|
||||
addons:
|
||||
gpuOperator:
|
||||
enabled: true
|
||||
hami:
|
||||
enabled: true
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.values.gpu-operator.devicePlugin.enabled
|
||||
value: false
|
||||
|
||||
- it: should allow user overrides to merge with hami defaults
|
||||
set:
|
||||
addons:
|
||||
gpuOperator:
|
||||
enabled: true
|
||||
valuesOverride:
|
||||
gpu-operator:
|
||||
driver:
|
||||
enabled: false
|
||||
hami:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.values.gpu-operator.devicePlugin.enabled
|
||||
value: false
|
||||
- equal:
|
||||
path: spec.values.gpu-operator.driver.enabled
|
||||
value: false
|
||||
|
||||
- it: should let user explicitly override devicePlugin.enabled to true with hami enabled
|
||||
set:
|
||||
addons:
|
||||
gpuOperator:
|
||||
enabled: true
|
||||
valuesOverride:
|
||||
gpu-operator:
|
||||
devicePlugin:
|
||||
enabled: true
|
||||
driver:
|
||||
enabled: false
|
||||
hami:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.values.gpu-operator.devicePlugin.enabled
|
||||
value: true
|
||||
- equal:
|
||||
path: spec.values.gpu-operator.driver.enabled
|
||||
value: false
|
||||
|
||||
- it: should not render when gpuOperator is disabled
|
||||
set:
|
||||
addons:
|
||||
gpuOperator:
|
||||
enabled: false
|
||||
valuesOverride: {}
|
||||
hami:
|
||||
enabled: false
|
||||
valuesOverride: {}
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
|
|
@ -1,153 +0,0 @@
|
|||
suite: HAMi HelmRelease tests
|
||||
templates:
|
||||
- templates/helmreleases/hami.yaml
|
||||
values:
|
||||
- values-ci.yaml
|
||||
tests:
|
||||
- it: should not render when hami is disabled
|
||||
set:
|
||||
addons:
|
||||
hami:
|
||||
enabled: false
|
||||
valuesOverride: {}
|
||||
gpuOperator:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
|
||||
- it: should render HelmRelease when hami is enabled
|
||||
set:
|
||||
addons:
|
||||
hami:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
gpuOperator:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 1
|
||||
- isKind:
|
||||
of: HelmRelease
|
||||
|
||||
- it: should fail when gpuOperator is not enabled
|
||||
set:
|
||||
addons:
|
||||
hami:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
gpuOperator:
|
||||
enabled: false
|
||||
valuesOverride: {}
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: "addons.hami requires addons.gpuOperator to be enabled"
|
||||
|
||||
- it: should have correct metadata labels
|
||||
set:
|
||||
addons:
|
||||
hami:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
gpuOperator:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
asserts:
|
||||
- equal:
|
||||
path: metadata.labels["cozystack.io/repository"]
|
||||
value: system
|
||||
- equal:
|
||||
path: metadata.labels["sharding.fluxcd.io/key"]
|
||||
value: tenants
|
||||
|
||||
- it: should use ExternalArtifact chartRef
|
||||
set:
|
||||
addons:
|
||||
hami:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
gpuOperator:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.chartRef.kind
|
||||
value: ExternalArtifact
|
||||
- equal:
|
||||
path: spec.chartRef.namespace
|
||||
value: cozy-system
|
||||
|
||||
- it: should target cozy-hami namespace
|
||||
set:
|
||||
addons:
|
||||
hami:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
gpuOperator:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.targetNamespace
|
||||
value: cozy-hami
|
||||
- equal:
|
||||
path: spec.storageNamespace
|
||||
value: cozy-hami
|
||||
|
||||
- it: should depend on gpu-operator and cilium
|
||||
release:
|
||||
name: test
|
||||
namespace: test-ns
|
||||
set:
|
||||
addons:
|
||||
hami:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
gpuOperator:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.dependsOn
|
||||
content:
|
||||
name: test-cilium
|
||||
namespace: test-ns
|
||||
- contains:
|
||||
path: spec.dependsOn
|
||||
content:
|
||||
name: test-gpu-operator
|
||||
namespace: test-ns
|
||||
|
||||
- it: should not render spec.values when valuesOverride is empty
|
||||
set:
|
||||
addons:
|
||||
hami:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
gpuOperator:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 1
|
||||
- notExists:
|
||||
path: spec.values
|
||||
|
||||
- it: should pass through valuesOverride
|
||||
set:
|
||||
addons:
|
||||
hami:
|
||||
enabled: true
|
||||
valuesOverride:
|
||||
hami:
|
||||
devicePlugin:
|
||||
deviceSplitCount: 5
|
||||
gpuOperator:
|
||||
enabled: true
|
||||
valuesOverride: {}
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.values.hami.devicePlugin.deviceSplitCount
|
||||
value: 5
|
||||
|
|
@ -149,7 +149,6 @@
|
|||
"fluxcd",
|
||||
"gatewayAPI",
|
||||
"gpuOperator",
|
||||
"hami",
|
||||
"ingressNginx",
|
||||
"monitoringAgents",
|
||||
"velero",
|
||||
|
|
@ -269,28 +268,6 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"hami": {
|
||||
"description": "HAMi GPU virtualization middleware.",
|
||||
"type": "object",
|
||||
"default": {},
|
||||
"required": [
|
||||
"enabled",
|
||||
"valuesOverride"
|
||||
],
|
||||
"properties": {
|
||||
"enabled": {
|
||||
"description": "Enable HAMi (requires GPU Operator).",
|
||||
"type": "boolean",
|
||||
"default": false
|
||||
},
|
||||
"valuesOverride": {
|
||||
"description": "Custom Helm values overrides.",
|
||||
"type": "object",
|
||||
"default": {},
|
||||
"x-kubernetes-preserve-unknown-fields": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"ingressNginx": {
|
||||
"description": "Ingress-NGINX controller.",
|
||||
"type": "object",
|
||||
|
|
|
|||
|
|
@ -94,10 +94,6 @@ host: ""
|
|||
## @field {bool} enabled - Enable FluxCD.
|
||||
## @field {object} valuesOverride - Custom Helm values overrides.
|
||||
|
||||
## @typedef {struct} HAMiAddon - HAMi GPU virtualization middleware.
|
||||
## @field {bool} enabled - Enable HAMi (requires GPU Operator).
|
||||
## @field {object} valuesOverride - Custom Helm values overrides.
|
||||
|
||||
## @typedef {struct} MonitoringAgentsAddon - Monitoring agents (Fluent Bit, VMAgents).
|
||||
## @field {bool} enabled - Enable monitoring agents.
|
||||
## @field {object} valuesOverride - Custom Helm values overrides.
|
||||
|
|
@ -118,7 +114,6 @@ host: ""
|
|||
## @field {GatewayAPIAddon} gatewayAPI - Gateway API addon.
|
||||
## @field {IngressNginxAddon} ingressNginx - Ingress-NGINX controller.
|
||||
## @field {GPUOperatorAddon} gpuOperator - NVIDIA GPU Operator.
|
||||
## @field {HAMiAddon} hami - HAMi GPU virtualization middleware.
|
||||
## @field {FluxCDAddon} fluxcd - FluxCD GitOps operator.
|
||||
## @field {MonitoringAgentsAddon} monitoringAgents - Monitoring agents.
|
||||
## @field {VerticalPodAutoscalerAddon} verticalPodAutoscaler - Vertical Pod Autoscaler.
|
||||
|
|
@ -142,9 +137,6 @@ addons:
|
|||
gpuOperator:
|
||||
enabled: false
|
||||
valuesOverride: {}
|
||||
hami:
|
||||
enabled: false
|
||||
valuesOverride: {}
|
||||
fluxcd:
|
||||
enabled: false
|
||||
valuesOverride: {}
|
||||
|
|
|
|||
|
|
@ -1,24 +0,0 @@
|
|||
---
|
||||
apiVersion: cozystack.io/v1alpha1
|
||||
kind: PackageSource
|
||||
metadata:
|
||||
name: cozystack.hami
|
||||
spec:
|
||||
sourceRef:
|
||||
kind: OCIRepository
|
||||
name: cozystack-packages
|
||||
namespace: cozy-system
|
||||
path: /
|
||||
variants:
|
||||
- name: default
|
||||
dependsOn:
|
||||
- cozystack.gpu-operator
|
||||
components:
|
||||
- name: hami
|
||||
path: system/hami
|
||||
valuesFiles:
|
||||
- values.yaml
|
||||
install:
|
||||
privileged: true
|
||||
namespace: cozy-hami
|
||||
releaseName: hami
|
||||
|
|
@ -52,8 +52,6 @@ spec:
|
|||
path: system/cilium
|
||||
- name: kubernetes-gpu-operator
|
||||
path: system/gpu-operator
|
||||
- name: kubernetes-hami
|
||||
path: system/hami
|
||||
- name: kubernetes-vertical-pod-autoscaler
|
||||
path: system/vertical-pod-autoscaler
|
||||
- name: kubernetes-prometheus-operator-crds
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@
|
|||
{{include "cozystack.platform.package.default" (list "cozystack.kubevirt-cdi" $) }}
|
||||
{{include "cozystack.platform.package.optional.default" (list "cozystack.vm-default-images" $) }}
|
||||
{{include "cozystack.platform.package.optional.default" (list "cozystack.gpu-operator" $) }}
|
||||
{{include "cozystack.platform.package.optional.default" (list "cozystack.hami" $) }}
|
||||
{{include "cozystack.platform.package.default" (list "cozystack.kamaji" $) }}
|
||||
{{include "cozystack.platform.package.default" (list "cozystack.capi-operator" $) }}
|
||||
{{include "cozystack.platform.package.default" (list "cozystack.capi-provider-bootstrap-kubeadm" $) }}
|
||||
|
|
|
|||
|
|
@ -105,6 +105,9 @@
|
|||
{{- /* cozystack-api DaemonSet */ -}}
|
||||
{{- $apiValues := dict "cozystackAPI" (dict "nodeSelector" $genericNodeSelector) -}}
|
||||
{{- $_ := set $cozystackEngineComponents "cozystack-api" (dict "values" $apiValues) -}}
|
||||
{{- /* lineage-controller-webhook DaemonSet */ -}}
|
||||
{{- $lineageValues := dict "lineageControllerWebhook" (dict "nodeSelector" $genericNodeSelector) -}}
|
||||
{{- $_ := set $cozystackEngineComponents "lineage-controller-webhook" (dict "values" $lineageValues) -}}
|
||||
{{- end -}}
|
||||
{{- if .Values.authentication.oidc.enabled }}
|
||||
{{include "cozystack.platform.package" (list "cozystack.cozystack-engine" "oidc" $ $cozystackEngineComponents) }}
|
||||
|
|
|
|||
|
|
@ -1,3 +0,0 @@
|
|||
apiVersion: v2
|
||||
name: cozy-hami
|
||||
version: 0.0.0 # Placeholder, the actual version will be automatically set during the build process
|
||||
|
|
@ -1,43 +0,0 @@
|
|||
export NAME=hami
|
||||
export NAMESPACE=cozy-$(NAME)
|
||||
|
||||
include ../../../hack/common-envs.mk
|
||||
include ../../../hack/package.mk
|
||||
|
||||
# When bumping the HAMi version, run `make update` and then review
|
||||
# the resulting diff in `charts/hami/`. The recipe below reproduces the
|
||||
# top-level vendoring overrides automatically:
|
||||
#
|
||||
# 1. Removes the broken hami-dra subchart. Upstream's NVIDIA DRA driver
|
||||
# path requires kubelet DRA support that cozystack does not enable
|
||||
# and has no upstream fix tracked. See commit 3c5521e.
|
||||
# 2. Empties Chart.yaml dependencies and drops Chart.lock so Helm does
|
||||
# not try to re-pull hami-dra at build time. See commit 2734dc0.
|
||||
# 3. Strips dra/hami-dra/podSecurityPolicy blocks from the upstream
|
||||
# values.yaml since the corresponding code paths are gone. PSP is
|
||||
# removed from Kubernetes 1.25+ and is unused by cozystack.
|
||||
#
|
||||
# Template-level patches are NOT reproduced automatically:
|
||||
#
|
||||
# * Scheduler templates have `{{- if .Values.dra.enabled }}` blocks
|
||||
# that need to be removed because the dra value is gone (commit
|
||||
# 2734dc0 stripped them).
|
||||
# * device-plugin/monitorservice.yaml uses `indent` with leading
|
||||
# whitespace; it must be rewritten to `nindent` for the labels block
|
||||
# to render correctly when devicePlugin.service.labels is set
|
||||
# (commit 3685254).
|
||||
#
|
||||
# After `make update`, run `git diff -- charts/hami/templates/` and
|
||||
# replay these template patches against the new upstream version, then
|
||||
# verify with `helm unittest`. If upstream restructured the affected
|
||||
# files, the patches may need to be redesigned rather than reapplied.
|
||||
|
||||
update:
|
||||
rm -rf charts
|
||||
helm repo add hami-charts https://project-hami.github.io/HAMi/
|
||||
helm repo update hami-charts
|
||||
helm pull hami-charts/hami --untar --untardir charts
|
||||
rm -rf charts/hami/charts/hami-dra
|
||||
yq --inplace '.dependencies = []' charts/hami/Chart.yaml
|
||||
rm -f charts/hami/Chart.lock
|
||||
yq --inplace 'del(.dra) | del(.["hami-dra"]) | del(.podSecurityPolicy)' charts/hami/values.yaml
|
||||
|
|
@ -1,82 +0,0 @@
|
|||
# HAMi — GPU Virtualization Middleware
|
||||
|
||||
[HAMi](https://github.com/Project-HAMi/HAMi) (Heterogeneous AI Computing Virtualization Middleware) is a CNCF Sandbox project that enables fractional GPU sharing in Kubernetes. It allows workloads to request specific amounts of GPU memory and compute cores instead of claiming entire GPUs.
|
||||
|
||||
## Architecture
|
||||
|
||||
HAMi consists of four components:
|
||||
|
||||
- **MutatingWebhook** — intercepts pod creation, injects `schedulerName: hami-scheduler`
|
||||
- **Scheduler Extender** — extends kube-scheduler with GPU-aware Filter and Bind logic
|
||||
- **Device Plugin** (DaemonSet) — registers vGPU resources via the Kubernetes Device Plugin API
|
||||
- **HAMi-core** (`libvgpu.so`) — `LD_PRELOAD` library injected into workload containers, intercepts CUDA API calls to enforce memory and compute isolation
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- GPU Operator must be enabled (`addons.gpuOperator.enabled: true`)
|
||||
- NVIDIA driver >= 440 on host nodes
|
||||
- nvidia-container-toolkit configured as the default container runtime
|
||||
- GPU nodes labeled with `gpu=on`
|
||||
|
||||
## Known Limitations
|
||||
|
||||
### glibc < 2.34 requirement for workload containers
|
||||
|
||||
HAMi-core uses `LD_PRELOAD` to intercept `dlsym()` for CUDA symbol resolution. The fallback code path relies on `_dl_sym`, a private glibc internal symbol that was removed in glibc 2.34 when libdl and libpthread were merged into libc.so.
|
||||
|
||||
**This limitation affects workload containers only**, not the host OS or HAMi's own components.
|
||||
|
||||
| Distribution | glibc | Result |
|
||||
| --------------- | ----- | -------------------------------------------- |
|
||||
| Ubuntu 18.04 | 2.27 | Full isolation (memory + compute) |
|
||||
| Ubuntu 20.04 | 2.31 | Full isolation (memory + compute) |
|
||||
| Ubuntu 22.04 | 2.35 | Memory isolation works, compute breaks |
|
||||
| Ubuntu 24.04 | 2.39 | Both memory and compute isolation break |
|
||||
| Alpine (musl) | N/A | Completely incompatible (`dlvsym` absent) |
|
||||
|
||||
Most modern ML/AI base images (CUDA 12.x, PyTorch 2.x, TensorFlow 2.x) use Ubuntu 22.04+ with glibc >= 2.35, which means compute isolation will not work with these images until the upstream fix is merged.
|
||||
|
||||
**Upstream tracking issues:**
|
||||
|
||||
- [HAMi-core#174](https://github.com/Project-HAMi/HAMi-core/issues/174) — `_dl_sym` removal in glibc 2.34 breaks HAMi-core's CUDA symbol resolution at the symbol level
|
||||
- [HAMi#1190](https://github.com/Project-HAMi/HAMi/issues/1190) — maintainer thread confirming the empirical per-glibc-version isolation behavior shown in the table above
|
||||
|
||||
### musl libc (Alpine) incompatibility
|
||||
|
||||
HAMi-core is completely incompatible with musl libc. The `dlvsym()` function used by HAMi-core is a glibc extension not available in musl. Only glibc-based container images (Debian, Ubuntu, RHEL, etc.) can use HAMi GPU isolation.
|
||||
|
||||
## Usage
|
||||
|
||||
Enable HAMi in your tenant Kubernetes cluster values:
|
||||
|
||||
```yaml
|
||||
addons:
|
||||
gpuOperator:
|
||||
enabled: true
|
||||
hami:
|
||||
enabled: true
|
||||
```
|
||||
|
||||
When HAMi is enabled, GPU Operator's built-in device plugin is automatically disabled to avoid conflicts. This default is preserved by setting `addons.gpuOperator.valuesOverride.gpu-operator.devicePlugin.enabled: false`; advanced topologies that partition GPU pools (e.g. some nodes use HAMi while others run the standard NVIDIA device plugin via node selectors) can re-enable it explicitly through `valuesOverride`.
|
||||
|
||||
### Requesting fractional GPU resources
|
||||
|
||||
```yaml
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 1
|
||||
nvidia.com/gpumem: 3000 # 3000 MB of GPU memory
|
||||
nvidia.com/gpucores: 30 # 30% of GPU compute cores
|
||||
```
|
||||
|
||||
## Parameters
|
||||
|
||||
Default values shown below are inherited from the upstream HAMi chart and may change with upstream updates.
|
||||
|
||||
| Name | Description | Default |
|
||||
| --- | --- | --- |
|
||||
| `hami.devicePlugin.runtimeClassName` | RuntimeClass for device plugin pods | `nvidia` |
|
||||
| `hami.devicePlugin.deviceSplitCount` | Max virtual GPUs per physical GPU | `10` |
|
||||
| `hami.devicePlugin.deviceMemoryScaling` | Memory overcommit factor (> 1.0 enables overcommit) | `1` |
|
||||
| `hami.scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy` | Node packing strategy (`binpack` or `spread`) | `binpack` |
|
||||
| `hami.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy` | GPU packing strategy (`binpack` or `spread`) | `spread` |
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
apiVersion: v2
|
||||
appVersion: 2.8.1
|
||||
dependencies: []
|
||||
description: Heterogeneous AI Computing Virtualization Middleware
|
||||
keywords:
|
||||
- vgpu
|
||||
- gpu
|
||||
kubeVersion: '>= 1.18.0-0'
|
||||
maintainers:
|
||||
- email: archlitchi@gmail.com
|
||||
name: limengxuan
|
||||
- email: xiaozhang0210@hotmail.com
|
||||
name: zhangxiao
|
||||
name: hami
|
||||
sources:
|
||||
- https://github.com/Project-HAMi/HAMi
|
||||
type: application
|
||||
version: 2.8.1
|
||||
|
|
@ -1,237 +0,0 @@
|
|||
# HAMi Helm Chart Values Documentation
|
||||
|
||||
This document provides detailed descriptions of all configurable values parameters for the HAMi Helm Chart.
|
||||
|
||||
## Global Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `global.imageRegistry` | Global Docker image registry | `""` |
|
||||
| `global.imagePullSecrets` | Global Docker image pull secrets | `[]` |
|
||||
| `global.imageTag` | Image tag | `"v2.8.1"` |
|
||||
| `global.gpuHookPath` | GPU Hook path | `/usr/local` |
|
||||
| `global.labels` | Global labels | `{}` |
|
||||
| `global.annotations` | Global annotations | `{}` |
|
||||
| `global.managedNodeSelectorEnable` | Whether to enable managed node selector | `false` |
|
||||
| `global.managedNodeSelector.usage` | Managed node selector usage | `"gpu"` |
|
||||
| `nameOverride` | Name override | `""` |
|
||||
| `fullnameOverride` | Full name override | `""` |
|
||||
| `namespaceOverride` | Namespace override | `""` |
|
||||
|
||||
## Resource Name Configuration
|
||||
|
||||
### NVIDIA GPU Resources
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `resourceName` | GPU resource name | `"nvidia.com/gpu"` |
|
||||
| `resourceMem` | GPU memory resource name | `"nvidia.com/gpumem"` |
|
||||
| `resourceMemPercentage` | GPU memory percentage resource name | `"nvidia.com/gpumem-percentage"` |
|
||||
| `resourceCores` | GPU core resource name | `"nvidia.com/gpucores"` |
|
||||
| `resourcePriority` | GPU priority resource name | `"nvidia.com/priority"` |
|
||||
|
||||
### Cambricon MLU Resources
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `mluResourceName` | MLU resource name | `"cambricon.com/vmlu"` |
|
||||
| `mluResourceMem` | MLU memory resource name | `"cambricon.com/mlu.smlu.vmemory"` |
|
||||
| `mluResourceCores` | MLU core resource name | `"cambricon.com/mlu.smlu.vcore"` |
|
||||
|
||||
### Hygon DCU Resources
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `dcuResourceName` | DCU resource name | `"hygon.com/dcunum"` |
|
||||
| `dcuResourceMem` | DCU memory resource name | `"hygon.com/dcumem"` |
|
||||
| `dcuResourceCores` | DCU core resource name | `"hygon.com/dcucores"` |
|
||||
|
||||
### Metax GPU Resources
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `metaxResourceName` | GPU resource name | `"metax-tech.com/sgpu"` |
|
||||
| `metaxResourceCore` | GPU core resource name | `"metax-tech.com/vcore"` |
|
||||
| `metaxResourceMem` | GPU memory resource name | `"metax-tech.com/vmemory"` |
|
||||
| `metaxsGPUTopologyAware` | GPU topology awareness | `"false"` |
|
||||
|
||||
### Enflame GCU Resources
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `enflameResourceNameVGCU` | vGCU resource name | `"enflame.com/vgcu"` |
|
||||
| `enflameResourceNameVGCUPercentage` | vGCU percentage resource name | `"enflame.com/vgcu-percentage"` |
|
||||
|
||||
### Kunlunxin XPU Resources
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `kunlunResourceName` | XPU resource name | `"kunlunxin.com/xpu"` |
|
||||
|
||||
## Scheduler Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `schedulerName` | Scheduler name | `"hami-scheduler"` |
|
||||
| `scheduler.nodeName` | Define node name, scheduler will schedule to this node | `""` |
|
||||
| `scheduler.overwriteEnv` | Whether to overwrite environment variables | `"false"` |
|
||||
| `scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy` | Node scheduler policy | `binpack` |
|
||||
| `scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy` | GPU scheduler policy | `spread` |
|
||||
| `scheduler.metricsBindAddress` | Metrics bind address | `":9395"` |
|
||||
| `scheduler.forceOverwriteDefaultScheduler` | Whether to force overwrite default scheduler | `true` |
|
||||
| `scheduler.livenessProbe` | Whether to enable liveness probe | `false` |
|
||||
| `scheduler.leaderElect` | Whether to enable leader election | `true` |
|
||||
| `scheduler.replicas` | Number of replicas | `1` |
|
||||
|
||||
### Kube Scheduler Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `scheduler.kubeScheduler.enabled` | Whether to run kube-scheduler container in scheduler pod | `true` |
|
||||
| `scheduler.kubeScheduler.image.registry` | Kube scheduler image registry | `"registry.cn-hangzhou.aliyuncs.com"` |
|
||||
| `scheduler.kubeScheduler.image.repository` | Kube scheduler image repository | `"google_containers/kube-scheduler"` |
|
||||
| `scheduler.kubeScheduler.image.tag` | Kube scheduler image tag | `""` |
|
||||
| `scheduler.kubeScheduler.image.pullPolicy` | Kube scheduler image pull policy | `IfNotPresent` |
|
||||
| `scheduler.kubeScheduler.image.pullSecrets` | Kube scheduler image pull secrets | `[]` |
|
||||
| `scheduler.kubeScheduler.extraNewArgs` | Extra new arguments | `["--config=/config/config.yaml", "-v=4"]` |
|
||||
| `scheduler.kubeScheduler.extraArgs` | Extra arguments | `["--policy-config-file=/config/config.json", "-v=4"]` |
|
||||
|
||||
### Extender Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `scheduler.extender.image.registry` | Scheduler extender image registry | `"docker.io"` |
|
||||
| `scheduler.extender.image.repository` | Scheduler extender image repository | `"projecthami/hami"` |
|
||||
| `scheduler.extender.image.tag` | Scheduler extender image tag | `""` |
|
||||
| `scheduler.extender.image.pullPolicy` | Scheduler extender image pull policy | `IfNotPresent` |
|
||||
| `scheduler.extender.image.pullSecrets` | Scheduler extender image pull secrets | `[]` |
|
||||
| `scheduler.extender.extraArgs` | Scheduler extender extra arguments | `["--debug", "-v=4"]` |
|
||||
|
||||
### Admission Webhook Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `scheduler.admissionWebhook.enabled` | Whether to enable admission webhook | `true` |
|
||||
| `scheduler.admissionWebhook.customURL.enabled` | Whether to enable custom URL | `false` |
|
||||
| `scheduler.admissionWebhook.customURL.host` | Custom URL host | `127.0.0.1` |
|
||||
| `scheduler.admissionWebhook.customURL.port` | Custom URL port | `31998` |
|
||||
| `scheduler.admissionWebhook.customURL.path` | Custom URL path | `/webhook` |
|
||||
| `scheduler.admissionWebhook.reinvocationPolicy` | Reinvocation policy | `Never` |
|
||||
| `scheduler.admissionWebhook.failurePolicy` | Failure policy | `Ignore` |
|
||||
|
||||
### TLS Certificate Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `scheduler.certManager.enabled` | Whether to use cert-manager to generate self-signed certificates | `false` |
|
||||
| `scheduler.patch.enabled` | Whether to use kube-webhook-certgen to generate self-signed certificates | `true` |
|
||||
| `scheduler.patch.image.registry` | Certgen image registry | `"docker.io"` |
|
||||
| `scheduler.patch.image.repository` | Certgen image repository | `"jettech/kube-webhook-certgen"` |
|
||||
| `scheduler.patch.image.tag` | Certgen image tag | `"v1.5.2"` |
|
||||
| `scheduler.patch.image.pullPolicy` | Certgen image pull policy | `IfNotPresent` |
|
||||
| `scheduler.patch.imageNew.registry` | New certgen image registry | `"docker.io"` |
|
||||
| `scheduler.patch.imageNew.repository` | New certgen image repository | `"liangjw/kube-webhook-certgen"` |
|
||||
| `scheduler.patch.imageNew.tag` | New certgen image tag | `"v1.1.1"` |
|
||||
|
||||
### Scheduler Service Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `scheduler.service.type` | Service type | `NodePort` |
|
||||
| `scheduler.service.httpPort` | HTTP port | `443` |
|
||||
| `scheduler.service.schedulerPort` | Scheduler NodePort | `31998` |
|
||||
| `scheduler.service.monitorPort` | Monitor port | `31993` |
|
||||
| `scheduler.service.monitorTargetPort` | Monitor target port | `9395` |
|
||||
|
||||
## Device Plugin Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `devicePlugin.image.registry` | Device plugin image registry | `"docker.io"` |
|
||||
| `devicePlugin.image.repository` | Device plugin image repository | `"projecthami/hami"` |
|
||||
| `devicePlugin.image.tag` | Device plugin image tag | `""` |
|
||||
| `devicePlugin.image.pullPolicy` | Device plugin image pull policy | `IfNotPresent` |
|
||||
| `devicePlugin.image.pullSecrets` | Device plugin image pull secrets | `[]` |
|
||||
|
||||
### Monitor Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `devicePlugin.monitor.image.registry` | Monitor image registry | `"docker.io"` |
|
||||
| `devicePlugin.monitor.image.repository` | Monitor image repository | `"projecthami/hami"` |
|
||||
| `devicePlugin.monitor.image.tag` | Monitor image tag | `""` |
|
||||
| `devicePlugin.monitor.image.pullPolicy` | Monitor image pull policy | `IfNotPresent` |
|
||||
| `devicePlugin.monitor.image.pullSecrets` | Monitor image pull secrets | `[]` |
|
||||
| `devicePlugin.monitor.ctrPath` | Container path | `/usr/local/vgpu/containers` |
|
||||
| `devicePlugin.monitor.extraArgs` | Monitor extra arguments | `["-v=4"]` |
|
||||
| `devicePlugin.monitor.extraEnvs` | Monitor extra environments | `{}` |
|
||||
|
||||
### Device Plugin Other Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `devicePlugin.deviceSplitCount` | Integer type, default value: 10. Maximum number of tasks assigned to a single GPU device | `10` |
|
||||
| `devicePlugin.deviceMemoryScaling` | Device memory scaling ratio | `1` |
|
||||
| `devicePlugin.deviceCoreScaling` | Device core scaling ratio | `1` |
|
||||
| `devicePlugin.runtimeClassName` | Runtime class name | `""` |
|
||||
| `devicePlugin.createRuntimeClass` | Whether to create runtime class | `false` |
|
||||
| `devicePlugin.migStrategy` | String type, "none" means ignore MIG functionality, "mixed" means allocate MIG devices through independent resources | `"none"` |
|
||||
| `devicePlugin.disablecorelimit` | String type, "true" means disable core limit, "false" means enable core limit | `"false"` |
|
||||
| `devicePlugin.passDeviceSpecsEnabled` | Whether to enable passing device specs | `false` |
|
||||
| `devicePlugin.extraArgs` | Device plugin extra arguments | `["-v=4"]` |
|
||||
| `devicePlugin.nodeConfiguration.config` | Node configuration for device plugin by json | An example of default configuration. |
|
||||
| `devicePlugin.nodeConfiguration.externalConfigName` | Node configuration for device plugin by external congimap | `""` |
|
||||
| `devicePlugin.extraEnvs` | Device plugin extra environments | `{}` |
|
||||
|
||||
### Device Plugin Service Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `devicePlugin.service.type` | Service type | `NodePort` |
|
||||
| `devicePlugin.service.httpPort` | HTTP port | `31992` |
|
||||
|
||||
### Device Plugin Deployment Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `devicePlugin.pluginPath` | Plugin path | `/var/lib/kubelet/device-plugins` |
|
||||
| `devicePlugin.libPath` | Library path | `/usr/local/vgpu` |
|
||||
| `devicePlugin.nvidiaNodeSelector` | NVIDIA node selector | `{"gpu": "on"}` |
|
||||
| `devicePlugin.updateStrategy.type` | Update strategy type | `RollingUpdate` |
|
||||
| `devicePlugin.updateStrategy.rollingUpdate.maxUnavailable` | Maximum unavailable count | `1` |
|
||||
|
||||
## Device Configuration
|
||||
|
||||
### AWS Neuron
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `devices.awsneuron.customresources` | Custom resources | `["aws.amazon.com/neuron", "aws.amazon.com/neuroncore"]` |
|
||||
|
||||
### Kunlunxin
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `devices.kunlun.enabled` | Whether to enable | `true` |
|
||||
| `devices.kunlun.customresources` | Custom resources | `["kunlunxin.com/xpu"]` |
|
||||
|
||||
### Mthreads
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `devices.mthreads.enabled` | Whether to enable | `true` |
|
||||
| `devices.mthreads.customresources` | Custom resources | `["mthreads.com/vgpu"]` |
|
||||
|
||||
### NVIDIA
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `devices.nvidia.gpuCorePolicy` | GPU core policy | `default` |
|
||||
| `devices.nvidia.libCudaLogLevel` | CUDA library log level | `1` |
|
||||
|
||||
### Huawei Ascend
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `devices.ascend.enabled` | Whether to enable | `false` |
|
||||
| `devices.ascend.image` | Image | `""` |
|
||||
| `devices.ascend.imagePullPolicy` | Image pull policy | `IfNotPresent` |
|
||||
| `devices.ascend.extraArgs` | Extra arguments | `[]` |
|
||||
| `devices.ascend.nodeSelector` | Node selector | `{"ascend": "on"}` |
|
||||
| `devices.ascend.tolerations` | Tolerations | `[]` |
|
||||
| `devices.ascend.customresources` | Custom resources | `["huawei.com/Ascend910A", "huawei.com/Ascend910A-memory", ...]` |
|
||||
|
||||
### Iluvatar
|
||||
| Parameter | Description | Default Value |
|
||||
|-----------|-------------|---------------|
|
||||
| `devices.iluvatar.enabled` | Whether to enable | `false` |
|
||||
| `devices.iluvatar.customresources` | Custom resources | `["iluvatar.ai/BI-V150-vgpu", "iluvatar.ai/BI-V150.vMem","iluvatar.ai/BI-V150.vCore", ...]` |
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
** Please be patient while the chart is being deployed **
|
||||
Resource name: {{ .Values.resourceName }}
|
||||
|
||||
|
|
@ -1,49 +0,0 @@
|
|||
{{/*
|
||||
Return the proper image name
|
||||
{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" $) }}
|
||||
*/}}
|
||||
{{- define "common.images.image" -}}
|
||||
{{- $registryName := .imageRoot.registry -}}
|
||||
{{- $repositoryName := .imageRoot.repository -}}
|
||||
{{- $tag := .imageRoot.tag | toString -}}
|
||||
{{- if .global }}
|
||||
{{- if .global.imageRegistry }}
|
||||
{{- $registryName = .global.imageRegistry -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- if .tag }}
|
||||
{{- $tag = .tag | toString -}}
|
||||
{{- end -}}
|
||||
{{- if $registryName }}
|
||||
{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}}
|
||||
{{- else -}}
|
||||
{{- printf "%s:%s" $repositoryName $tag -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead)
|
||||
{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }}
|
||||
*/}}
|
||||
{{- define "common.images.pullSecrets" -}}
|
||||
{{- $pullSecrets := list }}
|
||||
|
||||
{{- if .global }}
|
||||
{{- range .global.imagePullSecrets -}}
|
||||
{{- $pullSecrets = append $pullSecrets . -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- range .images -}}
|
||||
{{- range .pullSecrets -}}
|
||||
{{- $pullSecrets = append $pullSecrets . -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- if (not (empty $pullSecrets)) }}
|
||||
imagePullSecrets:
|
||||
{{- range $pullSecrets }}
|
||||
- name: {{ . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
|
|
@ -1,163 +0,0 @@
|
|||
{{/*
|
||||
Expand the name of the chart.
|
||||
*/}}
|
||||
{{- define "hami-vgpu.name" -}}
|
||||
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create a default fully qualified app name.
|
||||
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
|
||||
If release name contains chart name it will be used as a full name.
|
||||
*/}}
|
||||
{{- define "hami-vgpu.fullname" -}}
|
||||
{{- if .Values.fullnameOverride -}}
|
||||
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
|
||||
{{- else }}
|
||||
{{- $name := default .Chart.Name .Values.nameOverride -}}
|
||||
{{- if contains $name .Release.Name }}
|
||||
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
|
||||
{{- else -}}
|
||||
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Allow the release namespace to be overridden for multi-namespace deployments in combined charts
|
||||
*/}}
|
||||
{{- define "hami-vgpu.namespace" -}}
|
||||
{{- if .Values.namespaceOverride -}}
|
||||
{{- .Values.namespaceOverride -}}
|
||||
{{- else -}}
|
||||
{{- .Release.Namespace -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
The app name for Scheduler
|
||||
*/}}
|
||||
{{- define "hami-vgpu.scheduler" -}}
|
||||
{{- printf "%s-scheduler" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
The app name for DevicePlugin
|
||||
*/}}
|
||||
{{- define "hami-vgpu.device-plugin" -}}
|
||||
{{- printf "%s-device-plugin" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
The app name for MockDevicePlugin
|
||||
*/}}
|
||||
{{- define "hami-vgpu.mock-device-plugin" -}}
|
||||
{{- printf "%s-mock-device-plugin" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
The tls secret name for Scheduler
|
||||
*/}}
|
||||
{{- define "hami-vgpu.scheduler.tls" -}}
|
||||
{{- printf "%s-scheduler-tls" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
The webhook name
|
||||
*/}}
|
||||
{{- define "hami-vgpu.scheduler.webhook" -}}
|
||||
{{- printf "%s-webhook" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create chart name and version as used by the chart label.
|
||||
*/}}
|
||||
{{- define "hami-vgpu.chart" -}}
|
||||
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Common labels
|
||||
*/}}
|
||||
{{- define "hami-vgpu.labels" -}}
|
||||
helm.sh/chart: {{ include "hami-vgpu.chart" . }}
|
||||
{{ include "hami-vgpu.selectorLabels" . }}
|
||||
{{- if .Chart.AppVersion }}
|
||||
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
|
||||
{{- end }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Selector labels
|
||||
*/}}
|
||||
{{- define "hami-vgpu.selectorLabels" -}}
|
||||
app.kubernetes.io/name: {{ include "hami-vgpu.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
{{- end }}
|
||||
|
||||
|
||||
{{/*
|
||||
Resolve the tag for kubeScheduler.
|
||||
*/}}
|
||||
{{- define "resolvedKubeSchedulerTag" -}}
|
||||
{{- if .Values.scheduler.kubeScheduler.image.tag }}
|
||||
{{- .Values.scheduler.kubeScheduler.image.tag | trim -}}
|
||||
{{- else }}
|
||||
{{- include "strippedKubeVersion" . | trim -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Return the stripped Kubernetes version string by removing extra parts after semantic version number.
|
||||
v1.31.1+k3s1 -> v1.31.1
|
||||
v1.30.8-eks-2d5f260 -> v1.30.8
|
||||
v1.31.1 -> v1.31.1
|
||||
*/}}
|
||||
{{- define "strippedKubeVersion" -}}
|
||||
{{ regexReplaceAll "^(v[0-9]+\\.[0-9]+\\.[0-9]+)(.*)$" .Capabilities.KubeVersion.Version "$1" }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.scheduler.kubeScheduler.image" -}}
|
||||
{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.kubeScheduler.image "global" .Values.global "tag" (include "resolvedKubeSchedulerTag" .)) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.scheduler.extender.image" -}}
|
||||
{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.extender.image "global" .Values.global "tag" .Values.global.imageTag) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.devicePlugin.image" -}}
|
||||
{{ include "common.images.image" (dict "imageRoot" .Values.devicePlugin.image "global" .Values.global "tag" .Values.global.imageTag) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.mockDevicePlugin.image" -}}
|
||||
{{ include "common.images.image" (dict "imageRoot" .Values.mockDevicePlugin.image "global" .Values.global "tag" .Values.mockDevicePlugin.tag) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.devicePlugin.monitor.image" -}}
|
||||
{{ include "common.images.image" (dict "imageRoot" .Values.devicePlugin.monitor.image "global" .Values.global "tag" .Values.global.imageTag) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.scheduler.patch.image" -}}
|
||||
{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.patch.image "global" .Values.global) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.scheduler.patch.new.image" -}}
|
||||
{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.patch.imageNew "global" .Values.global) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.scheduler.extender.imagePullSecrets" -}}
|
||||
{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.extender.image) "global" .Values.global) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.devicePlugin.imagePullSecrets" -}}
|
||||
{{ include "common.images.pullSecrets" (dict "images" (list .Values.devicePlugin.image) "global" .Values.global) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.scheduler.patch.imagePullSecrets" -}}
|
||||
{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.patch.image) "global" .Values.global) }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "hami.scheduler.patch.new.imagePullSecrets" -}}
|
||||
{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.patch.imageNew) "global" .Values.global) }}
|
||||
{{- end -}}
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
{{- if and .Values.devicePlugin.enabled (not .Values.devicePlugin.nodeConfiguration.externalConfigName) -}}
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.device-plugin" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-device-plugin
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
data:
|
||||
config.json: |
|
||||
{{- .Values.devicePlugin.nodeConfiguration.config | nindent 4 }}
|
||||
{{- end }}
|
||||
|
|
@ -1,55 +0,0 @@
|
|||
{{- if .Values.mockDevicePlugin.enabled }}
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.mock-device-plugin" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/component: hami-mock-device-plugin
|
||||
{{- include "hami-vgpu.selectorLabels" . | nindent 6 }}
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
scheduler.alpha.kubernetes.io/critical-pod: ""
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-mock-device-plugin
|
||||
{{- include "hami-vgpu.selectorLabels" . | nindent 8 }}
|
||||
spec:
|
||||
serviceAccountName: {{ include "hami-vgpu.mock-device-plugin" . }}
|
||||
tolerations:
|
||||
- key: CriticalAddonsOnly
|
||||
operator: Exists
|
||||
containers:
|
||||
- image: {{ include "hami.mockDevicePlugin.image" . }}
|
||||
imagePullPolicy: {{ .Values.mockDevicePlugin.image.pullPolicy }}
|
||||
name: hami-mock-dp-cntr
|
||||
env:
|
||||
- name: NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
command:
|
||||
- ./k8s-device-plugin
|
||||
- -v=5
|
||||
- --device-config-file=/device-config.yaml
|
||||
volumeMounts:
|
||||
- name: dp
|
||||
mountPath: /var/lib/kubelet/device-plugins
|
||||
- name: sys
|
||||
mountPath: /sys
|
||||
- name: device-config
|
||||
mountPath: /device-config.yaml
|
||||
subPath: device-config.yaml
|
||||
volumes:
|
||||
- name: dp
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/device-plugins
|
||||
- name: sys
|
||||
hostPath:
|
||||
path: /sys
|
||||
- name: device-config
|
||||
configMap:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}-device
|
||||
{{- end -}}
|
||||
|
|
@ -1,262 +0,0 @@
|
|||
{{- if .Values.devicePlugin.enabled }}
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.device-plugin" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-device-plugin
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
{{- with .Values.global.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.annotations }}
|
||||
annotations: {{ toYaml .Values.global.annotations | nindent 4}}
|
||||
{{- end }}
|
||||
spec:
|
||||
updateStrategy:
|
||||
{{- with .Values.devicePlugin.updateStrategy }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/component: hami-device-plugin
|
||||
{{- include "hami-vgpu.selectorLabels" . | nindent 6 }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-device-plugin
|
||||
hami.io/webhook: ignore
|
||||
{{- include "hami-vgpu.selectorLabels" . | nindent 8 }}
|
||||
annotations:
|
||||
{{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
|
||||
checksum/hami-scheduler-newversion-config: {{ include (print $.Template.BasePath "/scheduler/configmapnew.yaml") . | sha256sum }}
|
||||
{{- else }}
|
||||
checksum/hami-scheduler-config: {{ include (print $.Template.BasePath "/scheduler/configmap.yaml") . | sha256sum }}
|
||||
{{- end }}
|
||||
checksum/hami-scheduler-device-config: {{ include (print $.Template.BasePath "/scheduler/device-configmap.yaml") . | sha256sum }}
|
||||
{{- if .Values.devicePlugin.podAnnotations }}
|
||||
{{- toYaml .Values.devicePlugin.podAnnotations | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if .Values.devicePlugin.runtimeClassName }}
|
||||
runtimeClassName: {{ .Values.devicePlugin.runtimeClassName }}
|
||||
{{- end }}
|
||||
serviceAccountName: {{ include "hami-vgpu.device-plugin" . }}
|
||||
priorityClassName: system-node-critical
|
||||
hostPID: true
|
||||
hostNetwork: true
|
||||
{{- include "hami.devicePlugin.imagePullSecrets" . | nindent 6 }}
|
||||
{{- if .Values.devicePlugin.gpuOperatorToolkitReady.enabled }}
|
||||
initContainers:
|
||||
- name: toolkit-validation
|
||||
image: {{ include "hami.devicePlugin.image" . }}
|
||||
imagePullPolicy: {{ .Values.devicePlugin.image.pullPolicy }}
|
||||
securityContext:
|
||||
privileged: true
|
||||
runAsUser: 0
|
||||
command: ["sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
echo "Waiting for NVIDIA Toolkit to be ready..."
|
||||
until [ -f {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }}/toolkit-ready ]; do
|
||||
echo "Waiting for {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }}/toolkit-ready..."
|
||||
sleep 5
|
||||
done
|
||||
echo "NVIDIA Toolkit is ready!"
|
||||
volumeMounts:
|
||||
- name: nvidia-validations
|
||||
mountPath: {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath | quote }}
|
||||
mountPropagation: HostToContainer
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: device-plugin
|
||||
image: {{ include "hami.devicePlugin.image" . }}
|
||||
imagePullPolicy: {{ .Values.devicePlugin.image.pullPolicy }}
|
||||
lifecycle:
|
||||
postStart:
|
||||
exec:
|
||||
command: ["/bin/sh","-c", {{ printf "/k8s-vgpu/bin/vgpu-init.sh %s/vgpu/" .Values.global.gpuHookPath | quote }}]
|
||||
command:
|
||||
- nvidia-device-plugin
|
||||
- --config-file=/device-config.yaml
|
||||
- --mig-strategy={{ .Values.devicePlugin.migStrategy }}
|
||||
- --disable-core-limit={{ .Values.devicePlugin.disablecorelimit }}
|
||||
{{- range .Values.devicePlugin.extraArgs }}
|
||||
- {{ . }}
|
||||
{{- end }}
|
||||
env:
|
||||
- name: NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
- name: NVIDIA_MIG_MONITOR_DEVICES
|
||||
value: all
|
||||
- name: DEVICE_LIST_STRATEGY
|
||||
value: {{ .Values.devicePlugin.deviceListStrategy }}
|
||||
- name: HOOK_PATH
|
||||
value: {{ .Values.global.gpuHookPath }}
|
||||
{{- if typeIs "bool" .Values.devicePlugin.passDeviceSpecsEnabled }}
|
||||
- name: PASS_DEVICE_SPECS
|
||||
value: {{ .Values.devicePlugin.passDeviceSpecsEnabled | quote }}
|
||||
{{- end }}
|
||||
{{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }}
|
||||
- name: NVIDIA_DRIVER_ROOT
|
||||
value: {{ .Values.devicePlugin.nvidiaDriverRoot }}
|
||||
{{- end }}
|
||||
{{- if typeIs "string" .Values.devicePlugin.nvidiaHookPath }}
|
||||
- name: NVIDIA_CDI_HOOK_PATH
|
||||
value: {{ .Values.devicePlugin.nvidiaHookPath }}
|
||||
{{- end }}
|
||||
{{- if typeIs "bool" .Values.devicePlugin.gdrcopyEnabled }}
|
||||
- name: GDRCOPY_ENABLED
|
||||
value: {{ .Values.devicePlugin.gdrcopyEnabled | quote }}
|
||||
{{- end }}
|
||||
{{- if typeIs "bool" .Values.devicePlugin.gdsEnabled }}
|
||||
- name: GDS_ENABLED
|
||||
value: {{ .Values.devicePlugin.gdsEnabled | quote }}
|
||||
{{- end }}
|
||||
{{- if typeIs "bool" .Values.devicePlugin.mofedEnabled }}
|
||||
- name: MOFED_ENABLED
|
||||
value: {{ .Values.devicePlugin.mofedEnabled | quote }}
|
||||
{{- end }}
|
||||
{{- if eq (.Values.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy | default "spread") "topology-aware" }}
|
||||
- name: ENABLE_TOPOLOGY_SCORE
|
||||
value: "true"
|
||||
{{- end }}
|
||||
{{- with .Values.devicePlugin.extraEnvs }}
|
||||
{{- . | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
securityContext:
|
||||
privileged: true
|
||||
allowPrivilegeEscalation: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
add: ["SYS_ADMIN"]
|
||||
resources:
|
||||
{{- toYaml .Values.devicePlugin.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
- name: device-plugin
|
||||
mountPath: /var/lib/kubelet/device-plugins
|
||||
- name: lib
|
||||
mountPath: {{ printf "%s%s" .Values.global.gpuHookPath "/vgpu" }}
|
||||
- name: usrbin
|
||||
mountPath: /usrbin
|
||||
- name: deviceconfig
|
||||
mountPath: /config
|
||||
- name: hosttmp
|
||||
mountPath: /tmp
|
||||
- name: device-config
|
||||
mountPath: /device-config.yaml
|
||||
subPath: device-config.yaml
|
||||
- name: cdi-root
|
||||
mountPath: /var/run/cdi
|
||||
{{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }}
|
||||
# We always mount the driver root at /driver-root in the container.
|
||||
# This is required for CDI detection to work correctly.
|
||||
- name: driver-root
|
||||
mountPath: /driver-root
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
- name: vgpu-monitor
|
||||
image: {{ include "hami.devicePlugin.monitor.image" . }}
|
||||
imagePullPolicy: {{ .Values.devicePlugin.monitor.image.pullPolicy }}
|
||||
command:
|
||||
- "vGPUmonitor"
|
||||
{{- range .Values.devicePlugin.monitor.extraArgs }}
|
||||
- {{ . }}
|
||||
{{- end }}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
add: ["SYS_ADMIN"]
|
||||
env:
|
||||
- name: NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
- name: NVIDIA_VISIBLE_DEVICES
|
||||
value: "all"
|
||||
- name: NVIDIA_MIG_MONITOR_DEVICES
|
||||
value: "all"
|
||||
- name: HOOK_PATH
|
||||
value: "{{ .Values.global.gpuHookPath }}/vgpu"
|
||||
- name: HAMI_RESYNC_INTERVAL
|
||||
value: {{ .Values.devicePlugin.monitor.resyncInterval | default "5m" | quote }}
|
||||
{{- with .Values.devicePlugin.monitor.extraEnvs }}
|
||||
{{- . | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
resources:
|
||||
{{- toYaml .Values.devicePlugin.monitor.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
- name: ctrs
|
||||
mountPath: {{ .Values.devicePlugin.monitor.ctrPath }}
|
||||
- name: dockers
|
||||
mountPath: /run/docker
|
||||
- name: containerds
|
||||
mountPath: /run/containerd
|
||||
- name: sysinfo
|
||||
mountPath: /sysinfo
|
||||
- name: hostvar
|
||||
mountPath: /hostvar
|
||||
- name: hosttmp
|
||||
mountPath: /tmp
|
||||
volumes:
|
||||
- name: ctrs
|
||||
hostPath:
|
||||
path: {{ .Values.devicePlugin.monitor.ctrPath }}
|
||||
- name: hosttmp
|
||||
hostPath:
|
||||
path: /tmp
|
||||
- name: dockers
|
||||
hostPath:
|
||||
path: /run/docker
|
||||
- name: containerds
|
||||
hostPath:
|
||||
path: /run/containerd
|
||||
- name: device-plugin
|
||||
hostPath:
|
||||
path: {{ .Values.devicePlugin.pluginPath }}
|
||||
- name: lib
|
||||
hostPath:
|
||||
path: {{ .Values.devicePlugin.libPath }}
|
||||
{{- if .Values.devicePlugin.gpuOperatorToolkitReady.enabled }}
|
||||
- name: nvidia-validations
|
||||
hostPath:
|
||||
path: {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }}
|
||||
type: DirectoryOrCreate
|
||||
{{- end }}
|
||||
{{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }}
|
||||
- name: driver-root
|
||||
hostPath:
|
||||
path: {{ .Values.devicePlugin.nvidiaDriverRoot }}
|
||||
type: Directory
|
||||
{{- end }}
|
||||
- name: cdi-root
|
||||
hostPath:
|
||||
path: /var/run/cdi
|
||||
type: DirectoryOrCreate
|
||||
- name: usrbin
|
||||
hostPath:
|
||||
path: /usr/bin
|
||||
- name: sysinfo
|
||||
hostPath:
|
||||
path: /sys
|
||||
- name: hostvar
|
||||
hostPath:
|
||||
path: /var
|
||||
- name: deviceconfig
|
||||
configMap:
|
||||
name: {{ .Values.devicePlugin.nodeConfiguration.externalConfigName | default (include "hami-vgpu.device-plugin" .) }}
|
||||
- name: device-config
|
||||
configMap:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}-device
|
||||
{{- if .Values.devicePlugin.nvidiaNodeSelector }}
|
||||
nodeSelector: {{ toYaml .Values.devicePlugin.nvidiaNodeSelector | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.devicePlugin.tolerations }}
|
||||
tolerations: {{ toYaml .Values.devicePlugin.tolerations | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -1,28 +0,0 @@
|
|||
{{- if .Values.devicePlugin.enabled -}}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.device-plugin" . }}-monitor
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- pods
|
||||
verbs:
|
||||
- get
|
||||
- create
|
||||
- watch
|
||||
- list
|
||||
- update
|
||||
- patch
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- nodes
|
||||
verbs:
|
||||
- get
|
||||
- update
|
||||
- list
|
||||
- patch
|
||||
{{- end -}}
|
||||
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
{{- if .Values.devicePlugin.enabled -}}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.device-plugin" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: "hami-device-plugin"
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: {{ include "hami-vgpu.device-plugin" . }}-monitor
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "hami-vgpu.device-plugin" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
{{- end -}}
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
{{- if .Values.devicePlugin.enabled -}}
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.device-plugin" . }}-monitor
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-device-plugin
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
{{- with .Values.devicePlugin.service.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- if .Values.devicePlugin.service.annotations }} # Use devicePlugin instead of scheduler
|
||||
annotations: {{ toYaml .Values.devicePlugin.service.annotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
type: {{ .Values.devicePlugin.service.type | default "NodePort" }} # Default type is NodePort
|
||||
ports:
|
||||
- name: monitorport
|
||||
port: {{ .Values.devicePlugin.service.httpPort | default 31992 }} # Default HTTP port is 31992
|
||||
targetPort: 9394
|
||||
{{- if eq (.Values.devicePlugin.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort
|
||||
nodePort: {{ .Values.devicePlugin.service.httpPort | default 31992 }}
|
||||
{{- end }}
|
||||
protocol: TCP
|
||||
selector:
|
||||
app.kubernetes.io/component: hami-device-plugin
|
||||
{{- include "hami-vgpu.selectorLabels" . | nindent 4 }}
|
||||
{{- end -}}
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
{{- if .Values.devicePlugin.enabled -}}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.device-plugin" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: "hami-device-plugin"
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
{{- end -}}
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
{{- if and .Values.devicePlugin.enabled .Values.devicePlugin.createRuntimeClass .Values.devicePlugin.runtimeClassName -}}
|
||||
apiVersion: node.k8s.io/v1
|
||||
kind: RuntimeClass
|
||||
metadata:
|
||||
name: {{ .Values.devicePlugin.runtimeClassName }}
|
||||
annotations:
|
||||
helm.sh/hook: pre-install,pre-upgrade
|
||||
handler: nvidia
|
||||
{{- end -}}
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
{{- if .Values.scheduler.admissionWebhook.enabled -}}
|
||||
{{- if .Values.scheduler.certManager.enabled }}
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}-serving-cert
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-scheduler
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
spec:
|
||||
dnsNames:
|
||||
- {{ include "hami-vgpu.scheduler" . }}.{{ include "hami-vgpu.namespace" . }}.svc
|
||||
- {{ include "hami-vgpu.scheduler" . }}.{{ include "hami-vgpu.namespace" . }}.svc.cluster.local
|
||||
issuerRef:
|
||||
kind: Issuer
|
||||
name: {{ include "hami-vgpu.scheduler" . }}-selfsigned-issuer
|
||||
secretName: {{ include "hami-vgpu.scheduler.tls" . }}
|
||||
---
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Issuer
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}-selfsigned-issuer
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-scheduler
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
spec:
|
||||
selfSigned: {}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: "hami-scheduler"
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["pods", "configmaps"]
|
||||
verbs: ["get", "list", "watch", "patch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["pods/binding"]
|
||||
verbs: ["create"]
|
||||
- apiGroups: [""]
|
||||
resources: ["nodes"]
|
||||
verbs: ["get", "list", "patch", "watch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["events"]
|
||||
verbs: ["create", "get", "list"]
|
||||
- apiGroups: [""]
|
||||
resources: ["resourcequotas"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
---
|
||||
{{- if .Values.mockDevicePlugin.enabled }}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.mock-device-plugin" . }}
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["nodes"]
|
||||
verbs: ["get", "update", "list", "patch"]
|
||||
{{- end -}}
|
||||
|
|
@ -1,47 +0,0 @@
|
|||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}-kube
|
||||
labels:
|
||||
app.kubernetes.io/component: "hami-scheduler"
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: system:kube-scheduler
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}-volume
|
||||
labels:
|
||||
app.kubernetes.io/component: "hami-scheduler"
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: system:volume-scheduler
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: "hami-scheduler"
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
|
|
@ -1,142 +0,0 @@
|
|||
{{- if .Values.scheduler.kubeScheduler.enabled -}}
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-scheduler
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
data:
|
||||
config.json: |
|
||||
{
|
||||
"kind": "Policy",
|
||||
"apiVersion": "v1",
|
||||
"extenders": [
|
||||
{
|
||||
{{- if .Values.scheduler.admissionWebhook.enabled }}
|
||||
"urlPrefix": "https://127.0.0.1:443",
|
||||
"enableHttps": true,
|
||||
"tlsConfig": {
|
||||
"insecure": true
|
||||
},
|
||||
{{- else }}
|
||||
"urlPrefix": "http://127.0.0.1:80",
|
||||
"enableHttps": false,
|
||||
{{- end }}
|
||||
"filterVerb": "filter",
|
||||
"bindVerb": "bind",
|
||||
"weight": 1,
|
||||
"nodeCacheCapable": true,
|
||||
"httpTimeout": 30000000000,
|
||||
"managedResources": [
|
||||
{{- range .Values.devices.amd.customresources }}
|
||||
{
|
||||
"name": "{{ . }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{{- end }}
|
||||
{{- if .Values.devices.ascend.enabled }}
|
||||
{{- range .Values.devices.ascend.customresources }}
|
||||
{
|
||||
"name": "{{ . }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.devices.mthreads.enabled }}
|
||||
{{- range .Values.devices.mthreads.customresources }}
|
||||
{
|
||||
"name": "{{ . }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.devices.enflame.enabled }}
|
||||
{{- range .Values.devices.enflame.customresources }}
|
||||
{
|
||||
"name": "{{ . }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.devices.kunlun.enabled }}
|
||||
{{- range .Values.devices.kunlun.customresources }}
|
||||
{
|
||||
"name": "{{ . }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- range .Values.devices.awsneuron.customresources }}
|
||||
{
|
||||
"name": "{{ . }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{{- end }}
|
||||
{{- if .Values.devices.iluvatar.enabled }}
|
||||
{{- range .Values.devices.iluvatar.customresources }}
|
||||
{
|
||||
"name": "{{ . }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{
|
||||
"name": "{{ .Values.resourceName }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "{{ .Values.resourceMem }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "{{ .Values.resourceCores }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "{{ .Values.resourceMemPercentage }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "{{ .Values.resourcePriority }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "{{ .Values.mluResourceName }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "{{ .Values.dcuResourceName }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "{{ .Values.dcuResourceMem }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "{{ .Values.dcuResourceCores }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "metax-tech.com/gpu",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "{{ .Values.metaxResourceName }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "{{ .Values.metaxResourceCore }}",
|
||||
"ignoredByScheduler": true
|
||||
},
|
||||
{
|
||||
"name": "{{ .Values.metaxResourceMem }}",
|
||||
"ignoredByScheduler": true
|
||||
}
|
||||
],
|
||||
"ignoreable": false
|
||||
}
|
||||
]
|
||||
}
|
||||
{{- end }}
|
||||
|
|
@ -1,102 +0,0 @@
|
|||
{{- if .Values.scheduler.kubeScheduler.enabled -}}
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}-newversion
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-scheduler
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
data:
|
||||
config.yaml: |
|
||||
{{- if gt (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 25}}
|
||||
apiVersion: kubescheduler.config.k8s.io/v1
|
||||
{{- else }}
|
||||
apiVersion: kubescheduler.config.k8s.io/v1beta2
|
||||
{{- end }}
|
||||
kind: KubeSchedulerConfiguration
|
||||
leaderElection:
|
||||
leaderElect: false
|
||||
profiles:
|
||||
- schedulerName: {{ .Values.schedulerName }}
|
||||
extenders:
|
||||
{{- if .Values.scheduler.admissionWebhook.enabled }}
|
||||
- urlPrefix: "https://127.0.0.1:443"
|
||||
enableHTTPS: true
|
||||
tlsConfig:
|
||||
insecure: true
|
||||
{{- else }}
|
||||
- urlPrefix: "http://127.0.0.1:80"
|
||||
enableHTTPS: false
|
||||
{{- end }}
|
||||
filterVerb: filter
|
||||
bindVerb: bind
|
||||
nodeCacheCapable: true
|
||||
weight: 1
|
||||
httpTimeout: 30s
|
||||
managedResources:
|
||||
- name: {{ .Values.resourceName }}
|
||||
ignoredByScheduler: true
|
||||
- name: {{ .Values.resourceMem }}
|
||||
ignoredByScheduler: true
|
||||
- name: {{ .Values.resourceCores }}
|
||||
ignoredByScheduler: true
|
||||
- name: {{ .Values.resourceMemPercentage }}
|
||||
ignoredByScheduler: true
|
||||
- name: {{ .Values.resourcePriority }}
|
||||
ignoredByScheduler: true
|
||||
- name: {{ .Values.mluResourceName }}
|
||||
ignoredByScheduler: true
|
||||
- name: {{ .Values.dcuResourceName }}
|
||||
ignoredByScheduler: true
|
||||
- name: {{ .Values.dcuResourceMem }}
|
||||
ignoredByScheduler: true
|
||||
- name: {{ .Values.dcuResourceCores }}
|
||||
ignoredByScheduler: true
|
||||
- name: "metax-tech.com/gpu"
|
||||
ignoredByScheduler: true
|
||||
- name: {{ .Values.metaxResourceName }}
|
||||
ignoredByScheduler: true
|
||||
- name: {{ .Values.metaxResourceCore }}
|
||||
ignoredByScheduler: true
|
||||
- name: {{ .Values.metaxResourceMem }}
|
||||
ignoredByScheduler: true
|
||||
{{- if .Values.devices.ascend.enabled }}
|
||||
{{- range .Values.devices.ascend.customresources }}
|
||||
- name: {{ . }}
|
||||
ignoredByScheduler: true
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.devices.mthreads.enabled }}
|
||||
{{- range .Values.devices.mthreads.customresources }}
|
||||
- name: {{ . }}
|
||||
ignoredByScheduler: true
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.devices.enflame.enabled }}
|
||||
{{- range .Values.devices.enflame.customresources }}
|
||||
- name: {{ . }}
|
||||
ignoredByScheduler: true
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.devices.kunlun.enabled }}
|
||||
{{- range .Values.devices.kunlun.customresources }}
|
||||
- name: {{ . }}
|
||||
ignoredByScheduler: true
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- range .Values.devices.awsneuron.customresources }}
|
||||
- name: {{ . }}
|
||||
ignoredByScheduler: true
|
||||
{{- end }}
|
||||
{{- if .Values.devices.iluvatar.enabled }}
|
||||
{{- range .Values.devices.iluvatar.customresources }}
|
||||
- name: {{ . }}
|
||||
ignoredByScheduler: true
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- range .Values.devices.amd.customresources }}
|
||||
- name: {{ . }}
|
||||
ignoredByScheduler: true
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -1,225 +0,0 @@
|
|||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-scheduler
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
{{- with .Values.global.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.annotations }}
|
||||
annotations: {{ toYaml .Values.global.annotations | nindent 4}}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if .Values.scheduler.leaderElect }}
|
||||
replicas: {{ .Values.scheduler.replicas }}
|
||||
{{- else }}
|
||||
replicas: 1
|
||||
{{- end }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/component: hami-scheduler
|
||||
{{- include "hami-vgpu.selectorLabels" . | nindent 6 }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-scheduler
|
||||
{{- include "hami-vgpu.selectorLabels" . | nindent 8 }}
|
||||
hami.io/webhook: ignore
|
||||
annotations:
|
||||
{{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
|
||||
checksum/hami-scheduler-newversion-config: {{ include (print $.Template.BasePath "/scheduler/configmapnew.yaml") . | sha256sum }}
|
||||
{{- else }}
|
||||
checksum/hami-scheduler-config: {{ include (print $.Template.BasePath "/scheduler/configmap.yaml") . | sha256sum }}
|
||||
{{- end }}
|
||||
checksum/hami-scheduler-device-config: {{ include (print $.Template.BasePath "/scheduler/device-configmap.yaml") . | sha256sum }}
|
||||
{{- if .Values.scheduler.podAnnotations }}
|
||||
{{- toYaml .Values.scheduler.podAnnotations | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
serviceAccountName: {{ include "hami-vgpu.scheduler" . }}
|
||||
priorityClassName: system-node-critical
|
||||
{{- include "hami.scheduler.extender.imagePullSecrets" . | nindent 6 }}
|
||||
containers:
|
||||
{{- if .Values.scheduler.kubeScheduler.enabled }}
|
||||
- name: kube-scheduler
|
||||
image: {{ include "hami.scheduler.kubeScheduler.image" . }}
|
||||
imagePullPolicy: {{ .Values.scheduler.kubeScheduler.image.pullPolicy }}
|
||||
command:
|
||||
- kube-scheduler
|
||||
{{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
|
||||
{{- range .Values.scheduler.kubeScheduler.extraNewArgs }}
|
||||
- {{ . }}
|
||||
{{- end }}
|
||||
{{- else }}
|
||||
- --scheduler-name={{ .Values.schedulerName }}
|
||||
{{- range .Values.scheduler.kubeScheduler.extraArgs }}
|
||||
- {{ . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
- --leader-elect={{ .Values.scheduler.leaderElect }}
|
||||
- --leader-elect-resource-name={{ .Values.schedulerName }}
|
||||
- --leader-elect-resource-namespace={{ include "hami-vgpu.namespace" . }}
|
||||
resources:
|
||||
{{- toYaml .Values.scheduler.kubeScheduler.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
- name: scheduler-config
|
||||
mountPath: /config
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.livenessProbe }}
|
||||
livenessProbe:
|
||||
failureThreshold: 8
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: 10259
|
||||
scheme: HTTPS
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
successThreshold: 1
|
||||
timeoutSeconds: 15
|
||||
{{- end }}
|
||||
- name: vgpu-scheduler-extender
|
||||
image: {{ include "hami.scheduler.extender.image" . }}
|
||||
imagePullPolicy: {{ .Values.scheduler.extender.image.pullPolicy }}
|
||||
env:
|
||||
{{- if .Values.scheduler.nodeLockExpire }}
|
||||
- name: HAMI_NODELOCK_EXPIRE
|
||||
value: "{{ .Values.scheduler.nodeLockExpire }}"
|
||||
{{- end }}
|
||||
{{- if .Values.global.managedNodeSelectorEnable }}
|
||||
{{- range $key, $value := .Values.global.managedNodeSelector }}
|
||||
- name: NODE_SELECTOR_{{ $key | upper | replace "-" "_" }}
|
||||
value: "{{ $value }}"
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
command:
|
||||
- scheduler
|
||||
{{- if .Values.scheduler.admissionWebhook.enabled }}
|
||||
- --http_bind=0.0.0.0:443
|
||||
- --cert_file=/tls/tls.crt
|
||||
- --key_file=/tls/tls.key
|
||||
{{- else }}
|
||||
- --http_bind=0.0.0.0:80
|
||||
{{- end }}
|
||||
- --scheduler-name={{ .Values.schedulerName }}
|
||||
- --metrics-bind-address={{ .Values.scheduler.metricsBindAddress }}
|
||||
- --node-scheduler-policy={{ .Values.scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy }}
|
||||
- --gpu-scheduler-policy={{ .Values.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy }}
|
||||
- --force-overwrite-default-scheduler={{ .Values.scheduler.forceOverwriteDefaultScheduler}}
|
||||
- --device-config-file=/device-config.yaml
|
||||
- --leader-elect={{ .Values.scheduler.leaderElect }}
|
||||
- --leader-elect-resource-name={{ .Values.schedulerName }}
|
||||
- --leader-elect-resource-namespace={{ include "hami-vgpu.namespace" . }}
|
||||
{{- if .Values.devices.ascend.enabled }}
|
||||
- --enable-ascend=true
|
||||
{{- end }}
|
||||
{{- if .Values.devices.iluvatar.enabled }}
|
||||
- --enable-iluvatar=true
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.nodeLabelSelector }}
|
||||
- --node-label-selector={{- $first := true -}}
|
||||
{{- range $key, $value := .Values.scheduler.nodeLabelSelector -}}
|
||||
{{- if not $first }},{{ end -}}
|
||||
{{- $key }}={{ $value -}}
|
||||
{{- $first = false -}}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- range .Values.scheduler.extender.extraArgs }}
|
||||
- {{ . }}
|
||||
{{- end }}
|
||||
ports:
|
||||
{{- if .Values.scheduler.admissionWebhook.enabled }}
|
||||
- name: https
|
||||
containerPort: 443
|
||||
protocol: TCP
|
||||
{{- else }}
|
||||
- name: http
|
||||
containerPort: 80
|
||||
protocol: TCP
|
||||
{{- end }}
|
||||
- name: metrics
|
||||
containerPort: {{ last (splitList ":" .Values.scheduler.metricsBindAddress) | int }}
|
||||
protocol: TCP
|
||||
resources:
|
||||
{{- toYaml .Values.scheduler.extender.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
- name: device-config
|
||||
mountPath: /device-config.yaml
|
||||
subPath: device-config.yaml
|
||||
{{- if .Values.scheduler.admissionWebhook.enabled }}
|
||||
- name: tls-config
|
||||
mountPath: /tls
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.livenessProbe }}
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
{{- if .Values.scheduler.admissionWebhook.enabled }}
|
||||
port: https
|
||||
scheme: HTTPS
|
||||
{{- else }}
|
||||
port: http
|
||||
scheme: HTTP
|
||||
{{- end }}
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
failureThreshold: 3
|
||||
timeoutSeconds: 5
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.leaderElect }}
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /readyz
|
||||
{{- if .Values.scheduler.admissionWebhook.enabled }}
|
||||
port: https
|
||||
scheme: HTTPS
|
||||
{{- else }}
|
||||
port: http
|
||||
scheme: HTTP
|
||||
{{- end }}
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
failureThreshold: 3
|
||||
timeoutSeconds: 5
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.leaderElect }}
|
||||
affinity:
|
||||
podAntiAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
- labelSelector:
|
||||
matchExpressions:
|
||||
- key: app.kubernetes.io/component
|
||||
operator: In
|
||||
values:
|
||||
- hami-scheduler
|
||||
topologyKey: "kubernetes.io/hostname"
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- if .Values.scheduler.admissionWebhook.enabled }}
|
||||
- name: tls-config
|
||||
secret:
|
||||
secretName: {{ template "hami-vgpu.scheduler.tls" . }}
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.kubeScheduler.enabled }}
|
||||
- name: scheduler-config
|
||||
configMap:
|
||||
{{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
|
||||
name: {{ template "hami-vgpu.scheduler" . }}-newversion
|
||||
{{- else }}
|
||||
name: {{ template "hami-vgpu.scheduler" . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
- name: device-config
|
||||
configMap:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}-device
|
||||
{{- if .Values.scheduler.nodeSelector }}
|
||||
nodeSelector: {{ toYaml .Values.scheduler.nodeSelector | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.tolerations }}
|
||||
tolerations: {{ toYaml .Values.scheduler.tolerations | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.nodeName }}
|
||||
nodeName: {{ .Values.scheduler.nodeName }}
|
||||
{{- end }}
|
||||
|
|
@ -1,408 +0,0 @@
|
|||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}-device
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-scheduler
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
data:
|
||||
device-config.yaml: |-
|
||||
{{- if .Files.Glob "files/device-config.yaml" }}
|
||||
{{- .Files.Get "files/device-config.yaml" | nindent 4}}
|
||||
{{- else }}
|
||||
nvidia:
|
||||
resourceCountName: {{ .Values.resourceName }}
|
||||
resourceMemoryName: {{ .Values.resourceMem }}
|
||||
resourceMemoryPercentageName: {{ .Values.resourceMemPercentage }}
|
||||
resourceCoreName: {{ .Values.resourceCores }}
|
||||
resourcePriorityName: {{ .Values.resourcePriority }}
|
||||
overwriteEnv: false
|
||||
defaultMemory: 0
|
||||
defaultCores: 0
|
||||
defaultGPUNum: 1
|
||||
memoryFactor: 1
|
||||
deviceSplitCount: {{ .Values.devicePlugin.deviceSplitCount }}
|
||||
deviceMemoryScaling: {{ .Values.devicePlugin.deviceMemoryScaling }}
|
||||
deviceCoreScaling: {{ .Values.devicePlugin.deviceCoreScaling }}
|
||||
gpuCorePolicy: {{ .Values.devices.nvidia.gpuCorePolicy }}
|
||||
libCudaLogLevel: {{ .Values.devices.nvidia.libCudaLogLevel }}
|
||||
runtimeClassName: "{{ .Values.devicePlugin.runtimeClassName }}"
|
||||
knownMigGeometries:
|
||||
- models: [ "A30" ]
|
||||
allowedGeometries:
|
||||
-
|
||||
- name: 1g.6gb
|
||||
core: 25
|
||||
memory: 6144
|
||||
count: 4
|
||||
-
|
||||
- name: 2g.12gb
|
||||
core: 50
|
||||
memory: 12288
|
||||
count: 2
|
||||
-
|
||||
- name: 4g.24gb
|
||||
core: 100
|
||||
memory: 24576
|
||||
count: 1
|
||||
- models: [ "A100-SXM4-40GB", "A100-40GB-PCIe", "A100-PCIE-40GB"]
|
||||
allowedGeometries:
|
||||
-
|
||||
- name: 1g.5gb
|
||||
core: 14
|
||||
memory: 5120
|
||||
count: 7
|
||||
-
|
||||
- name: 1g.5gb
|
||||
core: 14
|
||||
memory: 5120
|
||||
count: 1
|
||||
- name: 2g.10gb
|
||||
core: 28
|
||||
memory: 10240
|
||||
count: 3
|
||||
-
|
||||
- name: 3g.20gb
|
||||
core: 42
|
||||
memory: 20480
|
||||
count: 2
|
||||
-
|
||||
- name: 7g.40gb
|
||||
core: 100
|
||||
memory: 40960
|
||||
count: 1
|
||||
- models: [ "A100-SXM4-80GB", "A100-80GB-PCIe", "A100-PCIE-80GB"]
|
||||
allowedGeometries:
|
||||
-
|
||||
- name: 1g.10gb
|
||||
core: 14
|
||||
memory: 10240
|
||||
count: 7
|
||||
-
|
||||
- name: 1g.10gb
|
||||
core: 14
|
||||
memory: 10240
|
||||
count: 1
|
||||
- name: 2g.20gb
|
||||
core: 28
|
||||
memory: 20480
|
||||
count: 3
|
||||
-
|
||||
- name: 3g.40gb
|
||||
core: 42
|
||||
memory: 40960
|
||||
count: 2
|
||||
-
|
||||
- name: 7g.79gb
|
||||
core: 100
|
||||
memory: 80896
|
||||
count: 1
|
||||
- models: [ "H100-PCIE-80GB", "H100-SXM5-80GB"]
|
||||
allowedGeometries:
|
||||
-
|
||||
- name: 1g.10gb
|
||||
core: 14
|
||||
memory: 10240
|
||||
count: 7
|
||||
-
|
||||
- name: 1g.10gb
|
||||
core: 14
|
||||
memory: 10240
|
||||
count: 1
|
||||
- name: 2g.20gb
|
||||
core: 28
|
||||
memory: 20480
|
||||
count: 3
|
||||
-
|
||||
- name: 3g.40gb
|
||||
core: 42
|
||||
memory: 40960
|
||||
count: 2
|
||||
-
|
||||
- name: 7g.80gb
|
||||
core: 100
|
||||
memory: 81920
|
||||
count: 1
|
||||
- models: [ "H100-PCIE-94GB", "H100-SXM5-94GB"]
|
||||
allowedGeometries:
|
||||
-
|
||||
- name: 1g.12gb
|
||||
core: 14
|
||||
memory: 12288
|
||||
count: 7
|
||||
-
|
||||
- name: 1g.12gb
|
||||
core: 14
|
||||
memory: 12288
|
||||
count: 1
|
||||
- name: 2g.24gb
|
||||
core: 28
|
||||
memory: 24576
|
||||
count: 3
|
||||
-
|
||||
- name: 3g.47gb
|
||||
core: 42
|
||||
memory: 48128
|
||||
count: 2
|
||||
-
|
||||
- name: 7g.94gb
|
||||
core: 100
|
||||
memory: 96256
|
||||
count: 1
|
||||
- models: [ "H20", "H100 on GH200"]
|
||||
allowedGeometries:
|
||||
-
|
||||
- name: 1g.12gb
|
||||
core: 14
|
||||
memory: 12288
|
||||
count: 7
|
||||
-
|
||||
- name: 1g.12gb
|
||||
core: 14
|
||||
memory: 12288
|
||||
count: 1
|
||||
- name: 2g.24gb
|
||||
core: 28
|
||||
memory: 24576
|
||||
count: 3
|
||||
-
|
||||
- name: 3g.48gb
|
||||
core: 42
|
||||
memory: 49152
|
||||
count: 2
|
||||
-
|
||||
- name: 7g.96gb
|
||||
core: 100
|
||||
memory: 98304
|
||||
count: 1
|
||||
- models: [ "H200 NVL", "H200-SXM5"]
|
||||
allowedGeometries:
|
||||
-
|
||||
- name: 1g.18gb
|
||||
core: 14
|
||||
memory: 18432
|
||||
count: 7
|
||||
-
|
||||
- name: 1g.18gb
|
||||
core: 14
|
||||
memory: 18432
|
||||
count: 1
|
||||
- name: 2g.35gb
|
||||
core: 28
|
||||
memory: 35840
|
||||
count: 3
|
||||
-
|
||||
- name: 3g.71gb
|
||||
core: 42
|
||||
memory: 72704
|
||||
count: 2
|
||||
-
|
||||
- name: 7g.141gb
|
||||
core: 100
|
||||
memory: 144384
|
||||
count: 1
|
||||
- models: [ "B200" ]
|
||||
allowedGeometries:
|
||||
-
|
||||
- name: 1g.23gb
|
||||
core: 14
|
||||
memory: 23552
|
||||
count: 7
|
||||
-
|
||||
- name: 1g.23gb
|
||||
core: 14
|
||||
memory: 23552
|
||||
count: 1
|
||||
- name: 2g.45gb
|
||||
core: 28
|
||||
memory: 46080
|
||||
count: 3
|
||||
-
|
||||
- name: 3g.90gb
|
||||
core: 42
|
||||
memory: 92160
|
||||
count: 2
|
||||
-
|
||||
- name: 7g.180gb
|
||||
core: 100
|
||||
memory: 184320
|
||||
count: 1
|
||||
cambricon:
|
||||
resourceCountName: {{ .Values.mluResourceName }}
|
||||
resourceMemoryName: {{ .Values.mluResourceMem }}
|
||||
resourceCoreName: {{ .Values.mluResourceCores }}
|
||||
hygon:
|
||||
resourceCountName: {{ .Values.dcuResourceName }}
|
||||
resourceMemoryName: {{ .Values.dcuResourceMem }}
|
||||
resourceCoreName: {{ .Values.dcuResourceCores }}
|
||||
memoryFactor: 1
|
||||
metax:
|
||||
resourceCountName: "metax-tech.com/gpu"
|
||||
resourceVCountName: {{ .Values.metaxResourceName }}
|
||||
resourceVMemoryName: {{ .Values.metaxResourceMem }}
|
||||
resourceVCoreName: {{ .Values.metaxResourceCore }}
|
||||
sgpuTopologyAware: {{ .Values.metaxsGPUTopologyAware }}
|
||||
enflame:
|
||||
resourceNameGCU: "enflame.com/gcu"
|
||||
resourceNameVGCU: {{ .Values.enflameResourceNameVGCU }}
|
||||
resourceNameVGCUPercentage: {{ .Values.enflameResourceNameVGCUPercentage }}
|
||||
mthreads:
|
||||
resourceCountName: "mthreads.com/vgpu"
|
||||
resourceMemoryName: "mthreads.com/sgpu-memory"
|
||||
resourceCoreName: "mthreads.com/sgpu-core"
|
||||
iluvatars:
|
||||
- chipName: MR-V100
|
||||
commonWord: MR-V100
|
||||
resourceCountName: iluvatar.ai/MR-V100-vgpu
|
||||
resourceMemoryName: iluvatar.ai/MR-V100.vMem
|
||||
resourceCoreName: iluvatar.ai/MR-V100.vCore
|
||||
- chipName: MR-V50
|
||||
commonWord: MR-V50
|
||||
resourceCountName: iluvatar.ai/MR-V50-vgpu
|
||||
resourceMemoryName: iluvatar.ai/MR-V50.vMem
|
||||
resourceCoreName: iluvatar.ai/MR-V50.vCore
|
||||
- chipName: BI-V150
|
||||
commonWord: BI-V150
|
||||
resourceCountName: iluvatar.ai/BI-V150-vgpu
|
||||
resourceMemoryName: iluvatar.ai/BI-V150.vMem
|
||||
resourceCoreName: iluvatar.ai/BI-V150.vCore
|
||||
- chipName: BI-V100
|
||||
commonWord: BI-V100
|
||||
resourceCountName: iluvatar.ai/BI-V100-vgpu
|
||||
resourceMemoryName: iluvatar.ai/BI-V100.vMem
|
||||
resourceCoreName: iluvatar.ai/BI-V100.vCore
|
||||
kunlun:
|
||||
resourceCountName: {{ .Values.kunlunResourceName }}
|
||||
resourceVCountName: {{ .Values.kunlunResourceVCountName }}
|
||||
resourceVMemoryName: {{ .Values.kunlunResourceVMemoryName }}
|
||||
awsneuron:
|
||||
resourceCountName: "aws.amazon.com/neuron"
|
||||
resourceCoreName: "aws.amazon.com/neuroncore"
|
||||
amd:
|
||||
resourceCountName: "amd.com/gpu"
|
||||
vnpus:
|
||||
- chipName: 910A
|
||||
commonWord: Ascend910A
|
||||
resourceName: huawei.com/Ascend910A
|
||||
resourceMemoryName: huawei.com/Ascend910A-memory
|
||||
memoryAllocatable: 32768
|
||||
memoryCapacity: 32768
|
||||
memoryFactor: 1
|
||||
aiCore: 30
|
||||
templates:
|
||||
- name: vir02
|
||||
memory: 2184
|
||||
aiCore: 2
|
||||
- name: vir04
|
||||
memory: 4369
|
||||
aiCore: 4
|
||||
- name: vir08
|
||||
memory: 8738
|
||||
aiCore: 8
|
||||
- name: vir16
|
||||
memory: 17476
|
||||
aiCore: 16
|
||||
- chipName: 910B2
|
||||
commonWord: Ascend910B2
|
||||
resourceName: huawei.com/Ascend910B2
|
||||
resourceMemoryName: huawei.com/Ascend910B2-memory
|
||||
memoryAllocatable: 65536
|
||||
memoryCapacity: 65536
|
||||
memoryFactor: 1
|
||||
aiCore: 24
|
||||
aiCPU: 6
|
||||
templates:
|
||||
- name: vir03_1c_8g
|
||||
memory: 8192
|
||||
aiCore: 3
|
||||
aiCPU: 1
|
||||
- name: vir06_1c_16g
|
||||
memory: 16384
|
||||
aiCore: 6
|
||||
aiCPU: 1
|
||||
- name: vir12_3c_32g
|
||||
memory: 32768
|
||||
aiCore: 12
|
||||
aiCPU: 3
|
||||
- chipName: 910B3
|
||||
commonWord: Ascend910B3
|
||||
resourceName: huawei.com/Ascend910B3
|
||||
resourceMemoryName: huawei.com/Ascend910B3-memory
|
||||
memoryAllocatable: 65536
|
||||
memoryCapacity: 65536
|
||||
memoryFactor: 1
|
||||
aiCore: 20
|
||||
aiCPU: 7
|
||||
templates:
|
||||
- name: vir05_1c_16g
|
||||
memory: 16384
|
||||
aiCore: 5
|
||||
aiCPU: 1
|
||||
- name: vir10_3c_32g
|
||||
memory: 32768
|
||||
aiCore: 10
|
||||
aiCPU: 3
|
||||
- chipName: 910B4-1
|
||||
commonWord: Ascend910B4-1
|
||||
resourceName: huawei.com/Ascend910B4-1
|
||||
resourceMemoryName: huawei.com/Ascend910B4-1-memory
|
||||
memoryAllocatable: 65536
|
||||
memoryCapacity: 65536
|
||||
memoryFactor: 1
|
||||
aiCore: 20
|
||||
aiCPU: 7
|
||||
templates:
|
||||
# NOTE: Names of vnpu template for 910B4-1 are fixed by Ascend runtime and must not be changed.
|
||||
# The memory is used for scheduling so the correct values must be set.
|
||||
# Template vir05_1c_8g actually provides 16GB memory,
|
||||
- name: vir05_1c_8g
|
||||
memory: 16384
|
||||
aiCore: 5
|
||||
aiCPU: 1
|
||||
# Template vir10_3c_16g actually provides 32GB memory
|
||||
- name: vir10_3c_16g
|
||||
memory: 32768
|
||||
aiCore: 10
|
||||
aiCPU: 3
|
||||
- chipName: 910B4
|
||||
commonWord: Ascend910B4
|
||||
resourceName: huawei.com/Ascend910B4
|
||||
resourceMemoryName: huawei.com/Ascend910B4-memory
|
||||
memoryAllocatable: 32768
|
||||
memoryCapacity: 32768
|
||||
memoryFactor: 1
|
||||
aiCore: 20
|
||||
aiCPU: 7
|
||||
templates:
|
||||
- name: vir05_1c_8g
|
||||
memory: 8192
|
||||
aiCore: 5
|
||||
aiCPU: 1
|
||||
- name: vir10_3c_16g
|
||||
memory: 16384
|
||||
aiCore: 10
|
||||
aiCPU: 3
|
||||
- chipName: 310P3
|
||||
commonWord: Ascend310P
|
||||
resourceName: huawei.com/Ascend310P
|
||||
resourceMemoryName: huawei.com/Ascend310P-memory
|
||||
memoryAllocatable: 21527
|
||||
memoryCapacity: 24576
|
||||
memoryFactor: 1
|
||||
aiCore: 8
|
||||
aiCPU: 7
|
||||
templates:
|
||||
- name: vir01
|
||||
memory: 3072
|
||||
aiCore: 1
|
||||
aiCPU: 1
|
||||
- name: vir02
|
||||
memory: 6144
|
||||
aiCore: 2
|
||||
aiCPU: 2
|
||||
- name: vir04
|
||||
memory: 12288
|
||||
aiCore: 4
|
||||
aiCPU: 4
|
||||
{{ end }}
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
annotations:
|
||||
"helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
labels:
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: admission-webhook
|
||||
rules:
|
||||
- apiGroups:
|
||||
- admissionregistration.k8s.io
|
||||
resources:
|
||||
#- validatingwebhookconfigurations
|
||||
- mutatingwebhookconfigurations
|
||||
verbs:
|
||||
- get
|
||||
- update
|
||||
{{- end }}
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
annotations:
|
||||
"helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
labels:
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: admission-webhook
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
{{- end }}
|
||||
|
|
@ -1,68 +0,0 @@
|
|||
{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}}
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission-create
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
annotations:
|
||||
"helm.sh/hook": pre-install,pre-upgrade
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
labels:
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: admission-webhook
|
||||
spec:
|
||||
{{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }}
|
||||
# Alpha feature since k8s 1.12
|
||||
ttlSecondsAfterFinished: 0
|
||||
{{- end }}
|
||||
template:
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission-create
|
||||
{{- if .Values.scheduler.patch.podAnnotations }}
|
||||
annotations: {{ toYaml .Values.scheduler.patch.podAnnotations | nindent 8 }}
|
||||
{{- end }}
|
||||
labels:
|
||||
{{- include "hami-vgpu.labels" . | nindent 8 }}
|
||||
app.kubernetes.io/component: admission-webhook
|
||||
hami.io/webhook: ignore
|
||||
spec:
|
||||
{{- if .Values.scheduler.patch.priorityClassName }}
|
||||
priorityClassName: {{ .Values.scheduler.patch.priorityClassName }}
|
||||
{{- end }}
|
||||
{{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
|
||||
{{- include "hami.scheduler.patch.new.imagePullSecrets" . | nindent 6 }}
|
||||
{{- else }}
|
||||
{{- include "hami.scheduler.patch.imagePullSecrets" . | nindent 6 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: create
|
||||
{{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
|
||||
image: {{ include "hami.scheduler.patch.new.image" . }}
|
||||
imagePullPolicy: {{ .Values.scheduler.patch.imageNew.pullPolicy }}
|
||||
{{- else }}
|
||||
image: {{ include "hami.scheduler.patch.image" . }}
|
||||
imagePullPolicy: {{ .Values.scheduler.patch.image.pullPolicy }}
|
||||
{{- end }}
|
||||
args:
|
||||
- create
|
||||
- --cert-name=tls.crt
|
||||
- --key-name=tls.key
|
||||
{{- if .Values.scheduler.admissionWebhook.customURL.enabled }}
|
||||
- --host={{ printf "%s.%s.svc,127.0.0.1,%s" (include "hami-vgpu.scheduler" .) (include "hami-vgpu.namespace" .) .Values.scheduler.admissionWebhook.customURL.host}}
|
||||
{{- else }}
|
||||
- --host={{ printf "%s.%s.svc,127.0.0.1" (include "hami-vgpu.scheduler" .) (include "hami-vgpu.namespace" .) }}
|
||||
{{- end }}
|
||||
- --namespace={{ include "hami-vgpu.namespace" . }}
|
||||
- --secret-name={{ include "hami-vgpu.scheduler.tls" . }}
|
||||
restartPolicy: OnFailure
|
||||
serviceAccountName: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
{{- if .Values.scheduler.patch.nodeSelector }}
|
||||
nodeSelector: {{ toYaml .Values.scheduler.patch.nodeSelector | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.patch.tolerations }}
|
||||
tolerations: {{ toYaml .Values.scheduler.patch.tolerations | nindent 8 }}
|
||||
{{- end }}
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: {{ .Values.scheduler.patch.runAsUser }}
|
||||
{{- end }}
|
||||
|
|
@ -1,63 +0,0 @@
|
|||
{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}}
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission-patch
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
annotations:
|
||||
"helm.sh/hook": post-install,post-upgrade
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
labels:
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: admission-webhook
|
||||
spec:
|
||||
{{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }}
|
||||
# Alpha feature since k8s 1.12
|
||||
ttlSecondsAfterFinished: 0
|
||||
{{- end }}
|
||||
template:
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission-patch
|
||||
{{- if .Values.scheduler.patch.podAnnotations }}
|
||||
annotations: {{ toYaml .Values.scheduler.patch.podAnnotations | nindent 8 }}
|
||||
{{- end }}
|
||||
labels:
|
||||
{{- include "hami-vgpu.labels" . | nindent 8 }}
|
||||
app.kubernetes.io/component: admission-webhook
|
||||
hami.io/webhook: ignore
|
||||
spec:
|
||||
{{- if .Values.scheduler.patch.priorityClassName }}
|
||||
priorityClassName: {{ .Values.scheduler.patch.priorityClassName }}
|
||||
{{- end }}
|
||||
{{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
|
||||
{{- include "hami.scheduler.patch.new.imagePullSecrets" . | nindent 6 }}
|
||||
{{- else }}
|
||||
{{- include "hami.scheduler.patch.imagePullSecrets" . | nindent 6 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: patch
|
||||
{{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
|
||||
image: {{ include "hami.scheduler.patch.new.image" . }}
|
||||
imagePullPolicy: {{ .Values.scheduler.patch.imageNew.pullPolicy }}
|
||||
{{- else }}
|
||||
image: {{ include "hami.scheduler.patch.image" . }}
|
||||
imagePullPolicy: {{ .Values.scheduler.patch.image.pullPolicy }}
|
||||
{{- end }}
|
||||
args:
|
||||
- patch
|
||||
- --webhook-name={{ include "hami-vgpu.scheduler.webhook" . }}
|
||||
- --namespace={{ include "hami-vgpu.namespace" . }}
|
||||
- --patch-validating=false
|
||||
- --secret-name={{ include "hami-vgpu.scheduler.tls" . }}
|
||||
restartPolicy: OnFailure
|
||||
serviceAccountName: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
{{- if .Values.scheduler.patch.nodeSelector }}
|
||||
nodeSelector: {{ toYaml .Values.scheduler.patch.nodeSelector | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.patch.tolerations }}
|
||||
tolerations: {{ toYaml .Values.scheduler.patch.tolerations | nindent 8 }}
|
||||
{{- end }}
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: {{ .Values.scheduler.patch.runAsUser }}
|
||||
{{- end }}
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
annotations:
|
||||
"helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
labels:
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: admission-webhook
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- secrets
|
||||
verbs:
|
||||
- get
|
||||
- create
|
||||
{{- end }}
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
annotations:
|
||||
"helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
labels:
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: admission-webhook
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
{{- end }}
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.fullname" . }}-admission
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
annotations:
|
||||
"helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
labels:
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: admission-webhook
|
||||
{{- end }}
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: "hami-scheduler"
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
rules:
|
||||
- apiGroups: ["coordination.k8s.io"]
|
||||
resources: ["leases"]
|
||||
verbs: ["create", "list", "watch", "get", "update", "patch"]
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: "hami-scheduler"
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
---
|
||||
{{- if .Values.mockDevicePlugin.enabled }}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.mock-device-plugin" . }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: {{ include "hami-vgpu.mock-device-plugin" . }}
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "hami-vgpu.mock-device-plugin" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
{{- end -}}
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: hami-scheduler
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
{{- if .Values.scheduler.service.labels }}
|
||||
{{ toYaml .Values.scheduler.service.labels | indent 4 }}
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.service.annotations }}
|
||||
annotations: {{ toYaml .Values.scheduler.service.annotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
type: {{ .Values.scheduler.service.type | default "NodePort" }} # Default type is NodePort
|
||||
ports:
|
||||
- name: http
|
||||
port: {{ .Values.scheduler.service.httpPort | default 443 }} # Default HTTP port is 443
|
||||
targetPort: {{ .Values.scheduler.service.httpTargetPort | default 443 }}
|
||||
{{- if eq (.Values.scheduler.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort
|
||||
nodePort: {{ .Values.scheduler.service.schedulerPort | default 31998 }}
|
||||
{{- end }}
|
||||
protocol: TCP
|
||||
- name: monitor
|
||||
port: {{ .Values.scheduler.service.monitorPort | default 31993 }} # Default monitoring port is 31993
|
||||
targetPort: {{ .Values.scheduler.service.monitorTargetPort | default 9395 }}
|
||||
{{- if eq (.Values.scheduler.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort
|
||||
nodePort: {{ .Values.scheduler.service.monitorPort | default 31993 }}
|
||||
{{- end }}
|
||||
protocol: TCP
|
||||
selector:
|
||||
app.kubernetes.io/component: hami-scheduler
|
||||
{{- include "hami-vgpu.selectorLabels" . | nindent 4 }}
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
labels:
|
||||
app.kubernetes.io/component: "hami-scheduler"
|
||||
{{- include "hami-vgpu.labels" . | nindent 4 }}
|
||||
---
|
||||
{{- if .Values.mockDevicePlugin.enabled }}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "hami-vgpu.mock-device-plugin" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
{{- end -}}
|
||||
|
|
@ -1,57 +0,0 @@
|
|||
{{- if .Values.scheduler.admissionWebhook.enabled -}}
|
||||
apiVersion: admissionregistration.k8s.io/v1
|
||||
kind: MutatingWebhookConfiguration
|
||||
metadata:
|
||||
{{- if .Values.scheduler.certManager.enabled }}
|
||||
annotations:
|
||||
cert-manager.io/inject-ca-from: {{ include "hami-vgpu.namespace" . }}/{{ include "hami-vgpu.scheduler" . }}-serving-cert
|
||||
{{- end }}
|
||||
name: {{ include "hami-vgpu.scheduler.webhook" . }}
|
||||
webhooks:
|
||||
- admissionReviewVersions:
|
||||
- v1beta1
|
||||
clientConfig:
|
||||
{{- if .Values.scheduler.admissionWebhook.customURL.enabled }}
|
||||
url: https://{{ .Values.scheduler.admissionWebhook.customURL.host}}:{{.Values.scheduler.admissionWebhook.customURL.port}}{{.Values.scheduler.admissionWebhook.customURL.path}}
|
||||
{{- else }}
|
||||
service:
|
||||
name: {{ include "hami-vgpu.scheduler" . }}
|
||||
namespace: {{ include "hami-vgpu.namespace" . }}
|
||||
path: /webhook
|
||||
port: {{ .Values.scheduler.service.httpPort }}
|
||||
{{- end }}
|
||||
failurePolicy: {{ .Values.scheduler.admissionWebhook.failurePolicy }}
|
||||
matchPolicy: Equivalent
|
||||
name: vgpu.hami.io
|
||||
namespaceSelector:
|
||||
matchExpressions:
|
||||
- key: hami.io/webhook
|
||||
operator: NotIn
|
||||
values:
|
||||
- ignore
|
||||
{{- if .Values.scheduler.admissionWebhook.whitelistNamespaces }}
|
||||
- key: kubernetes.io/metadata.name
|
||||
operator: NotIn
|
||||
values:
|
||||
{{- toYaml .Values.scheduler.admissionWebhook.whitelistNamespaces | nindent 10 }}
|
||||
{{- end }}
|
||||
objectSelector:
|
||||
matchExpressions:
|
||||
- key: hami.io/webhook
|
||||
operator: NotIn
|
||||
values:
|
||||
- ignore
|
||||
reinvocationPolicy: {{ .Values.scheduler.admissionWebhook.reinvocationPolicy }}
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
apiVersions:
|
||||
- v1
|
||||
operations:
|
||||
- CREATE
|
||||
resources:
|
||||
- pods
|
||||
scope: '*'
|
||||
sideEffects: None
|
||||
timeoutSeconds: 10
|
||||
{{- end }}
|
||||
|
|
@ -1,455 +0,0 @@
|
|||
## @section Global configuration
|
||||
## @param global.imageRegistry Global Docker image registry
|
||||
## @param global.imagePullSecrets Global Docker image pull secrets
|
||||
global:
|
||||
## @param global.imageRegistry Global Docker image registry
|
||||
imageRegistry: ""
|
||||
## E.g.
|
||||
## imagePullSecrets:
|
||||
## - myRegistryKeySecretName
|
||||
## @param global.imagePullSecrets Global Docker image pull secrets
|
||||
imagePullSecrets: []
|
||||
imageTag: "v2.8.1"
|
||||
gpuHookPath: /usr/local
|
||||
labels: {}
|
||||
annotations: {}
|
||||
managedNodeSelectorEnable: false
|
||||
managedNodeSelector:
|
||||
usage: "gpu"
|
||||
|
||||
nameOverride: ""
|
||||
fullnameOverride: ""
|
||||
namespaceOverride: ""
|
||||
|
||||
#Nvidia GPU Parameters
|
||||
resourceName: "nvidia.com/gpu"
|
||||
resourceMem: "nvidia.com/gpumem"
|
||||
resourceMemPercentage: "nvidia.com/gpumem-percentage"
|
||||
resourceCores: "nvidia.com/gpucores"
|
||||
resourcePriority: "nvidia.com/priority"
|
||||
|
||||
#MLU Parameters
|
||||
mluResourceName: "cambricon.com/vmlu"
|
||||
mluResourceMem: "cambricon.com/mlu.smlu.vmemory"
|
||||
mluResourceCores: "cambricon.com/mlu.smlu.vcore"
|
||||
|
||||
#Hygon DCU Parameters
|
||||
dcuResourceName: "hygon.com/dcunum"
|
||||
dcuResourceMem: "hygon.com/dcumem"
|
||||
dcuResourceCores: "hygon.com/dcucores"
|
||||
|
||||
#Metax sGPU Parameters
|
||||
metaxResourceName: "metax-tech.com/sgpu"
|
||||
metaxResourceCore: "metax-tech.com/vcore"
|
||||
metaxResourceMem: "metax-tech.com/vmemory"
|
||||
metaxsGPUTopologyAware: "false"
|
||||
|
||||
#Enflame VGCU Parameters
|
||||
enflameResourceNameVGCU: "enflame.com/vgcu"
|
||||
enflameResourceNameVGCUPercentage: "enflame.com/vgcu-percentage"
|
||||
|
||||
#Kunlun XPU Parameters
|
||||
kunlunResourceName: "kunlunxin.com/xpu"
|
||||
kunlunResourceVCountName: "kunlunxin.com/vxpu"
|
||||
kunlunResourceVMemoryName: "kunlunxin.com/vxpu-memory"
|
||||
|
||||
schedulerName: "hami-scheduler"
|
||||
|
||||
scheduler:
|
||||
# @param nodeName defines the node name and the nvidia-vgpu-scheduler-scheduler will schedule to the node.
|
||||
# if we install the nvidia-vgpu-scheduler-scheduler as default scheduler, we need to remove the k8s default
|
||||
# scheduler pod from the cluster first, we must specify node name to skip the schedule workflow.
|
||||
nodeName: ""
|
||||
#nodeLabelSelector:
|
||||
# "gpu": "on"
|
||||
overwriteEnv: "false"
|
||||
defaultSchedulerPolicy:
|
||||
nodeSchedulerPolicy: binpack
|
||||
gpuSchedulerPolicy: spread
|
||||
metricsBindAddress: ":9395"
|
||||
# If set to false, When Pod.Spec.SchedulerName equals to the const DefaultSchedulerName in k8s.io/api/core/v1 package, webhook will not overwrite it
|
||||
forceOverwriteDefaultScheduler: true
|
||||
livenessProbe: false
|
||||
leaderElect: true
|
||||
# when leaderElect is true, replicas is available, otherwise replicas is 1.
|
||||
replicas: 1
|
||||
kubeScheduler:
|
||||
# @param enabled indicate whether to run kube-scheduler container in the scheduler pod, it's true by default.
|
||||
enabled: true
|
||||
## @param image.registry kube scheduler image registry
|
||||
## @param image.repository kube scheduler image repository
|
||||
## @param image.tag kube scheduler image tag (immutable tags are recommended)
|
||||
## @param image.pullPolicy kube scheduler image pull policy
|
||||
## @param image.pullSecrets Specify docker-registry secret names as an array
|
||||
image:
|
||||
registry: "registry.cn-hangzhou.aliyuncs.com"
|
||||
repository: "google_containers/kube-scheduler"
|
||||
tag: ""
|
||||
## Specify a imagePullPolicy
|
||||
## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
|
||||
## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
|
||||
##
|
||||
pullPolicy: IfNotPresent
|
||||
## Optionally specify an array of imagePullSecrets.
|
||||
## Secrets must be manually created in the namespace.
|
||||
## Example:
|
||||
## pullSecrets:
|
||||
## - myRegistryKeySecretName
|
||||
##
|
||||
pullSecrets: []
|
||||
resources: {}
|
||||
# If you do want to specify resources, uncomment the following lines, adjust them as necessary.
|
||||
# and remove the curly braces after 'resources:'.
|
||||
# limits:
|
||||
# cpu: 1000m
|
||||
# memory: 1000Mi
|
||||
# requests:
|
||||
# cpu: 100m
|
||||
# memory: 100Mi
|
||||
extraNewArgs:
|
||||
- --config=/config/config.yaml
|
||||
- -v=4
|
||||
extraArgs:
|
||||
- --policy-config-file=/config/config.json
|
||||
- -v=4
|
||||
extender:
|
||||
## @param image.registry scheduler extender image registry
|
||||
## @param image.repository scheduler extender image repository
|
||||
## @param image.tag scheduler extender image tag (immutable tags are recommended)
|
||||
## @param image.pullPolicy scheduler extender image pull policy
|
||||
## @param image.pullSecrets Specify docker-registry secret names as an array
|
||||
image:
|
||||
registry: "docker.io"
|
||||
repository: "projecthami/hami"
|
||||
tag: ""
|
||||
## Specify a imagePullPolicy
|
||||
## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
|
||||
## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
|
||||
##
|
||||
pullPolicy: IfNotPresent
|
||||
## Optionally specify an array of imagePullSecrets.
|
||||
## Secrets must be manually created in the namespace.
|
||||
## Example:
|
||||
## pullSecrets:
|
||||
## - myRegistryKeySecretName
|
||||
##
|
||||
pullSecrets: []
|
||||
resources: {}
|
||||
# If you do want to specify resources, uncomment the following lines, adjust them as necessary,
|
||||
# and remove the curly braces after 'resources:'.
|
||||
# limits:
|
||||
# cpu: 1000m
|
||||
# memory: 1000Mi
|
||||
# requests:
|
||||
# cpu: 100m
|
||||
# memory: 100Mi
|
||||
extraArgs:
|
||||
- --debug
|
||||
- -v=4
|
||||
nodeLockExpire: "5m"
|
||||
podAnnotations: {}
|
||||
tolerations: []
|
||||
#serviceAccountName: "hami-vgpu-scheduler-sa"
|
||||
admissionWebhook:
|
||||
# If set to false, the admission webhook is not installed and any pods that should use HAMi must be
|
||||
# configured to use the right 'schedulerName' and other device-specific configurations.
|
||||
enabled: true
|
||||
customURL:
|
||||
enabled: false
|
||||
# must be an endpoint using https.
|
||||
# should generate host certs here
|
||||
host: 127.0.0.1 # hostname or ip, can be your node'IP if you want to use https://<nodeIP>:<schedulerPort>/<path>
|
||||
port: 31998
|
||||
path: /webhook
|
||||
whitelistNamespaces:
|
||||
# Specify the namespaces that the webhook will not be applied to.
|
||||
# - default
|
||||
# - kube-system
|
||||
# - istio-system
|
||||
reinvocationPolicy: Never
|
||||
failurePolicy: Ignore
|
||||
## TLS Certificate Option 1: Use cert-manager to generate self-signed certificate.
|
||||
## If enabled, always takes precedence over options 2.
|
||||
certManager:
|
||||
enabled: false
|
||||
## TLS Certificate Option 2: Use kube-webhook-certgen to generate self-signed certificate.
|
||||
## If true and certManager.enabled is false, Helm will automatically create a self-signed cert and secret for you.
|
||||
patch:
|
||||
enabled: true
|
||||
## @param image.registry scheduler extender image registry
|
||||
## @param image.repository scheduler extender image repository
|
||||
## @param image.tag scheduler extender image tag (immutable tags are recommended)
|
||||
## @param image.pullPolicy scheduler extender image pull policy
|
||||
## @param image.pullSecrets Specify docker-registry secret names as an array
|
||||
image:
|
||||
registry: "docker.io"
|
||||
repository: "jettech/kube-webhook-certgen"
|
||||
tag: "v1.5.2"
|
||||
## Specify a imagePullPolicy
|
||||
## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
|
||||
## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
|
||||
##
|
||||
pullPolicy: IfNotPresent
|
||||
## Optionally specify an array of imagePullSecrets.
|
||||
## Secrets must be manually created in the namespace.
|
||||
## Example:
|
||||
## pullSecrets:
|
||||
## - myRegistryKeySecretName
|
||||
##
|
||||
pullSecrets: []
|
||||
## @param image.registry scheduler extender image registry
|
||||
## @param image.repository scheduler extender image repository
|
||||
## @param image.tag scheduler extender image tag (immutable tags are recommended)
|
||||
## @param image.pullPolicy scheduler extender image pull policy
|
||||
## @param image.pullSecrets Specify docker-registry secret names as an array
|
||||
imageNew:
|
||||
registry: "docker.io"
|
||||
repository: "liangjw/kube-webhook-certgen"
|
||||
tag: "v1.1.1"
|
||||
## Specify a imagePullPolicy
|
||||
## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
|
||||
## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
|
||||
##
|
||||
pullPolicy: IfNotPresent
|
||||
## Optionally specify an array of imagePullSecrets.
|
||||
## Secrets must be manually created in the namespace.
|
||||
## Example:
|
||||
## pullSecrets:
|
||||
## - myRegistryKeySecretName
|
||||
##
|
||||
pullSecrets: []
|
||||
priorityClassName: ""
|
||||
podAnnotations: {}
|
||||
nodeSelector: {}
|
||||
tolerations: []
|
||||
runAsUser: 2000
|
||||
service:
|
||||
type: NodePort # Default type is NodePort, can be changed to ClusterIP
|
||||
httpPort: 443 # HTTP port
|
||||
schedulerPort: 31998 # NodePort for HTTP
|
||||
monitorPort: 31993 # Monitoring port
|
||||
monitorTargetPort: 9395
|
||||
httpTargetPort: 443
|
||||
labels: {}
|
||||
annotations: {}
|
||||
|
||||
devicePlugin:
|
||||
enabled: true
|
||||
gpuOperatorToolkitReady:
|
||||
enabled: false
|
||||
hostPath: "/run/nvidia/validations"
|
||||
## @param image.registry devicePlugin image registry
|
||||
## @param image.repository devicePlugin image repository
|
||||
## @param image.tag devicePlugin image tag (immutable tags are recommended)
|
||||
## @param image.pullPolicy devicePlugin image pull policy
|
||||
## @param image.pullSecrets Specify docker-registry secret names as an array
|
||||
image:
|
||||
registry: "docker.io"
|
||||
repository: "projecthami/hami"
|
||||
tag: ""
|
||||
## Specify a imagePullPolicy
|
||||
## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
|
||||
## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
|
||||
##
|
||||
pullPolicy: IfNotPresent
|
||||
## Optionally specify an array of imagePullSecrets.
|
||||
## Secrets must be manually created in the namespace.
|
||||
## Example:
|
||||
## pullSecrets:
|
||||
## - myRegistryKeySecretName
|
||||
##
|
||||
pullSecrets: []
|
||||
monitor:
|
||||
## @param image.registry monitor image registry
|
||||
## @param image.repository monitor image repository
|
||||
## @param image.tag monitor image tag (immutable tags are recommended)
|
||||
## @param image.pullPolicy monitor image pull policy
|
||||
## @param image.pullSecrets Specify docker-registry secret names as an array
|
||||
image:
|
||||
registry: "docker.io"
|
||||
repository: "projecthami/hami"
|
||||
tag: ""
|
||||
## Specify a imagePullPolicy
|
||||
## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
|
||||
## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
|
||||
##
|
||||
pullPolicy: IfNotPresent
|
||||
## Optionally specify an array of imagePullSecrets.
|
||||
## Secrets must be manually created in the namespace.
|
||||
## Example:
|
||||
## pullSecrets:
|
||||
## - myRegistryKeySecretName
|
||||
##
|
||||
pullSecrets: []
|
||||
ctrPath: /usr/local/vgpu/containers
|
||||
resyncInterval: "5m"
|
||||
extraArgs:
|
||||
- -v=4
|
||||
extraEnvs: {}
|
||||
resources: {}
|
||||
# If you do want to specify resources, uncomment the following lines, adjust them as necessary.
|
||||
# and remove the curly braces after 'resources:'.
|
||||
# limits:
|
||||
# cpu: 1000m
|
||||
# memory: 1000Mi
|
||||
# requests:
|
||||
# cpu: 100m
|
||||
# memory: 100Mi
|
||||
|
||||
deviceSplitCount: 10
|
||||
deviceMemoryScaling: 1
|
||||
deviceCoreScaling: 1
|
||||
# Node configuration for device plugin, Priority: externalConfigName > config > default config
|
||||
nodeConfiguration:
|
||||
# If you want to use a custom config.json, you can set the content here.
|
||||
# If this is set, it will override the default config.json(An example is as follows).
|
||||
config: |
|
||||
{
|
||||
"nodeconfig": [
|
||||
{
|
||||
"name": "your-node-name",
|
||||
"operatingmode": "hami-core",
|
||||
"devicememoryscaling": 1,
|
||||
"devicesplitcount": 10,
|
||||
"migstrategy": "none",
|
||||
"filterdevices": {
|
||||
"uuid": [],
|
||||
"index": []
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
# If you want to use an existing ConfigMap, you can set the name here.
|
||||
# If this is set, the chart will not create the ConfigMap and will use the existing one.
|
||||
externalConfigName: ""
|
||||
# The runtime class name to be used by the device plugin, and added to the pod.spec.runtimeClassName of applications utilizing NVIDIA GPUs
|
||||
runtimeClassName: ""
|
||||
# Whether to create runtime class, name comes from runtimeClassName when it is set
|
||||
createRuntimeClass: false
|
||||
migStrategy: "none"
|
||||
disablecorelimit: "false"
|
||||
passDeviceSpecsEnabled: false
|
||||
deviceListStrategy: "envvar"
|
||||
nvidiaHookPath: null
|
||||
nvidiaDriverRoot: null
|
||||
gdrcopyEnabled: null
|
||||
gdsEnabled: null
|
||||
mofedEnabled: null
|
||||
|
||||
extraArgs:
|
||||
- -v=4
|
||||
extraEnvs: {}
|
||||
service:
|
||||
type: NodePort # Default type is NodePort, can be changed to ClusterIP
|
||||
httpPort: 31992
|
||||
labels: {}
|
||||
annotations: {}
|
||||
|
||||
pluginPath: /var/lib/kubelet/device-plugins
|
||||
libPath: /usr/local/vgpu
|
||||
|
||||
podAnnotations: {}
|
||||
nvidiaNodeSelector:
|
||||
gpu: "on"
|
||||
tolerations: []
|
||||
# The updateStrategy for DevicePlugin DaemonSet.
|
||||
# If you want to update the DaemonSet by manual, set type as "OnDelete".
|
||||
# We recommend use OnDelete update strategy because DevicePlugin pod restart will cause business pod restart, this behavior is destructive.
|
||||
# Otherwise, you can use RollingUpdate update strategy to rolling update DevicePlugin pod.
|
||||
updateStrategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxUnavailable: 1
|
||||
|
||||
resources: {}
|
||||
# If you do want to specify resources, uncomment the following lines, adjust them as necessary.
|
||||
# and remove the curly braces after 'resources:'.
|
||||
# limits:
|
||||
# cpu: 1000m
|
||||
# memory: 1000Mi
|
||||
# requests:
|
||||
# cpu: 100m
|
||||
# memory: 100Mi
|
||||
|
||||
mockDevicePlugin:
|
||||
enabled: false
|
||||
image:
|
||||
registry: "docker.io"
|
||||
repository: "projecthami/mock-device-plugin"
|
||||
tag: "1.0.1"
|
||||
## Specify a imagePullPolicy
|
||||
## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
|
||||
## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
|
||||
##
|
||||
pullPolicy: IfNotPresent
|
||||
## Optionally specify an array of imagePullSecrets.
|
||||
## Secrets must be manually created in the namespace.
|
||||
## Example:
|
||||
## pullSecrets:
|
||||
## - myRegistryKeySecretName
|
||||
##
|
||||
pullSecrets: []
|
||||
devices:
|
||||
amd:
|
||||
customresources:
|
||||
- amd.com/gpu
|
||||
- amd.com/gpu-memory
|
||||
awsneuron:
|
||||
customresources:
|
||||
- aws.amazon.com/neuron
|
||||
- aws.amazon.com/neuroncore
|
||||
kunlun:
|
||||
enabled: true
|
||||
customresources:
|
||||
- kunlunxin.com/xpu
|
||||
- kunlunxin.com/vxpu
|
||||
- kunlunxin.com/vxpu-memory
|
||||
enflame:
|
||||
enabled: true
|
||||
customresources:
|
||||
- enflame.com/vgcu
|
||||
- enflame.com/vgcu-percentage
|
||||
- enflame.com/gcu
|
||||
mthreads:
|
||||
enabled: true
|
||||
customresources:
|
||||
- mthreads.com/vgpu
|
||||
nvidia:
|
||||
gpuCorePolicy: default
|
||||
libCudaLogLevel: 1
|
||||
ascend:
|
||||
enabled: false
|
||||
image: ""
|
||||
imagePullPolicy: IfNotPresent
|
||||
extraArgs: []
|
||||
nodeSelector:
|
||||
ascend: "on"
|
||||
tolerations: []
|
||||
customresources:
|
||||
- huawei.com/Ascend910A
|
||||
- huawei.com/Ascend910A-memory
|
||||
- huawei.com/Ascend910B2
|
||||
- huawei.com/Ascend910B2-memory
|
||||
- huawei.com/Ascend910B3
|
||||
- huawei.com/Ascend910B3-memory
|
||||
- huawei.com/Ascend910B4
|
||||
- huawei.com/Ascend910B4-memory
|
||||
- huawei.com/Ascend910B4-1
|
||||
- huawei.com/Ascend910B4-1-memory
|
||||
- huawei.com/Ascend310P
|
||||
- huawei.com/Ascend310P-memory
|
||||
iluvatar:
|
||||
enabled: false
|
||||
customresources:
|
||||
- iluvatar.ai/BI-V100-vgpu
|
||||
- iluvatar.ai/BI-V100.vCore
|
||||
- iluvatar.ai/BI-V100.vMem
|
||||
- iluvatar.ai/BI-V150-vgpu
|
||||
- iluvatar.ai/BI-V150.vCore
|
||||
- iluvatar.ai/BI-V150.vMem
|
||||
- iluvatar.ai/MR-V100-vgpu
|
||||
- iluvatar.ai/MR-V100.vCore
|
||||
- iluvatar.ai/MR-V100.vMem
|
||||
- iluvatar.ai/MR-V50-vgpu
|
||||
- iluvatar.ai/MR-V50.vCore
|
||||
- iluvatar.ai/MR-V50.vMem
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
hami:
|
||||
scheduler:
|
||||
kubeScheduler:
|
||||
image:
|
||||
registry: registry.k8s.io
|
||||
repository: kube-scheduler
|
||||
devicePlugin:
|
||||
runtimeClassName: nvidia
|
||||
updateStrategy:
|
||||
type: RollingUpdate
|
||||
# See https://github.com/Project-HAMi/HAMi/blob/v2.8.1/deployments/charts/hami/values.yaml
|
||||
# Each entry: {"name": "node-name", "operatingmode": "hami-core", "devicesplitcount": N, ...}
|
||||
nodeConfiguration:
|
||||
config: |
|
||||
{
|
||||
"nodeconfig": []
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
|
|
@ -4,13 +4,8 @@ NAMESPACE=cozy-system
|
|||
include ../../../hack/common-envs.mk
|
||||
include ../../../hack/package.mk
|
||||
|
||||
.PHONY: test
|
||||
|
||||
image: image-lineage-controller-webhook
|
||||
|
||||
test:
|
||||
helm unittest .
|
||||
|
||||
image-lineage-controller-webhook:
|
||||
docker buildx build -f images/lineage-controller-webhook/Dockerfile ../../.. \
|
||||
--tag $(REGISTRY)/lineage-controller-webhook:$(call settag,$(TAG)) \
|
||||
|
|
|
|||
|
|
@ -1,67 +0,0 @@
|
|||
# lineage-controller-webhook
|
||||
|
||||
Cozystack system package for the **lineage controller webhook** — a mutating
|
||||
admission webhook that stamps "lineage" labels onto tenant workloads, linking
|
||||
each resource back to the Cozystack `Application` that ultimately owns it.
|
||||
|
||||
The webhook intercepts CREATE and UPDATE on `pods`, `secrets`, `services`,
|
||||
`persistentvolumeclaims`, `ingresses.networking.k8s.io`, and
|
||||
`workloadmonitors.cozystack.io` outside system namespaces. For each request it
|
||||
walks the ownership graph upward (Kubernetes `ownerReferences`, then the
|
||||
`HelmRelease` Flux installed the resource with, then the Cozystack
|
||||
`Application`-derived CRD that produced the HelmRelease) and writes the
|
||||
discovered application's group, kind and name as labels
|
||||
(`apps.cozystack.io/application.{group,kind,name}`) on the incoming object.
|
||||
Those labels let the aggregated API server, the Cozystack dashboard, and other
|
||||
lineage-aware consumers reason about which application a resource belongs to.
|
||||
|
||||
The webhook serves TLS on port 9443 with a cert-manager issued certificate, and
|
||||
runs with `failurePolicy: Fail` — every CREATE/UPDATE on the resources above
|
||||
is gated on the webhook being reachable.
|
||||
|
||||
## Topology
|
||||
|
||||
The chart ships a single shape, modelled on `cozystack-api`:
|
||||
|
||||
- **Deployment** with two replicas (override via `replicas`).
|
||||
- **Soft `nodeAffinity`** preferring `node-role.kubernetes.io/control-plane`
|
||||
(`Exists`, so it matches both Talos's empty value and k3s/kubeadm's `"true"`).
|
||||
Soft means: the pod lands on a control-plane node when one is reachable, and
|
||||
on any worker otherwise — no override needed for managed Kubernetes,
|
||||
Cozy-in-Cozy tenants, or any other cluster where control-plane nodes aren't
|
||||
visible.
|
||||
- **Permissive `tolerations`** (`[{operator: Exists}]`) so the pod can land on
|
||||
tainted control-plane nodes when the soft affinity is satisfiable.
|
||||
- **Soft `podAntiAffinity`** on `kubernetes.io/hostname` so replicas spread
|
||||
across nodes when possible (best-effort).
|
||||
- **`PodDisruptionBudget`** with `maxUnavailable: 1`, unconditional. At
|
||||
`replicas: 1` it's a useful no-op (allows full drain); at `replicas >= 2`
|
||||
it caps disruption to one pod.
|
||||
- **`Service` with `spec.trafficDistribution: PreferClose`** so the apiserver
|
||||
prefers a webhook endpoint on its own node when one exists, and transparently
|
||||
falls over to a remote endpoint otherwise. Requires Kubernetes ≥ 1.31; on
|
||||
older clusters the field is silently ignored and traffic uses default
|
||||
cluster-wide distribution.
|
||||
|
||||
## Parameters
|
||||
|
||||
All keys live under the top-level `lineageControllerWebhook:` map.
|
||||
|
||||
| Name | Description | Value |
|
||||
| ----------------------------- | --------------------------------------------------------- | -------------------------------------------------------------------------------- |
|
||||
| `image` | Container image (digest-pinned) | `ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.3.0@sha256:e898…6fb0` |
|
||||
| `debug` | Enable `--zap-log-level=debug` instead of `info` | `false` |
|
||||
| `replicas` | Deployment replica count | `2` |
|
||||
| `localK8sAPIEndpoint.enabled` | **Deprecated.** See note below. | `false` |
|
||||
|
||||
### `localK8sAPIEndpoint.enabled` (deprecated)
|
||||
|
||||
When enabled, this injects `KUBERNETES_SERVICE_HOST=status.hostIP` and
|
||||
`KUBERNETES_SERVICE_PORT=6443` so the webhook talks to the kube-apiserver on
|
||||
its own node. It was originally added to avoid latency on the
|
||||
webhook-to-apiserver path, but it is only valid when the pod is actually
|
||||
scheduled on an apiserver-bearing node — which the chart's soft control-plane
|
||||
affinity no longer guarantees. With this flag enabled and the pod scheduled
|
||||
off a control-plane node, the controller will crash-loop dialing a non-
|
||||
apiserver hostIP. Slated for removal once the latency motivation is addressed
|
||||
in the webhook itself; **leave disabled**.
|
||||
|
|
@ -1,11 +1,10 @@
|
|||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: lineage-controller-webhook
|
||||
labels:
|
||||
app: lineage-controller-webhook
|
||||
spec:
|
||||
replicas: {{ .Values.lineageControllerWebhook.replicas }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app: lineage-controller-webhook
|
||||
|
|
@ -14,25 +13,13 @@ spec:
|
|||
labels:
|
||||
app: lineage-controller-webhook
|
||||
spec:
|
||||
serviceAccountName: lineage-controller-webhook
|
||||
{{- with .Values.lineageControllerWebhook.nodeSelector }}
|
||||
nodeSelector:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
tolerations:
|
||||
- operator: Exists
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- weight: 100
|
||||
preference:
|
||||
matchExpressions:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
podAntiAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- weight: 100
|
||||
podAffinityTerm:
|
||||
topologyKey: kubernetes.io/hostname
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
app: lineage-controller-webhook
|
||||
serviceAccountName: lineage-controller-webhook
|
||||
containers:
|
||||
- name: lineage-controller-webhook
|
||||
image: "{{ .Values.lineageControllerWebhook.image }}"
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: lineage-controller-webhook
|
||||
spec:
|
||||
maxUnavailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: lineage-controller-webhook
|
||||
|
|
@ -5,7 +5,7 @@ metadata:
|
|||
labels:
|
||||
app: lineage-controller-webhook
|
||||
spec:
|
||||
trafficDistribution: PreferClose
|
||||
internalTrafficPolicy: Local
|
||||
type: ClusterIP
|
||||
ports:
|
||||
- port: 443
|
||||
|
|
|
|||
|
|
@ -1,30 +0,0 @@
|
|||
suite: lineage-controller-webhook service
|
||||
templates:
|
||||
- templates/service.yaml
|
||||
|
||||
release:
|
||||
name: lineage-controller-webhook
|
||||
namespace: cozy-system
|
||||
|
||||
tests:
|
||||
- it: renders a ClusterIP service on 443 -> 9443 with PreferClose traffic distribution
|
||||
asserts:
|
||||
- isKind:
|
||||
of: Service
|
||||
- equal:
|
||||
path: spec.type
|
||||
value: ClusterIP
|
||||
- equal:
|
||||
path: spec.trafficDistribution
|
||||
value: PreferClose
|
||||
- notExists:
|
||||
path: spec.internalTrafficPolicy
|
||||
- equal:
|
||||
path: spec.ports[0].port
|
||||
value: 443
|
||||
- equal:
|
||||
path: spec.ports[0].targetPort
|
||||
value: 9443
|
||||
- equal:
|
||||
path: spec.selector.app
|
||||
value: lineage-controller-webhook
|
||||
|
|
@ -1,99 +0,0 @@
|
|||
suite: lineage-controller-webhook workload + PDB
|
||||
templates:
|
||||
- templates/workload.yaml
|
||||
- templates/pdb.yaml
|
||||
|
||||
release:
|
||||
name: lineage-controller-webhook
|
||||
namespace: cozy-system
|
||||
|
||||
tests:
|
||||
# ---- Default Deployment shape ----------------------------------------------
|
||||
|
||||
- it: defaults render a 2-replica Deployment with soft control-plane nodeAffinity, soft podAntiAffinity, and Exists tolerations
|
||||
template: templates/workload.yaml
|
||||
asserts:
|
||||
- isKind:
|
||||
of: Deployment
|
||||
- equal:
|
||||
path: spec.replicas
|
||||
value: 2
|
||||
- equal:
|
||||
path: spec.template.spec.tolerations
|
||||
value:
|
||||
- operator: Exists
|
||||
- equal:
|
||||
path: spec.template.spec.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution
|
||||
value:
|
||||
- weight: 100
|
||||
preference:
|
||||
matchExpressions:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
- notExists:
|
||||
path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution
|
||||
- equal:
|
||||
path: spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.topologyKey
|
||||
value: kubernetes.io/hostname
|
||||
|
||||
- it: defaults inject no localK8sAPIEndpoint env vars (knob is opt-in)
|
||||
template: templates/workload.yaml
|
||||
asserts:
|
||||
- notExists:
|
||||
path: spec.template.spec.containers[0].env
|
||||
|
||||
# ---- PDB --------------------------------------------------------------------
|
||||
|
||||
- it: PDB renders unconditionally with maxUnavailable=1 at the default replica count
|
||||
template: templates/pdb.yaml
|
||||
asserts:
|
||||
- isKind:
|
||||
of: PodDisruptionBudget
|
||||
- equal:
|
||||
path: spec.maxUnavailable
|
||||
value: 1
|
||||
- notExists:
|
||||
path: spec.minAvailable
|
||||
|
||||
- it: PDB still renders at replicas=1 (no-op, but consistent shape)
|
||||
set:
|
||||
lineageControllerWebhook.replicas: 1
|
||||
template: templates/pdb.yaml
|
||||
asserts:
|
||||
- isKind:
|
||||
of: PodDisruptionBudget
|
||||
- equal:
|
||||
path: spec.maxUnavailable
|
||||
value: 1
|
||||
|
||||
# ---- Replica override -------------------------------------------------------
|
||||
|
||||
- it: replicas value drives spec.replicas
|
||||
set:
|
||||
lineageControllerWebhook.replicas: 5
|
||||
template: templates/workload.yaml
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.replicas
|
||||
value: 5
|
||||
|
||||
# ---- Deprecated localK8sAPIEndpoint knob -----------------------------------
|
||||
|
||||
- it: localK8sAPIEndpoint.enabled=true injects KUBERNETES_SERVICE_HOST and PORT env vars
|
||||
set:
|
||||
lineageControllerWebhook.localK8sAPIEndpoint.enabled: true
|
||||
template: templates/workload.yaml
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: KUBERNETES_SERVICE_HOST
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
apiVersion: v1
|
||||
fieldPath: status.hostIP
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: KUBERNETES_SERVICE_PORT
|
||||
value: "6443"
|
||||
|
|
@ -1,13 +1,10 @@
|
|||
lineageControllerWebhook:
|
||||
image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.3.0@sha256:e8984709686a5eaf19b89da378d7b8c688ea5607e0783a88d9c9e4ccfca96fb0
|
||||
debug: false
|
||||
replicas: 2
|
||||
# DEPRECATED. Injects KUBERNETES_SERVICE_HOST=status.hostIP and
|
||||
# KUBERNETES_SERVICE_PORT=6443 so the webhook talks to the apiserver on its
|
||||
# own node — only valid when the pod is actually scheduled on an apiserver-
|
||||
# bearing node. Now that the chart's nodeAffinity to control-plane is soft
|
||||
# (preferred, not required), the pod can land off-control-plane and crash-
|
||||
# loop dialing a non-apiserver hostIP. Slated for removal once webhook
|
||||
# performance work obviates the locality motivation. Leave disabled.
|
||||
localK8sAPIEndpoint:
|
||||
enabled: false
|
||||
enabled: true
|
||||
# nodeSelector for the DaemonSet
|
||||
# Talos uses empty value: "node-role.kubernetes.io/control-plane": ""
|
||||
# Generic k8s (k3s, kubeadm) uses: "node-role.kubernetes.io/control-plane": "true"
|
||||
nodeSelector:
|
||||
node-role.kubernetes.io/control-plane: ""
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue