diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml index d9898616..a2c09b32 100644 --- a/.github/workflows/pull-requests.yaml +++ b/.github/workflows/pull-requests.yaml @@ -313,6 +313,20 @@ jobs: name: image-list path: /tmp/${{ env.SANDBOX_NAME }}/_out/images.txt + # ▸ Open an SSH breakpoint to the failing sandbox so maintainers can attach + # to the live runner, inspect Talos/Cozystack state and resume with + # `breakpoint resume`. Times out after 30m. + # Configured via the BREAKPOINT_ENDPOINT repository variable — in fork + # PRs repository variables are not exposed, so the step is skipped and + # forks cannot reach the self-hosted rendezvous server. + - name: Breakpoint on E2E failure + if: failure() && vars.BREAKPOINT_ENDPOINT != '' + uses: namespacelabs/breakpoint-action@v0 + with: + duration: 30m + endpoint: ${{ vars.BREAKPOINT_ENDPOINT }} + authorized-users: kvaps, gecube, kingdonb, lllamnyp, aobort, tym83, klinch0, nbykov0, matthieu-robin, mattia-eleuteri + # ▸ Tear down environment (always runs) - name: Tear down sandbox if: always() diff --git a/api/apps/v1alpha1/kubernetes/types.go b/api/apps/v1alpha1/kubernetes/types.go index 88a3c97e..ea4a2b64 100644 --- a/api/apps/v1alpha1/kubernetes/types.go +++ b/api/apps/v1alpha1/kubernetes/types.go @@ -69,9 +69,6 @@ type Addons struct { // NVIDIA GPU Operator. // +kubebuilder:default:={} GpuOperator GPUOperatorAddon `json:"gpuOperator"` - // HAMi GPU virtualization middleware. - // +kubebuilder:default:={} - Hami HAMiAddon `json:"hami"` // Ingress-NGINX controller. // +kubebuilder:default:={} IngressNginx IngressNginxAddon `json:"ingressNginx"` @@ -163,15 +160,6 @@ type GatewayAPIAddon struct { Enabled bool `json:"enabled"` } -type HAMiAddon struct { - // Enable HAMi (requires GPU Operator). - // +kubebuilder:default:=false - Enabled bool `json:"enabled"` - // Custom Helm values overrides. - // +kubebuilder:default:={} - ValuesOverride k8sRuntime.RawExtension `json:"valuesOverride"` -} - type Images struct { // Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag. // +kubebuilder:default:="" diff --git a/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go b/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go index e021fbaa..2cbe3ba1 100644 --- a/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go +++ b/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go @@ -49,7 +49,6 @@ func (in *Addons) DeepCopyInto(out *Addons) { in.Fluxcd.DeepCopyInto(&out.Fluxcd) out.GatewayAPI = in.GatewayAPI in.GpuOperator.DeepCopyInto(&out.GpuOperator) - in.Hami.DeepCopyInto(&out.Hami) in.IngressNginx.DeepCopyInto(&out.IngressNginx) in.MonitoringAgents.DeepCopyInto(&out.MonitoringAgents) in.Velero.DeepCopyInto(&out.Velero) @@ -136,7 +135,6 @@ func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) { } in.Addons.DeepCopyInto(&out.Addons) in.ControlPlane.DeepCopyInto(&out.ControlPlane) - out.Images = in.Images } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigSpec. @@ -262,37 +260,6 @@ func (in *GatewayAPIAddon) DeepCopy() *GatewayAPIAddon { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *HAMiAddon) DeepCopyInto(out *HAMiAddon) { - *out = *in - in.ValuesOverride.DeepCopyInto(&out.ValuesOverride) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HAMiAddon. -func (in *HAMiAddon) DeepCopy() *HAMiAddon { - if in == nil { - return nil - } - out := new(HAMiAddon) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Images) DeepCopyInto(out *Images) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Images. -func (in *Images) DeepCopy() *Images { - if in == nil { - return nil - } - out := new(Images) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IngressNginxAddon) DeepCopyInto(out *IngressNginxAddon) { *out = *in diff --git a/packages/apps/kubernetes/README.md b/packages/apps/kubernetes/README.md index bed1117a..ecbbc31b 100644 --- a/packages/apps/kubernetes/README.md +++ b/packages/apps/kubernetes/README.md @@ -128,9 +128,6 @@ See the reference for components utilized in this service: | `addons.gpuOperator` | NVIDIA GPU Operator. | `object` | `{}` | | `addons.gpuOperator.enabled` | Enable GPU Operator. | `bool` | `false` | | `addons.gpuOperator.valuesOverride` | Custom Helm values overrides. | `object` | `{}` | -| `addons.hami` | HAMi GPU virtualization middleware. | `object` | `{}` | -| `addons.hami.enabled` | Enable HAMi (requires GPU Operator). | `bool` | `false` | -| `addons.hami.valuesOverride` | Custom Helm values overrides. | `object` | `{}` | | `addons.fluxcd` | FluxCD GitOps operator. | `object` | `{}` | | `addons.fluxcd.enabled` | Enable FluxCD. | `bool` | `false` | | `addons.fluxcd.valuesOverride` | Custom Helm values overrides. | `object` | `{}` | diff --git a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml index 655dc868..e243bfad 100644 --- a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml @@ -1,11 +1,3 @@ -{{- define "cozystack.defaultGpuOperatorValues" -}} -{{- if .Values.addons.hami.enabled }} -gpu-operator: - devicePlugin: - enabled: false -{{- end }} -{{- end }} - {{- if and .Values.addons.gpuOperator.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease @@ -37,12 +29,9 @@ spec: force: true remediation: retries: -1 - {{- $defaults := fromYaml (include "cozystack.defaultGpuOperatorValues" .) }} - {{- $overrides := deepCopy (default (dict) .Values.addons.gpuOperator.valuesOverride) }} - {{- $merged := mergeOverwrite (default (dict) $defaults) $overrides }} - {{- if $merged }} + {{- with .Values.addons.gpuOperator.valuesOverride }} values: - {{- toYaml $merged | nindent 4 }} + {{- toYaml . | nindent 4 }} {{- end }} dependsOn: diff --git a/packages/apps/kubernetes/templates/helmreleases/hami.yaml b/packages/apps/kubernetes/templates/helmreleases/hami.yaml deleted file mode 100644 index f1538c7c..00000000 --- a/packages/apps/kubernetes/templates/helmreleases/hami.yaml +++ /dev/null @@ -1,49 +0,0 @@ -{{- if and .Values.addons.hami.enabled .Values._namespace.etcd }} -{{- if not .Values.addons.gpuOperator.enabled }} -{{- fail "addons.hami requires addons.gpuOperator to be enabled" }} -{{- end }} -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: {{ .Release.Name }}-hami - labels: - cozystack.io/repository: system - cozystack.io/target-cluster-name: {{ .Release.Name }} - sharding.fluxcd.io/key: tenants -spec: - releaseName: hami - chartRef: - kind: ExternalArtifact - name: cozystack-kubernetes-application-kubevirt-kubernetes-hami - namespace: cozy-system - kubeConfig: - secretRef: - name: {{ .Release.Name }}-admin-kubeconfig - key: super-admin.svc - targetNamespace: cozy-hami - storageNamespace: cozy-hami - interval: 5m - timeout: 10m - install: - createNamespace: true - remediation: - retries: -1 - upgrade: - force: true - remediation: - retries: -1 - {{- with .Values.addons.hami.valuesOverride }} - values: - {{- toYaml . | nindent 4 }} - {{- end }} - - dependsOn: - {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }} - - name: {{ .Release.Name }} - namespace: {{ .Release.Namespace }} - {{- end }} - - name: {{ .Release.Name }}-cilium - namespace: {{ .Release.Namespace }} - - name: {{ .Release.Name }}-gpu-operator - namespace: {{ .Release.Namespace }} -{{- end }} diff --git a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml deleted file mode 100644 index 44528470..00000000 --- a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml +++ /dev/null @@ -1,99 +0,0 @@ -suite: GPU Operator HelmRelease HAMi integration tests -templates: - - templates/helmreleases/gpu-operator.yaml -values: - - values-ci.yaml -tests: - - it: should disable devicePlugin when hami is enabled - set: - addons: - gpuOperator: - enabled: true - valuesOverride: {} - hami: - enabled: true - valuesOverride: {} - asserts: - - equal: - path: spec.values.gpu-operator.devicePlugin.enabled - value: false - - - it: should not have values when hami is disabled and no overrides - set: - addons: - gpuOperator: - enabled: true - valuesOverride: {} - hami: - enabled: false - valuesOverride: {} - asserts: - - notExists: - path: spec.values - - - it: should apply hami defaults when valuesOverride key is omitted - set: - addons: - gpuOperator: - enabled: true - hami: - enabled: true - asserts: - - equal: - path: spec.values.gpu-operator.devicePlugin.enabled - value: false - - - it: should allow user overrides to merge with hami defaults - set: - addons: - gpuOperator: - enabled: true - valuesOverride: - gpu-operator: - driver: - enabled: false - hami: - enabled: true - valuesOverride: {} - asserts: - - equal: - path: spec.values.gpu-operator.devicePlugin.enabled - value: false - - equal: - path: spec.values.gpu-operator.driver.enabled - value: false - - - it: should let user explicitly override devicePlugin.enabled to true with hami enabled - set: - addons: - gpuOperator: - enabled: true - valuesOverride: - gpu-operator: - devicePlugin: - enabled: true - driver: - enabled: false - hami: - enabled: true - valuesOverride: {} - asserts: - - equal: - path: spec.values.gpu-operator.devicePlugin.enabled - value: true - - equal: - path: spec.values.gpu-operator.driver.enabled - value: false - - - it: should not render when gpuOperator is disabled - set: - addons: - gpuOperator: - enabled: false - valuesOverride: {} - hami: - enabled: false - valuesOverride: {} - asserts: - - hasDocuments: - count: 0 diff --git a/packages/apps/kubernetes/tests/hami_test.yaml b/packages/apps/kubernetes/tests/hami_test.yaml deleted file mode 100644 index 27f7c75b..00000000 --- a/packages/apps/kubernetes/tests/hami_test.yaml +++ /dev/null @@ -1,153 +0,0 @@ -suite: HAMi HelmRelease tests -templates: - - templates/helmreleases/hami.yaml -values: - - values-ci.yaml -tests: - - it: should not render when hami is disabled - set: - addons: - hami: - enabled: false - valuesOverride: {} - gpuOperator: - enabled: true - valuesOverride: {} - asserts: - - hasDocuments: - count: 0 - - - it: should render HelmRelease when hami is enabled - set: - addons: - hami: - enabled: true - valuesOverride: {} - gpuOperator: - enabled: true - valuesOverride: {} - asserts: - - hasDocuments: - count: 1 - - isKind: - of: HelmRelease - - - it: should fail when gpuOperator is not enabled - set: - addons: - hami: - enabled: true - valuesOverride: {} - gpuOperator: - enabled: false - valuesOverride: {} - asserts: - - failedTemplate: - errorMessage: "addons.hami requires addons.gpuOperator to be enabled" - - - it: should have correct metadata labels - set: - addons: - hami: - enabled: true - valuesOverride: {} - gpuOperator: - enabled: true - valuesOverride: {} - asserts: - - equal: - path: metadata.labels["cozystack.io/repository"] - value: system - - equal: - path: metadata.labels["sharding.fluxcd.io/key"] - value: tenants - - - it: should use ExternalArtifact chartRef - set: - addons: - hami: - enabled: true - valuesOverride: {} - gpuOperator: - enabled: true - valuesOverride: {} - asserts: - - equal: - path: spec.chartRef.kind - value: ExternalArtifact - - equal: - path: spec.chartRef.namespace - value: cozy-system - - - it: should target cozy-hami namespace - set: - addons: - hami: - enabled: true - valuesOverride: {} - gpuOperator: - enabled: true - valuesOverride: {} - asserts: - - equal: - path: spec.targetNamespace - value: cozy-hami - - equal: - path: spec.storageNamespace - value: cozy-hami - - - it: should depend on gpu-operator and cilium - release: - name: test - namespace: test-ns - set: - addons: - hami: - enabled: true - valuesOverride: {} - gpuOperator: - enabled: true - valuesOverride: {} - asserts: - - contains: - path: spec.dependsOn - content: - name: test-cilium - namespace: test-ns - - contains: - path: spec.dependsOn - content: - name: test-gpu-operator - namespace: test-ns - - - it: should not render spec.values when valuesOverride is empty - set: - addons: - hami: - enabled: true - valuesOverride: {} - gpuOperator: - enabled: true - valuesOverride: {} - asserts: - - hasDocuments: - count: 1 - - notExists: - path: spec.values - - - it: should pass through valuesOverride - set: - addons: - hami: - enabled: true - valuesOverride: - hami: - devicePlugin: - deviceSplitCount: 5 - gpuOperator: - enabled: true - valuesOverride: {} - asserts: - - equal: - path: spec.values.hami.devicePlugin.deviceSplitCount - value: 5 diff --git a/packages/apps/kubernetes/values.schema.json b/packages/apps/kubernetes/values.schema.json index a43deaec..81e9e2b8 100644 --- a/packages/apps/kubernetes/values.schema.json +++ b/packages/apps/kubernetes/values.schema.json @@ -149,7 +149,6 @@ "fluxcd", "gatewayAPI", "gpuOperator", - "hami", "ingressNginx", "monitoringAgents", "velero", @@ -269,28 +268,6 @@ } } }, - "hami": { - "description": "HAMi GPU virtualization middleware.", - "type": "object", - "default": {}, - "required": [ - "enabled", - "valuesOverride" - ], - "properties": { - "enabled": { - "description": "Enable HAMi (requires GPU Operator).", - "type": "boolean", - "default": false - }, - "valuesOverride": { - "description": "Custom Helm values overrides.", - "type": "object", - "default": {}, - "x-kubernetes-preserve-unknown-fields": true - } - } - }, "ingressNginx": { "description": "Ingress-NGINX controller.", "type": "object", diff --git a/packages/apps/kubernetes/values.yaml b/packages/apps/kubernetes/values.yaml index d609476c..68ba07b0 100644 --- a/packages/apps/kubernetes/values.yaml +++ b/packages/apps/kubernetes/values.yaml @@ -94,10 +94,6 @@ host: "" ## @field {bool} enabled - Enable FluxCD. ## @field {object} valuesOverride - Custom Helm values overrides. -## @typedef {struct} HAMiAddon - HAMi GPU virtualization middleware. -## @field {bool} enabled - Enable HAMi (requires GPU Operator). -## @field {object} valuesOverride - Custom Helm values overrides. - ## @typedef {struct} MonitoringAgentsAddon - Monitoring agents (Fluent Bit, VMAgents). ## @field {bool} enabled - Enable monitoring agents. ## @field {object} valuesOverride - Custom Helm values overrides. @@ -118,7 +114,6 @@ host: "" ## @field {GatewayAPIAddon} gatewayAPI - Gateway API addon. ## @field {IngressNginxAddon} ingressNginx - Ingress-NGINX controller. ## @field {GPUOperatorAddon} gpuOperator - NVIDIA GPU Operator. -## @field {HAMiAddon} hami - HAMi GPU virtualization middleware. ## @field {FluxCDAddon} fluxcd - FluxCD GitOps operator. ## @field {MonitoringAgentsAddon} monitoringAgents - Monitoring agents. ## @field {VerticalPodAutoscalerAddon} verticalPodAutoscaler - Vertical Pod Autoscaler. @@ -142,9 +137,6 @@ addons: gpuOperator: enabled: false valuesOverride: {} - hami: - enabled: false - valuesOverride: {} fluxcd: enabled: false valuesOverride: {} diff --git a/packages/core/platform/sources/hami.yaml b/packages/core/platform/sources/hami.yaml deleted file mode 100644 index 0184e405..00000000 --- a/packages/core/platform/sources/hami.yaml +++ /dev/null @@ -1,24 +0,0 @@ ---- -apiVersion: cozystack.io/v1alpha1 -kind: PackageSource -metadata: - name: cozystack.hami -spec: - sourceRef: - kind: OCIRepository - name: cozystack-packages - namespace: cozy-system - path: / - variants: - - name: default - dependsOn: - - cozystack.gpu-operator - components: - - name: hami - path: system/hami - valuesFiles: - - values.yaml - install: - privileged: true - namespace: cozy-hami - releaseName: hami diff --git a/packages/core/platform/sources/kubernetes-application.yaml b/packages/core/platform/sources/kubernetes-application.yaml index 088383ad..9787cb19 100644 --- a/packages/core/platform/sources/kubernetes-application.yaml +++ b/packages/core/platform/sources/kubernetes-application.yaml @@ -52,8 +52,6 @@ spec: path: system/cilium - name: kubernetes-gpu-operator path: system/gpu-operator - - name: kubernetes-hami - path: system/hami - name: kubernetes-vertical-pod-autoscaler path: system/vertical-pod-autoscaler - name: kubernetes-prometheus-operator-crds diff --git a/packages/core/platform/templates/bundles/iaas.yaml b/packages/core/platform/templates/bundles/iaas.yaml index 66ec98b2..ced0a322 100644 --- a/packages/core/platform/templates/bundles/iaas.yaml +++ b/packages/core/platform/templates/bundles/iaas.yaml @@ -10,7 +10,6 @@ {{include "cozystack.platform.package.default" (list "cozystack.kubevirt-cdi" $) }} {{include "cozystack.platform.package.optional.default" (list "cozystack.vm-default-images" $) }} {{include "cozystack.platform.package.optional.default" (list "cozystack.gpu-operator" $) }} -{{include "cozystack.platform.package.optional.default" (list "cozystack.hami" $) }} {{include "cozystack.platform.package.default" (list "cozystack.kamaji" $) }} {{include "cozystack.platform.package.default" (list "cozystack.capi-operator" $) }} {{include "cozystack.platform.package.default" (list "cozystack.capi-provider-bootstrap-kubeadm" $) }} diff --git a/packages/core/platform/templates/bundles/system.yaml b/packages/core/platform/templates/bundles/system.yaml index 6c587c82..43ea7266 100644 --- a/packages/core/platform/templates/bundles/system.yaml +++ b/packages/core/platform/templates/bundles/system.yaml @@ -105,6 +105,9 @@ {{- /* cozystack-api DaemonSet */ -}} {{- $apiValues := dict "cozystackAPI" (dict "nodeSelector" $genericNodeSelector) -}} {{- $_ := set $cozystackEngineComponents "cozystack-api" (dict "values" $apiValues) -}} +{{- /* lineage-controller-webhook DaemonSet */ -}} +{{- $lineageValues := dict "lineageControllerWebhook" (dict "nodeSelector" $genericNodeSelector) -}} +{{- $_ := set $cozystackEngineComponents "lineage-controller-webhook" (dict "values" $lineageValues) -}} {{- end -}} {{- if .Values.authentication.oidc.enabled }} {{include "cozystack.platform.package" (list "cozystack.cozystack-engine" "oidc" $ $cozystackEngineComponents) }} diff --git a/packages/system/hami/Chart.yaml b/packages/system/hami/Chart.yaml deleted file mode 100644 index 3fcf4c5d..00000000 --- a/packages/system/hami/Chart.yaml +++ /dev/null @@ -1,3 +0,0 @@ -apiVersion: v2 -name: cozy-hami -version: 0.0.0 # Placeholder, the actual version will be automatically set during the build process diff --git a/packages/system/hami/Makefile b/packages/system/hami/Makefile deleted file mode 100644 index 83663a66..00000000 --- a/packages/system/hami/Makefile +++ /dev/null @@ -1,43 +0,0 @@ -export NAME=hami -export NAMESPACE=cozy-$(NAME) - -include ../../../hack/common-envs.mk -include ../../../hack/package.mk - -# When bumping the HAMi version, run `make update` and then review -# the resulting diff in `charts/hami/`. The recipe below reproduces the -# top-level vendoring overrides automatically: -# -# 1. Removes the broken hami-dra subchart. Upstream's NVIDIA DRA driver -# path requires kubelet DRA support that cozystack does not enable -# and has no upstream fix tracked. See commit 3c5521e. -# 2. Empties Chart.yaml dependencies and drops Chart.lock so Helm does -# not try to re-pull hami-dra at build time. See commit 2734dc0. -# 3. Strips dra/hami-dra/podSecurityPolicy blocks from the upstream -# values.yaml since the corresponding code paths are gone. PSP is -# removed from Kubernetes 1.25+ and is unused by cozystack. -# -# Template-level patches are NOT reproduced automatically: -# -# * Scheduler templates have `{{- if .Values.dra.enabled }}` blocks -# that need to be removed because the dra value is gone (commit -# 2734dc0 stripped them). -# * device-plugin/monitorservice.yaml uses `indent` with leading -# whitespace; it must be rewritten to `nindent` for the labels block -# to render correctly when devicePlugin.service.labels is set -# (commit 3685254). -# -# After `make update`, run `git diff -- charts/hami/templates/` and -# replay these template patches against the new upstream version, then -# verify with `helm unittest`. If upstream restructured the affected -# files, the patches may need to be redesigned rather than reapplied. - -update: - rm -rf charts - helm repo add hami-charts https://project-hami.github.io/HAMi/ - helm repo update hami-charts - helm pull hami-charts/hami --untar --untardir charts - rm -rf charts/hami/charts/hami-dra - yq --inplace '.dependencies = []' charts/hami/Chart.yaml - rm -f charts/hami/Chart.lock - yq --inplace 'del(.dra) | del(.["hami-dra"]) | del(.podSecurityPolicy)' charts/hami/values.yaml diff --git a/packages/system/hami/README.md b/packages/system/hami/README.md deleted file mode 100644 index 68669ac7..00000000 --- a/packages/system/hami/README.md +++ /dev/null @@ -1,82 +0,0 @@ -# HAMi — GPU Virtualization Middleware - -[HAMi](https://github.com/Project-HAMi/HAMi) (Heterogeneous AI Computing Virtualization Middleware) is a CNCF Sandbox project that enables fractional GPU sharing in Kubernetes. It allows workloads to request specific amounts of GPU memory and compute cores instead of claiming entire GPUs. - -## Architecture - -HAMi consists of four components: - -- **MutatingWebhook** — intercepts pod creation, injects `schedulerName: hami-scheduler` -- **Scheduler Extender** — extends kube-scheduler with GPU-aware Filter and Bind logic -- **Device Plugin** (DaemonSet) — registers vGPU resources via the Kubernetes Device Plugin API -- **HAMi-core** (`libvgpu.so`) — `LD_PRELOAD` library injected into workload containers, intercepts CUDA API calls to enforce memory and compute isolation - -## Prerequisites - -- GPU Operator must be enabled (`addons.gpuOperator.enabled: true`) -- NVIDIA driver >= 440 on host nodes -- nvidia-container-toolkit configured as the default container runtime -- GPU nodes labeled with `gpu=on` - -## Known Limitations - -### glibc < 2.34 requirement for workload containers - -HAMi-core uses `LD_PRELOAD` to intercept `dlsym()` for CUDA symbol resolution. The fallback code path relies on `_dl_sym`, a private glibc internal symbol that was removed in glibc 2.34 when libdl and libpthread were merged into libc.so. - -**This limitation affects workload containers only**, not the host OS or HAMi's own components. - -| Distribution | glibc | Result | -| --------------- | ----- | -------------------------------------------- | -| Ubuntu 18.04 | 2.27 | Full isolation (memory + compute) | -| Ubuntu 20.04 | 2.31 | Full isolation (memory + compute) | -| Ubuntu 22.04 | 2.35 | Memory isolation works, compute breaks | -| Ubuntu 24.04 | 2.39 | Both memory and compute isolation break | -| Alpine (musl) | N/A | Completely incompatible (`dlvsym` absent) | - -Most modern ML/AI base images (CUDA 12.x, PyTorch 2.x, TensorFlow 2.x) use Ubuntu 22.04+ with glibc >= 2.35, which means compute isolation will not work with these images until the upstream fix is merged. - -**Upstream tracking issues:** - -- [HAMi-core#174](https://github.com/Project-HAMi/HAMi-core/issues/174) — `_dl_sym` removal in glibc 2.34 breaks HAMi-core's CUDA symbol resolution at the symbol level -- [HAMi#1190](https://github.com/Project-HAMi/HAMi/issues/1190) — maintainer thread confirming the empirical per-glibc-version isolation behavior shown in the table above - -### musl libc (Alpine) incompatibility - -HAMi-core is completely incompatible with musl libc. The `dlvsym()` function used by HAMi-core is a glibc extension not available in musl. Only glibc-based container images (Debian, Ubuntu, RHEL, etc.) can use HAMi GPU isolation. - -## Usage - -Enable HAMi in your tenant Kubernetes cluster values: - -```yaml -addons: - gpuOperator: - enabled: true - hami: - enabled: true -``` - -When HAMi is enabled, GPU Operator's built-in device plugin is automatically disabled to avoid conflicts. This default is preserved by setting `addons.gpuOperator.valuesOverride.gpu-operator.devicePlugin.enabled: false`; advanced topologies that partition GPU pools (e.g. some nodes use HAMi while others run the standard NVIDIA device plugin via node selectors) can re-enable it explicitly through `valuesOverride`. - -### Requesting fractional GPU resources - -```yaml -resources: - limits: - nvidia.com/gpu: 1 - nvidia.com/gpumem: 3000 # 3000 MB of GPU memory - nvidia.com/gpucores: 30 # 30% of GPU compute cores -``` - -## Parameters - -Default values shown below are inherited from the upstream HAMi chart and may change with upstream updates. - -| Name | Description | Default | -| --- | --- | --- | -| `hami.devicePlugin.runtimeClassName` | RuntimeClass for device plugin pods | `nvidia` | -| `hami.devicePlugin.deviceSplitCount` | Max virtual GPUs per physical GPU | `10` | -| `hami.devicePlugin.deviceMemoryScaling` | Memory overcommit factor (> 1.0 enables overcommit) | `1` | -| `hami.scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy` | Node packing strategy (`binpack` or `spread`) | `binpack` | -| `hami.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy` | GPU packing strategy (`binpack` or `spread`) | `spread` | diff --git a/packages/system/hami/charts/hami/Chart.yaml b/packages/system/hami/charts/hami/Chart.yaml deleted file mode 100644 index 55f32ab6..00000000 --- a/packages/system/hami/charts/hami/Chart.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: v2 -appVersion: 2.8.1 -dependencies: [] -description: Heterogeneous AI Computing Virtualization Middleware -keywords: -- vgpu -- gpu -kubeVersion: '>= 1.18.0-0' -maintainers: -- email: archlitchi@gmail.com - name: limengxuan -- email: xiaozhang0210@hotmail.com - name: zhangxiao -name: hami -sources: -- https://github.com/Project-HAMi/HAMi -type: application -version: 2.8.1 diff --git a/packages/system/hami/charts/hami/README.md b/packages/system/hami/charts/hami/README.md deleted file mode 100644 index e8217290..00000000 --- a/packages/system/hami/charts/hami/README.md +++ /dev/null @@ -1,237 +0,0 @@ -# HAMi Helm Chart Values Documentation - -This document provides detailed descriptions of all configurable values parameters for the HAMi Helm Chart. - -## Global Configuration - -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `global.imageRegistry` | Global Docker image registry | `""` | -| `global.imagePullSecrets` | Global Docker image pull secrets | `[]` | -| `global.imageTag` | Image tag | `"v2.8.1"` | -| `global.gpuHookPath` | GPU Hook path | `/usr/local` | -| `global.labels` | Global labels | `{}` | -| `global.annotations` | Global annotations | `{}` | -| `global.managedNodeSelectorEnable` | Whether to enable managed node selector | `false` | -| `global.managedNodeSelector.usage` | Managed node selector usage | `"gpu"` | -| `nameOverride` | Name override | `""` | -| `fullnameOverride` | Full name override | `""` | -| `namespaceOverride` | Namespace override | `""` | - -## Resource Name Configuration - -### NVIDIA GPU Resources -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `resourceName` | GPU resource name | `"nvidia.com/gpu"` | -| `resourceMem` | GPU memory resource name | `"nvidia.com/gpumem"` | -| `resourceMemPercentage` | GPU memory percentage resource name | `"nvidia.com/gpumem-percentage"` | -| `resourceCores` | GPU core resource name | `"nvidia.com/gpucores"` | -| `resourcePriority` | GPU priority resource name | `"nvidia.com/priority"` | - -### Cambricon MLU Resources -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `mluResourceName` | MLU resource name | `"cambricon.com/vmlu"` | -| `mluResourceMem` | MLU memory resource name | `"cambricon.com/mlu.smlu.vmemory"` | -| `mluResourceCores` | MLU core resource name | `"cambricon.com/mlu.smlu.vcore"` | - -### Hygon DCU Resources -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `dcuResourceName` | DCU resource name | `"hygon.com/dcunum"` | -| `dcuResourceMem` | DCU memory resource name | `"hygon.com/dcumem"` | -| `dcuResourceCores` | DCU core resource name | `"hygon.com/dcucores"` | - -### Metax GPU Resources -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `metaxResourceName` | GPU resource name | `"metax-tech.com/sgpu"` | -| `metaxResourceCore` | GPU core resource name | `"metax-tech.com/vcore"` | -| `metaxResourceMem` | GPU memory resource name | `"metax-tech.com/vmemory"` | -| `metaxsGPUTopologyAware` | GPU topology awareness | `"false"` | - -### Enflame GCU Resources -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `enflameResourceNameVGCU` | vGCU resource name | `"enflame.com/vgcu"` | -| `enflameResourceNameVGCUPercentage` | vGCU percentage resource name | `"enflame.com/vgcu-percentage"` | - -### Kunlunxin XPU Resources -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `kunlunResourceName` | XPU resource name | `"kunlunxin.com/xpu"` | - -## Scheduler Configuration - -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `schedulerName` | Scheduler name | `"hami-scheduler"` | -| `scheduler.nodeName` | Define node name, scheduler will schedule to this node | `""` | -| `scheduler.overwriteEnv` | Whether to overwrite environment variables | `"false"` | -| `scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy` | Node scheduler policy | `binpack` | -| `scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy` | GPU scheduler policy | `spread` | -| `scheduler.metricsBindAddress` | Metrics bind address | `":9395"` | -| `scheduler.forceOverwriteDefaultScheduler` | Whether to force overwrite default scheduler | `true` | -| `scheduler.livenessProbe` | Whether to enable liveness probe | `false` | -| `scheduler.leaderElect` | Whether to enable leader election | `true` | -| `scheduler.replicas` | Number of replicas | `1` | - -### Kube Scheduler Configuration - -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `scheduler.kubeScheduler.enabled` | Whether to run kube-scheduler container in scheduler pod | `true` | -| `scheduler.kubeScheduler.image.registry` | Kube scheduler image registry | `"registry.cn-hangzhou.aliyuncs.com"` | -| `scheduler.kubeScheduler.image.repository` | Kube scheduler image repository | `"google_containers/kube-scheduler"` | -| `scheduler.kubeScheduler.image.tag` | Kube scheduler image tag | `""` | -| `scheduler.kubeScheduler.image.pullPolicy` | Kube scheduler image pull policy | `IfNotPresent` | -| `scheduler.kubeScheduler.image.pullSecrets` | Kube scheduler image pull secrets | `[]` | -| `scheduler.kubeScheduler.extraNewArgs` | Extra new arguments | `["--config=/config/config.yaml", "-v=4"]` | -| `scheduler.kubeScheduler.extraArgs` | Extra arguments | `["--policy-config-file=/config/config.json", "-v=4"]` | - -### Extender Configuration - -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `scheduler.extender.image.registry` | Scheduler extender image registry | `"docker.io"` | -| `scheduler.extender.image.repository` | Scheduler extender image repository | `"projecthami/hami"` | -| `scheduler.extender.image.tag` | Scheduler extender image tag | `""` | -| `scheduler.extender.image.pullPolicy` | Scheduler extender image pull policy | `IfNotPresent` | -| `scheduler.extender.image.pullSecrets` | Scheduler extender image pull secrets | `[]` | -| `scheduler.extender.extraArgs` | Scheduler extender extra arguments | `["--debug", "-v=4"]` | - -### Admission Webhook Configuration - -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `scheduler.admissionWebhook.enabled` | Whether to enable admission webhook | `true` | -| `scheduler.admissionWebhook.customURL.enabled` | Whether to enable custom URL | `false` | -| `scheduler.admissionWebhook.customURL.host` | Custom URL host | `127.0.0.1` | -| `scheduler.admissionWebhook.customURL.port` | Custom URL port | `31998` | -| `scheduler.admissionWebhook.customURL.path` | Custom URL path | `/webhook` | -| `scheduler.admissionWebhook.reinvocationPolicy` | Reinvocation policy | `Never` | -| `scheduler.admissionWebhook.failurePolicy` | Failure policy | `Ignore` | - -### TLS Certificate Configuration - -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `scheduler.certManager.enabled` | Whether to use cert-manager to generate self-signed certificates | `false` | -| `scheduler.patch.enabled` | Whether to use kube-webhook-certgen to generate self-signed certificates | `true` | -| `scheduler.patch.image.registry` | Certgen image registry | `"docker.io"` | -| `scheduler.patch.image.repository` | Certgen image repository | `"jettech/kube-webhook-certgen"` | -| `scheduler.patch.image.tag` | Certgen image tag | `"v1.5.2"` | -| `scheduler.patch.image.pullPolicy` | Certgen image pull policy | `IfNotPresent` | -| `scheduler.patch.imageNew.registry` | New certgen image registry | `"docker.io"` | -| `scheduler.patch.imageNew.repository` | New certgen image repository | `"liangjw/kube-webhook-certgen"` | -| `scheduler.patch.imageNew.tag` | New certgen image tag | `"v1.1.1"` | - -### Scheduler Service Configuration - -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `scheduler.service.type` | Service type | `NodePort` | -| `scheduler.service.httpPort` | HTTP port | `443` | -| `scheduler.service.schedulerPort` | Scheduler NodePort | `31998` | -| `scheduler.service.monitorPort` | Monitor port | `31993` | -| `scheduler.service.monitorTargetPort` | Monitor target port | `9395` | - -## Device Plugin Configuration - -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `devicePlugin.image.registry` | Device plugin image registry | `"docker.io"` | -| `devicePlugin.image.repository` | Device plugin image repository | `"projecthami/hami"` | -| `devicePlugin.image.tag` | Device plugin image tag | `""` | -| `devicePlugin.image.pullPolicy` | Device plugin image pull policy | `IfNotPresent` | -| `devicePlugin.image.pullSecrets` | Device plugin image pull secrets | `[]` | - -### Monitor Configuration - -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `devicePlugin.monitor.image.registry` | Monitor image registry | `"docker.io"` | -| `devicePlugin.monitor.image.repository` | Monitor image repository | `"projecthami/hami"` | -| `devicePlugin.monitor.image.tag` | Monitor image tag | `""` | -| `devicePlugin.monitor.image.pullPolicy` | Monitor image pull policy | `IfNotPresent` | -| `devicePlugin.monitor.image.pullSecrets` | Monitor image pull secrets | `[]` | -| `devicePlugin.monitor.ctrPath` | Container path | `/usr/local/vgpu/containers` | -| `devicePlugin.monitor.extraArgs` | Monitor extra arguments | `["-v=4"]` | -| `devicePlugin.monitor.extraEnvs` | Monitor extra environments | `{}` | - -### Device Plugin Other Configuration - -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `devicePlugin.deviceSplitCount` | Integer type, default value: 10. Maximum number of tasks assigned to a single GPU device | `10` | -| `devicePlugin.deviceMemoryScaling` | Device memory scaling ratio | `1` | -| `devicePlugin.deviceCoreScaling` | Device core scaling ratio | `1` | -| `devicePlugin.runtimeClassName` | Runtime class name | `""` | -| `devicePlugin.createRuntimeClass` | Whether to create runtime class | `false` | -| `devicePlugin.migStrategy` | String type, "none" means ignore MIG functionality, "mixed" means allocate MIG devices through independent resources | `"none"` | -| `devicePlugin.disablecorelimit` | String type, "true" means disable core limit, "false" means enable core limit | `"false"` | -| `devicePlugin.passDeviceSpecsEnabled` | Whether to enable passing device specs | `false` | -| `devicePlugin.extraArgs` | Device plugin extra arguments | `["-v=4"]` | -| `devicePlugin.nodeConfiguration.config` | Node configuration for device plugin by json | An example of default configuration. | -| `devicePlugin.nodeConfiguration.externalConfigName` | Node configuration for device plugin by external congimap | `""` | -| `devicePlugin.extraEnvs` | Device plugin extra environments | `{}` | - -### Device Plugin Service Configuration - -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `devicePlugin.service.type` | Service type | `NodePort` | -| `devicePlugin.service.httpPort` | HTTP port | `31992` | - -### Device Plugin Deployment Configuration - -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `devicePlugin.pluginPath` | Plugin path | `/var/lib/kubelet/device-plugins` | -| `devicePlugin.libPath` | Library path | `/usr/local/vgpu` | -| `devicePlugin.nvidiaNodeSelector` | NVIDIA node selector | `{"gpu": "on"}` | -| `devicePlugin.updateStrategy.type` | Update strategy type | `RollingUpdate` | -| `devicePlugin.updateStrategy.rollingUpdate.maxUnavailable` | Maximum unavailable count | `1` | - -## Device Configuration - -### AWS Neuron -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `devices.awsneuron.customresources` | Custom resources | `["aws.amazon.com/neuron", "aws.amazon.com/neuroncore"]` | - -### Kunlunxin -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `devices.kunlun.enabled` | Whether to enable | `true` | -| `devices.kunlun.customresources` | Custom resources | `["kunlunxin.com/xpu"]` | - -### Mthreads -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `devices.mthreads.enabled` | Whether to enable | `true` | -| `devices.mthreads.customresources` | Custom resources | `["mthreads.com/vgpu"]` | - -### NVIDIA -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `devices.nvidia.gpuCorePolicy` | GPU core policy | `default` | -| `devices.nvidia.libCudaLogLevel` | CUDA library log level | `1` | - -### Huawei Ascend -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `devices.ascend.enabled` | Whether to enable | `false` | -| `devices.ascend.image` | Image | `""` | -| `devices.ascend.imagePullPolicy` | Image pull policy | `IfNotPresent` | -| `devices.ascend.extraArgs` | Extra arguments | `[]` | -| `devices.ascend.nodeSelector` | Node selector | `{"ascend": "on"}` | -| `devices.ascend.tolerations` | Tolerations | `[]` | -| `devices.ascend.customresources` | Custom resources | `["huawei.com/Ascend910A", "huawei.com/Ascend910A-memory", ...]` | - -### Iluvatar -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `devices.iluvatar.enabled` | Whether to enable | `false` | -| `devices.iluvatar.customresources` | Custom resources | `["iluvatar.ai/BI-V150-vgpu", "iluvatar.ai/BI-V150.vMem","iluvatar.ai/BI-V150.vCore", ...]` | diff --git a/packages/system/hami/charts/hami/templates/NOTES.txt b/packages/system/hami/charts/hami/templates/NOTES.txt deleted file mode 100644 index 15bb1218..00000000 --- a/packages/system/hami/charts/hami/templates/NOTES.txt +++ /dev/null @@ -1,3 +0,0 @@ -** Please be patient while the chart is being deployed ** -Resource name: {{ .Values.resourceName }} - diff --git a/packages/system/hami/charts/hami/templates/_commons.tpl b/packages/system/hami/charts/hami/templates/_commons.tpl deleted file mode 100644 index b68018e4..00000000 --- a/packages/system/hami/charts/hami/templates/_commons.tpl +++ /dev/null @@ -1,49 +0,0 @@ -{{/* -Return the proper image name -{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" $) }} -*/}} -{{- define "common.images.image" -}} -{{- $registryName := .imageRoot.registry -}} -{{- $repositoryName := .imageRoot.repository -}} -{{- $tag := .imageRoot.tag | toString -}} -{{- if .global }} - {{- if .global.imageRegistry }} - {{- $registryName = .global.imageRegistry -}} - {{- end -}} -{{- end -}} -{{- if .tag }} - {{- $tag = .tag | toString -}} -{{- end -}} -{{- if $registryName }} -{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} -{{- else -}} -{{- printf "%s:%s" $repositoryName $tag -}} -{{- end -}} -{{- end -}} - -{{/* -Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead) -{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }} -*/}} -{{- define "common.images.pullSecrets" -}} - {{- $pullSecrets := list }} - - {{- if .global }} - {{- range .global.imagePullSecrets -}} - {{- $pullSecrets = append $pullSecrets . -}} - {{- end -}} - {{- end -}} - - {{- range .images -}} - {{- range .pullSecrets -}} - {{- $pullSecrets = append $pullSecrets . -}} - {{- end -}} - {{- end -}} - - {{- if (not (empty $pullSecrets)) }} -imagePullSecrets: - {{- range $pullSecrets }} - - name: {{ . }} - {{- end }} - {{- end }} -{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/_helpers.tpl b/packages/system/hami/charts/hami/templates/_helpers.tpl deleted file mode 100644 index ffcc61da..00000000 --- a/packages/system/hami/charts/hami/templates/_helpers.tpl +++ /dev/null @@ -1,163 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "hami-vgpu.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "hami-vgpu.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Allow the release namespace to be overridden for multi-namespace deployments in combined charts -*/}} -{{- define "hami-vgpu.namespace" -}} - {{- if .Values.namespaceOverride -}} - {{- .Values.namespaceOverride -}} - {{- else -}} - {{- .Release.Namespace -}} - {{- end -}} -{{- end -}} - -{{/* -The app name for Scheduler -*/}} -{{- define "hami-vgpu.scheduler" -}} -{{- printf "%s-scheduler" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -The app name for DevicePlugin -*/}} -{{- define "hami-vgpu.device-plugin" -}} -{{- printf "%s-device-plugin" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* - The app name for MockDevicePlugin - */}} -{{- define "hami-vgpu.mock-device-plugin" -}} -{{- printf "%s-mock-device-plugin" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -The tls secret name for Scheduler -*/}} -{{- define "hami-vgpu.scheduler.tls" -}} -{{- printf "%s-scheduler-tls" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -The webhook name -*/}} -{{- define "hami-vgpu.scheduler.webhook" -}} -{{- printf "%s-webhook" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "hami-vgpu.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "hami-vgpu.labels" -}} -helm.sh/chart: {{ include "hami-vgpu.chart" . }} -{{ include "hami-vgpu.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "hami-vgpu.selectorLabels" -}} -app.kubernetes.io/name: {{ include "hami-vgpu.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - - -{{/* - Resolve the tag for kubeScheduler. -*/}} -{{- define "resolvedKubeSchedulerTag" -}} -{{- if .Values.scheduler.kubeScheduler.image.tag }} -{{- .Values.scheduler.kubeScheduler.image.tag | trim -}} -{{- else }} -{{- include "strippedKubeVersion" . | trim -}} -{{- end }} -{{- end }} - -{{/* - Return the stripped Kubernetes version string by removing extra parts after semantic version number. - v1.31.1+k3s1 -> v1.31.1 - v1.30.8-eks-2d5f260 -> v1.30.8 - v1.31.1 -> v1.31.1 -*/}} -{{- define "strippedKubeVersion" -}} -{{ regexReplaceAll "^(v[0-9]+\\.[0-9]+\\.[0-9]+)(.*)$" .Capabilities.KubeVersion.Version "$1" }} -{{- end -}} - -{{- define "hami.scheduler.kubeScheduler.image" -}} -{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.kubeScheduler.image "global" .Values.global "tag" (include "resolvedKubeSchedulerTag" .)) }} -{{- end -}} - -{{- define "hami.scheduler.extender.image" -}} -{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.extender.image "global" .Values.global "tag" .Values.global.imageTag) }} -{{- end -}} - -{{- define "hami.devicePlugin.image" -}} -{{ include "common.images.image" (dict "imageRoot" .Values.devicePlugin.image "global" .Values.global "tag" .Values.global.imageTag) }} -{{- end -}} - -{{- define "hami.mockDevicePlugin.image" -}} -{{ include "common.images.image" (dict "imageRoot" .Values.mockDevicePlugin.image "global" .Values.global "tag" .Values.mockDevicePlugin.tag) }} -{{- end -}} - -{{- define "hami.devicePlugin.monitor.image" -}} -{{ include "common.images.image" (dict "imageRoot" .Values.devicePlugin.monitor.image "global" .Values.global "tag" .Values.global.imageTag) }} -{{- end -}} - -{{- define "hami.scheduler.patch.image" -}} -{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.patch.image "global" .Values.global) }} -{{- end -}} - -{{- define "hami.scheduler.patch.new.image" -}} -{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.patch.imageNew "global" .Values.global) }} -{{- end -}} - -{{- define "hami.scheduler.extender.imagePullSecrets" -}} -{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.extender.image) "global" .Values.global) }} -{{- end -}} - -{{- define "hami.devicePlugin.imagePullSecrets" -}} -{{ include "common.images.pullSecrets" (dict "images" (list .Values.devicePlugin.image) "global" .Values.global) }} -{{- end -}} - -{{- define "hami.scheduler.patch.imagePullSecrets" -}} -{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.patch.image) "global" .Values.global) }} -{{- end -}} - -{{- define "hami.scheduler.patch.new.imagePullSecrets" -}} -{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.patch.imageNew) "global" .Values.global) }} -{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml b/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml deleted file mode 100644 index 74631e24..00000000 --- a/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- if and .Values.devicePlugin.enabled (not .Values.devicePlugin.nodeConfiguration.externalConfigName) -}} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "hami-vgpu.device-plugin" . }} - namespace: {{ include "hami-vgpu.namespace" . }} - labels: - app.kubernetes.io/component: hami-device-plugin - {{- include "hami-vgpu.labels" . | nindent 4 }} -data: - config.json: | -{{- .Values.devicePlugin.nodeConfiguration.config | nindent 4 }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/daemonsetmock.yaml b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetmock.yaml deleted file mode 100644 index a0dc4ff3..00000000 --- a/packages/system/hami/charts/hami/templates/device-plugin/daemonsetmock.yaml +++ /dev/null @@ -1,55 +0,0 @@ -{{- if .Values.mockDevicePlugin.enabled }} -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: {{ include "hami-vgpu.mock-device-plugin" . }} - namespace: {{ include "hami-vgpu.namespace" . }} -spec: - selector: - matchLabels: - app.kubernetes.io/component: hami-mock-device-plugin - {{- include "hami-vgpu.selectorLabels" . | nindent 6 }} - template: - metadata: - annotations: - scheduler.alpha.kubernetes.io/critical-pod: "" - labels: - app.kubernetes.io/component: hami-mock-device-plugin - {{- include "hami-vgpu.selectorLabels" . | nindent 8 }} - spec: - serviceAccountName: {{ include "hami-vgpu.mock-device-plugin" . }} - tolerations: - - key: CriticalAddonsOnly - operator: Exists - containers: - - image: {{ include "hami.mockDevicePlugin.image" . }} - imagePullPolicy: {{ .Values.mockDevicePlugin.image.pullPolicy }} - name: hami-mock-dp-cntr - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - command: - - ./k8s-device-plugin - - -v=5 - - --device-config-file=/device-config.yaml - volumeMounts: - - name: dp - mountPath: /var/lib/kubelet/device-plugins - - name: sys - mountPath: /sys - - name: device-config - mountPath: /device-config.yaml - subPath: device-config.yaml - volumes: - - name: dp - hostPath: - path: /var/lib/kubelet/device-plugins - - name: sys - hostPath: - path: /sys - - name: device-config - configMap: - name: {{ include "hami-vgpu.scheduler" . }}-device -{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml deleted file mode 100644 index 1f4c24f3..00000000 --- a/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml +++ /dev/null @@ -1,262 +0,0 @@ -{{- if .Values.devicePlugin.enabled }} -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: {{ include "hami-vgpu.device-plugin" . }} - namespace: {{ include "hami-vgpu.namespace" . }} - labels: - app.kubernetes.io/component: hami-device-plugin - {{- include "hami-vgpu.labels" . | nindent 4 }} - {{- with .Values.global.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} - {{- if .Values.global.annotations }} - annotations: {{ toYaml .Values.global.annotations | nindent 4}} - {{- end }} -spec: - updateStrategy: - {{- with .Values.devicePlugin.updateStrategy }} - {{- toYaml . | nindent 4 }} - {{- end }} - selector: - matchLabels: - app.kubernetes.io/component: hami-device-plugin - {{- include "hami-vgpu.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - app.kubernetes.io/component: hami-device-plugin - hami.io/webhook: ignore - {{- include "hami-vgpu.selectorLabels" . | nindent 8 }} - annotations: - {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} - checksum/hami-scheduler-newversion-config: {{ include (print $.Template.BasePath "/scheduler/configmapnew.yaml") . | sha256sum }} - {{- else }} - checksum/hami-scheduler-config: {{ include (print $.Template.BasePath "/scheduler/configmap.yaml") . | sha256sum }} - {{- end }} - checksum/hami-scheduler-device-config: {{ include (print $.Template.BasePath "/scheduler/device-configmap.yaml") . | sha256sum }} - {{- if .Values.devicePlugin.podAnnotations }} - {{- toYaml .Values.devicePlugin.podAnnotations | nindent 8 }} - {{- end }} - spec: - {{- if .Values.devicePlugin.runtimeClassName }} - runtimeClassName: {{ .Values.devicePlugin.runtimeClassName }} - {{- end }} - serviceAccountName: {{ include "hami-vgpu.device-plugin" . }} - priorityClassName: system-node-critical - hostPID: true - hostNetwork: true - {{- include "hami.devicePlugin.imagePullSecrets" . | nindent 6 }} - {{- if .Values.devicePlugin.gpuOperatorToolkitReady.enabled }} - initContainers: - - name: toolkit-validation - image: {{ include "hami.devicePlugin.image" . }} - imagePullPolicy: {{ .Values.devicePlugin.image.pullPolicy }} - securityContext: - privileged: true - runAsUser: 0 - command: ["sh", "-c"] - args: - - | - echo "Waiting for NVIDIA Toolkit to be ready..." - until [ -f {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }}/toolkit-ready ]; do - echo "Waiting for {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }}/toolkit-ready..." - sleep 5 - done - echo "NVIDIA Toolkit is ready!" - volumeMounts: - - name: nvidia-validations - mountPath: {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath | quote }} - mountPropagation: HostToContainer - readOnly: true - {{- end }} - containers: - - name: device-plugin - image: {{ include "hami.devicePlugin.image" . }} - imagePullPolicy: {{ .Values.devicePlugin.image.pullPolicy }} - lifecycle: - postStart: - exec: - command: ["/bin/sh","-c", {{ printf "/k8s-vgpu/bin/vgpu-init.sh %s/vgpu/" .Values.global.gpuHookPath | quote }}] - command: - - nvidia-device-plugin - - --config-file=/device-config.yaml - - --mig-strategy={{ .Values.devicePlugin.migStrategy }} - - --disable-core-limit={{ .Values.devicePlugin.disablecorelimit }} - {{- range .Values.devicePlugin.extraArgs }} - - {{ . }} - {{- end }} - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: NVIDIA_MIG_MONITOR_DEVICES - value: all - - name: DEVICE_LIST_STRATEGY - value: {{ .Values.devicePlugin.deviceListStrategy }} - - name: HOOK_PATH - value: {{ .Values.global.gpuHookPath }} - {{- if typeIs "bool" .Values.devicePlugin.passDeviceSpecsEnabled }} - - name: PASS_DEVICE_SPECS - value: {{ .Values.devicePlugin.passDeviceSpecsEnabled | quote }} - {{- end }} - {{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }} - - name: NVIDIA_DRIVER_ROOT - value: {{ .Values.devicePlugin.nvidiaDriverRoot }} - {{- end }} - {{- if typeIs "string" .Values.devicePlugin.nvidiaHookPath }} - - name: NVIDIA_CDI_HOOK_PATH - value: {{ .Values.devicePlugin.nvidiaHookPath }} - {{- end }} - {{- if typeIs "bool" .Values.devicePlugin.gdrcopyEnabled }} - - name: GDRCOPY_ENABLED - value: {{ .Values.devicePlugin.gdrcopyEnabled | quote }} - {{- end }} - {{- if typeIs "bool" .Values.devicePlugin.gdsEnabled }} - - name: GDS_ENABLED - value: {{ .Values.devicePlugin.gdsEnabled | quote }} - {{- end }} - {{- if typeIs "bool" .Values.devicePlugin.mofedEnabled }} - - name: MOFED_ENABLED - value: {{ .Values.devicePlugin.mofedEnabled | quote }} - {{- end }} - {{- if eq (.Values.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy | default "spread") "topology-aware" }} - - name: ENABLE_TOPOLOGY_SCORE - value: "true" - {{- end }} - {{- with .Values.devicePlugin.extraEnvs }} - {{- . | toYaml | nindent 12 }} - {{- end }} - securityContext: - privileged: true - allowPrivilegeEscalation: true - capabilities: - drop: ["ALL"] - add: ["SYS_ADMIN"] - resources: - {{- toYaml .Values.devicePlugin.resources | nindent 12 }} - volumeMounts: - - name: device-plugin - mountPath: /var/lib/kubelet/device-plugins - - name: lib - mountPath: {{ printf "%s%s" .Values.global.gpuHookPath "/vgpu" }} - - name: usrbin - mountPath: /usrbin - - name: deviceconfig - mountPath: /config - - name: hosttmp - mountPath: /tmp - - name: device-config - mountPath: /device-config.yaml - subPath: device-config.yaml - - name: cdi-root - mountPath: /var/run/cdi - {{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }} - # We always mount the driver root at /driver-root in the container. - # This is required for CDI detection to work correctly. - - name: driver-root - mountPath: /driver-root - readOnly: true - {{- end }} - - name: vgpu-monitor - image: {{ include "hami.devicePlugin.monitor.image" . }} - imagePullPolicy: {{ .Values.devicePlugin.monitor.image.pullPolicy }} - command: - - "vGPUmonitor" - {{- range .Values.devicePlugin.monitor.extraArgs }} - - {{ . }} - {{- end }} - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: ["ALL"] - add: ["SYS_ADMIN"] - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: NVIDIA_VISIBLE_DEVICES - value: "all" - - name: NVIDIA_MIG_MONITOR_DEVICES - value: "all" - - name: HOOK_PATH - value: "{{ .Values.global.gpuHookPath }}/vgpu" - - name: HAMI_RESYNC_INTERVAL - value: {{ .Values.devicePlugin.monitor.resyncInterval | default "5m" | quote }} - {{- with .Values.devicePlugin.monitor.extraEnvs }} - {{- . | toYaml | nindent 12 }} - {{- end }} - resources: - {{- toYaml .Values.devicePlugin.monitor.resources | nindent 12 }} - volumeMounts: - - name: ctrs - mountPath: {{ .Values.devicePlugin.monitor.ctrPath }} - - name: dockers - mountPath: /run/docker - - name: containerds - mountPath: /run/containerd - - name: sysinfo - mountPath: /sysinfo - - name: hostvar - mountPath: /hostvar - - name: hosttmp - mountPath: /tmp - volumes: - - name: ctrs - hostPath: - path: {{ .Values.devicePlugin.monitor.ctrPath }} - - name: hosttmp - hostPath: - path: /tmp - - name: dockers - hostPath: - path: /run/docker - - name: containerds - hostPath: - path: /run/containerd - - name: device-plugin - hostPath: - path: {{ .Values.devicePlugin.pluginPath }} - - name: lib - hostPath: - path: {{ .Values.devicePlugin.libPath }} - {{- if .Values.devicePlugin.gpuOperatorToolkitReady.enabled }} - - name: nvidia-validations - hostPath: - path: {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }} - type: DirectoryOrCreate - {{- end }} - {{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }} - - name: driver-root - hostPath: - path: {{ .Values.devicePlugin.nvidiaDriverRoot }} - type: Directory - {{- end }} - - name: cdi-root - hostPath: - path: /var/run/cdi - type: DirectoryOrCreate - - name: usrbin - hostPath: - path: /usr/bin - - name: sysinfo - hostPath: - path: /sys - - name: hostvar - hostPath: - path: /var - - name: deviceconfig - configMap: - name: {{ .Values.devicePlugin.nodeConfiguration.externalConfigName | default (include "hami-vgpu.device-plugin" .) }} - - name: device-config - configMap: - name: {{ include "hami-vgpu.scheduler" . }}-device - {{- if .Values.devicePlugin.nvidiaNodeSelector }} - nodeSelector: {{ toYaml .Values.devicePlugin.nvidiaNodeSelector | nindent 8 }} - {{- end }} - {{- if .Values.devicePlugin.tolerations }} - tolerations: {{ toYaml .Values.devicePlugin.tolerations | nindent 8 }} - {{- end }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml deleted file mode 100644 index 6ac757f2..00000000 --- a/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{- if .Values.devicePlugin.enabled -}} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "hami-vgpu.device-plugin" . }}-monitor -rules: - - apiGroups: - - "" - resources: - - pods - verbs: - - get - - create - - watch - - list - - update - - patch - - apiGroups: - - "" - resources: - - nodes - verbs: - - get - - update - - list - - patch -{{- end -}} - diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml deleted file mode 100644 index 2f0a14ba..00000000 --- a/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml +++ /dev/null @@ -1,17 +0,0 @@ -{{- if .Values.devicePlugin.enabled -}} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "hami-vgpu.device-plugin" . }} - labels: - app.kubernetes.io/component: "hami-device-plugin" - {{- include "hami-vgpu.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "hami-vgpu.device-plugin" . }}-monitor -subjects: - - kind: ServiceAccount - name: {{ include "hami-vgpu.device-plugin" . }} - namespace: {{ include "hami-vgpu.namespace" . }} -{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml deleted file mode 100644 index 88f1b214..00000000 --- a/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml +++ /dev/null @@ -1,29 +0,0 @@ -{{- if .Values.devicePlugin.enabled -}} -apiVersion: v1 -kind: Service -metadata: - name: {{ include "hami-vgpu.device-plugin" . }}-monitor - namespace: {{ include "hami-vgpu.namespace" . }} - labels: - app.kubernetes.io/component: hami-device-plugin - {{- include "hami-vgpu.labels" . | nindent 4 }} - {{- with .Values.devicePlugin.service.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} - {{- if .Values.devicePlugin.service.annotations }} # Use devicePlugin instead of scheduler - annotations: {{ toYaml .Values.devicePlugin.service.annotations | nindent 4 }} - {{- end }} -spec: - type: {{ .Values.devicePlugin.service.type | default "NodePort" }} # Default type is NodePort - ports: - - name: monitorport - port: {{ .Values.devicePlugin.service.httpPort | default 31992 }} # Default HTTP port is 31992 - targetPort: 9394 - {{- if eq (.Values.devicePlugin.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort - nodePort: {{ .Values.devicePlugin.service.httpPort | default 31992 }} - {{- end }} - protocol: TCP - selector: - app.kubernetes.io/component: hami-device-plugin - {{- include "hami-vgpu.selectorLabels" . | nindent 4 }} -{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml deleted file mode 100644 index 6e3c2a44..00000000 --- a/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml +++ /dev/null @@ -1,10 +0,0 @@ -{{- if .Values.devicePlugin.enabled -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "hami-vgpu.device-plugin" . }} - namespace: {{ include "hami-vgpu.namespace" . }} - labels: - app.kubernetes.io/component: "hami-device-plugin" - {{- include "hami-vgpu.labels" . | nindent 4 }} -{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml b/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml deleted file mode 100644 index ebcc2c9c..00000000 --- a/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml +++ /dev/null @@ -1,9 +0,0 @@ -{{- if and .Values.devicePlugin.enabled .Values.devicePlugin.createRuntimeClass .Values.devicePlugin.runtimeClassName -}} -apiVersion: node.k8s.io/v1 -kind: RuntimeClass -metadata: - name: {{ .Values.devicePlugin.runtimeClassName }} - annotations: - helm.sh/hook: pre-install,pre-upgrade -handler: nvidia -{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml b/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml deleted file mode 100644 index e6d28721..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml +++ /dev/null @@ -1,31 +0,0 @@ -{{- if .Values.scheduler.admissionWebhook.enabled -}} -{{- if .Values.scheduler.certManager.enabled }} -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: {{ include "hami-vgpu.scheduler" . }}-serving-cert - namespace: {{ include "hami-vgpu.namespace" . }} - labels: - app.kubernetes.io/component: hami-scheduler - {{- include "hami-vgpu.labels" . | nindent 4 }} -spec: - dnsNames: - - {{ include "hami-vgpu.scheduler" . }}.{{ include "hami-vgpu.namespace" . }}.svc - - {{ include "hami-vgpu.scheduler" . }}.{{ include "hami-vgpu.namespace" . }}.svc.cluster.local - issuerRef: - kind: Issuer - name: {{ include "hami-vgpu.scheduler" . }}-selfsigned-issuer - secretName: {{ include "hami-vgpu.scheduler.tls" . }} ---- -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - name: {{ include "hami-vgpu.scheduler" . }}-selfsigned-issuer - namespace: {{ include "hami-vgpu.namespace" . }} - labels: - app.kubernetes.io/component: hami-scheduler - {{- include "hami-vgpu.labels" . | nindent 4 }} -spec: - selfSigned: {} -{{- end }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml b/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml deleted file mode 100644 index 81c4fddf..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "hami-vgpu.scheduler" . }} - labels: - app.kubernetes.io/component: "hami-scheduler" - {{- include "hami-vgpu.labels" . | nindent 4 }} -rules: - - apiGroups: [""] - resources: ["pods", "configmaps"] - verbs: ["get", "list", "watch", "patch"] - - apiGroups: [""] - resources: ["pods/binding"] - verbs: ["create"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "patch", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["create", "get", "list"] - - apiGroups: [""] - resources: ["resourcequotas"] - verbs: ["get", "list", "watch"] ---- -{{- if .Values.mockDevicePlugin.enabled }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "hami-vgpu.mock-device-plugin" . }} -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "update", "list", "patch"] -{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml deleted file mode 100644 index a81d425c..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml +++ /dev/null @@ -1,47 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "hami-vgpu.scheduler" . }}-kube - labels: - app.kubernetes.io/component: "hami-scheduler" - {{- include "hami-vgpu.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:kube-scheduler -subjects: - - kind: ServiceAccount - name: {{ include "hami-vgpu.scheduler" . }} - namespace: {{ include "hami-vgpu.namespace" . }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "hami-vgpu.scheduler" . }}-volume - labels: - app.kubernetes.io/component: "hami-scheduler" - {{- include "hami-vgpu.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:volume-scheduler -subjects: - - kind: ServiceAccount - name: {{ include "hami-vgpu.scheduler" . }} - namespace: {{ include "hami-vgpu.namespace" . }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "hami-vgpu.scheduler" . }} - labels: - app.kubernetes.io/component: "hami-scheduler" - {{- include "hami-vgpu.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "hami-vgpu.scheduler" . }} -subjects: - - kind: ServiceAccount - name: {{ include "hami-vgpu.scheduler" . }} - namespace: {{ include "hami-vgpu.namespace" . }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml b/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml deleted file mode 100644 index 109ecbce..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml +++ /dev/null @@ -1,142 +0,0 @@ -{{- if .Values.scheduler.kubeScheduler.enabled -}} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "hami-vgpu.scheduler" . }} - namespace: {{ include "hami-vgpu.namespace" . }} - labels: - app.kubernetes.io/component: hami-scheduler - {{- include "hami-vgpu.labels" . | nindent 4 }} -data: - config.json: | - { - "kind": "Policy", - "apiVersion": "v1", - "extenders": [ - { - {{- if .Values.scheduler.admissionWebhook.enabled }} - "urlPrefix": "https://127.0.0.1:443", - "enableHttps": true, - "tlsConfig": { - "insecure": true - }, - {{- else }} - "urlPrefix": "http://127.0.0.1:80", - "enableHttps": false, - {{- end }} - "filterVerb": "filter", - "bindVerb": "bind", - "weight": 1, - "nodeCacheCapable": true, - "httpTimeout": 30000000000, - "managedResources": [ - {{- range .Values.devices.amd.customresources }} - { - "name": "{{ . }}", - "ignoredByScheduler": true - }, - {{- end }} - {{- if .Values.devices.ascend.enabled }} - {{- range .Values.devices.ascend.customresources }} - { - "name": "{{ . }}", - "ignoredByScheduler": true - }, - {{- end }} - {{- end }} - {{- if .Values.devices.mthreads.enabled }} - {{- range .Values.devices.mthreads.customresources }} - { - "name": "{{ . }}", - "ignoredByScheduler": true - }, - {{- end }} - {{- end }} - {{- if .Values.devices.enflame.enabled }} - {{- range .Values.devices.enflame.customresources }} - { - "name": "{{ . }}", - "ignoredByScheduler": true - }, - {{- end }} - {{- end }} - {{- if .Values.devices.kunlun.enabled }} - {{- range .Values.devices.kunlun.customresources }} - { - "name": "{{ . }}", - "ignoredByScheduler": true - }, - {{- end }} - {{- end }} - {{- range .Values.devices.awsneuron.customresources }} - { - "name": "{{ . }}", - "ignoredByScheduler": true - }, - {{- end }} - {{- if .Values.devices.iluvatar.enabled }} - {{- range .Values.devices.iluvatar.customresources }} - { - "name": "{{ . }}", - "ignoredByScheduler": true - }, - {{- end }} - {{- end }} - { - "name": "{{ .Values.resourceName }}", - "ignoredByScheduler": true - }, - { - "name": "{{ .Values.resourceMem }}", - "ignoredByScheduler": true - }, - { - "name": "{{ .Values.resourceCores }}", - "ignoredByScheduler": true - }, - { - "name": "{{ .Values.resourceMemPercentage }}", - "ignoredByScheduler": true - }, - { - "name": "{{ .Values.resourcePriority }}", - "ignoredByScheduler": true - }, - { - "name": "{{ .Values.mluResourceName }}", - "ignoredByScheduler": true - }, - { - "name": "{{ .Values.dcuResourceName }}", - "ignoredByScheduler": true - }, - { - "name": "{{ .Values.dcuResourceMem }}", - "ignoredByScheduler": true - }, - { - "name": "{{ .Values.dcuResourceCores }}", - "ignoredByScheduler": true - }, - { - "name": "metax-tech.com/gpu", - "ignoredByScheduler": true - }, - { - "name": "{{ .Values.metaxResourceName }}", - "ignoredByScheduler": true - }, - { - "name": "{{ .Values.metaxResourceCore }}", - "ignoredByScheduler": true - }, - { - "name": "{{ .Values.metaxResourceMem }}", - "ignoredByScheduler": true - } - ], - "ignoreable": false - } - ] - } -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml b/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml deleted file mode 100644 index 6f6db097..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml +++ /dev/null @@ -1,102 +0,0 @@ -{{- if .Values.scheduler.kubeScheduler.enabled -}} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "hami-vgpu.scheduler" . }}-newversion - namespace: {{ include "hami-vgpu.namespace" . }} - labels: - app.kubernetes.io/component: hami-scheduler - {{- include "hami-vgpu.labels" . | nindent 4 }} -data: - config.yaml: | - {{- if gt (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 25}} - apiVersion: kubescheduler.config.k8s.io/v1 - {{- else }} - apiVersion: kubescheduler.config.k8s.io/v1beta2 - {{- end }} - kind: KubeSchedulerConfiguration - leaderElection: - leaderElect: false - profiles: - - schedulerName: {{ .Values.schedulerName }} - extenders: - {{- if .Values.scheduler.admissionWebhook.enabled }} - - urlPrefix: "https://127.0.0.1:443" - enableHTTPS: true - tlsConfig: - insecure: true - {{- else }} - - urlPrefix: "http://127.0.0.1:80" - enableHTTPS: false - {{- end }} - filterVerb: filter - bindVerb: bind - nodeCacheCapable: true - weight: 1 - httpTimeout: 30s - managedResources: - - name: {{ .Values.resourceName }} - ignoredByScheduler: true - - name: {{ .Values.resourceMem }} - ignoredByScheduler: true - - name: {{ .Values.resourceCores }} - ignoredByScheduler: true - - name: {{ .Values.resourceMemPercentage }} - ignoredByScheduler: true - - name: {{ .Values.resourcePriority }} - ignoredByScheduler: true - - name: {{ .Values.mluResourceName }} - ignoredByScheduler: true - - name: {{ .Values.dcuResourceName }} - ignoredByScheduler: true - - name: {{ .Values.dcuResourceMem }} - ignoredByScheduler: true - - name: {{ .Values.dcuResourceCores }} - ignoredByScheduler: true - - name: "metax-tech.com/gpu" - ignoredByScheduler: true - - name: {{ .Values.metaxResourceName }} - ignoredByScheduler: true - - name: {{ .Values.metaxResourceCore }} - ignoredByScheduler: true - - name: {{ .Values.metaxResourceMem }} - ignoredByScheduler: true - {{- if .Values.devices.ascend.enabled }} - {{- range .Values.devices.ascend.customresources }} - - name: {{ . }} - ignoredByScheduler: true - {{- end }} - {{- end }} - {{- if .Values.devices.mthreads.enabled }} - {{- range .Values.devices.mthreads.customresources }} - - name: {{ . }} - ignoredByScheduler: true - {{- end }} - {{- end }} - {{- if .Values.devices.enflame.enabled }} - {{- range .Values.devices.enflame.customresources }} - - name: {{ . }} - ignoredByScheduler: true - {{- end }} - {{- end }} - {{- if .Values.devices.kunlun.enabled }} - {{- range .Values.devices.kunlun.customresources }} - - name: {{ . }} - ignoredByScheduler: true - {{- end }} - {{- end }} - {{- range .Values.devices.awsneuron.customresources }} - - name: {{ . }} - ignoredByScheduler: true - {{- end }} - {{- if .Values.devices.iluvatar.enabled }} - {{- range .Values.devices.iluvatar.customresources }} - - name: {{ . }} - ignoredByScheduler: true - {{- end }} - {{- end }} - {{- range .Values.devices.amd.customresources }} - - name: {{ . }} - ignoredByScheduler: true - {{- end }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml b/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml deleted file mode 100644 index 1d89e189..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml +++ /dev/null @@ -1,225 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "hami-vgpu.scheduler" . }} - namespace: {{ include "hami-vgpu.namespace" . }} - labels: - app.kubernetes.io/component: hami-scheduler - {{- include "hami-vgpu.labels" . | nindent 4 }} - {{- with .Values.global.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} - {{- if .Values.global.annotations }} - annotations: {{ toYaml .Values.global.annotations | nindent 4}} - {{- end }} -spec: - {{- if .Values.scheduler.leaderElect }} - replicas: {{ .Values.scheduler.replicas }} - {{- else }} - replicas: 1 - {{- end }} - selector: - matchLabels: - app.kubernetes.io/component: hami-scheduler - {{- include "hami-vgpu.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - app.kubernetes.io/component: hami-scheduler - {{- include "hami-vgpu.selectorLabels" . | nindent 8 }} - hami.io/webhook: ignore - annotations: - {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} - checksum/hami-scheduler-newversion-config: {{ include (print $.Template.BasePath "/scheduler/configmapnew.yaml") . | sha256sum }} - {{- else }} - checksum/hami-scheduler-config: {{ include (print $.Template.BasePath "/scheduler/configmap.yaml") . | sha256sum }} - {{- end }} - checksum/hami-scheduler-device-config: {{ include (print $.Template.BasePath "/scheduler/device-configmap.yaml") . | sha256sum }} - {{- if .Values.scheduler.podAnnotations }} - {{- toYaml .Values.scheduler.podAnnotations | nindent 8 }} - {{- end }} - spec: - serviceAccountName: {{ include "hami-vgpu.scheduler" . }} - priorityClassName: system-node-critical - {{- include "hami.scheduler.extender.imagePullSecrets" . | nindent 6 }} - containers: - {{- if .Values.scheduler.kubeScheduler.enabled }} - - name: kube-scheduler - image: {{ include "hami.scheduler.kubeScheduler.image" . }} - imagePullPolicy: {{ .Values.scheduler.kubeScheduler.image.pullPolicy }} - command: - - kube-scheduler - {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} - {{- range .Values.scheduler.kubeScheduler.extraNewArgs }} - - {{ . }} - {{- end }} - {{- else }} - - --scheduler-name={{ .Values.schedulerName }} - {{- range .Values.scheduler.kubeScheduler.extraArgs }} - - {{ . }} - {{- end }} - {{- end }} - - --leader-elect={{ .Values.scheduler.leaderElect }} - - --leader-elect-resource-name={{ .Values.schedulerName }} - - --leader-elect-resource-namespace={{ include "hami-vgpu.namespace" . }} - resources: - {{- toYaml .Values.scheduler.kubeScheduler.resources | nindent 12 }} - volumeMounts: - - name: scheduler-config - mountPath: /config - {{- end }} - {{- if .Values.scheduler.livenessProbe }} - livenessProbe: - failureThreshold: 8 - httpGet: - path: /healthz - port: 10259 - scheme: HTTPS - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 15 - {{- end }} - - name: vgpu-scheduler-extender - image: {{ include "hami.scheduler.extender.image" . }} - imagePullPolicy: {{ .Values.scheduler.extender.image.pullPolicy }} - env: - {{- if .Values.scheduler.nodeLockExpire }} - - name: HAMI_NODELOCK_EXPIRE - value: "{{ .Values.scheduler.nodeLockExpire }}" - {{- end }} - {{- if .Values.global.managedNodeSelectorEnable }} - {{- range $key, $value := .Values.global.managedNodeSelector }} - - name: NODE_SELECTOR_{{ $key | upper | replace "-" "_" }} - value: "{{ $value }}" - {{- end }} - {{- end }} - command: - - scheduler - {{- if .Values.scheduler.admissionWebhook.enabled }} - - --http_bind=0.0.0.0:443 - - --cert_file=/tls/tls.crt - - --key_file=/tls/tls.key - {{- else }} - - --http_bind=0.0.0.0:80 - {{- end }} - - --scheduler-name={{ .Values.schedulerName }} - - --metrics-bind-address={{ .Values.scheduler.metricsBindAddress }} - - --node-scheduler-policy={{ .Values.scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy }} - - --gpu-scheduler-policy={{ .Values.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy }} - - --force-overwrite-default-scheduler={{ .Values.scheduler.forceOverwriteDefaultScheduler}} - - --device-config-file=/device-config.yaml - - --leader-elect={{ .Values.scheduler.leaderElect }} - - --leader-elect-resource-name={{ .Values.schedulerName }} - - --leader-elect-resource-namespace={{ include "hami-vgpu.namespace" . }} - {{- if .Values.devices.ascend.enabled }} - - --enable-ascend=true - {{- end }} - {{- if .Values.devices.iluvatar.enabled }} - - --enable-iluvatar=true - {{- end }} - {{- if .Values.scheduler.nodeLabelSelector }} - - --node-label-selector={{- $first := true -}} - {{- range $key, $value := .Values.scheduler.nodeLabelSelector -}} - {{- if not $first }},{{ end -}} - {{- $key }}={{ $value -}} - {{- $first = false -}} - {{- end -}} - {{- end }} - {{- range .Values.scheduler.extender.extraArgs }} - - {{ . }} - {{- end }} - ports: - {{- if .Values.scheduler.admissionWebhook.enabled }} - - name: https - containerPort: 443 - protocol: TCP - {{- else }} - - name: http - containerPort: 80 - protocol: TCP - {{- end }} - - name: metrics - containerPort: {{ last (splitList ":" .Values.scheduler.metricsBindAddress) | int }} - protocol: TCP - resources: - {{- toYaml .Values.scheduler.extender.resources | nindent 12 }} - volumeMounts: - - name: device-config - mountPath: /device-config.yaml - subPath: device-config.yaml - {{- if .Values.scheduler.admissionWebhook.enabled }} - - name: tls-config - mountPath: /tls - {{- end }} - {{- if .Values.scheduler.livenessProbe }} - livenessProbe: - httpGet: - path: /healthz - {{- if .Values.scheduler.admissionWebhook.enabled }} - port: https - scheme: HTTPS - {{- else }} - port: http - scheme: HTTP - {{- end }} - initialDelaySeconds: 10 - periodSeconds: 10 - failureThreshold: 3 - timeoutSeconds: 5 - {{- end }} - {{- if .Values.scheduler.leaderElect }} - readinessProbe: - httpGet: - path: /readyz - {{- if .Values.scheduler.admissionWebhook.enabled }} - port: https - scheme: HTTPS - {{- else }} - port: http - scheme: HTTP - {{- end }} - initialDelaySeconds: 10 - periodSeconds: 10 - failureThreshold: 3 - timeoutSeconds: 5 - {{- end }} - {{- if .Values.scheduler.leaderElect }} - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app.kubernetes.io/component - operator: In - values: - - hami-scheduler - topologyKey: "kubernetes.io/hostname" - {{- end }} - volumes: - {{- if .Values.scheduler.admissionWebhook.enabled }} - - name: tls-config - secret: - secretName: {{ template "hami-vgpu.scheduler.tls" . }} - {{- end }} - {{- if .Values.scheduler.kubeScheduler.enabled }} - - name: scheduler-config - configMap: - {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} - name: {{ template "hami-vgpu.scheduler" . }}-newversion - {{- else }} - name: {{ template "hami-vgpu.scheduler" . }} - {{- end }} - {{- end }} - - name: device-config - configMap: - name: {{ include "hami-vgpu.scheduler" . }}-device - {{- if .Values.scheduler.nodeSelector }} - nodeSelector: {{ toYaml .Values.scheduler.nodeSelector | nindent 8 }} - {{- end }} - {{- if .Values.scheduler.tolerations }} - tolerations: {{ toYaml .Values.scheduler.tolerations | nindent 8 }} - {{- end }} - {{- if .Values.scheduler.nodeName }} - nodeName: {{ .Values.scheduler.nodeName }} - {{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml b/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml deleted file mode 100644 index 873b813b..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml +++ /dev/null @@ -1,408 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "hami-vgpu.scheduler" . }}-device - namespace: {{ include "hami-vgpu.namespace" . }} - labels: - app.kubernetes.io/component: hami-scheduler - {{- include "hami-vgpu.labels" . | nindent 4 }} -data: - device-config.yaml: |- - {{- if .Files.Glob "files/device-config.yaml" }} - {{- .Files.Get "files/device-config.yaml" | nindent 4}} - {{- else }} - nvidia: - resourceCountName: {{ .Values.resourceName }} - resourceMemoryName: {{ .Values.resourceMem }} - resourceMemoryPercentageName: {{ .Values.resourceMemPercentage }} - resourceCoreName: {{ .Values.resourceCores }} - resourcePriorityName: {{ .Values.resourcePriority }} - overwriteEnv: false - defaultMemory: 0 - defaultCores: 0 - defaultGPUNum: 1 - memoryFactor: 1 - deviceSplitCount: {{ .Values.devicePlugin.deviceSplitCount }} - deviceMemoryScaling: {{ .Values.devicePlugin.deviceMemoryScaling }} - deviceCoreScaling: {{ .Values.devicePlugin.deviceCoreScaling }} - gpuCorePolicy: {{ .Values.devices.nvidia.gpuCorePolicy }} - libCudaLogLevel: {{ .Values.devices.nvidia.libCudaLogLevel }} - runtimeClassName: "{{ .Values.devicePlugin.runtimeClassName }}" - knownMigGeometries: - - models: [ "A30" ] - allowedGeometries: - - - - name: 1g.6gb - core: 25 - memory: 6144 - count: 4 - - - - name: 2g.12gb - core: 50 - memory: 12288 - count: 2 - - - - name: 4g.24gb - core: 100 - memory: 24576 - count: 1 - - models: [ "A100-SXM4-40GB", "A100-40GB-PCIe", "A100-PCIE-40GB"] - allowedGeometries: - - - - name: 1g.5gb - core: 14 - memory: 5120 - count: 7 - - - - name: 1g.5gb - core: 14 - memory: 5120 - count: 1 - - name: 2g.10gb - core: 28 - memory: 10240 - count: 3 - - - - name: 3g.20gb - core: 42 - memory: 20480 - count: 2 - - - - name: 7g.40gb - core: 100 - memory: 40960 - count: 1 - - models: [ "A100-SXM4-80GB", "A100-80GB-PCIe", "A100-PCIE-80GB"] - allowedGeometries: - - - - name: 1g.10gb - core: 14 - memory: 10240 - count: 7 - - - - name: 1g.10gb - core: 14 - memory: 10240 - count: 1 - - name: 2g.20gb - core: 28 - memory: 20480 - count: 3 - - - - name: 3g.40gb - core: 42 - memory: 40960 - count: 2 - - - - name: 7g.79gb - core: 100 - memory: 80896 - count: 1 - - models: [ "H100-PCIE-80GB", "H100-SXM5-80GB"] - allowedGeometries: - - - - name: 1g.10gb - core: 14 - memory: 10240 - count: 7 - - - - name: 1g.10gb - core: 14 - memory: 10240 - count: 1 - - name: 2g.20gb - core: 28 - memory: 20480 - count: 3 - - - - name: 3g.40gb - core: 42 - memory: 40960 - count: 2 - - - - name: 7g.80gb - core: 100 - memory: 81920 - count: 1 - - models: [ "H100-PCIE-94GB", "H100-SXM5-94GB"] - allowedGeometries: - - - - name: 1g.12gb - core: 14 - memory: 12288 - count: 7 - - - - name: 1g.12gb - core: 14 - memory: 12288 - count: 1 - - name: 2g.24gb - core: 28 - memory: 24576 - count: 3 - - - - name: 3g.47gb - core: 42 - memory: 48128 - count: 2 - - - - name: 7g.94gb - core: 100 - memory: 96256 - count: 1 - - models: [ "H20", "H100 on GH200"] - allowedGeometries: - - - - name: 1g.12gb - core: 14 - memory: 12288 - count: 7 - - - - name: 1g.12gb - core: 14 - memory: 12288 - count: 1 - - name: 2g.24gb - core: 28 - memory: 24576 - count: 3 - - - - name: 3g.48gb - core: 42 - memory: 49152 - count: 2 - - - - name: 7g.96gb - core: 100 - memory: 98304 - count: 1 - - models: [ "H200 NVL", "H200-SXM5"] - allowedGeometries: - - - - name: 1g.18gb - core: 14 - memory: 18432 - count: 7 - - - - name: 1g.18gb - core: 14 - memory: 18432 - count: 1 - - name: 2g.35gb - core: 28 - memory: 35840 - count: 3 - - - - name: 3g.71gb - core: 42 - memory: 72704 - count: 2 - - - - name: 7g.141gb - core: 100 - memory: 144384 - count: 1 - - models: [ "B200" ] - allowedGeometries: - - - - name: 1g.23gb - core: 14 - memory: 23552 - count: 7 - - - - name: 1g.23gb - core: 14 - memory: 23552 - count: 1 - - name: 2g.45gb - core: 28 - memory: 46080 - count: 3 - - - - name: 3g.90gb - core: 42 - memory: 92160 - count: 2 - - - - name: 7g.180gb - core: 100 - memory: 184320 - count: 1 - cambricon: - resourceCountName: {{ .Values.mluResourceName }} - resourceMemoryName: {{ .Values.mluResourceMem }} - resourceCoreName: {{ .Values.mluResourceCores }} - hygon: - resourceCountName: {{ .Values.dcuResourceName }} - resourceMemoryName: {{ .Values.dcuResourceMem }} - resourceCoreName: {{ .Values.dcuResourceCores }} - memoryFactor: 1 - metax: - resourceCountName: "metax-tech.com/gpu" - resourceVCountName: {{ .Values.metaxResourceName }} - resourceVMemoryName: {{ .Values.metaxResourceMem }} - resourceVCoreName: {{ .Values.metaxResourceCore }} - sgpuTopologyAware: {{ .Values.metaxsGPUTopologyAware }} - enflame: - resourceNameGCU: "enflame.com/gcu" - resourceNameVGCU: {{ .Values.enflameResourceNameVGCU }} - resourceNameVGCUPercentage: {{ .Values.enflameResourceNameVGCUPercentage }} - mthreads: - resourceCountName: "mthreads.com/vgpu" - resourceMemoryName: "mthreads.com/sgpu-memory" - resourceCoreName: "mthreads.com/sgpu-core" - iluvatars: - - chipName: MR-V100 - commonWord: MR-V100 - resourceCountName: iluvatar.ai/MR-V100-vgpu - resourceMemoryName: iluvatar.ai/MR-V100.vMem - resourceCoreName: iluvatar.ai/MR-V100.vCore - - chipName: MR-V50 - commonWord: MR-V50 - resourceCountName: iluvatar.ai/MR-V50-vgpu - resourceMemoryName: iluvatar.ai/MR-V50.vMem - resourceCoreName: iluvatar.ai/MR-V50.vCore - - chipName: BI-V150 - commonWord: BI-V150 - resourceCountName: iluvatar.ai/BI-V150-vgpu - resourceMemoryName: iluvatar.ai/BI-V150.vMem - resourceCoreName: iluvatar.ai/BI-V150.vCore - - chipName: BI-V100 - commonWord: BI-V100 - resourceCountName: iluvatar.ai/BI-V100-vgpu - resourceMemoryName: iluvatar.ai/BI-V100.vMem - resourceCoreName: iluvatar.ai/BI-V100.vCore - kunlun: - resourceCountName: {{ .Values.kunlunResourceName }} - resourceVCountName: {{ .Values.kunlunResourceVCountName }} - resourceVMemoryName: {{ .Values.kunlunResourceVMemoryName }} - awsneuron: - resourceCountName: "aws.amazon.com/neuron" - resourceCoreName: "aws.amazon.com/neuroncore" - amd: - resourceCountName: "amd.com/gpu" - vnpus: - - chipName: 910A - commonWord: Ascend910A - resourceName: huawei.com/Ascend910A - resourceMemoryName: huawei.com/Ascend910A-memory - memoryAllocatable: 32768 - memoryCapacity: 32768 - memoryFactor: 1 - aiCore: 30 - templates: - - name: vir02 - memory: 2184 - aiCore: 2 - - name: vir04 - memory: 4369 - aiCore: 4 - - name: vir08 - memory: 8738 - aiCore: 8 - - name: vir16 - memory: 17476 - aiCore: 16 - - chipName: 910B2 - commonWord: Ascend910B2 - resourceName: huawei.com/Ascend910B2 - resourceMemoryName: huawei.com/Ascend910B2-memory - memoryAllocatable: 65536 - memoryCapacity: 65536 - memoryFactor: 1 - aiCore: 24 - aiCPU: 6 - templates: - - name: vir03_1c_8g - memory: 8192 - aiCore: 3 - aiCPU: 1 - - name: vir06_1c_16g - memory: 16384 - aiCore: 6 - aiCPU: 1 - - name: vir12_3c_32g - memory: 32768 - aiCore: 12 - aiCPU: 3 - - chipName: 910B3 - commonWord: Ascend910B3 - resourceName: huawei.com/Ascend910B3 - resourceMemoryName: huawei.com/Ascend910B3-memory - memoryAllocatable: 65536 - memoryCapacity: 65536 - memoryFactor: 1 - aiCore: 20 - aiCPU: 7 - templates: - - name: vir05_1c_16g - memory: 16384 - aiCore: 5 - aiCPU: 1 - - name: vir10_3c_32g - memory: 32768 - aiCore: 10 - aiCPU: 3 - - chipName: 910B4-1 - commonWord: Ascend910B4-1 - resourceName: huawei.com/Ascend910B4-1 - resourceMemoryName: huawei.com/Ascend910B4-1-memory - memoryAllocatable: 65536 - memoryCapacity: 65536 - memoryFactor: 1 - aiCore: 20 - aiCPU: 7 - templates: - # NOTE: Names of vnpu template for 910B4-1 are fixed by Ascend runtime and must not be changed. - # The memory is used for scheduling so the correct values must be set. - # Template vir05_1c_8g actually provides 16GB memory, - - name: vir05_1c_8g - memory: 16384 - aiCore: 5 - aiCPU: 1 - # Template vir10_3c_16g actually provides 32GB memory - - name: vir10_3c_16g - memory: 32768 - aiCore: 10 - aiCPU: 3 - - chipName: 910B4 - commonWord: Ascend910B4 - resourceName: huawei.com/Ascend910B4 - resourceMemoryName: huawei.com/Ascend910B4-memory - memoryAllocatable: 32768 - memoryCapacity: 32768 - memoryFactor: 1 - aiCore: 20 - aiCPU: 7 - templates: - - name: vir05_1c_8g - memory: 8192 - aiCore: 5 - aiCPU: 1 - - name: vir10_3c_16g - memory: 16384 - aiCore: 10 - aiCPU: 3 - - chipName: 310P3 - commonWord: Ascend310P - resourceName: huawei.com/Ascend310P - resourceMemoryName: huawei.com/Ascend310P-memory - memoryAllocatable: 21527 - memoryCapacity: 24576 - memoryFactor: 1 - aiCore: 8 - aiCPU: 7 - templates: - - name: vir01 - memory: 3072 - aiCore: 1 - aiCPU: 1 - - name: vir02 - memory: 6144 - aiCore: 2 - aiCPU: 2 - - name: vir04 - memory: 12288 - aiCore: 4 - aiCPU: 4 - {{ end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml deleted file mode 100644 index 77e891cc..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml +++ /dev/null @@ -1,21 +0,0 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "hami-vgpu.fullname" . }}-admission - annotations: - "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded - labels: - {{- include "hami-vgpu.labels" . | nindent 4 }} - app.kubernetes.io/component: admission-webhook -rules: - - apiGroups: - - admissionregistration.k8s.io - resources: - #- validatingwebhookconfigurations - - mutatingwebhookconfigurations - verbs: - - get - - update -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml deleted file mode 100644 index 2b82f926..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml +++ /dev/null @@ -1,20 +0,0 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "hami-vgpu.fullname" . }}-admission - annotations: - "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded - labels: - {{- include "hami-vgpu.labels" . | nindent 4 }} - app.kubernetes.io/component: admission-webhook -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "hami-vgpu.fullname" . }}-admission -subjects: - - kind: ServiceAccount - name: {{ include "hami-vgpu.fullname" . }}-admission - namespace: {{ include "hami-vgpu.namespace" . }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml deleted file mode 100644 index 0e36d95f..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml +++ /dev/null @@ -1,68 +0,0 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ include "hami-vgpu.fullname" . }}-admission-create - namespace: {{ include "hami-vgpu.namespace" . }} - annotations: - "helm.sh/hook": pre-install,pre-upgrade - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded - labels: - {{- include "hami-vgpu.labels" . | nindent 4 }} - app.kubernetes.io/component: admission-webhook -spec: - {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} - # Alpha feature since k8s 1.12 - ttlSecondsAfterFinished: 0 - {{- end }} - template: - metadata: - name: {{ include "hami-vgpu.fullname" . }}-admission-create - {{- if .Values.scheduler.patch.podAnnotations }} - annotations: {{ toYaml .Values.scheduler.patch.podAnnotations | nindent 8 }} - {{- end }} - labels: - {{- include "hami-vgpu.labels" . | nindent 8 }} - app.kubernetes.io/component: admission-webhook - hami.io/webhook: ignore - spec: - {{- if .Values.scheduler.patch.priorityClassName }} - priorityClassName: {{ .Values.scheduler.patch.priorityClassName }} - {{- end }} - {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} - {{- include "hami.scheduler.patch.new.imagePullSecrets" . | nindent 6 }} - {{- else }} - {{- include "hami.scheduler.patch.imagePullSecrets" . | nindent 6 }} - {{- end }} - containers: - - name: create - {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} - image: {{ include "hami.scheduler.patch.new.image" . }} - imagePullPolicy: {{ .Values.scheduler.patch.imageNew.pullPolicy }} - {{- else }} - image: {{ include "hami.scheduler.patch.image" . }} - imagePullPolicy: {{ .Values.scheduler.patch.image.pullPolicy }} - {{- end }} - args: - - create - - --cert-name=tls.crt - - --key-name=tls.key - {{- if .Values.scheduler.admissionWebhook.customURL.enabled }} - - --host={{ printf "%s.%s.svc,127.0.0.1,%s" (include "hami-vgpu.scheduler" .) (include "hami-vgpu.namespace" .) .Values.scheduler.admissionWebhook.customURL.host}} - {{- else }} - - --host={{ printf "%s.%s.svc,127.0.0.1" (include "hami-vgpu.scheduler" .) (include "hami-vgpu.namespace" .) }} - {{- end }} - - --namespace={{ include "hami-vgpu.namespace" . }} - - --secret-name={{ include "hami-vgpu.scheduler.tls" . }} - restartPolicy: OnFailure - serviceAccountName: {{ include "hami-vgpu.fullname" . }}-admission - {{- if .Values.scheduler.patch.nodeSelector }} - nodeSelector: {{ toYaml .Values.scheduler.patch.nodeSelector | nindent 8 }} - {{- end }} - {{- if .Values.scheduler.patch.tolerations }} - tolerations: {{ toYaml .Values.scheduler.patch.tolerations | nindent 8 }} - {{- end }} - securityContext: - runAsNonRoot: true - runAsUser: {{ .Values.scheduler.patch.runAsUser }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml deleted file mode 100644 index ce52042c..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml +++ /dev/null @@ -1,63 +0,0 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ include "hami-vgpu.fullname" . }}-admission-patch - namespace: {{ include "hami-vgpu.namespace" . }} - annotations: - "helm.sh/hook": post-install,post-upgrade - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded - labels: - {{- include "hami-vgpu.labels" . | nindent 4 }} - app.kubernetes.io/component: admission-webhook -spec: - {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} - # Alpha feature since k8s 1.12 - ttlSecondsAfterFinished: 0 - {{- end }} - template: - metadata: - name: {{ include "hami-vgpu.fullname" . }}-admission-patch - {{- if .Values.scheduler.patch.podAnnotations }} - annotations: {{ toYaml .Values.scheduler.patch.podAnnotations | nindent 8 }} - {{- end }} - labels: - {{- include "hami-vgpu.labels" . | nindent 8 }} - app.kubernetes.io/component: admission-webhook - hami.io/webhook: ignore - spec: - {{- if .Values.scheduler.patch.priorityClassName }} - priorityClassName: {{ .Values.scheduler.patch.priorityClassName }} - {{- end }} - {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} - {{- include "hami.scheduler.patch.new.imagePullSecrets" . | nindent 6 }} - {{- else }} - {{- include "hami.scheduler.patch.imagePullSecrets" . | nindent 6 }} - {{- end }} - containers: - - name: patch - {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} - image: {{ include "hami.scheduler.patch.new.image" . }} - imagePullPolicy: {{ .Values.scheduler.patch.imageNew.pullPolicy }} - {{- else }} - image: {{ include "hami.scheduler.patch.image" . }} - imagePullPolicy: {{ .Values.scheduler.patch.image.pullPolicy }} - {{- end }} - args: - - patch - - --webhook-name={{ include "hami-vgpu.scheduler.webhook" . }} - - --namespace={{ include "hami-vgpu.namespace" . }} - - --patch-validating=false - - --secret-name={{ include "hami-vgpu.scheduler.tls" . }} - restartPolicy: OnFailure - serviceAccountName: {{ include "hami-vgpu.fullname" . }}-admission - {{- if .Values.scheduler.patch.nodeSelector }} - nodeSelector: {{ toYaml .Values.scheduler.patch.nodeSelector | nindent 8 }} - {{- end }} - {{- if .Values.scheduler.patch.tolerations }} - tolerations: {{ toYaml .Values.scheduler.patch.tolerations | nindent 8 }} - {{- end }} - securityContext: - runAsNonRoot: true - runAsUser: {{ .Values.scheduler.patch.runAsUser }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml deleted file mode 100644 index 56682f8e..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml +++ /dev/null @@ -1,21 +0,0 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ include "hami-vgpu.fullname" . }}-admission - namespace: {{ include "hami-vgpu.namespace" . }} - annotations: - "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded - labels: - {{- include "hami-vgpu.labels" . | nindent 4 }} - app.kubernetes.io/component: admission-webhook -rules: - - apiGroups: - - "" - resources: - - secrets - verbs: - - get - - create -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml deleted file mode 100644 index 7239b128..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml +++ /dev/null @@ -1,21 +0,0 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ include "hami-vgpu.fullname" . }}-admission - namespace: {{ include "hami-vgpu.namespace" . }} - annotations: - "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded - labels: - {{- include "hami-vgpu.labels" . | nindent 4 }} - app.kubernetes.io/component: admission-webhook -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ include "hami-vgpu.fullname" . }}-admission -subjects: - - kind: ServiceAccount - name: {{ include "hami-vgpu.fullname" . }}-admission - namespace: {{ include "hami-vgpu.namespace" . }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml deleted file mode 100644 index 857c6a8d..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "hami-vgpu.fullname" . }}-admission - namespace: {{ include "hami-vgpu.namespace" . }} - annotations: - "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded - labels: - {{- include "hami-vgpu.labels" . | nindent 4 }} - app.kubernetes.io/component: admission-webhook -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/role.yaml b/packages/system/hami/charts/hami/templates/scheduler/role.yaml deleted file mode 100644 index 5f7d7e44..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/role.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ include "hami-vgpu.scheduler" . }} - namespace: {{ include "hami-vgpu.namespace" . }} - labels: - app.kubernetes.io/component: "hami-scheduler" - {{- include "hami-vgpu.labels" . | nindent 4 }} -rules: - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["create", "list", "watch", "get", "update", "patch"] diff --git a/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml deleted file mode 100644 index 96e175a1..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ include "hami-vgpu.scheduler" . }} - namespace: {{ include "hami-vgpu.namespace" . }} - labels: - app.kubernetes.io/component: "hami-scheduler" - {{- include "hami-vgpu.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ include "hami-vgpu.scheduler" . }} -subjects: - - kind: ServiceAccount - name: {{ include "hami-vgpu.scheduler" . }} - namespace: {{ include "hami-vgpu.namespace" . }} ---- -{{- if .Values.mockDevicePlugin.enabled }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "hami-vgpu.mock-device-plugin" . }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "hami-vgpu.mock-device-plugin" . }} -subjects: - - kind: ServiceAccount - name: {{ include "hami-vgpu.mock-device-plugin" . }} - namespace: {{ include "hami-vgpu.namespace" . }} -{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/scheduler/service.yaml b/packages/system/hami/charts/hami/templates/scheduler/service.yaml deleted file mode 100644 index d7538fed..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/service.yaml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "hami-vgpu.scheduler" . }} - namespace: {{ include "hami-vgpu.namespace" . }} - labels: - app.kubernetes.io/component: hami-scheduler - {{- include "hami-vgpu.labels" . | nindent 4 }} - {{- if .Values.scheduler.service.labels }} - {{ toYaml .Values.scheduler.service.labels | indent 4 }} - {{- end }} - {{- if .Values.scheduler.service.annotations }} - annotations: {{ toYaml .Values.scheduler.service.annotations | nindent 4 }} - {{- end }} -spec: - type: {{ .Values.scheduler.service.type | default "NodePort" }} # Default type is NodePort - ports: - - name: http - port: {{ .Values.scheduler.service.httpPort | default 443 }} # Default HTTP port is 443 - targetPort: {{ .Values.scheduler.service.httpTargetPort | default 443 }} - {{- if eq (.Values.scheduler.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort - nodePort: {{ .Values.scheduler.service.schedulerPort | default 31998 }} - {{- end }} - protocol: TCP - - name: monitor - port: {{ .Values.scheduler.service.monitorPort | default 31993 }} # Default monitoring port is 31993 - targetPort: {{ .Values.scheduler.service.monitorTargetPort | default 9395 }} - {{- if eq (.Values.scheduler.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort - nodePort: {{ .Values.scheduler.service.monitorPort | default 31993 }} - {{- end }} - protocol: TCP - selector: - app.kubernetes.io/component: hami-scheduler - {{- include "hami-vgpu.selectorLabels" . | nindent 4 }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml b/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml deleted file mode 100644 index 0435c003..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "hami-vgpu.scheduler" . }} - namespace: {{ include "hami-vgpu.namespace" . }} - labels: - app.kubernetes.io/component: "hami-scheduler" - {{- include "hami-vgpu.labels" . | nindent 4 }} ---- -{{- if .Values.mockDevicePlugin.enabled }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "hami-vgpu.mock-device-plugin" . }} - namespace: {{ include "hami-vgpu.namespace" . }} -{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml b/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml deleted file mode 100644 index db9f8029..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml +++ /dev/null @@ -1,57 +0,0 @@ -{{- if .Values.scheduler.admissionWebhook.enabled -}} -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - {{- if .Values.scheduler.certManager.enabled }} - annotations: - cert-manager.io/inject-ca-from: {{ include "hami-vgpu.namespace" . }}/{{ include "hami-vgpu.scheduler" . }}-serving-cert - {{- end }} - name: {{ include "hami-vgpu.scheduler.webhook" . }} -webhooks: - - admissionReviewVersions: - - v1beta1 - clientConfig: - {{- if .Values.scheduler.admissionWebhook.customURL.enabled }} - url: https://{{ .Values.scheduler.admissionWebhook.customURL.host}}:{{.Values.scheduler.admissionWebhook.customURL.port}}{{.Values.scheduler.admissionWebhook.customURL.path}} - {{- else }} - service: - name: {{ include "hami-vgpu.scheduler" . }} - namespace: {{ include "hami-vgpu.namespace" . }} - path: /webhook - port: {{ .Values.scheduler.service.httpPort }} - {{- end }} - failurePolicy: {{ .Values.scheduler.admissionWebhook.failurePolicy }} - matchPolicy: Equivalent - name: vgpu.hami.io - namespaceSelector: - matchExpressions: - - key: hami.io/webhook - operator: NotIn - values: - - ignore - {{- if .Values.scheduler.admissionWebhook.whitelistNamespaces }} - - key: kubernetes.io/metadata.name - operator: NotIn - values: - {{- toYaml .Values.scheduler.admissionWebhook.whitelistNamespaces | nindent 10 }} - {{- end }} - objectSelector: - matchExpressions: - - key: hami.io/webhook - operator: NotIn - values: - - ignore - reinvocationPolicy: {{ .Values.scheduler.admissionWebhook.reinvocationPolicy }} - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - CREATE - resources: - - pods - scope: '*' - sideEffects: None - timeoutSeconds: 10 -{{- end }} diff --git a/packages/system/hami/charts/hami/values.yaml b/packages/system/hami/charts/hami/values.yaml deleted file mode 100644 index 01caca0b..00000000 --- a/packages/system/hami/charts/hami/values.yaml +++ /dev/null @@ -1,455 +0,0 @@ -## @section Global configuration -## @param global.imageRegistry Global Docker image registry -## @param global.imagePullSecrets Global Docker image pull secrets -global: - ## @param global.imageRegistry Global Docker image registry - imageRegistry: "" - ## E.g. - ## imagePullSecrets: - ## - myRegistryKeySecretName - ## @param global.imagePullSecrets Global Docker image pull secrets - imagePullSecrets: [] - imageTag: "v2.8.1" - gpuHookPath: /usr/local - labels: {} - annotations: {} - managedNodeSelectorEnable: false - managedNodeSelector: - usage: "gpu" - -nameOverride: "" -fullnameOverride: "" -namespaceOverride: "" - -#Nvidia GPU Parameters -resourceName: "nvidia.com/gpu" -resourceMem: "nvidia.com/gpumem" -resourceMemPercentage: "nvidia.com/gpumem-percentage" -resourceCores: "nvidia.com/gpucores" -resourcePriority: "nvidia.com/priority" - -#MLU Parameters -mluResourceName: "cambricon.com/vmlu" -mluResourceMem: "cambricon.com/mlu.smlu.vmemory" -mluResourceCores: "cambricon.com/mlu.smlu.vcore" - -#Hygon DCU Parameters -dcuResourceName: "hygon.com/dcunum" -dcuResourceMem: "hygon.com/dcumem" -dcuResourceCores: "hygon.com/dcucores" - -#Metax sGPU Parameters -metaxResourceName: "metax-tech.com/sgpu" -metaxResourceCore: "metax-tech.com/vcore" -metaxResourceMem: "metax-tech.com/vmemory" -metaxsGPUTopologyAware: "false" - -#Enflame VGCU Parameters -enflameResourceNameVGCU: "enflame.com/vgcu" -enflameResourceNameVGCUPercentage: "enflame.com/vgcu-percentage" - -#Kunlun XPU Parameters -kunlunResourceName: "kunlunxin.com/xpu" -kunlunResourceVCountName: "kunlunxin.com/vxpu" -kunlunResourceVMemoryName: "kunlunxin.com/vxpu-memory" - -schedulerName: "hami-scheduler" - -scheduler: - # @param nodeName defines the node name and the nvidia-vgpu-scheduler-scheduler will schedule to the node. - # if we install the nvidia-vgpu-scheduler-scheduler as default scheduler, we need to remove the k8s default - # scheduler pod from the cluster first, we must specify node name to skip the schedule workflow. - nodeName: "" - #nodeLabelSelector: - # "gpu": "on" - overwriteEnv: "false" - defaultSchedulerPolicy: - nodeSchedulerPolicy: binpack - gpuSchedulerPolicy: spread - metricsBindAddress: ":9395" - # If set to false, When Pod.Spec.SchedulerName equals to the const DefaultSchedulerName in k8s.io/api/core/v1 package, webhook will not overwrite it - forceOverwriteDefaultScheduler: true - livenessProbe: false - leaderElect: true - # when leaderElect is true, replicas is available, otherwise replicas is 1. - replicas: 1 - kubeScheduler: - # @param enabled indicate whether to run kube-scheduler container in the scheduler pod, it's true by default. - enabled: true - ## @param image.registry kube scheduler image registry - ## @param image.repository kube scheduler image repository - ## @param image.tag kube scheduler image tag (immutable tags are recommended) - ## @param image.pullPolicy kube scheduler image pull policy - ## @param image.pullSecrets Specify docker-registry secret names as an array - image: - registry: "registry.cn-hangzhou.aliyuncs.com" - repository: "google_containers/kube-scheduler" - tag: "" - ## Specify a imagePullPolicy - ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' - ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images - ## - pullPolicy: IfNotPresent - ## Optionally specify an array of imagePullSecrets. - ## Secrets must be manually created in the namespace. - ## Example: - ## pullSecrets: - ## - myRegistryKeySecretName - ## - pullSecrets: [] - resources: {} - # If you do want to specify resources, uncomment the following lines, adjust them as necessary. - # and remove the curly braces after 'resources:'. -# limits: -# cpu: 1000m -# memory: 1000Mi -# requests: -# cpu: 100m -# memory: 100Mi - extraNewArgs: - - --config=/config/config.yaml - - -v=4 - extraArgs: - - --policy-config-file=/config/config.json - - -v=4 - extender: - ## @param image.registry scheduler extender image registry - ## @param image.repository scheduler extender image repository - ## @param image.tag scheduler extender image tag (immutable tags are recommended) - ## @param image.pullPolicy scheduler extender image pull policy - ## @param image.pullSecrets Specify docker-registry secret names as an array - image: - registry: "docker.io" - repository: "projecthami/hami" - tag: "" - ## Specify a imagePullPolicy - ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' - ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images - ## - pullPolicy: IfNotPresent - ## Optionally specify an array of imagePullSecrets. - ## Secrets must be manually created in the namespace. - ## Example: - ## pullSecrets: - ## - myRegistryKeySecretName - ## - pullSecrets: [] - resources: {} - # If you do want to specify resources, uncomment the following lines, adjust them as necessary, - # and remove the curly braces after 'resources:'. -# limits: -# cpu: 1000m -# memory: 1000Mi -# requests: -# cpu: 100m -# memory: 100Mi - extraArgs: - - --debug - - -v=4 - nodeLockExpire: "5m" - podAnnotations: {} - tolerations: [] - #serviceAccountName: "hami-vgpu-scheduler-sa" - admissionWebhook: - # If set to false, the admission webhook is not installed and any pods that should use HAMi must be - # configured to use the right 'schedulerName' and other device-specific configurations. - enabled: true - customURL: - enabled: false - # must be an endpoint using https. - # should generate host certs here - host: 127.0.0.1 # hostname or ip, can be your node'IP if you want to use https://:/ - port: 31998 - path: /webhook - whitelistNamespaces: - # Specify the namespaces that the webhook will not be applied to. - # - default - # - kube-system - # - istio-system - reinvocationPolicy: Never - failurePolicy: Ignore - ## TLS Certificate Option 1: Use cert-manager to generate self-signed certificate. - ## If enabled, always takes precedence over options 2. - certManager: - enabled: false - ## TLS Certificate Option 2: Use kube-webhook-certgen to generate self-signed certificate. - ## If true and certManager.enabled is false, Helm will automatically create a self-signed cert and secret for you. - patch: - enabled: true - ## @param image.registry scheduler extender image registry - ## @param image.repository scheduler extender image repository - ## @param image.tag scheduler extender image tag (immutable tags are recommended) - ## @param image.pullPolicy scheduler extender image pull policy - ## @param image.pullSecrets Specify docker-registry secret names as an array - image: - registry: "docker.io" - repository: "jettech/kube-webhook-certgen" - tag: "v1.5.2" - ## Specify a imagePullPolicy - ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' - ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images - ## - pullPolicy: IfNotPresent - ## Optionally specify an array of imagePullSecrets. - ## Secrets must be manually created in the namespace. - ## Example: - ## pullSecrets: - ## - myRegistryKeySecretName - ## - pullSecrets: [] - ## @param image.registry scheduler extender image registry - ## @param image.repository scheduler extender image repository - ## @param image.tag scheduler extender image tag (immutable tags are recommended) - ## @param image.pullPolicy scheduler extender image pull policy - ## @param image.pullSecrets Specify docker-registry secret names as an array - imageNew: - registry: "docker.io" - repository: "liangjw/kube-webhook-certgen" - tag: "v1.1.1" - ## Specify a imagePullPolicy - ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' - ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images - ## - pullPolicy: IfNotPresent - ## Optionally specify an array of imagePullSecrets. - ## Secrets must be manually created in the namespace. - ## Example: - ## pullSecrets: - ## - myRegistryKeySecretName - ## - pullSecrets: [] - priorityClassName: "" - podAnnotations: {} - nodeSelector: {} - tolerations: [] - runAsUser: 2000 - service: - type: NodePort # Default type is NodePort, can be changed to ClusterIP - httpPort: 443 # HTTP port - schedulerPort: 31998 # NodePort for HTTP - monitorPort: 31993 # Monitoring port - monitorTargetPort: 9395 - httpTargetPort: 443 - labels: {} - annotations: {} - -devicePlugin: - enabled: true - gpuOperatorToolkitReady: - enabled: false - hostPath: "/run/nvidia/validations" - ## @param image.registry devicePlugin image registry - ## @param image.repository devicePlugin image repository - ## @param image.tag devicePlugin image tag (immutable tags are recommended) - ## @param image.pullPolicy devicePlugin image pull policy - ## @param image.pullSecrets Specify docker-registry secret names as an array - image: - registry: "docker.io" - repository: "projecthami/hami" - tag: "" - ## Specify a imagePullPolicy - ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' - ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images - ## - pullPolicy: IfNotPresent - ## Optionally specify an array of imagePullSecrets. - ## Secrets must be manually created in the namespace. - ## Example: - ## pullSecrets: - ## - myRegistryKeySecretName - ## - pullSecrets: [] - monitor: - ## @param image.registry monitor image registry - ## @param image.repository monitor image repository - ## @param image.tag monitor image tag (immutable tags are recommended) - ## @param image.pullPolicy monitor image pull policy - ## @param image.pullSecrets Specify docker-registry secret names as an array - image: - registry: "docker.io" - repository: "projecthami/hami" - tag: "" - ## Specify a imagePullPolicy - ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' - ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images - ## - pullPolicy: IfNotPresent - ## Optionally specify an array of imagePullSecrets. - ## Secrets must be manually created in the namespace. - ## Example: - ## pullSecrets: - ## - myRegistryKeySecretName - ## - pullSecrets: [] - ctrPath: /usr/local/vgpu/containers - resyncInterval: "5m" - extraArgs: - - -v=4 - extraEnvs: {} - resources: {} - # If you do want to specify resources, uncomment the following lines, adjust them as necessary. - # and remove the curly braces after 'resources:'. -# limits: -# cpu: 1000m -# memory: 1000Mi -# requests: -# cpu: 100m -# memory: 100Mi - - deviceSplitCount: 10 - deviceMemoryScaling: 1 - deviceCoreScaling: 1 - # Node configuration for device plugin, Priority: externalConfigName > config > default config - nodeConfiguration: - # If you want to use a custom config.json, you can set the content here. - # If this is set, it will override the default config.json(An example is as follows). - config: | - { - "nodeconfig": [ - { - "name": "your-node-name", - "operatingmode": "hami-core", - "devicememoryscaling": 1, - "devicesplitcount": 10, - "migstrategy": "none", - "filterdevices": { - "uuid": [], - "index": [] - } - } - ] - } - # If you want to use an existing ConfigMap, you can set the name here. - # If this is set, the chart will not create the ConfigMap and will use the existing one. - externalConfigName: "" - # The runtime class name to be used by the device plugin, and added to the pod.spec.runtimeClassName of applications utilizing NVIDIA GPUs - runtimeClassName: "" - # Whether to create runtime class, name comes from runtimeClassName when it is set - createRuntimeClass: false - migStrategy: "none" - disablecorelimit: "false" - passDeviceSpecsEnabled: false - deviceListStrategy: "envvar" - nvidiaHookPath: null - nvidiaDriverRoot: null - gdrcopyEnabled: null - gdsEnabled: null - mofedEnabled: null - - extraArgs: - - -v=4 - extraEnvs: {} - service: - type: NodePort # Default type is NodePort, can be changed to ClusterIP - httpPort: 31992 - labels: {} - annotations: {} - - pluginPath: /var/lib/kubelet/device-plugins - libPath: /usr/local/vgpu - - podAnnotations: {} - nvidiaNodeSelector: - gpu: "on" - tolerations: [] - # The updateStrategy for DevicePlugin DaemonSet. - # If you want to update the DaemonSet by manual, set type as "OnDelete". - # We recommend use OnDelete update strategy because DevicePlugin pod restart will cause business pod restart, this behavior is destructive. - # Otherwise, you can use RollingUpdate update strategy to rolling update DevicePlugin pod. - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - - resources: {} - # If you do want to specify resources, uncomment the following lines, adjust them as necessary. - # and remove the curly braces after 'resources:'. -# limits: -# cpu: 1000m -# memory: 1000Mi -# requests: -# cpu: 100m -# memory: 100Mi - -mockDevicePlugin: - enabled: false - image: - registry: "docker.io" - repository: "projecthami/mock-device-plugin" - tag: "1.0.1" - ## Specify a imagePullPolicy - ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' - ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images - ## - pullPolicy: IfNotPresent - ## Optionally specify an array of imagePullSecrets. - ## Secrets must be manually created in the namespace. - ## Example: - ## pullSecrets: - ## - myRegistryKeySecretName - ## - pullSecrets: [] -devices: - amd: - customresources: - - amd.com/gpu - - amd.com/gpu-memory - awsneuron: - customresources: - - aws.amazon.com/neuron - - aws.amazon.com/neuroncore - kunlun: - enabled: true - customresources: - - kunlunxin.com/xpu - - kunlunxin.com/vxpu - - kunlunxin.com/vxpu-memory - enflame: - enabled: true - customresources: - - enflame.com/vgcu - - enflame.com/vgcu-percentage - - enflame.com/gcu - mthreads: - enabled: true - customresources: - - mthreads.com/vgpu - nvidia: - gpuCorePolicy: default - libCudaLogLevel: 1 - ascend: - enabled: false - image: "" - imagePullPolicy: IfNotPresent - extraArgs: [] - nodeSelector: - ascend: "on" - tolerations: [] - customresources: - - huawei.com/Ascend910A - - huawei.com/Ascend910A-memory - - huawei.com/Ascend910B2 - - huawei.com/Ascend910B2-memory - - huawei.com/Ascend910B3 - - huawei.com/Ascend910B3-memory - - huawei.com/Ascend910B4 - - huawei.com/Ascend910B4-memory - - huawei.com/Ascend910B4-1 - - huawei.com/Ascend910B4-1-memory - - huawei.com/Ascend310P - - huawei.com/Ascend310P-memory - iluvatar: - enabled: false - customresources: - - iluvatar.ai/BI-V100-vgpu - - iluvatar.ai/BI-V100.vCore - - iluvatar.ai/BI-V100.vMem - - iluvatar.ai/BI-V150-vgpu - - iluvatar.ai/BI-V150.vCore - - iluvatar.ai/BI-V150.vMem - - iluvatar.ai/MR-V100-vgpu - - iluvatar.ai/MR-V100.vCore - - iluvatar.ai/MR-V100.vMem - - iluvatar.ai/MR-V50-vgpu - - iluvatar.ai/MR-V50.vCore - - iluvatar.ai/MR-V50.vMem diff --git a/packages/system/hami/values.yaml b/packages/system/hami/values.yaml deleted file mode 100644 index 76e8d631..00000000 --- a/packages/system/hami/values.yaml +++ /dev/null @@ -1,17 +0,0 @@ -hami: - scheduler: - kubeScheduler: - image: - registry: registry.k8s.io - repository: kube-scheduler - devicePlugin: - runtimeClassName: nvidia - updateStrategy: - type: RollingUpdate - # See https://github.com/Project-HAMi/HAMi/blob/v2.8.1/deployments/charts/hami/values.yaml - # Each entry: {"name": "node-name", "operatingmode": "hami-core", "devicesplitcount": N, ...} - nodeConfiguration: - config: | - { - "nodeconfig": [] - } diff --git a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml index f81eaff0..1631c86e 100644 --- a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml +++ b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml @@ -22,7 +22,7 @@ spec: singular: kubernetes plural: kuberneteses openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"ephemeralStorage":"20Gi","gpus":[],"instanceType":"u1.medium","maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"]}},"additionalProperties":{"type":"object","required":["ephemeralStorage","instanceType","maxReplicas","minReplicas","resources"],"properties":{"ephemeralStorage":{"description":"Ephemeral storage size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string"}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium"},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `.` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","hami","ingressNginx","monitoringAgents","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"hami":{"description":"HAMi GPU virtualization middleware.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable HAMi (requires GPU Operator).","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"large","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"images":{"description":"Optional image overrides for air-gapped or rate-limited registries.","type":"object","default":{},"properties":{"waitForKubeconfig":{"description":"Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.","type":"string","default":""}}}}} + {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"ephemeralStorage":"20Gi","gpus":[],"instanceType":"u1.medium","maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"]}},"additionalProperties":{"type":"object","required":["ephemeralStorage","instanceType","maxReplicas","minReplicas","resources"],"properties":{"ephemeralStorage":{"description":"Ephemeral storage size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string"}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium"},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `.` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","ingressNginx","monitoringAgents","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"large","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"images":{"description":"Optional image overrides for air-gapped or rate-limited registries.","type":"object","default":{},"properties":{"waitForKubeconfig":{"description":"Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.","type":"string","default":""}}}}} release: prefix: kubernetes- labels: @@ -40,7 +40,7 @@ spec: tags: - compute icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODFfMjg0NSkiLz4KPHBhdGggZD0iTTcxLjk5NjggMTlDNzAuMzAzOSAxOS4wMDAyIDY4LjkzMTIgMjAuNTMyMiA2OC45MzE0IDIyLjQyMjFDNjguOTMxNCAyMi40NTExIDY4LjkzNzMgMjIuNDc4OCA2OC45Mzc5IDIyLjUwNzZDNjguOTM1NCAyMi43NjQ0IDY4LjkyMzEgMjMuMDczNyA2OC45MzE0IDIzLjI5NzNDNjguOTcxNyAyNC4zODczIDY5LjIwODIgMjUuMjIxNiA2OS4zNTA2IDI2LjIyNThDNjkuNjA4NCAyOC4zNzUyIDY5LjgyNDUgMzAuMTU2OSA2OS42OTEyIDMxLjgxM0M2OS41NjE1IDMyLjQzNzUgNjkuMTAzNyAzMy4wMDg2IDY4LjY5NTYgMzMuNDA1Nkw2OC42MjM1IDM0LjcwODZDNjYuNzgzOSAzNC44NjE3IDY0LjkzMTkgMzUuMTQyMSA2My4wODIxIDM1LjU2NDFDNTUuMTIyNiAzNy4zNzk4IDQ4LjI2OTUgNDEuNDk5MSA0My4wNTIgNDcuMDYwOUM0Mi43MTM0IDQ2LjgyODggNDIuMTIxMSA0Ni40MDE5IDQxLjk0NSA0Ni4yNzEyQzQxLjM5NzcgNDYuMzQ1NCA0MC44NDQ1IDQ2LjUxNTEgNDAuMTI0MSA0Ni4wOTM1QzM4Ljc1MjIgNDUuMTY1NyAzNy41MDI4IDQzLjg4NTEgMzUuOTkxIDQyLjM0MjRDMzUuMjk4MiA0MS42MDQ0IDM0Ljc5NjYgNDAuOTAxOCAzMy45NzM1IDQwLjE5MDRDMzMuNzg2NiA0MC4wMjg5IDMzLjUwMTQgMzkuODEwNCAzMy4yOTIzIDM5LjY0NDJDMzIuNjQ4OSAzOS4xMjg4IDMxLjg5IDM4Ljg2IDMxLjE1NyAzOC44MzQ4QzMwLjIxNDcgMzguODAyNCAyOS4zMDc1IDM5LjE3MjUgMjguNzEzOCAzOS45MjA2QzI3LjY1ODQgNDEuMjUwNiAyNy45OTYzIDQzLjI4MzMgMjkuNDY3MSA0NC40NjE0QzI5LjQ4MiA0NC40NzM0IDI5LjQ5NzkgNDQuNDgyNyAyOS41MTI5IDQ0LjQ5NDNDMjkuNzE1IDQ0LjY1ODkgMjkuOTYyNSA0NC44Njk4IDMwLjE0ODMgNDUuMDA3NkMzMS4wMjE3IDQ1LjY1NTUgMzEuODE5NSA0NS45ODcyIDMyLjY4OTcgNDYuNTAxNUMzNC41MjMxIDQ3LjYzOTEgMzYuMDQzIDQ4LjU4MjMgMzcuMjQ4NiA0OS43MTk2QzM3LjcxOTQgNTAuMjIzNyAzNy44MDE2IDUxLjExMjIgMzcuODY0MyA1MS40OTY0TDM4Ljg0NjggNTIuMzc4MkMzMy41ODcyIDYwLjMzMDggMzEuMTUzIDcwLjE1MzkgMzIuNTkxNSA4MC4xNjI3TDMxLjMwNzcgODAuNTM3OEMzMC45NjkzIDgwLjk3NjggMzAuNDkxMiA4MS42Njc2IDI5Ljk5MTEgODEuODczOEMyOC40MTM4IDgyLjM3MjkgMjYuNjM4NyA4Mi41NTYyIDI0LjQ5NTYgODIuNzgxOUMyMy40ODk0IDgyLjg2NiAyMi42MjEzIDgyLjgxNTggMjEuNTU0NiA4My4wMTg4QzIxLjMxOTggODMuMDYzNSAyMC45OTI3IDgzLjE0OTEgMjAuNzM1OCA4My4yMDk3QzIwLjcyNjkgODMuMjExNiAyMC43MTg2IDgzLjIxNDIgMjAuNzA5NiA4My4yMTYyQzIwLjY5NTYgODMuMjE5NSAyMC42NzcyIDgzLjIyNjMgMjAuNjYzOCA4My4yMjk0QzE4Ljg1NyA4My42NjggMTcuNjk2MyA4NS4zMzY1IDE4LjA2OTkgODYuOTgwNUMxOC40NDM3IDg4LjYyNDggMjAuMjA4NiA4OS42MjQ4IDIyLjAyNjIgODkuMjMxMkMyMi4wMzkzIDg5LjIyODIgMjIuMDU4NCA4OS4yMjc3IDIyLjA3MiA4OS4yMjQ2QzIyLjA5MjYgODkuMjE5OSAyMi4xMTA2IDg5LjIwOTkgMjIuMTMxIDg5LjIwNDlDMjIuMzg0NCA4OS4xNDkgMjIuNzAxOSA4OS4wODY4IDIyLjkyMzYgODkuMDI3MkMyMy45NzIzIDg4Ljc0NTEgMjQuNzMxOCA4OC4zMzA2IDI1LjY3NDYgODcuOTY3N0MyNy43MDI5IDg3LjIzNjggMjkuMzgyOCA4Ni42MjYyIDMxLjAxOTUgODYuMzg4M0MzMS43MDMgODYuMzM0NSAzMi40MjMyIDg2LjgxMiAzMi43ODE0IDg3LjAxMzRMMzQuMTE3NyA4Ni43ODMxQzM3LjE5MjYgOTYuMzYxMyA0My42MzY2IDEwNC4xMDMgNTEuNzk2MyAxMDguOTYxTDUxLjIzOTYgMTEwLjMwM0M1MS40NDAzIDExMC44MjQgNTEuNjYxNiAxMTEuNTMgNTEuNTEyMSAxMTIuMDQ1QzUwLjkxNzEgMTEzLjU5NSA0OS44OTggMTE1LjIzMSA0OC43Mzc0IDExNy4wNTVDNDguMTc1NSAxMTcuODk4IDQ3LjYwMDQgMTE4LjU1MiA0Ny4wOTM0IDExOS41MTZDNDYuOTcyIDExOS43NDcgNDYuODE3NSAxMjAuMTAyIDQ2LjcwMDQgMTIwLjM0NkM0NS45MTI1IDEyMi4wMzkgNDYuNDkwNCAxMjMuOTkgNDguMDAzOCAxMjQuNzIyQzQ5LjUyNjggMTI1LjQ1OSA1MS40MTcxIDEyNC42ODIgNTIuMjM1MiAxMjIuOTg1QzUyLjIzNjQgMTIyLjk4MiA1Mi4yNDA2IDEyMi45OCA1Mi4yNDE3IDEyMi45NzhDNTIuMjQyNiAxMjIuOTc2IDUyLjI0MDkgMTIyLjk3MyA1Mi4yNDE3IDEyMi45NzFDNTIuMzU4MiAxMjIuNzMxIDUyLjUyMzMgMTIyLjQxNSA1Mi42MjE2IDEyMi4xODhDNTMuMDU2IDEyMS4xODkgNTMuMjAwNSAxMjAuMzMyIDUzLjUwNTkgMTE5LjM2NUM1NC4zMTcgMTE3LjMxOCA1NC43NjI2IDExNS4xNyA1NS44NzkxIDExMy44MzJDNTYuMTg0OSAxMTMuNDY2IDU2LjY4MzMgMTEzLjMyNSA1Ny4yMDAxIDExMy4xODZMNTcuODk0NCAxMTEuOTIyQzY1LjAwOCAxMTQuNjY1IDcyLjk3MDUgMTE1LjQwMiA4MC45MjQ1IDExMy41ODdDODIuNzM5MSAxMTMuMTczIDg0LjQ5MDggMTEyLjYzNyA4Ni4xODQzIDExMS45OTRDODYuMzc5NCAxMTIuMzQyIDg2Ljc0MiAxMTMuMDExIDg2LjgzOTMgMTEzLjE3OUM4Ny4zNjQ0IDExMy4zNTEgODcuOTM3NyAxMTMuNDM5IDg4LjQwNDcgMTE0LjEzM0M4OS4yNDAxIDExNS41NjcgODkuODExNCAxMTcuMjYzIDkwLjUwNzMgMTE5LjMxMkM5MC44MTI4IDEyMC4yNzkgOTAuOTYzOCAxMjEuMTM2IDkxLjM5ODEgMTIyLjEzNkM5MS40OTcxIDEyMi4zNjMgOTEuNjYxNCAxMjIuNjg0IDkxLjc3OCAxMjIuOTI1QzkyLjU5NDQgMTI0LjYyOCA5NC40OTA3IDEyNS40MDcgOTYuMDE1OSAxMjQuNjY5Qzk3LjUyOTIgMTIzLjkzNyA5OC4xMDc3IDEyMS45ODYgOTcuMzE5NCAxMjAuMjkzQzk3LjIwMjMgMTIwLjA0OSA5Ny4wNDEyIDExOS42OTUgOTYuOTE5OCAxMTkuNDY0Qzk2LjQxMjcgMTE4LjQ5OSA5NS44Mzc3IDExNy44NTIgOTUuMjc1OCAxMTcuMDA5Qzk0LjExNTIgMTE1LjE4NSA5My4xNTI2IDExMy42NyA5Mi41NTc1IDExMi4xMkM5Mi4zMDg3IDExMS4zMiA5Mi41OTk1IDExMC44MjMgOTIuNzkzMyAxMTAuMzAzQzkyLjY3NzIgMTEwLjE3IDkyLjQyODggMTA5LjQxNCA5Mi4yODI0IDEwOS4wNTlDMTAwLjc2MiAxMDQuMDI5IDEwNy4wMTcgOTUuOTk4NSAxMDkuOTU1IDg2LjcyMzlDMTEwLjM1MSA4Ni43ODY1IDExMS4wNDEgODYuOTA5MSAxMTEuMjY1IDg2Ljk1NDJDMTExLjcyNiA4Ni42NDg3IDExMi4xNDkgODYuMjUwMSAxMTIuOTgxIDg2LjMxNTlDMTE0LjYxNyA4Ni41NTM3IDExNi4yOTcgODcuMTY0NSAxMTguMzI2IDg3Ljg5NTNDMTE5LjI2OCA4OC4yNTgxIDEyMC4wMjggODguNjc5MyAxMjEuMDc3IDg4Ljk2MTRDMTIxLjI5OCA4OS4wMjEgMTIxLjYxNiA4OS4wNzY2IDEyMS44NjkgODkuMTMyNUMxMjEuODg5IDg5LjEzNzUgMTIxLjkwOCA4OS4xNDc1IDEyMS45MjggODkuMTUyMkMxMjEuOTQyIDg5LjE1NTMgMTIxLjk2MSA4OS4xNTU4IDEyMS45NzQgODkuMTU4OEMxMjMuNzkyIDg5LjU1MiAxMjUuNTU3IDg4LjU1MjYgMTI1LjkzIDg2LjkwODFDMTI2LjMwMyA4NS4yNjQxIDEyNS4xNDMgODMuNTk1MiAxMjMuMzM2IDgzLjE1N0MxMjMuMDc0IDgzLjA5NyAxMjIuNzAxIDgyLjk5NSAxMjIuNDQ2IDgyLjk0NjVDMTIxLjM3OSA4Mi43NDM1IDEyMC41MTEgODIuNzkzNSAxMTkuNTA1IDgyLjcwOTVDMTE3LjM2MSA4Mi40ODM5IDExNS41ODYgODIuMzAwNCAxMTQuMDA5IDgxLjgwMTRDMTEzLjM2NiA4MS41NTA3IDExMi45MDggODAuNzgxOSAxMTIuNjg2IDgwLjQ2NTVMMTExLjQ0OCA4MC4xMDM1QzExMi4wOSA3NS40MzggMTExLjkxNyA3MC41ODI1IDExMC44MDYgNjUuNzI0M0MxMDkuNjg1IDYwLjgyMDggMTA3LjcwNCA1Ni4zMzYxIDEwNS4wNjIgNTIuMzg0OEMxMDUuMzc5IDUyLjA5NDggMTA1Ljk3OSA1MS41NjEyIDEwNi4xNDkgNTEuNDA0M0MxMDYuMTk5IDUwLjg1MTcgMTA2LjE1NiA1MC4yNzIyIDEwNi43MjUgNDkuNjYwM0MxMDcuOTMxIDQ4LjUyMyAxMDkuNDUxIDQ3LjU3OTkgMTExLjI4NCA0Ni40NDIzQzExMi4xNTQgNDUuOTI3OSAxMTIuOTU5IDQ1LjU5NjQgMTEzLjgzMiA0NC45NDg0QzExNC4wMyA0NC44MDE5IDExNC4yOTkgNDQuNTY5OSAxMTQuNTA3IDQ0LjQwMjJDMTE1Ljk3NyA0My4yMjM3IDExNi4zMTYgNDEuMTkxMSAxMTUuMjYgMzkuODYxNEMxMTQuMjA0IDM4LjUzMTcgMTEyLjE1OSAzOC40MDY1IDExMC42ODggMzkuNTg1QzExMC40NzkgMzkuNzUxNiAxMTAuMTk1IDM5Ljk2ODggMTEwLjAwNyA0MC4xMzEyQzEwOS4xODQgNDAuODQyNiAxMDguNjc2IDQxLjU0NTIgMTA3Ljk4MyA0Mi4yODMyQzEwNi40NzEgNDMuODI1OSAxMDUuMjIyIDQ1LjExMyAxMDMuODUgNDYuMDQwOUMxMDMuMjU1IDQ2LjM4ODUgMTAyLjM4NSA0Ni4yNjgyIDEwMS45OSA0Ni4yNDQ5TDEwMC44MjQgNDcuMDgwNkM5NC4xNzUzIDQwLjA3NjMgODUuMTIzNSAzNS41OTgyIDc1LjM3NjYgMzQuNzI4M0M3NS4zNDk0IDM0LjMxNzkgNzUuMzEzNyAzMy41NzYxIDc1LjMwNDYgMzMuMzUyOUM3NC45MDU2IDMyLjk2OTMgNzQuNDIzNSAzMi42NDE4IDc0LjMwMjQgMzEuODEzQzc0LjE2OTEgMzAuMTU2OSA3NC4zOTE3IDI4LjM3NTIgNzQuNjQ5NiAyNi4yMjU4Qzc0Ljc5MTkgMjUuMjIxNiA3NS4wMjg0IDI0LjM4NzMgNzUuMDY4OCAyMy4yOTczQzc1LjA3OCAyMy4wNDk1IDc1LjA2MzIgMjIuNjkgNzUuMDYyMiAyMi40MjIxQzc1LjA2MiAyMC41MzIyIDczLjY4OTggMTguOTk5OCA3MS45OTY4IDE5Wk02OC4xNTg1IDQyLjg4ODZMNjcuMjQ4IDU5LjA0NDdMNjcuMTgyNSA1OS4wNzc2QzY3LjEyMTQgNjAuNTIyOSA2NS45Mzc1IDYxLjY3NyA2NC40ODM5IDYxLjY3N0M2My44ODg0IDYxLjY3NyA2My4zMzg4IDYxLjQ4NDkgNjIuODkyMiA2MS4xNTcxTDYyLjg2NiA2MS4xNzAzTDQ5LjY4MDcgNTEuNzc5NEM1My43MzMxIDQ3Ljc3NTkgNTguOTE2NCA0NC44MTcyIDY0Ljg5IDQzLjQ1NDZDNjUuOTgxMiA0My4yMDU2IDY3LjA3MTkgNDMuMDIwOSA2OC4xNTg1IDQyLjg4ODZaTTc1Ljg0MTcgNDIuODg4NkM4Mi44MTU5IDQzLjc1MDQgODkuMjY1NyA0Ni45MjMyIDk0LjIwODEgNTEuNzg2TDgxLjEwOCA2MS4xMTc2TDgxLjA2MjEgNjEuMDk3OUM3OS44OTk0IDYxLjk1MTIgNzguMjYxMSA2MS43Mzk0IDc3LjM1NDggNjAuNTk3OEM3Ni45ODM1IDYwLjEzMDEgNzYuNzg4NyA1OS41ODAxIDc2Ljc2NTMgNTkuMDI0OUw3Ni43NTIyIDU5LjAxODRMNzUuODQxNyA0Mi44ODg2Wk00NC44OTkxIDU3LjgxNEw1Ni45MzgyIDY4LjYzM0w1Ni45MjUxIDY4LjY5ODhDNTguMDExNyA2OS42NDc5IDU4LjE3MiA3MS4yOTQ5IDU3LjI2NTcgNzIuNDM2OEM1Ni44OTQ0IDcyLjkwNDUgNTYuMzk3NSA3My4yMTgyIDU1Ljg2MzkgNzMuMzY0N0w1NS44NTA4IDczLjQxNzNMNDAuNDE4OCA3Ny44OTIzQzM5LjYzMzQgNzAuNjc2NSA0MS4zMjYxIDYzLjY2MjEgNDQuODk5MSA1Ny44MTRaTTk5LjAwOTQgNTcuODIwNkMxMDAuNzk4IDYwLjczMzYgMTAyLjE1MyA2My45ODcxIDEwMi45NTkgNjcuNTE0M0MxMDMuNzU2IDcwLjk5OTEgMTAzLjk1NiA3NC40Nzc4IDEwMy42MjcgNzcuODM5N0w4OC4xMTY2IDczLjM1MTVMODguMTAzNSA3My4yODU3Qzg2LjcxNDUgNzIuOTA0MyA4NS44NjA5IDcxLjQ4NDggODYuMTg0MyA3MC4wNjExQzg2LjMxNjggNjkuNDc3OCA4Ni42MjQ5IDY4Ljk4NDQgODcuMDQyMyA2OC42MTk4TDg3LjAzNTggNjguNTg2OUw5OS4wMDk0IDU3LjgyMDZaTTY5LjUyNzQgNjkuNDY4OEg3NC40NTk2TDc3LjUyNTEgNzMuMzE4Nkw3Ni40MjQ3IDc4LjEyMjZMNzEuOTk2OCA4MC4yNjE0TDY3LjU1NTggNzguMTE2MUw2Ni40NTU0IDczLjMxMkw2OS41Mjc0IDY5LjQ2ODhaTTg1LjMzOTMgODIuNjQzN0M4NS41NDg5IDgyLjYzMzEgODUuNzU3NiA4Mi42NTIgODUuOTYxNiA4Mi42ODk4TDg1Ljk4NzggODIuNjU2OUwxMDEuOTUgODUuMzY4MkM5OS42MTQyIDkxLjk2MjQgOTUuMTQ0IDk3LjY3NSA4OS4xNzExIDEwMS40OThMODIuOTc0NyA4Ni40NjA2TDgyLjk5NDQgODYuNDM0M0M4Mi40MjUyIDg1LjEwNTUgODIuOTk0OCA4My41NDcyIDg0LjMwNDQgODIuOTEzNUM4NC42Mzk3IDgyLjc1MTMgODQuOTkgODIuNjYxNCA4NS4zMzkzIDgyLjY0MzdaTTU4LjUyOTggODIuNzA5NUM1OS43NDggODIuNzI2NyA2MC44NDA2IDgzLjU3NjEgNjEuMTIzNyA4NC44MjJDNjEuMjU2MiA4NS40MDUyIDYxLjE5MTcgODUuOTgzMSA2MC45NzMgODYuNDkzNUw2MS4wMTg5IDg2LjU1MjdMNTQuODg4IDEwMS40MzlDNDkuMTU1OSA5Ny43NDMyIDQ0LjU5MDQgOTIuMjA5OSA0Mi4xNDgxIDg1LjQyMDhMNTcuOTczMSA4Mi43MjI3TDU3Ljk5OTMgODIuNzU1NkM1OC4xNzYzIDgyLjcyMjkgNTguMzU1OCA4Mi43MDcxIDU4LjUyOTggODIuNzA5NVpNNzEuODk4NiA4OS4yMzEyQzcyLjMyMjkgODkuMjE1NSA3Mi43NTM0IDg5LjMwMyA3My4xNjI3IDg5LjUwMUM3My42OTkyIDg5Ljc2MDYgNzQuMTEzNiA5MC4xNjkyIDc0LjM3NDUgOTAuNjU5Mkg3NC40MzM0TDgyLjIzNDYgMTA0LjgyMUM4MS4yMjIxIDEwNS4xNjIgODAuMTgxMyAxMDUuNDU0IDc5LjExNjcgMTA1LjY5N0M3My4xNTA1IDEwNy4wNTggNjcuMjAzMiAxMDYuNjQ1IDYxLjgxOCAxMDQuODAyTDY5LjU5OTUgOTAuNjY1OEg2OS42MTI2QzcwLjA3OTUgODkuNzg4OCA3MC45NjUgODkuMjY1NiA3MS44OTg2IDg5LjIzMTJaIiBmaWxsPSJ3aGl0ZSIgc3Ryb2tlPSJ3aGl0ZSIgc3Ryb2tlLXdpZHRoPSIwLjI1Ii8+CjxkZWZzPgo8bGluZWFyR3JhZGllbnQgaWQ9InBhaW50MF9saW5lYXJfNjgxXzI4NDUiIHgxPSIxMCIgeTE9IjE1LjUiIHgyPSIxNDQiIHkyPSIxMzEuNSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiPgo8c3RvcCBzdG9wLWNvbG9yPSIjNEQ4N0ZGIi8+CjxzdG9wIG9mZnNldD0iMSIgc3RvcC1jb2xvcj0iIzA1NDdEMCIvPgo8L2xpbmVhckdyYWRpZW50Pgo8L2RlZnM+Cjwvc3ZnPgo= - keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "nodeGroups"], ["spec", "nodeGroups", "md0"], ["spec", "nodeGroups", "md0", "minReplicas"], ["spec", "nodeGroups", "md0", "maxReplicas"], ["spec", "nodeGroups", "md0", "instanceType"], ["spec", "nodeGroups", "md0", "ephemeralStorage"], ["spec", "nodeGroups", "md0", "roles"], ["spec", "nodeGroups", "md0", "resources"], ["spec", "nodeGroups", "md0", "gpus"], ["spec", "version"], ["spec", "host"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "hami"], ["spec", "addons", "hami", "enabled"], ["spec", "addons", "hami", "valuesOverride"], ["spec", "addons", "fluxcd"], ["spec", "addons", "fluxcd", "enabled"], ["spec", "addons", "fluxcd", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"], ["spec", "images"], ["spec", "images", "waitForKubeconfig"]] + keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "nodeGroups"], ["spec", "nodeGroups", "md0"], ["spec", "nodeGroups", "md0", "minReplicas"], ["spec", "nodeGroups", "md0", "maxReplicas"], ["spec", "nodeGroups", "md0", "instanceType"], ["spec", "nodeGroups", "md0", "ephemeralStorage"], ["spec", "nodeGroups", "md0", "roles"], ["spec", "nodeGroups", "md0", "resources"], ["spec", "nodeGroups", "md0", "gpus"], ["spec", "version"], ["spec", "host"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "fluxcd"], ["spec", "addons", "fluxcd", "enabled"], ["spec", "addons", "fluxcd", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"], ["spec", "images"], ["spec", "images", "waitForKubeconfig"]] secrets: exclude: [] include: diff --git a/packages/system/lineage-controller-webhook/Makefile b/packages/system/lineage-controller-webhook/Makefile index cdc1cf01..d5bada31 100644 --- a/packages/system/lineage-controller-webhook/Makefile +++ b/packages/system/lineage-controller-webhook/Makefile @@ -4,13 +4,8 @@ NAMESPACE=cozy-system include ../../../hack/common-envs.mk include ../../../hack/package.mk -.PHONY: test - image: image-lineage-controller-webhook -test: - helm unittest . - image-lineage-controller-webhook: docker buildx build -f images/lineage-controller-webhook/Dockerfile ../../.. \ --tag $(REGISTRY)/lineage-controller-webhook:$(call settag,$(TAG)) \ diff --git a/packages/system/lineage-controller-webhook/README.md b/packages/system/lineage-controller-webhook/README.md deleted file mode 100644 index 1e43c9c8..00000000 --- a/packages/system/lineage-controller-webhook/README.md +++ /dev/null @@ -1,67 +0,0 @@ -# lineage-controller-webhook - -Cozystack system package for the **lineage controller webhook** — a mutating -admission webhook that stamps "lineage" labels onto tenant workloads, linking -each resource back to the Cozystack `Application` that ultimately owns it. - -The webhook intercepts CREATE and UPDATE on `pods`, `secrets`, `services`, -`persistentvolumeclaims`, `ingresses.networking.k8s.io`, and -`workloadmonitors.cozystack.io` outside system namespaces. For each request it -walks the ownership graph upward (Kubernetes `ownerReferences`, then the -`HelmRelease` Flux installed the resource with, then the Cozystack -`Application`-derived CRD that produced the HelmRelease) and writes the -discovered application's group, kind and name as labels -(`apps.cozystack.io/application.{group,kind,name}`) on the incoming object. -Those labels let the aggregated API server, the Cozystack dashboard, and other -lineage-aware consumers reason about which application a resource belongs to. - -The webhook serves TLS on port 9443 with a cert-manager issued certificate, and -runs with `failurePolicy: Fail` — every CREATE/UPDATE on the resources above -is gated on the webhook being reachable. - -## Topology - -The chart ships a single shape, modelled on `cozystack-api`: - -- **Deployment** with two replicas (override via `replicas`). -- **Soft `nodeAffinity`** preferring `node-role.kubernetes.io/control-plane` - (`Exists`, so it matches both Talos's empty value and k3s/kubeadm's `"true"`). - Soft means: the pod lands on a control-plane node when one is reachable, and - on any worker otherwise — no override needed for managed Kubernetes, - Cozy-in-Cozy tenants, or any other cluster where control-plane nodes aren't - visible. -- **Permissive `tolerations`** (`[{operator: Exists}]`) so the pod can land on - tainted control-plane nodes when the soft affinity is satisfiable. -- **Soft `podAntiAffinity`** on `kubernetes.io/hostname` so replicas spread - across nodes when possible (best-effort). -- **`PodDisruptionBudget`** with `maxUnavailable: 1`, unconditional. At - `replicas: 1` it's a useful no-op (allows full drain); at `replicas >= 2` - it caps disruption to one pod. -- **`Service` with `spec.trafficDistribution: PreferClose`** so the apiserver - prefers a webhook endpoint on its own node when one exists, and transparently - falls over to a remote endpoint otherwise. Requires Kubernetes ≥ 1.31; on - older clusters the field is silently ignored and traffic uses default - cluster-wide distribution. - -## Parameters - -All keys live under the top-level `lineageControllerWebhook:` map. - -| Name | Description | Value | -| ----------------------------- | --------------------------------------------------------- | -------------------------------------------------------------------------------- | -| `image` | Container image (digest-pinned) | `ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.3.0@sha256:e898…6fb0` | -| `debug` | Enable `--zap-log-level=debug` instead of `info` | `false` | -| `replicas` | Deployment replica count | `2` | -| `localK8sAPIEndpoint.enabled` | **Deprecated.** See note below. | `false` | - -### `localK8sAPIEndpoint.enabled` (deprecated) - -When enabled, this injects `KUBERNETES_SERVICE_HOST=status.hostIP` and -`KUBERNETES_SERVICE_PORT=6443` so the webhook talks to the kube-apiserver on -its own node. It was originally added to avoid latency on the -webhook-to-apiserver path, but it is only valid when the pod is actually -scheduled on an apiserver-bearing node — which the chart's soft control-plane -affinity no longer guarantees. With this flag enabled and the pod scheduled -off a control-plane node, the controller will crash-loop dialing a non- -apiserver hostIP. Slated for removal once the latency motivation is addressed -in the webhook itself; **leave disabled**. diff --git a/packages/system/lineage-controller-webhook/templates/workload.yaml b/packages/system/lineage-controller-webhook/templates/daemonset.yaml similarity index 67% rename from packages/system/lineage-controller-webhook/templates/workload.yaml rename to packages/system/lineage-controller-webhook/templates/daemonset.yaml index 097919c6..860aee6e 100644 --- a/packages/system/lineage-controller-webhook/templates/workload.yaml +++ b/packages/system/lineage-controller-webhook/templates/daemonset.yaml @@ -1,11 +1,10 @@ apiVersion: apps/v1 -kind: Deployment +kind: DaemonSet metadata: name: lineage-controller-webhook labels: app: lineage-controller-webhook spec: - replicas: {{ .Values.lineageControllerWebhook.replicas }} selector: matchLabels: app: lineage-controller-webhook @@ -14,25 +13,13 @@ spec: labels: app: lineage-controller-webhook spec: - serviceAccountName: lineage-controller-webhook + {{- with .Values.lineageControllerWebhook.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} tolerations: - operator: Exists - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - preference: - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - topologyKey: kubernetes.io/hostname - labelSelector: - matchLabels: - app: lineage-controller-webhook + serviceAccountName: lineage-controller-webhook containers: - name: lineage-controller-webhook image: "{{ .Values.lineageControllerWebhook.image }}" diff --git a/packages/system/lineage-controller-webhook/templates/pdb.yaml b/packages/system/lineage-controller-webhook/templates/pdb.yaml deleted file mode 100644 index 0786c8ff..00000000 --- a/packages/system/lineage-controller-webhook/templates/pdb.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - name: lineage-controller-webhook -spec: - maxUnavailable: 1 - selector: - matchLabels: - app: lineage-controller-webhook diff --git a/packages/system/lineage-controller-webhook/templates/service.yaml b/packages/system/lineage-controller-webhook/templates/service.yaml index fbb3ad54..c5df90fb 100644 --- a/packages/system/lineage-controller-webhook/templates/service.yaml +++ b/packages/system/lineage-controller-webhook/templates/service.yaml @@ -5,7 +5,7 @@ metadata: labels: app: lineage-controller-webhook spec: - trafficDistribution: PreferClose + internalTrafficPolicy: Local type: ClusterIP ports: - port: 443 diff --git a/packages/system/lineage-controller-webhook/tests/service_test.yaml b/packages/system/lineage-controller-webhook/tests/service_test.yaml deleted file mode 100644 index 3cdbddd8..00000000 --- a/packages/system/lineage-controller-webhook/tests/service_test.yaml +++ /dev/null @@ -1,30 +0,0 @@ -suite: lineage-controller-webhook service -templates: - - templates/service.yaml - -release: - name: lineage-controller-webhook - namespace: cozy-system - -tests: - - it: renders a ClusterIP service on 443 -> 9443 with PreferClose traffic distribution - asserts: - - isKind: - of: Service - - equal: - path: spec.type - value: ClusterIP - - equal: - path: spec.trafficDistribution - value: PreferClose - - notExists: - path: spec.internalTrafficPolicy - - equal: - path: spec.ports[0].port - value: 443 - - equal: - path: spec.ports[0].targetPort - value: 9443 - - equal: - path: spec.selector.app - value: lineage-controller-webhook diff --git a/packages/system/lineage-controller-webhook/tests/workload_test.yaml b/packages/system/lineage-controller-webhook/tests/workload_test.yaml deleted file mode 100644 index bcb24014..00000000 --- a/packages/system/lineage-controller-webhook/tests/workload_test.yaml +++ /dev/null @@ -1,99 +0,0 @@ -suite: lineage-controller-webhook workload + PDB -templates: - - templates/workload.yaml - - templates/pdb.yaml - -release: - name: lineage-controller-webhook - namespace: cozy-system - -tests: - # ---- Default Deployment shape ---------------------------------------------- - - - it: defaults render a 2-replica Deployment with soft control-plane nodeAffinity, soft podAntiAffinity, and Exists tolerations - template: templates/workload.yaml - asserts: - - isKind: - of: Deployment - - equal: - path: spec.replicas - value: 2 - - equal: - path: spec.template.spec.tolerations - value: - - operator: Exists - - equal: - path: spec.template.spec.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution - value: - - weight: 100 - preference: - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - - notExists: - path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution - - equal: - path: spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.topologyKey - value: kubernetes.io/hostname - - - it: defaults inject no localK8sAPIEndpoint env vars (knob is opt-in) - template: templates/workload.yaml - asserts: - - notExists: - path: spec.template.spec.containers[0].env - - # ---- PDB -------------------------------------------------------------------- - - - it: PDB renders unconditionally with maxUnavailable=1 at the default replica count - template: templates/pdb.yaml - asserts: - - isKind: - of: PodDisruptionBudget - - equal: - path: spec.maxUnavailable - value: 1 - - notExists: - path: spec.minAvailable - - - it: PDB still renders at replicas=1 (no-op, but consistent shape) - set: - lineageControllerWebhook.replicas: 1 - template: templates/pdb.yaml - asserts: - - isKind: - of: PodDisruptionBudget - - equal: - path: spec.maxUnavailable - value: 1 - - # ---- Replica override ------------------------------------------------------- - - - it: replicas value drives spec.replicas - set: - lineageControllerWebhook.replicas: 5 - template: templates/workload.yaml - asserts: - - equal: - path: spec.replicas - value: 5 - - # ---- Deprecated localK8sAPIEndpoint knob ----------------------------------- - - - it: localK8sAPIEndpoint.enabled=true injects KUBERNETES_SERVICE_HOST and PORT env vars - set: - lineageControllerWebhook.localK8sAPIEndpoint.enabled: true - template: templates/workload.yaml - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: KUBERNETES_SERVICE_HOST - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.hostIP - - contains: - path: spec.template.spec.containers[0].env - content: - name: KUBERNETES_SERVICE_PORT - value: "6443" diff --git a/packages/system/lineage-controller-webhook/values.yaml b/packages/system/lineage-controller-webhook/values.yaml index e56d332d..c5671548 100644 --- a/packages/system/lineage-controller-webhook/values.yaml +++ b/packages/system/lineage-controller-webhook/values.yaml @@ -1,13 +1,10 @@ lineageControllerWebhook: image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.3.0@sha256:e8984709686a5eaf19b89da378d7b8c688ea5607e0783a88d9c9e4ccfca96fb0 debug: false - replicas: 2 - # DEPRECATED. Injects KUBERNETES_SERVICE_HOST=status.hostIP and - # KUBERNETES_SERVICE_PORT=6443 so the webhook talks to the apiserver on its - # own node — only valid when the pod is actually scheduled on an apiserver- - # bearing node. Now that the chart's nodeAffinity to control-plane is soft - # (preferred, not required), the pod can land off-control-plane and crash- - # loop dialing a non-apiserver hostIP. Slated for removal once webhook - # performance work obviates the locality motivation. Leave disabled. localK8sAPIEndpoint: - enabled: false + enabled: true + # nodeSelector for the DaemonSet + # Talos uses empty value: "node-role.kubernetes.io/control-plane": "" + # Generic k8s (k3s, kubeadm) uses: "node-role.kubernetes.io/control-plane": "true" + nodeSelector: + node-role.kubernetes.io/control-plane: ""