From 7ab462f67ecfecb81fe9649498add08e68fe291d Mon Sep 17 00:00:00 2001 From: tym83 <6355522@gmail.com> Date: Mon, 6 Apr 2026 20:21:53 +0500 Subject: [PATCH 001/250] Add Matthieu Robin (@matthieu-robin) as Maintainer Ref: #2344 Co-Authored-By: Claude Opus 4.6 (1M context) --- MAINTAINERS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 9c44daab..e5c07714 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -10,3 +10,4 @@ | Timur Tukaev | [@tym83](https://github.com/tym83) | Ænix | Cozystack Website, Marketing, Community Management | | Kirill Klinchenkov | [@klinch0](https://github.com/klinch0) | Ænix | Core Maintainer | | Nikita Bykov | [@nbykov0](https://github.com/nbykov0) | Ænix | Maintainer of ARM and stuff | +| Matthieu Robin | [@matthieu-robin](https://github.com/matthieu-robin) | Hidora | Managed Applications, Platform Quality & Benchmarking | From ae88fe3779bb8d990f29958cc7e87f6a72834de5 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 13 Apr 2026 11:42:42 +0500 Subject: [PATCH 002/250] [linstor-gui] Add package for LINBIT linstor-gui web UI Ships LINBIT's LINSTOR web UI (GPL-3.0) as an opt-in system package under packages/system/linstor-gui so operators can manage LINSTOR nodes, resources, and volumes from a browser instead of the linstor CLI. The UI is built from the upstream pkg.linbit.com tarball onto an nginx-unprivileged base image. A chart-supplied nginx.conf proxies /v1 and /metrics to the LINSTOR controller REST API over mTLS using the existing linstor-client-tls secret created by the linstor package. Only a ClusterIP Service is created; no Ingress is shipped, because LINSTOR's controller API is a privileged cluster-wide storage surface and auth depends on the deployment's OIDC setup. Operators opt in via bundles.enabledPackages and wire up ingress + auth themselves (port-forward works out of the box). Co-Authored-By: Claude Opus 4.6 Signed-off-by: Myasnikov Daniil --- .../core/platform/sources/linstor-gui.yaml | 21 ++++ .../platform/templates/bundles/system.yaml | 1 + packages/system/linstor-gui/Chart.yaml | 3 + packages/system/linstor-gui/Makefile | 32 ++++++ packages/system/linstor-gui/README.md | 43 ++++++++ .../linstor-gui/images/linstor-gui/Dockerfile | 19 ++++ .../system/linstor-gui/templates/_helpers.tpl | 17 +++ .../templates/configmap-nginx.yaml | 82 ++++++++++++++ .../linstor-gui/templates/deployment.yaml | 101 ++++++++++++++++++ .../system/linstor-gui/templates/service.yaml | 15 +++ .../linstor-gui/templates/serviceaccount.yaml | 7 ++ .../linstor-gui/tests/deployment_test.yaml | 95 ++++++++++++++++ packages/system/linstor-gui/values.yaml | 17 +++ 13 files changed, 453 insertions(+) create mode 100644 packages/core/platform/sources/linstor-gui.yaml create mode 100644 packages/system/linstor-gui/Chart.yaml create mode 100644 packages/system/linstor-gui/Makefile create mode 100644 packages/system/linstor-gui/README.md create mode 100644 packages/system/linstor-gui/images/linstor-gui/Dockerfile create mode 100644 packages/system/linstor-gui/templates/_helpers.tpl create mode 100644 packages/system/linstor-gui/templates/configmap-nginx.yaml create mode 100644 packages/system/linstor-gui/templates/deployment.yaml create mode 100644 packages/system/linstor-gui/templates/service.yaml create mode 100644 packages/system/linstor-gui/templates/serviceaccount.yaml create mode 100644 packages/system/linstor-gui/tests/deployment_test.yaml create mode 100644 packages/system/linstor-gui/values.yaml diff --git a/packages/core/platform/sources/linstor-gui.yaml b/packages/core/platform/sources/linstor-gui.yaml new file mode 100644 index 00000000..005e1fb7 --- /dev/null +++ b/packages/core/platform/sources/linstor-gui.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: cozystack.io/v1alpha1 +kind: PackageSource +metadata: + name: cozystack.linstor-gui +spec: + sourceRef: + kind: OCIRepository + name: cozystack-packages + namespace: cozy-system + path: / + variants: + - name: default + dependsOn: + - cozystack.linstor + components: + - name: linstor-gui + path: system/linstor-gui + install: + namespace: cozy-linstor + releaseName: linstor-gui diff --git a/packages/core/platform/templates/bundles/system.yaml b/packages/core/platform/templates/bundles/system.yaml index 795b523f..754851af 100644 --- a/packages/core/platform/templates/bundles/system.yaml +++ b/packages/core/platform/templates/bundles/system.yaml @@ -149,6 +149,7 @@ {{include "cozystack.platform.package.optional.default" (list "cozystack.external-dns-application" $) }} {{include "cozystack.platform.package.optional.default" (list "cozystack.external-secrets-operator" $) }} {{include "cozystack.platform.package.optional.default" (list "cozystack.velero" $) }} +{{include "cozystack.platform.package.optional.default" (list "cozystack.linstor-gui" $) }} {{- if has "cozystack.bootbox" (default (list) .Values.bundles.enabledPackages) }} {{include "cozystack.platform.package.default" (list "cozystack.bootbox-application" $) }} {{include "cozystack.platform.package.default" (list "cozystack.bootbox" $) }} diff --git a/packages/system/linstor-gui/Chart.yaml b/packages/system/linstor-gui/Chart.yaml new file mode 100644 index 00000000..e361a819 --- /dev/null +++ b/packages/system/linstor-gui/Chart.yaml @@ -0,0 +1,3 @@ +apiVersion: v2 +name: cozy-linstor-gui +version: 0.0.0 # Placeholder, the actual version will be automatically set during the build process diff --git a/packages/system/linstor-gui/Makefile b/packages/system/linstor-gui/Makefile new file mode 100644 index 00000000..2937ae0a --- /dev/null +++ b/packages/system/linstor-gui/Makefile @@ -0,0 +1,32 @@ +export NAME=linstor-gui +export NAMESPACE=cozy-linstor + +include ../../../hack/common-envs.mk +include ../../../hack/package.mk + +LINSTOR_GUI_VERSION ?= 2.3.0 + +image: image-linstor-gui + +image-linstor-gui: + docker buildx build images/linstor-gui \ + --provenance false \ + --builder=$(BUILDER) \ + --platform=linux/amd64,linux/arm64 \ + --build-arg LINSTOR_GUI_VERSION=$(LINSTOR_GUI_VERSION) \ + --tag $(REGISTRY)/linstor-gui:$(call settag,$(LINSTOR_GUI_VERSION)) \ + --tag $(REGISTRY)/linstor-gui:$(call settag,$(LINSTOR_GUI_VERSION)-$(TAG)) \ + --cache-from type=registry,ref=$(REGISTRY)/linstor-gui:latest \ + --cache-to type=inline \ + --metadata-file images/linstor-gui.json \ + --push=$(PUSH) \ + --label "org.opencontainers.image.source=https://github.com/cozystack/cozystack" \ + --load=$(LOAD) + REPOSITORY="$(REGISTRY)/linstor-gui" \ + yq -i '.image.repository = strenv(REPOSITORY)' values.yaml + TAG="$(call settag,$(LINSTOR_GUI_VERSION))@$$(yq e '."containerimage.digest"' images/linstor-gui.json -o json -r)" \ + yq -i '.image.tag = strenv(TAG)' values.yaml + rm -f images/linstor-gui.json + +test: + helm unittest . diff --git a/packages/system/linstor-gui/README.md b/packages/system/linstor-gui/README.md new file mode 100644 index 00000000..e4de2314 --- /dev/null +++ b/packages/system/linstor-gui/README.md @@ -0,0 +1,43 @@ +# linstor-gui + +Cozystack system package for [LINBIT/linstor-gui](https://github.com/LINBIT/linstor-gui) +— a web UI for managing LINSTOR nodes, resources, volumes and snapshots. + +Installed alongside the `linstor` package in the `cozy-linstor` namespace. The UI +proxies the LINSTOR controller REST API at `https://linstor-controller.cozy-linstor.svc:3371` +using mTLS with the `linstor-client-tls` secret created by the `linstor` package. + +## Exposing the UI + +This package only creates a `ClusterIP` Service. It does **not** ship an ingress, +because authentication depends on the deployment's Keycloak / OIDC setup and +LINSTOR's controller API is a privileged cluster-wide storage management +surface. Cluster admins should wire up ingress + auth explicitly, for example: + +```bash +kubectl -n cozy-linstor port-forward svc/linstor-gui 3373:80 +``` + +then open . + +## Parameters + +### Image + +| Name | Description | Value | +| ------------------ | ---------------------------------------------------------- | ----------------------------------------- | +| `image.repository` | LINSTOR GUI container image repository | `ghcr.io/cozystack/cozystack/linstor-gui` | +| `image.tag` | LINSTOR GUI container image tag (digest recommended) | `2.3.0` | + +### Deployment + +| Name | Description | Value | +| ---------- | ------------------------------- | ----- | +| `replicas` | Number of linstor-gui replicas | `1` | + +### LINSTOR controller connection + +| Name | Description | Value | +| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------- | +| `linstor.endpoint` | In-cluster URL of the LINSTOR controller REST API (HTTPS, mTLS) | `https://linstor-controller.cozy-linstor.svc:3371` | +| `linstor.clientSecret` | Kubernetes Secret with `tls.crt`, `tls.key`, `ca.crt` used as the mTLS client certificate against the LINSTOR controller. Created by the `linstor` package. | `linstor-client-tls` | diff --git a/packages/system/linstor-gui/images/linstor-gui/Dockerfile b/packages/system/linstor-gui/images/linstor-gui/Dockerfile new file mode 100644 index 00000000..aadc1162 --- /dev/null +++ b/packages/system/linstor-gui/images/linstor-gui/Dockerfile @@ -0,0 +1,19 @@ +# Upstream: https://github.com/LINBIT/linstor-gui (GPL-3.0) +# Serves the pre-built LINSTOR GUI tarball published by LINBIT from a hardened +# nginx-unprivileged image. nginx.conf is supplied by the chart via ConfigMap, +# so the upstream docker-entrypoint.sh / nginx.conf.template is not used. +FROM nginxinc/nginx-unprivileged:1.29-alpine + +ARG LINSTOR_GUI_VERSION +USER root +RUN apk add --no-cache curl tar && \ + curl -fsSL "https://pkg.linbit.com/downloads/linstor/linstor-gui-${LINSTOR_GUI_VERSION}.tar.gz" -o /tmp/linstor-gui.tar.gz && \ + mkdir -p /usr/share/nginx/html && \ + tar -xzf /tmp/linstor-gui.tar.gz -C /usr/share/nginx/html --strip-components=2 "linstor-gui-${LINSTOR_GUI_VERSION}/dist" && \ + rm -f /tmp/linstor-gui.tar.gz && \ + apk del curl tar && \ + chown -R 101:0 /usr/share/nginx/html + +USER 101 +EXPOSE 3373 +CMD ["nginx", "-g", "daemon off;"] diff --git a/packages/system/linstor-gui/templates/_helpers.tpl b/packages/system/linstor-gui/templates/_helpers.tpl new file mode 100644 index 00000000..6cad99ee --- /dev/null +++ b/packages/system/linstor-gui/templates/_helpers.tpl @@ -0,0 +1,17 @@ +{{/* +Common labels +*/}} +{{- define "linstor-gui.labels" -}} +app.kubernetes.io/name: linstor-gui +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/part-of: cozystack +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "linstor-gui.selectorLabels" -}} +app.kubernetes.io/name: linstor-gui +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/packages/system/linstor-gui/templates/configmap-nginx.yaml b/packages/system/linstor-gui/templates/configmap-nginx.yaml new file mode 100644 index 00000000..0b2d97f9 --- /dev/null +++ b/packages/system/linstor-gui/templates/configmap-nginx.yaml @@ -0,0 +1,82 @@ +{{/* +Parse the LINSTOR endpoint so nginx can set `proxy_ssl_name` correctly. +The endpoint is expected to be an https:// URL like + https://linstor-controller.cozy-linstor.svc:3371 +*/}} +{{- $endpoint := trimPrefix "https://" (trimPrefix "http://" .Values.linstor.endpoint) -}} +{{- $endpointHost := (splitList ":" $endpoint) | first -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: linstor-gui-nginx + labels: + {{- include "linstor-gui.labels" . | nindent 4 }} +data: + nginx.conf: | + worker_processes auto; + pid /tmp/nginx.pid; + + events { + worker_connections 1024; + } + + http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + server_tokens off; + + client_body_temp_path /tmp/client_body; + proxy_temp_path /tmp/proxy; + fastcgi_temp_path /tmp/fastcgi; + uwsgi_temp_path /tmp/uwsgi; + scgi_temp_path /tmp/scgi; + + sendfile on; + keepalive_timeout 65; + + server { + listen 3373; + server_name _; + + root /usr/share/nginx/html; + index index.html; + + # Static UI assets + location / { + try_files $uri $uri/ /index.html; + } + + # Proxy LINSTOR REST API over mTLS to the controller + location /v1 { + proxy_pass {{ .Values.linstor.endpoint }}; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_ssl_certificate /etc/linstor/client/tls.crt; + proxy_ssl_certificate_key /etc/linstor/client/tls.key; + proxy_ssl_trusted_certificate /etc/linstor/client/ca.crt; + proxy_ssl_verify on; + proxy_ssl_verify_depth 2; + proxy_ssl_server_name on; + proxy_ssl_name {{ $endpointHost }}; + } + + location /metrics { + proxy_pass {{ .Values.linstor.endpoint }}; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_ssl_certificate /etc/linstor/client/tls.crt; + proxy_ssl_certificate_key /etc/linstor/client/tls.key; + proxy_ssl_trusted_certificate /etc/linstor/client/ca.crt; + proxy_ssl_verify on; + proxy_ssl_verify_depth 2; + proxy_ssl_server_name on; + proxy_ssl_name {{ $endpointHost }}; + } + + location = /healthz { + access_log off; + return 200 'ok'; + add_header Content-Type text/plain; + } + } + } diff --git a/packages/system/linstor-gui/templates/deployment.yaml b/packages/system/linstor-gui/templates/deployment.yaml new file mode 100644 index 00000000..b1407bdb --- /dev/null +++ b/packages/system/linstor-gui/templates/deployment.yaml @@ -0,0 +1,101 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: linstor-gui + labels: + {{- include "linstor-gui.labels" . | nindent 4 }} + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: {{ .Values.replicas }} + selector: + matchLabels: + {{- include "linstor-gui.selectorLabels" . | nindent 6 }} + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + template: + metadata: + annotations: + checksum/nginx-config: {{ include (print $.Template.BasePath "/configmap-nginx.yaml") . | sha256sum }} + labels: + {{- include "linstor-gui.labels" . | nindent 8 }} + spec: + serviceAccountName: linstor-gui + automountServiceAccountToken: false + priorityClassName: system-cluster-critical + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + {{- include "linstor-gui.selectorLabels" . | nindent 20 }} + topologyKey: kubernetes.io/hostname + containers: + - name: linstor-gui + image: {{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 3373 + protocol: TCP + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 5 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 2 + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 101 + runAsGroup: 101 + capabilities: + drop: + - ALL + volumeMounts: + - name: nginx-config + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + readOnly: true + - name: linstor-client-tls + mountPath: /etc/linstor/client + readOnly: true + - name: tmp + mountPath: /tmp + - name: nginx-cache + mountPath: /var/cache/nginx + volumes: + - name: nginx-config + configMap: + name: linstor-gui-nginx + - name: linstor-client-tls + secret: + secretName: {{ .Values.linstor.clientSecret }} + defaultMode: 0400 + - name: tmp + emptyDir: {} + - name: nginx-cache + emptyDir: {} diff --git a/packages/system/linstor-gui/templates/service.yaml b/packages/system/linstor-gui/templates/service.yaml new file mode 100644 index 00000000..66ae2b2d --- /dev/null +++ b/packages/system/linstor-gui/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: linstor-gui + labels: + {{- include "linstor-gui.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - name: http + port: 80 + targetPort: http + protocol: TCP + selector: + {{- include "linstor-gui.selectorLabels" . | nindent 4 }} diff --git a/packages/system/linstor-gui/templates/serviceaccount.yaml b/packages/system/linstor-gui/templates/serviceaccount.yaml new file mode 100644 index 00000000..8e472671 --- /dev/null +++ b/packages/system/linstor-gui/templates/serviceaccount.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: linstor-gui + labels: + {{- include "linstor-gui.labels" . | nindent 4 }} +automountServiceAccountToken: false diff --git a/packages/system/linstor-gui/tests/deployment_test.yaml b/packages/system/linstor-gui/tests/deployment_test.yaml new file mode 100644 index 00000000..420ec3e1 --- /dev/null +++ b/packages/system/linstor-gui/tests/deployment_test.yaml @@ -0,0 +1,95 @@ +suite: linstor-gui deployment +templates: + - templates/deployment.yaml + - templates/service.yaml + - templates/configmap-nginx.yaml + +tests: + - it: renders a ClusterIP service on port 80 -> http + template: templates/service.yaml + release: + name: linstor-gui + namespace: cozy-linstor + asserts: + - isKind: + of: Service + - equal: + path: spec.type + value: ClusterIP + - equal: + path: spec.ports[0].port + value: 80 + - equal: + path: spec.ports[0].targetPort + value: http + + - it: mounts the LINSTOR client TLS secret into the pod + template: templates/deployment.yaml + release: + name: linstor-gui + namespace: cozy-linstor + asserts: + - isKind: + of: Deployment + - contains: + path: spec.template.spec.volumes + content: + name: linstor-client-tls + secret: + secretName: linstor-client-tls + defaultMode: 0400 + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: linstor-client-tls + mountPath: /etc/linstor/client + readOnly: true + + - it: runs as a non-root user with a read-only root filesystem + template: templates/deployment.yaml + release: + name: linstor-gui + namespace: cozy-linstor + asserts: + - equal: + path: spec.template.spec.containers[0].securityContext.runAsNonRoot + value: true + - equal: + path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem + value: true + - equal: + path: spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation + value: false + + - it: proxies /v1 to the configured LINSTOR controller with mTLS + template: templates/configmap-nginx.yaml + release: + name: linstor-gui + namespace: cozy-linstor + asserts: + - isKind: + of: ConfigMap + - matchRegex: + path: data["nginx.conf"] + pattern: "location /v1" + - matchRegex: + path: data["nginx.conf"] + pattern: "proxy_pass https://linstor-controller.cozy-linstor.svc:3371" + - matchRegex: + path: data["nginx.conf"] + pattern: "proxy_ssl_certificate /etc/linstor/client/tls.crt" + + - it: overrides linstor endpoint and client secret + template: templates/configmap-nginx.yaml + release: + name: linstor-gui + namespace: cozy-linstor + set: + linstor.endpoint: https://my-controller.example.svc:3371 + asserts: + - matchRegex: + path: data["nginx.conf"] + pattern: "proxy_pass https://my-controller.example.svc:3371" + - matchRegex: + path: data["nginx.conf"] + pattern: "proxy_ssl_name my-controller.example.svc" diff --git a/packages/system/linstor-gui/values.yaml b/packages/system/linstor-gui/values.yaml new file mode 100644 index 00000000..e3c49aea --- /dev/null +++ b/packages/system/linstor-gui/values.yaml @@ -0,0 +1,17 @@ +## @section Image +## @param image.repository LINSTOR GUI container image repository +## @param image.tag LINSTOR GUI container image tag (digest recommended) +image: + repository: ghcr.io/cozystack/cozystack/linstor-gui + tag: 2.3.0 + +## @section Deployment +## @param replicas Number of linstor-gui replicas +replicas: 1 + +## @section LINSTOR controller connection +## @param linstor.endpoint In-cluster URL of the LINSTOR controller REST API (HTTPS, mTLS) +## @param linstor.clientSecret Kubernetes Secret with `tls.crt`, `tls.key`, `ca.crt` used as the mTLS client certificate against the LINSTOR controller. Created by the `linstor` package. +linstor: + endpoint: "https://linstor-controller.cozy-linstor.svc:3371" + clientSecret: "linstor-client-tls" From ba9f9e9f2ced7eb411e491ae51d2518d04b5a3ac Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 13 Apr 2026 11:59:46 +0500 Subject: [PATCH 003/250] [linstor-gui] Address review comments on #2382 - Dockerfile: chown html root to 101:101 (match runAsGroup, not 101:0) - Makefile: declare .PHONY targets (image, image-linstor-gui, test) - deployment: drop priorityClassName system-cluster-critical (GUI is not control-plane critical; chart does not need to reserve the slot) - configmap-nginx: - fail the render if linstor.endpoint is not https:// so a misconfig cannot silently downgrade the mTLS-protected controller path - hoist proxy_set_header and proxy_ssl_* directives to the server block instead of duplicating across /v1 and /metrics locations Co-Authored-By: Claude Opus 4.6 Signed-off-by: Myasnikov Daniil --- packages/system/linstor-gui/Makefile | 2 + .../linstor-gui/images/linstor-gui/Dockerfile | 2 +- .../templates/configmap-nginx.yaml | 38 +++++++++---------- .../linstor-gui/templates/deployment.yaml | 1 - 4 files changed, 20 insertions(+), 23 deletions(-) diff --git a/packages/system/linstor-gui/Makefile b/packages/system/linstor-gui/Makefile index 2937ae0a..e202f4cc 100644 --- a/packages/system/linstor-gui/Makefile +++ b/packages/system/linstor-gui/Makefile @@ -6,6 +6,8 @@ include ../../../hack/package.mk LINSTOR_GUI_VERSION ?= 2.3.0 +.PHONY: image image-linstor-gui test + image: image-linstor-gui image-linstor-gui: diff --git a/packages/system/linstor-gui/images/linstor-gui/Dockerfile b/packages/system/linstor-gui/images/linstor-gui/Dockerfile index aadc1162..2082ef0d 100644 --- a/packages/system/linstor-gui/images/linstor-gui/Dockerfile +++ b/packages/system/linstor-gui/images/linstor-gui/Dockerfile @@ -12,7 +12,7 @@ RUN apk add --no-cache curl tar && \ tar -xzf /tmp/linstor-gui.tar.gz -C /usr/share/nginx/html --strip-components=2 "linstor-gui-${LINSTOR_GUI_VERSION}/dist" && \ rm -f /tmp/linstor-gui.tar.gz && \ apk del curl tar && \ - chown -R 101:0 /usr/share/nginx/html + chown -R 101:101 /usr/share/nginx/html USER 101 EXPOSE 3373 diff --git a/packages/system/linstor-gui/templates/configmap-nginx.yaml b/packages/system/linstor-gui/templates/configmap-nginx.yaml index 0b2d97f9..e7ba02bd 100644 --- a/packages/system/linstor-gui/templates/configmap-nginx.yaml +++ b/packages/system/linstor-gui/templates/configmap-nginx.yaml @@ -1,9 +1,12 @@ {{/* Parse the LINSTOR endpoint so nginx can set `proxy_ssl_name` correctly. -The endpoint is expected to be an https:// URL like - https://linstor-controller.cozy-linstor.svc:3371 +The endpoint must be an https:// URL — http:// would downgrade the +mTLS-protected controller path. */}} -{{- $endpoint := trimPrefix "https://" (trimPrefix "http://" .Values.linstor.endpoint) -}} +{{- if not (hasPrefix "https://" .Values.linstor.endpoint) -}} +{{- fail "linstor.endpoint must start with https:// to enforce mTLS to the LINSTOR controller" -}} +{{- end -}} +{{- $endpoint := trimPrefix "https://" .Values.linstor.endpoint -}} {{- $endpointHost := (splitList ":" $endpoint) | first -}} apiVersion: v1 kind: ConfigMap @@ -41,6 +44,17 @@ data: root /usr/share/nginx/html; index index.html; + # Shared proxy + mTLS settings for the LINSTOR controller upstream + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_ssl_certificate /etc/linstor/client/tls.crt; + proxy_ssl_certificate_key /etc/linstor/client/tls.key; + proxy_ssl_trusted_certificate /etc/linstor/client/ca.crt; + proxy_ssl_verify on; + proxy_ssl_verify_depth 2; + proxy_ssl_server_name on; + proxy_ssl_name {{ $endpointHost }}; + # Static UI assets location / { try_files $uri $uri/ /index.html; @@ -49,28 +63,10 @@ data: # Proxy LINSTOR REST API over mTLS to the controller location /v1 { proxy_pass {{ .Values.linstor.endpoint }}; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_ssl_certificate /etc/linstor/client/tls.crt; - proxy_ssl_certificate_key /etc/linstor/client/tls.key; - proxy_ssl_trusted_certificate /etc/linstor/client/ca.crt; - proxy_ssl_verify on; - proxy_ssl_verify_depth 2; - proxy_ssl_server_name on; - proxy_ssl_name {{ $endpointHost }}; } location /metrics { proxy_pass {{ .Values.linstor.endpoint }}; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_ssl_certificate /etc/linstor/client/tls.crt; - proxy_ssl_certificate_key /etc/linstor/client/tls.key; - proxy_ssl_trusted_certificate /etc/linstor/client/ca.crt; - proxy_ssl_verify on; - proxy_ssl_verify_depth 2; - proxy_ssl_server_name on; - proxy_ssl_name {{ $endpointHost }}; } location = /healthz { diff --git a/packages/system/linstor-gui/templates/deployment.yaml b/packages/system/linstor-gui/templates/deployment.yaml index b1407bdb..963f437a 100644 --- a/packages/system/linstor-gui/templates/deployment.yaml +++ b/packages/system/linstor-gui/templates/deployment.yaml @@ -25,7 +25,6 @@ spec: spec: serviceAccountName: linstor-gui automountServiceAccountToken: false - priorityClassName: system-cluster-critical securityContext: runAsNonRoot: true seccompProfile: From 4b76a93dc1bfdfe7ec9fc2714c99e4cef84d303b Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 13 Apr 2026 12:41:58 +0500 Subject: [PATCH 004/250] [linstor-gui] Fix secret mount mode so nginx can read mTLS client certs The linstor-client-tls secret was mounted with defaultMode 0400 and owned by root, so the nginx process (UID 101) got EACCES on tls.crt and crash-looped with: [emerg] cannot load certificate "/etc/linstor/client/tls.crt": BIO_new_file() failed ... Permission denied Set defaultMode to 0444. The secret volume is pod-local and readOnlyRootFilesystem is on, so making it world-readable inside the pod is not a broader disclosure. Verified on dev10: pod Ready, /healthz 200, /v1/nodes proxied through mTLS to linstor-controller:3371 returns real node data. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Myasnikov Daniil --- packages/system/linstor-gui/templates/deployment.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/system/linstor-gui/templates/deployment.yaml b/packages/system/linstor-gui/templates/deployment.yaml index 963f437a..838e87bd 100644 --- a/packages/system/linstor-gui/templates/deployment.yaml +++ b/packages/system/linstor-gui/templates/deployment.yaml @@ -93,7 +93,9 @@ spec: - name: linstor-client-tls secret: secretName: {{ .Values.linstor.clientSecret }} - defaultMode: 0400 + # 0444 so nginx (UID 101) can read the mTLS client keypair; + # the secret volume is pod-local, not a broader disclosure. + defaultMode: 0444 - name: tmp emptyDir: {} - name: nginx-cache From bbeaaf3dab4078c1c9a3c9635dd36176a74f344b Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 13 Apr 2026 12:49:48 +0500 Subject: [PATCH 005/250] [linstor-gui] Update test to match 0444 secret mount mode Follow-up to 4b76a93d: the assertion still expected defaultMode 0400 and would fail in CI. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Myasnikov Daniil --- packages/system/linstor-gui/tests/deployment_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/system/linstor-gui/tests/deployment_test.yaml b/packages/system/linstor-gui/tests/deployment_test.yaml index 420ec3e1..05013d2a 100644 --- a/packages/system/linstor-gui/tests/deployment_test.yaml +++ b/packages/system/linstor-gui/tests/deployment_test.yaml @@ -37,7 +37,7 @@ tests: name: linstor-client-tls secret: secretName: linstor-client-tls - defaultMode: 0400 + defaultMode: 0444 - contains: path: spec.template.spec.containers[0].volumeMounts content: From 727401c2c63b2e4f6cfafbe8aff4d0771001d52a Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 13 Apr 2026 20:42:14 +0500 Subject: [PATCH 006/250] [linstor-gui] Address coderabbit feedback on /healthz and reload annotation - Fix /healthz Content-Type header: nginx silently drops `add_header` after `return` because the response is already finalized. Switch to `default_type text/plain;` placed BEFORE the `return 200` so the probe response is actually labeled as text/plain. - Drop the redundant `checksum/nginx-config` pod-template annotation: the deployment already carries `reloader.stakater.com/auto: "true"`, so Stakater Reloader handles ConfigMap rollouts. Matches the sister `linstor` package convention. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Myasnikov Daniil --- packages/system/linstor-gui/templates/configmap-nginx.yaml | 5 ++++- packages/system/linstor-gui/templates/deployment.yaml | 2 -- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/system/linstor-gui/templates/configmap-nginx.yaml b/packages/system/linstor-gui/templates/configmap-nginx.yaml index e7ba02bd..0ba8e4fe 100644 --- a/packages/system/linstor-gui/templates/configmap-nginx.yaml +++ b/packages/system/linstor-gui/templates/configmap-nginx.yaml @@ -71,8 +71,11 @@ data: location = /healthz { access_log off; + # default_type must come BEFORE `return` — `add_header` after a + # `return` is silently dropped by nginx because the response is + # already finalized. + default_type text/plain; return 200 'ok'; - add_header Content-Type text/plain; } } } diff --git a/packages/system/linstor-gui/templates/deployment.yaml b/packages/system/linstor-gui/templates/deployment.yaml index 838e87bd..92e2f7a6 100644 --- a/packages/system/linstor-gui/templates/deployment.yaml +++ b/packages/system/linstor-gui/templates/deployment.yaml @@ -18,8 +18,6 @@ spec: maxSurge: 1 template: metadata: - annotations: - checksum/nginx-config: {{ include (print $.Template.BasePath "/configmap-nginx.yaml") . | sha256sum }} labels: {{- include "linstor-gui.labels" . | nindent 8 }} spec: From 6bf3c86aad5e26ce28386f0b013cd62c3a61fe80 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Mon, 13 Apr 2026 22:50:08 +0300 Subject: [PATCH 007/250] docs: adopt Conventional Commits across contributing docs Refocus docs/agents/contributing.md on project-side conventions only and drop personal agent-behavior guidance. Switch commit format from [component] prefix to Conventional Commits (type(scope): description) in contributing.md, overview.md, AGENTS.md, .gemini/styleguide.md and the PR template. Reference .github/PULL_REQUEST_TEMPLATE.md instead of duplicating the PR body template in contributing.md. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- .gemini/styleguide.md | 20 +-- .github/PULL_REQUEST_TEMPLATE.md | 14 +- AGENTS.md | 2 +- docs/agents/contributing.md | 219 ++++--------------------------- docs/agents/overview.md | 3 +- 5 files changed, 48 insertions(+), 210 deletions(-) diff --git a/.gemini/styleguide.md b/.gemini/styleguide.md index 00201bfb..7185a625 100644 --- a/.gemini/styleguide.md +++ b/.gemini/styleguide.md @@ -33,14 +33,18 @@ Similarly, never propose edits to: ## Commit and PR Requirements -Each commit must start with a component prefix: `[component] Brief description`. +Each commit must follow [Conventional Commits](https://www.conventionalcommits.org/) format: `type(scope): brief description`. -Valid prefixes: +Valid types: `feat`, `fix`, `docs`, `style`, `refactor`, `perf`, `test`, `build`, `ci`, `chore`. -- System: `[dashboard]`, `[platform]`, `[cilium]`, `[kube-ovn]`, `[linstor]`, `[fluxcd]`, `[cluster-api]` -- Apps: `[postgres]`, `[mariadb]`, `[redis]`, `[kafka]`, `[clickhouse]`, `[kubernetes]`, `[virtual-machine]` -- Meta: `[tests]`, `[ci]`, `[docs]`, `[maintenance]` -- Package-specific: any `[]` matching a directory under `packages/` +Valid scopes: + +- System: `dashboard`, `platform`, `cilium`, `kube-ovn`, `linstor`, `fluxcd`, `cluster-api` +- Apps: `postgres`, `mariadb`, `redis`, `kafka`, `clickhouse`, `kubernetes`, `virtual-machine` +- Meta: `api`, `hack`, `tests`, `ci`, `docs` +- Package-specific: any `` matching a directory under `packages/` + +Breaking changes: append `!` after type/scope (`feat(api)!: ...`) or add a `BREAKING CHANGE:` footer. Each commit must have a `Signed-off-by:` trailer (produced by `git commit --signoff`). @@ -48,11 +52,11 @@ PR body must contain a release note block: ````text ```release-note -[component] Human-readable changelog entry +type(scope): human-readable changelog entry ``` ```` -Flag any PR whose commits lack the prefix or signoff, or whose body has no release-note block. +Flag any PR whose commits lack the Conventional Commits format or signoff, or whose body has no release-note block. ## Helm Chart Conventions diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 9a52457c..3588adc1 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,8 +1,10 @@ ```release-note -[] + ``` \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index eb1febf7..fe9aa8e1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -53,7 +53,7 @@ working with the **Cozystack** project. ### Conventions - **Helm Charts**: Umbrella pattern, vendored upstream charts in `charts/` - **Go Code**: Controller-runtime patterns, kubebuilder style -- **Git Commits**: `[component] Description` format with `--signoff` +- **Git Commits**: Conventional Commits (`type(scope): description`) with `--signoff` ### What NOT to Do - ❌ Edit `/vendor/`, `zz_generated.*.go`, upstream charts directly diff --git a/docs/agents/contributing.md b/docs/agents/contributing.md index d5a4366e..946ae838 100644 --- a/docs/agents/contributing.md +++ b/docs/agents/contributing.md @@ -1,167 +1,60 @@ -# Instructions for AI Agents +# Contributing Conventions for AI Agents -Guidelines for AI agents contributing to Cozystack. +Project-side conventions for commits, branches, and pull requests in Cozystack. ## Checklist for Creating a Pull Request -- [ ] Changes are made and tested -- [ ] Commit message uses correct `[component]` prefix +- [ ] Commit message follows Conventional Commits format - [ ] Commit is signed off with `--signoff` - [ ] Branch is rebased on `upstream/main` (no extra commits) - [ ] PR body includes description and release note -- [ ] PR is pushed and created with `gh pr create` -## How to Commit and Create Pull Requests +## Commit Format -### 1. Make Your Changes - -Edit the necessary files in the codebase. - -### 2. Commit with Proper Format - -Use the `[component]` prefix and `--signoff` flag: +Follow [Conventional Commits](https://www.conventionalcommits.org/) with `--signoff`: ```bash -git commit --signoff -m "[component] Brief description of changes" +git commit --signoff -m "type(scope): brief description" ``` -**Component prefixes:** -- System: `[dashboard]`, `[platform]`, `[cilium]`, `[kube-ovn]`, `[linstor]`, `[fluxcd]`, `[cluster-api]` -- Apps: `[postgres]`, `[mariadb]`, `[redis]`, `[kafka]`, `[clickhouse]`, `[virtual-machine]`, `[kubernetes]` -- Other: `[tests]`, `[ci]`, `[docs]`, `[maintenance]` +**Types:** `feat`, `fix`, `docs`, `style`, `refactor`, `perf`, `test`, `build`, `ci`, `chore` + +**Scopes:** +- System: `dashboard`, `platform`, `cilium`, `kube-ovn`, `linstor`, `fluxcd`, `cluster-api` +- Apps: `postgres`, `mariadb`, `redis`, `kafka`, `clickhouse`, `virtual-machine`, `kubernetes` +- Other: `api`, `hack`, `tests`, `ci`, `docs` + +Breaking changes: append `!` after type/scope (`feat(api)!: ...`) or add a `BREAKING CHANGE:` footer. **Examples:** ```bash -git commit --signoff -m "[dashboard] Add config hash annotations to restart pods on config changes" -git commit --signoff -m "[postgres] Update operator to version 1.2.3" -git commit --signoff -m "[docs] Add installation guide" +git commit --signoff -m "feat(dashboard): add config hash annotations to restart pods on config changes" +git commit --signoff -m "fix(postgres): update operator to version 1.2.3" +git commit --signoff -m "docs: add installation guide" ``` -### 3. Rebase on upstream/main (if needed) +## Rebasing on upstream/main -If your branch has extra commits, clean it up: +If the branch has extra commits, clean it up: ```bash -# Fetch latest git fetch upstream - -# Create clean branch from upstream/main git checkout -b my-feature upstream/main - -# Cherry-pick only your commit git cherry-pick - -# Force push to your branch git push -f origin my-feature:my-branch-name ``` -### 4. Push Your Branch +## Pull Request Body -```bash -git push origin -``` +Fill in the template at [`.github/PULL_REQUEST_TEMPLATE.md`](../../.github/PULL_REQUEST_TEMPLATE.md). It includes the required `release-note` block. -### 5. Create Pull Request +Create the PR with `gh pr create --title "type(scope): brief description" --body-file `. -Write the PR body to a temporary file: +## Fetching Unresolved Review Comments -```bash -cat > /tmp/pr_body.md << 'EOF' -## What this PR does +Cozystack uses GitHub review threads with resolution status. Only unresolved threads are actionable — resolved threads are already handled. -Brief description of the changes. - -Changes: -- Change 1 -- Change 2 - -### Release note - -```release-note -[component] Description for changelog -``` -EOF -``` - -Create the PR: - -```bash -gh pr create --title "[component] Brief description" --body-file /tmp/pr_body.md -``` - -Clean up: - -```bash -rm /tmp/pr_body.md -``` - -## Addressing AI Bot Reviewer Comments - -When the user asks to fix comments from AI bot reviewers (like Qodo, Copilot, etc.): - -### 1. Get PR Comments - -View all comments on the pull request: - -```bash -gh pr view --comments -``` - -Or for the current branch: - -```bash -gh pr view --comments -``` - -### 2. Review Each Comment Carefully - -**Important**: Do NOT blindly apply all suggestions. Each comment should be evaluated: - -- **Consider context** - Does the suggestion make sense for this specific case? -- **Check project conventions** - Does it align with Cozystack patterns? -- **Evaluate impact** - Will this improve code quality or introduce issues? -- **Question validity** - AI bots can be wrong or miss context - -**When to apply:** -- ✅ Legitimate bugs or security issues -- ✅ Clear improvements to code quality -- ✅ Better error handling or edge cases -- ✅ Conformance to project conventions - -**When to skip:** -- ❌ Stylistic preferences that don't match project style -- ❌ Over-engineering simple code -- ❌ Changes that break existing patterns -- ❌ Suggestions that show misunderstanding of the code - -### 3. Apply Valid Fixes - -Make changes addressing the valid comments. Use your judgment. - -### 4. Leave Changes Uncommitted - -**Critical**: Do NOT commit or push the changes automatically. - -Leave the changes in the working directory so the user can: -- Review the fixes -- Decide whether to commit them -- Make additional adjustments if needed - -```bash -# After making changes, show status but DON'T commit -git status -git diff -``` - -The user will commit and push when ready. - -## Code Review Comments - -When asked to fix code review comments, **always work only with unresolved (open) comments**. Resolved comments should be ignored as they have already been addressed. - -### Getting Unresolved Review Comments - -Use GitHub GraphQL API to fetch only unresolved review comments from a pull request: +The REST endpoint `/pulls/{pr}/reviews` returns review summaries, not individual review comments. Use the GraphQL API to access `reviewThreads` with `isResolved` status: ```bash gh api graphql -F owner=cozystack -F repo=cozystack -F pr= -f query=' @@ -189,27 +82,7 @@ query($owner: String!, $repo: String!, $pr: Int!) { }' --jq '.data.repository.pullRequest.reviewThreads.nodes[] | select(.isResolved == false) | .comments.nodes[]' ``` -### Filtering for Unresolved Comments - -The key filter is `select(.isResolved == false)` which ensures only unresolved review threads are processed. Each thread can contain multiple comments, but if the thread is resolved, all its comments should be ignored. - -### Working with Review Comments - -1. **Fetch unresolved comments** using the GraphQL query above -2. **Parse the results** to identify: - - File path (`path`) - - Line number (`line` or `originalLine`) - - Comment text (`bodyText`) - - Author (`author.login`) -3. **Address each unresolved comment** by: - - Locating the relevant code section - - Making the requested changes - - Ensuring the fix addresses the concern raised -4. **Do NOT process resolved comments** - they have already been handled - -### Example: Compact List of Unresolved Comments - -For a quick overview of unresolved comments: +Compact one-line variant: ```bash gh api graphql -F owner=cozystack -F repo=cozystack -F pr= -f query=' @@ -233,43 +106,3 @@ query($owner: String!, $repo: String!, $pr: Int!) { } }' --jq '.data.repository.pullRequest.reviewThreads.nodes[] | select(.isResolved == false) | .comments.nodes[] | "\(.path):\(.line // "N/A") - \(.author.login): \(.bodyText[:150])"' ``` - -### Important Notes - -- **REST API limitation**: The REST endpoint `/pulls/{pr}/reviews` returns review summaries, not individual review comments. Use GraphQL API for accessing `reviewThreads` with `isResolved` status. -- **Thread-based resolution**: Comments are organized in threads. If a thread is resolved (`isResolved: true`), ignore all comments in that thread. -- **Always filter**: Never process comments from resolved threads, even if they appear in the results. - -### Example Workflow - -```bash -# Get PR comments -gh pr view 1234 --comments - -# Review comments and identify valid ones -# Make necessary changes to address valid comments -# ... edit files ... - -# Show what was changed (but don't commit) -git status -git diff - -# Tell the user what was fixed and what was skipped -``` - -## Git Permissions - -Request these permissions when needed: -- `git_write` - For commit, rebase, cherry-pick, branch operations -- `network` - For push, fetch, pull operations - -## Common Issues - -**PR has extra commits?** -→ Rebase on `upstream/main` and cherry-pick only your commits - -**Wrong commit message?** -→ `git commit --amend --signoff -m "[correct] message"` then `git push -f` - -**Need to update PR?** -→ `gh pr edit --body "new description"` diff --git a/docs/agents/overview.md b/docs/agents/overview.md index b6d69e68..35798961 100644 --- a/docs/agents/overview.md +++ b/docs/agents/overview.md @@ -78,11 +78,10 @@ packages/// - Add proper error handling and structured logging ### Git Commits -- Use format: `[component] Description` +- Follow [Conventional Commits](https://www.conventionalcommits.org/) format: `type(scope): description` - Always use `--signoff` flag - Reference PR numbers when available - Keep commits atomic and focused -- Follow conventional commit format for changelogs ### Documentation From 525cc9eab2139b2076c50631f3e2117a51034862 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Mon, 13 Apr 2026 22:50:50 +0300 Subject: [PATCH 008/250] docs(agents): document Assisted-By trailer for AI-authored commits Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- docs/agents/contributing.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/agents/contributing.md b/docs/agents/contributing.md index 946ae838..17da488c 100644 --- a/docs/agents/contributing.md +++ b/docs/agents/contributing.md @@ -33,6 +33,18 @@ git commit --signoff -m "fix(postgres): update operator to version 1.2.3" git commit --signoff -m "docs: add installation guide" ``` +### AI Agent Attribution + +When an AI agent authors or materially assists with a commit, add an `Assisted-By:` trailer naming the model: + +``` +Assisted-By: Claude +Assisted-By: GPT-5 +Assisted-By: Gemini +``` + +This sits alongside the `Signed-off-by:` trailer produced by `--signoff`. Use one trailer per model if multiple contributed. + ## Rebasing on upstream/main If the branch has extra commits, clean it up: From 90a9d6e905ad95d8a8edba7f4b95ed3ef3316371 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Mon, 13 Apr 2026 23:09:27 +0300 Subject: [PATCH 009/250] docs(contributing): sync scope lists and fix lint nits Address review feedback: - Sync scope lists between PR template and contributing guide - Add 'maintenance' to Other scopes in contributing guide - Add scope to the docs example for format consistency - Simplify rebase push example to drop unnecessary refspec mapping - Add 'text' language tag to trailer code fence (MD040) Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- .github/PULL_REQUEST_TEMPLATE.md | 6 +++--- docs/agents/contributing.md | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 3588adc1..7710149c 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,9 +1,9 @@ + +## Fixes + +* **[build] Filter git describe to match only v* tags**: Adds `--match 'v*'` to all `git describe` calls in `hack/common-envs.mk`. The `api/apps/v1alpha1/*` subtags share the same commit as release tags, causing `git describe --exact-match` to pick `api/apps/v1alpha1/vX.Y.Z` instead of `vX.Y.Z`, producing invalid Docker image tags ([**@kvaps**](https://github.com/kvaps) in #2386, #2388). + +## Development, Testing, and CI/CD + +* **[ci] Replace cozystack-bot PAT with cozystack-ci GitHub App**: Replaces the long-lived `cozystack-bot` personal access token with short-lived, scoped tokens from the `cozystack-ci` GitHub App across all release workflows. Improves security and auditability of CI operations ([**@tym83**](https://github.com/tym83) in #2351). + +* **[ci] Replace GH_PAT with cozystack-ci GitHub App token in pull-requests workflow**: Switches the pull-requests release workflow to use the cozystack-ci GitHub App token instead of the personal access token ([**@kvaps**](https://github.com/kvaps) in #2383). + +* **[ci] Use cozystack org noreply email for bot commits**: Updates CI workflows to use the cozystack organization noreply email for bot commits ([**@kvaps**](https://github.com/kvaps) in #2392). + +--- + +**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.1.5...v1.1.6 diff --git a/docs/changelogs/v1.2.2.md b/docs/changelogs/v1.2.2.md new file mode 100644 index 00000000..3cf1731c --- /dev/null +++ b/docs/changelogs/v1.2.2.md @@ -0,0 +1,63 @@ + + +## Features and Improvements + +* **[linstor] Update piraeus-server to v1.33.2 with selected backports**: Bumps LINSTOR server from v1.33.1 to v1.33.2 and adds backported patches for improved storage reliability: a stale bitmap adjust retry mechanism for automatic recovery after bitmap attach errors, LUKS2 header sizing and optimal I/O size detection improvements for more reliable disk formatting, and the maintainer implementation backport. All patches verified against upstream v1.33.1 with `git apply --check` and `gradlew compileJava` ([**@kvaps**](https://github.com/kvaps) in #2331, #2377). + +* **[postgres] Fix system PostgreSQL images to 17.7-standard-trixie**: Hardcodes PostgreSQL 17.7-standard-trixie images for system PostgreSQL instances. This ensures system databases use the correct image variant consistent with the monitoring stack requirements introduced in v1.2.1 ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2369). + +## Fixes + +* **[cilium] Opt-out of cri-containerd.apparmor.d for nsenter init containers**: On Ubuntu 22.04+, Debian, and other distributions that load the `cri-containerd.apparmor.d` AppArmor profile by default for containerd workloads, the kernel denied `nsenter` namespace entry in cilium-agent init containers (`mount-cgroup`, `apply-sysctl-overwrites`, `clean-cilium-state`), causing the agent to land in `Init:CrashLoopBackOff` and cascading platform failures. Per-container `container.apparmor.security.beta.kubernetes.io` annotations now opt the affected containers out of this profile, applied only on non-Talos cilium variants (`cilium-generic`, `kubeovn-cilium-generic`). The vendored daemonset template is also patched to strip the upstream `semverCompare "<1.30.0"` AppArmor block, preventing duplicate annotation keys. Talos variants are untouched as Talos does not load the AppArmor LSM ([**@lexfrei**](https://github.com/lexfrei) in #2370, #2378). + +* **[virtual-machine] Exclude external VM services from Cilium BPF LB**: Adds the `service.kubernetes.io/service-proxy-name: "cozy-proxy"` label to VM LoadBalancer services when `external: true`, telling Cilium to skip BPF processing entirely for these services. This fixes two issues: inter-tenant connectivity via public LB IPs (Cilium's DNAT caused cross-tenant pod-to-pod flow classification, triggering CiliumClusterwideNetworkPolicy blocks) and WholeIP broken on Cilium 1.19+ (wildcard service drop entries blocked traffic to LB IPs on undeclared ports before it reached netfilter/cozy-proxy). MetalLB L2 advertisement and kube-ovn routing remain unaffected ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #2357, #2361). + +* **[monitoring] Fix infra dashboards missing in default variant**: The default platform variant deploys the monitoring chart to the `cozy-monitoring` namespace, but the dashboard rendering condition introduced in #2197 only checked for `tenant-root`. Infrastructure dashboards were not rendered in the default variant. The `cozy-monitoring` namespace is now included in the rendering condition, consistent with the existing pattern in `vmagent.yaml` ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #2365, #2367). + +* **[build] Filter git describe to match only v* tags**: Adds `--match 'v*'` to all `git describe` calls in `hack/common-envs.mk`. The `api/apps/v1alpha1/*` subtags share the same commit as release tags, causing `git describe --exact-match` to pick `api/apps/v1alpha1/vX.Y.Z` instead of `vX.Y.Z`, producing invalid Docker image tags ([**@kvaps**](https://github.com/kvaps) in #2386, #2389). + +## Development, Testing, and CI/CD + +* **[ci] Replace cozystack-bot PAT with cozystack-ci GitHub App**: Replaces the long-lived `cozystack-bot` personal access token with short-lived, scoped tokens from the `cozystack-ci` GitHub App across all release workflows (`tags.yaml`, `auto-release.yaml`, `pull-requests-release.yaml`). Improves security and auditability of CI operations ([**@tym83**](https://github.com/tym83) in #2351). + +* **[ci] Use cozystack org noreply email for bot commits**: Updates CI workflows to use the cozystack organization noreply email for bot commits ([**@kvaps**](https://github.com/kvaps) in #2392, #2393). + +* **[ci] Replace GH_PAT with cozystack-ci GitHub App token in pull-requests workflow**: Switches the pull-requests release workflow to use the cozystack-ci GitHub App token instead of the personal access token ([**@kvaps**](https://github.com/kvaps) in #2383, #2384). + +## Documentation + +* **[website] Add ApplicationDefinition naming convention reference**: Added documentation on ApplicationDefinition naming conventions and auto-restart behavior for the applicationdefinition-controller ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#478). + +* **[website] Document Talos / talosctl / Cozystack version pairing**: Added documentation covering Talos, talosctl, and Cozystack version compatibility matrix for installation ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#484). + +* **[website] Fix KubeOVN MASTER_NODES example path and key in troubleshooting**: Corrected the MASTER_NODES example path and key in the KubeOVN troubleshooting guide ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#483). + +* **[website] Prefix bundle package names with cozystack. in v1 examples**: Updated documentation examples to use the correct `cozystack.` prefix for bundle package names in enabled/disabledPackages ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#482). + +* **[website] Finish isolated-field removal and document opt-in policy labels**: Removed the obsolete `isolated` field from tenant documentation and documented the new opt-in policy labels approach ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#481). + +* **[website] Add --take-ownership flag and describe networking.* fields**: Added documentation for the `--take-ownership` flag and described the `networking.*` fields in the installation guide ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#480). + +* **[website] Add bonding (LACP) configuration how-to guide**: Added a guide for configuring network bonding with LACP on Cozystack installations ([**@sircthulhu**](https://github.com/sircthulhu) in cozystack/website#459). + +* **[website] Improve registry mirrors for tenant Kubernetes in air-gapped guide**: Improved documentation for configuring registry mirrors in tenant Kubernetes clusters for air-gapped environments ([**@sircthulhu**](https://github.com/sircthulhu) in cozystack/website#461). + +* **[website] Update backup/restore documentation for VMI/VMDisk**: Updated backup documentation with information related to VM instance and VM disk restore improvements ([**@androndo**](https://github.com/androndo) in cozystack/website#466). + +* **[website] Add updated OpenAPI spec**: Updated the OpenAPI specification for managed applications reference ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#469). + +* **[website] Add OSS Health pages and OpenSSF badge**: Added OSS Health section with OpenSSF Scorecard and Best Practices badge to the website footer ([**@tym83**](https://github.com/tym83) in cozystack/website#470). + +* **[website] Add CozySummit Virtual 2026 program announcement**: Published the CozySummit Virtual 2026 program announcement blog post ([**@tym83**](https://github.com/tym83) in cozystack/website#472). + +* **[website] Add missing release announcements for v0.1–v0.41**: Backfilled missing release announcement blog posts for Cozystack versions v0.1 through v0.41 ([**@tym83**](https://github.com/tym83) in cozystack/website#468). + +* **[talm] Render templates online in apply to resolve lookups**: Fixed talm `apply` command to render templates online, resolving template lookup failures when using modeline templates ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/talm#119). + +* **[talm] Update default Talos image to v1.12.6**: Updated the default Talos image version to v1.12.6 in talm ([**@kvaps**](https://github.com/kvaps) in cozystack/talm@03e9b6e). + +--- + +**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.2.1...v1.2.2 From bec35e3aade31fdca231915fc6ba95647a621cef Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 8 Apr 2026 13:15:43 +0500 Subject: [PATCH 019/250] [vm-default-images] Added new optional package Signed-off-by: Myasnikov Daniil --- api/apps/v1alpha1/vmdisk/types.go | 11 +- hack/cdi_golden_image_create.sh | 2 +- .../dashboard/customformsoverride.go | 62 ++++++- .../dashboard/customformsoverride_test.go | 102 +++++++++++ packages/apps/vm-disk/README.md | 24 +-- packages/apps/vm-disk/templates/dv.yaml | 8 +- packages/apps/vm-disk/values.schema.json | 17 +- packages/apps/vm-disk/values.yaml | 8 +- .../platform/sources/vm-default-images.yaml | 22 +++ packages/system/vm-default-images/Chart.yaml | 5 + packages/system/vm-default-images/Makefile | 4 + .../vm-default-images/templates/dv.yaml | 45 +++++ packages/system/vm-default-images/values.yaml | 167 ++++++++++++++++++ 13 files changed, 448 insertions(+), 29 deletions(-) create mode 100644 packages/core/platform/sources/vm-default-images.yaml create mode 100644 packages/system/vm-default-images/Chart.yaml create mode 100644 packages/system/vm-default-images/Makefile create mode 100644 packages/system/vm-default-images/templates/dv.yaml create mode 100644 packages/system/vm-default-images/values.yaml diff --git a/api/apps/v1alpha1/vmdisk/types.go b/api/apps/v1alpha1/vmdisk/types.go index aefac1e4..c493f49c 100644 --- a/api/apps/v1alpha1/vmdisk/types.go +++ b/api/apps/v1alpha1/vmdisk/types.go @@ -32,21 +32,28 @@ type ConfigSpec struct { } type Source struct { + // Clone an existing vm-disk. + Disk *SourceDisk `json:"disk,omitempty"` // Download image from an HTTP source. Http *SourceHTTP `json:"http,omitempty"` - // Use image by name. + // Use image by name from default collection. Image *SourceImage `json:"image,omitempty"` // Upload local image. Upload *SourceUpload `json:"upload,omitempty"` } +type SourceDisk struct { + // Name of the vm-disk to clone. + Name string `json:"name"` +} + type SourceHTTP struct { // URL to download the image. Url string `json:"url"` } type SourceImage struct { - // Name of the image to use (uploaded as "golden image" or from the list: `ubuntu`, `fedora`, `cirros`, `alpine`, `talos`). + // Name of the image to use. Name string `json:"name"` } diff --git a/hack/cdi_golden_image_create.sh b/hack/cdi_golden_image_create.sh index be1558d0..f19f9970 100644 --- a/hack/cdi_golden_image_create.sh +++ b/hack/cdi_golden_image_create.sh @@ -16,7 +16,7 @@ kubectl create -f - < Date: Wed, 8 Apr 2026 14:26:51 +0500 Subject: [PATCH 020/250] [vm-disk] Fix vm-disk-rd Signed-off-by: Myasnikov Daniil --- packages/system/vm-disk-rd/cozyrds/vm-disk.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/system/vm-disk-rd/cozyrds/vm-disk.yaml b/packages/system/vm-disk-rd/cozyrds/vm-disk.yaml index 4d0d7e37..3ce046ed 100644 --- a/packages/system/vm-disk-rd/cozyrds/vm-disk.yaml +++ b/packages/system/vm-disk-rd/cozyrds/vm-disk.yaml @@ -8,7 +8,7 @@ spec: singular: vmdisk plural: vmdisks openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"source":{"description":"The source image location used to create a disk.","type":"object","default":{},"properties":{"http":{"description":"Download image from an HTTP source.","type":"object","required":["url"],"properties":{"url":{"description":"URL to download the image.","type":"string"}}},"image":{"description":"Use image by name.","type":"object","required":["name"],"properties":{"name":{"description":"Name of the image to use (uploaded as \"golden image\" or from the list: `ubuntu`, `fedora`, `cirros`, `alpine`, `talos`).","type":"string"}}},"upload":{"description":"Upload local image.","type":"object"}}},"optical":{"description":"Defines if disk should be considered optical.","type":"boolean","default":false},"storage":{"description":"The size of the disk allocated for the virtual machine.","default":"5Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"}}} + {"title":"Chart Values","type":"object","properties":{"source":{"description":"The source image location used to create a disk.","type":"object","default":{},"properties":{"disk":{"description":"Clone an existing vm-disk.","type":"object","required":["name"],"properties":{"name":{"description":"Name of the vm-disk to clone.","type":"string"}}},"http":{"description":"Download image from an HTTP source.","type":"object","required":["url"],"properties":{"url":{"description":"URL to download the image.","type":"string"}}},"image":{"description":"Use image by name from default collection.","type":"object","required":["name"],"properties":{"name":{"description":"Name of the image to use.","type":"string"}}},"upload":{"description":"Upload local image.","type":"object"}}},"optical":{"description":"Defines if disk should be considered optical.","type":"boolean","default":false},"storage":{"description":"The size of the disk allocated for the virtual machine.","default":"5Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"}}} release: prefix: vm-disk- labels: From 6b6118ce56f559e9136c2543eec075f95b2495e6 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 13 Apr 2026 20:44:13 +0500 Subject: [PATCH 021/250] [platform] Add migration 38 to rename vm-image-* DataVolumes to vm-default-images-* The vm-disk package previously referenced golden-image DataVolumes with the prefix "vm-image-" in the cozy-public namespace. The new vm-default-images package creates them as "vm-default-images-", so any existing vm-disk that sources an image by name would fail to find its PVC after upgrade. Migration 38 renames every DataVolume in cozy-public whose name starts with "vm-image-" to "vm-default-images-" (dropping the old object and re-creating with the new name) so live vm-disk resources keep resolving their source PVC transparently. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Myasnikov Daniil --- .../platform/images/migrations/migrations/38 | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100755 packages/core/platform/images/migrations/migrations/38 diff --git a/packages/core/platform/images/migrations/migrations/38 b/packages/core/platform/images/migrations/migrations/38 new file mode 100755 index 00000000..59536cb1 --- /dev/null +++ b/packages/core/platform/images/migrations/migrations/38 @@ -0,0 +1,52 @@ +#!/bin/sh +# Migration 38 --> 39 +# Rename DataVolumes/PVCs in cozy-public from vm-image- to vm-default-images-. +# +# The vm-disk package previously looked up golden images with the prefix "vm-image-". +# The new vm-default-images package uses the prefix "vm-default-images-". +# This migration renames any existing DataVolumes so that live vm-disk resources +# continue to resolve their source PVC after upgrade. + +set -euo pipefail + +NAMESPACE="cozy-public" + +# Skip if CDI DataVolume CRD is not installed +if ! kubectl api-resources --api-group=cdi.kubevirt.io -o name 2>/dev/null | grep -q '^datavolumes\.'; then + echo "CDI DataVolume CRD not found, skipping rename" + kubectl create configmap -n cozy-system cozystack-version \ + --from-literal=version=39 --dry-run=client -o yaml | kubectl apply -f- + exit 0 +fi + +# Find DataVolumes whose name starts with "vm-image-" +DVNAMES=$(kubectl get datavolumes.cdi.kubevirt.io -n "$NAMESPACE" \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null \ + | grep '^vm-image-' || true) + +if [ -z "$DVNAMES" ]; then + echo "No vm-image-* DataVolumes found in $NAMESPACE, nothing to rename" +else + for DVNAME in $DVNAMES; do + NEWNAME="vm-default-images-${DVNAME#vm-image-}" + echo "Renaming DataVolume $DVNAME -> $NEWNAME in namespace $NAMESPACE" + + # Export existing DV spec, swap name, and apply as new object + kubectl get datavolumes.cdi.kubevirt.io -n "$NAMESPACE" "$DVNAME" -o json \ + | jq --arg new "$NEWNAME" ' + del(.metadata.resourceVersion, .metadata.uid, .metadata.creationTimestamp, + .metadata.generation, .metadata.selfLink, .metadata.managedFields, + .status) + | .metadata.name = $new + ' \ + | kubectl apply -f - + + # Delete the old DataVolume + kubectl delete datavolumes.cdi.kubevirt.io -n "$NAMESPACE" "$DVNAME" --ignore-not-found=true + echo "Renamed $DVNAME -> $NEWNAME" + done +fi + +# Stamp version +kubectl create configmap -n cozy-system cozystack-version \ + --from-literal=version=39 --dry-run=client -o yaml | kubectl apply -f- From b9a17295cccda01f2644399f55236e6e4ab58db4 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 13 Apr 2026 20:44:19 +0500 Subject: [PATCH 022/250] [platform] Wire vm-default-images into iaas bundle The PackageSource for vm-default-images existed but was never included in any bundle, so it would never be installed. Add it to iaas.yaml immediately after kubevirt-cdi (which it depends on) so it is deployed together with the rest of the KubeVirt IaaS stack. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Myasnikov Daniil --- packages/core/platform/templates/bundles/iaas.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core/platform/templates/bundles/iaas.yaml b/packages/core/platform/templates/bundles/iaas.yaml index bd002c16..7485eb91 100644 --- a/packages/core/platform/templates/bundles/iaas.yaml +++ b/packages/core/platform/templates/bundles/iaas.yaml @@ -8,6 +8,7 @@ {{- end -}} {{include "cozystack.platform.package" (list "cozystack.kubevirt" "default" $ $kubevirtComponents) }} {{include "cozystack.platform.package.default" (list "cozystack.kubevirt-cdi" $) }} +{{include "cozystack.platform.package.default" (list "cozystack.vm-default-images" $) }} {{include "cozystack.platform.package.optional.default" (list "cozystack.gpu-operator" $) }} {{include "cozystack.platform.package.default" (list "cozystack.kamaji" $) }} {{include "cozystack.platform.package.default" (list "cozystack.capi-operator" $) }} From 15dfc2521ed5c9fa281dafc42be01dca242ecd32 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 13 Apr 2026 20:44:25 +0500 Subject: [PATCH 023/250] [vm-disk] Remove unused DataVolume lookup in dv.yaml The \$dv variable assigned by the lookup call on the image PVC was never read. Remove the dead line to keep the template tidy. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Myasnikov Daniil --- packages/apps/vm-disk/templates/dv.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/apps/vm-disk/templates/dv.yaml b/packages/apps/vm-disk/templates/dv.yaml index e1227dcd..521a82b8 100644 --- a/packages/apps/vm-disk/templates/dv.yaml +++ b/packages/apps/vm-disk/templates/dv.yaml @@ -21,7 +21,6 @@ spec: {{- end }} source: {{- if hasKey .Values.source "image" }} - {{- $dv := lookup "cdi.kubevirt.io/v1beta1" "DataVolume" "cozy-public" (printf "vm-default-images-%s" .Values.source.image.name) }} pvc: name: vm-default-images-{{ required "A valid .Values.source.image.name entry required!" .Values.source.image.name }} namespace: cozy-public From 551b4e65ac439c8dc310fa583f6961b3ad2ad2a0 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 13 Apr 2026 20:44:32 +0500 Subject: [PATCH 024/250] [vm-default-images] Fix CentOS Stream image URLs The x86_64 architecture token was placed inside the filename segment (CentOS-Stream-GenericCloud-x86_64-N-latest.x86_64.qcow2) which does not match the actual CentOS mirror layout. Correct both CentOS Stream 9 and 10 URLs to the canonical form without the extra architecture infix. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Myasnikov Daniil --- packages/system/vm-default-images/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/system/vm-default-images/values.yaml b/packages/system/vm-default-images/values.yaml index 8d66c281..a4677bc8 100644 --- a/packages/system/vm-default-images/values.yaml +++ b/packages/system/vm-default-images/values.yaml @@ -112,7 +112,7 @@ images: architecture: amd64 description: "Debian 13 (Trixie) generic cloud image" - name: centos-stream-9 - url: https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-x86_64-9-latest.x86_64.qcow2 + url: https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2 storage: 20Gi os: family: Linux @@ -121,7 +121,7 @@ images: architecture: amd64 description: "CentOS Stream 9 GenericCloud image" - name: centos-stream-10 - url: https://cloud.centos.org/centos/10-stream/x86_64/images/CentOS-Stream-GenericCloud-x86_64-10-latest.x86_64.qcow2 + url: https://cloud.centos.org/centos/10-stream/x86_64/images/CentOS-Stream-GenericCloud-10-latest.x86_64.qcow2 storage: 20Gi os: family: Linux From 776b4f2c8d9d24c5a5e9501f7276bede666c86df Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 13 Apr 2026 20:44:39 +0500 Subject: [PATCH 025/250] [dashboard] Guard nested type assertions in VMDisk customformsoverride Accessing imgName["properties"].(map[string]any) and diskName["properties"].(map[string]any) without an ok-check would panic if the schema does not contain a "properties" key. Wrap both assertions in ok-guarded form consistent with the outer defensive style. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Myasnikov Daniil --- .../dashboard/customformsoverride.go | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/internal/controller/dashboard/customformsoverride.go b/internal/controller/dashboard/customformsoverride.go index 13809248..0a20ae71 100644 --- a/internal/controller/dashboard/customformsoverride.go +++ b/internal/controller/dashboard/customformsoverride.go @@ -256,24 +256,28 @@ func applyListInputOverrides(schema map[string]any, kind string, openAPIProps ma if sourceObj, ok := specProps["source"].(map[string]any); ok { if imgProps, ok := sourceObj["properties"].(map[string]any); ok { if imgName, ok := imgProps["image"].(map[string]any); ok { - imgName["properties"].(map[string]any)["name"] = map[string]any{ - "type": "listInput", - "customProps": map[string]any{ - "valueUri": "/api/clusters/{cluster}/k8s/apis/cdi.kubevirt.io/v1beta1/namespaces/cozy-public/datavolumes", - "keysToValue": []any{"metadata", "annotations", "vm-default-images.cozystack.io/name"}, - "keysToLabel": []any{"metadata", "annotations", "vm-default-images.cozystack.io/description"}, - }, + if imgNameProps, ok := imgName["properties"].(map[string]any); ok { + imgNameProps["name"] = map[string]any{ + "type": "listInput", + "customProps": map[string]any{ + "valueUri": "/api/clusters/{cluster}/k8s/apis/cdi.kubevirt.io/v1beta1/namespaces/cozy-public/datavolumes", + "keysToValue": []any{"metadata", "annotations", "vm-default-images.cozystack.io/name"}, + "keysToLabel": []any{"metadata", "annotations", "vm-default-images.cozystack.io/description"}, + }, + } } } // Override source.disk.name to be an API-backed dropdown listing VMDisk resources if diskName, ok := imgProps["disk"].(map[string]any); ok { - diskName["properties"].(map[string]any)["name"] = map[string]any{ - "type": "listInput", - "customProps": map[string]any{ - "valueUri": "/api/clusters/{cluster}/k8s/apis/apps.cozystack.io/v1alpha1/namespaces/{namespace}/vmdisks", - "keysToValue": []any{"metadata", "name"}, - "keysToLabel": []any{"metadata", "name"}, - }, + if diskNameProps, ok := diskName["properties"].(map[string]any); ok { + diskNameProps["name"] = map[string]any{ + "type": "listInput", + "customProps": map[string]any{ + "valueUri": "/api/clusters/{cluster}/k8s/apis/apps.cozystack.io/v1alpha1/namespaces/{namespace}/vmdisks", + "keysToValue": []any{"metadata", "name"}, + "keysToLabel": []any{"metadata", "name"}, + }, + } } } } From eeda31eb95430105e3798a93eb956c134e67d65b Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 13 Apr 2026 20:44:47 +0500 Subject: [PATCH 026/250] [api] Regenerate deepcopy and CRD manifests via make generate Running make generate at repo root regenerates the zz_generated.deepcopy.go files. Key fix: Source.DeepCopyInto in the vmdisk package now handles the new Disk *SourceDisk pointer field, which was previously missing and would cause pointer aliasing on deep-copy. Other generated changes are incidental updates to vminstance (Networks field added, Subnet renamed to Network) and backups CRD manifests picked up by controller-gen from the current HEAD. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Myasnikov Daniil --- .../v1alpha1/vmdisk/zz_generated.deepcopy.go | 20 ++++++++++ .../vminstance/zz_generated.deepcopy.go | 37 +++++++++-------- .../v1alpha1/vpc/zz_generated.deepcopy.go | 40 +++++++++++++++++++ api/backups/v1alpha1/zz_generated.deepcopy.go | 17 +++++++- .../backups.cozystack.io_backups.yaml | 16 ++++---- 5 files changed, 105 insertions(+), 25 deletions(-) diff --git a/api/apps/v1alpha1/vmdisk/zz_generated.deepcopy.go b/api/apps/v1alpha1/vmdisk/zz_generated.deepcopy.go index e8ca986e..301bb1d9 100644 --- a/api/apps/v1alpha1/vmdisk/zz_generated.deepcopy.go +++ b/api/apps/v1alpha1/vmdisk/zz_generated.deepcopy.go @@ -70,6 +70,11 @@ func (in *ConfigSpec) DeepCopy() *ConfigSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Source) DeepCopyInto(out *Source) { *out = *in + if in.Disk != nil { + in, out := &in.Disk, &out.Disk + *out = new(SourceDisk) + **out = **in + } if in.Http != nil { in, out := &in.Http, &out.Http *out = new(SourceHTTP) @@ -97,6 +102,21 @@ func (in *Source) DeepCopy() *Source { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SourceDisk) DeepCopyInto(out *SourceDisk) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SourceDisk. +func (in *SourceDisk) DeepCopy() *SourceDisk { + if in == nil { + return nil + } + out := new(SourceDisk) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SourceHTTP) DeepCopyInto(out *SourceHTTP) { *out = *in diff --git a/api/apps/v1alpha1/vminstance/zz_generated.deepcopy.go b/api/apps/v1alpha1/vminstance/zz_generated.deepcopy.go index 00b2a710..86d5ddfc 100644 --- a/api/apps/v1alpha1/vminstance/zz_generated.deepcopy.go +++ b/api/apps/v1alpha1/vminstance/zz_generated.deepcopy.go @@ -63,9 +63,14 @@ func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) { *out = make([]Disk, len(*in)) copy(*out, *in) } + if in.Networks != nil { + in, out := &in.Networks, &out.Networks + *out = make([]Network, len(*in)) + copy(*out, *in) + } if in.Subnets != nil { in, out := &in.Subnets, &out.Subnets - *out = make([]Subnet, len(*in)) + *out = make([]Network, len(*in)) copy(*out, *in) } if in.Gpus != nil { @@ -121,6 +126,21 @@ func (in *GPU) DeepCopy() *GPU { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Network) DeepCopyInto(out *Network) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Network. +func (in *Network) DeepCopy() *Network { + if in == nil { + return nil + } + out := new(Network) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Resources) DeepCopyInto(out *Resources) { *out = *in @@ -138,18 +158,3 @@ func (in *Resources) DeepCopy() *Resources { in.DeepCopyInto(out) return out } - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Subnet) DeepCopyInto(out *Subnet) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Subnet. -func (in *Subnet) DeepCopy() *Subnet { - if in == nil { - return nil - } - out := new(Subnet) - in.DeepCopyInto(out) - return out -} diff --git a/api/apps/v1alpha1/vpc/zz_generated.deepcopy.go b/api/apps/v1alpha1/vpc/zz_generated.deepcopy.go index cd718167..3cda29c4 100644 --- a/api/apps/v1alpha1/vpc/zz_generated.deepcopy.go +++ b/api/apps/v1alpha1/vpc/zz_generated.deepcopy.go @@ -58,6 +58,16 @@ func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) { *out = make([]Subnet, len(*in)) copy(*out, *in) } + if in.Peers != nil { + in, out := &in.Peers, &out.Peers + *out = make([]Peer, len(*in)) + copy(*out, *in) + } + if in.Routes != nil { + in, out := &in.Routes, &out.Routes + *out = make([]Route, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigSpec. @@ -70,6 +80,36 @@ func (in *ConfigSpec) DeepCopy() *ConfigSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Peer) DeepCopyInto(out *Peer) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Peer. +func (in *Peer) DeepCopy() *Peer { + if in == nil { + return nil + } + out := new(Peer) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Route) DeepCopyInto(out *Route) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Route. +func (in *Route) DeepCopy() *Route { + if in == nil { + return nil + } + out := new(Route) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Subnet) DeepCopyInto(out *Subnet) { *out = *in diff --git a/api/backups/v1alpha1/zz_generated.deepcopy.go b/api/backups/v1alpha1/zz_generated.deepcopy.go index b7387772..89f6171f 100644 --- a/api/backups/v1alpha1/zz_generated.deepcopy.go +++ b/api/backups/v1alpha1/zz_generated.deepcopy.go @@ -1,5 +1,21 @@ //go:build !ignore_autogenerated +/* +Copyright 2025 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + // Code generated by controller-gen. DO NOT EDIT. package v1alpha1 @@ -645,4 +661,3 @@ func (in *RestoreJobStatus) DeepCopy() *RestoreJobStatus { in.DeepCopyInto(out) return out } - diff --git a/packages/system/backup-controller/definitions/backups.cozystack.io_backups.yaml b/packages/system/backup-controller/definitions/backups.cozystack.io_backups.yaml index 6e031a84..d48df533 100644 --- a/packages/system/backup-controller/definitions/backups.cozystack.io_backups.yaml +++ b/packages/system/backup-controller/definitions/backups.cozystack.io_backups.yaml @@ -142,14 +142,6 @@ spec: required: - uri type: object - underlyingResources: - description: |- - Holds application-specific resource metadata discovered during backup. - The payload is a self-typed JSON object carrying an inlined TypeMeta - (kind/apiVersion) so the consuming controller can dispatch on the - application kind. - type: object - x-kubernetes-preserve-unknown-fields: true conditions: description: Conditions represents the latest available observations of a Backup's state. @@ -213,6 +205,14 @@ spec: Phase is a simple, high-level summary of the backup's state. Typical values are: Pending, Ready, Failed. type: string + underlyingResources: + description: |- + UnderlyingResources holds application-specific resource metadata discovered + during backup (e.g., VM disks, network configuration). The payload is a + self-typed JSON object carrying an inlined TypeMeta (kind/apiVersion) so + the consuming controller can dispatch on the application kind. + type: object + x-kubernetes-preserve-unknown-fields: true type: object type: object selectableFields: From 2def6cda67e169a065e9b8c3a8f98fb17e702983 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Tue, 14 Apr 2026 10:46:12 +0500 Subject: [PATCH 027/250] [vm-default-images] Default to replicated storageClass and document storage requirements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The package is wired into the iaas bundle where LINSTOR replicated storage is available. Set storageClass to "replicated" by default and add a note that the full image set needs ~320Gi (16 images × 20Gi). Co-Authored-By: Claude Opus 4.6 Signed-off-by: Myasnikov Daniil --- packages/system/vm-default-images/values.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/system/vm-default-images/values.yaml b/packages/system/vm-default-images/values.yaml index a4677bc8..9d777f0b 100644 --- a/packages/system/vm-default-images/values.yaml +++ b/packages/system/vm-default-images/values.yaml @@ -3,7 +3,9 @@ ## ## @param {string} storageClass - Default StorageClass for all images. If empty, uses the cluster default StorageClass. -storageClass: "" +## NOTE: The default set of images requires approximately 320Gi of storage (16 images × 20Gi each). +## Adjust the image list or per-image storage sizes to match your cluster capacity. +storageClass: "replicated" ## @typedef {struct} ImageOS - Operating system metadata for a Golden Image. ## @field {string} [family] - OS family (e.g. "Linux", "Windows"). From 0b5a1df3c288de84808a7247aa103e0b797cc5cf Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Tue, 14 Apr 2026 20:00:25 +0500 Subject: [PATCH 028/250] [platform] Bump migration targetVersion to 39 for migration 38 Address review feedback from lexfrei on migrations/38:2: Migration 38 requires targetVersion 39 to execute, since run-migrations.sh iterates seq $CURRENT_VERSION $((TARGET_VERSION - 1)). Co-Authored-By: Claude Opus 4.6 Signed-off-by: Myasnikov Daniil --- packages/core/platform/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/platform/values.yaml b/packages/core/platform/values.yaml index 77eb00d7..c21b4f35 100644 --- a/packages/core/platform/values.yaml +++ b/packages/core/platform/values.yaml @@ -6,7 +6,7 @@ sourceRef: migrations: enabled: false image: ghcr.io/cozystack/cozystack/platform-migrations:v1.2.1@sha256:e8fcf006a4451fc0e961455e9b27a61b7103ee49b1a81fe5e4662ffed093fad6 - targetVersion: 38 + targetVersion: 39 # Bundle deployment configuration bundles: system: From cc3af6c0db7646ffb85b427b61f8ea7210e558aa Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Tue, 14 Apr 2026 23:10:10 +0500 Subject: [PATCH 029/250] [docs] Address review comments on v1.2.2 changelog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix LINSTOR verification version: v1.33.1 → v1.33.2 (upstream base) - Move postgres image fix entry from Features to Fixes, add missing #2364 ref - Fix ApplicationDefinition docs description to match actual website#478 scope Co-Authored-By: Claude Opus 4.6 Signed-off-by: Myasnikov Daniil --- docs/changelogs/v1.2.2.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/changelogs/v1.2.2.md b/docs/changelogs/v1.2.2.md index 3cf1731c..2b6a1ae5 100644 --- a/docs/changelogs/v1.2.2.md +++ b/docs/changelogs/v1.2.2.md @@ -4,12 +4,12 @@ https://github.com/cozystack/cozystack/releases/tag/v1.2.2 ## Features and Improvements -* **[linstor] Update piraeus-server to v1.33.2 with selected backports**: Bumps LINSTOR server from v1.33.1 to v1.33.2 and adds backported patches for improved storage reliability: a stale bitmap adjust retry mechanism for automatic recovery after bitmap attach errors, LUKS2 header sizing and optimal I/O size detection improvements for more reliable disk formatting, and the maintainer implementation backport. All patches verified against upstream v1.33.1 with `git apply --check` and `gradlew compileJava` ([**@kvaps**](https://github.com/kvaps) in #2331, #2377). - -* **[postgres] Fix system PostgreSQL images to 17.7-standard-trixie**: Hardcodes PostgreSQL 17.7-standard-trixie images for system PostgreSQL instances. This ensures system databases use the correct image variant consistent with the monitoring stack requirements introduced in v1.2.1 ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2369). +* **[linstor] Update piraeus-server to v1.33.2 with selected backports**: Bumps LINSTOR server from v1.33.1 to v1.33.2 and adds backported patches for improved storage reliability: a stale bitmap adjust retry mechanism for automatic recovery after bitmap attach errors, LUKS2 header sizing and optimal I/O size detection improvements for more reliable disk formatting, and the maintainer implementation backport. All patches verified against upstream v1.33.2 with `git apply --check` and `gradlew compileJava` ([**@kvaps**](https://github.com/kvaps) in #2331, #2377). ## Fixes +* **[postgres] Fix system PostgreSQL images to 17.7-standard-trixie**: Hardcodes PostgreSQL 17.7-standard-trixie images for system PostgreSQL instances. This ensures system databases use the correct image variant consistent with the monitoring stack requirements introduced in v1.2.1 ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2364, #2369). + * **[cilium] Opt-out of cri-containerd.apparmor.d for nsenter init containers**: On Ubuntu 22.04+, Debian, and other distributions that load the `cri-containerd.apparmor.d` AppArmor profile by default for containerd workloads, the kernel denied `nsenter` namespace entry in cilium-agent init containers (`mount-cgroup`, `apply-sysctl-overwrites`, `clean-cilium-state`), causing the agent to land in `Init:CrashLoopBackOff` and cascading platform failures. Per-container `container.apparmor.security.beta.kubernetes.io` annotations now opt the affected containers out of this profile, applied only on non-Talos cilium variants (`cilium-generic`, `kubeovn-cilium-generic`). The vendored daemonset template is also patched to strip the upstream `semverCompare "<1.30.0"` AppArmor block, preventing duplicate annotation keys. Talos variants are untouched as Talos does not load the AppArmor LSM ([**@lexfrei**](https://github.com/lexfrei) in #2370, #2378). * **[virtual-machine] Exclude external VM services from Cilium BPF LB**: Adds the `service.kubernetes.io/service-proxy-name: "cozy-proxy"` label to VM LoadBalancer services when `external: true`, telling Cilium to skip BPF processing entirely for these services. This fixes two issues: inter-tenant connectivity via public LB IPs (Cilium's DNAT caused cross-tenant pod-to-pod flow classification, triggering CiliumClusterwideNetworkPolicy blocks) and WholeIP broken on Cilium 1.19+ (wildcard service drop entries blocked traffic to LB IPs on undeclared ports before it reached netfilter/cozy-proxy). MetalLB L2 advertisement and kube-ovn routing remain unaffected ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #2357, #2361). @@ -28,7 +28,7 @@ https://github.com/cozystack/cozystack/releases/tag/v1.2.2 ## Documentation -* **[website] Add ApplicationDefinition naming convention reference**: Added documentation on ApplicationDefinition naming conventions and auto-restart behavior for the applicationdefinition-controller ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#478). +* **[website] Add ApplicationDefinition naming convention reference**: Added reference documentation on ApplicationDefinition naming conventions and how `cozystack-api` resolves kinds to their backing definitions ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#478). * **[website] Document Talos / talosctl / Cozystack version pairing**: Added documentation covering Talos, talosctl, and Cozystack version compatibility matrix for installation ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#484). From 1cbe69661a9b65363af17cb9245b1f219271f50a Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 15 Apr 2026 00:06:31 +0500 Subject: [PATCH 030/250] [webhook] Add deletion protection validating handler Signed-off-by: Myasnikov Daniil --- .../validate_deletion.go | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 internal/lineagecontrollerwebhook/validate_deletion.go diff --git a/internal/lineagecontrollerwebhook/validate_deletion.go b/internal/lineagecontrollerwebhook/validate_deletion.go new file mode 100644 index 00000000..076a74f7 --- /dev/null +++ b/internal/lineagecontrollerwebhook/validate_deletion.go @@ -0,0 +1,105 @@ +/* +Copyright 2025 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lineagecontrollerwebhook + +import ( + "context" + "fmt" + "net/http" + + admissionv1 "k8s.io/api/admission/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +const ( + DeletionProtectedLabel = "cozystack.io/deletion-protected" +) + +// DeletionProtectionWebhook denies DELETE requests on resources that carry +// the cozystack.io/deletion-protected label. The objectSelector on the +// ValidatingWebhookConfiguration ensures this handler is only called for +// labeled resources. +type DeletionProtectionWebhook struct{} + +func (h *DeletionProtectionWebhook) Handle(ctx context.Context, req admission.Request) admission.Response { + logger := log.FromContext(ctx).WithValues( + "kind", req.Kind.Kind, + "namespace", req.Namespace, + "name", req.Name, + "operation", req.Operation, + ) + + if req.Operation != admissionv1.Delete { + return admission.Allowed("not a DELETE operation") + } + + var identifier string + if req.Namespace != "" { + identifier = fmt.Sprintf("%s %s/%s", req.Kind.Kind, req.Namespace, req.Name) + } else { + identifier = fmt.Sprintf("%s %s", req.Kind.Kind, req.Name) + } + + msg := fmt.Sprintf( + "deletion of %s is protected by cozystack. "+ + "To force-delete, first remove the label: "+ + "kubectl label %s %s cozystack.io/deletion-protected-", + identifier, + kindToResourceArg(req.Kind.Kind, req.Namespace), + req.Name, + ) + + logger.Info("DENIED deletion of protected resource", "resource", identifier) + + return admission.Response{ + AdmissionResponse: admissionv1.AdmissionResponse{ + Allowed: false, + Result: &metav1.Status{ + Status: metav1.StatusFailure, + Message: msg, + Reason: metav1.StatusReasonForbidden, + Code: http.StatusForbidden, + }, + }, + } +} + +func kindToResourceArg(kind, namespace string) string { + switch kind { + case "Namespace": + return "namespace" + case "ConfigMap": + return "configmap -n " + namespace + case "HelmRelease": + return "helmrelease.helm.toolkit.fluxcd.io -n " + namespace + case "CustomResourceDefinition": + return "crd" + case "LinstorCluster": + return "linstorcluster.piraeus.io -n " + namespace + case "ClusterIssuer": + return "clusterissuer.cert-manager.io" + case "OCIRepository": + return "ocirepository.source.toolkit.fluxcd.io -n " + namespace + default: + if namespace != "" { + return kind + " -n " + namespace + } + return kind + } +} From b1b6cf40f7a5d06e63ac7af1d185edbc6db32a36 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 15 Apr 2026 00:08:03 +0500 Subject: [PATCH 031/250] [webhook] Add unit tests for deletion protection handler Signed-off-by: Myasnikov Daniil --- .../validate_deletion_test.go | 165 ++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 internal/lineagecontrollerwebhook/validate_deletion_test.go diff --git a/internal/lineagecontrollerwebhook/validate_deletion_test.go b/internal/lineagecontrollerwebhook/validate_deletion_test.go new file mode 100644 index 00000000..6d48a27b --- /dev/null +++ b/internal/lineagecontrollerwebhook/validate_deletion_test.go @@ -0,0 +1,165 @@ +/* +Copyright 2025 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lineagecontrollerwebhook + +import ( + "context" + "net/http" + "testing" + + admissionv1 "k8s.io/api/admission/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +func TestDeletionProtectionWebhook_Handle(t *testing.T) { + log.SetLogger(zap.New(zap.UseDevMode(true))) + ctx := context.Background() + + handler := &DeletionProtectionWebhook{} + + tests := []struct { + name string + req admission.Request + wantAllowed bool + wantCode int32 + }{ + { + name: "deny DELETE on protected namespace", + req: admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Delete, + Name: "tenant-root", + Kind: metav1.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "Namespace", + }, + }, + }, + wantAllowed: false, + wantCode: http.StatusForbidden, + }, + { + name: "deny DELETE on protected HelmRelease", + req: admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Delete, + Name: "tenant-root", + Namespace: "tenant-root", + Kind: metav1.GroupVersionKind{ + Group: "helm.toolkit.fluxcd.io", + Version: "v2", + Kind: "HelmRelease", + }, + }, + }, + wantAllowed: false, + wantCode: http.StatusForbidden, + }, + { + name: "deny DELETE on protected CRD", + req: admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Delete, + Name: "packages.cozystack.io", + Kind: metav1.GroupVersionKind{ + Group: "apiextensions.k8s.io", + Version: "v1", + Kind: "CustomResourceDefinition", + }, + }, + }, + wantAllowed: false, + wantCode: http.StatusForbidden, + }, + { + name: "deny DELETE on protected LinstorCluster", + req: admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Delete, + Name: "linstorcluster", + Namespace: "cozy-linstor", + Kind: metav1.GroupVersionKind{ + Group: "piraeus.io", + Version: "v1", + Kind: "LinstorCluster", + }, + }, + }, + wantAllowed: false, + wantCode: http.StatusForbidden, + }, + { + name: "allow non-DELETE operation", + req: admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Update, + Name: "tenant-root", + Kind: metav1.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "Namespace", + }, + }, + }, + wantAllowed: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resp := handler.Handle(ctx, tt.req) + + if resp.Allowed != tt.wantAllowed { + t.Errorf("Allowed = %v, want %v", resp.Allowed, tt.wantAllowed) + } + if !tt.wantAllowed && resp.Result != nil && resp.Result.Code != tt.wantCode { + t.Errorf("Code = %d, want %d", resp.Result.Code, tt.wantCode) + } + }) + } +} + +func TestKindToResourceArg(t *testing.T) { + tests := []struct { + kind string + namespace string + want string + }{ + {"Namespace", "", "namespace"}, + {"ConfigMap", "cozy-system", "configmap -n cozy-system"}, + {"HelmRelease", "tenant-root", "helmrelease.helm.toolkit.fluxcd.io -n tenant-root"}, + {"CustomResourceDefinition", "", "crd"}, + {"LinstorCluster", "cozy-linstor", "linstorcluster.piraeus.io -n cozy-linstor"}, + {"ClusterIssuer", "", "clusterissuer.cert-manager.io"}, + {"OCIRepository", "cozy-system", "ocirepository.source.toolkit.fluxcd.io -n cozy-system"}, + {"Unknown", "ns", "Unknown -n ns"}, + {"Unknown", "", "Unknown"}, + } + + for _, tt := range tests { + t.Run(tt.kind, func(t *testing.T) { + got := kindToResourceArg(tt.kind, tt.namespace) + if got != tt.want { + t.Errorf("kindToResourceArg(%q, %q) = %q, want %q", tt.kind, tt.namespace, got, tt.want) + } + }) + } +} From f36b7d64d598c8abd606608c270b73d9279bee87 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 15 Apr 2026 00:08:49 +0500 Subject: [PATCH 032/250] [webhook] Register deletion protection handler on /validate-deletion Signed-off-by: Myasnikov Daniil --- cmd/lineage-controller-webhook/main.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/lineage-controller-webhook/main.go b/cmd/lineage-controller-webhook/main.go index ffe0942b..5f33ee1b 100644 --- a/cmd/lineage-controller-webhook/main.go +++ b/cmd/lineage-controller-webhook/main.go @@ -160,6 +160,10 @@ func main() { os.Exit(1) } + deletionProtectionWebhook := &lcw.DeletionProtectionWebhook{} + mgr.GetWebhookServer().Register("/validate-deletion", &webhook.Admission{Handler: deletionProtectionWebhook}) + setupLog.Info("registered deletion protection webhook at /validate-deletion") + // +kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { From 0ce56ae3385a787380274a5a31906faae983abcc Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 15 Apr 2026 00:10:12 +0500 Subject: [PATCH 033/250] [webhook] Add ValidatingWebhookConfiguration for deletion protection Label-based objectSelector (cozystack.io/deletion-protected=true) ensures zero overhead for non-protected resources. Reuses the existing lineage-controller-webhook Service and TLS certificates. Signed-off-by: Myasnikov Daniil --- .../validatingwebhookconfiguration.yaml | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 packages/system/lineage-controller-webhook/templates/validatingwebhookconfiguration.yaml diff --git a/packages/system/lineage-controller-webhook/templates/validatingwebhookconfiguration.yaml b/packages/system/lineage-controller-webhook/templates/validatingwebhookconfiguration.yaml new file mode 100644 index 00000000..3895fcb9 --- /dev/null +++ b/packages/system/lineage-controller-webhook/templates/validatingwebhookconfiguration.yaml @@ -0,0 +1,53 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: deletion-protection + annotations: + cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/lineage-controller-webhook + labels: + app: lineage-controller-webhook +webhooks: + - name: deletion-protection.cozystack.io + admissionReviewVersions: ["v1"] + sideEffects: None + clientConfig: + service: + name: lineage-controller-webhook + namespace: {{ .Release.Namespace }} + path: /validate-deletion + rules: + - operations: ["DELETE"] + apiGroups: [""] + apiVersions: ["v1"] + resources: ["namespaces", "configmaps"] + scope: "*" + - operations: ["DELETE"] + apiGroups: ["helm.toolkit.fluxcd.io"] + apiVersions: ["v2"] + resources: ["helmreleases"] + scope: Namespaced + - operations: ["DELETE"] + apiGroups: ["apiextensions.k8s.io"] + apiVersions: ["v1"] + resources: ["customresourcedefinitions"] + scope: Cluster + - operations: ["DELETE"] + apiGroups: ["piraeus.io"] + apiVersions: ["v1"] + resources: ["linstorclusters"] + scope: "*" + - operations: ["DELETE"] + apiGroups: ["cert-manager.io"] + apiVersions: ["v1"] + resources: ["clusterissuers"] + scope: Cluster + - operations: ["DELETE"] + apiGroups: ["source.toolkit.fluxcd.io"] + apiVersions: ["v1"] + resources: ["ocirepositories"] + scope: "*" + failurePolicy: Fail + timeoutSeconds: 5 + objectSelector: + matchLabels: + cozystack.io/deletion-protected: "true" From 6654ca7a676548ad8015903e51323df710b409d9 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 15 Apr 2026 00:10:18 +0500 Subject: [PATCH 034/250] [platform] Add deletion protection label to tenant-root namespace and HelmRelease Signed-off-by: Myasnikov Daniil --- packages/system/cozystack-basics/templates/tenant-root.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/system/cozystack-basics/templates/tenant-root.yaml b/packages/system/cozystack-basics/templates/tenant-root.yaml index 93f5129e..1caf909b 100644 --- a/packages/system/cozystack-basics/templates/tenant-root.yaml +++ b/packages/system/cozystack-basics/templates/tenant-root.yaml @@ -3,6 +3,8 @@ apiVersion: v1 kind: Namespace metadata: name: tenant-root + labels: + cozystack.io/deletion-protected: "true" --- apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease @@ -10,6 +12,7 @@ metadata: annotations: helm.sh/resource-policy: keep labels: + cozystack.io/deletion-protected: "true" sharding.fluxcd.io/key: tenants apps.cozystack.io/application.kind: Tenant apps.cozystack.io/application.group: apps.cozystack.io From facfc90666bf30d4759440803fc39d426df72dbc Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 15 Apr 2026 00:10:23 +0500 Subject: [PATCH 035/250] [installer] Add deletion protection label to cozy-system namespace Signed-off-by: Myasnikov Daniil --- packages/core/installer/templates/cozystack-operator.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core/installer/templates/cozystack-operator.yaml b/packages/core/installer/templates/cozystack-operator.yaml index aded0995..a9877044 100644 --- a/packages/core/installer/templates/cozystack-operator.yaml +++ b/packages/core/installer/templates/cozystack-operator.yaml @@ -8,6 +8,7 @@ kind: Namespace metadata: name: cozy-system labels: + cozystack.io/deletion-protected: "true" cozystack.io/system: "true" pod-security.kubernetes.io/enforce: privileged annotations: From 14a909dba38df93a0b1d22a9e6e72f55deef9ca4 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 15 Apr 2026 00:10:30 +0500 Subject: [PATCH 036/250] [platform] Add deletion protection label to cozystack-version ConfigMap and cozystack-packages OCIRepository Signed-off-by: Myasnikov Daniil --- packages/core/platform/templates/cozystack-version.yaml | 2 ++ packages/core/platform/templates/repository.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/packages/core/platform/templates/cozystack-version.yaml b/packages/core/platform/templates/cozystack-version.yaml index 09b845c2..1f93a1cd 100644 --- a/packages/core/platform/templates/cozystack-version.yaml +++ b/packages/core/platform/templates/cozystack-version.yaml @@ -6,6 +6,8 @@ kind: ConfigMap metadata: name: cozystack-version namespace: {{ .Release.Namespace }} + labels: + cozystack.io/deletion-protected: "true" annotations: helm.sh/resource-policy: keep data: diff --git a/packages/core/platform/templates/repository.yaml b/packages/core/platform/templates/repository.yaml index 9bd9f5a1..882a0a90 100644 --- a/packages/core/platform/templates/repository.yaml +++ b/packages/core/platform/templates/repository.yaml @@ -13,5 +13,7 @@ kind: {{ $kind }} metadata: name: cozystack-packages namespace: cozy-system + labels: + cozystack.io/deletion-protected: "true" spec: {{- $sourceRepo.spec | toYaml | nindent 2 }} From 45764355580fd49295999c4f23cbc01d9a152a5a Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 15 Apr 2026 00:10:35 +0500 Subject: [PATCH 037/250] [linstor] Add deletion protection label to LinstorCluster Signed-off-by: Myasnikov Daniil --- packages/system/linstor/templates/cluster.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/system/linstor/templates/cluster.yaml b/packages/system/linstor/templates/cluster.yaml index e45b1dff..1bfced6c 100644 --- a/packages/system/linstor/templates/cluster.yaml +++ b/packages/system/linstor/templates/cluster.yaml @@ -2,6 +2,8 @@ apiVersion: piraeus.io/v1 kind: LinstorCluster metadata: name: linstorcluster + labels: + cozystack.io/deletion-protected: "true" spec: #nodeSelector: # node-role.kubernetes.io/linstor: "" From 5153c1599ba7fff37bec831aaddc3028ff96ea41 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 15 Apr 2026 00:10:40 +0500 Subject: [PATCH 038/250] [cert-manager] Add deletion protection label to ClusterIssuers Signed-off-by: Myasnikov Daniil --- .../templates/cluster-issuers.yaml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/packages/system/cert-manager-issuers/templates/cluster-issuers.yaml b/packages/system/cert-manager-issuers/templates/cluster-issuers.yaml index 442f1fb3..864e90ea 100644 --- a/packages/system/cert-manager-issuers/templates/cluster-issuers.yaml +++ b/packages/system/cert-manager-issuers/templates/cluster-issuers.yaml @@ -2,8 +2,10 @@ apiVersion: cert-manager.io/v1 kind: ClusterIssuer -metadata: - name: letsencrypt-prod +metadata: + name: letsencrypt-prod + labels: + cozystack.io/deletion-protected: "true" spec: acme: privateKeySecretRef: @@ -26,8 +28,10 @@ spec: apiVersion: cert-manager.io/v1 kind: ClusterIssuer -metadata: - name: letsencrypt-stage +metadata: + name: letsencrypt-stage + labels: + cozystack.io/deletion-protected: "true" spec: acme: privateKeySecretRef: @@ -50,7 +54,9 @@ spec: apiVersion: cert-manager.io/v1 kind: ClusterIssuer -metadata: - name: selfsigned-cluster-issuer +metadata: + name: selfsigned-cluster-issuer + labels: + cozystack.io/deletion-protected: "true" spec: selfSigned: {} From 4a4354a7e18a5898463aa4af1bfd0b8280a21333 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 15 Apr 2026 00:10:46 +0500 Subject: [PATCH 039/250] [operator] Add deletion protection label to CRDs during install Signed-off-by: Myasnikov Daniil --- internal/crdinstall/install.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/internal/crdinstall/install.go b/internal/crdinstall/install.go index 5143f2e6..516a63fa 100644 --- a/internal/crdinstall/install.go +++ b/internal/crdinstall/install.go @@ -89,6 +89,16 @@ func Install(ctx context.Context, k8sClient client.Client, writeEmbeddedManifest } } + // Add deletion protection label to all CRDs + for _, obj := range objects { + labels := obj.GetLabels() + if labels == nil { + labels = make(map[string]string) + } + labels["cozystack.io/deletion-protected"] = "true" + obj.SetLabels(labels) + } + logger.Info("Applying Cozystack CRDs", "count", len(objects)) for _, obj := range objects { patchOptions := &client.PatchOptions{ From 2157452da3c5c2f64a49fb03249ab23a83b9106a Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 15 Apr 2026 00:18:37 +0500 Subject: [PATCH 040/250] [webhook] Fix review issues: LinstorCluster scope, constant export, whitespace - LinstorCluster is cluster-scoped: fix webhook rule scope to Cluster, remove namespace from kindToResourceArg and test - OCIRepository webhook rule: scope "*" -> Namespaced - Unexport DeletionProtectedLabel constant (only used within package) - Normalize trailing whitespace in cert-manager-issuers template Signed-off-by: Myasnikov Daniil --- .../lineagecontrollerwebhook/validate_deletion.go | 7 ++++--- .../validate_deletion_test.go | 3 +-- .../templates/cluster-issuers.yaml | 12 ++++++------ .../templates/validatingwebhookconfiguration.yaml | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/internal/lineagecontrollerwebhook/validate_deletion.go b/internal/lineagecontrollerwebhook/validate_deletion.go index 076a74f7..5ebe4d29 100644 --- a/internal/lineagecontrollerwebhook/validate_deletion.go +++ b/internal/lineagecontrollerwebhook/validate_deletion.go @@ -28,7 +28,7 @@ import ( ) const ( - DeletionProtectedLabel = "cozystack.io/deletion-protected" + deletionProtectedLabel = "cozystack.io/deletion-protected" ) // DeletionProtectionWebhook denies DELETE requests on resources that carry @@ -59,10 +59,11 @@ func (h *DeletionProtectionWebhook) Handle(ctx context.Context, req admission.Re msg := fmt.Sprintf( "deletion of %s is protected by cozystack. "+ "To force-delete, first remove the label: "+ - "kubectl label %s %s cozystack.io/deletion-protected-", + "kubectl label %s %s %s-", identifier, kindToResourceArg(req.Kind.Kind, req.Namespace), req.Name, + deletionProtectedLabel, ) logger.Info("DENIED deletion of protected resource", "resource", identifier) @@ -91,7 +92,7 @@ func kindToResourceArg(kind, namespace string) string { case "CustomResourceDefinition": return "crd" case "LinstorCluster": - return "linstorcluster.piraeus.io -n " + namespace + return "linstorcluster.piraeus.io" case "ClusterIssuer": return "clusterissuer.cert-manager.io" case "OCIRepository": diff --git a/internal/lineagecontrollerwebhook/validate_deletion_test.go b/internal/lineagecontrollerwebhook/validate_deletion_test.go index 6d48a27b..ee4b8ed1 100644 --- a/internal/lineagecontrollerwebhook/validate_deletion_test.go +++ b/internal/lineagecontrollerwebhook/validate_deletion_test.go @@ -95,7 +95,6 @@ func TestDeletionProtectionWebhook_Handle(t *testing.T) { AdmissionRequest: admissionv1.AdmissionRequest{ Operation: admissionv1.Delete, Name: "linstorcluster", - Namespace: "cozy-linstor", Kind: metav1.GroupVersionKind{ Group: "piraeus.io", Version: "v1", @@ -147,7 +146,7 @@ func TestKindToResourceArg(t *testing.T) { {"ConfigMap", "cozy-system", "configmap -n cozy-system"}, {"HelmRelease", "tenant-root", "helmrelease.helm.toolkit.fluxcd.io -n tenant-root"}, {"CustomResourceDefinition", "", "crd"}, - {"LinstorCluster", "cozy-linstor", "linstorcluster.piraeus.io -n cozy-linstor"}, + {"LinstorCluster", "", "linstorcluster.piraeus.io"}, {"ClusterIssuer", "", "clusterissuer.cert-manager.io"}, {"OCIRepository", "cozy-system", "ocirepository.source.toolkit.fluxcd.io -n cozy-system"}, {"Unknown", "ns", "Unknown -n ns"}, diff --git a/packages/system/cert-manager-issuers/templates/cluster-issuers.yaml b/packages/system/cert-manager-issuers/templates/cluster-issuers.yaml index 864e90ea..f3457b8e 100644 --- a/packages/system/cert-manager-issuers/templates/cluster-issuers.yaml +++ b/packages/system/cert-manager-issuers/templates/cluster-issuers.yaml @@ -1,7 +1,7 @@ {{- $solver := (index .Values._cluster "solver") | default "http01" }} -apiVersion: cert-manager.io/v1 -kind: ClusterIssuer +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer metadata: name: letsencrypt-prod labels: @@ -26,8 +26,8 @@ spec: --- -apiVersion: cert-manager.io/v1 -kind: ClusterIssuer +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer metadata: name: letsencrypt-stage labels: @@ -52,8 +52,8 @@ spec: --- -apiVersion: cert-manager.io/v1 -kind: ClusterIssuer +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer metadata: name: selfsigned-cluster-issuer labels: diff --git a/packages/system/lineage-controller-webhook/templates/validatingwebhookconfiguration.yaml b/packages/system/lineage-controller-webhook/templates/validatingwebhookconfiguration.yaml index 3895fcb9..24aa264e 100644 --- a/packages/system/lineage-controller-webhook/templates/validatingwebhookconfiguration.yaml +++ b/packages/system/lineage-controller-webhook/templates/validatingwebhookconfiguration.yaml @@ -35,7 +35,7 @@ webhooks: apiGroups: ["piraeus.io"] apiVersions: ["v1"] resources: ["linstorclusters"] - scope: "*" + scope: Cluster - operations: ["DELETE"] apiGroups: ["cert-manager.io"] apiVersions: ["v1"] @@ -45,7 +45,7 @@ webhooks: apiGroups: ["source.toolkit.fluxcd.io"] apiVersions: ["v1"] resources: ["ocirepositories"] - scope: "*" + scope: Namespaced failurePolicy: Fail timeoutSeconds: 5 objectSelector: From 1bae0775be1992769ff8e90a32a15594d69b4b01 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 9 Apr 2026 20:25:20 +0300 Subject: [PATCH 041/250] feat(application): add WorkloadsReady condition and Events tab Add two features to improve application observability in the dashboard: 1. WorkloadsReady condition on Application status - Query WorkloadMonitor resources to determine if all pods are running - Expose WorkloadsReady as a separate condition alongside Ready - Ready continues to reflect HelmRelease state only (no override) to preserve backward compatibility with tooling and avoid false-negatives during normal startup windows when pods are still coming up - Handle three states: operational, not operational, unknown (pending) - Fail-open on WorkloadMonitor query errors - Integrate with Application Watch to emit MODIFIED events on WorkloadMonitor changes (with initial-events-end safety) - Register cozystack.io/v1alpha1 types in API server scheme 2. Events tab in dashboard - Show Kubernetes Events scoped to application namespace - Use status.namespace for Tenant applications - Include both lastTimestamp and eventTime columns for K8s compat 3. Bug fix: WorkloadMonitor Operational status persistence - Operational was written to stale 'monitor' instead of 'fresh' inside RetryOnConflict, so it was never persisted to the cluster Closes #2359 Closes #2360 Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- .gitignore | 1 + internal/controller/dashboard/factory.go | 32 ++ internal/controller/dashboard/factory_test.go | 60 +++ .../controller/dashboard/static_refactored.go | 11 + .../controller/workloadmonitor_controller.go | 4 +- .../workloadmonitor_controller_test.go | 180 +++++++++ pkg/apiserver/apiserver.go | 7 + pkg/registry/apps/application/rest.go | 186 ++++++++++ .../apps/application/rest_conditions_test.go | 331 +++++++++++++++++ .../apps/application/rest_watch_test.go | 223 ++++++++++++ .../apps/application/rest_workloads_test.go | 342 ++++++++++++++++++ 11 files changed, 1375 insertions(+), 2 deletions(-) create mode 100644 internal/controller/dashboard/factory_test.go create mode 100644 internal/controller/workloadmonitor_controller_test.go create mode 100644 pkg/registry/apps/application/rest_conditions_test.go create mode 100644 pkg/registry/apps/application/rest_watch_test.go create mode 100644 pkg/registry/apps/application/rest_workloads_test.go diff --git a/.gitignore b/.gitignore index 4e1e3119..61003de5 100644 --- a/.gitignore +++ b/.gitignore @@ -83,3 +83,4 @@ tmp/ # build revision marker (generated by make image-packages) packages/core/platform/.build-revision +.claude/ diff --git a/internal/controller/dashboard/factory.go b/internal/controller/dashboard/factory.go index 539e503b..fe0cadfd 100644 --- a/internal/controller/dashboard/factory.go +++ b/internal/controller/dashboard/factory.go @@ -47,6 +47,7 @@ func (m *Manager) ensureFactory(ctx context.Context, crd *cozyv1alpha1.Applicati if prefix, ok := vncTabPrefix(kind); ok { tabs = append(tabs, vncTab(prefix)) } + tabs = append(tabs, eventsTab(kind)) tabs = append(tabs, yamlTab(g, v, plural)) // Use unified factory creation @@ -358,6 +359,37 @@ func secretsTab(kind string) map[string]any { } } +// eventsTab shows Kubernetes Events scoped to the application's namespace. +// Events are namespace-scoped because Kubernetes Events don't carry application +// labels and cannot be filtered by label selector. In Cozystack's multi-tenancy +// model, each tenant namespace maps to a single application scope, so namespace +// filtering provides the correct event scope. +// For Tenant applications, events are fetched from status.namespace (the tenant's +// own namespace) instead of the parent namespace where the Tenant object lives. +func eventsTab(kind string) map[string]any { + nsPlaceholder := "{3}" + if kind == "Tenant" { + nsPlaceholder = "{reqsJsonPath[0]['.status.namespace']}" + } + return map[string]any{ + "key": "events", + "label": "Events", + "children": []any{ + map[string]any{ + "type": "EnrichedTable", + "data": map[string]any{ + "id": "events-table", + "fetchUrl": "/api/clusters/{2}/k8s/api/v1/namespaces/" + nsPlaceholder + "/events", + "cluster": "{2}", + "baseprefix": "/openapi-ui", + "customizationId": "factory-details-events", + "pathToItems": []any{"items"}, + }, + }, + }, + } +} + func yamlTab(group, version, plural string) map[string]any { return map[string]any{ "key": "yaml", diff --git a/internal/controller/dashboard/factory_test.go b/internal/controller/dashboard/factory_test.go new file mode 100644 index 00000000..b3742564 --- /dev/null +++ b/internal/controller/dashboard/factory_test.go @@ -0,0 +1,60 @@ +package dashboard + +import ( + "testing" +) + +func TestEventsTab_Structure(t *testing.T) { + tab := eventsTab("PostgreSQL") + + if tab["key"] != "events" { + t.Errorf("expected key=events, got %v", tab["key"]) + } + if tab["label"] != "Events" { + t.Errorf("expected label=Events, got %v", tab["label"]) + } + + children, ok := tab["children"].([]any) + if !ok || len(children) != 1 { + t.Fatal("expected exactly 1 child in events tab") + } + + table, ok := children[0].(map[string]any) + if !ok { + t.Fatal("child is not a map") + } + if table["type"] != "EnrichedTable" { + t.Errorf("expected type=EnrichedTable, got %v", table["type"]) + } + + data, ok := table["data"].(map[string]any) + if !ok { + t.Fatal("table data is not a map") + } + if data["id"] != "events-table" { + t.Errorf("expected id=events-table, got %v", data["id"]) + } + if data["fetchUrl"] != "/api/clusters/{2}/k8s/api/v1/namespaces/{3}/events" { + t.Errorf("unexpected fetchUrl for non-Tenant: %v", data["fetchUrl"]) + } + if data["customizationId"] != "factory-details-events" { + t.Errorf("expected customizationId=factory-details-events, got %v", data["customizationId"]) + } + + pathToItems, ok := data["pathToItems"].([]any) + if !ok || len(pathToItems) != 1 || pathToItems[0] != "items" { + t.Errorf("expected pathToItems=[items], got %v", data["pathToItems"]) + } +} + +func TestEventsTab_TenantUsesStatusNamespace(t *testing.T) { + tab := eventsTab("Tenant") + children := tab["children"].([]any) + table := children[0].(map[string]any) + data := table["data"].(map[string]any) + + expectedURL := "/api/clusters/{2}/k8s/api/v1/namespaces/{reqsJsonPath[0]['.status.namespace']}/events" + if data["fetchUrl"] != expectedURL { + t.Errorf("expected Tenant fetchUrl to use status.namespace, got %v", data["fetchUrl"]) + } +} diff --git a/internal/controller/dashboard/static_refactored.go b/internal/controller/dashboard/static_refactored.go index 06c4239f..a73026bc 100644 --- a/internal/controller/dashboard/static_refactored.go +++ b/internal/controller/dashboard/static_refactored.go @@ -224,6 +224,17 @@ func CreateAllCustomColumnsOverrides() []*dashboardv1alpha1.CustomColumnsOverrid createStringColumn("Name", ".name"), }), + // Factory details events + createCustomColumnsOverride("factory-details-events", []any{ + createTimestampColumn("Last Seen", ".lastTimestamp"), + createTimestampColumn("Event Time", ".eventTime"), + createStringColumn("Type", ".type"), + createStringColumn("Reason", ".reason"), + createStringColumn("Object", ".involvedObject.kind"), + createStringColumn("Name", ".involvedObject.name"), + createStringColumn("Message", ".message"), + }), + // Factory status conditions createCustomColumnsOverride("factory-status-conditions", []any{ createStringColumn("Type", ".type"), diff --git a/internal/controller/workloadmonitor_controller.go b/internal/controller/workloadmonitor_controller.go index a684df9b..0cd3103e 100644 --- a/internal/controller/workloadmonitor_controller.go +++ b/internal/controller/workloadmonitor_controller.go @@ -389,9 +389,9 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ fresh.Status.AvailableReplicas = availableReplicas // Default to operational = true, but check MinReplicas if set - monitor.Status.Operational = pointer.Bool(true) + fresh.Status.Operational = pointer.Bool(true) if monitor.Spec.MinReplicas != nil && availableReplicas < *monitor.Spec.MinReplicas { - monitor.Status.Operational = pointer.Bool(false) + fresh.Status.Operational = pointer.Bool(false) } return r.Status().Update(ctx, fresh) }) diff --git a/internal/controller/workloadmonitor_controller_test.go b/internal/controller/workloadmonitor_controller_test.go new file mode 100644 index 00000000..e1ed0c0d --- /dev/null +++ b/internal/controller/workloadmonitor_controller_test.go @@ -0,0 +1,180 @@ +package controller + +import ( + "context" + "testing" + + cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +func TestReconcile_OperationalStatusPersisted(t *testing.T) { + scheme := runtime.NewScheme() + _ = cozyv1alpha1.AddToScheme(scheme) + _ = corev1.AddToScheme(scheme) + + minReplicas := int32(2) + monitor := &cozyv1alpha1.WorkloadMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-monitor", + Namespace: "default", + }, + Spec: cozyv1alpha1.WorkloadMonitorSpec{ + Selector: map[string]string{"app": "test"}, + MinReplicas: &minReplicas, + }, + } + + // Create one pod that is ready — availableReplicas=1 < minReplicas=2, so Operational should be false + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod-1", + Namespace: "default", + Labels: map[string]string{"app": "test"}, + }, + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + {Type: corev1.PodReady, Status: corev1.ConditionTrue}, + }, + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(monitor, pod). + WithStatusSubresource(monitor). + Build() + + reconciler := &WorkloadMonitorReconciler{Client: fakeClient, Scheme: scheme} + req := reconcile.Request{NamespacedName: types.NamespacedName{Name: "test-monitor", Namespace: "default"}} + + _, err := reconciler.Reconcile(context.TODO(), req) + if err != nil { + t.Fatalf("Reconcile returned error: %v", err) + } + + // Fetch the monitor back from fake client and check Operational is persisted + updated := &cozyv1alpha1.WorkloadMonitor{} + if err := fakeClient.Get(context.TODO(), req.NamespacedName, updated); err != nil { + t.Fatalf("Failed to get updated WorkloadMonitor: %v", err) + } + + if updated.Status.Operational == nil { + t.Fatal("Expected Operational to be set, got nil") + } + if *updated.Status.Operational { + t.Error("Expected Operational=false (1 available < 2 minReplicas), got true") + } + if updated.Status.ObservedReplicas != 1 { + t.Errorf("Expected ObservedReplicas=1, got %d", updated.Status.ObservedReplicas) + } + if updated.Status.AvailableReplicas != 1 { + t.Errorf("Expected AvailableReplicas=1, got %d", updated.Status.AvailableReplicas) + } +} + +func TestReconcile_OperationalTrue_WhenEnoughReplicas(t *testing.T) { + scheme := runtime.NewScheme() + _ = cozyv1alpha1.AddToScheme(scheme) + _ = corev1.AddToScheme(scheme) + + minReplicas := int32(1) + monitor := &cozyv1alpha1.WorkloadMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-monitor", + Namespace: "default", + }, + Spec: cozyv1alpha1.WorkloadMonitorSpec{ + Selector: map[string]string{"app": "test"}, + MinReplicas: &minReplicas, + }, + } + + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod-1", + Namespace: "default", + Labels: map[string]string{"app": "test"}, + }, + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + {Type: corev1.PodReady, Status: corev1.ConditionTrue}, + }, + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(monitor, pod). + WithStatusSubresource(monitor). + Build() + + reconciler := &WorkloadMonitorReconciler{Client: fakeClient, Scheme: scheme} + req := reconcile.Request{NamespacedName: types.NamespacedName{Name: "test-monitor", Namespace: "default"}} + + _, err := reconciler.Reconcile(context.TODO(), req) + if err != nil { + t.Fatalf("Reconcile returned error: %v", err) + } + + updated := &cozyv1alpha1.WorkloadMonitor{} + if err := fakeClient.Get(context.TODO(), req.NamespacedName, updated); err != nil { + t.Fatalf("Failed to get updated WorkloadMonitor: %v", err) + } + + if updated.Status.Operational == nil { + t.Fatal("Expected Operational to be set, got nil") + } + if !*updated.Status.Operational { + t.Error("Expected Operational=true (1 available >= 1 minReplicas), got false") + } +} + +func TestReconcile_OperationalTrue_WhenNoMinReplicas(t *testing.T) { + scheme := runtime.NewScheme() + _ = cozyv1alpha1.AddToScheme(scheme) + _ = corev1.AddToScheme(scheme) + + monitor := &cozyv1alpha1.WorkloadMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-monitor", + Namespace: "default", + }, + Spec: cozyv1alpha1.WorkloadMonitorSpec{ + Selector: map[string]string{"app": "test"}, + // No MinReplicas — should default to operational=true + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(monitor). + WithStatusSubresource(monitor). + Build() + + reconciler := &WorkloadMonitorReconciler{Client: fakeClient, Scheme: scheme} + req := reconcile.Request{NamespacedName: types.NamespacedName{Name: "test-monitor", Namespace: "default"}} + + _, err := reconciler.Reconcile(context.TODO(), req) + if err != nil { + t.Fatalf("Reconcile returned error: %v", err) + } + + updated := &cozyv1alpha1.WorkloadMonitor{} + if err := fakeClient.Get(context.TODO(), req.NamespacedName, updated); err != nil { + t.Fatalf("Failed to get updated WorkloadMonitor: %v", err) + } + + if updated.Status.Operational == nil { + t.Fatal("Expected Operational to be set, got nil") + } + if !*updated.Status.Operational { + t.Error("Expected Operational=true (no MinReplicas constraint), got false") + } +} + diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index bf02e642..005fc640 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -21,6 +21,7 @@ import ( "fmt" "time" + cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" helmv2 "github.com/fluxcd/helm-controller/api/v2" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -79,6 +80,11 @@ func init() { if err := rbacv1.AddToScheme(mgrScheme); err != nil { panic(fmt.Errorf("Failed to add RBAC types to scheme: %w", err)) } + + // Register Cozystack types for WorkloadMonitor queries. + if err := cozyv1alpha1.AddToScheme(mgrScheme); err != nil { + panic(fmt.Errorf("failed to add Cozystack types to scheme: %w", err)) + } // Add unversioned types. metav1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) @@ -168,6 +174,7 @@ func (c completedConfig) New() (*CozyServer, error) { &corev1.Namespace{}, &corev1.Service{}, &rbacv1.RoleBinding{}, + &cozyv1alpha1.WorkloadMonitor{}, ); err != nil { return nil, fmt.Errorf("failed to get informers: %w", err) } diff --git a/pkg/registry/apps/application/rest.go b/pkg/registry/apps/application/rest.go index 470da560..583a4a8c 100644 --- a/pkg/registry/apps/application/rest.go +++ b/pkg/registry/apps/application/rest.go @@ -26,6 +26,7 @@ import ( "sync" "time" + cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" helmv2 "github.com/fluxcd/helm-controller/api/v2" corev1 "k8s.io/api/core/v1" metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion" @@ -703,9 +704,28 @@ func (r *REST) Watch(ctx context.Context, options *metainternalversion.ListOptio } customW.underlying = helmWatcher + // Start watch on WorkloadMonitor to detect pod readiness changes + wmLabelSelector := labels.NewSelector().Add(*appKindReq, *appGroupReq) + wmList := &cozyv1alpha1.WorkloadMonitorList{} + wmWatcher, err := r.w.Watch(ctx, wmList, &client.ListOptions{ + Namespace: namespace, + LabelSelector: wmLabelSelector, + }) + if err != nil { + klog.Warningf("Failed to set up WorkloadMonitor watch, workload status changes won't trigger events: %v", err) + // Non-fatal: proceed without WorkloadMonitor watch + wmWatcher = nil + } + go func() { + // Capture wmWatcher for cleanup; the variable may be set to nil + // inside the loop when the channel closes, so defer must use this copy. + wmWatcherForCleanup := wmWatcher defer close(customW.resultChan) defer customW.underlying.Stop() + if wmWatcherForCleanup != nil { + defer wmWatcherForCleanup.Stop() + } // Track whether we've sent the initial-events-end bookmark initialEventsEndSent := !sendInitialEvents // If not sendInitialEvents, consider it already sent @@ -879,6 +899,77 @@ func (r *REST) Watch(ctx context.Context, options *metainternalversion.ListOptio return } + case wmEvent, ok := <-wmResultChan(wmWatcher): + if !ok { + klog.V(4).Info("WorkloadMonitor watcher closed") + wmWatcher = nil + continue + } + if wmEvent.Type == watch.Bookmark || wmEvent.Type == watch.Error { + if wmEvent.Type == watch.Error { + klog.V(4).Infof("WorkloadMonitor watch error event: %v", wmEvent.Object) + } + continue + } + // Don't emit WM-triggered events until the initial snapshot is + // complete — the watch-list contract requires all ADDED events + // followed by the initial-events-end bookmark before any live updates. + if !initialEventsEndSent { + continue + } + wm, ok := wmEvent.Object.(*cozyv1alpha1.WorkloadMonitor) + if !ok { + continue + } + // All WM event types (Added/Modified/Deleted) produce a Modified + // Application event because the Application itself is what changed + // from the client's perspective. + wmAppName, hasLabel := wm.Labels[ApplicationNameLabel] + if !hasLabel { + continue + } + // Filter: skip WorkloadMonitor events for applications not matching + // the watch scope (single-resource or field-selector filtered watches) + hrName := r.releaseConfig.Prefix + wmAppName + if filterByName != "" && hrName != filterByName { + continue + } + if resourceName != "" && wmAppName != resourceName { + continue + } + hr := &helmv2.HelmRelease{} + if err := r.c.Get(ctx, client.ObjectKey{Namespace: wm.Namespace, Name: hrName}, hr); err != nil { + klog.V(4).Infof("Cannot find HelmRelease %s/%s for WorkloadMonitor event: %v", wm.Namespace, hrName, err) + continue + } + app, err := r.ConvertHelmReleaseToApplication(ctx, hr) + if err != nil { + klog.V(4).Infof("Error converting HelmRelease for WorkloadMonitor event: %v", err) + continue + } + // Apply label selector filtering (same as HelmRelease event path) + if options.LabelSelector != nil { + sel, err := labels.Parse(options.LabelSelector.String()) + if err != nil { + klog.Errorf("Invalid label selector: %v", err) + continue + } + if !sel.Matches(labels.Set(app.Labels)) { + continue + } + } + // Use the WorkloadMonitor's ResourceVersion for the emitted event + // so clients see a monotonically increasing RV and don't skip this update. + app.SetResourceVersion(wm.GetResourceVersion()) + lastResourceVersion = wm.GetResourceVersion() + select { + case customW.resultChan <- watch.Event{Type: watch.Modified, Object: &app}: + case <-customW.stopChan: + return + case <-ctx.Done(): + return + } + case <-customW.stopChan: return case <-ctx.Done(): @@ -891,6 +982,15 @@ func (r *REST) Watch(ctx context.Context, options *metainternalversion.ListOptio return customW, nil } +// wmResultChan returns the result channel of a WorkloadMonitor watcher, or a nil +// channel (which blocks forever in select) if the watcher is nil. +func wmResultChan(w watch.Interface) <-chan watch.Event { + if w == nil { + return nil + } + return w.ResultChan() +} + // customWatcher wraps the original watcher and filters/converts events type customWatcher struct { resultChan chan watch.Event @@ -1150,6 +1250,56 @@ func (r *REST) convertHelmReleaseToApplication(ctx context.Context, hr *helmv2.H }) } } + // Enrich conditions with WorkloadMonitor operational status + ws, wsErr := r.getWorkloadsOperational(ctx, hr.Namespace, app.Name) + if wsErr != nil { + // Fail-open: if we can't query WorkloadMonitors (e.g., informer cache not ready), + // don't override Ready. Prefer operational availability over safety. + // The WorkloadsReady=Unknown condition still signals the issue to the user. + klog.Warningf("Failed to check workload monitors for %s/%s: %v", hr.Namespace, app.Name, wsErr) + conditions = append(conditions, metav1.Condition{ + Type: "WorkloadsReady", + Status: metav1.ConditionUnknown, + LastTransitionTime: metav1.Now(), + Reason: "Error", + Message: fmt.Sprintf("Failed to check workload status: %v", wsErr), + }) + } else if ws.found { + // LastTransitionTime is set to the current time because the Application + // resource is virtual (computed on-the-fly from HelmRelease). There is no + // persistent condition state to track actual transitions. This is consistent + // with how computed/virtual API resources work in Kubernetes. + workloadsCondition := metav1.Condition{ + Type: "WorkloadsReady", + LastTransitionTime: metav1.Now(), + Reason: "WorkloadMonitorCheck", + } + switch { + case !ws.operational: + // Concrete failure takes priority over unknown/pending state + workloadsCondition.Status = metav1.ConditionFalse + workloadsCondition.Message = "One or more workloads are not operational" + case ws.unknown: + workloadsCondition.Status = metav1.ConditionUnknown + workloadsCondition.Reason = "Pending" + workloadsCondition.Message = "One or more workloads have not been reconciled yet" + default: + workloadsCondition.Status = metav1.ConditionTrue + workloadsCondition.Message = "All workloads are operational" + } + conditions = append(conditions, workloadsCondition) + + // Intentionally do NOT override the Ready condition based on WorkloadsReady. + // Ready continues to reflect HelmRelease state only, which: + // - preserves backward compatibility with existing tooling (kubectl wait, + // GitOps health checks) that expect Ready to match HelmRelease + // - avoids false-negative Ready=False during normal startup windows where + // pods are still coming up but WorkloadMonitor has already reported + // Operational=false due to availableReplicas < MinReplicas + // WorkloadsReady is a separate condition that surfaces workload health + // independently — users and dashboards can observe it for operational visibility. + } + app.SetConditions(conditions) // Add namespace field for Tenant applications @@ -1166,6 +1316,42 @@ func (r *REST) convertHelmReleaseToApplication(ctx context.Context, hr *helmv2.H return app, nil } +// workloadsStatus holds the aggregated operational status of WorkloadMonitors. +type workloadsStatus struct { + operational bool + found bool + unknown bool // true when at least one monitor has nil Operational (not yet reconciled) +} + +// getWorkloadsOperational checks WorkloadMonitor resources for an application and returns +// aggregated operational status. If no monitors exist, returns found=false. +func (r *REST) getWorkloadsOperational(ctx context.Context, namespace, appName string) (workloadsStatus, error) { + monitors := &cozyv1alpha1.WorkloadMonitorList{} + if err := r.c.List(ctx, monitors, + client.InNamespace(namespace), + client.MatchingLabels{ + appsv1alpha1.ApplicationKindLabel: r.kindName, + appsv1alpha1.ApplicationGroupLabel: r.gvk.Group, + appsv1alpha1.ApplicationNameLabel: appName, + }, + ); err != nil { + return workloadsStatus{}, err + } + if len(monitors.Items) == 0 { + return workloadsStatus{operational: true, found: false}, nil + } + operational := true + unknown := false + for _, m := range monitors.Items { + if m.Status.Operational == nil { + unknown = true + } else if !*m.Status.Operational { + operational = false + } + } + return workloadsStatus{operational: operational, found: true, unknown: unknown}, nil +} + // convertApplicationToHelmRelease implements the actual conversion logic func (r *REST) convertApplicationToHelmRelease(app *appsv1alpha1.Application) (*helmv2.HelmRelease, error) { helmRelease := &helmv2.HelmRelease{ diff --git a/pkg/registry/apps/application/rest_conditions_test.go b/pkg/registry/apps/application/rest_conditions_test.go new file mode 100644 index 00000000..dbb99d52 --- /dev/null +++ b/pkg/registry/apps/application/rest_conditions_test.go @@ -0,0 +1,331 @@ +package application + +import ( + "context" + "testing" + + cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" + helmv2 "github.com/fluxcd/helm-controller/api/v2" + appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" + "github.com/cozystack/cozystack/pkg/config" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func findCondition(conditions []metav1.Condition, condType string) *metav1.Condition { + for i := range conditions { + if conditions[i].Type == condType { + return &conditions[i] + } + } + return nil +} + +func makeHelmRelease(name, namespace string) *helmv2.HelmRelease { + hr := &helmv2.HelmRelease{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: map[string]string{ + ApplicationKindLabel: "PostgreSQL", + ApplicationGroupLabel: "apps.cozystack.io", + ApplicationNameLabel: "mydb", + }, + }, + } + return hr +} + +func TestConvertConditions_WorkloadsReadyAdded(t *testing.T) { + monitor := &cozyv1alpha1.WorkloadMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mon-1", + Namespace: "default", + Labels: map[string]string{ + appsv1alpha1.ApplicationKindLabel: "PostgreSQL", + appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", + appsv1alpha1.ApplicationNameLabel: "mydb", + }, + }, + Status: cozyv1alpha1.WorkloadMonitorStatus{ + Operational: ptr.To(true), + }, + } + + r := newTestRESTWithSchemes(monitor) + hr := makeHelmRelease("postgresql-mydb", "default") + hr.Status.Conditions = []metav1.Condition{ + {Type: "Ready", Status: metav1.ConditionTrue, Reason: "Succeeded", Message: "Release applied"}, + {Type: "Released", Status: metav1.ConditionTrue, Reason: "Succeeded", Message: "Released"}, + } + + app, err := r.convertHelmReleaseToApplication(context.TODO(), hr) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + wc := findCondition(app.GetConditions(), "WorkloadsReady") + if wc == nil { + t.Fatal("expected WorkloadsReady condition to be present") + } + if wc.Status != metav1.ConditionTrue { + t.Errorf("expected WorkloadsReady=True, got %s", wc.Status) + } + + rc := findCondition(app.GetConditions(), "Ready") + if rc == nil { + t.Fatal("expected Ready condition to be present") + } + if rc.Status != metav1.ConditionTrue { + t.Errorf("expected Ready=True, got %s", rc.Status) + } +} + +func TestConvertConditions_ReadyNotOverriddenWhenWorkloadsNotReady(t *testing.T) { + // Ready must reflect HelmRelease state only. WorkloadsReady is a separate + // signal that users/dashboards can observe independently. + monitor := &cozyv1alpha1.WorkloadMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mon-1", + Namespace: "default", + Labels: map[string]string{ + appsv1alpha1.ApplicationKindLabel: "PostgreSQL", + appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", + appsv1alpha1.ApplicationNameLabel: "mydb", + }, + }, + Status: cozyv1alpha1.WorkloadMonitorStatus{ + Operational: ptr.To(false), + }, + } + + r := newTestRESTWithSchemes(monitor) + hr := makeHelmRelease("postgresql-mydb", "default") + hr.Status.Conditions = []metav1.Condition{ + {Type: "Ready", Status: metav1.ConditionTrue, Reason: "Succeeded", Message: "Release applied"}, + } + + app, err := r.convertHelmReleaseToApplication(context.TODO(), hr) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + wc := findCondition(app.GetConditions(), "WorkloadsReady") + if wc == nil { + t.Fatal("expected WorkloadsReady condition to be present") + } + if wc.Status != metav1.ConditionFalse { + t.Errorf("expected WorkloadsReady=False, got %s", wc.Status) + } + + rc := findCondition(app.GetConditions(), "Ready") + if rc == nil { + t.Fatal("expected Ready condition to be present") + } + if rc.Status != metav1.ConditionTrue { + t.Errorf("expected Ready=True (reflects HelmRelease only), got %s", rc.Status) + } + if rc.Reason != "Succeeded" { + t.Errorf("expected Ready.Reason=Succeeded (unchanged), got %s", rc.Reason) + } +} + +func TestConvertConditions_NoOverrideWhenNoMonitors(t *testing.T) { + r := newTestRESTWithSchemes() + hr := makeHelmRelease("postgresql-mydb", "default") + hr.Status.Conditions = []metav1.Condition{ + {Type: "Ready", Status: metav1.ConditionTrue, Reason: "Succeeded", Message: "Release applied"}, + } + + app, err := r.convertHelmReleaseToApplication(context.TODO(), hr) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + wc := findCondition(app.GetConditions(), "WorkloadsReady") + if wc != nil { + t.Error("expected no WorkloadsReady condition when no monitors exist") + } + + rc := findCondition(app.GetConditions(), "Ready") + if rc == nil { + t.Fatal("expected Ready condition to be present") + } + if rc.Status != metav1.ConditionTrue { + t.Errorf("expected Ready=True unchanged, got %s", rc.Status) + } +} + +func TestConvertConditions_ReadyStaysTrue_WhenAllOperational(t *testing.T) { + monitor := &cozyv1alpha1.WorkloadMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mon-1", + Namespace: "default", + Labels: map[string]string{ + appsv1alpha1.ApplicationKindLabel: "PostgreSQL", + appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", + appsv1alpha1.ApplicationNameLabel: "mydb", + }, + }, + Status: cozyv1alpha1.WorkloadMonitorStatus{ + Operational: ptr.To(true), + }, + } + + r := newTestRESTWithSchemes(monitor) + hr := makeHelmRelease("postgresql-mydb", "default") + hr.Status.Conditions = []metav1.Condition{ + {Type: "Ready", Status: metav1.ConditionTrue, Reason: "Succeeded", Message: "Release applied"}, + {Type: "Released", Status: metav1.ConditionTrue, Reason: "Succeeded", Message: "Released"}, + } + + app, err := r.convertHelmReleaseToApplication(context.TODO(), hr) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + rc := findCondition(app.GetConditions(), "Ready") + if rc == nil { + t.Fatal("expected Ready condition") + } + if rc.Status != metav1.ConditionTrue { + t.Errorf("expected Ready=True when all workloads operational, got %s", rc.Status) + } + if rc.Reason != "Succeeded" { + t.Errorf("expected Ready.Reason=Succeeded (unchanged), got %s", rc.Reason) + } +} + +func TestConvertConditions_WorkloadsReadyTimestampIsNonZero(t *testing.T) { + monitor := &cozyv1alpha1.WorkloadMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mon-1", + Namespace: "default", + Labels: map[string]string{ + appsv1alpha1.ApplicationKindLabel: "PostgreSQL", + appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", + appsv1alpha1.ApplicationNameLabel: "mydb", + }, + }, + Status: cozyv1alpha1.WorkloadMonitorStatus{ + Operational: ptr.To(true), + }, + } + + r := newTestRESTWithSchemes(monitor) + hr := makeHelmRelease("postgresql-mydb", "default") + // No Ready condition — HR still being reconciled + hr.Status.Conditions = []metav1.Condition{} + + app, err := r.convertHelmReleaseToApplication(context.TODO(), hr) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + wc := findCondition(app.GetConditions(), "WorkloadsReady") + if wc == nil { + t.Fatal("expected WorkloadsReady condition") + } + if wc.LastTransitionTime.IsZero() { + t.Error("expected non-zero LastTransitionTime") + } +} + +func TestConvertConditions_WorkloadsReadyUnknownWhenNilOperational(t *testing.T) { + monitor := &cozyv1alpha1.WorkloadMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mon-1", + Namespace: "default", + Labels: map[string]string{ + appsv1alpha1.ApplicationKindLabel: "PostgreSQL", + appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", + appsv1alpha1.ApplicationNameLabel: "mydb", + }, + }, + Status: cozyv1alpha1.WorkloadMonitorStatus{ + Operational: nil, // Not yet reconciled + }, + } + + r := newTestRESTWithSchemes(monitor) + hr := makeHelmRelease("postgresql-mydb", "default") + hr.Status.Conditions = []metav1.Condition{ + {Type: "Ready", Status: metav1.ConditionTrue, Reason: "Succeeded", Message: "ok"}, + } + + app, err := r.convertHelmReleaseToApplication(context.TODO(), hr) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + wc := findCondition(app.GetConditions(), "WorkloadsReady") + if wc == nil { + t.Fatal("expected WorkloadsReady condition") + } + if wc.Status != metav1.ConditionUnknown { + t.Errorf("expected WorkloadsReady=Unknown for nil Operational, got %s", wc.Status) + } + + // Ready should NOT be overridden for unknown — prefer availability during startup + rc := findCondition(app.GetConditions(), "Ready") + if rc == nil { + t.Fatal("expected Ready condition") + } + if rc.Status != metav1.ConditionTrue { + t.Errorf("expected Ready=True when workloads unknown (pending), got %s", rc.Status) + } +} + +func TestConvertConditions_WorkloadsReadyUnknownOnError(t *testing.T) { + // Create a client with only HelmRelease scheme — WorkloadMonitor List will fail + scheme := runtime.NewScheme() + _ = helmv2.AddToScheme(scheme) + // Deliberately NOT registering cozyv1alpha1 so that List returns an error + + c := fake.NewClientBuilder().WithScheme(scheme).Build() + r := NewREST(c, nil, &config.Resource{ + Application: config.ApplicationConfig{ + Kind: "PostgreSQL", + Plural: "postgresqls", + Singular: "postgresql", + }, + Release: config.ReleaseConfig{ + Prefix: "postgresql-", + }, + }) + + hr := makeHelmRelease("postgresql-mydb", "default") + hr.CreationTimestamp = metav1.Now() + hr.Status.Conditions = []metav1.Condition{ + {Type: "Ready", Status: metav1.ConditionTrue, Reason: "Succeeded", Message: "ok"}, + } + + app, err := r.convertHelmReleaseToApplication(context.TODO(), hr) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + wc := findCondition(app.GetConditions(), "WorkloadsReady") + if wc == nil { + t.Fatal("expected WorkloadsReady condition with Unknown status on error") + } + if wc.Status != metav1.ConditionUnknown { + t.Errorf("expected WorkloadsReady=Unknown, got %s", wc.Status) + } + if wc.Reason != "Error" { + t.Errorf("expected reason=Error, got %s", wc.Reason) + } + + // Ready should NOT be overridden on error (fail-open: prefer availability) + rc := findCondition(app.GetConditions(), "Ready") + if rc == nil { + t.Fatal("expected Ready condition") + } + if rc.Status != metav1.ConditionTrue { + t.Errorf("expected Ready=True (fail-open on error), got %s", rc.Status) + } +} + + diff --git a/pkg/registry/apps/application/rest_watch_test.go b/pkg/registry/apps/application/rest_watch_test.go new file mode 100644 index 00000000..8fe2a04a --- /dev/null +++ b/pkg/registry/apps/application/rest_watch_test.go @@ -0,0 +1,223 @@ +package application + +import ( + "context" + "testing" + "time" + + cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" + appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" + "github.com/cozystack/cozystack/pkg/config" + helmv2 "github.com/fluxcd/helm-controller/api/v2" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestWmResultChan_NilWatcher(t *testing.T) { + ch := wmResultChan(nil) + if ch != nil { + t.Error("expected nil channel for nil watcher") + } +} + +func TestWmResultChan_ValidWatcher(t *testing.T) { + fw := watch.NewFake() + ch := wmResultChan(fw) + if ch == nil { + t.Error("expected non-nil channel for valid watcher") + } + fw.Stop() +} + +// TestWatchIntegration_WorkloadMonitorTriggersModifiedEvent verifies the +// full path: WM event → label lookup → HelmRelease Get → Application conversion. +func TestWatchIntegration_WorkloadMonitorTriggersModifiedEvent(t *testing.T) { + scheme := runtime.NewScheme() + _ = cozyv1alpha1.AddToScheme(scheme) + _ = helmv2.AddToScheme(scheme) + + hr := &helmv2.HelmRelease{ + ObjectMeta: metav1.ObjectMeta{ + Name: "postgresql-mydb", + Namespace: "default", + Labels: map[string]string{ + ApplicationKindLabel: "PostgreSQL", + ApplicationGroupLabel: "apps.cozystack.io", + ApplicationNameLabel: "mydb", + }, + }, + } + hr.Status.Conditions = []metav1.Condition{ + {Type: "Ready", Status: metav1.ConditionTrue, Reason: "Succeeded", Message: "ok"}, + } + + wm := &cozyv1alpha1.WorkloadMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mon-1", + Namespace: "default", + Labels: map[string]string{ + appsv1alpha1.ApplicationKindLabel: "PostgreSQL", + appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", + appsv1alpha1.ApplicationNameLabel: "mydb", + }, + }, + Status: cozyv1alpha1.WorkloadMonitorStatus{ + Operational: ptr.To(false), + }, + } + + c := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(hr, wm).Build() + + r := newTestRESTWithSchemesFromClient(c) + + // Simulate the Watch goroutine path: extract app name from WM labels, + // construct HelmRelease name, look up HR, convert to Application + wmAppName := wm.Labels[ApplicationNameLabel] + hrName := r.releaseConfig.Prefix + wmAppName + + foundHR := &helmv2.HelmRelease{} + if err := c.Get(context.TODO(), types.NamespacedName{Namespace: "default", Name: hrName}, foundHR); err != nil { + t.Fatalf("failed to get HelmRelease: %v", err) + } + app, err := r.ConvertHelmReleaseToApplication(context.TODO(), foundHR) + if err != nil { + t.Fatalf("failed to convert: %v", err) + } + if app.Name != "mydb" { + t.Errorf("expected app name 'mydb', got %q", app.Name) + } + + // Verify WorkloadsReady is False due to non-operational WM + wc := findCondition(app.GetConditions(), "WorkloadsReady") + if wc == nil { + t.Fatal("expected WorkloadsReady condition") + } + if wc.Status != metav1.ConditionFalse { + t.Errorf("expected WorkloadsReady=False, got %s", wc.Status) + } +} + +// TestWatchIntegration_MonitorDeletionDropsWorkloadsReady verifies that when a +// WorkloadMonitor is deleted, the Application's WorkloadsReady condition +// disappears. Ready condition always reflects HelmRelease state regardless. +func TestWatchIntegration_MonitorDeletionDropsWorkloadsReady(t *testing.T) { + scheme := runtime.NewScheme() + _ = cozyv1alpha1.AddToScheme(scheme) + _ = helmv2.AddToScheme(scheme) + + hr := &helmv2.HelmRelease{ + ObjectMeta: metav1.ObjectMeta{ + Name: "postgresql-mydb", + Namespace: "default", + Labels: map[string]string{ + ApplicationKindLabel: "PostgreSQL", + ApplicationGroupLabel: "apps.cozystack.io", + ApplicationNameLabel: "mydb", + }, + }, + } + hr.Status.Conditions = []metav1.Condition{ + {Type: "Ready", Status: metav1.ConditionTrue, Reason: "Succeeded", Message: "ok"}, + } + + nonOpMonitor := &cozyv1alpha1.WorkloadMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mon-1", + Namespace: "default", + Labels: map[string]string{ + appsv1alpha1.ApplicationKindLabel: "PostgreSQL", + appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", + appsv1alpha1.ApplicationNameLabel: "mydb", + }, + }, + Status: cozyv1alpha1.WorkloadMonitorStatus{ + Operational: ptr.To(false), + }, + } + + c := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(hr, nonOpMonitor).Build() + r := newTestRESTWithSchemesFromClient(c) + + // Step 1: With non-operational monitor, WorkloadsReady=False, Ready=True + app1, err := r.convertHelmReleaseToApplication(context.TODO(), hr) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + wc1 := findCondition(app1.GetConditions(), "WorkloadsReady") + if wc1 == nil || wc1.Status != metav1.ConditionFalse { + t.Fatalf("expected WorkloadsReady=False with non-operational monitor, got %v", wc1) + } + rc1 := findCondition(app1.GetConditions(), "Ready") + if rc1 == nil || rc1.Status != metav1.ConditionTrue { + t.Fatalf("expected Ready=True (reflects HelmRelease), got %v", rc1) + } + + // Step 2: Delete the monitor + if err := c.Delete(context.TODO(), nonOpMonitor); err != nil { + t.Fatalf("failed to delete monitor: %v", err) + } + + // Step 3: WorkloadsReady should disappear, Ready stays True + app2, err := r.convertHelmReleaseToApplication(context.TODO(), hr) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + wc2 := findCondition(app2.GetConditions(), "WorkloadsReady") + if wc2 != nil { + t.Error("expected no WorkloadsReady condition after monitor deletion") + } + rc2 := findCondition(app2.GetConditions(), "Ready") + if rc2 == nil { + t.Fatal("expected Ready condition") + } + if rc2.Status != metav1.ConditionTrue { + t.Errorf("expected Ready=True after monitor deletion, got %s", rc2.Status) + } +} + +// TestWatchIntegration_WMWatcherCloseProducesNilChannel verifies that +// after wmWatcher is set to nil, wmResultChan returns nil channel. +func TestWatchIntegration_WMWatcherCloseProducesNilChannel(t *testing.T) { + fw := watch.NewFake() + ch := wmResultChan(fw) + if ch == nil { + t.Fatal("expected non-nil channel before close") + } + + fw.Stop() + + timeout := time.After(time.Second) + for { + select { + case _, ok := <-ch: + if !ok { + var nilWatcher watch.Interface + nilCh := wmResultChan(nilWatcher) + if nilCh != nil { + t.Error("expected nil channel after watcher set to nil") + } + return + } + case <-timeout: + t.Fatal("timeout waiting for watcher channel to close") + } + } +} + +func newTestRESTWithSchemesFromClient(c client.Client) *REST { + return NewREST(c, nil, &config.Resource{ + Application: config.ApplicationConfig{ + Kind: "PostgreSQL", + Plural: "postgresqls", + Singular: "postgresql", + }, + Release: config.ReleaseConfig{ + Prefix: "postgresql-", + }, + }) +} diff --git a/pkg/registry/apps/application/rest_workloads_test.go b/pkg/registry/apps/application/rest_workloads_test.go new file mode 100644 index 00000000..30dd85c6 --- /dev/null +++ b/pkg/registry/apps/application/rest_workloads_test.go @@ -0,0 +1,342 @@ +package application + +import ( + "context" + "testing" + + cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" + appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" + "github.com/cozystack/cozystack/pkg/config" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + helmv2 "github.com/fluxcd/helm-controller/api/v2" +) + +func newTestRESTWithSchemes(objs ...runtime.Object) *REST { + scheme := runtime.NewScheme() + _ = cozyv1alpha1.AddToScheme(scheme) + _ = helmv2.AddToScheme(scheme) + + builder := fake.NewClientBuilder().WithScheme(scheme) + for _, obj := range objs { + builder = builder.WithRuntimeObjects(obj) + } + c := builder.Build() + + return NewREST(c, nil, &config.Resource{ + Application: config.ApplicationConfig{ + Kind: "PostgreSQL", + Plural: "postgresqls", + Singular: "postgresql", + }, + Release: config.ReleaseConfig{ + Prefix: "postgresql-", + }, + }) +} + +func TestGetWorkloadsOperational_NoMonitors(t *testing.T) { + r := newTestRESTWithSchemes() + ws, err := r.getWorkloadsOperational(context.TODO(), "default", "mydb") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if ws.found { + t.Error("expected found=false when no monitors exist") + } + if !ws.operational { + t.Error("expected operational=true when no monitors exist") + } +} + +func TestGetWorkloadsOperational_AllOperational(t *testing.T) { + m1 := &cozyv1alpha1.WorkloadMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mon-1", + Namespace: "default", + Labels: map[string]string{ + appsv1alpha1.ApplicationKindLabel: "PostgreSQL", + appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", + appsv1alpha1.ApplicationNameLabel: "mydb", + }, + }, + Status: cozyv1alpha1.WorkloadMonitorStatus{ + Operational: ptr.To(true), + }, + } + m2 := &cozyv1alpha1.WorkloadMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mon-2", + Namespace: "default", + Labels: map[string]string{ + appsv1alpha1.ApplicationKindLabel: "PostgreSQL", + appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", + appsv1alpha1.ApplicationNameLabel: "mydb", + }, + }, + Status: cozyv1alpha1.WorkloadMonitorStatus{ + Operational: ptr.To(true), + }, + } + + r := newTestRESTWithSchemes(m1, m2) + ws, err := r.getWorkloadsOperational(context.TODO(), "default", "mydb") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !ws.found { + t.Error("expected found=true") + } + if !ws.operational { + t.Error("expected operational=true when all monitors are operational") + } +} + +func TestGetWorkloadsOperational_SomeNotOperational(t *testing.T) { + m1 := &cozyv1alpha1.WorkloadMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mon-1", + Namespace: "default", + Labels: map[string]string{ + appsv1alpha1.ApplicationKindLabel: "PostgreSQL", + appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", + appsv1alpha1.ApplicationNameLabel: "mydb", + }, + }, + Status: cozyv1alpha1.WorkloadMonitorStatus{ + Operational: ptr.To(true), + }, + } + m2 := &cozyv1alpha1.WorkloadMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mon-2", + Namespace: "default", + Labels: map[string]string{ + appsv1alpha1.ApplicationKindLabel: "PostgreSQL", + appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", + appsv1alpha1.ApplicationNameLabel: "mydb", + }, + }, + Status: cozyv1alpha1.WorkloadMonitorStatus{ + Operational: ptr.To(false), + }, + } + + r := newTestRESTWithSchemes(m1, m2) + ws, err := r.getWorkloadsOperational(context.TODO(), "default", "mydb") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !ws.found { + t.Error("expected found=true") + } + if ws.operational { + t.Error("expected operational=false when at least one monitor is not operational") + } +} + +func TestGetWorkloadsOperational_OperationalNil(t *testing.T) { + m := &cozyv1alpha1.WorkloadMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mon-1", + Namespace: "default", + Labels: map[string]string{ + appsv1alpha1.ApplicationKindLabel: "PostgreSQL", + appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", + appsv1alpha1.ApplicationNameLabel: "mydb", + }, + }, + Status: cozyv1alpha1.WorkloadMonitorStatus{ + Operational: nil, // Not yet reconciled + }, + } + + r := newTestRESTWithSchemes(m) + ws, err := r.getWorkloadsOperational(context.TODO(), "default", "mydb") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !ws.found { + t.Error("expected found=true") + } + if !ws.unknown { + t.Error("expected unknown=true when Operational is nil") + } +} + +func TestGetWorkloadsOperational_MixedNilAndOperational(t *testing.T) { + m1 := &cozyv1alpha1.WorkloadMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mon-1", + Namespace: "default", + Labels: map[string]string{ + appsv1alpha1.ApplicationKindLabel: "PostgreSQL", + appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", + appsv1alpha1.ApplicationNameLabel: "mydb", + }, + }, + Status: cozyv1alpha1.WorkloadMonitorStatus{ + Operational: ptr.To(true), + }, + } + m2 := &cozyv1alpha1.WorkloadMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mon-2", + Namespace: "default", + Labels: map[string]string{ + appsv1alpha1.ApplicationKindLabel: "PostgreSQL", + appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", + appsv1alpha1.ApplicationNameLabel: "mydb", + }, + }, + Status: cozyv1alpha1.WorkloadMonitorStatus{ + Operational: nil, + }, + } + + r := newTestRESTWithSchemes(m1, m2) + ws, err := r.getWorkloadsOperational(context.TODO(), "default", "mydb") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !ws.unknown { + t.Error("expected unknown=true when at least one monitor has nil Operational") + } +} + +func TestGetWorkloadsOperational_MixedFailedAndPending(t *testing.T) { + m1 := &cozyv1alpha1.WorkloadMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mon-1", + Namespace: "default", + Labels: map[string]string{ + appsv1alpha1.ApplicationKindLabel: "PostgreSQL", + appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", + appsv1alpha1.ApplicationNameLabel: "mydb", + }, + }, + Status: cozyv1alpha1.WorkloadMonitorStatus{ + Operational: ptr.To(false), // Confirmed failure + }, + } + m2 := &cozyv1alpha1.WorkloadMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mon-2", + Namespace: "default", + Labels: map[string]string{ + appsv1alpha1.ApplicationKindLabel: "PostgreSQL", + appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", + appsv1alpha1.ApplicationNameLabel: "mydb", + }, + }, + Status: cozyv1alpha1.WorkloadMonitorStatus{ + Operational: nil, // Not yet reconciled + }, + } + + r := newTestRESTWithSchemes(m1, m2) + ws, err := r.getWorkloadsOperational(context.TODO(), "default", "mydb") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if ws.operational { + t.Error("expected operational=false when at least one monitor is explicitly failed") + } + if !ws.unknown { + t.Error("expected unknown=true when at least one monitor has nil Operational") + } +} + +func TestConvertConditions_MixedFailedAndPendingShowsFalse(t *testing.T) { + mFailed := &cozyv1alpha1.WorkloadMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mon-failed", + Namespace: "default", + Labels: map[string]string{ + appsv1alpha1.ApplicationKindLabel: "PostgreSQL", + appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", + appsv1alpha1.ApplicationNameLabel: "mydb", + }, + }, + Status: cozyv1alpha1.WorkloadMonitorStatus{ + Operational: ptr.To(false), + }, + } + mPending := &cozyv1alpha1.WorkloadMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mon-pending", + Namespace: "default", + Labels: map[string]string{ + appsv1alpha1.ApplicationKindLabel: "PostgreSQL", + appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", + appsv1alpha1.ApplicationNameLabel: "mydb", + }, + }, + Status: cozyv1alpha1.WorkloadMonitorStatus{ + Operational: nil, + }, + } + + r := newTestRESTWithSchemes(mFailed, mPending) + hr := &helmv2.HelmRelease{ + ObjectMeta: metav1.ObjectMeta{ + Name: "postgresql-mydb", + Namespace: "default", + Labels: map[string]string{ + ApplicationKindLabel: "PostgreSQL", + ApplicationGroupLabel: "apps.cozystack.io", + ApplicationNameLabel: "mydb", + }, + }, + } + hr.Status.Conditions = []metav1.Condition{ + {Type: "Ready", Status: metav1.ConditionTrue, Reason: "Succeeded", Message: "ok"}, + } + + app, err := r.convertHelmReleaseToApplication(context.TODO(), hr) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Concrete failure should take priority over unknown + wc := findCondition(app.GetConditions(), "WorkloadsReady") + if wc == nil { + t.Fatal("expected WorkloadsReady condition") + } + if wc.Status != metav1.ConditionFalse { + t.Errorf("expected WorkloadsReady=False (failure takes priority over unknown), got %s", wc.Status) + } +} + +func TestGetWorkloadsOperational_DifferentApp_NotFound(t *testing.T) { + m := &cozyv1alpha1.WorkloadMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mon-1", + Namespace: "default", + Labels: map[string]string{ + appsv1alpha1.ApplicationKindLabel: "PostgreSQL", + appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", + appsv1alpha1.ApplicationNameLabel: "other-db", + }, + }, + Status: cozyv1alpha1.WorkloadMonitorStatus{ + Operational: ptr.To(false), + }, + } + + r := newTestRESTWithSchemes(m) + ws, err := r.getWorkloadsOperational(context.TODO(), "default", "mydb") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if ws.found { + t.Error("expected found=false for different app name") + } + if !ws.operational { + t.Error("expected operational=true when no matching monitors found") + } +} + From dc3387c635eef5ee795c31a0909910eed052dd5f Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Mon, 13 Apr 2026 22:27:01 +0300 Subject: [PATCH 042/250] fix(workload-monitor): use fresh spec for MinReplicas check on retry Inside the RetryOnConflict block, derive the operational status from fresh.Spec.MinReplicas instead of the stale monitor.Spec.MinReplicas so that concurrent spec updates observed by the retry are respected. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- internal/controller/workloadmonitor_controller.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/controller/workloadmonitor_controller.go b/internal/controller/workloadmonitor_controller.go index 0cd3103e..9a967e65 100644 --- a/internal/controller/workloadmonitor_controller.go +++ b/internal/controller/workloadmonitor_controller.go @@ -388,9 +388,11 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ fresh.Status.ObservedReplicas = observedReplicas fresh.Status.AvailableReplicas = availableReplicas - // Default to operational = true, but check MinReplicas if set + // Default to operational = true, but check MinReplicas if set. + // Use fresh.Spec to avoid making decisions based on a stale cached copy + // when the spec was updated between the initial read and this retry. fresh.Status.Operational = pointer.Bool(true) - if monitor.Spec.MinReplicas != nil && availableReplicas < *monitor.Spec.MinReplicas { + if fresh.Spec.MinReplicas != nil && availableReplicas < *fresh.Spec.MinReplicas { fresh.Status.Operational = pointer.Bool(false) } return r.Status().Update(ctx, fresh) From d369b64d718beee24ff4dc57582af5710c05d1d5 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Mon, 13 Apr 2026 22:42:11 +0300 Subject: [PATCH 043/250] fix(application): address review feedback for WorkloadMonitor integration Tenant applications live in a child namespace computed from the Tenant name, not in the HelmRelease's namespace. Scope the WorkloadMonitor watch cluster-wide when kindName is Tenant and reverse-map WM events back to the owning HelmRelease via computeTenantNamespace. The condition-enrichment path now looks up monitors in the computed child namespace too, so WorkloadsReady is populated for Tenants. Buffer WorkloadMonitor events that arrive before the initial-events-end bookmark and replay them once the bookmark is emitted, to preserve the watch-list contract while not dropping workload-state transitions that happen during the snapshot window. Pass the fresh WorkloadMonitor object from the watch event into the conversion path so WorkloadsReady reflects the state that triggered the event, even when the cache client is lagging behind the watch client. Derive WorkloadsReady.LastTransitionTime from a stable source (max of HelmRelease creation and condition timestamps, plus monitor timestamps) instead of metav1.Now(), so repeated conversions of the same underlying state produce identical timestamps. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- pkg/registry/apps/application/rest.go | 196 +++++++++++++++--- .../apps/application/rest_conditions_test.go | 14 +- .../apps/application/rest_validation_test.go | 4 +- .../apps/application/rest_watch_test.go | 4 +- .../apps/application/rest_workloads_test.go | 16 +- 5 files changed, 184 insertions(+), 50 deletions(-) diff --git a/pkg/registry/apps/application/rest.go b/pkg/registry/apps/application/rest.go index 583a4a8c..0728ea13 100644 --- a/pkg/registry/apps/application/rest.go +++ b/pkg/registry/apps/application/rest.go @@ -704,13 +704,19 @@ func (r *REST) Watch(ctx context.Context, options *metainternalversion.ListOptio } customW.underlying = helmWatcher - // Start watch on WorkloadMonitor to detect pod readiness changes + // Start watch on WorkloadMonitor to detect pod readiness changes. + // For Tenant applications the WorkloadMonitor lives in a computed child + // namespace (see computeTenantNamespace), not in the HelmRelease namespace, + // so scoping the watch to `namespace` would miss all events. Use a + // cluster-wide watch in that case — label selectors still restrict the + // stream to the relevant kind/group. wmLabelSelector := labels.NewSelector().Add(*appKindReq, *appGroupReq) wmList := &cozyv1alpha1.WorkloadMonitorList{} - wmWatcher, err := r.w.Watch(ctx, wmList, &client.ListOptions{ - Namespace: namespace, - LabelSelector: wmLabelSelector, - }) + wmListOpts := &client.ListOptions{LabelSelector: wmLabelSelector} + if r.kindName != "Tenant" { + wmListOpts.Namespace = namespace + } + wmWatcher, err := r.w.Watch(ctx, wmList, wmListOpts) if err != nil { klog.Warningf("Failed to set up WorkloadMonitor watch, workload status changes won't trigger events: %v", err) // Non-fatal: proceed without WorkloadMonitor watch @@ -731,6 +737,16 @@ func (r *REST) Watch(ctx context.Context, options *metainternalversion.ListOptio initialEventsEndSent := !sendInitialEvents // If not sendInitialEvents, consider it already sent var lastResourceVersion string + // Buffer of WorkloadMonitor events that arrived before the initial + // snapshot finished. The watch-list contract requires the stream to + // deliver all ADDED events followed by the initial-events-end bookmark + // before any live updates, so we hold WM-triggered Modified events and + // replay them once the bookmark has been emitted. Without this, a + // workload whose status flips during the snapshot window (after the + // Application ADDED but before the bookmark) and then stops changing + // would leave the client with a stale WorkloadsReady forever. + var pendingWMEvents []watch.Event + // Get the starting resourceVersion from options // If client provides resourceVersion (e.g., from a previous List), we should skip // objects with resourceVersion <= startingRV (client already has them) @@ -741,6 +757,19 @@ func (r *REST) Watch(ctx context.Context, options *metainternalversion.ListOptio } } + drainPendingWMEvents := func() { + for _, ev := range pendingWMEvents { + select { + case customW.resultChan <- ev: + case <-customW.stopChan: + return + case <-ctx.Done(): + return + } + } + pendingWMEvents = nil + } + // Helper function to send initial-events-end bookmark sendInitialEventsEndBookmark := func() { if initialEventsEndSent { @@ -765,8 +794,11 @@ func (r *REST) Watch(ctx context.Context, options *metainternalversion.ListOptio select { case customW.resultChan <- bookmarkEvent: case <-customW.stopChan: + return case <-ctx.Done(): + return } + drainPendingWMEvents() } // Process watch events @@ -794,8 +826,10 @@ func (r *REST) Watch(ctx context.Context, options *metainternalversion.ListOptio APIVersion: appsv1alpha1.SchemeGroupVersion.String(), Kind: r.kindName, } + justFlipped := false if !initialEventsEndSent { initialEventsEndSent = true + justFlipped = true bookmarkApp.SetAnnotations(map[string]string{ "k8s.io/initial-events-end": "true", }) @@ -812,6 +846,9 @@ func (r *REST) Watch(ctx context.Context, options *metainternalversion.ListOptio case <-ctx.Done(): return } + if justFlipped { + drainPendingWMEvents() + } } continue } @@ -911,12 +948,6 @@ func (r *REST) Watch(ctx context.Context, options *metainternalversion.ListOptio } continue } - // Don't emit WM-triggered events until the initial snapshot is - // complete — the watch-list contract requires all ADDED events - // followed by the initial-events-end bookmark before any live updates. - if !initialEventsEndSent { - continue - } wm, ok := wmEvent.Object.(*cozyv1alpha1.WorkloadMonitor) if !ok { continue @@ -937,12 +968,37 @@ func (r *REST) Watch(ctx context.Context, options *metainternalversion.ListOptio if resourceName != "" && wmAppName != resourceName { continue } - hr := &helmv2.HelmRelease{} - if err := r.c.Get(ctx, client.ObjectKey{Namespace: wm.Namespace, Name: hrName}, hr); err != nil { - klog.V(4).Infof("Cannot find HelmRelease %s/%s for WorkloadMonitor event: %v", wm.Namespace, hrName, err) + + // Locate the owning HelmRelease. For most application kinds the + // WorkloadMonitor and its HelmRelease live in the same namespace, + // but Tenant workloads live in a child namespace (see + // computeTenantNamespace) while the HelmRelease remains in the + // parent/requested namespace — so the WM-to-HR namespace mapping + // differs. + hrNS := wm.Namespace + if r.kindName == "Tenant" { + hrNS = namespace + // Filter out WM events whose child namespace does not + // correspond to the Tenant in our watched namespace, since + // the cluster-wide WM watch delivers events for all tenants. + if r.computeTenantNamespace(namespace, wmAppName) != wm.Namespace { + continue + } + if !fieldFilter.MatchesNamespace(hrNS) { + continue + } + } else if !fieldFilter.MatchesNamespace(hrNS) { continue } - app, err := r.ConvertHelmReleaseToApplication(ctx, hr) + hr := &helmv2.HelmRelease{} + if err := r.c.Get(ctx, client.ObjectKey{Namespace: hrNS, Name: hrName}, hr); err != nil { + klog.V(4).Infof("Cannot find HelmRelease %s/%s for WorkloadMonitor event: %v", hrNS, hrName, err) + continue + } + // Pass the fresh WorkloadMonitor so conversion uses the latest + // operational status even if the cache (r.c) has not yet + // observed this watch event. + app, err := r.ConvertHelmReleaseToApplicationWithMonitor(ctx, hr, wm) if err != nil { klog.V(4).Infof("Error converting HelmRelease for WorkloadMonitor event: %v", err) continue @@ -961,9 +1017,17 @@ func (r *REST) Watch(ctx context.Context, options *metainternalversion.ListOptio // Use the WorkloadMonitor's ResourceVersion for the emitted event // so clients see a monotonically increasing RV and don't skip this update. app.SetResourceVersion(wm.GetResourceVersion()) + outEvent := watch.Event{Type: watch.Modified, Object: &app} + // Buffer WM-triggered events that arrive before the + // initial-events-end bookmark. They will be replayed in order + // immediately after the bookmark is emitted. + if !initialEventsEndSent { + pendingWMEvents = append(pendingWMEvents, outEvent) + continue + } lastResourceVersion = wm.GetResourceVersion() select { - case customW.resultChan <- watch.Event{Type: watch.Modified, Object: &app}: + case customW.resultChan <- outEvent: case <-customW.stopChan: return case <-ctx.Done(): @@ -1094,12 +1158,21 @@ func filterPrefixedMap(original map[string]string, prefix string) map[string]str return processed } -// ConvertHelmReleaseToApplication converts a HelmRelease to an Application +// ConvertHelmReleaseToApplication converts a HelmRelease to an Application. func (r *REST) ConvertHelmReleaseToApplication(ctx context.Context, hr *helmv2.HelmRelease) (appsv1alpha1.Application, error) { + return r.ConvertHelmReleaseToApplicationWithMonitor(ctx, hr, nil) +} + +// ConvertHelmReleaseToApplicationWithMonitor converts a HelmRelease to an +// Application, optionally overriding the cached copy of a WorkloadMonitor with +// a fresher version received from the watch client. This prevents the emitted +// Application object from carrying stale WorkloadsReady data when r.c (cache) +// lags behind r.w (watch). +func (r *REST) ConvertHelmReleaseToApplicationWithMonitor(ctx context.Context, hr *helmv2.HelmRelease, freshMonitor *cozyv1alpha1.WorkloadMonitor) (appsv1alpha1.Application, error) { klog.V(6).Infof("Converting HelmRelease to Application for resource %s", hr.GetName()) // Convert HelmRelease struct to Application struct - app, err := r.convertHelmReleaseToApplication(ctx, hr) + app, err := r.convertHelmReleaseToApplication(ctx, hr, freshMonitor) if err != nil { klog.Errorf("Error converting from HelmRelease to Application: %v", err) return appsv1alpha1.Application{}, err @@ -1212,8 +1285,11 @@ func (r *REST) validateTenantNamespaceLength(currentNamespace, tenantName string return allErrs } -// convertHelmReleaseToApplication implements the actual conversion logic -func (r *REST) convertHelmReleaseToApplication(ctx context.Context, hr *helmv2.HelmRelease) (appsv1alpha1.Application, error) { +// convertHelmReleaseToApplication implements the actual conversion logic. +// The optional freshMonitor is used to override the cache copy of a +// WorkloadMonitor when a newer version was delivered via the watch client — +// see ConvertHelmReleaseToApplicationWithMonitor for the rationale. +func (r *REST) convertHelmReleaseToApplication(ctx context.Context, hr *helmv2.HelmRelease, freshMonitor *cozyv1alpha1.WorkloadMonitor) (appsv1alpha1.Application, error) { // Filter out internal keys (starting with "_") from spec filteredSpec := filterInternalKeys(hr.Spec.Values) @@ -1250,28 +1326,49 @@ func (r *REST) convertHelmReleaseToApplication(ctx context.Context, hr *helmv2.H }) } } - // Enrich conditions with WorkloadMonitor operational status - ws, wsErr := r.getWorkloadsOperational(ctx, hr.Namespace, app.Name) + // Enrich conditions with WorkloadMonitor operational status. + // Tenant workloads live in a child namespace (computed from the Tenant name), + // not in the same namespace as the owning HelmRelease — look there instead. + workloadsNS := hr.Namespace + if r.kindName == "Tenant" { + workloadsNS = r.computeTenantNamespace(hr.Namespace, app.Name) + } + ws, wsErr := r.getWorkloadsOperational(ctx, workloadsNS, app.Name, freshMonitor) + // Derive a stable LastTransitionTime: use the owning HelmRelease's own + // condition update time (or CreationTimestamp as a floor) so that repeated + // conversions of the same underlying state produce identical timestamps. + wrTransition := hr.CreationTimestamp + for _, c := range hr.GetConditions() { + if c.LastTransitionTime.After(wrTransition.Time) { + wrTransition = c.LastTransitionTime + } + } + if ws.transitionTime.After(wrTransition.Time) { + wrTransition = ws.transitionTime + } + if wrTransition.IsZero() { + // Fallback for objects that somehow have no timestamps at all + // (e.g. hand-crafted test fixtures). In production HelmReleases + // always carry a CreationTimestamp, so the stable branch above + // is used. + wrTransition = metav1.Now() + } if wsErr != nil { // Fail-open: if we can't query WorkloadMonitors (e.g., informer cache not ready), // don't override Ready. Prefer operational availability over safety. // The WorkloadsReady=Unknown condition still signals the issue to the user. - klog.Warningf("Failed to check workload monitors for %s/%s: %v", hr.Namespace, app.Name, wsErr) + klog.Warningf("Failed to check workload monitors for %s/%s: %v", workloadsNS, app.Name, wsErr) conditions = append(conditions, metav1.Condition{ Type: "WorkloadsReady", Status: metav1.ConditionUnknown, - LastTransitionTime: metav1.Now(), + LastTransitionTime: wrTransition, Reason: "Error", Message: fmt.Sprintf("Failed to check workload status: %v", wsErr), }) } else if ws.found { - // LastTransitionTime is set to the current time because the Application - // resource is virtual (computed on-the-fly from HelmRelease). There is no - // persistent condition state to track actual transitions. This is consistent - // with how computed/virtual API resources work in Kubernetes. workloadsCondition := metav1.Condition{ Type: "WorkloadsReady", - LastTransitionTime: metav1.Now(), + LastTransitionTime: wrTransition, Reason: "WorkloadMonitorCheck", } switch { @@ -1321,11 +1418,20 @@ type workloadsStatus struct { operational bool found bool unknown bool // true when at least one monitor has nil Operational (not yet reconciled) + // transitionTime is the most recent metadata update time across the + // matching monitors. Used as WorkloadsReady.LastTransitionTime so that + // repeated conversions for the same underlying state produce stable + // timestamps (preserving the Kubernetes contract that identical + // resource versions represent identical content). + transitionTime metav1.Time } // getWorkloadsOperational checks WorkloadMonitor resources for an application and returns // aggregated operational status. If no monitors exist, returns found=false. -func (r *REST) getWorkloadsOperational(ctx context.Context, namespace, appName string) (workloadsStatus, error) { +// When freshOverride is non-nil, its status replaces the cached copy for the +// corresponding monitor — this keeps the result consistent with watch events +// when the cache (r.c) lags behind the watch client (r.w). +func (r *REST) getWorkloadsOperational(ctx context.Context, namespace, appName string, freshOverride *cozyv1alpha1.WorkloadMonitor) (workloadsStatus, error) { monitors := &cozyv1alpha1.WorkloadMonitorList{} if err := r.c.List(ctx, monitors, client.InNamespace(namespace), @@ -1337,19 +1443,47 @@ func (r *REST) getWorkloadsOperational(ctx context.Context, namespace, appName s ); err != nil { return workloadsStatus{}, err } + // Ensure the freshOverride is represented in the aggregation even when + // the cache has not yet observed it (brand-new resource) or is behind. + replaced := false + if freshOverride != nil { + for i := range monitors.Items { + if monitors.Items[i].UID == freshOverride.UID || + (monitors.Items[i].Name == freshOverride.Name && monitors.Items[i].Namespace == freshOverride.Namespace) { + monitors.Items[i] = *freshOverride + replaced = true + break + } + } + if !replaced { + monitors.Items = append(monitors.Items, *freshOverride) + } + } if len(monitors.Items) == 0 { return workloadsStatus{operational: true, found: false}, nil } operational := true unknown := false + var latest metav1.Time for _, m := range monitors.Items { if m.Status.Operational == nil { unknown = true } else if !*m.Status.Operational { operational = false } + // Pick the most recent monitor mtime as a stable transition time. + if t := latestMonitorTime(&m); t.After(latest.Time) { + latest = t + } } - return workloadsStatus{operational: operational, found: true, unknown: unknown}, nil + return workloadsStatus{operational: operational, found: true, unknown: unknown, transitionTime: latest}, nil +} + +// latestMonitorTime returns the most recent timestamp associated with a +// WorkloadMonitor — currently only the object creation time is guaranteed. +// Status does not carry a transition time, so we fall back to CreationTimestamp. +func latestMonitorTime(m *cozyv1alpha1.WorkloadMonitor) metav1.Time { + return m.CreationTimestamp } // convertApplicationToHelmRelease implements the actual conversion logic diff --git a/pkg/registry/apps/application/rest_conditions_test.go b/pkg/registry/apps/application/rest_conditions_test.go index dbb99d52..185f59b2 100644 --- a/pkg/registry/apps/application/rest_conditions_test.go +++ b/pkg/registry/apps/application/rest_conditions_test.go @@ -61,7 +61,7 @@ func TestConvertConditions_WorkloadsReadyAdded(t *testing.T) { {Type: "Released", Status: metav1.ConditionTrue, Reason: "Succeeded", Message: "Released"}, } - app, err := r.convertHelmReleaseToApplication(context.TODO(), hr) + app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -107,7 +107,7 @@ func TestConvertConditions_ReadyNotOverriddenWhenWorkloadsNotReady(t *testing.T) {Type: "Ready", Status: metav1.ConditionTrue, Reason: "Succeeded", Message: "Release applied"}, } - app, err := r.convertHelmReleaseToApplication(context.TODO(), hr) + app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -139,7 +139,7 @@ func TestConvertConditions_NoOverrideWhenNoMonitors(t *testing.T) { {Type: "Ready", Status: metav1.ConditionTrue, Reason: "Succeeded", Message: "Release applied"}, } - app, err := r.convertHelmReleaseToApplication(context.TODO(), hr) + app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -181,7 +181,7 @@ func TestConvertConditions_ReadyStaysTrue_WhenAllOperational(t *testing.T) { {Type: "Released", Status: metav1.ConditionTrue, Reason: "Succeeded", Message: "Released"}, } - app, err := r.convertHelmReleaseToApplication(context.TODO(), hr) + app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -219,7 +219,7 @@ func TestConvertConditions_WorkloadsReadyTimestampIsNonZero(t *testing.T) { // No Ready condition — HR still being reconciled hr.Status.Conditions = []metav1.Condition{} - app, err := r.convertHelmReleaseToApplication(context.TODO(), hr) + app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -255,7 +255,7 @@ func TestConvertConditions_WorkloadsReadyUnknownWhenNilOperational(t *testing.T) {Type: "Ready", Status: metav1.ConditionTrue, Reason: "Succeeded", Message: "ok"}, } - app, err := r.convertHelmReleaseToApplication(context.TODO(), hr) + app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -302,7 +302,7 @@ func TestConvertConditions_WorkloadsReadyUnknownOnError(t *testing.T) { {Type: "Ready", Status: metav1.ConditionTrue, Reason: "Succeeded", Message: "ok"}, } - app, err := r.convertHelmReleaseToApplication(context.TODO(), hr) + app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/pkg/registry/apps/application/rest_validation_test.go b/pkg/registry/apps/application/rest_validation_test.go index 088990ce..b806161c 100644 --- a/pkg/registry/apps/application/rest_validation_test.go +++ b/pkg/registry/apps/application/rest_validation_test.go @@ -190,7 +190,7 @@ func TestConvertHelmReleaseToApplication_TenantNamespaceKindGate(t *testing.T) { }, } - app, err := r.convertHelmReleaseToApplication(context.Background(), hr) + app, err := r.convertHelmReleaseToApplication(context.Background(), hr, nil) if err != nil { t.Fatalf("convertHelmReleaseToApplication: %v", err) } @@ -208,7 +208,7 @@ func TestConvertHelmReleaseToApplication_TenantNamespaceKindGate(t *testing.T) { }, } - app, err := r.convertHelmReleaseToApplication(context.Background(), hr) + app, err := r.convertHelmReleaseToApplication(context.Background(), hr, nil) if err != nil { t.Fatalf("convertHelmReleaseToApplication: %v", err) } diff --git a/pkg/registry/apps/application/rest_watch_test.go b/pkg/registry/apps/application/rest_watch_test.go index 8fe2a04a..93a51f3c 100644 --- a/pkg/registry/apps/application/rest_watch_test.go +++ b/pkg/registry/apps/application/rest_watch_test.go @@ -144,7 +144,7 @@ func TestWatchIntegration_MonitorDeletionDropsWorkloadsReady(t *testing.T) { r := newTestRESTWithSchemesFromClient(c) // Step 1: With non-operational monitor, WorkloadsReady=False, Ready=True - app1, err := r.convertHelmReleaseToApplication(context.TODO(), hr) + app1, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -163,7 +163,7 @@ func TestWatchIntegration_MonitorDeletionDropsWorkloadsReady(t *testing.T) { } // Step 3: WorkloadsReady should disappear, Ready stays True - app2, err := r.convertHelmReleaseToApplication(context.TODO(), hr) + app2, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/pkg/registry/apps/application/rest_workloads_test.go b/pkg/registry/apps/application/rest_workloads_test.go index 30dd85c6..b706c323 100644 --- a/pkg/registry/apps/application/rest_workloads_test.go +++ b/pkg/registry/apps/application/rest_workloads_test.go @@ -40,7 +40,7 @@ func newTestRESTWithSchemes(objs ...runtime.Object) *REST { func TestGetWorkloadsOperational_NoMonitors(t *testing.T) { r := newTestRESTWithSchemes() - ws, err := r.getWorkloadsOperational(context.TODO(), "default", "mydb") + ws, err := r.getWorkloadsOperational(context.TODO(), "default", "mydb", nil) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -83,7 +83,7 @@ func TestGetWorkloadsOperational_AllOperational(t *testing.T) { } r := newTestRESTWithSchemes(m1, m2) - ws, err := r.getWorkloadsOperational(context.TODO(), "default", "mydb") + ws, err := r.getWorkloadsOperational(context.TODO(), "default", "mydb", nil) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -126,7 +126,7 @@ func TestGetWorkloadsOperational_SomeNotOperational(t *testing.T) { } r := newTestRESTWithSchemes(m1, m2) - ws, err := r.getWorkloadsOperational(context.TODO(), "default", "mydb") + ws, err := r.getWorkloadsOperational(context.TODO(), "default", "mydb", nil) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -155,7 +155,7 @@ func TestGetWorkloadsOperational_OperationalNil(t *testing.T) { } r := newTestRESTWithSchemes(m) - ws, err := r.getWorkloadsOperational(context.TODO(), "default", "mydb") + ws, err := r.getWorkloadsOperational(context.TODO(), "default", "mydb", nil) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -198,7 +198,7 @@ func TestGetWorkloadsOperational_MixedNilAndOperational(t *testing.T) { } r := newTestRESTWithSchemes(m1, m2) - ws, err := r.getWorkloadsOperational(context.TODO(), "default", "mydb") + ws, err := r.getWorkloadsOperational(context.TODO(), "default", "mydb", nil) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -238,7 +238,7 @@ func TestGetWorkloadsOperational_MixedFailedAndPending(t *testing.T) { } r := newTestRESTWithSchemes(m1, m2) - ws, err := r.getWorkloadsOperational(context.TODO(), "default", "mydb") + ws, err := r.getWorkloadsOperational(context.TODO(), "default", "mydb", nil) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -296,7 +296,7 @@ func TestConvertConditions_MixedFailedAndPendingShowsFalse(t *testing.T) { {Type: "Ready", Status: metav1.ConditionTrue, Reason: "Succeeded", Message: "ok"}, } - app, err := r.convertHelmReleaseToApplication(context.TODO(), hr) + app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -328,7 +328,7 @@ func TestGetWorkloadsOperational_DifferentApp_NotFound(t *testing.T) { } r := newTestRESTWithSchemes(m) - ws, err := r.getWorkloadsOperational(context.TODO(), "default", "mydb") + ws, err := r.getWorkloadsOperational(context.TODO(), "default", "mydb", nil) if err != nil { t.Fatalf("unexpected error: %v", err) } From e6ba9817be1a42281e73dbedd68aa1ba0b2f9192 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 15 Apr 2026 19:47:25 +0500 Subject: [PATCH 044/250] [platform] Make vm-default-images opt-in, not default in iaas bundle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The package imports ~320Gi of golden-image PVCs (ubuntu-noble, fedora, debian, centos, etc.) as soon as it's installed. On small test and dev clusters that's enough to consume the entire replicated storage pool, after which no tenant PVCs — including the ones E2E itself provisions — can be bound. It's also unreasonable to force that cost on every iaas user: many deployments don't need prebuilt images at all, and the ones that do often want to curate their own subset. Switch the bundle entry from 'package.default' to 'package.optional.default', matching the treatment already applied to gpu-operator directly below it. Users who want the golden images can opt in via: bundles: enabledPackages: - cozystack.vm-default-images The package, its Source, and migration 38 all stay in place — nothing else changes for users who explicitly enable it. Users who previously relied on the bundle auto-installing it will need to add the package to enabledPackages on upgrade; this is called out in the release note. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Myasnikov Daniil --- packages/core/platform/templates/bundles/iaas.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/platform/templates/bundles/iaas.yaml b/packages/core/platform/templates/bundles/iaas.yaml index 7485eb91..ced0a322 100644 --- a/packages/core/platform/templates/bundles/iaas.yaml +++ b/packages/core/platform/templates/bundles/iaas.yaml @@ -8,7 +8,7 @@ {{- end -}} {{include "cozystack.platform.package" (list "cozystack.kubevirt" "default" $ $kubevirtComponents) }} {{include "cozystack.platform.package.default" (list "cozystack.kubevirt-cdi" $) }} -{{include "cozystack.platform.package.default" (list "cozystack.vm-default-images" $) }} +{{include "cozystack.platform.package.optional.default" (list "cozystack.vm-default-images" $) }} {{include "cozystack.platform.package.optional.default" (list "cozystack.gpu-operator" $) }} {{include "cozystack.platform.package.default" (list "cozystack.kamaji" $) }} {{include "cozystack.platform.package.default" (list "cozystack.capi-operator" $) }} From 8fc82b18c0fc40a58ab2f170abecf22527e943db Mon Sep 17 00:00:00 2001 From: tym83 <6355522@gmail.com> Date: Mon, 6 Apr 2026 20:14:59 +0500 Subject: [PATCH 045/250] Add Mattia Eleuteri (@mattia-eleuteri) as Maintainer Ref: #2343 Co-Authored-By: Claude Opus 4.6 (1M context) -e Signed-off-by: tym83 <6355522@gmail.com> --- MAINTAINERS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index e5c07714..d7061941 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -11,3 +11,4 @@ | Kirill Klinchenkov | [@klinch0](https://github.com/klinch0) | Ænix | Core Maintainer | | Nikita Bykov | [@nbykov0](https://github.com/nbykov0) | Ænix | Maintainer of ARM and stuff | | Matthieu Robin | [@matthieu-robin](https://github.com/matthieu-robin) | Hidora | Managed Applications, Platform Quality & Benchmarking | +| Mattia Eleuteri | [@mattia-eleuteri](https://github.com/mattia-eleuteri) | Hidora | CSI, Storage, Networking & Security | From 2e6a68541135681a95ee6edf2f7adbfa4514f84b Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 16 Apr 2026 00:34:52 +0300 Subject: [PATCH 046/250] docs(ci): require screenshots for UI changes in PR template Add a Screenshots section to the pull request template that requires contributors to attach screenshots or screen recordings when their changes affect the UI. Co-Authored-By: Claude Signed-off-by: ZverGuy --- .github/PULL_REQUEST_TEMPLATE.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 7710149c..b9475846 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -13,6 +13,11 @@ ## What this PR does +### Screenshots + + + ### Release note + +# Cozystack v1.3.0-rc.1 + +Cozystack v1.3.0-rc.1 is the first release candidate for v1.3.0, bringing **storage-aware scheduling** via the LINSTOR scheduler extender, a managed **LINSTOR GUI** web UI with Keycloak SSO, a **VM Default Images** catalog for out-of-the-box virtual machine provisioning, **WorkloadsReady conditions** with a real-time Events tab in the dashboard, and **cross-namespace VM backup restore** capabilities. Additional highlights include stricter tenant name validation, VM network selector improvements, Keycloak theme injection and SMTP configuration, and a comprehensive host runtime preflight check. + +> **Note:** Fixes marked with *(backported to v1.2.x)* were also included in v1.2.1 or v1.2.2 patch releases. + +## Feature Highlights + +### Storage-Aware Scheduling via LINSTOR Extender + +The `cozystack-scheduler` now calls the **LINSTOR scheduler extender** for storage-locality-aware pod placement. When a pod declares both a `SchedulingClass` and LINSTOR-backed PVCs, the scheduler consults LINSTOR to prefer nodes where volume replicas already exist — reducing cross-node replication traffic and improving I/O latency for storage-heavy workloads ([**@lllamnyp**](https://github.com/lllamnyp) in #2330). + +### LINSTOR GUI: Managed Web UI for Storage Administration + +A new opt-in `linstor-gui` system package deploys **LINBIT's linstor-gui web UI** alongside the LINSTOR controller with mTLS client authentication, non-root security context, and ClusterIP-only service. An optional **Keycloak-protected Ingress** (via oauth2-proxy) can be enabled for SSO-authenticated browser access when OIDC is configured on the platform ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2382, #2390). + +### VM Default Images: Out-of-the-Box VM Provisioning + +The new `vm-default-images` package provides a curated set of **cluster-wide virtual machine images** (Ubuntu, Debian, CentOS Stream, and others) as pre-populated DataVolumes. The package is opt-in via the `iaas` bundle and defaults to replicated storage for high availability. A companion migration (migration 38) renames legacy `vm-image-*` DataVolumes to the new `vm-default-images-*` naming scheme. The `vm-disk` chart also gains a new "disk" source type for cloning from existing vm-disks in the same namespace ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2258). + +### WorkloadsReady Condition and Events Tab + +Applications now expose a **WorkloadsReady** condition on their status by querying associated WorkloadMonitor resources, giving operators a single place to check whether all underlying workloads (Deployments, StatefulSets, DaemonSets) are healthy. The dashboard gains a new **Events tab** showing namespace-scoped Kubernetes events for each application, with fallback to `.firstTimestamp` when `.eventTime` is absent. A bug where WorkloadMonitor's `Operational` status was never persisted is also fixed ([**@lexfrei**](https://github.com/lexfrei) in #2356). + +### Cross-Namespace VM Backup Restore + +The backup system now supports **restoring VMInstance backups into a different namespace** (cross-namespace copy restores), with IP/MAC preservation and safe rename semantics. In-place backup/restores for VMDisk and VMInstance are improved: HelmReleases and DataVolumes are properly handled, and Velero failure messages are propagated to the Application status. The backup status structure has been refactored to store underlying resources as a generic opaque JSON object, enabling arbitrary application-specific metadata ([**@androndo**](https://github.com/androndo) in #2251, #2329, #2319). + +## Major Features and Improvements + +* **[api] Reject tenant names with dashes at Create time**: Enforces alphanumeric-only naming for Tenants at the API level, preventing names with hyphens that would silently fail during Helm reconciliation. A corresponding regex tightening and regression test suite hardens the validation ([**@lexfrei**](https://github.com/lexfrei) in #2380). + +* **[platform] Validate computed tenant namespace length**: Rejects Tenant creation when the computed ancestor-chain namespace would exceed the 63-character Kubernetes namespace limit, preventing opaque HelmRelease reconcile errors downstream ([**@lexfrei**](https://github.com/lexfrei) in #2376). + +* **[vm-instance] Rename subnets to networks and add dropdown selector**: Renames the misleading `subnets` field to `networks` in VMInstance for clarity, adds a dropdown selector for available networks in the dashboard form, and includes a migration to copy existing `subnets` values. The old field remains supported for backward compatibility ([**@sircthulhu**](https://github.com/sircthulhu) in #2263). + +* **[keycloak] Enable injecting themes**: Cozystack administrators can now inject custom Keycloak themes via `initContainers` for UI white-labeling and customization ([**@lllamnyp**](https://github.com/lllamnyp) in #2142). + +* **[keycloak-configure] Add email verification and SMTP configuration**: Adds configurable Keycloak settings for user self-registration, email verification, and SMTP server configuration, enabling automated user onboarding flows ([**@BROngineer**](https://github.com/BROngineer) in #2318). + +* **[postgres] Hardcode PostgreSQL 17 for monitoring databases**: Pins PostgreSQL 17.7 images for system databases (Grafana, Alerta, Harbor, Keycloak, SeaweedFS) and adds migration 37 to backfill `spec.version=v17` for existing PostgreSQL resources, preventing CNPG from defaulting to PostgreSQL 18 *(backported to v1.2.1)* ([**@IvanHunters**](https://github.com/IvanHunters) in #2304). + +* **[hack] Add host runtime preflight check**: New `check-host-runtime.sh` script and `make preflight` target that warns operators when a standalone containerd or docker runtime is running alongside the embedded k3s runtime, helping diagnose container runtime conflicts ([**@lexfrei**](https://github.com/lexfrei) in #2371). + +* **[hack] Add check-readiness.sh diagnostic script**: A new diagnostic script for tracking platform reconciliation by checking readiness of Packages, ArtifactGenerators, ExternalArtifacts, and HelmReleases, with support for watch mode and continuous monitoring ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2294). + +* **[mariadb] Always enable replication for consistent service naming**: MariaDB now always enables replication, creating `-primary`/`-secondary` services even for single-replica instances. This fixes dashboard visibility and backup functionality for single-replica setups ([**@sircthulhu**](https://github.com/sircthulhu) in #2279). + +* **[platform] Prevent installed packages deletion**: Adds `helm.sh/resource-policy: keep` annotation to packages, preventing automatic deletion when packages are disabled and restoring documented behavior *(backported to v1.2.1)* ([**@kvaps**](https://github.com/kvaps) in #2273). + +## Bug Fixes + +* **[cilium] Opt-out of cri-containerd.apparmor.d for nsenter init containers**: Opts cilium-agent init containers out of the `cri-containerd.apparmor.d` AppArmor profile on non-Talos variants, fixing `Init:CrashLoopBackOff` on Ubuntu 22.04+ and Debian *(backported to v1.2.2)* ([**@lexfrei**](https://github.com/lexfrei) in #2370). + +* **[virtual-machine] Exclude external VM services from Cilium BPF LB**: Adds `service-proxy-name: cozy-proxy` label to VM LoadBalancer services, telling Cilium to skip BPF processing. Fixes inter-tenant connectivity via public LB IPs and WholeIP functionality on Cilium 1.19+ *(backported to v1.2.2)* ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #2357). + +* **[monitoring] Fix infra dashboards missing in default variant**: Includes `cozy-monitoring` namespace in the dashboard rendering condition, fixing infrastructure Grafana dashboards not rendering in the default platform variant *(backported to v1.2.2)* ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #2365). + +* **[postgres] Fix system PostgreSQL images to 17.7-standard-trixie**: Normalizes system PostgreSQL image tags to use `17.7-standard-trixie` variant with migration logic for existing CNPG clusters *(backported to v1.2.2)* ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2364). + +* **[build] Filter git describe to match only v\* tags**: Adds `--match 'v*'` to `git describe` calls, preventing API subtags from being picked up instead of release tags and producing invalid Docker image tags *(backported to v1.2.2)* ([**@kvaps**](https://github.com/kvaps) in #2386). + +* **[platform] Fix resource allocation ratios not propagated to packages**: Restores propagation of CPU, memory, and ephemeral-storage allocation ratios to managed applications and KubeVirt, which were silently ignored since the bundle restructure *(backported to v1.2.1)* ([**@sircthulhu**](https://github.com/sircthulhu) in #2296). + +* **[kubernetes] Set explicit ephemeral-storage on virt-launcher pods**: Sets explicit `domain.resources` with ephemeral-storage on VirtualMachine spec to prevent virt-launcher pods from being evicted due to LimitRange defaults being too low for actual emptyDisk capacity ([**@kvaps**](https://github.com/kvaps) in #2317). + +* **[multus] Pin master CNI to 05-cilium.conflist**: Prevents a boot-time race condition where multus could auto-detect kube-ovn's conflist instead of Cilium's *(backported to v1.2.1)* ([**@kvaps**](https://github.com/kvaps) in #2315). + +* **[multus] Build custom image with DEL cache fix**: Fixes sandbox cleanup deadlock when CNI ADD never completes, preventing stale sandbox name reservations from permanently blocking pod creation *(backported to v1.2.1)* ([**@kvaps**](https://github.com/kvaps) in #2313). + +* **[linstor] Set verify-alg to crc32c**: Prevents DRBD connection failures on kernels where `crct10dif` is unavailable (e.g., Talos v1.12.6 with kernel 6.18.18) *(backported to v1.2.1)* ([**@kvaps**](https://github.com/kvaps) in #2303). + +* **[linstor] Preserve TCP ports during toggle-disk operations**: Fixes TCP port mismatches after toggle-disk operations that could cause DRBD resources to enter StandAlone state *(backported to v1.2.1)* ([**@kvaps**](https://github.com/kvaps) in #2292). + +## Dependencies & Version Updates + +* **[linstor] Update piraeus-server to v1.33.2 with selected backports**: Bumps LINSTOR server from v1.33.1 to v1.33.2 with backported patches for stale bitmap adjust retry, LUKS2 header sizing, and optimal I/O size detection *(backported to v1.2.2)* ([**@kvaps**](https://github.com/kvaps) in #2331). + +* **[kamaji] Update to 26.3.5-edge, drop upstreamed patches**: Updates Kamaji from edge-26.2.4 to 26.3.5-edge and removes two patches accepted upstream. Adds configurable probe tuning and DataStore readiness conditions ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2260). + +* **[talm] Release v0.23.0, v0.23.1, v0.24.0** (github.com/cozystack/talm): Migrates to the Talos v1.12 multi-document machine config format ([**@lexfrei**](https://github.com/lexfrei) in cozystack/talm#116); fixes template rendering in `apply` command to resolve lookups ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/talm#119); bumps dependencies and modernizes codebase ([**@lexfrei**](https://github.com/lexfrei) in cozystack/talm#124). + +* **[ansible-cozystack] Release v1.2.1, v1.2.2** (github.com/cozystack/ansible-cozystack): Exposes `publishing.externalIPs` and tenant-root ingress via role variables ([**@lexfrei**](https://github.com/lexfrei) in cozystack/ansible-cozystack#30); adds comprehensive node prerequisites audit ([**@lexfrei**](https://github.com/lexfrei) in cozystack/ansible-cozystack#27); replaces `ansible.utils.ipaddr` with a stdlib-based test plugin ([**@lexfrei**](https://github.com/lexfrei) in cozystack/ansible-cozystack#24). + +## Security + +* **docs: add SECURITY.md**: Adds vulnerability reporting procedures, disclosure expectations, and supported release lines ([**@kvaps**](https://github.com/kvaps) in #2230). + +* **docs: add OpenSSF Best Practices badge to README**: Adds the OpenSSF Best Practices passing badge to the project README ([**@lexfrei**](https://github.com/lexfrei) in #2320). + +## Development, Testing, and CI/CD + +* **[ci] Replace cozystack-bot PAT with cozystack-ci GitHub App**: Replaces the long-lived cozystack-bot personal access token with short-lived, scoped tokens from the cozystack-ci GitHub App across all CI release workflows ([**@tym83**](https://github.com/tym83) in #2351; [**@kvaps**](https://github.com/kvaps) in #2383, #2392). + +* **[ci] Add Gemini Code Assist and CodeRabbit configuration**: Adds repository-level configuration for AI code reviewers with ignore patterns for vendored/generated code and incremental review settings ([**@lexfrei**](https://github.com/lexfrei) in #2385). + +* **[ci] Make tags workflow idempotent on re-runs**: Fixes CI to force-update API subtags and handle re-runs gracefully ([**@kvaps**](https://github.com/kvaps)). + +* **[tests] Fix Kafka E2E test timeout and retry race condition**: Increases Kafka E2E test timeout from 60s to 300s and fixes a retry race condition where `kubectl apply` could hit a still-deleting resource ([**@lexfrei**](https://github.com/lexfrei) in #2358). + +* **docs: adopt Conventional Commits for commit and PR titles**: Standardizes commit and PR title format to `type(scope): description` across all contributing docs and the PR template ([**@lexfrei**](https://github.com/lexfrei) in #2395). + +* **docs(ci): require screenshots for UI changes in PR template**: Adds a mandatory screenshots section to the PR template for UI-related changes ([**@kitsunoff**](https://github.com/kitsunoff) in #2407). + +## Documentation + +* **[website] Add ApplicationDefinition naming convention reference**: Documents how `cozystack-api` resolves kinds to their backing definitions ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#478). + +* **[website] Document Talos / talosctl / Cozystack version pairing**: Adds version compatibility matrix for installation ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#484). + +* **[website] Document namespace layout and parent/child derivation**: Explains tenant namespace hierarchy and parent/child namespace derivation rules ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#479). + +* **[website] Document the checkbox-then-edit-CR customization pattern for tenants**: Describes the workflow for customizing tenant settings via the CR after initial checkbox-based creation ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#485). + +* **[website] Add custom Keycloak themes documentation**: Covers the theme image contract, configuration, `imagePullSecrets`, and theme activation in the Keycloak admin console ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#463). + +* **[website] Add bonding (LACP) configuration how-to guide**: Covers network bonding configuration for Cozystack installations ([**@sircthulhu**](https://github.com/sircthulhu) in cozystack/website#459). + +* **[website] Improve registry mirrors for tenant Kubernetes in air-gapped guide**: Improved documentation for configuring registry mirrors in air-gapped environments ([**@sircthulhu**](https://github.com/sircthulhu) in cozystack/website#461). + +* **[website] Rewrite guide for ApplicationDefinition API (external-apps)**: Comprehensive rewrite of the external apps guide using the ApplicationDefinition API ([**@kitsunoff**](https://github.com/kitsunoff) in cozystack/website#488). + +* **[website] Add documentation for Go types usage**: Guide for using generated Go types for Cozystack managed applications as a Go module ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#465). + +* **[website] Update backup/restore documentation for VMI/VMDisk**: Updated backup documentation with VM instance and VM disk restore improvements ([**@androndo**](https://github.com/androndo) in cozystack/website#466). + +* **[website] Add OSS Health pages and OpenSSF badge**: Added OSS Health section with OpenSSF Scorecard and Best Practices badge to the website ([**@tym83**](https://github.com/tym83) in cozystack/website#470). + +* **[website] Add CozySummit Virtual 2026 program announcement**: Published the CozySummit Virtual 2026 program announcement blog post ([**@tym83**](https://github.com/tym83) in cozystack/website#472). + +* **[website] Add missing release announcements for v0.1–v0.41**: Backfilled missing release announcement blog posts for historical Cozystack versions ([**@tym83**](https://github.com/tym83) in cozystack/website#468). + +* **[website] Fix broken links and stale anchors across v1 docs**: Fixes 14 broken links and stale talm anchors ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#486). + +* **[website] Prefix bundle package names with cozystack. in v1 examples**: Corrects package naming in documentation examples ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#482). + +* **[website] Finish isolated-field removal and document opt-in policy labels**: Removes obsolete `isolated` field from tenant documentation and documents the new approach ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#481). + +* **[website] Add --take-ownership flag and describe networking.* fields**: Documents the `--take-ownership` flag and `networking.*` fields in the installation guide ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#480). + +* **[website] Fix KubeOVN MASTER_NODES example path and key in troubleshooting**: Corrects the MASTER_NODES example path ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#483). + +* **[external-apps-example] Replace MongoDB example with Minecraft apps**: Refactors the external apps example to use ApplicationDefinition API with Minecraft server applications ([**@lexfrei**](https://github.com/lexfrei) in cozystack/external-apps-example#2). + +## Governance + +* **Add Mattia Eleuteri ([@mattia-eleuteri](https://github.com/mattia-eleuteri)) as Maintainer**: CSI, Storage, Networking & Security ([**@tym83**](https://github.com/tym83) in #2345). + +* **Add Matthieu Robin ([@matthieu-robin](https://github.com/matthieu-robin)) as Maintainer**: Managed applications, platform quality, and benchmarking ([**@tym83**](https://github.com/tym83) in #2346). + +## Contributors + +We'd like to thank all contributors who made this release possible: + +* [**@androndo**](https://github.com/androndo) +* [**@BROngineer**](https://github.com/BROngineer) +* [**@IvanHunters**](https://github.com/IvanHunters) +* [**@kitsunoff**](https://github.com/kitsunoff) +* [**@kvaps**](https://github.com/kvaps) +* [**@lexfrei**](https://github.com/lexfrei) +* [**@lllamnyp**](https://github.com/lllamnyp) +* [**@mattia-eleuteri**](https://github.com/mattia-eleuteri) +* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) +* [**@sircthulhu**](https://github.com/sircthulhu) +* [**@tym83**](https://github.com/tym83) + +--- + +**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.2.0...v1.3.0-rc.1 From 1e1bb3eb37e0cc3068fd87171f9c8c1ec884c0d6 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Thu, 16 Apr 2026 13:18:18 +0200 Subject: [PATCH 051/250] docs(agents): add external-apps-example and ansible-cozystack to changelog instructions Co-Authored-By: Claude Signed-off-by: Andrei Kvapil --- docs/agents/changelog.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/agents/changelog.md b/docs/agents/changelog.md index feb70597..35d59a00 100644 --- a/docs/agents/changelog.md +++ b/docs/agents/changelog.md @@ -22,7 +22,7 @@ When the user asks to generate a changelog, follow these steps in the specified - [ ] Step 5: Get the list of commits for the release period - [ ] Step 6: Check additional repositories (website is REQUIRED, optional repos if tags exist) - [ ] **MANDATORY**: Check website repository for documentation changes WITH authors and PR links via GitHub CLI - - [ ] **MANDATORY**: Check ALL optional repositories (talm, boot-to-talos, cozyhr, cozy-proxy) for tags during release period + - [ ] **MANDATORY**: Check ALL optional repositories (talm, boot-to-talos, cozyhr, cozy-proxy, external-apps-example, ansible-cozystack) for tags during release period - [ ] **MANDATORY**: For ALL commits from additional repos, get GitHub username via CLI, prioritizing PR author over commit author. - [ ] Step 7: Analyze commits (extract PR numbers, authors, user impact) - [ ] **MANDATORY**: For EVERY PR in main repo, get PR author via `gh pr view --json author --jq .author.login` (do NOT skip this step) @@ -148,6 +148,8 @@ Cozystack release may include changes from related repositories. Check and inclu - [https://github.com/cozystack/boot-to-talos](https://github.com/cozystack/boot-to-talos) - [https://github.com/cozystack/cozyhr](https://github.com/cozystack/cozyhr) - [https://github.com/cozystack/cozy-proxy](https://github.com/cozystack/cozy-proxy) +- [https://github.com/cozystack/external-apps-example](https://github.com/cozystack/external-apps-example) +- [https://github.com/cozystack/ansible-cozystack](https://github.com/cozystack/ansible-cozystack) **⚠️ IMPORTANT**: You MUST check ALL optional repositories for tags created during the release period. Do NOT skip this step even if you think there might not be any tags. Use the process below to verify. @@ -195,7 +197,7 @@ Cozystack release may include changes from related repositories. Check and inclu 3. **For optional repositories, check if tags exist during release period:** - **⚠️ MANDATORY: You MUST check ALL optional repositories (talm, boot-to-talos, cozyhr, cozy-proxy). Do NOT skip any repository!** + **⚠️ MANDATORY: You MUST check ALL optional repositories (talm, boot-to-talos, cozyhr, cozy-proxy, external-apps-example, ansible-cozystack). Do NOT skip any repository!** **Use the helper script:** ```bash @@ -208,7 +210,7 @@ Cozystack release may include changes from related repositories. Check and inclu ``` The script will: - - Check ALL optional repositories (talm, boot-to-talos, cozyhr, cozy-proxy) + - Check ALL optional repositories (talm, boot-to-talos, cozyhr, cozy-proxy, external-apps-example, ansible-cozystack) - Look for tags created during the release period - Get commits between tags (if tags exist) or by date range (if no tags) - Extract PR numbers from commit messages @@ -569,7 +571,7 @@ Create a new changelog file in the format matching previous versions: - [ ] Step 5 completed: **ALL commits included** (including merge commits and backports) - do not skip any commits - [ ] Step 5 completed: **Backports identified and handled correctly** - original PR author used, both original and backport PR numbers included - [ ] Step 6 completed: Website repository checked for documentation changes WITH authors and PR links via GitHub CLI -- [ ] Step 6 completed: **ALL** optional repositories (talm, boot-to-talos, cozyhr, cozy-proxy) checked for tags during release period +- [ ] Step 6 completed: **ALL** optional repositories (talm, boot-to-talos, cozyhr, cozy-proxy, external-apps-example, ansible-cozystack) checked for tags during release period - [ ] Step 6 completed: For ALL commits from additional repos, GitHub username obtained via GitHub CLI (not skipped). For commits with PR numbers, PR author used via `gh pr view` (not commit author) - [ ] Step 7 completed: For EVERY PR in main repo (including backports), PR author obtained via `gh pr view --json author --jq .author.login` (not skipped or assumed). Commit author NOT used - always use PR author - [ ] Step 7 completed: **Backports verified** - for each backport PR, original PR found and original PR author used in changelog @@ -628,7 +630,7 @@ Save the changelog to file `docs/changelogs/v.md` according to the vers - **Additional repositories (Step 6) - MANDATORY**: - **⚠️ CRITICAL**: Always check the **website** repository for documentation changes during the release period. This is a required step and MUST NOT be skipped. - - **⚠️ CRITICAL**: You MUST check ALL optional repositories (talm, boot-to-talos, cozyhr, cozy-proxy) for tags during the release period. Do NOT skip any repository even if you think there might not be tags. + - **⚠️ CRITICAL**: You MUST check ALL optional repositories (talm, boot-to-talos, cozyhr, cozy-proxy, external-apps-example, ansible-cozystack) for tags during the release period. Do NOT skip any repository even if you think there might not be tags. - **CRITICAL**: For ALL entries from additional repositories (website and optional), you MUST: - **MANDATORY**: Extract PR number from commit message first - **MANDATORY**: For commits with PR numbers, ALWAYS use `gh pr view --repo cozystack/ --json author --jq .author.login` to get PR author (not commit author) @@ -637,7 +639,7 @@ Save the changelog to file `docs/changelogs/v.md` according to the vers - **MANDATORY**: Do NOT use commit author for PRs - always use PR author - Include PR link or commit hash reference - Format: `* **[repo] Description**: details ([**@username**](https://github.com/username) in cozystack/repo#123)` - - For **optional repositories** (talm, boot-to-talos, cozyhr, cozy-proxy), you MUST check ALL of them for tags during the release period. Use the loop provided in Step 6 to check each repository systematically. + - For **optional repositories** (talm, boot-to-talos, cozyhr, cozy-proxy, external-apps-example, ansible-cozystack), you MUST check ALL of them for tags during the release period. Use the loop provided in Step 6 to check each repository systematically. - When including changes from additional repositories, use the format: `[repo-name] Description` and link to the repository's PR/issue if available - **Prefer PR numbers over commit hashes**: For commits from additional repositories, extract PR number from commit message using GitHub API. Use PR format (`cozystack/website#123`) instead of commit hash (`cozystack/website@abc1234`) when available - **Never add entries without author and PR/commit reference**: Every entry from additional repositories must have both author and link From 7797f4956918f1631c52714034ee1690b8fd3fd3 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 16 Apr 2026 20:20:41 +0300 Subject: [PATCH 052/250] test(api): assert parent HelmRelease Install/Upgrade Timeout >= 15m Adds a failing unit test for convertApplicationToHelmRelease asserting that Install.Timeout and Upgrade.Timeout are at least 15 minutes. The default flux helm-controller timeout is too short to cover cold-start Kamaji control-plane bootstrap (image pull + etcd bootstrap + apiserver Ready + admin-kubeconfig Secret generation) and causes install remediation loops. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- .../apps/application/rest_timeout_test.go | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 pkg/registry/apps/application/rest_timeout_test.go diff --git a/pkg/registry/apps/application/rest_timeout_test.go b/pkg/registry/apps/application/rest_timeout_test.go new file mode 100644 index 00000000..8ae13183 --- /dev/null +++ b/pkg/registry/apps/application/rest_timeout_test.go @@ -0,0 +1,53 @@ +package application + +import ( + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" + "github.com/cozystack/cozystack/pkg/config" +) + +func TestConvertApplicationToHelmRelease_SetsInstallAndUpgradeTimeout(t *testing.T) { + r := &REST{ + releaseConfig: config.ReleaseConfig{ + Prefix: "kubernetes-", + ChartRef: config.ChartRefConfig{ + Kind: "HelmChart", + Name: "kubernetes", + Namespace: "cozy-system", + }, + }, + } + + app := &appsv1alpha1.Application{ + ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "tenant-root"}, + } + + hr, err := r.convertApplicationToHelmRelease(app) + if err != nil { + t.Fatalf("convertApplicationToHelmRelease returned error: %v", err) + } + + if hr.Spec.Install == nil { + t.Fatal("Spec.Install must not be nil") + } + if hr.Spec.Install.Timeout == nil { + t.Fatal("Spec.Install.Timeout must be set to cover async admin-kubeconfig provisioning") + } + if hr.Spec.Install.Timeout.Duration < 15*time.Minute { + t.Errorf("Spec.Install.Timeout must be >= 15m (cold bootstrap budget), got %v", hr.Spec.Install.Timeout.Duration) + } + + if hr.Spec.Upgrade == nil { + t.Fatal("Spec.Upgrade must not be nil") + } + if hr.Spec.Upgrade.Timeout == nil { + t.Fatal("Spec.Upgrade.Timeout must be set to cover async admin-kubeconfig provisioning") + } + if hr.Spec.Upgrade.Timeout.Duration < 15*time.Minute { + t.Errorf("Spec.Upgrade.Timeout must be >= 15m (cold bootstrap budget), got %v", hr.Spec.Upgrade.Timeout.Duration) + } +} From e4f279f8e2cb08e271130b4a5193d9e0e9d9d3a7 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 16 Apr 2026 20:21:15 +0300 Subject: [PATCH 053/250] fix(api): set 15m Install/Upgrade Timeout for parent HelmRelease Parent HelmRelease created by cozystack-api for Kubernetes tenants contains CAPI/Kamaji resources (Cluster, KamajiControlPlane, MachineDeployment) that asynchronously provision the *-admin-kubeconfig Secret. Three Deployments in the same chart (cluster-autoscaler, kccm, kcsi-controller) mount that Secret directly, so the helm-wait cannot complete until control-plane bootstrap finishes. Default flux helm-controller timeout is too short for a cold-node first-tenant bootstrap (image pull + etcd bootstrap + apiserver Ready + admin-kubeconfig generation routinely exceed it). On timeout, install.remediation triggers uninstall, which removes the Cluster CR and restarts the cycle indefinitely. Bumping Install.Timeout and Upgrade.Timeout to 15m gives realistic bootstrap headroom while remaining bounded. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- pkg/registry/apps/application/rest.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/registry/apps/application/rest.go b/pkg/registry/apps/application/rest.go index 0728ea13..77d1d458 100644 --- a/pkg/registry/apps/application/rest.go +++ b/pkg/registry/apps/application/rest.go @@ -1509,11 +1509,13 @@ func (r *REST) convertApplicationToHelmRelease(app *appsv1alpha1.Application) (* }, Interval: metav1.Duration{Duration: 5 * time.Minute}, Install: &helmv2.Install{ + Timeout: &metav1.Duration{Duration: 15 * time.Minute}, Remediation: &helmv2.InstallRemediation{ Retries: -1, }, }, Upgrade: &helmv2.Upgrade{ + Timeout: &metav1.Duration{Duration: 15 * time.Minute}, Remediation: &helmv2.UpgradeRemediation{ Retries: -1, }, From 3e26234a1c98f138542471a7f4395532e15c7b46 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 16 Apr 2026 20:24:20 +0300 Subject: [PATCH 054/250] test(kubernetes): assert admin-kubeconfig wait pattern and etcd guard Adds failing helm unittest suite for packages/apps/kubernetes covering: - cluster-autoscaler, kccm, and csi controller Deployments mount the admin-kubeconfig Secret with optional: true - each of those Deployments has a wait-for-kubeconfig initContainer that mounts the same kubeconfig path - cluster.yaml renders a helm fail with a descriptive message when the tenant has no etcd DataStore (empty _namespace.etcd) Also wires up a test target in the chart Makefile so helm-unit-tests.sh picks it up. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- packages/apps/kubernetes/Makefile | 3 + .../tests/admin_kubeconfig_wait_test.yaml | 85 +++++++++++++++++++ packages/apps/kubernetes/tests/values-ci.yaml | 9 ++ 3 files changed, 97 insertions(+) create mode 100644 packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml create mode 100644 packages/apps/kubernetes/tests/values-ci.yaml diff --git a/packages/apps/kubernetes/Makefile b/packages/apps/kubernetes/Makefile index 01cf736d..0f9e6d57 100644 --- a/packages/apps/kubernetes/Makefile +++ b/packages/apps/kubernetes/Makefile @@ -4,6 +4,9 @@ KUBERNETES_PKG_TAG = $(shell awk '$$1 == "version:" {print $$2}' Chart.yaml) include ../../../hack/common-envs.mk include ../../../hack/package.mk +test: + helm unittest . + generate: cozyvalues-gen -m 'kubernetes' -v values.yaml -s values.schema.json -r README.md -g ../../../api/apps/v1alpha1/kubernetes/types.go ../../../hack/update-crd.sh diff --git a/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml b/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml new file mode 100644 index 00000000..e2d9f359 --- /dev/null +++ b/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml @@ -0,0 +1,85 @@ +suite: admin-kubeconfig wait guards + +release: + name: test + namespace: tenant-root + +values: + - values-ci.yaml + +tests: + - it: cluster-autoscaler mounts admin-kubeconfig as optional + template: templates/cluster-autoscaler/deployment.yaml + documentSelector: + path: kind + value: Deployment + asserts: + - equal: + path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.secretName + value: test-admin-kubeconfig + - equal: + path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.optional + value: true + + - it: cluster-autoscaler waits for admin-kubeconfig via initContainer + template: templates/cluster-autoscaler/deployment.yaml + documentSelector: + path: kind + value: Deployment + asserts: + - equal: + path: spec.template.spec.initContainers[0].name + value: wait-for-kubeconfig + - contains: + path: spec.template.spec.initContainers[0].volumeMounts + content: + name: kubeconfig + mountPath: /etc/kubernetes/kubeconfig + readOnly: true + + - it: kccm mounts admin-kubeconfig as optional + template: templates/kccm/manager.yaml + asserts: + - equal: + path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.secretName + value: test-admin-kubeconfig + - equal: + path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.optional + value: true + + - it: kccm waits for admin-kubeconfig via initContainer + template: templates/kccm/manager.yaml + asserts: + - equal: + path: spec.template.spec.initContainers[0].name + value: wait-for-kubeconfig + + - it: csi controller mounts admin-kubeconfig as optional + template: templates/csi/deploy.yaml + asserts: + - equal: + path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.secretName + value: test-admin-kubeconfig + - equal: + path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.optional + value: true + + - it: csi controller waits for admin-kubeconfig via initContainer + template: templates/csi/deploy.yaml + asserts: + - equal: + path: spec.template.spec.initContainers[0].name + value: wait-for-kubeconfig + + - it: cluster.yaml fails render when tenant has no etcd DataStore + template: templates/cluster.yaml + set: + _namespace: + etcd: "" + monitoring: "" + ingress: "" + seaweedfs: "" + host: "" + asserts: + - failedTemplate: + errorPattern: "requires a parent tenant with etcd enabled" diff --git a/packages/apps/kubernetes/tests/values-ci.yaml b/packages/apps/kubernetes/tests/values-ci.yaml new file mode 100644 index 00000000..13365e8c --- /dev/null +++ b/packages/apps/kubernetes/tests/values-ci.yaml @@ -0,0 +1,9 @@ +_namespace: + etcd: tenant-root + monitoring: "" + ingress: "" + seaweedfs: "" + host: "" +_cluster: + cluster-domain: cozy.local +nodeGroups: null From ca33cc4e3c4b032293fceec2b7d3c1eb13a241bb Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 16 Apr 2026 20:25:42 +0300 Subject: [PATCH 055/250] fix(kubernetes): wait for admin-kubeconfig before starting CP-side pods Three Deployments in the Kubernetes app chart mount the tenant admin-kubeconfig Secret directly as a volume: cluster-autoscaler, kccm, and the kcsi controller. That Secret is provisioned asynchronously by Kamaji after control-plane bootstrap, so on a fresh install the pods used to hit FailedMount and the parent HelmRelease ran out of its wait budget. Mark the Secret volume optional and add a wait-for-kubeconfig initContainer that polls the mounted path until the Secret appears. Kubelet remounts the optional Secret within its sync period once Kamaji publishes it, the init container exits, and the main container starts cleanly. The Deployment becomes Available without the helm-wait ever seeing a FailedMount. Pins a busybox image for the init container via images/busybox.tag (same format as the other pinned tags in this chart). Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- packages/apps/kubernetes/images/busybox.tag | 1 + .../cluster-autoscaler/deployment.yaml | 17 +++++++++++++++++ .../apps/kubernetes/templates/csi/deploy.yaml | 17 +++++++++++++++++ .../apps/kubernetes/templates/kccm/manager.yaml | 17 +++++++++++++++++ 4 files changed, 52 insertions(+) create mode 100644 packages/apps/kubernetes/images/busybox.tag diff --git a/packages/apps/kubernetes/images/busybox.tag b/packages/apps/kubernetes/images/busybox.tag new file mode 100644 index 00000000..e358c12e --- /dev/null +++ b/packages/apps/kubernetes/images/busybox.tag @@ -0,0 +1 @@ +busybox:1.37.0@sha256:1487d0af5f52b4ba31c7e465126ee2123fe3f2305d638e7827681e7cf6c83d5e diff --git a/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml b/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml index a00e0155..47c22f76 100644 --- a/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml +++ b/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml @@ -23,6 +23,22 @@ spec: - key: node-role.kubernetes.io/control-plane operator: Exists effect: "NoSchedule" + initContainers: + - name: wait-for-kubeconfig + image: "{{ $.Files.Get "images/busybox.tag" | trim }}" + command: + - sh + - -c + - | + set -eu + until [ -s /etc/kubernetes/kubeconfig/super-admin.svc ]; do + echo "waiting for admin kubeconfig to be provisioned by Kamaji..." + sleep 5 + done + volumeMounts: + - name: kubeconfig + mountPath: /etc/kubernetes/kubeconfig + readOnly: true containers: - image: "{{ $.Files.Get "images/cluster-autoscaler.tag" | trim }}" name: cluster-autoscaler @@ -56,6 +72,7 @@ spec: name: cloud-config - secret: secretName: {{ .Release.Name }}-admin-kubeconfig + optional: true name: kubeconfig serviceAccountName: {{ .Release.Name }}-cluster-autoscaler terminationGracePeriodSeconds: 10 diff --git a/packages/apps/kubernetes/templates/csi/deploy.yaml b/packages/apps/kubernetes/templates/csi/deploy.yaml index 938b6d67..2979c75c 100644 --- a/packages/apps/kubernetes/templates/csi/deploy.yaml +++ b/packages/apps/kubernetes/templates/csi/deploy.yaml @@ -24,6 +24,22 @@ spec: - key: node-role.kubernetes.io/control-plane operator: Exists effect: "NoSchedule" + initContainers: + - name: wait-for-kubeconfig + image: "{{ $.Files.Get "images/busybox.tag" | trim }}" + command: + - sh + - -c + - | + set -eu + until [ -s /etc/kubernetes/kubeconfig/super-admin.svc ]; do + echo "waiting for admin kubeconfig to be provisioned by Kamaji..." + sleep 5 + done + volumeMounts: + - name: kubeconfig + mountPath: /etc/kubernetes/kubeconfig + readOnly: true containers: - name: csi-driver imagePullPolicy: Always @@ -234,4 +250,5 @@ spec: emptyDir: {} - secret: secretName: {{ .Release.Name }}-admin-kubeconfig + optional: true name: kubeconfig diff --git a/packages/apps/kubernetes/templates/kccm/manager.yaml b/packages/apps/kubernetes/templates/kccm/manager.yaml index 81426d4e..5c7f15a4 100644 --- a/packages/apps/kubernetes/templates/kccm/manager.yaml +++ b/packages/apps/kubernetes/templates/kccm/manager.yaml @@ -22,6 +22,22 @@ spec: - key: node-role.kubernetes.io/control-plane operator: Exists effect: "NoSchedule" + initContainers: + - name: wait-for-kubeconfig + image: "{{ $.Files.Get "images/busybox.tag" | trim }}" + command: + - sh + - -c + - | + set -eu + until [ -s /etc/kubernetes/kubeconfig/super-admin.svc ]; do + echo "waiting for admin kubeconfig to be provisioned by Kamaji..." + sleep 5 + done + volumeMounts: + - name: kubeconfig + mountPath: /etc/kubernetes/kubeconfig + readOnly: true containers: - name: kubevirt-cloud-controller-manager args: @@ -55,5 +71,6 @@ spec: name: cloud-config - secret: secretName: {{ .Release.Name }}-admin-kubeconfig + optional: true name: kubeconfig serviceAccountName: {{ .Release.Name }}-kccm From cac514b60fc00717df37780d2183e1702b07e9c6 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 16 Apr 2026 20:26:09 +0300 Subject: [PATCH 056/250] fix(kubernetes): fail fast when tenant has no etcd DataStore When a Kubernetes tenant is created without a parent tenant that has etcd enabled, .Values._namespace.etcd is empty and the rendered KamajiControlPlane spec carries an empty dataStoreName. The Kamaji admission webhook then rejects every TenantControlPlane create with "tenant-root DataStore does not exist" and the control plane never comes up. Add a helm template-level guard that fails rendering with a descriptive, actionable error message before the HelmRelease even reaches the webhook. This also closes the narrow race where a Kubernetes HelmRelease reconciles before the etcd HelmRelease has created the DataStore CR - flux retries on its interval and picks up the DataStore once it appears. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- packages/apps/kubernetes/templates/cluster.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/apps/kubernetes/templates/cluster.yaml b/packages/apps/kubernetes/templates/cluster.yaml index 10d6fd80..99249c1a 100644 --- a/packages/apps/kubernetes/templates/cluster.yaml +++ b/packages/apps/kubernetes/templates/cluster.yaml @@ -1,4 +1,7 @@ {{- $etcd := .Values._namespace.etcd }} +{{- if not $etcd }} +{{- fail "Kubernetes tenant requires a parent tenant with etcd enabled: set .Values.etcd=true on tenant-root (or any ancestor tenant), wait for the etcd HelmRelease to reconcile (DataStore CR appears in the tenant namespace), then retry." }} +{{- end }} {{- $ingress := .Values._namespace.ingress }} {{- $host := .Values._namespace.host }} {{- $kubevirtmachinetemplateNames := list }} From a7d994365d067dee5d9546a9b95bb555b206a6af Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 16 Apr 2026 20:26:47 +0300 Subject: [PATCH 057/250] test(kubernetes): assert parent HelmRelease did not remediate in e2e Before cleanup, inspect the parent HelmRelease installFailures and upgradeFailures counters. A non-zero value means flux helm-controller hit its wait timeout, ran install/upgrade remediation (uninstall), and re-installed - the exact race condition this PR closes. Fail the bats test in that case so the signal surfaces in CI instead of being masked by a green retry. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- hack/e2e-apps/run-kubernetes.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/hack/e2e-apps/run-kubernetes.sh b/hack/e2e-apps/run-kubernetes.sh index be6dcd6f..f05ec370 100644 --- a/hack/e2e-apps/run-kubernetes.sh +++ b/hack/e2e-apps/run-kubernetes.sh @@ -320,6 +320,19 @@ EOF done kubectl wait hr kubernetes-${test_name}-ingress-nginx -n tenant-test --timeout=5m --for=condition=ready + # Guard: parent HelmRelease must not have entered an install/upgrade remediation cycle. + # A non-zero installFailures/upgradeFailures indicates the helm-wait budget expired while + # admin-kubeconfig was still being provisioned, which would trigger uninstall remediation + # and churn the Cluster CR. + install_failures=$(kubectl get hr -n tenant-test "kubernetes-${test_name}" -ojsonpath='{.status.installFailures}') + upgrade_failures=$(kubectl get hr -n tenant-test "kubernetes-${test_name}" -ojsonpath='{.status.upgradeFailures}') + if [ "${install_failures:-0}" != "0" ] && [ -n "${install_failures}" ] || \ + [ "${upgrade_failures:-0}" != "0" ] && [ -n "${upgrade_failures}" ]; then + echo "Parent HelmRelease entered remediation cycle: installFailures=${install_failures:-0}, upgradeFailures=${upgrade_failures:-0}" >&2 + kubectl -n tenant-test describe hr "kubernetes-${test_name}" >&2 + exit 1 + fi + # Clean up pkill -f "port-forward.*${port}:" 2>/dev/null || true rm -f "tenantkubeconfig-${test_name}" From f87834a3edba32945f22e8d419a9592e5303b34b Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 16 Apr 2026 20:48:47 +0300 Subject: [PATCH 058/250] fix(hack): group e2e remediation guard conditions correctly Shell && and || have equal precedence and left-to-right associativity, so the previous guard parsed as (((A && B) || C) && D) and silently passed on the canonical failure mode: install_failures=1 with an empty upgrade_failures. Extract the check into helmrelease_has_remediation_cycle() in a dedicated helper sourced from run-kubernetes.sh, and add unit tests under hack/remediation-guard.bats that pin the expected behavior for every combination of empty, zero, and positive counter values. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- hack/e2e-apps/remediation-guard.sh | 24 +++++++++ hack/e2e-apps/run-kubernetes.sh | 5 +- hack/remediation-guard.bats | 87 ++++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 2 deletions(-) create mode 100644 hack/e2e-apps/remediation-guard.sh create mode 100644 hack/remediation-guard.bats diff --git a/hack/e2e-apps/remediation-guard.sh b/hack/e2e-apps/remediation-guard.sh new file mode 100644 index 00000000..6da69f13 --- /dev/null +++ b/hack/e2e-apps/remediation-guard.sh @@ -0,0 +1,24 @@ +# Helpers for asserting that a Flux HelmRelease did not fall into an +# install/upgrade remediation cycle during an e2e run. +# +# A non-zero installFailures/upgradeFailures counter means flux +# helm-controller hit its wait timeout, ran remediation (uninstall), +# and re-installed. That is exactly the race this guard is meant to +# catch, so the function returns success (0) when a cycle is detected +# and failure (1) otherwise. +# +# Both arguments may be empty strings, the literal "0", or a positive +# integer. Shell's && and || have equal precedence with left-to-right +# associativity, so each half of the disjunction is grouped explicitly +# to avoid (A && B) || C && D parsing that masks the common +# install_failures=1, upgrade_failures="" case. + +helmrelease_has_remediation_cycle() { + install_failures="$1" + upgrade_failures="$2" + if { [ -n "${install_failures}" ] && [ "${install_failures}" != "0" ]; } || \ + { [ -n "${upgrade_failures}" ] && [ "${upgrade_failures}" != "0" ]; }; then + return 0 + fi + return 1 +} diff --git a/hack/e2e-apps/run-kubernetes.sh b/hack/e2e-apps/run-kubernetes.sh index f05ec370..811f42a7 100644 --- a/hack/e2e-apps/run-kubernetes.sh +++ b/hack/e2e-apps/run-kubernetes.sh @@ -1,3 +1,5 @@ +. hack/e2e-apps/remediation-guard.sh + run_kubernetes_test() { local version_expr="$1" local test_name="$2" @@ -326,8 +328,7 @@ EOF # and churn the Cluster CR. install_failures=$(kubectl get hr -n tenant-test "kubernetes-${test_name}" -ojsonpath='{.status.installFailures}') upgrade_failures=$(kubectl get hr -n tenant-test "kubernetes-${test_name}" -ojsonpath='{.status.upgradeFailures}') - if [ "${install_failures:-0}" != "0" ] && [ -n "${install_failures}" ] || \ - [ "${upgrade_failures:-0}" != "0" ] && [ -n "${upgrade_failures}" ]; then + if helmrelease_has_remediation_cycle "${install_failures}" "${upgrade_failures}"; then echo "Parent HelmRelease entered remediation cycle: installFailures=${install_failures:-0}, upgradeFailures=${upgrade_failures:-0}" >&2 kubectl -n tenant-test describe hr "kubernetes-${test_name}" >&2 exit 1 diff --git a/hack/remediation-guard.bats b/hack/remediation-guard.bats new file mode 100644 index 00000000..64aa83af --- /dev/null +++ b/hack/remediation-guard.bats @@ -0,0 +1,87 @@ +#!/usr/bin/env bats +# ----------------------------------------------------------------------------- +# Unit tests for hack/e2e-apps/remediation-guard.sh +# +# helmrelease_has_remediation_cycle is consumed from e2e tests to assert that +# the parent HelmRelease did not hit flux helm-controller's wait timeout and +# enter uninstall remediation. The function accepts two arguments (values of +# .status.installFailures and .status.upgradeFailures) and returns 0 when a +# remediation cycle is detected, 1 otherwise. +# +# Each argument can be empty (controller never populated the field), "0" +# (populated but never failed), or a positive integer. Shell's && and || +# have equal precedence with left-to-right associativity, which used to +# break this check on the most common failure mode - install_failures=1 +# and upgrade_failures="". These tests pin the correct behavior. +# +# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on +# its own line; there is no bats `run` or `$status`. Assertions are +# expressed as direct shell tests that exit non-zero on failure. +# +# Run with: hack/cozytest.sh hack/remediation-guard.bats +# ----------------------------------------------------------------------------- + +@test "no counters set returns not-detected" { + . hack/e2e-apps/remediation-guard.sh + rc=0 + helmrelease_has_remediation_cycle "" "" || rc=$? + [ "$rc" -eq 1 ] +} + +@test "both counters zero returns not-detected" { + . hack/e2e-apps/remediation-guard.sh + rc=0 + helmrelease_has_remediation_cycle "0" "0" || rc=$? + [ "$rc" -eq 1 ] +} + +@test "install zero upgrade empty returns not-detected" { + . hack/e2e-apps/remediation-guard.sh + rc=0 + helmrelease_has_remediation_cycle "0" "" || rc=$? + [ "$rc" -eq 1 ] +} + +@test "install empty upgrade zero returns not-detected" { + . hack/e2e-apps/remediation-guard.sh + rc=0 + helmrelease_has_remediation_cycle "" "0" || rc=$? + [ "$rc" -eq 1 ] +} + +@test "install one upgrade empty returns detected" { + # Canonical race: first install exceeded helm-wait, remediation fired, + # no upgrade has happened yet. + . hack/e2e-apps/remediation-guard.sh + rc=0 + helmrelease_has_remediation_cycle "1" "" || rc=$? + [ "$rc" -eq 0 ] +} + +@test "install empty upgrade one returns detected" { + . hack/e2e-apps/remediation-guard.sh + rc=0 + helmrelease_has_remediation_cycle "" "1" || rc=$? + [ "$rc" -eq 0 ] +} + +@test "install two upgrade zero returns detected" { + . hack/e2e-apps/remediation-guard.sh + rc=0 + helmrelease_has_remediation_cycle "2" "0" || rc=$? + [ "$rc" -eq 0 ] +} + +@test "install zero upgrade two returns detected" { + . hack/e2e-apps/remediation-guard.sh + rc=0 + helmrelease_has_remediation_cycle "0" "2" || rc=$? + [ "$rc" -eq 0 ] +} + +@test "both counters positive returns detected" { + . hack/e2e-apps/remediation-guard.sh + rc=0 + helmrelease_has_remediation_cycle "3" "5" || rc=$? + [ "$rc" -eq 0 ] +} From 73b80bfb940ec24459a826b32540f676a00bca36 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 16 Apr 2026 20:51:32 +0300 Subject: [PATCH 059/250] fix(api): scope 15m helm wait budget to Kubernetes Application kind The previous change set Install.Timeout and Upgrade.Timeout to 15m on every Application's parent HelmRelease, but the admin-kubeconfig race documented in #2412 is specific to the Kubernetes kind: only its parent chart creates CAPI/Kamaji resources whose admin-kubeconfig Secret is asynchronously provisioned and mounted by Deployments in the same chart. Other kinds (Qdrant, MongoDB, Postgres, ...) have no such race and should not have their failed installs linger three times longer before flux triggers remediation. Gate the timeout on r.kindName == "Kubernetes". Rewrite rest_timeout_test.go to cover both sides: Kubernetes must get a >= 15m timeout, other kinds must keep the flux defaults. Both tests also pin Install/Upgrade Remediation.Retries == -1 so a future edit that removes unbounded remediation would show up as a red test. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- pkg/registry/apps/application/rest.go | 16 ++++- .../apps/application/rest_timeout_test.go | 58 ++++++++++++++++--- 2 files changed, 64 insertions(+), 10 deletions(-) diff --git a/pkg/registry/apps/application/rest.go b/pkg/registry/apps/application/rest.go index 77d1d458..9b92d2c0 100644 --- a/pkg/registry/apps/application/rest.go +++ b/pkg/registry/apps/application/rest.go @@ -1509,13 +1509,11 @@ func (r *REST) convertApplicationToHelmRelease(app *appsv1alpha1.Application) (* }, Interval: metav1.Duration{Duration: 5 * time.Minute}, Install: &helmv2.Install{ - Timeout: &metav1.Duration{Duration: 15 * time.Minute}, Remediation: &helmv2.InstallRemediation{ Retries: -1, }, }, Upgrade: &helmv2.Upgrade{ - Timeout: &metav1.Duration{Duration: 15 * time.Minute}, Remediation: &helmv2.UpgradeRemediation{ Retries: -1, }, @@ -1530,6 +1528,20 @@ func (r *REST) convertApplicationToHelmRelease(app *appsv1alpha1.Application) (* }, } + // The Kubernetes Application's parent chart creates CAPI/Kamaji resources + // whose admin-kubeconfig Secret is provisioned asynchronously and mounted + // by Deployments in the same chart. On a cold node, Kamaji control-plane + // bootstrap routinely exceeds flux helm-controller's default wait budget, + // so install remediation loops uninstall the Cluster CR and churn. Extend + // the wait budget to 15m for Kubernetes only - other Application kinds + // without this race keep flux defaults, so their failed installs do not + // linger unnecessarily before remediation fires. + if r.kindName == "Kubernetes" { + timeout := metav1.Duration{Duration: 15 * time.Minute} + helmRelease.Spec.Install.Timeout = &timeout + helmRelease.Spec.Upgrade.Timeout = &timeout + } + return helmRelease, nil } diff --git a/pkg/registry/apps/application/rest_timeout_test.go b/pkg/registry/apps/application/rest_timeout_test.go index 8ae13183..fa6d5427 100644 --- a/pkg/registry/apps/application/rest_timeout_test.go +++ b/pkg/registry/apps/application/rest_timeout_test.go @@ -10,18 +10,22 @@ import ( "github.com/cozystack/cozystack/pkg/config" ) -func TestConvertApplicationToHelmRelease_SetsInstallAndUpgradeTimeout(t *testing.T) { - r := &REST{ +func newRESTForKind(kind, prefix string) *REST { + return &REST{ + kindName: kind, releaseConfig: config.ReleaseConfig{ - Prefix: "kubernetes-", + Prefix: prefix, ChartRef: config.ChartRefConfig{ Kind: "HelmChart", - Name: "kubernetes", + Name: "x", Namespace: "cozy-system", }, }, } +} +func TestConvertApplicationToHelmRelease_KubernetesKindGetsLongTimeout(t *testing.T) { + r := newRESTForKind("Kubernetes", "kubernetes-") app := &appsv1alpha1.Application{ ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "tenant-root"}, } @@ -35,19 +39,57 @@ func TestConvertApplicationToHelmRelease_SetsInstallAndUpgradeTimeout(t *testing t.Fatal("Spec.Install must not be nil") } if hr.Spec.Install.Timeout == nil { - t.Fatal("Spec.Install.Timeout must be set to cover async admin-kubeconfig provisioning") + t.Fatal("Spec.Install.Timeout must be set for Kubernetes kind") } if hr.Spec.Install.Timeout.Duration < 15*time.Minute { - t.Errorf("Spec.Install.Timeout must be >= 15m (cold bootstrap budget), got %v", hr.Spec.Install.Timeout.Duration) + t.Errorf("Spec.Install.Timeout must be >= 15m for Kubernetes, got %v", hr.Spec.Install.Timeout.Duration) } if hr.Spec.Upgrade == nil { t.Fatal("Spec.Upgrade must not be nil") } if hr.Spec.Upgrade.Timeout == nil { - t.Fatal("Spec.Upgrade.Timeout must be set to cover async admin-kubeconfig provisioning") + t.Fatal("Spec.Upgrade.Timeout must be set for Kubernetes kind") } if hr.Spec.Upgrade.Timeout.Duration < 15*time.Minute { - t.Errorf("Spec.Upgrade.Timeout must be >= 15m (cold bootstrap budget), got %v", hr.Spec.Upgrade.Timeout.Duration) + t.Errorf("Spec.Upgrade.Timeout must be >= 15m for Kubernetes, got %v", hr.Spec.Upgrade.Timeout.Duration) + } + + if hr.Spec.Install.Remediation == nil || hr.Spec.Install.Remediation.Retries != -1 { + t.Errorf("Spec.Install.Remediation.Retries must remain -1, got %+v", hr.Spec.Install.Remediation) + } + if hr.Spec.Upgrade.Remediation == nil || hr.Spec.Upgrade.Remediation.Retries != -1 { + t.Errorf("Spec.Upgrade.Remediation.Retries must remain -1, got %+v", hr.Spec.Upgrade.Remediation) + } +} + +func TestConvertApplicationToHelmRelease_NonKubernetesKindKeepsFluxDefaults(t *testing.T) { + // For Applications whose parent chart has no admin-kubeconfig race + // (Qdrant, MongoDB, Postgres, etc.), do NOT extend the helm-wait + // budget - otherwise failed installs would block three times as long + // before Flux starts remediating. + r := newRESTForKind("Qdrant", "qdrant-") + app := &appsv1alpha1.Application{ + ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "tenant-root"}, + } + + hr, err := r.convertApplicationToHelmRelease(app) + if err != nil { + t.Fatalf("convertApplicationToHelmRelease returned error: %v", err) + } + + if hr.Spec.Install != nil && hr.Spec.Install.Timeout != nil { + t.Errorf("Spec.Install.Timeout must be unset for non-Kubernetes kinds, got %v", hr.Spec.Install.Timeout.Duration) + } + if hr.Spec.Upgrade != nil && hr.Spec.Upgrade.Timeout != nil { + t.Errorf("Spec.Upgrade.Timeout must be unset for non-Kubernetes kinds, got %v", hr.Spec.Upgrade.Timeout.Duration) + } + + // But remediation must still be -1 across the board. + if hr.Spec.Install.Remediation == nil || hr.Spec.Install.Remediation.Retries != -1 { + t.Errorf("Spec.Install.Remediation.Retries must remain -1, got %+v", hr.Spec.Install.Remediation) + } + if hr.Spec.Upgrade.Remediation == nil || hr.Spec.Upgrade.Remediation.Retries != -1 { + t.Errorf("Spec.Upgrade.Remediation.Retries must remain -1, got %+v", hr.Spec.Upgrade.Remediation) } } From 03606091df5672b6e857057a349875d534990f3a Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 16 Apr 2026 20:53:17 +0300 Subject: [PATCH 060/250] chore(kubernetes): align busybox image with project convention Adds images/busybox/Dockerfile and an image-busybox Makefile target that mirror the same pattern as the rest of this chart's images (the Dockerfile pins the upstream busybox by digest; the Makefile target builds and tags for ghcr.io/cozystack/cozystack/busybox the same way cluster-autoscaler et al. are handled). Also wires it into the umbrella image target so 'make image' rebuilds everything. Until the first release build runs image-busybox and rewrites the .tag to point at ghcr.io, the .tag keeps a fully-qualified docker.io/library/busybox:1.37.0@sha256:... reference so pods do not silently resolve the short name via the default registry and pulls remain immutable by digest. The release workflow overwrites this file the same way it does for the other images. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- packages/apps/kubernetes/Makefile | 14 +++++++++++++- packages/apps/kubernetes/images/busybox.tag | 2 +- packages/apps/kubernetes/images/busybox/Dockerfile | 1 + 3 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 packages/apps/kubernetes/images/busybox/Dockerfile diff --git a/packages/apps/kubernetes/Makefile b/packages/apps/kubernetes/Makefile index 0f9e6d57..eb23c2ef 100644 --- a/packages/apps/kubernetes/Makefile +++ b/packages/apps/kubernetes/Makefile @@ -15,7 +15,7 @@ update: hack/update-versions.sh make generate -image: image-ubuntu-container-disk image-kubevirt-cloud-provider image-kubevirt-csi-driver image-cluster-autoscaler +image: image-ubuntu-container-disk image-kubevirt-cloud-provider image-kubevirt-csi-driver image-cluster-autoscaler image-busybox image-ubuntu-container-disk: $(foreach ver,$(KUBERNETES_VERSIONS), \ @@ -70,3 +70,15 @@ image-cluster-autoscaler: echo "$(REGISTRY)/cluster-autoscaler:$(call settag,$(KUBERNETES_PKG_TAG))@$$(yq e '."containerimage.digest"' images/cluster-autoscaler.json -o json -r)" \ > images/cluster-autoscaler.tag rm -f images/cluster-autoscaler.json + +image-busybox: + docker buildx build images/busybox \ + --tag $(REGISTRY)/busybox:$(call settag,$(KUBERNETES_PKG_TAG)) \ + --tag $(REGISTRY)/busybox:$(call settag,$(KUBERNETES_PKG_TAG)-$(TAG)) \ + --cache-from type=registry,ref=$(REGISTRY)/busybox:latest \ + --cache-to type=inline \ + --metadata-file images/busybox.json \ + $(BUILDX_ARGS) + echo "$(REGISTRY)/busybox:$(call settag,$(KUBERNETES_PKG_TAG))@$$(yq e '."containerimage.digest"' images/busybox.json -o json -r)" \ + > images/busybox.tag + rm -f images/busybox.json diff --git a/packages/apps/kubernetes/images/busybox.tag b/packages/apps/kubernetes/images/busybox.tag index e358c12e..39de220a 100644 --- a/packages/apps/kubernetes/images/busybox.tag +++ b/packages/apps/kubernetes/images/busybox.tag @@ -1 +1 @@ -busybox:1.37.0@sha256:1487d0af5f52b4ba31c7e465126ee2123fe3f2305d638e7827681e7cf6c83d5e +docker.io/library/busybox:1.37.0@sha256:1487d0af5f52b4ba31c7e465126ee2123fe3f2305d638e7827681e7cf6c83d5e diff --git a/packages/apps/kubernetes/images/busybox/Dockerfile b/packages/apps/kubernetes/images/busybox/Dockerfile new file mode 100644 index 00000000..0c57fd39 --- /dev/null +++ b/packages/apps/kubernetes/images/busybox/Dockerfile @@ -0,0 +1 @@ +FROM docker.io/library/busybox:1.37.0@sha256:1487d0af5f52b4ba31c7e465126ee2123fe3f2305d638e7827681e7cf6c83d5e From 97696b2b036be79ecafd8e96401dc2af3bc39d32 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 16 Apr 2026 20:54:10 +0300 Subject: [PATCH 061/250] test(kubernetes): add positive cluster render test and pin document kind Adds an assertion that cluster.yaml renders successfully when _namespace.etcd is set and produces a KamajiControlPlane whose dataStoreName equals the tenant's etcd DataStore name. Without this positive case a future edit that inverts or removes the existing etcd guard would pass the suite as long as the negative case still fails. Also adds documentSelector: kind=Deployment to the kccm and csi controller assertions so the jsonpath filter operates on a single document, matching the cluster-autoscaler case and removing reliance on helm-unittest filter-vs-single-value coercion. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- .../tests/admin_kubeconfig_wait_test.yaml | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml b/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml index e2d9f359..3b2b1d72 100644 --- a/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml +++ b/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml @@ -39,6 +39,9 @@ tests: - it: kccm mounts admin-kubeconfig as optional template: templates/kccm/manager.yaml + documentSelector: + path: kind + value: Deployment asserts: - equal: path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.secretName @@ -49,13 +52,25 @@ tests: - it: kccm waits for admin-kubeconfig via initContainer template: templates/kccm/manager.yaml + documentSelector: + path: kind + value: Deployment asserts: - equal: path: spec.template.spec.initContainers[0].name value: wait-for-kubeconfig + - contains: + path: spec.template.spec.initContainers[0].volumeMounts + content: + name: kubeconfig + mountPath: /etc/kubernetes/kubeconfig + readOnly: true - it: csi controller mounts admin-kubeconfig as optional template: templates/csi/deploy.yaml + documentSelector: + path: kind + value: Deployment asserts: - equal: path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.secretName @@ -66,10 +81,29 @@ tests: - it: csi controller waits for admin-kubeconfig via initContainer template: templates/csi/deploy.yaml + documentSelector: + path: kind + value: Deployment asserts: - equal: path: spec.template.spec.initContainers[0].name value: wait-for-kubeconfig + - contains: + path: spec.template.spec.initContainers[0].volumeMounts + content: + name: kubeconfig + mountPath: /etc/kubernetes/kubeconfig + readOnly: true + + - it: cluster.yaml renders and wires dataStoreName when tenant has etcd + template: templates/cluster.yaml + documentSelector: + path: kind + value: KamajiControlPlane + asserts: + - equal: + path: spec.dataStoreName + value: tenant-root - it: cluster.yaml fails render when tenant has no etcd DataStore template: templates/cluster.yaml From 6afc0eb370f827c55f478045d1aff2a329b25100 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 16 Apr 2026 20:55:17 +0300 Subject: [PATCH 062/250] fix(kubernetes): bound init wait and reword fail-fast message Cap the wait-for-kubeconfig init container at 20m. If Kamaji genuinely fails to produce the admin-kubeconfig Secret (misconfigured tenant, etcd outage after the guard already passed, Kamaji crash-loop), the pod now exits non-zero and goes CrashLoopBackOff so the failure is visible in dashboards, instead of silently sleeping in Init forever and leaving only the flux helm-wait timeout to surface the problem. Reword the etcd DataStore guard to reference the parent Tenant application's etcd flag (not .Values.etcd of the Kubernetes chart, which is a different chart). Update the helm unittest errorPattern accordingly. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- .../kubernetes/templates/cluster-autoscaler/deployment.yaml | 5 +++++ packages/apps/kubernetes/templates/cluster.yaml | 2 +- packages/apps/kubernetes/templates/csi/deploy.yaml | 5 +++++ packages/apps/kubernetes/templates/kccm/manager.yaml | 5 +++++ .../apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml | 2 +- 5 files changed, 17 insertions(+), 2 deletions(-) diff --git a/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml b/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml index 47c22f76..277b6df8 100644 --- a/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml +++ b/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml @@ -31,7 +31,12 @@ spec: - -c - | set -eu + deadline=$(( $(date +%s) + 1200 )) until [ -s /etc/kubernetes/kubeconfig/super-admin.svc ]; do + if [ "$(date +%s)" -ge "$deadline" ]; then + echo "admin kubeconfig was not provisioned within 20m; exiting so the pod goes CrashLoopBackOff and surfaces in dashboards" >&2 + exit 1 + fi echo "waiting for admin kubeconfig to be provisioned by Kamaji..." sleep 5 done diff --git a/packages/apps/kubernetes/templates/cluster.yaml b/packages/apps/kubernetes/templates/cluster.yaml index 99249c1a..1747df4e 100644 --- a/packages/apps/kubernetes/templates/cluster.yaml +++ b/packages/apps/kubernetes/templates/cluster.yaml @@ -1,6 +1,6 @@ {{- $etcd := .Values._namespace.etcd }} {{- if not $etcd }} -{{- fail "Kubernetes tenant requires a parent tenant with etcd enabled: set .Values.etcd=true on tenant-root (or any ancestor tenant), wait for the etcd HelmRelease to reconcile (DataStore CR appears in the tenant namespace), then retry." }} +{{- fail "Kubernetes tenant requires a parent Tenant application with etcd: true so the etcd module deploys a DataStore CR in the tenant namespace. Set spec.etcd: true on the root Tenant (or any ancestor Tenant), wait for the etcd HelmRelease to reconcile and the DataStore CR to appear, then retry." }} {{- end }} {{- $ingress := .Values._namespace.ingress }} {{- $host := .Values._namespace.host }} diff --git a/packages/apps/kubernetes/templates/csi/deploy.yaml b/packages/apps/kubernetes/templates/csi/deploy.yaml index 2979c75c..ff170cb8 100644 --- a/packages/apps/kubernetes/templates/csi/deploy.yaml +++ b/packages/apps/kubernetes/templates/csi/deploy.yaml @@ -32,7 +32,12 @@ spec: - -c - | set -eu + deadline=$(( $(date +%s) + 1200 )) until [ -s /etc/kubernetes/kubeconfig/super-admin.svc ]; do + if [ "$(date +%s)" -ge "$deadline" ]; then + echo "admin kubeconfig was not provisioned within 20m; exiting so the pod goes CrashLoopBackOff and surfaces in dashboards" >&2 + exit 1 + fi echo "waiting for admin kubeconfig to be provisioned by Kamaji..." sleep 5 done diff --git a/packages/apps/kubernetes/templates/kccm/manager.yaml b/packages/apps/kubernetes/templates/kccm/manager.yaml index 5c7f15a4..67321647 100644 --- a/packages/apps/kubernetes/templates/kccm/manager.yaml +++ b/packages/apps/kubernetes/templates/kccm/manager.yaml @@ -30,7 +30,12 @@ spec: - -c - | set -eu + deadline=$(( $(date +%s) + 1200 )) until [ -s /etc/kubernetes/kubeconfig/super-admin.svc ]; do + if [ "$(date +%s)" -ge "$deadline" ]; then + echo "admin kubeconfig was not provisioned within 20m; exiting so the pod goes CrashLoopBackOff and surfaces in dashboards" >&2 + exit 1 + fi echo "waiting for admin kubeconfig to be provisioned by Kamaji..." sleep 5 done diff --git a/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml b/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml index 3b2b1d72..2a43e13f 100644 --- a/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml +++ b/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml @@ -116,4 +116,4 @@ tests: host: "" asserts: - failedTemplate: - errorPattern: "requires a parent tenant with etcd enabled" + errorPattern: "requires a parent Tenant application with etcd: true" From b38ae605495ff30a71013037af94839c4944e815 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 16 Apr 2026 21:05:23 +0300 Subject: [PATCH 063/250] fix(kubernetes): soft-skip cluster resources when tenant has no DataStore A hard helm fail in cluster.yaml made every cold bootstrap racy: if the parent Tenant chart had not yet populated _namespace.etcd in cozystack-values when the Kubernetes HelmRelease first reconciled, the fail fired, install.remediation triggered, installFailures incremented and the new e2e remediation-guard flagged it as a bug. That directly contradicts the race the rest of this PR is trying to close. Replace fail with a graceful skip: render only a status-beacon ConfigMap (test-awaiting-etcd) when etcd is empty, wrap all CAPI/Kamaji resources in {{ if $etcd }}. The HelmRelease installs successfully and goes Ready; flux retries on its 5m interval and picks up the DataStore as soon as the Tenant chart finishes reconciling. Update the helm unittest: positive test still asserts dataStoreName on KamajiControlPlane; the negative test now asserts exactly one ConfigMap document with status=awaiting-etcd, no Cluster / KCP / KubevirtCluster / WorkloadMonitor rendered. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- .../apps/kubernetes/templates/cluster.yaml | 37 +++++++++++++++++-- .../tests/admin_kubeconfig_wait_test.yaml | 19 ++++++++-- 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/packages/apps/kubernetes/templates/cluster.yaml b/packages/apps/kubernetes/templates/cluster.yaml index 1747df4e..12da9818 100644 --- a/packages/apps/kubernetes/templates/cluster.yaml +++ b/packages/apps/kubernetes/templates/cluster.yaml @@ -1,7 +1,15 @@ {{- $etcd := .Values._namespace.etcd }} -{{- if not $etcd }} -{{- fail "Kubernetes tenant requires a parent Tenant application with etcd: true so the etcd module deploys a DataStore CR in the tenant namespace. Set spec.etcd: true on the root Tenant (or any ancestor Tenant), wait for the etcd HelmRelease to reconcile and the DataStore CR to appear, then retry." }} -{{- end }} +{{- /* + When $etcd is empty, the parent Tenant application has not populated + _namespace.etcd in cozystack-values yet - either the operator forgot to + set etcd: true on an ancestor Tenant, or the Tenant HelmRelease is still + reconciling. Either way, rendering a KamajiControlPlane with an empty + dataStoreName would be rejected by Kamaji's admission webhook and the + HelmRelease would fail to install, triggering remediation. Instead, emit + a single ConfigMap as a user-visible status beacon and skip the rest so + flux marks the HelmRelease Ready and retries its 5m reconcile loop until + the Tenant chart catches up. +*/}} {{- $ingress := .Values._namespace.ingress }} {{- $host := .Values._namespace.host }} {{- $kubevirtmachinetemplateNames := list }} @@ -87,6 +95,28 @@ spec: - name: default pod: {} {{- end }} +{{- if not $etcd }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-awaiting-etcd + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/name: kubernetes + app.kubernetes.io/instance: {{ .Release.Name }} + annotations: + cozystack.io/status-beacon: "true" +data: + status: "awaiting-etcd" + message: | + No DataStore is available for this tenant Kubernetes cluster. The parent + Tenant application has not populated _namespace.etcd. Set spec.etcd: true + on an ancestor Tenant (usually tenant-root) and wait for its HelmRelease + to reconcile - this HelmRelease will pick up the DataStore on its next + 5m reconcile loop and provision the cluster. +{{- else }} --- apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster @@ -407,3 +437,4 @@ metadata: spec: {{- .spec | toYaml | nindent 2 }} {{- end }} +{{- end }} diff --git a/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml b/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml index 2a43e13f..e7c03bd1 100644 --- a/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml +++ b/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml @@ -105,7 +105,12 @@ tests: path: spec.dataStoreName value: tenant-root - - it: cluster.yaml fails render when tenant has no etcd DataStore + - it: cluster.yaml skips Cluster resources when tenant has no etcd DataStore + # Must NOT fail rendering - the parent Tenant chart populates + # _namespace.etcd asynchronously, so rendering failures here would cause + # flux install remediation on every cold bootstrap. Instead, emit only a + # ConfigMap status beacon so the HelmRelease reports Ready while flux + # retries on its interval until the DataStore appears. template: templates/cluster.yaml set: _namespace: @@ -115,5 +120,13 @@ tests: seaweedfs: "" host: "" asserts: - - failedTemplate: - errorPattern: "requires a parent Tenant application with etcd: true" + - hasDocuments: + count: 1 + - isKind: + of: ConfigMap + - equal: + path: metadata.name + value: test-awaiting-etcd + - equal: + path: data.status + value: awaiting-etcd From 1757567218ac313a030da1f82e64541a3f9bfbae Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 16 Apr 2026 21:06:47 +0300 Subject: [PATCH 064/250] refactor(kubernetes): extract wait-for-kubeconfig init into shared helper The three control-plane-side Deployments (cluster-autoscaler, kccm, kcsi-controller) carried three copies of the same 20-line init container. That already drifted: the CSI copy used 4-space nesting while the other two used 2-space. Any future update to the image, the deadline, or the poll script had to land in three places or silently diverge. Extract the block into a new kubernetes.waitForAdminKubeconfig helper in templates/_helpers.tpl and include it at each call site. Tighten the deadline from 20m to 10m so it stays strictly below the 15m HelmRelease Install.Timeout and the CrashLoopBackOff surfaces in dashboards before flux remediation can fire. Also clarify the wait message so operators debugging a stuck init container do not chase Kamaji for what is actually kubelet's optional-Secret refresh cadence. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- .../apps/kubernetes/templates/_helpers.tpl | 40 +++++++++++++++++++ .../cluster-autoscaler/deployment.yaml | 21 +--------- .../apps/kubernetes/templates/csi/deploy.yaml | 21 +--------- .../kubernetes/templates/kccm/manager.yaml | 21 +--------- 4 files changed, 43 insertions(+), 60 deletions(-) diff --git a/packages/apps/kubernetes/templates/_helpers.tpl b/packages/apps/kubernetes/templates/_helpers.tpl index 36c06b64..89f06934 100644 --- a/packages/apps/kubernetes/templates/_helpers.tpl +++ b/packages/apps/kubernetes/templates/_helpers.tpl @@ -49,3 +49,43 @@ Selector labels app.kubernetes.io/name: {{ include "kubernetes.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} + +{{/* +wait-for-kubeconfig init container shared by the control-plane-side +Deployments (cluster-autoscaler, kccm, kcsi-controller) that mount the +*-admin-kubeconfig Secret provisioned asynchronously by Kamaji. The +Secret volume is declared optional so kubelet does not FailedMount while +Kamaji is still bootstrapping; this container polls the mounted path and +exits only when super-admin.svc appears, which happens after kubelet's +optional-Secret refresh cycle. + +The 10m deadline stays strictly below the 15m HelmRelease +Install.Timeout scoped to the Kubernetes Application kind so the +CrashLoopBackOff surfaces before flux remediation fires and uninstalls +the Cluster CR. + +Call site owns the surrounding volumes block; the kubeconfig volume +must exist on the pod and mount at /etc/kubernetes/kubeconfig. +*/}} +{{- define "kubernetes.waitForAdminKubeconfig" -}} +- name: wait-for-kubeconfig + image: "{{ .Files.Get "images/busybox.tag" | trim }}" + command: + - sh + - -c + - | + set -eu + deadline=$(( $(date +%s) + 600 )) + until [ -s /etc/kubernetes/kubeconfig/super-admin.svc ]; do + if [ "$(date +%s)" -ge "$deadline" ]; then + echo "admin kubeconfig was not provisioned within 10m; exiting so the pod goes CrashLoopBackOff and surfaces in dashboards" >&2 + exit 1 + fi + echo "waiting for admin kubeconfig (provisioned by Kamaji, visible after kubelet Secret refresh)..." + sleep 5 + done + volumeMounts: + - name: kubeconfig + mountPath: /etc/kubernetes/kubeconfig + readOnly: true +{{- end }} diff --git a/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml b/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml index 277b6df8..348b017d 100644 --- a/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml +++ b/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml @@ -24,26 +24,7 @@ spec: operator: Exists effect: "NoSchedule" initContainers: - - name: wait-for-kubeconfig - image: "{{ $.Files.Get "images/busybox.tag" | trim }}" - command: - - sh - - -c - - | - set -eu - deadline=$(( $(date +%s) + 1200 )) - until [ -s /etc/kubernetes/kubeconfig/super-admin.svc ]; do - if [ "$(date +%s)" -ge "$deadline" ]; then - echo "admin kubeconfig was not provisioned within 20m; exiting so the pod goes CrashLoopBackOff and surfaces in dashboards" >&2 - exit 1 - fi - echo "waiting for admin kubeconfig to be provisioned by Kamaji..." - sleep 5 - done - volumeMounts: - - name: kubeconfig - mountPath: /etc/kubernetes/kubeconfig - readOnly: true + {{- include "kubernetes.waitForAdminKubeconfig" $ | nindent 6 }} containers: - image: "{{ $.Files.Get "images/cluster-autoscaler.tag" | trim }}" name: cluster-autoscaler diff --git a/packages/apps/kubernetes/templates/csi/deploy.yaml b/packages/apps/kubernetes/templates/csi/deploy.yaml index ff170cb8..c1af7f13 100644 --- a/packages/apps/kubernetes/templates/csi/deploy.yaml +++ b/packages/apps/kubernetes/templates/csi/deploy.yaml @@ -25,26 +25,7 @@ spec: operator: Exists effect: "NoSchedule" initContainers: - - name: wait-for-kubeconfig - image: "{{ $.Files.Get "images/busybox.tag" | trim }}" - command: - - sh - - -c - - | - set -eu - deadline=$(( $(date +%s) + 1200 )) - until [ -s /etc/kubernetes/kubeconfig/super-admin.svc ]; do - if [ "$(date +%s)" -ge "$deadline" ]; then - echo "admin kubeconfig was not provisioned within 20m; exiting so the pod goes CrashLoopBackOff and surfaces in dashboards" >&2 - exit 1 - fi - echo "waiting for admin kubeconfig to be provisioned by Kamaji..." - sleep 5 - done - volumeMounts: - - name: kubeconfig - mountPath: /etc/kubernetes/kubeconfig - readOnly: true + {{- include "kubernetes.waitForAdminKubeconfig" $ | nindent 6 }} containers: - name: csi-driver imagePullPolicy: Always diff --git a/packages/apps/kubernetes/templates/kccm/manager.yaml b/packages/apps/kubernetes/templates/kccm/manager.yaml index 67321647..20ac0a1e 100644 --- a/packages/apps/kubernetes/templates/kccm/manager.yaml +++ b/packages/apps/kubernetes/templates/kccm/manager.yaml @@ -23,26 +23,7 @@ spec: operator: Exists effect: "NoSchedule" initContainers: - - name: wait-for-kubeconfig - image: "{{ $.Files.Get "images/busybox.tag" | trim }}" - command: - - sh - - -c - - | - set -eu - deadline=$(( $(date +%s) + 1200 )) - until [ -s /etc/kubernetes/kubeconfig/super-admin.svc ]; do - if [ "$(date +%s)" -ge "$deadline" ]; then - echo "admin kubeconfig was not provisioned within 20m; exiting so the pod goes CrashLoopBackOff and surfaces in dashboards" >&2 - exit 1 - fi - echo "waiting for admin kubeconfig to be provisioned by Kamaji..." - sleep 5 - done - volumeMounts: - - name: kubeconfig - mountPath: /etc/kubernetes/kubeconfig - readOnly: true + {{- include "kubernetes.waitForAdminKubeconfig" $ | nindent 6 }} containers: - name: kubevirt-cloud-controller-manager args: From 7b146cbe56c26511d2228c17cdb7c2201f480395 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 16 Apr 2026 21:09:35 +0300 Subject: [PATCH 065/250] feat(api): make HelmRelease Install/Upgrade timeout per-Application Replace the hardcoded r.kindName == "Kubernetes" switch in rest.go with a config-driven path. The ApplicationDefinition CR now accepts a release.cozystack.io/helm-install-timeout annotation that is parsed at cozystack-api startup into config.ReleaseConfig.HelmInstallTimeout and applied to both Install.Timeout and Upgrade.Timeout on the rendered HelmRelease. Applications that leave the annotation unset keep flux defaults so their failed installs remediate on the normal cadence - only the Kubernetes kind carries the override and gets a 15m budget. New kinds with a similar race can opt in by setting the same annotation; no rest.go patch needed. Kubernetes-rd sets the annotation to 15m. Table-driven test in rest_timeout_test.go covers three cases: Kubernetes with 15m, Qdrant unset, and an arbitrary future kind with 20m - all of which pin the Remediation.Retries == -1 contract as well. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- .../kubernetes-rd/cozyrds/kubernetes.yaml | 8 + pkg/cmd/server/start.go | 35 +++-- pkg/config/config.go | 8 + pkg/registry/apps/application/rest.go | 21 +-- .../apps/application/rest_timeout_test.go | 143 ++++++++++-------- 5 files changed, 133 insertions(+), 82 deletions(-) diff --git a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml index 5e9e8f94..1128abe8 100644 --- a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml +++ b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml @@ -2,6 +2,14 @@ apiVersion: cozystack.io/v1alpha1 kind: ApplicationDefinition metadata: name: kubernetes + annotations: + # Kubernetes tenants boot a Kamaji control plane whose admin-kubeconfig + # Secret is provisioned asynchronously. Cold Kamaji start (image pull + + # etcd + apiserver Ready) plus admin-kubeconfig generation can exceed + # Flux helm-controller's default wait budget, causing remediation loops + # that uninstall the Cluster CR. This override applied by cozystack-api + # to the HelmRelease Spec.Install.Timeout and Spec.Upgrade.Timeout. + release.cozystack.io/helm-install-timeout: "15m" spec: application: kind: Kubernetes diff --git a/pkg/cmd/server/start.go b/pkg/cmd/server/start.go index f7791332..3e905735 100644 --- a/pkg/cmd/server/start.go +++ b/pkg/cmd/server/start.go @@ -160,6 +160,31 @@ func (o *CozyServerOptions) Complete() error { // Convert to ResourceConfig o.ResourceConfig = &config.ResourceConfig{} for _, crd := range crdList.Items { + release := config.ReleaseConfig{ + Prefix: crd.Spec.Release.Prefix, + Labels: crd.Spec.Release.Labels, + ChartRef: config.ChartRefConfig{ + Kind: crd.Spec.Release.ChartRef.Kind, + Name: crd.Spec.Release.ChartRef.Name, + Namespace: crd.Spec.Release.ChartRef.Namespace, + }, + } + // Per-Application HelmRelease Install/Upgrade timeout. Applications + // whose parent chart contains asynchronously-provisioned resources + // the chart itself depends on (for example, the Kamaji-provisioned + // admin-kubeconfig Secret for Kubernetes tenants) need a longer + // wait budget than the Flux default. Consumed by the REST storage + // layer when building the HelmRelease Spec. + if raw, ok := crd.Annotations["release.cozystack.io/helm-install-timeout"]; ok && raw != "" { + d, err := time.ParseDuration(raw) + if err != nil { + return fmt.Errorf( + "ApplicationDefinition %q has invalid release.cozystack.io/helm-install-timeout %q: %w", + crd.Name, raw, err, + ) + } + release.HelmInstallTimeout = d + } resource := config.Resource{ Application: config.ApplicationConfig{ Kind: crd.Spec.Application.Kind, @@ -168,15 +193,7 @@ func (o *CozyServerOptions) Complete() error { ShortNames: []string{}, // TODO: implement shortnames OpenAPISchema: crd.Spec.Application.OpenAPISchema, }, - Release: config.ReleaseConfig{ - Prefix: crd.Spec.Release.Prefix, - Labels: crd.Spec.Release.Labels, - ChartRef: config.ChartRefConfig{ - Kind: crd.Spec.Release.ChartRef.Kind, - Name: crd.Spec.Release.ChartRef.Name, - Namespace: crd.Spec.Release.ChartRef.Namespace, - }, - }, + Release: release, } o.ResourceConfig.Resources = append(o.ResourceConfig.Resources, resource) } diff --git a/pkg/config/config.go b/pkg/config/config.go index 1e123e2c..16cf4f0c 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -16,6 +16,8 @@ limitations under the License. package config +import "time" + // ResourceConfig represents the structure of the configuration file. type ResourceConfig struct { Resources []Resource `yaml:"resources"` @@ -41,6 +43,12 @@ type ReleaseConfig struct { Prefix string `yaml:"prefix"` Labels map[string]string `yaml:"labels"` ChartRef ChartRefConfig `yaml:"chartRef"` + // HelmInstallTimeout overrides the Flux HelmRelease Install.Timeout and + // Upgrade.Timeout for this Application kind. When zero, flux defaults + // apply. Populated from the + // release.cozystack.io/helm-install-timeout annotation on the + // ApplicationDefinition at start-up. + HelmInstallTimeout time.Duration `yaml:"helmInstallTimeout,omitempty"` } // ChartRefConfig references a Flux source artifact for the Helm chart. diff --git a/pkg/registry/apps/application/rest.go b/pkg/registry/apps/application/rest.go index 9b92d2c0..406d3738 100644 --- a/pkg/registry/apps/application/rest.go +++ b/pkg/registry/apps/application/rest.go @@ -1528,16 +1528,17 @@ func (r *REST) convertApplicationToHelmRelease(app *appsv1alpha1.Application) (* }, } - // The Kubernetes Application's parent chart creates CAPI/Kamaji resources - // whose admin-kubeconfig Secret is provisioned asynchronously and mounted - // by Deployments in the same chart. On a cold node, Kamaji control-plane - // bootstrap routinely exceeds flux helm-controller's default wait budget, - // so install remediation loops uninstall the Cluster CR and churn. Extend - // the wait budget to 15m for Kubernetes only - other Application kinds - // without this race keep flux defaults, so their failed installs do not - // linger unnecessarily before remediation fires. - if r.kindName == "Kubernetes" { - timeout := metav1.Duration{Duration: 15 * time.Minute} + // Per-Application HelmRelease wait budget. When an ApplicationDefinition + // sets release.cozystack.io/helm-install-timeout, the annotation is + // parsed at startup into ReleaseConfig.HelmInstallTimeout and applied + // to both Install and Upgrade here. Applications that leave it unset + // (the common case) keep flux defaults, so their failed installs + // remediate on the normal cadence. Needed for the Kubernetes kind + // because its parent chart contains CAPI/Kamaji resources whose + // admin-kubeconfig Secret is provisioned asynchronously and Kamaji + // cold-start routinely exceeds flux's default wait budget. + if r.releaseConfig.HelmInstallTimeout > 0 { + timeout := metav1.Duration{Duration: r.releaseConfig.HelmInstallTimeout} helmRelease.Spec.Install.Timeout = &timeout helmRelease.Spec.Upgrade.Timeout = &timeout } diff --git a/pkg/registry/apps/application/rest_timeout_test.go b/pkg/registry/apps/application/rest_timeout_test.go index fa6d5427..8474d1cb 100644 --- a/pkg/registry/apps/application/rest_timeout_test.go +++ b/pkg/registry/apps/application/rest_timeout_test.go @@ -10,7 +10,7 @@ import ( "github.com/cozystack/cozystack/pkg/config" ) -func newRESTForKind(kind, prefix string) *REST { +func newRESTForTimeout(kind, prefix string, helmInstallTimeout time.Duration) *REST { return &REST{ kindName: kind, releaseConfig: config.ReleaseConfig{ @@ -20,76 +20,93 @@ func newRESTForKind(kind, prefix string) *REST { Name: "x", Namespace: "cozy-system", }, + HelmInstallTimeout: helmInstallTimeout, }, } } -func TestConvertApplicationToHelmRelease_KubernetesKindGetsLongTimeout(t *testing.T) { - r := newRESTForKind("Kubernetes", "kubernetes-") - app := &appsv1alpha1.Application{ - ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "tenant-root"}, +// Table-driven: every Application kind carries a per-CRD HelmRelease wait +// budget. The Kubernetes kind's parent chart contains CAPI/Kamaji resources +// whose admin-kubeconfig Secret is provisioned asynchronously, so its +// ApplicationDefinition sets release.cozystack.io/helm-install-timeout=15m +// (or longer). Other kinds leave the annotation unset and keep flux defaults +// so their failed installs remediate on the normal cadence. The test must +// cover both paths: a kind with the timeout set and one without. +func TestConvertApplicationToHelmRelease_AppliesReleaseConfigTimeout(t *testing.T) { + cases := []struct { + name string + kind string + prefix string + configured time.Duration + wantSet bool + }{ + { + name: "Kubernetes kind with 15m configured gets Install and Upgrade Timeout", + kind: "Kubernetes", + prefix: "kubernetes-", + configured: 15 * time.Minute, + wantSet: true, + }, + { + name: "Qdrant kind without configured timeout keeps flux defaults", + kind: "Qdrant", + prefix: "qdrant-", + configured: 0, + wantSet: false, + }, + { + name: "arbitrary future kind with 20m configured gets 20m", + kind: "TalosCluster", + prefix: "talos-", + configured: 20 * time.Minute, + wantSet: true, + }, } - hr, err := r.convertApplicationToHelmRelease(app) - if err != nil { - t.Fatalf("convertApplicationToHelmRelease returned error: %v", err) - } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := newRESTForTimeout(tc.kind, tc.prefix, tc.configured) + app := &appsv1alpha1.Application{ + ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "tenant-root"}, + } - if hr.Spec.Install == nil { - t.Fatal("Spec.Install must not be nil") - } - if hr.Spec.Install.Timeout == nil { - t.Fatal("Spec.Install.Timeout must be set for Kubernetes kind") - } - if hr.Spec.Install.Timeout.Duration < 15*time.Minute { - t.Errorf("Spec.Install.Timeout must be >= 15m for Kubernetes, got %v", hr.Spec.Install.Timeout.Duration) - } + hr, err := r.convertApplicationToHelmRelease(app) + if err != nil { + t.Fatalf("convertApplicationToHelmRelease returned error: %v", err) + } - if hr.Spec.Upgrade == nil { - t.Fatal("Spec.Upgrade must not be nil") - } - if hr.Spec.Upgrade.Timeout == nil { - t.Fatal("Spec.Upgrade.Timeout must be set for Kubernetes kind") - } - if hr.Spec.Upgrade.Timeout.Duration < 15*time.Minute { - t.Errorf("Spec.Upgrade.Timeout must be >= 15m for Kubernetes, got %v", hr.Spec.Upgrade.Timeout.Duration) - } + if hr.Spec.Install == nil || hr.Spec.Upgrade == nil { + t.Fatalf("Spec.Install/Upgrade must be non-nil") + } - if hr.Spec.Install.Remediation == nil || hr.Spec.Install.Remediation.Retries != -1 { - t.Errorf("Spec.Install.Remediation.Retries must remain -1, got %+v", hr.Spec.Install.Remediation) - } - if hr.Spec.Upgrade.Remediation == nil || hr.Spec.Upgrade.Remediation.Retries != -1 { - t.Errorf("Spec.Upgrade.Remediation.Retries must remain -1, got %+v", hr.Spec.Upgrade.Remediation) - } -} - -func TestConvertApplicationToHelmRelease_NonKubernetesKindKeepsFluxDefaults(t *testing.T) { - // For Applications whose parent chart has no admin-kubeconfig race - // (Qdrant, MongoDB, Postgres, etc.), do NOT extend the helm-wait - // budget - otherwise failed installs would block three times as long - // before Flux starts remediating. - r := newRESTForKind("Qdrant", "qdrant-") - app := &appsv1alpha1.Application{ - ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "tenant-root"}, - } - - hr, err := r.convertApplicationToHelmRelease(app) - if err != nil { - t.Fatalf("convertApplicationToHelmRelease returned error: %v", err) - } - - if hr.Spec.Install != nil && hr.Spec.Install.Timeout != nil { - t.Errorf("Spec.Install.Timeout must be unset for non-Kubernetes kinds, got %v", hr.Spec.Install.Timeout.Duration) - } - if hr.Spec.Upgrade != nil && hr.Spec.Upgrade.Timeout != nil { - t.Errorf("Spec.Upgrade.Timeout must be unset for non-Kubernetes kinds, got %v", hr.Spec.Upgrade.Timeout.Duration) - } - - // But remediation must still be -1 across the board. - if hr.Spec.Install.Remediation == nil || hr.Spec.Install.Remediation.Retries != -1 { - t.Errorf("Spec.Install.Remediation.Retries must remain -1, got %+v", hr.Spec.Install.Remediation) - } - if hr.Spec.Upgrade.Remediation == nil || hr.Spec.Upgrade.Remediation.Retries != -1 { - t.Errorf("Spec.Upgrade.Remediation.Retries must remain -1, got %+v", hr.Spec.Upgrade.Remediation) + if tc.wantSet { + if hr.Spec.Install.Timeout == nil { + t.Fatalf("Spec.Install.Timeout must be set when HelmInstallTimeout=%v", tc.configured) + } + if hr.Spec.Install.Timeout.Duration != tc.configured { + t.Errorf("Spec.Install.Timeout = %v, want %v", hr.Spec.Install.Timeout.Duration, tc.configured) + } + if hr.Spec.Upgrade.Timeout == nil { + t.Fatalf("Spec.Upgrade.Timeout must be set when HelmInstallTimeout=%v", tc.configured) + } + if hr.Spec.Upgrade.Timeout.Duration != tc.configured { + t.Errorf("Spec.Upgrade.Timeout = %v, want %v", hr.Spec.Upgrade.Timeout.Duration, tc.configured) + } + } else { + if hr.Spec.Install.Timeout != nil { + t.Errorf("Spec.Install.Timeout must be nil when HelmInstallTimeout=0, got %v", hr.Spec.Install.Timeout.Duration) + } + if hr.Spec.Upgrade.Timeout != nil { + t.Errorf("Spec.Upgrade.Timeout must be nil when HelmInstallTimeout=0, got %v", hr.Spec.Upgrade.Timeout.Duration) + } + } + + if hr.Spec.Install.Remediation == nil || hr.Spec.Install.Remediation.Retries != -1 { + t.Errorf("Spec.Install.Remediation.Retries must remain -1, got %+v", hr.Spec.Install.Remediation) + } + if hr.Spec.Upgrade.Remediation == nil || hr.Spec.Upgrade.Remediation.Retries != -1 { + t.Errorf("Spec.Upgrade.Remediation.Retries must remain -1, got %+v", hr.Spec.Upgrade.Remediation) + } + }) } } From bc5473d4fc9bb1b43be79ea5eb8d370d9e6c00b7 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 16 Apr 2026 21:11:38 +0300 Subject: [PATCH 066/250] test(kubernetes): chart-wide invariant for admin-kubeconfig guards The per-template unittests in packages/apps/kubernetes/tests/ assert that cluster-autoscaler, kccm, and the csi controller each mount the admin-kubeconfig Secret optional and carry the wait-for-kubeconfig init. That locks in today's three Deployments by name - a fourth Deployment that mounts the same Secret but forgets the guard would slip past them. Add a bats-unit test that renders the entire chart, enumerates every Deployment whose spec mounts a Secret ending in -admin-kubeconfig, and asserts optional:true plus wait-for-kubeconfig init on all of them. Verified by temporarily removing optional:true from csi/deploy.yaml: the test correctly flagged invariant-kcsi-controller as an offender. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- hack/admin-kubeconfig-invariant.bats | 72 ++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 hack/admin-kubeconfig-invariant.bats diff --git a/hack/admin-kubeconfig-invariant.bats b/hack/admin-kubeconfig-invariant.bats new file mode 100644 index 00000000..d699de83 --- /dev/null +++ b/hack/admin-kubeconfig-invariant.bats @@ -0,0 +1,72 @@ +#!/usr/bin/env bats +# ----------------------------------------------------------------------------- +# Chart-wide invariant for packages/apps/kubernetes: +# +# Every Deployment in this chart that mounts -admin-kubeconfig as a +# Secret volume MUST: +# - declare that volume optional: true (so kubelet does not FailedMount +# while Kamaji is still provisioning the Secret), AND +# - include the wait-for-kubeconfig init container (so the pod becomes +# Ready only after Kamaji publishes the Secret). +# +# The per-template unittests in packages/apps/kubernetes/tests/ lock in +# today's three Deployments (cluster-autoscaler, kccm, csi controller) by +# name. This invariant is stricter: any future Deployment added to this +# chart that mounts the same Secret but forgets the guard will fail here. +# +# Requires: helm, yq (mikefarah v4+), jq. All three are available on the +# project's CI runners and on the maintainer workstation. +# ----------------------------------------------------------------------------- + +@test "every Deployment mounting admin-kubeconfig has optional:true and wait-for-kubeconfig init" { + values_file="packages/apps/kubernetes/tests/values-ci.yaml" + [ -f "$values_file" ] + + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + + helm template invariant packages/apps/kubernetes \ + --namespace tenant-root \ + --values "$values_file" \ + 2>/dev/null > "$tmp/rendered.yaml" + + # yq streams one JSON object per input document. jq -s slurps the stream + # into an array so we can treat all Deployments as a single collection. + yq --output-format=json eval-all '.' "$tmp/rendered.yaml" \ + | jq -s --raw-output ' + map(select(.kind == "Deployment")) | + map({ + name: .metadata.name, + volumes: (.spec.template.spec.volumes // []), + initNames: ((.spec.template.spec.initContainers // []) | map(.name)), + }) | + map( + .name as $n | + .initNames as $ins | + (.volumes[] | select(.secret.secretName | test("-admin-kubeconfig$")?)) + | { + name: $n, + optional: (.secret.optional == true), + hasInit: ($ins | index("wait-for-kubeconfig") != null), + } + ) + ' > "$tmp/summary.json" + + # At least one Deployment must match; if a refactor removes every + # admin-kubeconfig volume from this chart, the test must be updated + # deliberately rather than silently passing. + matched=$(jq 'length' "$tmp/summary.json") + [ "$matched" -ge 1 ] + + offenders=$(jq --raw-output '.[] | select(.optional != true or .hasInit != true) | .name' "$tmp/summary.json") + + if [ -n "$offenders" ]; then + echo "Deployments mounting *-admin-kubeconfig without optional:true + wait-for-kubeconfig init:" >&2 + echo "$offenders" >&2 + echo "Full summary:" >&2 + cat "$tmp/summary.json" >&2 + exit 1 + fi + + echo "Invariant holds for $matched Deployment(s)" +} From 03426fbd718029e40d6680f299189e97f915a614 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 16 Apr 2026 21:13:06 +0300 Subject: [PATCH 067/250] test(hack): pin HelmRelease v2 status shape used by remediation guard run-kubernetes.sh extracts .status.installFailures and .status.upgradeFailures via kubectl -o jsonpath. If a future flux release renames the counters, kubectl returns empty, the guard reports no cycle, and e2e silently misses real remediation loops. Add a bats unit test that feeds a pinned HelmRelease v2 status snippet through the same jsonpath and asserts the extraction still yields the expected values. Also leave a pointer comment in run-kubernetes.sh so a future flux bump surfaces the version coupling in review. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- hack/e2e-apps/run-kubernetes.sh | 7 ++++++ hack/remediation-guard.bats | 44 +++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/hack/e2e-apps/run-kubernetes.sh b/hack/e2e-apps/run-kubernetes.sh index 811f42a7..e6002b53 100644 --- a/hack/e2e-apps/run-kubernetes.sh +++ b/hack/e2e-apps/run-kubernetes.sh @@ -326,6 +326,13 @@ EOF # A non-zero installFailures/upgradeFailures indicates the helm-wait budget expired while # admin-kubeconfig was still being provisioned, which would trigger uninstall remediation # and churn the Cluster CR. + # Flux helm-controller v2 status shape: .status.installFailures and + # .status.upgradeFailures are counters populated by the controller on + # every failed install/upgrade. If a future flux release renames them, + # kubectl returns the empty string and the guard silently passes. The + # shape is pinned by hack/remediation-guard.bats (see that file for + # details), and the vendored API types live under + # vendor/github.com/fluxcd/helm-controller/api/v2. install_failures=$(kubectl get hr -n tenant-test "kubernetes-${test_name}" -ojsonpath='{.status.installFailures}') upgrade_failures=$(kubectl get hr -n tenant-test "kubernetes-${test_name}" -ojsonpath='{.status.upgradeFailures}') if helmrelease_has_remediation_cycle "${install_failures}" "${upgrade_failures}"; then diff --git a/hack/remediation-guard.bats b/hack/remediation-guard.bats index 64aa83af..9c33452a 100644 --- a/hack/remediation-guard.bats +++ b/hack/remediation-guard.bats @@ -85,3 +85,47 @@ helmrelease_has_remediation_cycle "3" "5" || rc=$? [ "$rc" -eq 0 ] } + +@test "installFailures and upgradeFailures extraction pins HR v2 status shape" { + # Pins the Flux HelmRelease v2 status shape that run-kubernetes.sh relies + # on. If a future Flux version renames .status.installFailures (or + # .status.upgradeFailures), kubectl get -o jsonpath returns an empty + # string, the guard quietly says "no cycle", and real remediation loops + # slip past the e2e assertion. + # + # This test uses yq to read the exact path used in the e2e script. yq + # evaluates the same json-ish jsonpath against a pinned HR snippet, so + # the test fails loudly if the field ever disappears or moves. Cross + # reference: vendor/github.com/fluxcd/helm-controller/api/v2/ status + # struct field tags. + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + + cat > "$tmp/hr.yaml" <<'YAML' +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: kubernetes-test + namespace: tenant-test +spec: + interval: 5m +status: + installFailures: 2 + upgradeFailures: 0 + conditions: + - type: Ready + status: "False" + reason: UninstallSucceeded +YAML + + install_failures=$(yq '.status.installFailures' "$tmp/hr.yaml") + upgrade_failures=$(yq '.status.upgradeFailures' "$tmp/hr.yaml") + + [ "$install_failures" = "2" ] + [ "$upgrade_failures" = "0" ] + + . hack/e2e-apps/remediation-guard.sh + rc=0 + helmrelease_has_remediation_cycle "$install_failures" "$upgrade_failures" || rc=$? + [ "$rc" -eq 0 ] +} From 12632c60c76b77639c8e2edd7ab5bdb248398a7e Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 16 Apr 2026 21:25:50 +0300 Subject: [PATCH 068/250] fix(kubernetes): gate CP-side Deployments on tenant etcd DataStore The soft-skip wrap in cluster.yaml only silenced Cluster and KamajiControlPlane rendering when _namespace.etcd is empty. The three CP-side Deployments (cluster-autoscaler, kccm, kcsi-controller) still rendered, their wait-for-kubeconfig init containers CrashLoopBackOff'd forever (no KamajiControlPlane = no admin-kubeconfig Secret), HelmRelease hit its 15m wait timeout and triggered the very install remediation cycle the rest of this PR prevents. Self-contradiction. Wrap each of the three Deployment templates in {{- if .Values._namespace.etcd }}...{{- end }} so they render only when there is a DataStore to back them. Add an invariant bats test that renders the whole chart with etcd empty and asserts zero Deployments reference *-admin-kubeconfig. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- hack/admin-kubeconfig-invariant.bats | 45 +++++++++++++++++++ .../cluster-autoscaler/deployment.yaml | 12 +++++ .../apps/kubernetes/templates/csi/deploy.yaml | 2 + .../kubernetes/templates/kccm/manager.yaml | 2 + 4 files changed, 61 insertions(+) diff --git a/hack/admin-kubeconfig-invariant.bats b/hack/admin-kubeconfig-invariant.bats index d699de83..c027e2c4 100644 --- a/hack/admin-kubeconfig-invariant.bats +++ b/hack/admin-kubeconfig-invariant.bats @@ -70,3 +70,48 @@ echo "Invariant holds for $matched Deployment(s)" } + +@test "chart emits zero admin-kubeconfig Deployments when tenant has no etcd DataStore" { + # Without a DataStore (parent Tenant has not populated _namespace.etcd yet) + # the control-plane-side Deployments must NOT render at all. If they did, + # the wait-for-kubeconfig init would CrashLoopBackOff indefinitely - there + # would be no KamajiControlPlane to provision the Secret - consuming the + # HelmRelease wait budget and triggering exactly the remediation cycle the + # rest of this chart tries to avoid. This test renders the whole chart + # with etcd empty and asserts no Deployment references the admin-kubeconfig + # Secret. + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + + helm template invariant packages/apps/kubernetes \ + --namespace tenant-root \ + --set _namespace.etcd="" \ + --set _namespace.monitoring="" \ + --set _namespace.ingress="" \ + --set _namespace.seaweedfs="" \ + --set _namespace.host="" \ + --set _cluster.cluster-domain=cozy.local \ + --set 'nodeGroups=null' \ + 2>/dev/null > "$tmp/rendered.yaml" + + matched=$( + yq --output-format=json eval-all '.' "$tmp/rendered.yaml" \ + | jq -s ' + map(select(.kind == "Deployment")) | + map(select( + (.spec.template.spec.volumes // []) + | any(.secret.secretName | test("-admin-kubeconfig$")?) + )) | + length + ' + ) + + if [ "$matched" -ne 0 ]; then + echo "Expected zero Deployments mounting *-admin-kubeconfig when etcd is empty, got $matched:" >&2 + yq --output-format=json eval-all '.' "$tmp/rendered.yaml" \ + | jq -s 'map(select(.kind == "Deployment") | .metadata.name)' >&2 + exit 1 + fi + + echo "No admin-kubeconfig Deployments rendered for empty etcd (as expected)" +} diff --git a/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml b/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml index 348b017d..298d86db 100644 --- a/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml +++ b/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml @@ -1,3 +1,14 @@ +{{- /* + Gate the control-plane-side workloads on the parent tenant having an etcd + DataStore. Without it no KamajiControlPlane is ever created, Kamaji never + provisions -admin-kubeconfig, and rendering these Deployments would cause + the wait-for-kubeconfig init to CrashLoopBackOff indefinitely, consuming + the parent HelmRelease install timeout and triggering the very uninstall + remediation cycle this chart is supposed to avoid. Rendering them only + when $etcd is set keeps the HelmRelease Ready while flux retries on its + interval and picks up the DataStore as soon as the Tenant chart finishes. +*/}} +{{- if .Values._namespace.etcd }} --- apiVersion: apps/v1 kind: Deployment @@ -108,3 +119,4 @@ rules: - list - update - watch +{{- end }} diff --git a/packages/apps/kubernetes/templates/csi/deploy.yaml b/packages/apps/kubernetes/templates/csi/deploy.yaml index c1af7f13..de62104c 100644 --- a/packages/apps/kubernetes/templates/csi/deploy.yaml +++ b/packages/apps/kubernetes/templates/csi/deploy.yaml @@ -1,3 +1,4 @@ +{{- if .Values._namespace.etcd }} kind: Deployment apiVersion: apps/v1 metadata: @@ -238,3 +239,4 @@ spec: secretName: {{ .Release.Name }}-admin-kubeconfig optional: true name: kubeconfig +{{- end }} diff --git a/packages/apps/kubernetes/templates/kccm/manager.yaml b/packages/apps/kubernetes/templates/kccm/manager.yaml index 20ac0a1e..bd9e2798 100644 --- a/packages/apps/kubernetes/templates/kccm/manager.yaml +++ b/packages/apps/kubernetes/templates/kccm/manager.yaml @@ -1,3 +1,4 @@ +{{- if .Values._namespace.etcd }} apiVersion: apps/v1 kind: Deployment metadata: @@ -60,3 +61,4 @@ spec: optional: true name: kubeconfig serviceAccountName: {{ .Release.Name }}-kccm +{{- end }} From 1ddd2aaea97024f6803c60d29a946273b55e4185 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 16 Apr 2026 21:28:11 +0300 Subject: [PATCH 069/250] fix(hack): detect remediation via status.history, not transient counters Flux helm-controller's ClearFailures() zeroes installFailures and upgradeFailures on every successful reconciliation (see the upstream HelmReleaseStatus method). The previous guard ran after the HelmRelease was Ready, at which point the counters were always 0 - the assertion was vacuous and would have passed against a reverted fix. Switch to .status.history, which retains per-revision release Snapshots that survive a subsequent successful reconciliation. A remediation cycle leaves behind a Snapshot with status=uninstalled (the install-remediation code path) or status=failed (Helm release failure that remediation then uninstalled). Either one signals the race actually fired. Rewrite the bats unit tests to cover: empty history, deployed-only, deployed+superseded (happy path - not detected), single failed, single uninstalled, uninstalled-then-deployed, and deployed-then-failed (all detected). The pinned-shape test feeds a realistic HR status snippet through yq the same way run-kubernetes.sh does via kubectl -o jsonpath. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- hack/e2e-apps/remediation-guard.sh | 45 +++++--- hack/e2e-apps/run-kubernetes.sh | 24 +++-- hack/remediation-guard.bats | 158 ++++++++++++++--------------- 3 files changed, 120 insertions(+), 107 deletions(-) diff --git a/hack/e2e-apps/remediation-guard.sh b/hack/e2e-apps/remediation-guard.sh index 6da69f13..38463310 100644 --- a/hack/e2e-apps/remediation-guard.sh +++ b/hack/e2e-apps/remediation-guard.sh @@ -1,24 +1,39 @@ # Helpers for asserting that a Flux HelmRelease did not fall into an # install/upgrade remediation cycle during an e2e run. # -# A non-zero installFailures/upgradeFailures counter means flux -# helm-controller hit its wait timeout, ran remediation (uninstall), -# and re-installed. That is exactly the race this guard is meant to -# catch, so the function returns success (0) when a cycle is detected -# and failure (1) otherwise. +# Background: Flux helm-controller's ClearFailures() zeroes +# .status.installFailures / .status.upgradeFailures on every successful +# reconciliation (see the upstream ClearFailures method on +# HelmReleaseStatus). That makes those counters useless for a guard that +# runs after the HelmRelease has reached Ready - the values are always 0. # -# Both arguments may be empty strings, the literal "0", or a positive -# integer. Shell's && and || have equal precedence with left-to-right -# associativity, so each half of the disjunction is grouped explicitly -# to avoid (A && B) || C && D parsing that masks the common -# install_failures=1, upgrade_failures="" case. +# What survives a successful reconciliation is .status.history, a bounded +# list of release Snapshots. Each Snapshot carries a status field that +# tracks the Helm release state: deployed, superseded, failed, uninstalled, +# and so on. A remediation cycle leaves the footprint behind: a snapshot +# with status "uninstalled" (from install/upgrade remediation) or "failed" +# (Helm release failure that remediation then uninstalled). Those stay in +# history even after a subsequent successful reinstall. +# +# helmrelease_has_remediation_cycle takes a newline-delimited list of +# snapshot statuses (whatever the caller extracted via kubectl -o jsonpath +# or equivalent) and returns 0 (detected) when any entry is "failed" or +# "uninstalled", 1 otherwise. Empty input is treated as "no history yet, +# no cycle observed". helmrelease_has_remediation_cycle() { - install_failures="$1" - upgrade_failures="$2" - if { [ -n "${install_failures}" ] && [ "${install_failures}" != "0" ]; } || \ - { [ -n "${upgrade_failures}" ] && [ "${upgrade_failures}" != "0" ]; }; then - return 0 + statuses="$1" + if [ -z "${statuses}" ]; then + return 1 fi + while IFS= read -r status; do + case "${status}" in + failed|uninstalled) + return 0 + ;; + esac + done <&2 + # Flux helm-controller v2 retains per-revision release Snapshots in + # .status.history; each Snapshot's .status reflects the Helm release + # state (deployed/superseded/failed/uninstalled). A remediation cycle + # leaves a "failed" or "uninstalled" entry behind that survives a later + # successful reinstall, unlike the installFailures/upgradeFailures + # counters (which ClearFailures zeroes on every successful reconcile). + # The shape is pinned by hack/remediation-guard.bats; the upstream + # types are github.com/fluxcd/helm-controller/api v2 Snapshot. + history_statuses=$(kubectl get hr -n tenant-test "kubernetes-${test_name}" \ + -ojsonpath='{range .status.history[*]}{.status}{"\n"}{end}') + if helmrelease_has_remediation_cycle "${history_statuses}"; then + echo "Parent HelmRelease entered remediation cycle. History statuses:" >&2 + printf '%s\n' "${history_statuses}" >&2 kubectl -n tenant-test describe hr "kubernetes-${test_name}" >&2 exit 1 fi diff --git a/hack/remediation-guard.bats b/hack/remediation-guard.bats index 9c33452a..30e4e04c 100644 --- a/hack/remediation-guard.bats +++ b/hack/remediation-guard.bats @@ -2,17 +2,15 @@ # ----------------------------------------------------------------------------- # Unit tests for hack/e2e-apps/remediation-guard.sh # -# helmrelease_has_remediation_cycle is consumed from e2e tests to assert that -# the parent HelmRelease did not hit flux helm-controller's wait timeout and -# enter uninstall remediation. The function accepts two arguments (values of -# .status.installFailures and .status.upgradeFailures) and returns 0 when a -# remediation cycle is detected, 1 otherwise. +# helmrelease_has_remediation_cycle takes a newline-delimited list of +# HelmRelease history snapshot status values (deployed/superseded/failed/ +# uninstalled/...) and returns 0 when any entry is "failed" or "uninstalled" +# (meaning flux helm-controller performed install/upgrade remediation). # -# Each argument can be empty (controller never populated the field), "0" -# (populated but never failed), or a positive integer. Shell's && and || -# have equal precedence with left-to-right associativity, which used to -# break this check on the most common failure mode - install_failures=1 -# and upgrade_failures="". These tests pin the correct behavior. +# This is used by the e2e script after the HelmRelease reaches Ready. The +# failure/upgrade counters (.status.installFailures / .status.upgradeFailures) +# are useless there because flux's ClearFailures zeroes them on successful +# reconciliation; .status.history retains the snapshot trail. # # cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on # its own line; there is no bats `run` or `$status`. Assertions are @@ -21,83 +19,76 @@ # Run with: hack/cozytest.sh hack/remediation-guard.bats # ----------------------------------------------------------------------------- -@test "no counters set returns not-detected" { +@test "empty history returns not-detected" { . hack/e2e-apps/remediation-guard.sh - rc=0 - helmrelease_has_remediation_cycle "" "" || rc=$? - [ "$rc" -eq 1 ] + if helmrelease_has_remediation_cycle ""; then + echo "expected not-detected for empty history" >&2 + exit 1 + fi } -@test "both counters zero returns not-detected" { +@test "single deployed snapshot returns not-detected" { . hack/e2e-apps/remediation-guard.sh - rc=0 - helmrelease_has_remediation_cycle "0" "0" || rc=$? - [ "$rc" -eq 1 ] + if helmrelease_has_remediation_cycle "deployed"; then + echo "expected not-detected for deployed-only history" >&2 + exit 1 + fi } -@test "install zero upgrade empty returns not-detected" { +@test "deployed then superseded returns not-detected" { . hack/e2e-apps/remediation-guard.sh - rc=0 - helmrelease_has_remediation_cycle "0" "" || rc=$? - [ "$rc" -eq 1 ] + statuses=$(printf 'deployed\nsuperseded\n') + if helmrelease_has_remediation_cycle "${statuses}"; then + echo "expected not-detected for deployed+superseded history" >&2 + exit 1 + fi } -@test "install empty upgrade zero returns not-detected" { +@test "single failed snapshot returns detected" { . hack/e2e-apps/remediation-guard.sh - rc=0 - helmrelease_has_remediation_cycle "" "0" || rc=$? - [ "$rc" -eq 1 ] + if ! helmrelease_has_remediation_cycle "failed"; then + echo "expected detected when history contains failed snapshot" >&2 + exit 1 + fi } -@test "install one upgrade empty returns detected" { - # Canonical race: first install exceeded helm-wait, remediation fired, - # no upgrade has happened yet. +@test "single uninstalled snapshot returns detected" { + # The exact signature of the install-remediation race: the first install + # exceeded flux's wait budget, remediation uninstalled, the next retry + # eventually succeeded. History still carries the uninstalled snapshot. . hack/e2e-apps/remediation-guard.sh - rc=0 - helmrelease_has_remediation_cycle "1" "" || rc=$? - [ "$rc" -eq 0 ] + if ! helmrelease_has_remediation_cycle "uninstalled"; then + echo "expected detected when history contains uninstalled snapshot" >&2 + exit 1 + fi } -@test "install empty upgrade one returns detected" { +@test "uninstalled then deployed still returns detected" { . hack/e2e-apps/remediation-guard.sh - rc=0 - helmrelease_has_remediation_cycle "" "1" || rc=$? - [ "$rc" -eq 0 ] + statuses=$(printf 'uninstalled\ndeployed\n') + if ! helmrelease_has_remediation_cycle "${statuses}"; then + echo "expected detected despite later successful deploy" >&2 + exit 1 + fi } -@test "install two upgrade zero returns detected" { +@test "deployed then failed still returns detected" { . hack/e2e-apps/remediation-guard.sh - rc=0 - helmrelease_has_remediation_cycle "2" "0" || rc=$? - [ "$rc" -eq 0 ] + statuses=$(printf 'deployed\nfailed\n') + if ! helmrelease_has_remediation_cycle "${statuses}"; then + echo "expected detected when any entry is failed" >&2 + exit 1 + fi } -@test "install zero upgrade two returns detected" { - . hack/e2e-apps/remediation-guard.sh - rc=0 - helmrelease_has_remediation_cycle "0" "2" || rc=$? - [ "$rc" -eq 0 ] -} - -@test "both counters positive returns detected" { - . hack/e2e-apps/remediation-guard.sh - rc=0 - helmrelease_has_remediation_cycle "3" "5" || rc=$? - [ "$rc" -eq 0 ] -} - -@test "installFailures and upgradeFailures extraction pins HR v2 status shape" { - # Pins the Flux HelmRelease v2 status shape that run-kubernetes.sh relies - # on. If a future Flux version renames .status.installFailures (or - # .status.upgradeFailures), kubectl get -o jsonpath returns an empty - # string, the guard quietly says "no cycle", and real remediation loops - # slip past the e2e assertion. - # - # This test uses yq to read the exact path used in the e2e script. yq - # evaluates the same json-ish jsonpath against a pinned HR snippet, so - # the test fails loudly if the field ever disappears or moves. Cross - # reference: vendor/github.com/fluxcd/helm-controller/api/v2/ status - # struct field tags. +@test "installFailures extraction pins HR v2 status.history shape" { + # Pins the Flux HelmRelease v2 .status.history[].status shape that + # run-kubernetes.sh relies on. If a future flux release renames the + # field, the jsonpath returns nothing, the guard reports no cycle, + # and real remediation loops slip past the e2e assertion. This test + # uses yq to read the exact path used in the e2e script; the upstream + # Snapshot type lives at + # github.com/fluxcd/helm-controller/api/v2.Snapshot (via go.mod). tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT @@ -107,25 +98,30 @@ kind: HelmRelease metadata: name: kubernetes-test namespace: tenant-test -spec: - interval: 5m status: - installFailures: 2 - upgradeFailures: 0 - conditions: - - type: Ready - status: "False" - reason: UninstallSucceeded + history: + - name: kubernetes-test + namespace: tenant-test + version: 1 + status: uninstalled + - name: kubernetes-test + namespace: tenant-test + version: 2 + status: deployed YAML - install_failures=$(yq '.status.installFailures' "$tmp/hr.yaml") - upgrade_failures=$(yq '.status.upgradeFailures' "$tmp/hr.yaml") + # Default yq output is yaml scalar format, which for string values emits + # bare unquoted tokens - matching what kubectl -o jsonpath produces in + # e2e. Do not switch to JSON output here; that would quote the values + # and break the loop in helmrelease_has_remediation_cycle. + statuses=$(yq '.status.history[].status' "$tmp/hr.yaml") - [ "$install_failures" = "2" ] - [ "$upgrade_failures" = "0" ] + [ -n "$statuses" ] + echo "$statuses" | grep --quiet '^uninstalled$' . hack/e2e-apps/remediation-guard.sh - rc=0 - helmrelease_has_remediation_cycle "$install_failures" "$upgrade_failures" || rc=$? - [ "$rc" -eq 0 ] + if ! helmrelease_has_remediation_cycle "$statuses"; then + echo "expected detected for pinned HR snippet with uninstalled + deployed history" >&2 + exit 1 + fi } From 48312cc3696904235e87fcf2bc33d05e21a77b72 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 16 Apr 2026 21:29:49 +0300 Subject: [PATCH 070/250] build: wire go-unit-tests into make unit-tests CI runs make unit-tests on every PR, which already covers helm unittests and bats (hack/admin-kubeconfig-invariant.bats and hack/remediation-guard.bats are both picked up by the existing hack/*.bats glob). What was missing was any go test invocation. Add a go-unit-tests target scoped to pkg/registry, pkg/config, and pkg/cmd/server - the cozystack-api surface this repo actually owns and tests in-tree. Running go test ./... pulls in generated-code round-trip suites whose behavior is governed by generator tool versions outside this repo's control; those are better exercised from their own generator workflows. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- Makefile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0e94c5e6..10a55314 100644 --- a/Makefile +++ b/Makefile @@ -82,11 +82,19 @@ test: make -C packages/core/testing apply make -C packages/core/testing test -unit-tests: helm-unit-tests bats-unit-tests +unit-tests: helm-unit-tests bats-unit-tests go-unit-tests helm-unit-tests: hack/helm-unit-tests.sh +# Scoped go test over the cozystack-api surface that this repo owns. Kept +# narrow intentionally - running `go test ./...` pulls in generated code +# round-trip suites whose behavior depends on tool versions outside this +# repo's control (kubebuilder, openapi-gen, etc.) and is better exercised +# from their generator workflows. +go-unit-tests: + go test ./pkg/registry/... ./pkg/config/... ./pkg/cmd/server/... + # Discover every hack/*.bats file that is NOT an e2e test and run it # through cozytest.sh. Drop a new *.bats file in hack/ and it is picked # up automatically on the next `make unit-tests` run. From d2e8f7e86cc7157aa83a7130b3f9857f10518218 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 16 Apr 2026 21:31:29 +0300 Subject: [PATCH 071/250] chore(kubernetes): drop busybox mirror Containerfile, pin upstream directly The image-busybox Makefile target + images/busybox/Dockerfile wrapper just rebuilt an upstream busybox digest as ghcr.io/cozystack/cozystack/ busybox. Payload is a one-shot sh loop run once per pod; the pinned upstream digest is already immutable, so maintaining a private mirror adds churn (rebuild on every release) for no real hardening benefit. Delete the wrapper and reference docker.io/library/busybox: directly from images/busybox.tag. Document the choice in _helpers.tpl. Also drop the false coupling in the go table test: the "unrelated kind without configured timeout" case used the real kind name Qdrant, which tied the test to the Qdrant ApplicationDefinition for no reason. Switch to a clearly fictional kind so a future Qdrant timeout override does not break this assertion. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- packages/apps/kubernetes/Makefile | 13 +------------ packages/apps/kubernetes/images/busybox/Dockerfile | 1 - packages/apps/kubernetes/templates/_helpers.tpl | 9 ++++++++- pkg/registry/apps/application/rest_timeout_test.go | 10 +++++++--- 4 files changed, 16 insertions(+), 17 deletions(-) delete mode 100644 packages/apps/kubernetes/images/busybox/Dockerfile diff --git a/packages/apps/kubernetes/Makefile b/packages/apps/kubernetes/Makefile index eb23c2ef..4fea42ef 100644 --- a/packages/apps/kubernetes/Makefile +++ b/packages/apps/kubernetes/Makefile @@ -15,7 +15,7 @@ update: hack/update-versions.sh make generate -image: image-ubuntu-container-disk image-kubevirt-cloud-provider image-kubevirt-csi-driver image-cluster-autoscaler image-busybox +image: image-ubuntu-container-disk image-kubevirt-cloud-provider image-kubevirt-csi-driver image-cluster-autoscaler image-ubuntu-container-disk: $(foreach ver,$(KUBERNETES_VERSIONS), \ @@ -71,14 +71,3 @@ image-cluster-autoscaler: > images/cluster-autoscaler.tag rm -f images/cluster-autoscaler.json -image-busybox: - docker buildx build images/busybox \ - --tag $(REGISTRY)/busybox:$(call settag,$(KUBERNETES_PKG_TAG)) \ - --tag $(REGISTRY)/busybox:$(call settag,$(KUBERNETES_PKG_TAG)-$(TAG)) \ - --cache-from type=registry,ref=$(REGISTRY)/busybox:latest \ - --cache-to type=inline \ - --metadata-file images/busybox.json \ - $(BUILDX_ARGS) - echo "$(REGISTRY)/busybox:$(call settag,$(KUBERNETES_PKG_TAG))@$$(yq e '."containerimage.digest"' images/busybox.json -o json -r)" \ - > images/busybox.tag - rm -f images/busybox.json diff --git a/packages/apps/kubernetes/images/busybox/Dockerfile b/packages/apps/kubernetes/images/busybox/Dockerfile deleted file mode 100644 index 0c57fd39..00000000 --- a/packages/apps/kubernetes/images/busybox/Dockerfile +++ /dev/null @@ -1 +0,0 @@ -FROM docker.io/library/busybox:1.37.0@sha256:1487d0af5f52b4ba31c7e465126ee2123fe3f2305d638e7827681e7cf6c83d5e diff --git a/packages/apps/kubernetes/templates/_helpers.tpl b/packages/apps/kubernetes/templates/_helpers.tpl index 89f06934..e6ad9dcf 100644 --- a/packages/apps/kubernetes/templates/_helpers.tpl +++ b/packages/apps/kubernetes/templates/_helpers.tpl @@ -60,10 +60,17 @@ exits only when super-admin.svc appears, which happens after kubelet's optional-Secret refresh cycle. The 10m deadline stays strictly below the 15m HelmRelease -Install.Timeout scoped to the Kubernetes Application kind so the +Install.Timeout set by cozystack-api for the Kubernetes kind (via the +release.cozystack.io/helm-install-timeout annotation) so the CrashLoopBackOff surfaces before flux remediation fires and uninstalls the Cluster CR. +The pinned busybox image in images/busybox.tag points directly at +docker.io by digest (not mirrored to ghcr.io like the other .tag files +here): the payload is a one-shot sh loop, the digest pin makes the +pull immutable, and the cost of maintaining a private mirror of a tiny +upstream image that does not move often is not worth it. + Call site owns the surrounding volumes block; the kubeconfig volume must exist on the pod and mount at /etc/kubernetes/kubeconfig. */}} diff --git a/pkg/registry/apps/application/rest_timeout_test.go b/pkg/registry/apps/application/rest_timeout_test.go index 8474d1cb..85ac0b97 100644 --- a/pkg/registry/apps/application/rest_timeout_test.go +++ b/pkg/registry/apps/application/rest_timeout_test.go @@ -48,9 +48,13 @@ func TestConvertApplicationToHelmRelease_AppliesReleaseConfigTimeout(t *testing. wantSet: true, }, { - name: "Qdrant kind without configured timeout keeps flux defaults", - kind: "Qdrant", - prefix: "qdrant-", + // Fictional kind on purpose: the test is about the unset path + // regardless of which real Application kind ends up needing a + // timeout override. Using a real kind name would create false + // coupling to that Application's ApplicationDefinition. + name: "unrelated kind without configured timeout keeps flux defaults", + kind: "PlaceholderKindForDefaults", + prefix: "placeholder-", configured: 0, wantSet: false, }, From 64b216e10a2392394add9eb7fbf7e908f4c8864a Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 16 Apr 2026 21:46:01 +0300 Subject: [PATCH 072/250] fix(kubernetes): gate child HelmReleases on tenant etcd DataStore 17 child HelmReleases (cilium, coredns, csi, cert-manager, metrics-server, ...) referenced *-admin-kubeconfig via kubeConfig.secretRef and rendered even when _namespace.etcd was empty. On an etcd-less tenant each one sat in NotReady forever because the admin-kubeconfig Secret only exists after a KamajiControlPlane reconciles, and KamajiControlPlane now only renders when etcd is set. The outcome contradicted the "beacon only" contract claimed in the soft-skip commit. Extend the existing addon guards to also require _namespace.etcd, and wrap the four unconditional HelmReleases (csi, metrics-server, prometheus-operator-crds, volumesnapshot-crd) plus the always-on cilium/coredns HR resources in the same gate. Add an invariant bats test that renders the whole chart with etcd empty and asserts zero HelmReleases reference *-admin-kubeconfig. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- hack/admin-kubeconfig-invariant.bats | 40 +++++++++++++++++++ .../helmreleases/cert-manager-crds.yaml | 2 +- .../templates/helmreleases/cert-manager.yaml | 2 +- .../templates/helmreleases/cilium.yaml | 2 + .../templates/helmreleases/coredns.yaml | 2 + .../templates/helmreleases/csi.yaml | 2 + .../templates/helmreleases/fluxcd.yaml | 2 +- .../helmreleases/gateway-api-crds.yaml | 2 +- .../templates/helmreleases/gpu-operator.yaml | 2 +- .../templates/helmreleases/ingress-nginx.yaml | 2 +- .../helmreleases/metrics-server.yaml | 2 + .../helmreleases/monitoring-agents.yaml | 2 +- .../prometheus-operator-crds.yaml | 2 + .../templates/helmreleases/velero.yaml | 2 +- .../vertical-pod-autoscaler-crds.yaml | 2 +- .../helmreleases/vertical-pod-autoscaler.yaml | 2 +- .../victoria-metrics-operator.yaml | 2 +- .../helmreleases/volumesnapshot-crd.yaml | 2 + 18 files changed, 63 insertions(+), 11 deletions(-) diff --git a/hack/admin-kubeconfig-invariant.bats b/hack/admin-kubeconfig-invariant.bats index c027e2c4..4c809c41 100644 --- a/hack/admin-kubeconfig-invariant.bats +++ b/hack/admin-kubeconfig-invariant.bats @@ -115,3 +115,43 @@ echo "No admin-kubeconfig Deployments rendered for empty etcd (as expected)" } + +@test "chart emits zero admin-kubeconfig HelmReleases when tenant has no etcd DataStore" { + # Same principle as the Deployment variant above, extended to every child + # HelmRelease (cilium, coredns, csi, cert-manager, ...). They reference + # *-admin-kubeconfig via kubeConfig.secretRef and would otherwise sit in + # NotReady forever on an etcd-less tenant, polluting the HelmRelease list + # the operator sees and contradicting the "awaiting-etcd beacon only" + # contract of the soft-skip path. + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + + helm template invariant packages/apps/kubernetes \ + --namespace tenant-root \ + --set _namespace.etcd="" \ + --set _namespace.monitoring="" \ + --set _namespace.ingress="" \ + --set _namespace.seaweedfs="" \ + --set _namespace.host="" \ + --set _cluster.cluster-domain=cozy.local \ + --set 'nodeGroups=null' \ + 2>/dev/null > "$tmp/rendered.yaml" + + matched=$( + yq --output-format=json eval-all '.' "$tmp/rendered.yaml" \ + | jq -s ' + map(select(.kind == "HelmRelease")) | + map(select(.spec.kubeConfig.secretRef.name | test("-admin-kubeconfig$")?)) | + length + ' + ) + + if [ "$matched" -ne 0 ]; then + echo "Expected zero HelmReleases referencing *-admin-kubeconfig when etcd is empty, got $matched:" >&2 + yq --output-format=json eval-all '.' "$tmp/rendered.yaml" \ + | jq -s 'map(select(.kind == "HelmRelease") | .metadata.name)' >&2 + exit 1 + fi + + echo "No admin-kubeconfig HelmReleases rendered for empty etcd (as expected)" +} diff --git a/packages/apps/kubernetes/templates/helmreleases/cert-manager-crds.yaml b/packages/apps/kubernetes/templates/helmreleases/cert-manager-crds.yaml index be07a8b9..fd9dac7e 100644 --- a/packages/apps/kubernetes/templates/helmreleases/cert-manager-crds.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/cert-manager-crds.yaml @@ -1,4 +1,4 @@ -{{- if .Values.addons.certManager.enabled }} +{{- if and .Values.addons.certManager.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/cert-manager.yaml b/packages/apps/kubernetes/templates/helmreleases/cert-manager.yaml index 6857581a..700b666e 100644 --- a/packages/apps/kubernetes/templates/helmreleases/cert-manager.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/cert-manager.yaml @@ -8,7 +8,7 @@ cert-manager: {{- end }} {{- end }} -{{- if .Values.addons.certManager.enabled }} +{{- if and .Values.addons.certManager.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/cilium.yaml b/packages/apps/kubernetes/templates/helmreleases/cilium.yaml index d032f5b6..d8c90cbf 100644 --- a/packages/apps/kubernetes/templates/helmreleases/cilium.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/cilium.yaml @@ -14,6 +14,7 @@ cilium: {{- end }} {{- end }} +{{- if .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -55,3 +56,4 @@ spec: - name: {{ .Release.Name }}-gateway-api-crds namespace: {{ .Release.Namespace }} {{- end }} +{{- end }} diff --git a/packages/apps/kubernetes/templates/helmreleases/coredns.yaml b/packages/apps/kubernetes/templates/helmreleases/coredns.yaml index bdb6c682..0711a51d 100644 --- a/packages/apps/kubernetes/templates/helmreleases/coredns.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/coredns.yaml @@ -4,6 +4,7 @@ coredns: clusterIP: "10.95.0.10" {{- end }} +{{- if .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -42,3 +43,4 @@ spec: {{- end }} - name: {{ .Release.Name }}-cilium namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/packages/apps/kubernetes/templates/helmreleases/csi.yaml b/packages/apps/kubernetes/templates/helmreleases/csi.yaml index dd2c69a6..109d78e1 100644 --- a/packages/apps/kubernetes/templates/helmreleases/csi.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/csi.yaml @@ -1,3 +1,4 @@ +{{- if .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -39,3 +40,4 @@ spec: - name: {{ .Release.Name }} namespace: {{ .Release.Namespace }} {{- end }} +{{- end }} diff --git a/packages/apps/kubernetes/templates/helmreleases/fluxcd.yaml b/packages/apps/kubernetes/templates/helmreleases/fluxcd.yaml index 76499dfe..25fff01c 100644 --- a/packages/apps/kubernetes/templates/helmreleases/fluxcd.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/fluxcd.yaml @@ -1,4 +1,4 @@ -{{- if .Values.addons.fluxcd.enabled }} +{{- if and .Values.addons.fluxcd.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/gateway-api-crds.yaml b/packages/apps/kubernetes/templates/helmreleases/gateway-api-crds.yaml index 2bcc8d4d..b4172ed1 100644 --- a/packages/apps/kubernetes/templates/helmreleases/gateway-api-crds.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/gateway-api-crds.yaml @@ -1,4 +1,4 @@ -{{- if $.Values.addons.gatewayAPI.enabled }} +{{- if and $.Values.addons.gatewayAPI.enabled $.Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml index 5ef48912..e243bfad 100644 --- a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml @@ -1,4 +1,4 @@ -{{- if .Values.addons.gpuOperator.enabled }} +{{- if and .Values.addons.gpuOperator.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/ingress-nginx.yaml b/packages/apps/kubernetes/templates/helmreleases/ingress-nginx.yaml index 5cafff90..6e2183d3 100644 --- a/packages/apps/kubernetes/templates/helmreleases/ingress-nginx.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/ingress-nginx.yaml @@ -20,7 +20,7 @@ ingress-nginx: node-role.kubernetes.io/ingress-nginx: "" {{- end }} -{{- if .Values.addons.ingressNginx.enabled }} +{{- if and .Values.addons.ingressNginx.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/metrics-server.yaml b/packages/apps/kubernetes/templates/helmreleases/metrics-server.yaml index 3cc81a14..3e6f9660 100644 --- a/packages/apps/kubernetes/templates/helmreleases/metrics-server.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/metrics-server.yaml @@ -1,3 +1,4 @@ +{{- if .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -36,3 +37,4 @@ spec: namespace: {{ .Release.Namespace }} - name: {{ .Release.Name }}-prometheus-operator-crds namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/packages/apps/kubernetes/templates/helmreleases/monitoring-agents.yaml b/packages/apps/kubernetes/templates/helmreleases/monitoring-agents.yaml index ea84dec0..a811f7dd 100644 --- a/packages/apps/kubernetes/templates/helmreleases/monitoring-agents.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/monitoring-agents.yaml @@ -1,6 +1,6 @@ {{- $targetTenant := .Values._namespace.monitoring }} {{- $clusterDomain := (index .Values._cluster "cluster-domain") | default "cozy.local" }} -{{- if .Values.addons.monitoringAgents.enabled }} +{{- if and .Values.addons.monitoringAgents.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/prometheus-operator-crds.yaml b/packages/apps/kubernetes/templates/helmreleases/prometheus-operator-crds.yaml index 600a7994..3038a058 100644 --- a/packages/apps/kubernetes/templates/helmreleases/prometheus-operator-crds.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/prometheus-operator-crds.yaml @@ -1,3 +1,4 @@ +{{- if .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -31,3 +32,4 @@ spec: - name: {{ .Release.Name }} namespace: {{ .Release.Namespace }} {{- end }} +{{- end }} diff --git a/packages/apps/kubernetes/templates/helmreleases/velero.yaml b/packages/apps/kubernetes/templates/helmreleases/velero.yaml index ad236d53..781b9c49 100644 --- a/packages/apps/kubernetes/templates/helmreleases/velero.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/velero.yaml @@ -1,4 +1,4 @@ -{{- if .Values.addons.velero.enabled }} +{{- if and .Values.addons.velero.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler-crds.yaml b/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler-crds.yaml index a3b7a9b4..55a5faac 100644 --- a/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler-crds.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler-crds.yaml @@ -1,4 +1,4 @@ -{{- if .Values.addons.monitoringAgents.enabled }} +{{- if and .Values.addons.monitoringAgents.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler.yaml b/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler.yaml index 178df3e3..74fb5a39 100644 --- a/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler.yaml @@ -24,7 +24,7 @@ vertical-pod-autoscaler: memory: 1600Mi {{- end }} -{{- if .Values.addons.monitoringAgents.enabled }} +{{- if and .Values.addons.monitoringAgents.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/victoria-metrics-operator.yaml b/packages/apps/kubernetes/templates/helmreleases/victoria-metrics-operator.yaml index 99744277..7302f8f5 100644 --- a/packages/apps/kubernetes/templates/helmreleases/victoria-metrics-operator.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/victoria-metrics-operator.yaml @@ -1,4 +1,4 @@ -{{- if .Values.addons.monitoringAgents.enabled }} +{{- if and .Values.addons.monitoringAgents.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/volumesnapshot-crd.yaml b/packages/apps/kubernetes/templates/helmreleases/volumesnapshot-crd.yaml index 025f01b7..d50fd93c 100644 --- a/packages/apps/kubernetes/templates/helmreleases/volumesnapshot-crd.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/volumesnapshot-crd.yaml @@ -1,3 +1,4 @@ +{{- if .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -33,3 +34,4 @@ spec: - name: {{ .Release.Name }} namespace: {{ .Release.Namespace }} {{- end }} +{{- end }} From 6072723e1e03c0072fe3bd83e2a5723c50d6ebd8 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 16 Apr 2026 21:47:19 +0300 Subject: [PATCH 073/250] feat(config): extract + test annotation-timeout parser with Flux unit gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pulls the release.cozystack.io/helm-install-timeout parsing out of start.go into ParseHelmInstallTimeoutAnnotation in pkg/config. The helper rejects units that time.ParseDuration accepts but Flux helm-controller rejects (ns, us, µs): feeding one of those through would cause the HelmRelease admission webhook to reject the object at install time, giving a silent drop to flux defaults that is hard to debug. Fail loudly at cozystack-api startup instead. Adds a table-driven unit test covering: unset (empty), accepted units ms/s/m/h, compound 2h30m, decimal 1.5m, and the rejected shapes (bare digits, garbage, negative, ns/us/µs). The test lives in pkg/config so it runs under the existing go-unit-tests make target. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- pkg/cmd/server/start.go | 23 ++++---- pkg/config/config.go | 42 +++++++++++++- pkg/config/config_test.go | 119 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 173 insertions(+), 11 deletions(-) create mode 100644 pkg/config/config_test.go diff --git a/pkg/cmd/server/start.go b/pkg/cmd/server/start.go index 3e905735..7ebbbc89 100644 --- a/pkg/cmd/server/start.go +++ b/pkg/cmd/server/start.go @@ -174,17 +174,20 @@ func (o *CozyServerOptions) Complete() error { // the chart itself depends on (for example, the Kamaji-provisioned // admin-kubeconfig Secret for Kubernetes tenants) need a longer // wait budget than the Flux default. Consumed by the REST storage - // layer when building the HelmRelease Spec. - if raw, ok := crd.Annotations["release.cozystack.io/helm-install-timeout"]; ok && raw != "" { - d, err := time.ParseDuration(raw) - if err != nil { - return fmt.Errorf( - "ApplicationDefinition %q has invalid release.cozystack.io/helm-install-timeout %q: %w", - crd.Name, raw, err, - ) - } - release.HelmInstallTimeout = d + // layer when building the HelmRelease Spec. The parser rejects + // units Flux would reject at webhook time, so a bad annotation + // surfaces as a loud startup failure instead of a silent drop to + // defaults. + d, err := config.ParseHelmInstallTimeoutAnnotation( + crd.Annotations[config.HelmInstallTimeoutAnnotation], + ) + if err != nil { + return fmt.Errorf( + "ApplicationDefinition %q has invalid %s annotation: %w", + crd.Name, config.HelmInstallTimeoutAnnotation, err, + ) } + release.HelmInstallTimeout = d resource := config.Resource{ Application: config.ApplicationConfig{ Kind: crd.Spec.Application.Kind, diff --git a/pkg/config/config.go b/pkg/config/config.go index 16cf4f0c..21de27b2 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -16,7 +16,47 @@ limitations under the License. package config -import "time" +import ( + "fmt" + "regexp" + "time" +) + +// HelmInstallTimeoutAnnotation is the ApplicationDefinition metadata +// annotation key that overrides the Flux HelmRelease Install.Timeout and +// Upgrade.Timeout for a given Application kind. +const HelmInstallTimeoutAnnotation = "release.cozystack.io/helm-install-timeout" + +// helmTimeoutPattern mirrors the CRD validation pattern used by Flux +// helm-controller on HelmReleaseSpec.Install.Timeout (ms/s/m/h units only). +// time.ParseDuration accepts ns/us/µs, but Flux rejects them - parsing here +// with the same shape avoids feeding the controller a value it will later +// reject at webhook time. See +// github.com/fluxcd/helm-controller/api/v2 HelmReleaseSpec.Install.Timeout +// in the go module cache. +var helmTimeoutPattern = regexp.MustCompile(`^([0-9]+(\.[0-9]+)?(ms|s|m|h))+$`) + +// ParseHelmInstallTimeoutAnnotation parses the value of the +// release.cozystack.io/helm-install-timeout annotation. The empty string is +// treated as "unset" and returns (0, nil) so callers can leave +// HelmInstallTimeout zeroed and let flux defaults apply. Values accepted by +// time.ParseDuration but rejected by Flux (ns/us/µs) return a helpful +// error instead of silently parsing and failing later at HelmRelease +// admission. +func ParseHelmInstallTimeoutAnnotation(raw string) (time.Duration, error) { + if raw == "" { + return 0, nil + } + if !helmTimeoutPattern.MatchString(raw) { + return 0, fmt.Errorf("must match %s (Flux accepts ms/s/m/h units only), got %q", + helmTimeoutPattern, raw) + } + d, err := time.ParseDuration(raw) + if err != nil { + return 0, fmt.Errorf("time.ParseDuration(%q): %w", raw, err) + } + return d, nil +} // ResourceConfig represents the structure of the configuration file. type ResourceConfig struct { diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go new file mode 100644 index 00000000..eb6990af --- /dev/null +++ b/pkg/config/config_test.go @@ -0,0 +1,119 @@ +package config + +import ( + "strings" + "testing" + "time" +) + +// Cover the annotation parser used by cozystack-api at startup. The parser +// is consumed by pkg/cmd/server/start.go on every ApplicationDefinition; a +// typo here silently drops back to flux defaults and the Kubernetes tenant +// race described in cozystack#2412 reappears, so the table must exercise: +// - the unset path (empty string treated as "no override"), +// - every unit Flux accepts (ms, s, m, h), +// - compound forms (the CRD pattern accepts repeats), +// - units time.ParseDuration accepts but Flux rejects (ns, us, µs), +// - outright garbage. +func TestParseHelmInstallTimeoutAnnotation(t *testing.T) { + cases := []struct { + name string + input string + want time.Duration + wantErr bool + errMatch string + }{ + { + name: "empty string leaves flux defaults in place", + input: "", + want: 0, + }, + { + name: "minutes", + input: "15m", + want: 15 * time.Minute, + }, + { + name: "hours", + input: "1h", + want: time.Hour, + }, + { + name: "seconds", + input: "45s", + want: 45 * time.Second, + }, + { + name: "milliseconds", + input: "500ms", + want: 500 * time.Millisecond, + }, + { + name: "compound hour and minutes", + input: "2h30m", + want: 2*time.Hour + 30*time.Minute, + }, + { + name: "decimal minutes", + input: "1.5m", + want: 90 * time.Second, + }, + { + name: "nanoseconds rejected - Flux CRD pattern excludes ns", + input: "500ns", + wantErr: true, + errMatch: "Flux accepts ms/s/m/h units only", + }, + { + name: "microseconds rejected - Flux CRD pattern excludes us", + input: "500us", + wantErr: true, + errMatch: "Flux accepts ms/s/m/h units only", + }, + { + name: "microseconds unicode rejected", + input: "500µs", + wantErr: true, + errMatch: "Flux accepts ms/s/m/h units only", + }, + { + name: "bare digits rejected", + input: "15", + wantErr: true, + errMatch: "Flux accepts ms/s/m/h units only", + }, + { + name: "garbage rejected", + input: "abc", + wantErr: true, + errMatch: "Flux accepts ms/s/m/h units only", + }, + { + name: "negative rejected", + input: "-15m", + wantErr: true, + errMatch: "Flux accepts ms/s/m/h units only", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := ParseHelmInstallTimeoutAnnotation(tc.input) + if tc.wantErr { + if err == nil { + t.Fatalf("expected error, got duration=%v", got) + } + if tc.errMatch != "" && !strings.Contains(err.Error(), tc.errMatch) { + t.Errorf("error %q does not contain %q", err.Error(), tc.errMatch) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != tc.want { + t.Errorf("got %v, want %v", got, tc.want) + } + }) + } +} From b8aec9a9731885cae9217271381680f3e274f46f Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 16 Apr 2026 21:50:10 +0300 Subject: [PATCH 074/250] fix(kubernetes): history guard non-empty check + nits from review - Log .status.history regardless of content so a silently empty result (Flux field rename) shows up in CI logs, and treat empty history on a Ready HelmRelease as a distinct failure. A Ready HR by definition has at least one snapshot; anything else is a shape-drift signal. - Replace the unquoted heredoc in remediation-guard.sh with a printf | grep pipeline. printf %s treats statuses as literal payload (no $ expansion surprises for future callers), grep --quiet --extended-regexp returns exit status the caller can forward directly. - Share the etcd-absent values file between both invariant tests (packages/apps/kubernetes/tests/values-ci-no-etcd.yaml) instead of duplicating the --set block. - Fix typo "override applied" -> "override is applied" in the Kubernetes ApplicationDefinition. - Add a coupling comment in the ApplicationDefinition annotation that points at the wait-for-kubeconfig init deadline in _helpers.tpl, so a future operator raising the HR timeout updates the init deadline too. - Clarify the per-annotation timeout comment in rest.go so it stops implying the feature is Kubernetes-only (it is not - only today's one user is). Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- hack/admin-kubeconfig-invariant.bats | 16 ++------------ hack/e2e-apps/remediation-guard.sh | 18 ++++++++-------- hack/e2e-apps/run-kubernetes.sh | 12 +++++++++-- .../kubernetes/tests/values-ci-no-etcd.yaml | 9 ++++++++ .../kubernetes-rd/cozyrds/kubernetes.yaml | 8 ++++++- pkg/registry/apps/application/rest.go | 21 +++++++++++-------- 6 files changed, 49 insertions(+), 35 deletions(-) create mode 100644 packages/apps/kubernetes/tests/values-ci-no-etcd.yaml diff --git a/hack/admin-kubeconfig-invariant.bats b/hack/admin-kubeconfig-invariant.bats index 4c809c41..a4b98b00 100644 --- a/hack/admin-kubeconfig-invariant.bats +++ b/hack/admin-kubeconfig-invariant.bats @@ -85,13 +85,7 @@ helm template invariant packages/apps/kubernetes \ --namespace tenant-root \ - --set _namespace.etcd="" \ - --set _namespace.monitoring="" \ - --set _namespace.ingress="" \ - --set _namespace.seaweedfs="" \ - --set _namespace.host="" \ - --set _cluster.cluster-domain=cozy.local \ - --set 'nodeGroups=null' \ + --values packages/apps/kubernetes/tests/values-ci-no-etcd.yaml \ 2>/dev/null > "$tmp/rendered.yaml" matched=$( @@ -128,13 +122,7 @@ helm template invariant packages/apps/kubernetes \ --namespace tenant-root \ - --set _namespace.etcd="" \ - --set _namespace.monitoring="" \ - --set _namespace.ingress="" \ - --set _namespace.seaweedfs="" \ - --set _namespace.host="" \ - --set _cluster.cluster-domain=cozy.local \ - --set 'nodeGroups=null' \ + --values packages/apps/kubernetes/tests/values-ci-no-etcd.yaml \ 2>/dev/null > "$tmp/rendered.yaml" matched=$( diff --git a/hack/e2e-apps/remediation-guard.sh b/hack/e2e-apps/remediation-guard.sh index 38463310..b45561b9 100644 --- a/hack/e2e-apps/remediation-guard.sh +++ b/hack/e2e-apps/remediation-guard.sh @@ -26,14 +26,14 @@ helmrelease_has_remediation_cycle() { if [ -z "${statuses}" ]; then return 1 fi - while IFS= read -r status; do - case "${status}" in - failed|uninstalled) - return 0 - ;; - esac - done <}" + if [ -z "${history_statuses}" ]; then + echo "Unexpected empty .status.history on a Ready HelmRelease - Flux API shape may have changed." >&2 + kubectl -n tenant-test describe hr "kubernetes-${test_name}" >&2 + exit 1 + fi if helmrelease_has_remediation_cycle "${history_statuses}"; then - echo "Parent HelmRelease entered remediation cycle. History statuses:" >&2 - printf '%s\n' "${history_statuses}" >&2 + echo "Parent HelmRelease entered remediation cycle." >&2 kubectl -n tenant-test describe hr "kubernetes-${test_name}" >&2 exit 1 fi diff --git a/packages/apps/kubernetes/tests/values-ci-no-etcd.yaml b/packages/apps/kubernetes/tests/values-ci-no-etcd.yaml new file mode 100644 index 00000000..c7c8196f --- /dev/null +++ b/packages/apps/kubernetes/tests/values-ci-no-etcd.yaml @@ -0,0 +1,9 @@ +_namespace: + etcd: "" + monitoring: "" + ingress: "" + seaweedfs: "" + host: "" +_cluster: + cluster-domain: cozy.local +nodeGroups: null diff --git a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml index 1128abe8..aaa01247 100644 --- a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml +++ b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml @@ -7,8 +7,14 @@ metadata: # Secret is provisioned asynchronously. Cold Kamaji start (image pull + # etcd + apiserver Ready) plus admin-kubeconfig generation can exceed # Flux helm-controller's default wait budget, causing remediation loops - # that uninstall the Cluster CR. This override applied by cozystack-api + # that uninstall the Cluster CR. This override is applied by cozystack-api # to the HelmRelease Spec.Install.Timeout and Spec.Upgrade.Timeout. + # + # Coupling: the wait-for-kubeconfig init container in + # packages/apps/kubernetes/templates/_helpers.tpl hard-codes a 10m + # deadline chosen to stay strictly below this value so the pod's + # CrashLoopBackOff surfaces before flux remediation fires. If this + # annotation is raised, update that init deadline correspondingly. release.cozystack.io/helm-install-timeout: "15m" spec: application: diff --git a/pkg/registry/apps/application/rest.go b/pkg/registry/apps/application/rest.go index 406d3738..d233ecc3 100644 --- a/pkg/registry/apps/application/rest.go +++ b/pkg/registry/apps/application/rest.go @@ -1528,15 +1528,18 @@ func (r *REST) convertApplicationToHelmRelease(app *appsv1alpha1.Application) (* }, } - // Per-Application HelmRelease wait budget. When an ApplicationDefinition - // sets release.cozystack.io/helm-install-timeout, the annotation is - // parsed at startup into ReleaseConfig.HelmInstallTimeout and applied - // to both Install and Upgrade here. Applications that leave it unset - // (the common case) keep flux defaults, so their failed installs - // remediate on the normal cadence. Needed for the Kubernetes kind - // because its parent chart contains CAPI/Kamaji resources whose - // admin-kubeconfig Secret is provisioned asynchronously and Kamaji - // cold-start routinely exceeds flux's default wait budget. + // Per-Application HelmRelease wait budget. The mechanism is generic: + // an ApplicationDefinition that sets + // release.cozystack.io/helm-install-timeout gets Install.Timeout and + // Upgrade.Timeout populated from ReleaseConfig.HelmInstallTimeout + // (parsed at startup). Applications that leave it unset keep flux + // defaults so their failed installs remediate on the normal cadence. + // Today only kubernetes-rd carries the annotation because the + // Kubernetes Application's parent chart contains CAPI/Kamaji + // resources whose admin-kubeconfig Secret is provisioned + // asynchronously and Kamaji cold-start routinely exceeds flux's + // default wait budget; any future kind with the same shape can opt + // in by setting the same annotation. if r.releaseConfig.HelmInstallTimeout > 0 { timeout := metav1.Duration{Duration: r.releaseConfig.HelmInstallTimeout} helmRelease.Spec.Install.Timeout = &timeout From 39b8f0252b6fb07d510c83aacf48fc20d7fdd96c Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 16 Apr 2026 22:32:24 +0300 Subject: [PATCH 075/250] test(hack): rename remediation-guard bats test to match what it pins The test body asserts .status.history[].status extraction, but the test name still referenced the old installFailures counter (leftover from when the guard used that field before switching to status.history to avoid ClearFailures zeroing the counters on successful reconcile). Address review feedback from coderabbitai on hack/remediation-guard.bats:84: rename so grep for what the test actually pins matches the test name. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- hack/remediation-guard.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/remediation-guard.bats b/hack/remediation-guard.bats index 30e4e04c..092fe06d 100644 --- a/hack/remediation-guard.bats +++ b/hack/remediation-guard.bats @@ -81,7 +81,7 @@ fi } -@test "installFailures extraction pins HR v2 status.history shape" { +@test "status.history extraction pins HR v2 status.history shape" { # Pins the Flux HelmRelease v2 .status.history[].status shape that # run-kubernetes.sh relies on. If a future flux release renames the # field, the jsonpath returns nothing, the guard reports no cycle, From 9b54e4672364a0de7859c5273dd2b280374083ea Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Fri, 17 Apr 2026 11:32:44 +0500 Subject: [PATCH 076/250] fix(linstor): restrict linstor-gui to cozystack-cluster-admin group MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before this change oauth2-proxy fronting linstor-gui only enforced that the user could authenticate against the `cozy` Keycloak realm (`--email-domain=*`, no group restriction). Any realm user could reach the UI and, through it, the LINSTOR controller REST API — which the gatekeeper proxies with a static mTLS client cert and no per-user RBAC. Add `--allowed-group=cozystack-cluster-admin` and include `groups` in the OIDC scope so the claim is present at validation time. The `cozystack-cluster-admin` KeycloakRealmGroup and the `groups` client scope are already provisioned by keycloak-configure, so no cluster-wide changes are needed. Assisted-By: Claude Signed-off-by: Myasnikov Daniil --- packages/system/linstor-gui/README.md | 7 +++++-- packages/system/linstor-gui/templates/gatekeeper.yaml | 8 +++++++- packages/system/linstor-gui/tests/ingress_auth_test.yaml | 6 ++++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/packages/system/linstor-gui/README.md b/packages/system/linstor-gui/README.md index 2c9ac5d5..8f8053ab 100644 --- a/packages/system/linstor-gui/README.md +++ b/packages/system/linstor-gui/README.md @@ -13,8 +13,11 @@ using mTLS with the `linstor-client-tls` secret created by the `linstor` package The chart ships an `oauth2-proxy` based gatekeeper plus a `KeycloakClient` CRD so the UI can be published on `linstor-gui.` behind the cluster -Keycloak realm. Access is granted to any user who can authenticate against the -`cozy` realm. +Keycloak realm. Access is restricted to members of the +`cozystack-cluster-admin` Keycloak group — the same group that grants +cluster-admin RBAC on the host cluster. Authenticating against the `cozy` +realm alone is not sufficient; users outside that group receive a 403 from +oauth2-proxy before any request reaches the UI or the LINSTOR controller. To turn it on, add `linstor-gui` to `publishing.exposedServices` in the core `cozystack` values (same list that controls `dashboard`). OIDC must be diff --git a/packages/system/linstor-gui/templates/gatekeeper.yaml b/packages/system/linstor-gui/templates/gatekeeper.yaml index 0830a5fa..81451f41 100644 --- a/packages/system/linstor-gui/templates/gatekeeper.yaml +++ b/packages/system/linstor-gui/templates/gatekeeper.yaml @@ -13,6 +13,11 @@ Only rendered when oidc-enabled=true. If the cluster has OIDC disabled we deliberately do not ship an unauthenticated token-proxy fallback (unlike dashboard): linstor-gui surfaces raw storage state and should not be reachable via Ingress without Keycloak login. + +Access is further restricted to the cozystack-cluster-admin group +(--allowed-group below). The proxy authenticates with a static mTLS client +cert and LINSTOR itself has no per-user RBAC, so group membership is the +only boundary between a realm user and raw storage state. */ -}} {{- if eq $oidcEnabled "true" }} apiVersion: apps/v1 @@ -84,7 +89,8 @@ spec: - --cookie-secure=true - --cookie-secret=$(OAUTH2_PROXY_COOKIE_SECRET) - --skip-provider-button - - --scope=openid email profile offline_access + - --scope=openid email profile groups offline_access + - --allowed-group=cozystack-cluster-admin {{- if eq $oidcInsecureSkipVerify "true" }} - --ssl-insecure-skip-verify=true {{- end }} diff --git a/packages/system/linstor-gui/tests/ingress_auth_test.yaml b/packages/system/linstor-gui/tests/ingress_auth_test.yaml index fe735f50..2f0b1d73 100644 --- a/packages/system/linstor-gui/tests/ingress_auth_test.yaml +++ b/packages/system/linstor-gui/tests/ingress_auth_test.yaml @@ -109,6 +109,12 @@ tests: - contains: path: spec.template.spec.containers[0].args content: --oidc-issuer-url=https://keycloak.dev10.example.org/realms/cozy + - contains: + path: spec.template.spec.containers[0].args + content: --scope=openid email profile groups offline_access + - contains: + path: spec.template.spec.containers[0].args + content: --allowed-group=cozystack-cluster-admin - contains: path: spec.template.spec.containers[0].env content: From ee4e71b1393027376c3fc48a30d9262b2cf9fa69 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Mon, 13 Apr 2026 14:34:22 +0300 Subject: [PATCH 077/250] feat(bucket): add WorkloadMonitor for billing integration Add WorkloadMonitor CR to bucket Helm chart so that the billing pipeline can discover and track S3 buckets. Add instance label to BucketClaim metadata for WorkloadMonitor selector matching. Co-Authored-By: Claude Signed-off-by: ZverGuy --- packages/apps/bucket/templates/bucketclaim.yaml | 2 ++ packages/apps/bucket/templates/workloadmonitor.yaml | 13 +++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 packages/apps/bucket/templates/workloadmonitor.yaml diff --git a/packages/apps/bucket/templates/bucketclaim.yaml b/packages/apps/bucket/templates/bucketclaim.yaml index 2f57fd70..0639916f 100644 --- a/packages/apps/bucket/templates/bucketclaim.yaml +++ b/packages/apps/bucket/templates/bucketclaim.yaml @@ -4,6 +4,8 @@ apiVersion: objectstorage.k8s.io/v1alpha1 kind: BucketClaim metadata: name: {{ .Release.Name }} + labels: + app.kubernetes.io/instance: {{ .Release.Name }} spec: bucketClassName: {{ $seaweedfs }}{{- if $pool }}-{{ $pool }}{{- end }}{{- if .Values.locking }}-lock{{- end }} protocols: diff --git a/packages/apps/bucket/templates/workloadmonitor.yaml b/packages/apps/bucket/templates/workloadmonitor.yaml new file mode 100644 index 00000000..4e618ddd --- /dev/null +++ b/packages/apps/bucket/templates/workloadmonitor.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: cozystack.io/v1alpha1 +kind: WorkloadMonitor +metadata: + name: {{ $.Release.Name }} +spec: + replicas: 1 + minReplicas: 1 + kind: bucket + type: s3 + selector: + app.kubernetes.io/instance: {{ $.Release.Name }} + version: {{ $.Chart.Version }} From 92286213ab377aa4db4845c66b9a1b94f2f2a0ab Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Mon, 13 Apr 2026 14:41:10 +0300 Subject: [PATCH 078/250] feat(controller): add BucketClaim support to WorkloadMonitorReconciler Add reconcileBucketClaimForMonitor() that watches COSI BucketClaim objects and creates Workload CRDs with s3-buckets resource, following the same pattern as PVC and Service reconcilers. This enables the billing pipeline to discover and track S3 buckets per tenant. Changes: - Add COSI API types dependency (container-object-storage-interface-api) - Register cosiv1alpha1 scheme in controller main - Add BucketClaim watch in SetupWithManager - Add BucketClaim list + reconcile in Reconcile loop - Add RBAC annotation for objectstorage.k8s.io/bucketclaims - Add unit tests for BucketClaim reconciliation Co-Authored-By: Claude Signed-off-by: ZverGuy --- cmd/cozystack-controller/main.go | 2 + go.mod | 6 +- go.sum | 2 + .../controller/workloadmonitor_controller.go | 73 +++++++ .../workloadmonitor_controller_test.go | 204 ++++++++++++++++++ 5 files changed, 284 insertions(+), 3 deletions(-) diff --git a/cmd/cozystack-controller/main.go b/cmd/cozystack-controller/main.go index 77c4f9dc..bcaca9c9 100644 --- a/cmd/cozystack-controller/main.go +++ b/cmd/cozystack-controller/main.go @@ -42,6 +42,7 @@ import ( "github.com/cozystack/cozystack/internal/telemetry" helmv2 "github.com/fluxcd/helm-controller/api/v2" + cosiv1alpha1 "sigs.k8s.io/container-object-storage-interface-api/apis/objectstorage/v1alpha1" // +kubebuilder:scaffold:imports ) @@ -56,6 +57,7 @@ func init() { utilruntime.Must(cozystackiov1alpha1.AddToScheme(scheme)) utilruntime.Must(dashboard.AddToScheme(scheme)) utilruntime.Must(helmv2.AddToScheme(scheme)) + utilruntime.Must(cosiv1alpha1.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme } diff --git a/go.mod b/go.mod index 00e8e129..7b7d5bc4 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ module github.com/cozystack/cozystack go 1.25.0 require ( + github.com/cozystack/cozystack-scheduler/pkg/apis v0.1.1 github.com/emicklei/dot v1.10.0 github.com/fluxcd/helm-controller/api v1.4.3 github.com/fluxcd/source-controller/api v1.7.4 @@ -28,8 +29,10 @@ require ( k8s.io/klog/v2 v2.130.1 k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b k8s.io/utils v0.0.0-20250820121507-0af2bda4dd1d + sigs.k8s.io/container-object-storage-interface-api v0.1.0 sigs.k8s.io/controller-runtime v0.22.4 sigs.k8s.io/structured-merge-diff/v6 v6.3.0 + sigs.k8s.io/yaml v1.6.0 ) require ( @@ -42,10 +45,8 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect - github.com/cozystack/cozystack-scheduler/pkg/apis v0.1.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.12.2 // indirect - github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fluxcd/pkg/apis/acl v0.9.0 // indirect @@ -126,7 +127,6 @@ require ( sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/yaml v1.6.0 // indirect ) // See: issues.k8s.io/135537 diff --git a/go.sum b/go.sum index 52942562..81e1779f 100644 --- a/go.sum +++ b/go.sum @@ -321,6 +321,8 @@ k8s.io/utils v0.0.0-20250820121507-0af2bda4dd1d h1:wAhiDyZ4Tdtt7e46e9M5ZSAJ/MnPG k8s.io/utils v0.0.0-20250820121507-0af2bda4dd1d/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= +sigs.k8s.io/container-object-storage-interface-api v0.1.0 h1:8tB6JFQhbQIC1hwGQ+q4+tmSSNfjKemb7bFI6C0CK/4= +sigs.k8s.io/container-object-storage-interface-api v0.1.0/go.mod h1:YiB+i/UGkzqgODDhRG3u7jkbWkQcoUeLEJ7hwOT/2Qk= sigs.k8s.io/controller-runtime v0.22.4 h1:GEjV7KV3TY8e+tJ2LCTxUTanW4z/FmNB7l327UfMq9A= sigs.k8s.io/controller-runtime v0.22.4/go.mod h1:+QX1XUpTXN4mLoblf4tqr5CQcyHPAki2HLXqQMY6vh8= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= diff --git a/internal/controller/workloadmonitor_controller.go b/internal/controller/workloadmonitor_controller.go index 9a967e65..3b69d0fb 100644 --- a/internal/controller/workloadmonitor_controller.go +++ b/internal/controller/workloadmonitor_controller.go @@ -22,6 +22,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" + cosiv1alpha1 "sigs.k8s.io/container-object-storage-interface-api/apis/objectstorage/v1alpha1" ) // WorkloadMonitorReconciler reconciles a WorkloadMonitor object @@ -36,6 +37,12 @@ type WorkloadMonitorReconciler struct { // +kubebuilder:rbac:groups=cozystack.io,resources=workloads/status,verbs=get;update;patch // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch // +kubebuilder:rbac:groups=core,resources=persistentvolumeclaims,verbs=get;list;watch +// +kubebuilder:rbac:groups=objectstorage.k8s.io,resources=bucketclaims,verbs=get;list;watch + +// isBucketClaimReady checks if the BucketClaim has been provisioned. +func (r *WorkloadMonitorReconciler) isBucketClaimReady(bc *cosiv1alpha1.BucketClaim) bool { + return bc.Status.BucketReady +} // isServiceReady checks if the service has an external IP bound func (r *WorkloadMonitorReconciler) isServiceReady(svc *corev1.Service) bool { @@ -101,6 +108,47 @@ func updateOwnerReferences(obj metav1.Object, monitor client.Object) { obj.SetOwnerReferences(owners) } +// reconcileBucketClaimForMonitor creates or updates a Workload object for the given BucketClaim and WorkloadMonitor. +func (r *WorkloadMonitorReconciler) reconcileBucketClaimForMonitor( + ctx context.Context, + monitor *cozyv1alpha1.WorkloadMonitor, + bc cosiv1alpha1.BucketClaim, +) error { + logger := log.FromContext(ctx) + workload := &cozyv1alpha1.Workload{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("bucket-%s", bc.Name), + Namespace: bc.Namespace, + Labels: make(map[string]string, len(bc.Labels)), + }, + } + + resources := make(map[string]resource.Quantity) + resources["s3-buckets"] = resource.MustParse("1") + + _, err := ctrl.CreateOrUpdate(ctx, r.Client, workload, func() error { + updateOwnerReferences(workload.GetObjectMeta(), &bc) + + for k, v := range bc.Labels { + workload.Labels[k] = v + } + workload.Labels["workloads.cozystack.io/monitor"] = monitor.Name + + workload.Status.Kind = monitor.Spec.Kind + workload.Status.Type = monitor.Spec.Type + workload.Status.Resources = resources + workload.Status.Operational = r.isBucketClaimReady(&bc) + + return nil + }) + if err != nil { + logger.Error(err, "Failed to CreateOrUpdate Workload", "workload", workload.Name) + return err + } + + return nil +} + // reconcileServiceForMonitor creates or updates a Workload object for the given Service and WorkloadMonitor. func (r *WorkloadMonitorReconciler) reconcileServiceForMonitor( ctx context.Context, @@ -375,6 +423,26 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ } } + bucketClaimList := &cosiv1alpha1.BucketClaimList{} + if err := r.List( + ctx, + bucketClaimList, + client.InNamespace(monitor.Namespace), + client.MatchingLabels(monitor.Spec.Selector), + ); err != nil { + if !apierrors.IsNotFound(err) { + logger.Error(err, "Unable to list BucketClaims for WorkloadMonitor", "monitor", monitor.Name) + return ctrl.Result{}, err + } + } + + for _, bc := range bucketClaimList.Items { + if err := r.reconcileBucketClaimForMonitor(ctx, monitor, bc); err != nil { + logger.Error(err, "Failed to reconcile Workload for BucketClaim", "BucketClaim", bc.Name) + continue + } + } + // Update WorkloadMonitor status based on observed pods monitor.Status.ObservedReplicas = observedReplicas monitor.Status.AvailableReplicas = availableReplicas @@ -421,6 +489,11 @@ func (r *WorkloadMonitorReconciler) SetupWithManager(mgr ctrl.Manager) error { &corev1.PersistentVolumeClaim{}, handler.EnqueueRequestsFromMapFunc(mapObjectToMonitor(&corev1.PersistentVolumeClaim{}, r.Client)), ). + // Watch BucketClaims for S3 bucket billing + Watches( + &cosiv1alpha1.BucketClaim{}, + handler.EnqueueRequestsFromMapFunc(mapObjectToMonitor(&cosiv1alpha1.BucketClaim{}, r.Client)), + ). // Watch for changes to Workload objects we create (owned by WorkloadMonitor) Owns(&cozyv1alpha1.Workload{}). Complete(r) diff --git a/internal/controller/workloadmonitor_controller_test.go b/internal/controller/workloadmonitor_controller_test.go index e1ed0c0d..9df61cf3 100644 --- a/internal/controller/workloadmonitor_controller_test.go +++ b/internal/controller/workloadmonitor_controller_test.go @@ -6,9 +6,11 @@ import ( cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + cosiv1alpha1 "sigs.k8s.io/container-object-storage-interface-api/apis/objectstorage/v1alpha1" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -17,6 +19,7 @@ func TestReconcile_OperationalStatusPersisted(t *testing.T) { scheme := runtime.NewScheme() _ = cozyv1alpha1.AddToScheme(scheme) _ = corev1.AddToScheme(scheme) + _ = cosiv1alpha1.AddToScheme(scheme) minReplicas := int32(2) monitor := &cozyv1alpha1.WorkloadMonitor{ @@ -82,6 +85,7 @@ func TestReconcile_OperationalTrue_WhenEnoughReplicas(t *testing.T) { scheme := runtime.NewScheme() _ = cozyv1alpha1.AddToScheme(scheme) _ = corev1.AddToScheme(scheme) + _ = cosiv1alpha1.AddToScheme(scheme) minReplicas := int32(1) monitor := &cozyv1alpha1.WorkloadMonitor{ @@ -139,6 +143,7 @@ func TestReconcile_OperationalTrue_WhenNoMinReplicas(t *testing.T) { scheme := runtime.NewScheme() _ = cozyv1alpha1.AddToScheme(scheme) _ = corev1.AddToScheme(scheme) + _ = cosiv1alpha1.AddToScheme(scheme) monitor := &cozyv1alpha1.WorkloadMonitor{ ObjectMeta: metav1.ObjectMeta{ @@ -178,3 +183,202 @@ func TestReconcile_OperationalTrue_WhenNoMinReplicas(t *testing.T) { } } +func newTestScheme() *runtime.Scheme { + s := runtime.NewScheme() + _ = cozyv1alpha1.AddToScheme(s) + _ = corev1.AddToScheme(s) + _ = cosiv1alpha1.AddToScheme(s) + return s +} + +func TestReconcileBucketClaimCreatesWorkload(t *testing.T) { + s := newTestScheme() + + monitor := &cozyv1alpha1.WorkloadMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-bucket", + Namespace: "tenant-demo", + }, + Spec: cozyv1alpha1.WorkloadMonitorSpec{ + Kind: "bucket", + Type: "s3", + Selector: map[string]string{ + "app.kubernetes.io/instance": "my-bucket", + }, + }, + } + + bc := &cosiv1alpha1.BucketClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-bucket", + Namespace: "tenant-demo", + Labels: map[string]string{ + "app.kubernetes.io/instance": "my-bucket", + }, + }, + Spec: cosiv1alpha1.BucketClaimSpec{ + BucketClassName: "seaweedfs", + Protocols: []cosiv1alpha1.Protocol{cosiv1alpha1.ProtocolS3}, + }, + Status: cosiv1alpha1.BucketClaimStatus{ + BucketReady: true, + BucketName: "cosi-abc123", + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(s). + WithObjects(monitor, bc). + WithStatusSubresource(monitor). + Build() + + reconciler := &WorkloadMonitorReconciler{Client: fakeClient, Scheme: s} + req := reconcile.Request{NamespacedName: types.NamespacedName{ + Name: "my-bucket", + Namespace: "tenant-demo", + }} + + _, err := reconciler.Reconcile(context.TODO(), req) + if err != nil { + t.Fatalf("Reconcile returned error: %v", err) + } + + workload := &cozyv1alpha1.Workload{} + err = fakeClient.Get(context.TODO(), types.NamespacedName{ + Name: "bucket-my-bucket", + Namespace: "tenant-demo", + }, workload) + if err != nil { + t.Fatalf("expected Workload to be created, got error: %v", err) + } + + if workload.Status.Kind != "bucket" { + t.Errorf("expected Kind=bucket, got %q", workload.Status.Kind) + } + if workload.Status.Type != "s3" { + t.Errorf("expected Type=s3, got %q", workload.Status.Type) + } + if !workload.Status.Operational { + t.Error("expected Operational=true for ready BucketClaim") + } + + qty, ok := workload.Status.Resources["s3-buckets"] + if !ok { + t.Fatal("expected s3-buckets resource to be set") + } + if qty.Cmp(resource.MustParse("1")) != 0 { + t.Errorf("expected s3-buckets=1, got %s", qty.String()) + } +} + +func TestReconcileBucketClaimNotReady(t *testing.T) { + s := newTestScheme() + + monitor := &cozyv1alpha1.WorkloadMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-bucket", + Namespace: "tenant-demo", + }, + Spec: cozyv1alpha1.WorkloadMonitorSpec{ + Kind: "bucket", + Type: "s3", + Selector: map[string]string{ + "app.kubernetes.io/instance": "my-bucket", + }, + }, + } + + bc := &cosiv1alpha1.BucketClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-bucket", + Namespace: "tenant-demo", + Labels: map[string]string{ + "app.kubernetes.io/instance": "my-bucket", + }, + }, + Spec: cosiv1alpha1.BucketClaimSpec{ + BucketClassName: "seaweedfs", + Protocols: []cosiv1alpha1.Protocol{cosiv1alpha1.ProtocolS3}, + }, + Status: cosiv1alpha1.BucketClaimStatus{ + BucketReady: false, + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(s). + WithObjects(monitor, bc). + WithStatusSubresource(monitor). + Build() + + reconciler := &WorkloadMonitorReconciler{Client: fakeClient, Scheme: s} + req := reconcile.Request{NamespacedName: types.NamespacedName{ + Name: "my-bucket", + Namespace: "tenant-demo", + }} + + _, err := reconciler.Reconcile(context.TODO(), req) + if err != nil { + t.Fatalf("Reconcile returned error: %v", err) + } + + workload := &cozyv1alpha1.Workload{} + err = fakeClient.Get(context.TODO(), types.NamespacedName{ + Name: "bucket-my-bucket", + Namespace: "tenant-demo", + }, workload) + if err != nil { + t.Fatalf("expected Workload to be created, got error: %v", err) + } + + if workload.Status.Operational { + t.Error("expected Operational=false for not-ready BucketClaim") + } +} + +func TestReconcileNoBucketClaimSkips(t *testing.T) { + s := newTestScheme() + + monitor := &cozyv1alpha1.WorkloadMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-postgres", + Namespace: "tenant-demo", + }, + Spec: cozyv1alpha1.WorkloadMonitorSpec{ + Kind: "postgres", + Type: "postgres", + Selector: map[string]string{ + "app.kubernetes.io/instance": "my-postgres", + }, + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(s). + WithObjects(monitor). + WithStatusSubresource(monitor). + Build() + + reconciler := &WorkloadMonitorReconciler{Client: fakeClient, Scheme: s} + req := reconcile.Request{NamespacedName: types.NamespacedName{ + Name: "my-postgres", + Namespace: "tenant-demo", + }} + + _, err := reconciler.Reconcile(context.TODO(), req) + if err != nil { + t.Fatalf("Reconcile returned error: %v", err) + } + + workloadList := &cozyv1alpha1.WorkloadList{} + err = fakeClient.List(context.TODO(), workloadList) + if err != nil { + t.Fatalf("List returned error: %v", err) + } + + for _, w := range workloadList.Items { + if w.Status.Kind == "bucket" { + t.Error("expected no bucket workloads to be created for postgres monitor") + } + } +} From 0aab19f50003149d2bfc63137a09696b4777eb48 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Mon, 13 Apr 2026 14:45:06 +0300 Subject: [PATCH 079/250] feat(controller): add SeaweedFS bucket size metrics via Prometheus Query SeaweedFS_s3_bucket_size_bytes from a Prometheus-compatible API to populate s3-storage-bytes resource on bucket Workloads. The Prometheus URL is configurable via --prometheus-url flag. When set, bucket WorkloadMonitors are requeued every 60s to keep sizes current. When Prometheus is not configured, buckets still get tracked with s3-buckets=1 for existence-based billing. Co-Authored-By: Claude Signed-off-by: ZverGuy --- cmd/cozystack-controller/main.go | 9 +- .../controller/workloadmonitor_controller.go | 91 ++++++++++++- .../workloadmonitor_controller_test.go | 128 ++++++++++++++++++ 3 files changed, 225 insertions(+), 3 deletions(-) diff --git a/cmd/cozystack-controller/main.go b/cmd/cozystack-controller/main.go index bcaca9c9..1175e2b4 100644 --- a/cmd/cozystack-controller/main.go +++ b/cmd/cozystack-controller/main.go @@ -70,6 +70,7 @@ func main() { var disableTelemetry bool var telemetryEndpoint string var telemetryInterval string + var prometheusURL string var tlsOpts []func(*tls.Config) flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") @@ -87,6 +88,9 @@ func main() { "Endpoint for sending telemetry data") flag.StringVar(&telemetryInterval, "telemetry-interval", "15m", "Interval between telemetry data collection (e.g. 15m, 1h)") + flag.StringVar(&prometheusURL, "prometheus-url", "", + "Prometheus-compatible API URL for querying SeaweedFS bucket metrics (e.g. http://vmselect:8481). "+ + "If empty, S3 bucket size metrics are not collected.") opts := zap.Options{ Development: false, } @@ -180,8 +184,9 @@ func main() { } if err = (&controller.WorkloadMonitorReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + PrometheusURL: prometheusURL, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "WorkloadMonitor") os.Exit(1) diff --git a/internal/controller/workloadmonitor_controller.go b/internal/controller/workloadmonitor_controller.go index 3b69d0fb..960be5c4 100644 --- a/internal/controller/workloadmonitor_controller.go +++ b/internal/controller/workloadmonitor_controller.go @@ -4,7 +4,11 @@ import ( "context" "encoding/json" "fmt" + "io" + "net/http" + "net/url" "sort" + "time" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -29,6 +33,9 @@ import ( type WorkloadMonitorReconciler struct { client.Client Scheme *runtime.Scheme + // PrometheusURL is the base URL of a Prometheus-compatible API for querying + // SeaweedFS bucket metrics. If empty, bucket size metrics are not collected. + PrometheusURL string } // +kubebuilder:rbac:groups=cozystack.io,resources=workloadmonitors,verbs=get;list;watch;create;update;patch;delete @@ -108,6 +115,76 @@ func updateOwnerReferences(obj metav1.Object, monitor client.Object) { obj.SetOwnerReferences(owners) } +// queryBucketSizeBytes queries Prometheus for the logical size of a SeaweedFS bucket. +// Returns 0 if the metric is not available or PrometheusURL is not configured. +func (r *WorkloadMonitorReconciler) queryBucketSizeBytes(ctx context.Context, seaweedBucketName string) int64 { + if r.PrometheusURL == "" || seaweedBucketName == "" { + return 0 + } + logger := log.FromContext(ctx) + + query := fmt.Sprintf(`SeaweedFS_s3_bucket_size_bytes{bucket="%s"}`, seaweedBucketName) + u, err := url.Parse(r.PrometheusURL + "/api/v1/query") + if err != nil { + logger.Error(err, "Failed to parse Prometheus URL") + return 0 + } + u.RawQuery = url.Values{"query": {query}}.Encode() + + httpCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(httpCtx, http.MethodGet, u.String(), nil) + if err != nil { + logger.Error(err, "Failed to create Prometheus request") + return 0 + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + logger.V(1).Info("Failed to query Prometheus for bucket size", "bucket", seaweedBucketName, "error", err) + return 0 + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + logger.Error(err, "Failed to read Prometheus response") + return 0 + } + + // Parse Prometheus instant query response: + // {"status":"success","data":{"resultType":"vector","result":[{"metric":{...},"value":[timestamp,"value"]}]}} + var promResp struct { + Status string `json:"status"` + Data struct { + Result []struct { + Value [2]json.RawMessage `json:"value"` + } `json:"result"` + } `json:"data"` + } + if err := json.Unmarshal(body, &promResp); err != nil { + logger.Error(err, "Failed to parse Prometheus response") + return 0 + } + if promResp.Status != "success" || len(promResp.Data.Result) == 0 { + return 0 + } + + var valueStr string + if err := json.Unmarshal(promResp.Data.Result[0].Value[1], &valueStr); err != nil { + logger.Error(err, "Failed to parse Prometheus metric value") + return 0 + } + + qty, err := resource.ParseQuantity(valueStr) + if err != nil { + logger.Error(err, "Failed to parse metric value as quantity", "value", valueStr) + return 0 + } + return qty.Value() +} + // reconcileBucketClaimForMonitor creates or updates a Workload object for the given BucketClaim and WorkloadMonitor. func (r *WorkloadMonitorReconciler) reconcileBucketClaimForMonitor( ctx context.Context, @@ -126,6 +203,13 @@ func (r *WorkloadMonitorReconciler) reconcileBucketClaimForMonitor( resources := make(map[string]resource.Quantity) resources["s3-buckets"] = resource.MustParse("1") + // Query actual bucket size from SeaweedFS metrics via Prometheus. + // bc.Status.BucketName is the COSI Bucket name, which the COSI driver + // uses directly as the SeaweedFS bucket name. + if sizeBytes := r.queryBucketSizeBytes(ctx, bc.Status.BucketName); sizeBytes > 0 { + resources["s3-storage-bytes"] = *resource.NewQuantity(sizeBytes, resource.BinarySI) + } + _, err := ctrl.CreateOrUpdate(ctx, r.Client, workload, func() error { updateOwnerReferences(workload.GetObjectMeta(), &bc) @@ -470,7 +554,12 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, err } - // Return without requeue if we want purely event-driven reconciliations + // Requeue periodically if there are BucketClaims to keep sizes up to date. + // Bucket sizes come from Prometheus metrics that update every 60s. + if len(bucketClaimList.Items) > 0 && r.PrometheusURL != "" { + return ctrl.Result{RequeueAfter: 60 * time.Second}, nil + } + return ctrl.Result{}, nil } diff --git a/internal/controller/workloadmonitor_controller_test.go b/internal/controller/workloadmonitor_controller_test.go index 9df61cf3..fa7074cb 100644 --- a/internal/controller/workloadmonitor_controller_test.go +++ b/internal/controller/workloadmonitor_controller_test.go @@ -2,6 +2,9 @@ package controller import ( "context" + "fmt" + "net/http" + "net/http/httptest" "testing" cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" @@ -382,3 +385,128 @@ func TestReconcileNoBucketClaimSkips(t *testing.T) { } } } + +func TestQueryBucketSizeBytes(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"bucket":"cosi-abc123"},"value":[1713000000,"5368709120"]}]}}`) + })) + defer srv.Close() + + reconciler := &WorkloadMonitorReconciler{PrometheusURL: srv.URL} + size := reconciler.queryBucketSizeBytes(context.TODO(), "cosi-abc123") + if size != 5368709120 { + t.Errorf("expected 5368709120, got %d", size) + } +} + +func TestQueryBucketSizeBytesEmpty(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`) + })) + defer srv.Close() + + reconciler := &WorkloadMonitorReconciler{PrometheusURL: srv.URL} + size := reconciler.queryBucketSizeBytes(context.TODO(), "nonexistent") + if size != 0 { + t.Errorf("expected 0 for empty result, got %d", size) + } +} + +func TestQueryBucketSizeBytesNoPrometheus(t *testing.T) { + reconciler := &WorkloadMonitorReconciler{PrometheusURL: ""} + size := reconciler.queryBucketSizeBytes(context.TODO(), "cosi-abc123") + if size != 0 { + t.Errorf("expected 0 when PrometheusURL is empty, got %d", size) + } +} + +func TestReconcileBucketClaimWithPrometheus(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"bucket":"cosi-abc123"},"value":[1713000000,"1073741824"]}]}}`) + })) + defer srv.Close() + + s := newTestScheme() + + monitor := &cozyv1alpha1.WorkloadMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-bucket", + Namespace: "tenant-demo", + }, + Spec: cozyv1alpha1.WorkloadMonitorSpec{ + Kind: "bucket", + Type: "s3", + Selector: map[string]string{ + "app.kubernetes.io/instance": "my-bucket", + }, + }, + } + + bc := &cosiv1alpha1.BucketClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-bucket", + Namespace: "tenant-demo", + Labels: map[string]string{ + "app.kubernetes.io/instance": "my-bucket", + }, + }, + Spec: cosiv1alpha1.BucketClaimSpec{ + BucketClassName: "seaweedfs", + Protocols: []cosiv1alpha1.Protocol{cosiv1alpha1.ProtocolS3}, + }, + Status: cosiv1alpha1.BucketClaimStatus{ + BucketReady: true, + BucketName: "cosi-abc123", + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(s). + WithObjects(monitor, bc). + WithStatusSubresource(monitor). + Build() + + reconciler := &WorkloadMonitorReconciler{ + Client: fakeClient, + Scheme: s, + PrometheusURL: srv.URL, + } + req := reconcile.Request{NamespacedName: types.NamespacedName{ + Name: "my-bucket", + Namespace: "tenant-demo", + }} + + result, err := reconciler.Reconcile(context.TODO(), req) + if err != nil { + t.Fatalf("Reconcile returned error: %v", err) + } + + if result.RequeueAfter == 0 { + t.Error("expected RequeueAfter > 0 when PrometheusURL is set and buckets exist") + } + + workload := &cozyv1alpha1.Workload{} + err = fakeClient.Get(context.TODO(), types.NamespacedName{ + Name: "bucket-my-bucket", + Namespace: "tenant-demo", + }, workload) + if err != nil { + t.Fatalf("expected Workload to be created, got error: %v", err) + } + + sizeQty, ok := workload.Status.Resources["s3-storage-bytes"] + if !ok { + t.Fatal("expected s3-storage-bytes resource to be set") + } + if sizeQty.Value() != 1073741824 { + t.Errorf("expected s3-storage-bytes=1073741824 (1 GiB), got %d", sizeQty.Value()) + } + + bucketsQty, ok := workload.Status.Resources["s3-buckets"] + if !ok { + t.Fatal("expected s3-buckets resource to be set") + } + if bucketsQty.Cmp(resource.MustParse("1")) != 0 { + t.Errorf("expected s3-buckets=1, got %s", bucketsQty.String()) + } +} From ddbc5f6f830b3dcc33ebae3573b3513cb6b08494 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Mon, 13 Apr 2026 14:48:49 +0300 Subject: [PATCH 080/250] feat(chart): add prometheus-url flag to cozystack-controller deployment Pass --prometheus-url to the controller container when configured in values. This enables querying SeaweedFS bucket size metrics from a Prometheus-compatible API for S3 bucket billing. RBAC already covers BucketClaim access via existing wildcard rule (apiGroups: ['*'], resources: ['*'], verbs: ["get", "list", "watch"]). Co-Authored-By: Claude Signed-off-by: ZverGuy --- packages/system/cozystack-controller/templates/deployment.yaml | 3 +++ packages/system/cozystack-controller/values.yaml | 1 + 2 files changed, 4 insertions(+) diff --git a/packages/system/cozystack-controller/templates/deployment.yaml b/packages/system/cozystack-controller/templates/deployment.yaml index aab7bbe1..94567881 100644 --- a/packages/system/cozystack-controller/templates/deployment.yaml +++ b/packages/system/cozystack-controller/templates/deployment.yaml @@ -27,3 +27,6 @@ spec: {{- if .Values.cozystackController.disableTelemetry }} - --disable-telemetry {{- end }} + {{- if .Values.cozystackController.prometheusUrl }} + - --prometheus-url={{ .Values.cozystackController.prometheusUrl }} + {{- end }} diff --git a/packages/system/cozystack-controller/values.yaml b/packages/system/cozystack-controller/values.yaml index 6addae2b..2999730b 100644 --- a/packages/system/cozystack-controller/values.yaml +++ b/packages/system/cozystack-controller/values.yaml @@ -2,3 +2,4 @@ cozystackController: image: ghcr.io/cozystack/cozystack/cozystack-controller:v1.3.0-rc.1@sha256:5ab50893e9d0237d26f366c9d647da6337ca9b97bae764430571d4fb080f6200 debug: false disableTelemetry: false + prometheusUrl: "" From 04f340ab8461ffcedbbcad89fff162acca101133 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Mon, 13 Apr 2026 15:03:37 +0300 Subject: [PATCH 081/250] feat(controller): add physical storage size metric for S3 buckets Query SeaweedFS_s3_bucket_physical_size_bytes alongside the logical size metric. Physical size includes all replicas and reflects actual disk usage, while logical size reflects what the user stored. Refactor queryBucketSizeBytes into generic queryPrometheusMetric to reuse for both metrics. Co-Authored-By: Claude Signed-off-by: ZverGuy --- .../controller/workloadmonitor_controller.go | 22 +++++++++------ .../workloadmonitor_controller_test.go | 28 ++++++++++++++----- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/internal/controller/workloadmonitor_controller.go b/internal/controller/workloadmonitor_controller.go index 960be5c4..d7514552 100644 --- a/internal/controller/workloadmonitor_controller.go +++ b/internal/controller/workloadmonitor_controller.go @@ -115,21 +115,20 @@ func updateOwnerReferences(obj metav1.Object, monitor client.Object) { obj.SetOwnerReferences(owners) } -// queryBucketSizeBytes queries Prometheus for the logical size of a SeaweedFS bucket. +// queryPrometheusMetric queries a Prometheus-compatible API for a single instant value. // Returns 0 if the metric is not available or PrometheusURL is not configured. -func (r *WorkloadMonitorReconciler) queryBucketSizeBytes(ctx context.Context, seaweedBucketName string) int64 { - if r.PrometheusURL == "" || seaweedBucketName == "" { +func (r *WorkloadMonitorReconciler) queryPrometheusMetric(ctx context.Context, promQL string) int64 { + if r.PrometheusURL == "" { return 0 } logger := log.FromContext(ctx) - query := fmt.Sprintf(`SeaweedFS_s3_bucket_size_bytes{bucket="%s"}`, seaweedBucketName) u, err := url.Parse(r.PrometheusURL + "/api/v1/query") if err != nil { logger.Error(err, "Failed to parse Prometheus URL") return 0 } - u.RawQuery = url.Values{"query": {query}}.Encode() + u.RawQuery = url.Values{"query": {promQL}}.Encode() httpCtx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() @@ -142,7 +141,7 @@ func (r *WorkloadMonitorReconciler) queryBucketSizeBytes(ctx context.Context, se resp, err := http.DefaultClient.Do(req) if err != nil { - logger.V(1).Info("Failed to query Prometheus for bucket size", "bucket", seaweedBucketName, "error", err) + logger.V(1).Info("Failed to query Prometheus", "query", promQL, "error", err) return 0 } defer resp.Body.Close() @@ -203,11 +202,16 @@ func (r *WorkloadMonitorReconciler) reconcileBucketClaimForMonitor( resources := make(map[string]resource.Quantity) resources["s3-buckets"] = resource.MustParse("1") - // Query actual bucket size from SeaweedFS metrics via Prometheus. + // Query actual bucket sizes from SeaweedFS metrics via Prometheus. // bc.Status.BucketName is the COSI Bucket name, which the COSI driver // uses directly as the SeaweedFS bucket name. - if sizeBytes := r.queryBucketSizeBytes(ctx, bc.Status.BucketName); sizeBytes > 0 { - resources["s3-storage-bytes"] = *resource.NewQuantity(sizeBytes, resource.BinarySI) + if bn := bc.Status.BucketName; bn != "" { + if v := r.queryPrometheusMetric(ctx, fmt.Sprintf(`SeaweedFS_s3_bucket_size_bytes{bucket="%s"}`, bn)); v > 0 { + resources["s3-storage-bytes"] = *resource.NewQuantity(v, resource.BinarySI) + } + if v := r.queryPrometheusMetric(ctx, fmt.Sprintf(`SeaweedFS_s3_bucket_physical_size_bytes{bucket="%s"}`, bn)); v > 0 { + resources["s3-physical-storage-bytes"] = *resource.NewQuantity(v, resource.BinarySI) + } } _, err := ctrl.CreateOrUpdate(ctx, r.Client, workload, func() error { diff --git a/internal/controller/workloadmonitor_controller_test.go b/internal/controller/workloadmonitor_controller_test.go index fa7074cb..de9b78e5 100644 --- a/internal/controller/workloadmonitor_controller_test.go +++ b/internal/controller/workloadmonitor_controller_test.go @@ -386,35 +386,35 @@ func TestReconcileNoBucketClaimSkips(t *testing.T) { } } -func TestQueryBucketSizeBytes(t *testing.T) { +func TestQueryPrometheusMetric(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"bucket":"cosi-abc123"},"value":[1713000000,"5368709120"]}]}}`) })) defer srv.Close() reconciler := &WorkloadMonitorReconciler{PrometheusURL: srv.URL} - size := reconciler.queryBucketSizeBytes(context.TODO(), "cosi-abc123") + size := reconciler.queryPrometheusMetric(context.TODO(), `SeaweedFS_s3_bucket_size_bytes{bucket="cosi-abc123"}`) if size != 5368709120 { t.Errorf("expected 5368709120, got %d", size) } } -func TestQueryBucketSizeBytesEmpty(t *testing.T) { +func TestQueryPrometheusMetricEmpty(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`) })) defer srv.Close() reconciler := &WorkloadMonitorReconciler{PrometheusURL: srv.URL} - size := reconciler.queryBucketSizeBytes(context.TODO(), "nonexistent") + size := reconciler.queryPrometheusMetric(context.TODO(), `SeaweedFS_s3_bucket_size_bytes{bucket="nonexistent"}`) if size != 0 { t.Errorf("expected 0 for empty result, got %d", size) } } -func TestQueryBucketSizeBytesNoPrometheus(t *testing.T) { +func TestQueryPrometheusMetricNoURL(t *testing.T) { reconciler := &WorkloadMonitorReconciler{PrometheusURL: ""} - size := reconciler.queryBucketSizeBytes(context.TODO(), "cosi-abc123") + size := reconciler.queryPrometheusMetric(context.TODO(), `anything`) if size != 0 { t.Errorf("expected 0 when PrometheusURL is empty, got %d", size) } @@ -422,7 +422,13 @@ func TestQueryBucketSizeBytesNoPrometheus(t *testing.T) { func TestReconcileBucketClaimWithPrometheus(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"bucket":"cosi-abc123"},"value":[1713000000,"1073741824"]}]}}`) + query := r.URL.Query().Get("query") + switch { + case len(query) > 0 && query[0:len("SeaweedFS_s3_bucket_physical")] == "SeaweedFS_s3_bucket_physical": + fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"bucket":"cosi-abc123"},"value":[1713000000,"2147483648"]}]}}`) + default: + fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"bucket":"cosi-abc123"},"value":[1713000000,"1073741824"]}]}}`) + } })) defer srv.Close() @@ -509,4 +515,12 @@ func TestReconcileBucketClaimWithPrometheus(t *testing.T) { if bucketsQty.Cmp(resource.MustParse("1")) != 0 { t.Errorf("expected s3-buckets=1, got %s", bucketsQty.String()) } + + physQty, ok := workload.Status.Resources["s3-physical-storage-bytes"] + if !ok { + t.Fatal("expected s3-physical-storage-bytes resource to be set") + } + if physQty.Value() != 2147483648 { + t.Errorf("expected s3-physical-storage-bytes=2147483648 (2 GiB), got %d", physQty.Value()) + } } From bfdfe989e05c88b81569fad2f23aa9d0031c007a Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Mon, 13 Apr 2026 18:20:12 +0300 Subject: [PATCH 082/250] fix(controller): add HTTP status check and limit response body size - Check resp.StatusCode before parsing Prometheus response - Limit response body read to 1 MB via io.LimitReader - Use strings.HasPrefix in test instead of fragile slice indexing - Add TestQueryPrometheusMetricServerError for 500 responses Co-Authored-By: Claude Signed-off-by: ZverGuy --- .../controller/workloadmonitor_controller.go | 7 ++++++- .../workloadmonitor_controller_test.go | 16 +++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/internal/controller/workloadmonitor_controller.go b/internal/controller/workloadmonitor_controller.go index d7514552..900610e4 100644 --- a/internal/controller/workloadmonitor_controller.go +++ b/internal/controller/workloadmonitor_controller.go @@ -146,7 +146,12 @@ func (r *WorkloadMonitorReconciler) queryPrometheusMetric(ctx context.Context, p } defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK { + logger.V(1).Info("Prometheus returned non-OK status", "query", promQL, "status", resp.StatusCode) + return 0 + } + + body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) if err != nil { logger.Error(err, "Failed to read Prometheus response") return 0 diff --git a/internal/controller/workloadmonitor_controller_test.go b/internal/controller/workloadmonitor_controller_test.go index de9b78e5..e8feb0b0 100644 --- a/internal/controller/workloadmonitor_controller_test.go +++ b/internal/controller/workloadmonitor_controller_test.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "net/http/httptest" + "strings" "testing" cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" @@ -412,6 +413,19 @@ func TestQueryPrometheusMetricEmpty(t *testing.T) { } } +func TestQueryPrometheusMetricServerError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + + reconciler := &WorkloadMonitorReconciler{PrometheusURL: srv.URL} + size := reconciler.queryPrometheusMetric(context.TODO(), `SeaweedFS_s3_bucket_size_bytes{bucket="test"}`) + if size != 0 { + t.Errorf("expected 0 for server error, got %d", size) + } +} + func TestQueryPrometheusMetricNoURL(t *testing.T) { reconciler := &WorkloadMonitorReconciler{PrometheusURL: ""} size := reconciler.queryPrometheusMetric(context.TODO(), `anything`) @@ -424,7 +438,7 @@ func TestReconcileBucketClaimWithPrometheus(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { query := r.URL.Query().Get("query") switch { - case len(query) > 0 && query[0:len("SeaweedFS_s3_bucket_physical")] == "SeaweedFS_s3_bucket_physical": + case strings.HasPrefix(query, "SeaweedFS_s3_bucket_physical"): fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"bucket":"cosi-abc123"},"value":[1713000000,"2147483648"]}]}}`) default: fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"bucket":"cosi-abc123"},"value":[1713000000,"1073741824"]}]}}`) From 0e6b8f7ba1109e32dfa25f30a2863bd2731b3ac3 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Mon, 13 Apr 2026 18:23:59 +0300 Subject: [PATCH 083/250] fix(controller): address review findings for BucketClaim reconciler - Set replicas/minReplicas to 0 in bucket WorkloadMonitor (buckets have no pods, minReplicas=1 would mark monitor as not operational) - Remove dead IsNotFound check on List (List returns empty, not 404) - Trim trailing slash from PrometheusURL before path append - Add strings import for TrimRight Co-Authored-By: Claude Signed-off-by: ZverGuy --- internal/controller/workloadmonitor_controller.go | 9 ++++----- packages/apps/bucket/templates/workloadmonitor.yaml | 4 ++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/internal/controller/workloadmonitor_controller.go b/internal/controller/workloadmonitor_controller.go index 900610e4..1e011313 100644 --- a/internal/controller/workloadmonitor_controller.go +++ b/internal/controller/workloadmonitor_controller.go @@ -8,6 +8,7 @@ import ( "net/http" "net/url" "sort" + "strings" "time" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -123,7 +124,7 @@ func (r *WorkloadMonitorReconciler) queryPrometheusMetric(ctx context.Context, p } logger := log.FromContext(ctx) - u, err := url.Parse(r.PrometheusURL + "/api/v1/query") + u, err := url.Parse(strings.TrimRight(r.PrometheusURL, "/") + "/api/v1/query") if err != nil { logger.Error(err, "Failed to parse Prometheus URL") return 0 @@ -523,10 +524,8 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ client.InNamespace(monitor.Namespace), client.MatchingLabels(monitor.Spec.Selector), ); err != nil { - if !apierrors.IsNotFound(err) { - logger.Error(err, "Unable to list BucketClaims for WorkloadMonitor", "monitor", monitor.Name) - return ctrl.Result{}, err - } + logger.Error(err, "Unable to list BucketClaims for WorkloadMonitor", "monitor", monitor.Name) + return ctrl.Result{}, err } for _, bc := range bucketClaimList.Items { diff --git a/packages/apps/bucket/templates/workloadmonitor.yaml b/packages/apps/bucket/templates/workloadmonitor.yaml index 4e618ddd..a23f3147 100644 --- a/packages/apps/bucket/templates/workloadmonitor.yaml +++ b/packages/apps/bucket/templates/workloadmonitor.yaml @@ -4,8 +4,8 @@ kind: WorkloadMonitor metadata: name: {{ $.Release.Name }} spec: - replicas: 1 - minReplicas: 1 + replicas: 0 + minReplicas: 0 kind: bucket type: s3 selector: From 7f7f4b218d8e56066592af7326a0aaa0d39db89b Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Mon, 13 Apr 2026 18:42:50 +0300 Subject: [PATCH 084/250] refactor(controller): resolve Prometheus URL from namespace labels Replace static --prometheus-url flag with dynamic resolution from namespace.cozystack.io/monitoring label. Each tenant namespace knows which tenant hosts its monitoring stack, so the controller constructs the vmselect URL automatically. This correctly handles multi-tenant setups where different tenants may use different monitoring instances. - Remove PrometheusURL field from WorkloadMonitorReconciler struct - Remove --prometheus-url flag and prometheusUrl chart value - Add resolvePrometheusURL() that reads namespace label - queryPrometheusMetric() now accepts prometheusBaseURL as parameter - Add tests for resolvePrometheusURL with and without label Co-Authored-By: Claude Signed-off-by: ZverGuy --- cmd/cozystack-controller/main.go | 9 +- .../controller/workloadmonitor_controller.go | 47 ++++++-- .../workloadmonitor_controller_test.go | 111 +++++++++++------- .../templates/deployment.yaml | 3 - .../system/cozystack-controller/values.yaml | 1 - 5 files changed, 108 insertions(+), 63 deletions(-) diff --git a/cmd/cozystack-controller/main.go b/cmd/cozystack-controller/main.go index 1175e2b4..bcaca9c9 100644 --- a/cmd/cozystack-controller/main.go +++ b/cmd/cozystack-controller/main.go @@ -70,7 +70,6 @@ func main() { var disableTelemetry bool var telemetryEndpoint string var telemetryInterval string - var prometheusURL string var tlsOpts []func(*tls.Config) flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") @@ -88,9 +87,6 @@ func main() { "Endpoint for sending telemetry data") flag.StringVar(&telemetryInterval, "telemetry-interval", "15m", "Interval between telemetry data collection (e.g. 15m, 1h)") - flag.StringVar(&prometheusURL, "prometheus-url", "", - "Prometheus-compatible API URL for querying SeaweedFS bucket metrics (e.g. http://vmselect:8481). "+ - "If empty, S3 bucket size metrics are not collected.") opts := zap.Options{ Development: false, } @@ -184,9 +180,8 @@ func main() { } if err = (&controller.WorkloadMonitorReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - PrometheusURL: prometheusURL, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "WorkloadMonitor") os.Exit(1) diff --git a/internal/controller/workloadmonitor_controller.go b/internal/controller/workloadmonitor_controller.go index 1e011313..e86e5a75 100644 --- a/internal/controller/workloadmonitor_controller.go +++ b/internal/controller/workloadmonitor_controller.go @@ -30,13 +30,21 @@ import ( cosiv1alpha1 "sigs.k8s.io/container-object-storage-interface-api/apis/objectstorage/v1alpha1" ) +const ( + // namespaceMonitoringLabel is the namespace label that indicates which tenant + // namespace hosts the monitoring stack (VictoriaMetrics/Prometheus). + namespaceMonitoringLabel = "namespace.cozystack.io/monitoring" + // vmSelectService is the well-known service name for VictoriaMetrics vmselect + // within a monitoring namespace. Port 8481, path /select/0/prometheus. + vmSelectService = "vmselect-shortterm" + vmSelectPort = "8481" + vmSelectPath = "/select/0/prometheus" +) + // WorkloadMonitorReconciler reconciles a WorkloadMonitor object type WorkloadMonitorReconciler struct { client.Client Scheme *runtime.Scheme - // PrometheusURL is the base URL of a Prometheus-compatible API for querying - // SeaweedFS bucket metrics. If empty, bucket size metrics are not collected. - PrometheusURL string } // +kubebuilder:rbac:groups=cozystack.io,resources=workloadmonitors,verbs=get;list;watch;create;update;patch;delete @@ -116,15 +124,30 @@ func updateOwnerReferences(obj metav1.Object, monitor client.Object) { obj.SetOwnerReferences(owners) } +// resolvePrometheusURL returns the Prometheus-compatible API base URL for the given namespace. +// It reads the namespace.cozystack.io/monitoring label to find the monitoring namespace, +// then constructs the vmselect URL. Returns empty string if monitoring is not configured. +func (r *WorkloadMonitorReconciler) resolvePrometheusURL(ctx context.Context, namespace string) string { + ns := &corev1.Namespace{} + if err := r.Get(ctx, types.NamespacedName{Name: namespace}, ns); err != nil { + return "" + } + monitoringNS := ns.Labels[namespaceMonitoringLabel] + if monitoringNS == "" { + return "" + } + return fmt.Sprintf("http://%s.%s.svc:%s%s", vmSelectService, monitoringNS, vmSelectPort, vmSelectPath) +} + // queryPrometheusMetric queries a Prometheus-compatible API for a single instant value. -// Returns 0 if the metric is not available or PrometheusURL is not configured. -func (r *WorkloadMonitorReconciler) queryPrometheusMetric(ctx context.Context, promQL string) int64 { - if r.PrometheusURL == "" { +// Returns 0 if prometheusBaseURL is empty or the metric is not available. +func (r *WorkloadMonitorReconciler) queryPrometheusMetric(ctx context.Context, prometheusBaseURL, promQL string) int64 { + if prometheusBaseURL == "" { return 0 } logger := log.FromContext(ctx) - u, err := url.Parse(strings.TrimRight(r.PrometheusURL, "/") + "/api/v1/query") + u, err := url.Parse(strings.TrimRight(prometheusBaseURL, "/") + "/api/v1/query") if err != nil { logger.Error(err, "Failed to parse Prometheus URL") return 0 @@ -209,13 +232,17 @@ func (r *WorkloadMonitorReconciler) reconcileBucketClaimForMonitor( resources["s3-buckets"] = resource.MustParse("1") // Query actual bucket sizes from SeaweedFS metrics via Prometheus. + // The monitoring endpoint is resolved from the namespace label + // namespace.cozystack.io/monitoring, which points to the tenant + // namespace hosting VictoriaMetrics. // bc.Status.BucketName is the COSI Bucket name, which the COSI driver // uses directly as the SeaweedFS bucket name. if bn := bc.Status.BucketName; bn != "" { - if v := r.queryPrometheusMetric(ctx, fmt.Sprintf(`SeaweedFS_s3_bucket_size_bytes{bucket="%s"}`, bn)); v > 0 { + promURL := r.resolvePrometheusURL(ctx, bc.Namespace) + if v := r.queryPrometheusMetric(ctx, promURL, fmt.Sprintf(`SeaweedFS_s3_bucket_size_bytes{bucket="%s"}`, bn)); v > 0 { resources["s3-storage-bytes"] = *resource.NewQuantity(v, resource.BinarySI) } - if v := r.queryPrometheusMetric(ctx, fmt.Sprintf(`SeaweedFS_s3_bucket_physical_size_bytes{bucket="%s"}`, bn)); v > 0 { + if v := r.queryPrometheusMetric(ctx, promURL, fmt.Sprintf(`SeaweedFS_s3_bucket_physical_size_bytes{bucket="%s"}`, bn)); v > 0 { resources["s3-physical-storage-bytes"] = *resource.NewQuantity(v, resource.BinarySI) } } @@ -564,7 +591,7 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ // Requeue periodically if there are BucketClaims to keep sizes up to date. // Bucket sizes come from Prometheus metrics that update every 60s. - if len(bucketClaimList.Items) > 0 && r.PrometheusURL != "" { + if len(bucketClaimList.Items) > 0 { return ctrl.Result{RequeueAfter: 60 * time.Second}, nil } diff --git a/internal/controller/workloadmonitor_controller_test.go b/internal/controller/workloadmonitor_controller_test.go index e8feb0b0..c78efd18 100644 --- a/internal/controller/workloadmonitor_controller_test.go +++ b/internal/controller/workloadmonitor_controller_test.go @@ -5,7 +5,6 @@ import ( "fmt" "net/http" "net/http/httptest" - "strings" "testing" cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" @@ -393,8 +392,8 @@ func TestQueryPrometheusMetric(t *testing.T) { })) defer srv.Close() - reconciler := &WorkloadMonitorReconciler{PrometheusURL: srv.URL} - size := reconciler.queryPrometheusMetric(context.TODO(), `SeaweedFS_s3_bucket_size_bytes{bucket="cosi-abc123"}`) + reconciler := &WorkloadMonitorReconciler{} + size := reconciler.queryPrometheusMetric(context.TODO(), srv.URL, `SeaweedFS_s3_bucket_size_bytes{bucket="cosi-abc123"}`) if size != 5368709120 { t.Errorf("expected 5368709120, got %d", size) } @@ -406,8 +405,8 @@ func TestQueryPrometheusMetricEmpty(t *testing.T) { })) defer srv.Close() - reconciler := &WorkloadMonitorReconciler{PrometheusURL: srv.URL} - size := reconciler.queryPrometheusMetric(context.TODO(), `SeaweedFS_s3_bucket_size_bytes{bucket="nonexistent"}`) + reconciler := &WorkloadMonitorReconciler{} + size := reconciler.queryPrometheusMetric(context.TODO(), srv.URL, `SeaweedFS_s3_bucket_size_bytes{bucket="nonexistent"}`) if size != 0 { t.Errorf("expected 0 for empty result, got %d", size) } @@ -419,35 +418,78 @@ func TestQueryPrometheusMetricServerError(t *testing.T) { })) defer srv.Close() - reconciler := &WorkloadMonitorReconciler{PrometheusURL: srv.URL} - size := reconciler.queryPrometheusMetric(context.TODO(), `SeaweedFS_s3_bucket_size_bytes{bucket="test"}`) + reconciler := &WorkloadMonitorReconciler{} + size := reconciler.queryPrometheusMetric(context.TODO(), srv.URL, `SeaweedFS_s3_bucket_size_bytes{bucket="test"}`) if size != 0 { t.Errorf("expected 0 for server error, got %d", size) } } func TestQueryPrometheusMetricNoURL(t *testing.T) { - reconciler := &WorkloadMonitorReconciler{PrometheusURL: ""} - size := reconciler.queryPrometheusMetric(context.TODO(), `anything`) + reconciler := &WorkloadMonitorReconciler{} + size := reconciler.queryPrometheusMetric(context.TODO(), "", `anything`) if size != 0 { t.Errorf("expected 0 when PrometheusURL is empty, got %d", size) } } -func TestReconcileBucketClaimWithPrometheus(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - query := r.URL.Query().Get("query") - switch { - case strings.HasPrefix(query, "SeaweedFS_s3_bucket_physical"): - fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"bucket":"cosi-abc123"},"value":[1713000000,"2147483648"]}]}}`) - default: - fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"bucket":"cosi-abc123"},"value":[1713000000,"1073741824"]}]}}`) - } - })) - defer srv.Close() - +func TestResolvePrometheusURL(t *testing.T) { s := newTestScheme() + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tenant-demo", + Labels: map[string]string{ + "namespace.cozystack.io/monitoring": "tenant-root", + }, + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(s). + WithObjects(ns). + Build() + + reconciler := &WorkloadMonitorReconciler{Client: fakeClient, Scheme: s} + url := reconciler.resolvePrometheusURL(context.TODO(), "tenant-demo") + + expected := "http://vmselect-shortterm.tenant-root.svc:8481/select/0/prometheus" + if url != expected { + t.Errorf("expected %q, got %q", expected, url) + } +} + +func TestResolvePrometheusURLNoLabel(t *testing.T) { + s := newTestScheme() + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tenant-demo", + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(s). + WithObjects(ns). + Build() + + reconciler := &WorkloadMonitorReconciler{Client: fakeClient, Scheme: s} + url := reconciler.resolvePrometheusURL(context.TODO(), "tenant-demo") + + if url != "" { + t.Errorf("expected empty URL when no monitoring label, got %q", url) + } +} + +func TestReconcileBucketClaimRequeuesWhenBucketsExist(t *testing.T) { + s := newTestScheme() + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tenant-demo", + }, + } + monitor := &cozyv1alpha1.WorkloadMonitor{ ObjectMeta: metav1.ObjectMeta{ Name: "my-bucket", @@ -482,15 +524,11 @@ func TestReconcileBucketClaimWithPrometheus(t *testing.T) { fakeClient := fake.NewClientBuilder(). WithScheme(s). - WithObjects(monitor, bc). + WithObjects(ns, monitor, bc). WithStatusSubresource(monitor). Build() - reconciler := &WorkloadMonitorReconciler{ - Client: fakeClient, - Scheme: s, - PrometheusURL: srv.URL, - } + reconciler := &WorkloadMonitorReconciler{Client: fakeClient, Scheme: s} req := reconcile.Request{NamespacedName: types.NamespacedName{ Name: "my-bucket", Namespace: "tenant-demo", @@ -502,7 +540,7 @@ func TestReconcileBucketClaimWithPrometheus(t *testing.T) { } if result.RequeueAfter == 0 { - t.Error("expected RequeueAfter > 0 when PrometheusURL is set and buckets exist") + t.Error("expected RequeueAfter > 0 when buckets exist") } workload := &cozyv1alpha1.Workload{} @@ -514,12 +552,9 @@ func TestReconcileBucketClaimWithPrometheus(t *testing.T) { t.Fatalf("expected Workload to be created, got error: %v", err) } - sizeQty, ok := workload.Status.Resources["s3-storage-bytes"] - if !ok { - t.Fatal("expected s3-storage-bytes resource to be set") - } - if sizeQty.Value() != 1073741824 { - t.Errorf("expected s3-storage-bytes=1073741824 (1 GiB), got %d", sizeQty.Value()) + // Without monitoring label on namespace, only s3-buckets should be set (no size metrics) + if _, ok := workload.Status.Resources["s3-storage-bytes"]; ok { + t.Error("expected no s3-storage-bytes when monitoring is not configured") } bucketsQty, ok := workload.Status.Resources["s3-buckets"] @@ -529,12 +564,4 @@ func TestReconcileBucketClaimWithPrometheus(t *testing.T) { if bucketsQty.Cmp(resource.MustParse("1")) != 0 { t.Errorf("expected s3-buckets=1, got %s", bucketsQty.String()) } - - physQty, ok := workload.Status.Resources["s3-physical-storage-bytes"] - if !ok { - t.Fatal("expected s3-physical-storage-bytes resource to be set") - } - if physQty.Value() != 2147483648 { - t.Errorf("expected s3-physical-storage-bytes=2147483648 (2 GiB), got %d", physQty.Value()) - } } diff --git a/packages/system/cozystack-controller/templates/deployment.yaml b/packages/system/cozystack-controller/templates/deployment.yaml index 94567881..aab7bbe1 100644 --- a/packages/system/cozystack-controller/templates/deployment.yaml +++ b/packages/system/cozystack-controller/templates/deployment.yaml @@ -27,6 +27,3 @@ spec: {{- if .Values.cozystackController.disableTelemetry }} - --disable-telemetry {{- end }} - {{- if .Values.cozystackController.prometheusUrl }} - - --prometheus-url={{ .Values.cozystackController.prometheusUrl }} - {{- end }} diff --git a/packages/system/cozystack-controller/values.yaml b/packages/system/cozystack-controller/values.yaml index 2999730b..6addae2b 100644 --- a/packages/system/cozystack-controller/values.yaml +++ b/packages/system/cozystack-controller/values.yaml @@ -2,4 +2,3 @@ cozystackController: image: ghcr.io/cozystack/cozystack/cozystack-controller:v1.3.0-rc.1@sha256:5ab50893e9d0237d26f366c9d647da6337ca9b97bae764430571d4fb080f6200 debug: false disableTelemetry: false - prometheusUrl: "" From 40620b2911604392a392aa5f878a8cabaff01851 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Mon, 13 Apr 2026 19:04:00 +0300 Subject: [PATCH 085/250] fix(controller): fix nil map panic and scientific notation parsing - Initialize workload.Labels map inside CreateOrUpdate mutate function to prevent nil map panic when existing Workload has no labels - Use strconv.ParseFloat instead of resource.ParseQuantity for Prometheus metric values, which may use scientific notation (e.g. "1.048576e+06") that ParseQuantity does not support Co-Authored-By: Claude Signed-off-by: ZverGuy --- internal/controller/workloadmonitor_controller.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/internal/controller/workloadmonitor_controller.go b/internal/controller/workloadmonitor_controller.go index e86e5a75..61fe8a99 100644 --- a/internal/controller/workloadmonitor_controller.go +++ b/internal/controller/workloadmonitor_controller.go @@ -8,6 +8,7 @@ import ( "net/http" "net/url" "sort" + "strconv" "strings" "time" @@ -205,12 +206,12 @@ func (r *WorkloadMonitorReconciler) queryPrometheusMetric(ctx context.Context, p return 0 } - qty, err := resource.ParseQuantity(valueStr) + val, err := strconv.ParseFloat(valueStr, 64) if err != nil { - logger.Error(err, "Failed to parse metric value as quantity", "value", valueStr) + logger.Error(err, "Failed to parse metric value", "value", valueStr) return 0 } - return qty.Value() + return int64(val) } // reconcileBucketClaimForMonitor creates or updates a Workload object for the given BucketClaim and WorkloadMonitor. @@ -250,6 +251,9 @@ func (r *WorkloadMonitorReconciler) reconcileBucketClaimForMonitor( _, err := ctrl.CreateOrUpdate(ctx, r.Client, workload, func() error { updateOwnerReferences(workload.GetObjectMeta(), &bc) + if workload.Labels == nil { + workload.Labels = make(map[string]string) + } for k, v := range bc.Labels { workload.Labels[k] = v } From a6f74f3198bb4578139a433eb3036bb01827d1ea Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Tue, 14 Apr 2026 14:06:29 +0300 Subject: [PATCH 086/250] refactor(controller): remove s3-buckets count resource from bucket workloads Bucket existence is already tracked via LifetimeHours. The s3-buckets count resource produced a meaningless "s3-buckets-Hours" fallback type in billing. Only storage size metrics (s3-storage-bytes, s3-physical-storage-bytes) are now set on bucket Workloads. Co-Authored-By: Claude Signed-off-by: ZverGuy --- .../controller/workloadmonitor_controller.go | 1 - .../workloadmonitor_controller_test.go | 20 +++---------------- 2 files changed, 3 insertions(+), 18 deletions(-) diff --git a/internal/controller/workloadmonitor_controller.go b/internal/controller/workloadmonitor_controller.go index 61fe8a99..f1f64f29 100644 --- a/internal/controller/workloadmonitor_controller.go +++ b/internal/controller/workloadmonitor_controller.go @@ -230,7 +230,6 @@ func (r *WorkloadMonitorReconciler) reconcileBucketClaimForMonitor( } resources := make(map[string]resource.Quantity) - resources["s3-buckets"] = resource.MustParse("1") // Query actual bucket sizes from SeaweedFS metrics via Prometheus. // The monitoring endpoint is resolved from the namespace label diff --git a/internal/controller/workloadmonitor_controller_test.go b/internal/controller/workloadmonitor_controller_test.go index c78efd18..71294927 100644 --- a/internal/controller/workloadmonitor_controller_test.go +++ b/internal/controller/workloadmonitor_controller_test.go @@ -9,7 +9,6 @@ import ( cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -264,14 +263,6 @@ func TestReconcileBucketClaimCreatesWorkload(t *testing.T) { if !workload.Status.Operational { t.Error("expected Operational=true for ready BucketClaim") } - - qty, ok := workload.Status.Resources["s3-buckets"] - if !ok { - t.Fatal("expected s3-buckets resource to be set") - } - if qty.Cmp(resource.MustParse("1")) != 0 { - t.Errorf("expected s3-buckets=1, got %s", qty.String()) - } } func TestReconcileBucketClaimNotReady(t *testing.T) { @@ -552,16 +543,11 @@ func TestReconcileBucketClaimRequeuesWhenBucketsExist(t *testing.T) { t.Fatalf("expected Workload to be created, got error: %v", err) } - // Without monitoring label on namespace, only s3-buckets should be set (no size metrics) + // Without monitoring label on namespace, no size metrics should be set if _, ok := workload.Status.Resources["s3-storage-bytes"]; ok { t.Error("expected no s3-storage-bytes when monitoring is not configured") } - - bucketsQty, ok := workload.Status.Resources["s3-buckets"] - if !ok { - t.Fatal("expected s3-buckets resource to be set") - } - if bucketsQty.Cmp(resource.MustParse("1")) != 0 { - t.Errorf("expected s3-buckets=1, got %s", bucketsQty.String()) + if len(workload.Status.Resources) != 0 { + t.Errorf("expected empty resources without monitoring, got %v", workload.Status.Resources) } } From 260ce84e5f45357942389006a7d856947ecf7260 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Tue, 14 Apr 2026 14:19:52 +0300 Subject: [PATCH 087/250] fix(controller): distinguish empty buckets from monitoring unavailable Change queryPrometheusMetric to return (int64, bool) so callers can emit s3-storage-bytes=0 for empty buckets while omitting the field entirely when monitoring is not configured. Also: - Add RBAC marker for core/namespaces GET (used by resolvePrometheusURL) - Log namespace read errors instead of silently returning empty URL - Add TestQueryPrometheusMetricZeroValue test Co-Authored-By: Claude Signed-off-by: ZverGuy --- .../controller/workloadmonitor_controller.go | 35 ++++++++++------- .../workloadmonitor_controller_test.go | 39 ++++++++++++++----- 2 files changed, 49 insertions(+), 25 deletions(-) diff --git a/internal/controller/workloadmonitor_controller.go b/internal/controller/workloadmonitor_controller.go index f1f64f29..5b450d75 100644 --- a/internal/controller/workloadmonitor_controller.go +++ b/internal/controller/workloadmonitor_controller.go @@ -54,6 +54,7 @@ type WorkloadMonitorReconciler struct { // +kubebuilder:rbac:groups=cozystack.io,resources=workloads/status,verbs=get;update;patch // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch // +kubebuilder:rbac:groups=core,resources=persistentvolumeclaims,verbs=get;list;watch +// +kubebuilder:rbac:groups=core,resources=namespaces,verbs=get // +kubebuilder:rbac:groups=objectstorage.k8s.io,resources=bucketclaims,verbs=get;list;watch // isBucketClaimReady checks if the BucketClaim has been provisioned. @@ -129,8 +130,10 @@ func updateOwnerReferences(obj metav1.Object, monitor client.Object) { // It reads the namespace.cozystack.io/monitoring label to find the monitoring namespace, // then constructs the vmselect URL. Returns empty string if monitoring is not configured. func (r *WorkloadMonitorReconciler) resolvePrometheusURL(ctx context.Context, namespace string) string { + logger := log.FromContext(ctx) ns := &corev1.Namespace{} if err := r.Get(ctx, types.NamespacedName{Name: namespace}, ns); err != nil { + logger.V(1).Info("Failed to read namespace for monitoring resolution", "namespace", namespace, "error", err) return "" } monitoringNS := ns.Labels[namespaceMonitoringLabel] @@ -141,17 +144,19 @@ func (r *WorkloadMonitorReconciler) resolvePrometheusURL(ctx context.Context, na } // queryPrometheusMetric queries a Prometheus-compatible API for a single instant value. -// Returns 0 if prometheusBaseURL is empty or the metric is not available. -func (r *WorkloadMonitorReconciler) queryPrometheusMetric(ctx context.Context, prometheusBaseURL, promQL string) int64 { +// Returns the value and true if the metric was found, or 0 and false if the query +// failed or no metric exists. This distinction allows callers to emit a resource +// with value 0 for empty buckets vs omitting it when monitoring is unavailable. +func (r *WorkloadMonitorReconciler) queryPrometheusMetric(ctx context.Context, prometheusBaseURL, promQL string) (int64, bool) { if prometheusBaseURL == "" { - return 0 + return 0, false } logger := log.FromContext(ctx) u, err := url.Parse(strings.TrimRight(prometheusBaseURL, "/") + "/api/v1/query") if err != nil { logger.Error(err, "Failed to parse Prometheus URL") - return 0 + return 0, false } u.RawQuery = url.Values{"query": {promQL}}.Encode() @@ -161,25 +166,25 @@ func (r *WorkloadMonitorReconciler) queryPrometheusMetric(ctx context.Context, p req, err := http.NewRequestWithContext(httpCtx, http.MethodGet, u.String(), nil) if err != nil { logger.Error(err, "Failed to create Prometheus request") - return 0 + return 0, false } resp, err := http.DefaultClient.Do(req) if err != nil { logger.V(1).Info("Failed to query Prometheus", "query", promQL, "error", err) - return 0 + return 0, false } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { logger.V(1).Info("Prometheus returned non-OK status", "query", promQL, "status", resp.StatusCode) - return 0 + return 0, false } body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) if err != nil { logger.Error(err, "Failed to read Prometheus response") - return 0 + return 0, false } // Parse Prometheus instant query response: @@ -194,24 +199,24 @@ func (r *WorkloadMonitorReconciler) queryPrometheusMetric(ctx context.Context, p } if err := json.Unmarshal(body, &promResp); err != nil { logger.Error(err, "Failed to parse Prometheus response") - return 0 + return 0, false } if promResp.Status != "success" || len(promResp.Data.Result) == 0 { - return 0 + return 0, false } var valueStr string if err := json.Unmarshal(promResp.Data.Result[0].Value[1], &valueStr); err != nil { logger.Error(err, "Failed to parse Prometheus metric value") - return 0 + return 0, false } val, err := strconv.ParseFloat(valueStr, 64) if err != nil { logger.Error(err, "Failed to parse metric value", "value", valueStr) - return 0 + return 0, false } - return int64(val) + return int64(val), true } // reconcileBucketClaimForMonitor creates or updates a Workload object for the given BucketClaim and WorkloadMonitor. @@ -239,10 +244,10 @@ func (r *WorkloadMonitorReconciler) reconcileBucketClaimForMonitor( // uses directly as the SeaweedFS bucket name. if bn := bc.Status.BucketName; bn != "" { promURL := r.resolvePrometheusURL(ctx, bc.Namespace) - if v := r.queryPrometheusMetric(ctx, promURL, fmt.Sprintf(`SeaweedFS_s3_bucket_size_bytes{bucket="%s"}`, bn)); v > 0 { + if v, ok := r.queryPrometheusMetric(ctx, promURL, fmt.Sprintf(`SeaweedFS_s3_bucket_size_bytes{bucket="%s"}`, bn)); ok { resources["s3-storage-bytes"] = *resource.NewQuantity(v, resource.BinarySI) } - if v := r.queryPrometheusMetric(ctx, promURL, fmt.Sprintf(`SeaweedFS_s3_bucket_physical_size_bytes{bucket="%s"}`, bn)); v > 0 { + if v, ok := r.queryPrometheusMetric(ctx, promURL, fmt.Sprintf(`SeaweedFS_s3_bucket_physical_size_bytes{bucket="%s"}`, bn)); ok { resources["s3-physical-storage-bytes"] = *resource.NewQuantity(v, resource.BinarySI) } } diff --git a/internal/controller/workloadmonitor_controller_test.go b/internal/controller/workloadmonitor_controller_test.go index 71294927..f702c978 100644 --- a/internal/controller/workloadmonitor_controller_test.go +++ b/internal/controller/workloadmonitor_controller_test.go @@ -384,12 +384,31 @@ func TestQueryPrometheusMetric(t *testing.T) { defer srv.Close() reconciler := &WorkloadMonitorReconciler{} - size := reconciler.queryPrometheusMetric(context.TODO(), srv.URL, `SeaweedFS_s3_bucket_size_bytes{bucket="cosi-abc123"}`) + size, ok := reconciler.queryPrometheusMetric(context.TODO(), srv.URL, `SeaweedFS_s3_bucket_size_bytes{bucket="cosi-abc123"}`) + if !ok { + t.Fatal("expected ok=true for valid metric") + } if size != 5368709120 { t.Errorf("expected 5368709120, got %d", size) } } +func TestQueryPrometheusMetricZeroValue(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"bucket":"empty"},"value":[1713000000,"0"]}]}}`) + })) + defer srv.Close() + + reconciler := &WorkloadMonitorReconciler{} + size, ok := reconciler.queryPrometheusMetric(context.TODO(), srv.URL, `SeaweedFS_s3_bucket_size_bytes{bucket="empty"}`) + if !ok { + t.Fatal("expected ok=true for zero-value metric (empty bucket)") + } + if size != 0 { + t.Errorf("expected 0, got %d", size) + } +} + func TestQueryPrometheusMetricEmpty(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`) @@ -397,9 +416,9 @@ func TestQueryPrometheusMetricEmpty(t *testing.T) { defer srv.Close() reconciler := &WorkloadMonitorReconciler{} - size := reconciler.queryPrometheusMetric(context.TODO(), srv.URL, `SeaweedFS_s3_bucket_size_bytes{bucket="nonexistent"}`) - if size != 0 { - t.Errorf("expected 0 for empty result, got %d", size) + _, ok := reconciler.queryPrometheusMetric(context.TODO(), srv.URL, `SeaweedFS_s3_bucket_size_bytes{bucket="nonexistent"}`) + if ok { + t.Error("expected ok=false for empty result") } } @@ -410,17 +429,17 @@ func TestQueryPrometheusMetricServerError(t *testing.T) { defer srv.Close() reconciler := &WorkloadMonitorReconciler{} - size := reconciler.queryPrometheusMetric(context.TODO(), srv.URL, `SeaweedFS_s3_bucket_size_bytes{bucket="test"}`) - if size != 0 { - t.Errorf("expected 0 for server error, got %d", size) + _, ok := reconciler.queryPrometheusMetric(context.TODO(), srv.URL, `SeaweedFS_s3_bucket_size_bytes{bucket="test"}`) + if ok { + t.Error("expected ok=false for server error") } } func TestQueryPrometheusMetricNoURL(t *testing.T) { reconciler := &WorkloadMonitorReconciler{} - size := reconciler.queryPrometheusMetric(context.TODO(), "", `anything`) - if size != 0 { - t.Errorf("expected 0 when PrometheusURL is empty, got %d", size) + _, ok := reconciler.queryPrometheusMetric(context.TODO(), "", `anything`) + if ok { + t.Error("expected ok=false when PrometheusURL is empty") } } From ed9808fad6069a996db2ea3147d2ee762286c574 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Tue, 14 Apr 2026 14:44:38 +0300 Subject: [PATCH 088/250] perf(controller): resolve Prometheus URL once per reconcile, not per BucketClaim MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move resolvePrometheusURL call before the BucketClaim loop and pass the URL as parameter. Avoids redundant namespace lookups and reduces reconcile time from 2×N+1 to 2×N HTTP calls (where N = BucketClaims). Co-Authored-By: Claude Signed-off-by: ZverGuy --- internal/controller/workloadmonitor_controller.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/controller/workloadmonitor_controller.go b/internal/controller/workloadmonitor_controller.go index 5b450d75..cb070e0c 100644 --- a/internal/controller/workloadmonitor_controller.go +++ b/internal/controller/workloadmonitor_controller.go @@ -224,6 +224,7 @@ func (r *WorkloadMonitorReconciler) reconcileBucketClaimForMonitor( ctx context.Context, monitor *cozyv1alpha1.WorkloadMonitor, bc cosiv1alpha1.BucketClaim, + promURL string, ) error { logger := log.FromContext(ctx) workload := &cozyv1alpha1.Workload{ @@ -243,7 +244,6 @@ func (r *WorkloadMonitorReconciler) reconcileBucketClaimForMonitor( // bc.Status.BucketName is the COSI Bucket name, which the COSI driver // uses directly as the SeaweedFS bucket name. if bn := bc.Status.BucketName; bn != "" { - promURL := r.resolvePrometheusURL(ctx, bc.Namespace) if v, ok := r.queryPrometheusMetric(ctx, promURL, fmt.Sprintf(`SeaweedFS_s3_bucket_size_bytes{bucket="%s"}`, bn)); ok { resources["s3-storage-bytes"] = *resource.NewQuantity(v, resource.BinarySI) } @@ -563,8 +563,9 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, err } + bucketPromURL := r.resolvePrometheusURL(ctx, monitor.Namespace) for _, bc := range bucketClaimList.Items { - if err := r.reconcileBucketClaimForMonitor(ctx, monitor, bc); err != nil { + if err := r.reconcileBucketClaimForMonitor(ctx, monitor, bc, bucketPromURL); err != nil { logger.Error(err, "Failed to reconcile Workload for BucketClaim", "BucketClaim", bc.Name) continue } From 475d24b0296ccee0a0cd9b03553ab29df17d8d97 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Tue, 14 Apr 2026 14:47:15 +0300 Subject: [PATCH 089/250] perf(controller): batch all bucket metrics into single Prometheus query MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace 2×N per-bucket HTTP requests with a single query that fetches all SeaweedFS bucket size metrics at once: {__name__=~"SeaweedFS_s3_bucket_(size|physical_size)_bytes"} Results are keyed by bucket name in memory, then looked up per BucketClaim. This reduces HTTP round-trips from 2N+1 to 2 (one namespace lookup + one Prometheus query) regardless of bucket count. Co-Authored-By: Claude Signed-off-by: ZverGuy --- .../controller/workloadmonitor_controller.go | 113 +++++++++++------- .../workloadmonitor_controller_test.go | 72 +++++------ 2 files changed, 108 insertions(+), 77 deletions(-) diff --git a/internal/controller/workloadmonitor_controller.go b/internal/controller/workloadmonitor_controller.go index cb070e0c..63ca02cb 100644 --- a/internal/controller/workloadmonitor_controller.go +++ b/internal/controller/workloadmonitor_controller.go @@ -143,80 +143,109 @@ func (r *WorkloadMonitorReconciler) resolvePrometheusURL(ctx context.Context, na return fmt.Sprintf("http://%s.%s.svc:%s%s", vmSelectService, monitoringNS, vmSelectPort, vmSelectPath) } -// queryPrometheusMetric queries a Prometheus-compatible API for a single instant value. -// Returns the value and true if the metric was found, or 0 and false if the query -// failed or no metric exists. This distinction allows callers to emit a resource -// with value 0 for empty buckets vs omitting it when monitoring is unavailable. -func (r *WorkloadMonitorReconciler) queryPrometheusMetric(ctx context.Context, prometheusBaseURL, promQL string) (int64, bool) { +// bucketMetrics holds size metrics for a single bucket, keyed by metric name. +type bucketMetrics struct { + LogicalSize int64 + PhysicalSize int64 + HasLogical bool + HasPhysical bool +} + +// queryAllBucketMetrics fetches all SeaweedFS bucket size metrics in a single +// Prometheus query and returns them keyed by bucket name. This avoids 2×N HTTP +// round-trips when there are N buckets. +func (r *WorkloadMonitorReconciler) queryAllBucketMetrics(ctx context.Context, prometheusBaseURL string) map[string]*bucketMetrics { + result := make(map[string]*bucketMetrics) if prometheusBaseURL == "" { - return 0, false + return result } logger := log.FromContext(ctx) + query := `{__name__=~"SeaweedFS_s3_bucket_(size|physical_size)_bytes"}` u, err := url.Parse(strings.TrimRight(prometheusBaseURL, "/") + "/api/v1/query") if err != nil { logger.Error(err, "Failed to parse Prometheus URL") - return 0, false + return result } - u.RawQuery = url.Values{"query": {promQL}}.Encode() + u.RawQuery = url.Values{"query": {query}}.Encode() - httpCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + httpCtx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() req, err := http.NewRequestWithContext(httpCtx, http.MethodGet, u.String(), nil) if err != nil { logger.Error(err, "Failed to create Prometheus request") - return 0, false + return result } resp, err := http.DefaultClient.Do(req) if err != nil { - logger.V(1).Info("Failed to query Prometheus", "query", promQL, "error", err) - return 0, false + logger.V(1).Info("Failed to query Prometheus for bucket metrics", "error", err) + return result } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - logger.V(1).Info("Prometheus returned non-OK status", "query", promQL, "status", resp.StatusCode) - return 0, false + logger.V(1).Info("Prometheus returned non-OK status for bucket metrics", "status", resp.StatusCode) + return result } - body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20)) if err != nil { logger.Error(err, "Failed to read Prometheus response") - return 0, false + return result } - // Parse Prometheus instant query response: - // {"status":"success","data":{"resultType":"vector","result":[{"metric":{...},"value":[timestamp,"value"]}]}} var promResp struct { Status string `json:"status"` Data struct { Result []struct { - Value [2]json.RawMessage `json:"value"` + Metric map[string]string `json:"metric"` + Value [2]json.RawMessage `json:"value"` } `json:"result"` } `json:"data"` } if err := json.Unmarshal(body, &promResp); err != nil { logger.Error(err, "Failed to parse Prometheus response") - return 0, false + return result } - if promResp.Status != "success" || len(promResp.Data.Result) == 0 { - return 0, false + if promResp.Status != "success" { + return result } - var valueStr string - if err := json.Unmarshal(promResp.Data.Result[0].Value[1], &valueStr); err != nil { - logger.Error(err, "Failed to parse Prometheus metric value") - return 0, false + for _, r := range promResp.Data.Result { + bucket := r.Metric["bucket"] + metricName := r.Metric["__name__"] + if bucket == "" || metricName == "" { + continue + } + + var valueStr string + if err := json.Unmarshal(r.Value[1], &valueStr); err != nil { + continue + } + val, err := strconv.ParseFloat(valueStr, 64) + if err != nil { + continue + } + + bm, ok := result[bucket] + if !ok { + bm = &bucketMetrics{} + result[bucket] = bm + } + + switch metricName { + case "SeaweedFS_s3_bucket_size_bytes": + bm.LogicalSize = int64(val) + bm.HasLogical = true + case "SeaweedFS_s3_bucket_physical_size_bytes": + bm.PhysicalSize = int64(val) + bm.HasPhysical = true + } } - val, err := strconv.ParseFloat(valueStr, 64) - if err != nil { - logger.Error(err, "Failed to parse metric value", "value", valueStr) - return 0, false - } - return int64(val), true + return result } // reconcileBucketClaimForMonitor creates or updates a Workload object for the given BucketClaim and WorkloadMonitor. @@ -224,7 +253,7 @@ func (r *WorkloadMonitorReconciler) reconcileBucketClaimForMonitor( ctx context.Context, monitor *cozyv1alpha1.WorkloadMonitor, bc cosiv1alpha1.BucketClaim, - promURL string, + allMetrics map[string]*bucketMetrics, ) error { logger := log.FromContext(ctx) workload := &cozyv1alpha1.Workload{ @@ -237,18 +266,15 @@ func (r *WorkloadMonitorReconciler) reconcileBucketClaimForMonitor( resources := make(map[string]resource.Quantity) - // Query actual bucket sizes from SeaweedFS metrics via Prometheus. - // The monitoring endpoint is resolved from the namespace label - // namespace.cozystack.io/monitoring, which points to the tenant - // namespace hosting VictoriaMetrics. + // Look up pre-fetched bucket metrics by the SeaweedFS bucket name. // bc.Status.BucketName is the COSI Bucket name, which the COSI driver // uses directly as the SeaweedFS bucket name. - if bn := bc.Status.BucketName; bn != "" { - if v, ok := r.queryPrometheusMetric(ctx, promURL, fmt.Sprintf(`SeaweedFS_s3_bucket_size_bytes{bucket="%s"}`, bn)); ok { - resources["s3-storage-bytes"] = *resource.NewQuantity(v, resource.BinarySI) + if bm, ok := allMetrics[bc.Status.BucketName]; ok { + if bm.HasLogical { + resources["s3-storage-bytes"] = *resource.NewQuantity(bm.LogicalSize, resource.BinarySI) } - if v, ok := r.queryPrometheusMetric(ctx, promURL, fmt.Sprintf(`SeaweedFS_s3_bucket_physical_size_bytes{bucket="%s"}`, bn)); ok { - resources["s3-physical-storage-bytes"] = *resource.NewQuantity(v, resource.BinarySI) + if bm.HasPhysical { + resources["s3-physical-storage-bytes"] = *resource.NewQuantity(bm.PhysicalSize, resource.BinarySI) } } @@ -564,8 +590,9 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ } bucketPromURL := r.resolvePrometheusURL(ctx, monitor.Namespace) + allBucketMetrics := r.queryAllBucketMetrics(ctx, bucketPromURL) for _, bc := range bucketClaimList.Items { - if err := r.reconcileBucketClaimForMonitor(ctx, monitor, bc, bucketPromURL); err != nil { + if err := r.reconcileBucketClaimForMonitor(ctx, monitor, bc, allBucketMetrics); err != nil { logger.Error(err, "Failed to reconcile Workload for BucketClaim", "BucketClaim", bc.Name) continue } diff --git a/internal/controller/workloadmonitor_controller_test.go b/internal/controller/workloadmonitor_controller_test.go index f702c978..5bb3397e 100644 --- a/internal/controller/workloadmonitor_controller_test.go +++ b/internal/controller/workloadmonitor_controller_test.go @@ -377,69 +377,73 @@ func TestReconcileNoBucketClaimSkips(t *testing.T) { } } -func TestQueryPrometheusMetric(t *testing.T) { +func TestQueryAllBucketMetrics(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"bucket":"cosi-abc123"},"value":[1713000000,"5368709120"]}]}}`) + fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[ + {"metric":{"__name__":"SeaweedFS_s3_bucket_size_bytes","bucket":"bucket-aaa"},"value":[1713000000,"10485864"]}, + {"metric":{"__name__":"SeaweedFS_s3_bucket_physical_size_bytes","bucket":"bucket-aaa"},"value":[1713000000,"20971728"]}, + {"metric":{"__name__":"SeaweedFS_s3_bucket_size_bytes","bucket":"bucket-bbb"},"value":[1713000000,"0"]} + ]}}`) })) defer srv.Close() reconciler := &WorkloadMonitorReconciler{} - size, ok := reconciler.queryPrometheusMetric(context.TODO(), srv.URL, `SeaweedFS_s3_bucket_size_bytes{bucket="cosi-abc123"}`) + metrics := reconciler.queryAllBucketMetrics(context.TODO(), srv.URL) + + bm, ok := metrics["bucket-aaa"] if !ok { - t.Fatal("expected ok=true for valid metric") + t.Fatal("expected bucket-aaa in metrics") } - if size != 5368709120 { - t.Errorf("expected 5368709120, got %d", size) + if !bm.HasLogical || bm.LogicalSize != 10485864 { + t.Errorf("expected logical=10485864, got %d", bm.LogicalSize) + } + if !bm.HasPhysical || bm.PhysicalSize != 20971728 { + t.Errorf("expected physical=20971728, got %d", bm.PhysicalSize) + } + + bm2, ok := metrics["bucket-bbb"] + if !ok { + t.Fatal("expected bucket-bbb in metrics") + } + if !bm2.HasLogical || bm2.LogicalSize != 0 { + t.Errorf("expected logical=0 for empty bucket, got %d", bm2.LogicalSize) + } + if bm2.HasPhysical { + t.Error("expected no physical size for bucket-bbb") } } -func TestQueryPrometheusMetricZeroValue(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"bucket":"empty"},"value":[1713000000,"0"]}]}}`) - })) - defer srv.Close() - - reconciler := &WorkloadMonitorReconciler{} - size, ok := reconciler.queryPrometheusMetric(context.TODO(), srv.URL, `SeaweedFS_s3_bucket_size_bytes{bucket="empty"}`) - if !ok { - t.Fatal("expected ok=true for zero-value metric (empty bucket)") - } - if size != 0 { - t.Errorf("expected 0, got %d", size) - } -} - -func TestQueryPrometheusMetricEmpty(t *testing.T) { +func TestQueryAllBucketMetricsEmpty(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`) })) defer srv.Close() reconciler := &WorkloadMonitorReconciler{} - _, ok := reconciler.queryPrometheusMetric(context.TODO(), srv.URL, `SeaweedFS_s3_bucket_size_bytes{bucket="nonexistent"}`) - if ok { - t.Error("expected ok=false for empty result") + metrics := reconciler.queryAllBucketMetrics(context.TODO(), srv.URL) + if len(metrics) != 0 { + t.Errorf("expected empty metrics, got %d", len(metrics)) } } -func TestQueryPrometheusMetricServerError(t *testing.T) { +func TestQueryAllBucketMetricsServerError(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) })) defer srv.Close() reconciler := &WorkloadMonitorReconciler{} - _, ok := reconciler.queryPrometheusMetric(context.TODO(), srv.URL, `SeaweedFS_s3_bucket_size_bytes{bucket="test"}`) - if ok { - t.Error("expected ok=false for server error") + metrics := reconciler.queryAllBucketMetrics(context.TODO(), srv.URL) + if len(metrics) != 0 { + t.Errorf("expected empty metrics on error, got %d", len(metrics)) } } -func TestQueryPrometheusMetricNoURL(t *testing.T) { +func TestQueryAllBucketMetricsNoURL(t *testing.T) { reconciler := &WorkloadMonitorReconciler{} - _, ok := reconciler.queryPrometheusMetric(context.TODO(), "", `anything`) - if ok { - t.Error("expected ok=false when PrometheusURL is empty") + metrics := reconciler.queryAllBucketMetrics(context.TODO(), "") + if len(metrics) != 0 { + t.Errorf("expected empty metrics when URL is empty, got %d", len(metrics)) } } From 8183b4669a68aef38a606f0430101ead6b88439d Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Tue, 14 Apr 2026 14:59:45 +0300 Subject: [PATCH 090/250] perf(controller): skip Prometheus calls when no BucketClaims matched Wrap resolvePrometheusURL and queryAllBucketMetrics in a len(bucketClaimList.Items) > 0 guard. Avoids unnecessary namespace GET and HTTP request to Prometheus on every reconcile of non-bucket WorkloadMonitors (postgres, redis, kubernetes, etc.). Co-Authored-By: Claude Signed-off-by: ZverGuy --- internal/controller/workloadmonitor_controller.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/internal/controller/workloadmonitor_controller.go b/internal/controller/workloadmonitor_controller.go index 63ca02cb..af61ac47 100644 --- a/internal/controller/workloadmonitor_controller.go +++ b/internal/controller/workloadmonitor_controller.go @@ -589,12 +589,14 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, err } - bucketPromURL := r.resolvePrometheusURL(ctx, monitor.Namespace) - allBucketMetrics := r.queryAllBucketMetrics(ctx, bucketPromURL) - for _, bc := range bucketClaimList.Items { - if err := r.reconcileBucketClaimForMonitor(ctx, monitor, bc, allBucketMetrics); err != nil { - logger.Error(err, "Failed to reconcile Workload for BucketClaim", "BucketClaim", bc.Name) - continue + if len(bucketClaimList.Items) > 0 { + bucketPromURL := r.resolvePrometheusURL(ctx, monitor.Namespace) + allBucketMetrics := r.queryAllBucketMetrics(ctx, bucketPromURL) + for _, bc := range bucketClaimList.Items { + if err := r.reconcileBucketClaimForMonitor(ctx, monitor, bc, allBucketMetrics); err != nil { + logger.Error(err, "Failed to reconcile Workload for BucketClaim", "BucketClaim", bc.Name) + continue + } } } From c72fadec3e695154d64288c0a0150ef0f454db61 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Tue, 14 Apr 2026 16:30:20 +0300 Subject: [PATCH 091/250] fix(controller): scope Prometheus query to only matched bucket names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pass bucket names from BucketClaim.Status.BucketName into the PromQL query as a bucket=~"name1|name2" filter. This prevents O(N²) load where N WorkloadMonitors each fetch all buckets globally. Co-Authored-By: Claude Signed-off-by: ZverGuy --- .../controller/workloadmonitor_controller.go | 21 ++++++++++++------- .../workloadmonitor_controller_test.go | 8 +++---- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/internal/controller/workloadmonitor_controller.go b/internal/controller/workloadmonitor_controller.go index af61ac47..7629868d 100644 --- a/internal/controller/workloadmonitor_controller.go +++ b/internal/controller/workloadmonitor_controller.go @@ -151,17 +151,18 @@ type bucketMetrics struct { HasPhysical bool } -// queryAllBucketMetrics fetches all SeaweedFS bucket size metrics in a single -// Prometheus query and returns them keyed by bucket name. This avoids 2×N HTTP -// round-trips when there are N buckets. -func (r *WorkloadMonitorReconciler) queryAllBucketMetrics(ctx context.Context, prometheusBaseURL string) map[string]*bucketMetrics { +// queryAllBucketMetrics fetches SeaweedFS bucket size metrics for the given +// bucket names in a single Prometheus query and returns them keyed by bucket +// name. The query is scoped to only the requested buckets to avoid fetching +// metrics for buckets belonging to other WorkloadMonitors. +func (r *WorkloadMonitorReconciler) queryAllBucketMetrics(ctx context.Context, prometheusBaseURL string, bucketNames []string) map[string]*bucketMetrics { result := make(map[string]*bucketMetrics) - if prometheusBaseURL == "" { + if prometheusBaseURL == "" || len(bucketNames) == 0 { return result } logger := log.FromContext(ctx) - query := `{__name__=~"SeaweedFS_s3_bucket_(size|physical_size)_bytes"}` + query := fmt.Sprintf(`{__name__=~"SeaweedFS_s3_bucket_(size|physical_size)_bytes",bucket=~"%s"}`, strings.Join(bucketNames, "|")) u, err := url.Parse(strings.TrimRight(prometheusBaseURL, "/") + "/api/v1/query") if err != nil { logger.Error(err, "Failed to parse Prometheus URL") @@ -591,7 +592,13 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ if len(bucketClaimList.Items) > 0 { bucketPromURL := r.resolvePrometheusURL(ctx, monitor.Namespace) - allBucketMetrics := r.queryAllBucketMetrics(ctx, bucketPromURL) + var bucketNames []string + for _, bc := range bucketClaimList.Items { + if bc.Status.BucketName != "" { + bucketNames = append(bucketNames, bc.Status.BucketName) + } + } + allBucketMetrics := r.queryAllBucketMetrics(ctx, bucketPromURL, bucketNames) for _, bc := range bucketClaimList.Items { if err := r.reconcileBucketClaimForMonitor(ctx, monitor, bc, allBucketMetrics); err != nil { logger.Error(err, "Failed to reconcile Workload for BucketClaim", "BucketClaim", bc.Name) diff --git a/internal/controller/workloadmonitor_controller_test.go b/internal/controller/workloadmonitor_controller_test.go index 5bb3397e..2d487000 100644 --- a/internal/controller/workloadmonitor_controller_test.go +++ b/internal/controller/workloadmonitor_controller_test.go @@ -388,7 +388,7 @@ func TestQueryAllBucketMetrics(t *testing.T) { defer srv.Close() reconciler := &WorkloadMonitorReconciler{} - metrics := reconciler.queryAllBucketMetrics(context.TODO(), srv.URL) + metrics := reconciler.queryAllBucketMetrics(context.TODO(), srv.URL, []string{"bucket-aaa", "bucket-bbb"}) bm, ok := metrics["bucket-aaa"] if !ok { @@ -420,7 +420,7 @@ func TestQueryAllBucketMetricsEmpty(t *testing.T) { defer srv.Close() reconciler := &WorkloadMonitorReconciler{} - metrics := reconciler.queryAllBucketMetrics(context.TODO(), srv.URL) + metrics := reconciler.queryAllBucketMetrics(context.TODO(), srv.URL, []string{"bucket-aaa", "bucket-bbb"}) if len(metrics) != 0 { t.Errorf("expected empty metrics, got %d", len(metrics)) } @@ -433,7 +433,7 @@ func TestQueryAllBucketMetricsServerError(t *testing.T) { defer srv.Close() reconciler := &WorkloadMonitorReconciler{} - metrics := reconciler.queryAllBucketMetrics(context.TODO(), srv.URL) + metrics := reconciler.queryAllBucketMetrics(context.TODO(), srv.URL, []string{"bucket-aaa", "bucket-bbb"}) if len(metrics) != 0 { t.Errorf("expected empty metrics on error, got %d", len(metrics)) } @@ -441,7 +441,7 @@ func TestQueryAllBucketMetricsServerError(t *testing.T) { func TestQueryAllBucketMetricsNoURL(t *testing.T) { reconciler := &WorkloadMonitorReconciler{} - metrics := reconciler.queryAllBucketMetrics(context.TODO(), "") + metrics := reconciler.queryAllBucketMetrics(context.TODO(), "", nil) if len(metrics) != 0 { t.Errorf("expected empty metrics when URL is empty, got %d", len(metrics)) } From 3803e6e070e987f0f2729bd8ec9d039ae2d6aa65 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Tue, 14 Apr 2026 16:36:40 +0300 Subject: [PATCH 092/250] fix(controller): use dedicated HTTP client instead of http.DefaultClient Replace http.DefaultClient with a package-level *http.Client with an explicit 10-second timeout. Avoids sharing the process-wide default transport with other libraries. Co-Authored-By: Claude Signed-off-by: ZverGuy --- internal/controller/workloadmonitor_controller.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/controller/workloadmonitor_controller.go b/internal/controller/workloadmonitor_controller.go index 7629868d..3bc3c1aa 100644 --- a/internal/controller/workloadmonitor_controller.go +++ b/internal/controller/workloadmonitor_controller.go @@ -42,6 +42,10 @@ const ( vmSelectPath = "/select/0/prometheus" ) +// prometheusHTTPClient is a dedicated HTTP client for Prometheus queries, +// avoiding the shared http.DefaultClient global. +var prometheusHTTPClient = &http.Client{Timeout: 10 * time.Second} + // WorkloadMonitorReconciler reconciles a WorkloadMonitor object type WorkloadMonitorReconciler struct { client.Client @@ -179,7 +183,7 @@ func (r *WorkloadMonitorReconciler) queryAllBucketMetrics(ctx context.Context, p return result } - resp, err := http.DefaultClient.Do(req) + resp, err := prometheusHTTPClient.Do(req) if err != nil { logger.V(1).Info("Failed to query Prometheus for bucket metrics", "error", err) return result From 2c141d3f3804bd4e45fb784c0dd28dd017684e74 Mon Sep 17 00:00:00 2001 From: Andrey Kolkov Date: Fri, 17 Apr 2026 14:16:40 +0400 Subject: [PATCH 093/250] feat(platform): add resourcePreset labels Signed-off-by: Andrey Kolkov --- .../controller/workloadmonitor_controller.go | 60 +++- .../workloadmonitor_controller_test.go | 270 ++++++++++++++++++ .../clickhouse/templates/workloadmonitor.yaml | 4 + .../templates/workloadmonitor.yaml | 1 + packages/apps/harbor/templates/harbor.yaml | 4 + .../http-cache/templates/workloadmonitor.yaml | 4 + .../apps/kafka/templates/workloadmonitor.yaml | 4 + .../mariadb/templates/workloadmonitor.yaml | 2 + packages/apps/mongodb/templates/mongodb.yaml | 2 + .../apps/nats/templates/workloadmonitor.yaml | 2 + .../openbao/templates/workloadmonitor.yaml | 2 + .../apps/opensearch/templates/opensearch.yaml | 2 + packages/apps/postgres/templates/db.yaml | 2 + .../templates/dashboard-resourcemap.yaml | 2 + .../rabbitmq/templates/workloadmonitor.yaml | 2 + .../apps/redis/templates/redisfailover.yaml | 4 + .../templates/workloadmonitor.yaml | 2 + .../apps/vpn/templates/workloadmonitor.yaml | 2 + 18 files changed, 367 insertions(+), 4 deletions(-) diff --git a/internal/controller/workloadmonitor_controller.go b/internal/controller/workloadmonitor_controller.go index 3bc3c1aa..35906a87 100644 --- a/internal/controller/workloadmonitor_controller.go +++ b/internal/controller/workloadmonitor_controller.go @@ -35,6 +35,11 @@ const ( // namespaceMonitoringLabel is the namespace label that indicates which tenant // namespace hosts the monitoring stack (VictoriaMetrics/Prometheus). namespaceMonitoringLabel = "namespace.cozystack.io/monitoring" + workloadLabelPrefix = "workloads.cozystack.io/" + // workloadMonitorLabel is reserved: it names the WorkloadMonitor that owns + // the Workload and is always set by the reconciler, so it is never copied + // from monitor labels. + workloadMonitorLabel = workloadLabelPrefix + "monitor" // vmSelectService is the well-known service name for VictoriaMetrics vmselect // within a monitoring namespace. Port 8481, path /select/0/prometheus. vmSelectService = "vmselect-shortterm" @@ -283,16 +288,21 @@ func (r *WorkloadMonitorReconciler) reconcileBucketClaimForMonitor( } } + monitorLabels := r.getMonitorLabels(monitor) _, err := ctrl.CreateOrUpdate(ctx, r.Client, workload, func() error { updateOwnerReferences(workload.GetObjectMeta(), &bc) if workload.Labels == nil { workload.Labels = make(map[string]string) } + // Apply monitor-level labels first so source-object labels can override on conflict + for k, v := range monitorLabels { + workload.Labels[k] = v + } for k, v := range bc.Labels { workload.Labels[k] = v } - workload.Labels["workloads.cozystack.io/monitor"] = monitor.Name + workload.Labels[workloadMonitorLabel] = monitor.Name workload.Status.Kind = monitor.Spec.Kind workload.Status.Type = monitor.Spec.Type @@ -345,14 +355,22 @@ func (r *WorkloadMonitorReconciler) reconcileServiceForMonitor( resourceLabel = fmt.Sprintf("%s.ipaddresspool.metallb.io/requests.ipaddresses", resourceLabel) resources[resourceLabel] = quantity + monitorLabels := r.getMonitorLabels(monitor) _, err := ctrl.CreateOrUpdate(ctx, r.Client, workload, func() error { // Update owner references with the new monitor updateOwnerReferences(workload.GetObjectMeta(), &svc) + // Apply monitor-level labels first so source-object labels can override on conflict + if workload.Labels == nil { + workload.Labels = make(map[string]string) + } + for k, v := range monitorLabels { + workload.Labels[k] = v + } for k, v := range svc.Labels { workload.Labels[k] = v } - workload.Labels["workloads.cozystack.io/monitor"] = monitor.Name + workload.Labels[workloadMonitorLabel] = monitor.Name // Fill Workload status fields: workload.Status.Kind = monitor.Spec.Kind @@ -396,14 +414,22 @@ func (r *WorkloadMonitorReconciler) reconcilePVCForMonitor( resources[resourceLabel] = resourceQuantity } + monitorLabels := r.getMonitorLabels(monitor) _, err := ctrl.CreateOrUpdate(ctx, r.Client, workload, func() error { // Update owner references with the new monitor updateOwnerReferences(workload.GetObjectMeta(), &pvc) + // Apply monitor-level labels first so source-object labels can override on conflict + if workload.Labels == nil { + workload.Labels = make(map[string]string) + } + for k, v := range monitorLabels { + workload.Labels[k] = v + } for k, v := range pvc.Labels { workload.Labels[k] = v } - workload.Labels["workloads.cozystack.io/monitor"] = monitor.Name + workload.Labels[workloadMonitorLabel] = monitor.Name // Fill Workload status fields: workload.Status.Kind = monitor.Spec.Kind @@ -470,14 +496,22 @@ func (r *WorkloadMonitorReconciler) reconcilePodForMonitor( } metaLabels := r.getWorkloadMetadata(&pod) + monitorLabels := r.getMonitorLabels(monitor) _, err := ctrl.CreateOrUpdate(ctx, r.Client, workload, func() error { // Update owner references with the new monitor updateOwnerReferences(workload.GetObjectMeta(), &pod) + // Apply monitor-level labels first so source-object labels can override on conflict + if workload.Labels == nil { + workload.Labels = make(map[string]string) + } + for k, v := range monitorLabels { + workload.Labels[k] = v + } for k, v := range pod.Labels { workload.Labels[k] = v } - workload.Labels["workloads.cozystack.io/monitor"] = monitor.Name + workload.Labels[workloadMonitorLabel] = monitor.Name // Add workload meta to labels for k, v := range metaLabels { @@ -720,3 +754,21 @@ func (r *WorkloadMonitorReconciler) getWorkloadMetadata(obj client.Object) map[s } return labels } + +// getMonitorLabels extracts workloads.cozystack.io/* labels from a WorkloadMonitor +// so they can be propagated onto Workload objects created for pods, PVCs, services, +// or bucket claims. The monitor label "workloads.cozystack.io/monitor" is reserved +// and set separately per Workload, so it is excluded here. +func (r *WorkloadMonitorReconciler) getMonitorLabels(monitor *cozyv1alpha1.WorkloadMonitor) map[string]string { + labels := make(map[string]string) + for k, v := range monitor.GetLabels() { + if !strings.HasPrefix(k, workloadLabelPrefix) { + continue + } + if k == workloadMonitorLabel { + continue + } + labels[k] = v + } + return labels +} diff --git a/internal/controller/workloadmonitor_controller_test.go b/internal/controller/workloadmonitor_controller_test.go index 2d487000..355b0379 100644 --- a/internal/controller/workloadmonitor_controller_test.go +++ b/internal/controller/workloadmonitor_controller_test.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "net/http/httptest" + "strings" "testing" cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" @@ -141,6 +142,199 @@ func TestReconcile_OperationalTrue_WhenEnoughReplicas(t *testing.T) { } } +func TestGetMonitorLabels(t *testing.T) { + tests := []struct { + name string + labels map[string]string + expected map[string]string + }{ + { + name: "nil labels", + labels: nil, + expected: map[string]string{}, + }, + { + name: "only workloads.cozystack.io/* labels are propagated", + labels: map[string]string{ + "workloads.cozystack.io/resource-preset": "medium", + "app.kubernetes.io/name": "postgres", + "custom.example.com/team": "platform", + }, + expected: map[string]string{ + "workloads.cozystack.io/resource-preset": "medium", + }, + }, + { + name: "monitor label is reserved and excluded", + labels: map[string]string{ + "workloads.cozystack.io/resource-preset": "small", + "workloads.cozystack.io/monitor": "should-be-dropped", + }, + expected: map[string]string{ + "workloads.cozystack.io/resource-preset": "small", + }, + }, + { + name: "multiple workloads.cozystack.io labels propagate", + labels: map[string]string{ + "workloads.cozystack.io/resource-preset": "large", + "workloads.cozystack.io/tier": "db", + }, + expected: map[string]string{ + "workloads.cozystack.io/resource-preset": "large", + "workloads.cozystack.io/tier": "db", + }, + }, + { + name: "no matching labels returns empty map", + labels: map[string]string{ + "app.kubernetes.io/name": "postgres", + }, + expected: map[string]string{}, + }, + } + + r := &WorkloadMonitorReconciler{} + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + monitor := &cozyv1alpha1.WorkloadMonitor{ + ObjectMeta: metav1.ObjectMeta{Labels: tc.labels}, + } + got := r.getMonitorLabels(monitor) + if len(got) != len(tc.expected) { + t.Fatalf("expected %d labels, got %d (%v)", len(tc.expected), len(got), got) + } + for k, v := range tc.expected { + if gv, ok := got[k]; !ok || gv != v { + t.Errorf("expected label %q=%q, got %q", k, v, gv) + } + } + }) + } +} + +func TestReconcile_MonitorLabelsPropagatedToPodWorkload(t *testing.T) { + scheme := runtime.NewScheme() + _ = cozyv1alpha1.AddToScheme(scheme) + _ = corev1.AddToScheme(scheme) + + monitor := &cozyv1alpha1.WorkloadMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-monitor", + Namespace: "default", + Labels: map[string]string{ + "workloads.cozystack.io/resource-preset": "medium", + "app.kubernetes.io/name": "ignored-not-propagated", + }, + }, + Spec: cozyv1alpha1.WorkloadMonitorSpec{ + Selector: map[string]string{"app": "test"}, + Kind: "postgres", + Type: "postgres", + }, + } + + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod-1", + Namespace: "default", + Labels: map[string]string{ + "app": "test", + "app.kubernetes.io/name": "pod-wins-on-conflict", + }, + }, + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + {Type: corev1.PodReady, Status: corev1.ConditionTrue}, + }, + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(monitor, pod). + WithStatusSubresource(monitor). + Build() + + reconciler := &WorkloadMonitorReconciler{Client: fakeClient, Scheme: scheme} + req := reconcile.Request{NamespacedName: types.NamespacedName{Name: "test-monitor", Namespace: "default"}} + if _, err := reconciler.Reconcile(context.TODO(), req); err != nil { + t.Fatalf("Reconcile returned error: %v", err) + } + + workload := &cozyv1alpha1.Workload{} + if err := fakeClient.Get(context.TODO(), types.NamespacedName{Name: "pod-test-pod-1", Namespace: "default"}, workload); err != nil { + t.Fatalf("Failed to get Workload: %v", err) + } + + if got := workload.Labels["workloads.cozystack.io/resource-preset"]; got != "medium" { + t.Errorf("expected monitor label propagated, got %q", got) + } + // Non-workloads.cozystack.io monitor labels must not be copied + if _, ok := workload.Labels["app.kubernetes.io/name"]; !ok { + t.Error("expected pod label to be present on Workload") + } + // Source-object label takes precedence on conflict + if got := workload.Labels["app.kubernetes.io/name"]; got != "pod-wins-on-conflict" { + t.Errorf("expected pod label to win on conflict, got %q", got) + } + // Reserved monitor label is always set from the monitor name + if got := workload.Labels["workloads.cozystack.io/monitor"]; got != "test-monitor" { + t.Errorf("expected monitor-name label, got %q", got) + } +} + +func TestReconcile_BackwardCompat_NoMonitorLabels(t *testing.T) { + scheme := runtime.NewScheme() + _ = cozyv1alpha1.AddToScheme(scheme) + _ = corev1.AddToScheme(scheme) + + monitor := &cozyv1alpha1.WorkloadMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-monitor", + Namespace: "default", + }, + Spec: cozyv1alpha1.WorkloadMonitorSpec{ + Selector: map[string]string{"app": "test"}, + }, + } + + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod-1", + Namespace: "default", + Labels: map[string]string{"app": "test"}, + }, + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + {Type: corev1.PodReady, Status: corev1.ConditionTrue}, + }, + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(monitor, pod). + WithStatusSubresource(monitor). + Build() + + reconciler := &WorkloadMonitorReconciler{Client: fakeClient, Scheme: scheme} + req := reconcile.Request{NamespacedName: types.NamespacedName{Name: "test-monitor", Namespace: "default"}} + if _, err := reconciler.Reconcile(context.TODO(), req); err != nil { + t.Fatalf("Reconcile returned error: %v", err) + } + + workload := &cozyv1alpha1.Workload{} + if err := fakeClient.Get(context.TODO(), types.NamespacedName{Name: "pod-test-pod-1", Namespace: "default"}, workload); err != nil { + t.Fatalf("Failed to get Workload: %v", err) + } + for k := range workload.Labels { + if strings.HasPrefix(k, "workloads.cozystack.io/") && k != "workloads.cozystack.io/monitor" { + t.Errorf("unexpected workload label present: %q", k) + } + } +} + func TestReconcile_OperationalTrue_WhenNoMinReplicas(t *testing.T) { scheme := runtime.NewScheme() _ = cozyv1alpha1.AddToScheme(scheme) @@ -330,6 +524,82 @@ func TestReconcileBucketClaimNotReady(t *testing.T) { } } +func TestReconcile_MonitorLabelsPropagatedToBucketClaimWorkload(t *testing.T) { + s := newTestScheme() + + monitor := &cozyv1alpha1.WorkloadMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-bucket", + Namespace: "tenant-demo", + Labels: map[string]string{ + "workloads.cozystack.io/resource-preset": "medium", + "app.kubernetes.io/name": "ignored-not-propagated", + }, + }, + Spec: cozyv1alpha1.WorkloadMonitorSpec{ + Kind: "bucket", + Type: "s3", + Selector: map[string]string{ + "app.kubernetes.io/instance": "my-bucket", + }, + }, + } + + bc := &cosiv1alpha1.BucketClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-bucket", + Namespace: "tenant-demo", + Labels: map[string]string{ + "app.kubernetes.io/instance": "my-bucket", + "app.kubernetes.io/name": "bucket-wins-on-conflict", + }, + }, + Spec: cosiv1alpha1.BucketClaimSpec{ + BucketClassName: "seaweedfs", + Protocols: []cosiv1alpha1.Protocol{cosiv1alpha1.ProtocolS3}, + }, + Status: cosiv1alpha1.BucketClaimStatus{ + BucketReady: true, + BucketName: "cosi-abc123", + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(s). + WithObjects(monitor, bc). + WithStatusSubresource(monitor). + Build() + + reconciler := &WorkloadMonitorReconciler{Client: fakeClient, Scheme: s} + req := reconcile.Request{NamespacedName: types.NamespacedName{ + Name: "my-bucket", + Namespace: "tenant-demo", + }} + if _, err := reconciler.Reconcile(context.TODO(), req); err != nil { + t.Fatalf("Reconcile returned error: %v", err) + } + + workload := &cozyv1alpha1.Workload{} + if err := fakeClient.Get(context.TODO(), types.NamespacedName{ + Name: "bucket-my-bucket", + Namespace: "tenant-demo", + }, workload); err != nil { + t.Fatalf("Failed to get Workload: %v", err) + } + + if got := workload.Labels["workloads.cozystack.io/resource-preset"]; got != "medium" { + t.Errorf("expected monitor label propagated, got %q", got) + } + // Source-object label takes precedence on conflict + if got := workload.Labels["app.kubernetes.io/name"]; got != "bucket-wins-on-conflict" { + t.Errorf("expected bucket claim label to win on conflict, got %q", got) + } + // Reserved monitor label is always set from the monitor name + if got := workload.Labels["workloads.cozystack.io/monitor"]; got != "my-bucket" { + t.Errorf("expected monitor-name label, got %q", got) + } +} + func TestReconcileNoBucketClaimSkips(t *testing.T) { s := newTestScheme() diff --git a/packages/apps/clickhouse/templates/workloadmonitor.yaml b/packages/apps/clickhouse/templates/workloadmonitor.yaml index 9020ad41..62c951c2 100644 --- a/packages/apps/clickhouse/templates/workloadmonitor.yaml +++ b/packages/apps/clickhouse/templates/workloadmonitor.yaml @@ -3,6 +3,8 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ $.Release.Name }} + labels: + workloads.cozystack.io/resource-preset: {{ .Values.resourcesPreset | quote }} spec: replicas: {{ .Values.replicas }} minReplicas: 1 @@ -17,6 +19,8 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ $.Release.Name }}-keeper + labels: + workloads.cozystack.io/resource-preset: {{ .Values.clickhouseKeeper.resourcesPreset | quote }} spec: replicas: {{ .Values.clickhouseKeeper.replicas }} minReplicas: 1 diff --git a/packages/apps/foundationdb/templates/workloadmonitor.yaml b/packages/apps/foundationdb/templates/workloadmonitor.yaml index 306797f3..2e2a3d19 100644 --- a/packages/apps/foundationdb/templates/workloadmonitor.yaml +++ b/packages/apps/foundationdb/templates/workloadmonitor.yaml @@ -8,6 +8,7 @@ metadata: app.kubernetes.io/name: foundationdb app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} + workloads.cozystack.io/resource-preset: {{ .Values.resourcesPreset | quote }} spec: replicas: {{ .Values.cluster.processCounts.storage }} minReplicas: {{ include "foundationdb.minReplicas" . }} diff --git a/packages/apps/harbor/templates/harbor.yaml b/packages/apps/harbor/templates/harbor.yaml index ff8e88ad..bf780967 100644 --- a/packages/apps/harbor/templates/harbor.yaml +++ b/packages/apps/harbor/templates/harbor.yaml @@ -158,6 +158,8 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ $.Release.Name }}-core + labels: + workloads.cozystack.io/resource-preset: {{ .Values.core.resourcesPreset | quote }} spec: replicas: 1 minReplicas: 1 @@ -174,6 +176,8 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ $.Release.Name }}-registry + labels: + workloads.cozystack.io/resource-preset: {{ .Values.registry.resourcesPreset | quote }} spec: replicas: 1 minReplicas: 1 diff --git a/packages/apps/http-cache/templates/workloadmonitor.yaml b/packages/apps/http-cache/templates/workloadmonitor.yaml index 150d8bbe..a38a395a 100644 --- a/packages/apps/http-cache/templates/workloadmonitor.yaml +++ b/packages/apps/http-cache/templates/workloadmonitor.yaml @@ -3,6 +3,8 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ $.Release.Name }}-haproxy + labels: + workloads.cozystack.io/resource-preset: {{ .Values.haproxy.resourcesPreset | quote }} spec: replicas: {{ .Values.haproxy.replicas }} minReplicas: 1 @@ -16,6 +18,8 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ $.Release.Name }}-nginx + labels: + workloads.cozystack.io/resource-preset: {{ .Values.nginx.resourcesPreset | quote }} spec: replicas: {{ .Values.nginx.replicas }} minReplicas: 1 diff --git a/packages/apps/kafka/templates/workloadmonitor.yaml b/packages/apps/kafka/templates/workloadmonitor.yaml index 4b161b04..c31eb425 100644 --- a/packages/apps/kafka/templates/workloadmonitor.yaml +++ b/packages/apps/kafka/templates/workloadmonitor.yaml @@ -3,6 +3,8 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ $.Release.Name }} + labels: + workloads.cozystack.io/resource-preset: {{ .Values.kafka.resourcesPreset | quote }} spec: replicas: {{ .Values.replicas }} minReplicas: 1 @@ -19,6 +21,8 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ $.Release.Name }}-zookeeper + labels: + workloads.cozystack.io/resource-preset: {{ .Values.zookeeper.resourcesPreset | quote }} spec: replicas: {{ .Values.replicas }} minReplicas: 1 diff --git a/packages/apps/mariadb/templates/workloadmonitor.yaml b/packages/apps/mariadb/templates/workloadmonitor.yaml index b69139bf..36cb59f0 100644 --- a/packages/apps/mariadb/templates/workloadmonitor.yaml +++ b/packages/apps/mariadb/templates/workloadmonitor.yaml @@ -3,6 +3,8 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ $.Release.Name }} + labels: + workloads.cozystack.io/resource-preset: {{ .Values.resourcesPreset | quote }} spec: replicas: {{ .Values.replicas }} minReplicas: 1 diff --git a/packages/apps/mongodb/templates/mongodb.yaml b/packages/apps/mongodb/templates/mongodb.yaml index 67969758..9273e506 100644 --- a/packages/apps/mongodb/templates/mongodb.yaml +++ b/packages/apps/mongodb/templates/mongodb.yaml @@ -169,6 +169,8 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ .Release.Name }} + labels: + workloads.cozystack.io/resource-preset: {{ .Values.resourcesPreset | quote }} spec: {{- if .Values.sharding }} {{- $totalReplicas := 0 }} diff --git a/packages/apps/nats/templates/workloadmonitor.yaml b/packages/apps/nats/templates/workloadmonitor.yaml index 43d64a46..cb20b4a6 100644 --- a/packages/apps/nats/templates/workloadmonitor.yaml +++ b/packages/apps/nats/templates/workloadmonitor.yaml @@ -3,6 +3,8 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ $.Release.Name }} + labels: + workloads.cozystack.io/resource-preset: {{ .Values.resourcesPreset | quote }} spec: replicas: {{ .Values.replicas }} minReplicas: 1 diff --git a/packages/apps/openbao/templates/workloadmonitor.yaml b/packages/apps/openbao/templates/workloadmonitor.yaml index 0a9acf76..56f9ec8e 100644 --- a/packages/apps/openbao/templates/workloadmonitor.yaml +++ b/packages/apps/openbao/templates/workloadmonitor.yaml @@ -3,6 +3,8 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ $.Release.Name }} + labels: + workloads.cozystack.io/resource-preset: {{ .Values.resourcesPreset | quote }} spec: replicas: {{ .Values.replicas }} minReplicas: 1 diff --git a/packages/apps/opensearch/templates/opensearch.yaml b/packages/apps/opensearch/templates/opensearch.yaml index fcf431ac..836a3d4b 100644 --- a/packages/apps/opensearch/templates/opensearch.yaml +++ b/packages/apps/opensearch/templates/opensearch.yaml @@ -93,6 +93,8 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ .Release.Name }} + labels: + workloads.cozystack.io/resource-preset: {{ .Values.resourcesPreset | quote }} spec: replicas: {{ .Values.replicas }} minReplicas: 1 diff --git a/packages/apps/postgres/templates/db.yaml b/packages/apps/postgres/templates/db.yaml index 7557c436..5c40b2c7 100644 --- a/packages/apps/postgres/templates/db.yaml +++ b/packages/apps/postgres/templates/db.yaml @@ -84,6 +84,8 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ $.Release.Name }} + labels: + workloads.cozystack.io/resource-preset: {{ .Values.resourcesPreset | quote }} spec: replicas: {{ .Values.replicas }} minReplicas: 1 diff --git a/packages/apps/qdrant/templates/dashboard-resourcemap.yaml b/packages/apps/qdrant/templates/dashboard-resourcemap.yaml index b0a85bcc..0986951f 100644 --- a/packages/apps/qdrant/templates/dashboard-resourcemap.yaml +++ b/packages/apps/qdrant/templates/dashboard-resourcemap.yaml @@ -41,6 +41,8 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ .Release.Name }} + labels: + workloads.cozystack.io/resource-preset: {{ .Values.resourcesPreset | quote }} spec: minReplicas: 1 replicas: {{ .Values.replicas }} diff --git a/packages/apps/rabbitmq/templates/workloadmonitor.yaml b/packages/apps/rabbitmq/templates/workloadmonitor.yaml index 0f7462c7..66941153 100644 --- a/packages/apps/rabbitmq/templates/workloadmonitor.yaml +++ b/packages/apps/rabbitmq/templates/workloadmonitor.yaml @@ -3,6 +3,8 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ $.Release.Name }} + labels: + workloads.cozystack.io/resource-preset: {{ .Values.resourcesPreset | quote }} spec: replicas: {{ .Values.replicas }} minReplicas: 1 diff --git a/packages/apps/redis/templates/redisfailover.yaml b/packages/apps/redis/templates/redisfailover.yaml index 936217a3..160e030d 100644 --- a/packages/apps/redis/templates/redisfailover.yaml +++ b/packages/apps/redis/templates/redisfailover.yaml @@ -75,6 +75,8 @@ kind: WorkloadMonitor metadata: name: {{ $.Release.Name }}-redis namespace: {{ $.Release.Namespace }} + labels: + workloads.cozystack.io/resource-preset: {{ .Values.resourcesPreset | quote }} spec: minReplicas: 1 replicas: {{ .Values.replicas }} @@ -90,6 +92,8 @@ kind: WorkloadMonitor metadata: name: {{ $.Release.Name }}-sentinel namespace: {{ $.Release.Namespace }} + labels: + workloads.cozystack.io/resource-preset: {{ .Values.resourcesPreset | quote }} spec: minReplicas: 2 replicas: 3 diff --git a/packages/apps/tcp-balancer/templates/workloadmonitor.yaml b/packages/apps/tcp-balancer/templates/workloadmonitor.yaml index 41dce2ab..3478826b 100644 --- a/packages/apps/tcp-balancer/templates/workloadmonitor.yaml +++ b/packages/apps/tcp-balancer/templates/workloadmonitor.yaml @@ -3,6 +3,8 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ $.Release.Name }} + labels: + workloads.cozystack.io/resource-preset: {{ .Values.resourcesPreset | quote }} spec: replicas: {{ .Values.replicas }} minReplicas: 1 diff --git a/packages/apps/vpn/templates/workloadmonitor.yaml b/packages/apps/vpn/templates/workloadmonitor.yaml index a75f7940..7fc1ec7a 100644 --- a/packages/apps/vpn/templates/workloadmonitor.yaml +++ b/packages/apps/vpn/templates/workloadmonitor.yaml @@ -2,6 +2,8 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ $.Release.Name }} + labels: + workloads.cozystack.io/resource-preset: {{ .Values.resourcesPreset | quote }} spec: replicas: {{ .Values.replicas }} minReplicas: 1 From cb3e0adf6438a2520bf7a943c798da7de8104c42 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Fri, 17 Apr 2026 16:55:42 +0300 Subject: [PATCH 094/250] feat(monitoring): add GPU recording rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add VMRule with recording rules for DCGM metrics at three levels: - gpu.recording.30s: per-GPU aggregates over 30s windows - gpu.recording.cluster.1m: cluster-wide totals for overview panels - gpu.recording.namespace.1m: per-namespace aggregates for tenant reporting and GPU-hour calculations The rules are safe to ship on clusters without DCGM — they evaluate to empty series when no matching metrics are scraped. Used by dashboards/gpu/gpu-performance.json. Assisted-By: Claude Signed-off-by: Arsolitt --- .../alerts/gpu-recording.rules.yaml | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml new file mode 100644 index 00000000..871a46fc --- /dev/null +++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml @@ -0,0 +1,57 @@ +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMRule +metadata: + name: alerts-gpu-recording.rules +spec: + groups: + - name: gpu.recording.30s + interval: 30s + rules: + - record: gpu:util:avg + expr: avg by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_DEV_GPU_UTIL) + - record: gpu:mem_copy_util:avg + expr: avg by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_DEV_MEM_COPY_UTIL) + - record: gpu:fb_used_bytes:max + expr: max by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_DEV_FB_USED) * 1048576 + - record: gpu:fb_free_bytes:max + expr: max by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_DEV_FB_FREE) * 1048576 + - record: gpu:power_watts:avg + expr: avg by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_DEV_POWER_USAGE) + - record: gpu:temp_celsius:max + expr: max by (Hostname, gpu, UUID, modelName) (DCGM_FI_DEV_GPU_TEMP) + - record: gpu:tensor_active:avg + expr: avg by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_PROF_PIPE_TENSOR_ACTIVE) + - record: gpu:gr_engine_active:avg + expr: avg by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_PROF_GR_ENGINE_ACTIVE) + + - name: gpu.recording.cluster.1m + interval: 1m + rules: + - record: cluster:gpu_count:total + expr: count(group by (UUID) (DCGM_FI_DEV_GPU_UTIL)) + - record: cluster:gpu_count:allocated + expr: count(group by (UUID) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"})) + - record: cluster:gpu_count:free + expr: cluster:gpu_count:total - (cluster:gpu_count:allocated or vector(0)) + - record: cluster:gpu_util:avg + expr: avg(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}) + - record: cluster:gpu_power_watts:sum + expr: sum(DCGM_FI_DEV_POWER_USAGE) + + - name: gpu.recording.namespace.1m + interval: 1m + rules: + - record: namespace:gpu_count:sum + expr: count by (namespace) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}) + - record: namespace:gpu_util:avg + expr: avg by (namespace) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}) + - record: namespace:tensor_active:avg + expr: avg by (namespace) (DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace!="", namespace!~"cozy-.*|kube-.*"}) + - record: namespace:fb_used_bytes:sum + expr: sum by (namespace) (DCGM_FI_DEV_FB_USED{namespace!="", namespace!~"cozy-.*|kube-.*"}) * 1048576 + - record: namespace:power_watts:sum + expr: sum by (namespace) (DCGM_FI_DEV_POWER_USAGE{namespace!="", namespace!~"cozy-.*|kube-.*"}) + - record: namespace:energy_joules:sum + expr: sum by (namespace) (DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION{namespace!="", namespace!~"cozy-.*|kube-.*"}) / 1000 + - record: namespace:gpu_allocated_count:gauge + expr: count by (namespace) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}) From d1d19e9978f965b65ed910b3b6486469e2d07f8d Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Fri, 17 Apr 2026 16:55:51 +0300 Subject: [PATCH 095/250] feat(monitoring): add GPU performance Grafana dashboard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the gpu/gpu-performance dashboard and register it in the infra dashboard list. The dashboard provides: - Cluster overview: total/allocated GPUs, average utilization, aggregate power draw. - Utilization: GPU util (NVML), tensor pipe active (realistic load for LLM/AI workloads), graphics engine active, memory copy util. - Memory: VRAM used/free per GPU. - Power and temperature per GPU. - Health: XID errors, power and thermal throttling. The dashboard relies on DCGM_FI_* metrics plus the cluster:gpu_* and namespace:gpu_* recording rules added to monitoring-agents. The JSON follows the cozystack convention — Prometheus data source is selected via the $ds_prometheus template variable. Assisted-By: Claude Signed-off-by: Arsolitt --- dashboards/gpu/gpu-performance.json | 935 ++++++++++++++++++ .../system/monitoring/dashboards-infra.list | 1 + 2 files changed, 936 insertions(+) create mode 100644 dashboards/gpu/gpu-performance.json diff --git a/dashboards/gpu/gpu-performance.json b/dashboards/gpu/gpu-performance.json new file mode 100644 index 00000000..0b458d4e --- /dev/null +++ b/dashboards/gpu/gpu-performance.json @@ -0,0 +1,935 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "schemaVersion": 39, + "tags": [ + "gpu", + "dcgm" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "hide": 0, + "includeAll": false, + "label": "Prometheus", + "multi": false, + "name": "ds_prometheus", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "definition": "label_values(DCGM_FI_DEV_GPU_UTIL, Hostname)", + "hide": 0, + "includeAll": true, + "label": "Host", + "multi": true, + "name": "Hostname", + "options": [], + "query": { + "query": "label_values(DCGM_FI_DEV_GPU_UTIL, Hostname)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "definition": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)", + "hide": 0, + "includeAll": true, + "label": "Namespace", + "multi": true, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "GPU Performance", + "uid": "gpu-performance", + "weekStart": "", + "panels": [ + { + "type": "row", + "title": "Overview", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "collapsed": false, + "id": 1, + "panels": [] + }, + { + "type": "stat", + "title": "Total GPUs", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 2, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "targets": [ + { + "expr": "cluster:gpu_count:total", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + } + }, + { + "type": "stat", + "title": "Allocated", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 3, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "targets": [ + { + "expr": "cluster:gpu_count:allocated", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + } + }, + { + "type": "stat", + "title": "Average utilization", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 4, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "targets": [ + { + "expr": "cluster:gpu_util:avg", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "green", + "value": 30 + }, + { + "color": "orange", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + } + }, + { + "type": "stat", + "title": "Power draw", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 5, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "targets": [ + { + "expr": "cluster:gpu_power_watts:sum", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "watt", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + } + }, + { + "type": "row", + "title": "Utilization", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "collapsed": false, + "id": 10, + "panels": [] + }, + { + "type": "timeseries", + "title": "GPU Utilization (NVML)", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 11, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "targets": [ + { + "expr": "DCGM_FI_DEV_GPU_UTIL{Hostname=~\"$Hostname\", namespace=~\"$namespace\"}", + "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "timeseries", + "title": "Tensor Pipe Active (realistic load for LLM/AI)", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 12, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "targets": [ + { + "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 100", + "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "timeseries", + "title": "Graphics Engine Active", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 13, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "targets": [ + { + "expr": "DCGM_FI_PROF_GR_ENGINE_ACTIVE{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 100", + "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "timeseries", + "title": "Memory Copy Utilization", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 14, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "targets": [ + { + "expr": "DCGM_FI_DEV_MEM_COPY_UTIL{Hostname=~\"$Hostname\", namespace=~\"$namespace\"}", + "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "row", + "title": "Memory", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "collapsed": false, + "id": 20, + "panels": [] + }, + { + "type": "timeseries", + "title": "VRAM Used", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 21, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "targets": [ + { + "expr": "DCGM_FI_DEV_FB_USED{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 1048576", + "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 25, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "timeseries", + "title": "VRAM Free", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 22, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "targets": [ + { + "expr": "DCGM_FI_DEV_FB_FREE{Hostname=~\"$Hostname\"} * 1048576", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "min" + ] + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "row", + "title": "Power & Temperature", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "collapsed": false, + "id": 30, + "panels": [] + }, + { + "type": "timeseries", + "title": "Power Usage per GPU", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 31, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "targets": [ + { + "expr": "DCGM_FI_DEV_POWER_USAGE{Hostname=~\"$Hostname\"}", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "watt", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "timeseries", + "title": "GPU Temperature", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 32, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "targets": [ + { + "expr": "DCGM_FI_DEV_GPU_TEMP{Hostname=~\"$Hostname\"}", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "celsius", + "min": 20, + "max": 100, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 75 + }, + { + "color": "red", + "value": 85 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "row", + "title": "Health", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "collapsed": false, + "id": 40, + "panels": [] + }, + { + "type": "stat", + "title": "XID errors (latest)", + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 41 + }, + "id": 41, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "targets": [ + { + "expr": "max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS{Hostname=~\"$Hostname\"})", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + } + }, + { + "type": "timeseries", + "title": "Power Violation (\u00b5s/s throttled due to power)", + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 41 + }, + "id": 42, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "targets": [ + { + "expr": "rate(DCGM_FI_DEV_POWER_VIOLATION{Hostname=~\"$Hostname\"}[5m])", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "\u00b5s", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "timeseries", + "title": "Thermal Violation (\u00b5s/s throttled due to thermals)", + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 41 + }, + "id": 43, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "targets": [ + { + "expr": "rate(DCGM_FI_DEV_THERMAL_VIOLATION{Hostname=~\"$Hostname\"}[5m])", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "\u00b5s", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + } + ] +} \ No newline at end of file diff --git a/packages/system/monitoring/dashboards-infra.list b/packages/system/monitoring/dashboards-infra.list index b5b9f52a..581f9a05 100644 --- a/packages/system/monitoring/dashboards-infra.list +++ b/packages/system/monitoring/dashboards-infra.list @@ -32,3 +32,4 @@ hubble/overview hubble/dns-namespace hubble/l7-http-metrics hubble/network-overview +gpu/gpu-performance From 5d6654c6f4eb63447d91bbf448e6116317228931 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Fri, 17 Apr 2026 16:56:03 +0300 Subject: [PATCH 096/250] docs(gpu-operator): add native-pod Talos reference manifests Add reference manifests (not templates) under packages/system/gpu-operator/examples/ documenting one working configuration for running CUDA workloads directly in pods on a Talos cluster, with DCGM metrics that drive the gpu/gpu-performance dashboard. - values-native-talos.yaml: Cozystack Package values that disable the sandbox path, enable the device plugin, and wire DCGM to the custom metrics ConfigMap. - dcgm-custom-metrics.yaml: ConfigMap extending the default DCGM CSV with profiling, ECC, throttling and energy counters used by the dashboard and recording rules. - nvidia-driver-compat.yaml: DaemonSet that stages libnvidia-ml.so.1 and nvidia-smi from the Talos glibc tree into a location the gpu-operator validator inspects. Workaround for NVIDIA/gpu-operator#1687. - README.md: explains why these are shipped as references rather than first-class templates (sandbox vs native is a deployment choice), and how the pieces connect. The out-of-the-box values-talos.yaml still targets the sandbox (VFIO passthrough) scenario. Operators who want native pod GPU workloads can start from these references. Assisted-By: Claude Signed-off-by: Arsolitt --- .../system/gpu-operator/examples/README.md | 69 ++++++++++++++++ .../examples/dcgm-custom-metrics.yaml | 82 +++++++++++++++++++ .../examples/nvidia-driver-compat.yaml | 72 ++++++++++++++++ .../examples/values-native-talos.yaml | 45 ++++++++++ 4 files changed, 268 insertions(+) create mode 100644 packages/system/gpu-operator/examples/README.md create mode 100644 packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml create mode 100644 packages/system/gpu-operator/examples/nvidia-driver-compat.yaml create mode 100644 packages/system/gpu-operator/examples/values-native-talos.yaml diff --git a/packages/system/gpu-operator/examples/README.md b/packages/system/gpu-operator/examples/README.md new file mode 100644 index 00000000..dac3703e --- /dev/null +++ b/packages/system/gpu-operator/examples/README.md @@ -0,0 +1,69 @@ +# GPU operator — native pod workload on Talos (reference) + +The files in this directory are **not** templates. They are reference +artifacts that document one working configuration for running GPU +workloads directly in pods on a Talos-based Cozystack cluster, together +with the DCGM metrics needed by the `gpu/gpu-performance` Grafana +dashboard. + +The out-of-the-box `values-talos.yaml` for this package targets the +sandbox (VFIO passthrough to KubeVirt VMs) scenario. The files here +illustrate an alternative — running CUDA workloads in regular pods with +the NVIDIA device plugin — and the workarounds it currently requires on +Talos. + +## Files + +- [`values-native-talos.yaml`](./values-native-talos.yaml) — Cozystack + `Package` values that disable sandbox workloads, enable the device + plugin, point `hostPaths.driverInstallDir` at the staging location + used by the compat DaemonSet, and wire DCGM to the custom metrics + ConfigMap. +- [`dcgm-custom-metrics.yaml`](./dcgm-custom-metrics.yaml) — `ConfigMap` + with a DCGM metrics CSV that adds profiling, ECC, throttling and + energy counters on top of the upstream defaults. Required by the + recording rules in `packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml` + and by several panels in the `gpu/gpu-performance` dashboard. +- [`nvidia-driver-compat.yaml`](./nvidia-driver-compat.yaml) — DaemonSet + that stages `libnvidia-ml.so.1` and `nvidia-smi` from the Talos glibc + tree into a path where the NVIDIA GPU Operator validator expects + them. See the "Why the compat DaemonSet exists" section below. + +## Why these are reference, not templates + +Shipping these as first-class templates would silently impose +assumptions that do not hold for every user: + +- Whether the NVIDIA Talos system extension is installed on the nodes. +- Whether GPUs are exposed directly to pods or passed through to VMs. +- The exact path the installed driver ends up at (depends on the + extension version and Talos release). + +The sandbox-oriented `values-talos.yaml` remains the default. Operators +who want native pod GPU workloads can start from this directory and +adapt as needed. + +## Why the compat DaemonSet exists + +The NVIDIA GPU Operator validator checks for `libnvidia-ml.so.1` and +`bin/nvidia-smi` in the path given by `hostPaths.driverInstallDir`. +Talos installs them under `/usr/local/glibc/usr/lib/` and +`/usr/local/bin/`, which the validator does not look at. Until upstream +addresses [NVIDIA/gpu-operator#1687][1], the DaemonSet copies those +files into a directory the validator does inspect and creates the +`.driver-ctr-ready` flag file so the validator proceeds. + +[1]: https://github.com/NVIDIA/gpu-operator/issues/1687 + +## How the dashboard and recording rules fit in + +- `dashboards/gpu/gpu-performance.json` expects `DCGM_FI_*` metrics, + including profiling series (`DCGM_FI_PROF_PIPE_TENSOR_ACTIVE`, + `DCGM_FI_PROF_GR_ENGINE_ACTIVE`) and throttling counters + (`DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION`). + These are only emitted when DCGM Exporter is started with the custom + CSV in `dcgm-custom-metrics.yaml`. +- `packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml` + precomputes cluster-wide and per-namespace aggregations used by the + overview panels of the dashboard. The rules are safe to ship on any + cluster — they evaluate to empty series when DCGM is not scraped. diff --git a/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml b/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml new file mode 100644 index 00000000..14b89b5c --- /dev/null +++ b/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml @@ -0,0 +1,82 @@ +# Custom DCGM Exporter metrics CSV. Referenced from +# examples/values-native-talos.yaml via dcgmExporter.config.name. +# +# Extends the upstream default set with profiling counters, ECC, page +# retirement, row remap, energy and throttling violations — everything +# the gpu/gpu-performance dashboard and the GPU recording rules in +# monitoring-agents expect. +apiVersion: v1 +kind: ConfigMap +metadata: + name: dcgm-custom-metrics + namespace: cozy-gpu-operator +data: + dcgm-metrics.csv: | + # Format + # If line starts with a '#' it is considered a comment + # DCGM FIELD, Prometheus metric type, help message + + # Clocks + DCGM_FI_DEV_SM_CLOCK, gauge, SM clock frequency (in MHz). + DCGM_FI_DEV_MEM_CLOCK, gauge, Memory clock frequency (in MHz). + + # Temperature + DCGM_FI_DEV_MEMORY_TEMP, gauge, Memory temperature (in C). + DCGM_FI_DEV_GPU_TEMP, gauge, GPU temperature (in C). + + # Power + DCGM_FI_DEV_POWER_USAGE, gauge, Power draw (in W). + DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION, counter, Total energy consumption since boot (in mJ). + + # PCIE + DCGM_FI_DEV_PCIE_REPLAY_COUNTER, counter, Total number of PCIe retries. + + # Utilization (the sample period varies depending on the product) + DCGM_FI_DEV_GPU_UTIL, gauge, GPU utilization (in %). + DCGM_FI_DEV_MEM_COPY_UTIL, gauge, Memory utilization (in %). + DCGM_FI_DEV_ENC_UTIL, gauge, Encoder utilization (in %). + DCGM_FI_DEV_DEC_UTIL, gauge, Decoder utilization (in %). + + # Errors and violations + DCGM_FI_DEV_XID_ERRORS, gauge, Value of the last XID error encountered. + + # Memory usage + DCGM_FI_DEV_FB_FREE, gauge, Framebuffer memory free (in MiB). + DCGM_FI_DEV_FB_USED, gauge, Framebuffer memory used (in MiB). + DCGM_FI_DEV_FB_RESERVED, gauge, Framebuffer memory reserved (in MiB). + + # ECC (supported on datacenter-class GPUs such as A10) + DCGM_FI_DEV_ECC_SBE_VOL_TOTAL, counter, Total number of single-bit volatile ECC errors. + DCGM_FI_DEV_ECC_DBE_VOL_TOTAL, counter, Total number of double-bit volatile ECC errors. + DCGM_FI_DEV_ECC_SBE_AGG_TOTAL, counter, Total number of single-bit persistent ECC errors. + DCGM_FI_DEV_ECC_DBE_AGG_TOTAL, counter, Total number of double-bit persistent ECC errors. + + # Retired pages + DCGM_FI_DEV_RETIRED_SBE, counter, Total number of retired pages due to single-bit errors. + DCGM_FI_DEV_RETIRED_DBE, counter, Total number of retired pages due to double-bit errors. + DCGM_FI_DEV_RETIRED_PENDING, counter, Total number of pages pending retirement. + + # Row remapping (not applicable to GDDR6 but left for datacenter GPUs) + DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for uncorrectable errors. + DCGM_FI_DEV_CORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for correctable errors. + DCGM_FI_DEV_ROW_REMAP_FAILURE, gauge, Whether remapping of rows has failed. + + # Throttle / violation counters (crucial for SLA and tenant monitoring) + DCGM_FI_DEV_POWER_VIOLATION, counter, Throttling duration due to power constraints (in us). + DCGM_FI_DEV_THERMAL_VIOLATION, counter, Throttling duration due to thermal constraints (in us). + DCGM_FI_DEV_SYNC_BOOST_VIOLATION, counter, Throttling duration due to sync-boost constraints (in us). + DCGM_FI_DEV_BOARD_LIMIT_VIOLATION, counter, Throttling duration due to board limit constraints (in us). + DCGM_FI_DEV_LOW_UTIL_VIOLATION, counter, Throttling duration due to low utilization (in us). + DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (in us). + + # NVLink — enable only for GPUs that actually have NVLink (A10 has none). + # DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL, counter, Total number of NVLink bandwidth counters for all lanes. + + # DCP (profiling) metrics — supported from Ampere onwards. + DCGM_FI_PROF_GR_ENGINE_ACTIVE, gauge, Ratio of time the graphics engine is active. + DCGM_FI_PROF_SM_ACTIVE, gauge, The ratio of cycles an SM has at least 1 warp assigned. + DCGM_FI_PROF_SM_OCCUPANCY, gauge, The ratio of number of warps resident on an SM. + DCGM_FI_PROF_PIPE_TENSOR_ACTIVE, gauge, Ratio of cycles the tensor (HMMA) pipe is active. + DCGM_FI_PROF_DRAM_ACTIVE, gauge, Ratio of cycles the device memory interface is active sending or receiving data. + DCGM_FI_PROF_PCIE_TX_BYTES, counter, The number of bytes of active PCIe tx (transmit) data including both header and payload. + DCGM_FI_PROF_PCIE_RX_BYTES, counter, The number of bytes of active PCIe rx (read) data including both header and payload. diff --git a/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml b/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml new file mode 100644 index 00000000..e4718e0a --- /dev/null +++ b/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml @@ -0,0 +1,72 @@ +# Workaround for https://github.com/NVIDIA/gpu-operator/issues/1687 +# +# On Talos, the NVIDIA system extension installs driver files under +# /usr/local/glibc/usr/lib/ and /usr/local/bin/. The GPU Operator +# validator only looks for libnvidia-ml.so.1 and bin/nvidia-smi under +# the path configured as hostPaths.driverInstallDir. This DaemonSet +# stages the required files into /var/nvidia-driver on each node and +# creates the .driver-ctr-ready flag so the validator proceeds. +# +# Paired with examples/values-native-talos.yaml, which sets +# hostPaths.driverInstallDir to /var/nvidia-driver. +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: nvidia-driver-compat + namespace: cozy-gpu-operator + labels: + app: nvidia-driver-compat +spec: + selector: + matchLabels: + app: nvidia-driver-compat + template: + metadata: + labels: + app: nvidia-driver-compat + spec: + hostPID: true + priorityClassName: system-node-critical + tolerations: + - operator: Exists + initContainers: + - name: create-driver-tree + image: busybox:1.37 + command: + - sh + - -c + - | + set -e + GLIBC_LIB="/host/usr/local/glibc/usr/lib" + SRC_BIN="/host/usr/local/bin" + DST="/host/var/nvidia-driver" + + mkdir -p "$DST/bin" + + if [ -f "$GLIBC_LIB/libnvidia-ml.so.1" ]; then + cp -f "$GLIBC_LIB/libnvidia-ml.so.1" "$DST/libnvidia-ml.so.1" + echo "Copied libnvidia-ml.so.1" + fi + + if [ -f "$SRC_BIN/nvidia-smi" ]; then + cp -f "$SRC_BIN/nvidia-smi" "$DST/bin/nvidia-smi" + chmod +x "$DST/bin/nvidia-smi" + echo "Copied nvidia-smi" + fi + + mkdir -p /host/run/nvidia/validations + touch /host/run/nvidia/validations/.driver-ctr-ready + echo "Created driver-ctr-ready flag" + echo "Done" + securityContext: + privileged: true + volumeMounts: + - name: host-root + mountPath: /host + containers: + - name: pause + image: registry.k8s.io/pause:3.10 + volumes: + - name: host-root + hostPath: + path: / diff --git a/packages/system/gpu-operator/examples/values-native-talos.yaml b/packages/system/gpu-operator/examples/values-native-talos.yaml new file mode 100644 index 00000000..86e436c4 --- /dev/null +++ b/packages/system/gpu-operator/examples/values-native-talos.yaml @@ -0,0 +1,45 @@ +# Cozystack Package values for running GPU workloads natively in pods +# on Talos. This is the counterpart to values-talos.yaml, which targets +# the sandbox (VFIO passthrough) scenario. +# +# Prerequisites: +# - NVIDIA Talos system extension installed on GPU nodes. +# - examples/nvidia-driver-compat.yaml deployed to stage driver files +# where the gpu-operator validator looks for them. +# - examples/dcgm-custom-metrics.yaml applied so DCGM Exporter exports +# the full set of metrics used by the dashboard and recording rules. +apiVersion: cozystack.io/v1alpha1 +kind: Package +metadata: + name: cozystack.gpu-operator +spec: + variant: default + components: + gpu-operator: + values: + gpu-operator: + # The compat DaemonSet stages libnvidia-ml.so.1 and nvidia-smi + # under /var/nvidia-driver. Point the validator at that path. + hostPaths: + driverInstallDir: "/var/nvidia-driver" + # Disable the sandbox path — workloads run in pods, not VMs. + sandboxWorkloads: + enabled: false + vfioManager: + enabled: false + # The Talos extension provides the driver and runtime hooks, + # so the operator's own toolkit and driver components must be + # switched off to avoid conflicting installations. + toolkit: + enabled: false + devicePlugin: + enabled: true + # Export full set of DCGM metrics using the ConfigMap in + # examples/dcgm-custom-metrics.yaml. + dcgmExporter: + serviceMonitor: + enabled: true + interval: "15s" + honorLabels: true + config: + name: dcgm-custom-metrics From c1b9a06a3647c48df5a8e25327c38c05425504c8 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Fri, 17 Apr 2026 17:08:18 +0300 Subject: [PATCH 097/250] =?UTF-8?q?feat(monitoring):=20expand=20GPU=20dash?= =?UTF-8?q?boards=20=E2=80=94=20efficiency=20and=20quotas?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Revise gpu-performance and add two new dashboards, registered in dashboards-infra.list: - gpu-efficiency (GPU Efficiency Score) — utilization vs. capacity and workload efficiency signals. - gpu-quotas (GPU Quotas & Allocation) — per-namespace requested vs. used GPUs for tenant capacity planning. All three dashboards use the $ds_prometheus template variable, per the project convention. Assisted-By: Claude Signed-off-by: Arsolitt --- dashboards/gpu/gpu-efficiency.json | 1131 ++++++++++++ dashboards/gpu/gpu-performance.json | 1523 +++++++++++------ dashboards/gpu/gpu-quotas.json | 822 +++++++++ .../system/monitoring/dashboards-infra.list | 2 + 4 files changed, 2956 insertions(+), 522 deletions(-) create mode 100644 dashboards/gpu/gpu-efficiency.json create mode 100644 dashboards/gpu/gpu-quotas.json diff --git a/dashboards/gpu/gpu-efficiency.json b/dashboards/gpu/gpu-efficiency.json new file mode 100644 index 00000000..cc2bca6c --- /dev/null +++ b/dashboards/gpu/gpu-efficiency.json @@ -0,0 +1,1131 @@ +{ + "__inputs": [ + { + "name": "DS_VM-SHORTTERM", + "label": "vm-shortterm", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.4.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Tensor saturation, util/watt and throttling \u2014 reveals inefficient GPU workloads", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "\u041e\u0431\u0449\u0438\u0435 \u043c\u0435\u0442\u0440\u0438\u043a\u0438 \u044d\u0444\u0444\u0435\u043a\u0442\u0438\u0432\u043d\u043e\u0441\u0442\u0438", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "\u0421\u0440\u0435\u0434\u043d\u044f\u044f \u0437\u0430\u0433\u0440\u0443\u0437\u043a\u0430 \u0442\u0435\u043d\u0437\u043e\u0440\u043d\u044b\u0445 \u044f\u0434\u0435\u0440. <10% \u2014 GPU \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u044e\u0442\u0441\u044f \u043d\u0435\u044d\u0444\u0444\u0435\u043a\u0442\u0438\u0432\u043d\u043e (\u043a\u043b\u0438\u0435\u043d\u0442\u044b \u043c\u043e\u0433\u043b\u0438 \u0431\u044b \u043f\u0435\u0440\u0435\u0435\u0445\u0430\u0442\u044c \u043d\u0430 CPU \u0438\u043b\u0438 \u043e\u043f\u0442\u0438\u043c\u0438\u0437\u0438\u0440\u043e\u0432\u0430\u0442\u044c \u043a\u043e\u0434).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 10 + }, + { + "color": "yellow", + "value": 30 + }, + { + "color": "green", + "value": 60 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "avg(pod:tensor_saturation:avg5m)", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + } + } + ], + "title": "Cluster Tensor Saturation", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + }, + "description": "NVML utilization % \u043d\u0430 \u043a\u0430\u0436\u0434\u044b\u0439 \u0432\u0430\u0442\u0442. \u0412\u044b\u0441\u043e\u043a\u043e\u0435 \u0437\u043d\u0430\u0447\u0435\u043d\u0438\u0435 = \u044d\u0444\u0444\u0435\u043a\u0442\u0438\u0432\u043d\u0430\u044f \u043d\u0430\u0433\u0440\u0443\u0437\u043a\u0430.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 3, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "avg(pod:util_per_watt:avg5m)", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + } + } + ], + "title": "Avg Utilization per Watt", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "\u0414\u043e\u043b\u044f \u0432\u0440\u0435\u043c\u0435\u043d\u0438, \u043a\u043e\u0433\u0434\u0430 GPU \u0443\u043f\u0438\u0440\u0430\u044e\u0442\u0441\u044f \u0432 TDP \u0438 \u0442\u0435\u0440\u044f\u044e\u0442 \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c. >5% \u2014 \u043a\u043b\u0438\u0435\u043d\u0442\u044b \u043d\u0435\u0434\u043e\u043f\u043e\u043b\u0443\u0447\u0430\u044e\u0442 FLOPS.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "red", + "value": 20 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "avg(gpu:power_throttle_fraction:rate5m) * 100", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + } + } + ], + "title": "Avg Power Throttling", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 10, + "panels": [], + "title": "NVML vs Tensor (\u043a\u0442\u043e \u043e\u0431\u043c\u0430\u043d\u044b\u0432\u0430\u0435\u0442\u0441\u044f)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + }, + "description": "\u041a\u043b\u0430\u0441\u0441\u0438\u0447\u0435\u0441\u043a\u0430\u044f \u043c\u0435\u0442\u0440\u0438\u043a\u0430 \u0443\u0442\u0438\u043b\u0438\u0437\u0430\u0446\u0438\u0438. \u041f\u043e\u043a\u0430\u0437\u044b\u0432\u0430\u0435\u0442 \u0437\u0430\u043d\u044f\u0442\u043e\u0441\u0442\u044c \u043b\u044e\u0431\u043e\u0433\u043e \u0434\u0432\u0438\u0436\u043a\u0430 (SM, copy, encoder).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "DCGM_FI_DEV_GPU_UTIL{namespace=~\"$namespace\", namespace!=\"\"}", + "legendFormat": "{{namespace}}/{{pod}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + } + } + ], + "title": "NVML GPU Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "\u0420\u0435\u0430\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u0433\u0440\u0443\u0437\u043a\u0430 \u0442\u0435\u043d\u0437\u043e\u0440\u043d\u044b\u0445 \u044f\u0434\u0435\u0440 (HMMA). \u0414\u043b\u044f AI/LLM \u0438\u043d\u0444\u0435\u0440\u0435\u043d\u0441\u0430 \u0434\u043e\u043b\u0436\u043d\u0430 \u0431\u044b\u0442\u044c \u226520%, \u0438\u043d\u0430\u0447\u0435 workload \u043d\u0435 \u043e\u043f\u0442\u0438\u043c\u0438\u0437\u0438\u0440\u043e\u0432\u0430\u043d.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace=~\"$namespace\", namespace!=\"\"} * 100", + "legendFormat": "{{namespace}}/{{pod}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + } + } + ], + "title": "Tensor Pipe Active", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 20, + "panels": [], + "title": "Per-pod \u0440\u0435\u0439\u0442\u0438\u043d\u0433", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + }, + "description": "\u041a\u0442\u043e \u0436\u043c\u0451\u0442 \u0442\u0435\u043d\u0437\u043e\u0440\u043d\u044b\u0435 \u044f\u0434\u0440\u0430, \u0430 \u043a\u0442\u043e \u043d\u0435\u0442. \u0421\u043e\u0440\u0442\u0438\u0440\u043e\u0432\u043a\u0430 \u043e\u0442 \u043c\u0430\u043a\u0441\u0438\u043c\u0443\u043c\u0430.", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Saturation" + }, + "properties": [ + { + "id": "unit", + "value": "percent" + }, + { + "id": "min", + "value": 0 + }, + { + "id": "max", + "value": 100 + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 10 + }, + { + "color": "yellow", + "value": 30 + }, + { + "color": "green", + "value": 60 + } + ] + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 21, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Saturation" + } + ] + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "topk(20, pod:tensor_saturation:avg5m)", + "format": "table", + "instant": true, + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + } + } + ], + "title": "Tensor Saturation per pod (5m avg)", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Hostname": true, + "Time": true, + "UUID": true, + "__name__": true, + "cluster": true, + "container": true, + "endpoint": true, + "gpu": true, + "instance": true, + "job": true, + "modelName": true, + "prometheus": true, + "service": true, + "tenant": true, + "tier": true, + "uid": true, + "unit": true + }, + "indexByName": { + "Value": 2, + "namespace": 0, + "pod": 1 + }, + "renameByName": { + "Value": "Saturation", + "namespace": "Namespace", + "pod": "Pod" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "\u041d\u0430\u0441\u043a\u043e\u043b\u044c\u043a\u043e \u044d\u0444\u0444\u0435\u043a\u0442\u0438\u0432\u043d\u043e \u043a\u0430\u0436\u0434\u044b\u0439 \u043f\u043e\u0434 \u0442\u0440\u0430\u0442\u0438\u0442 \u0432\u0430\u0442\u0442\u044b. \u041d\u0438\u0437\u043a\u043e\u0435 \u0437\u043d\u0430\u0447\u0435\u043d\u0438\u0435 = \u043f\u043b\u043e\u0445\u0430\u044f \u043e\u043f\u0442\u0438\u043c\u0438\u0437\u0430\u0446\u0438\u044f.", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Util/Watt" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "decimals", + "value": 3 + }, + { + "id": "min", + "value": 0 + }, + { + "id": "max", + "value": 1.5 + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "green", + "value": 1 + } + ] + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 22, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Util/Watt" + } + ] + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "topk(20, pod:util_per_watt:avg5m)", + "format": "table", + "instant": true, + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + } + } + ], + "title": "Utilization / Watt per pod (5m avg)", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Hostname": true, + "Time": true, + "UUID": true, + "__name__": true, + "cluster": true, + "container": true, + "endpoint": true, + "gpu": true, + "instance": true, + "job": true, + "modelName": true, + "prometheus": true, + "service": true, + "tenant": true, + "tier": true, + "uid": true, + "unit": true + }, + "indexByName": { + "Value": 2, + "namespace": 0, + "pod": 1 + }, + "renameByName": { + "Value": "Util/Watt", + "namespace": "Namespace", + "pod": "Pod" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 30, + "panels": [], + "title": "Throttling", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + }, + "description": "\u0414\u043e\u043b\u044f \u0432\u0440\u0435\u043c\u0435\u043d\u0438, \u043a\u043e\u0433\u0434\u0430 GPU \u0434\u0440\u043e\u0441\u0441\u0435\u043b\u0438\u0440\u043e\u0432\u0430\u043d\u0430 \u043f\u043e \u043f\u0438\u0442\u0430\u043d\u0438\u044e. 1.0 = \u043f\u043e\u0441\u0442\u043e\u044f\u043d\u043d\u043e.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.05 + }, + { + "color": "red", + "value": 0.2 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 31, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "gpu:power_throttle_fraction:rate5m", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + } + } + ], + "title": "Power throttle fraction per GPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "\u0414\u043e\u043b\u044f \u0432\u0440\u0435\u043c\u0435\u043d\u0438, \u043a\u043e\u0433\u0434\u0430 GPU \u0434\u0440\u043e\u0441\u0441\u0435\u043b\u0438\u0440\u043e\u0432\u0430\u043d\u0430 \u043f\u043e \u0442\u0435\u043c\u043f\u0435\u0440\u0430\u0442\u0443\u0440\u0435.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.05 + }, + { + "color": "red", + "value": 0.2 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 32, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "gpu:thermal_throttle_fraction:rate5m", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + } + } + ], + "title": "Thermal throttle fraction per GPU", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 40, + "tags": [ + "gpu", + "efficiency", + "finops" + ], + "templating": { + "list": [ + { + "current": {}, + "includeAll": false, + "label": "Datasource", + "name": "ds_prometheus", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "vm-.*", + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + }, + "definition": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)", + "includeAll": true, + "label": "Namespace", + "multi": true, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "GPU Efficiency Score", + "uid": "gpu-efficiency", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/dashboards/gpu/gpu-performance.json b/dashboards/gpu/gpu-performance.json index 0b458d4e..906d6a11 100644 --- a/dashboards/gpu/gpu-performance.json +++ b/dashboards/gpu/gpu-performance.json @@ -1,120 +1,100 @@ { + "__inputs": [ + { + "name": "DS_VM-SHORTTERM", + "label": "vm-shortterm", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.4.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], "annotations": { - "list": [] + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": null, "links": [], - "liveNow": false, - "schemaVersion": 39, - "tags": [ - "gpu", - "dcgm" - ], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "default", - "value": "default" - }, - "hide": 0, - "includeAll": false, - "label": "Prometheus", - "multi": false, - "name": "ds_prometheus", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "definition": "label_values(DCGM_FI_DEV_GPU_UTIL, Hostname)", - "hide": 0, - "includeAll": true, - "label": "Host", - "multi": true, - "name": "Hostname", - "options": [], - "query": { - "query": "label_values(DCGM_FI_DEV_GPU_UTIL, Hostname)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "definition": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)", - "hide": 0, - "includeAll": true, - "label": "Namespace", - "multi": true, - "name": "namespace", - "options": [], - "query": { - "query": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "GPU Performance", - "uid": "gpu-performance", - "weekStart": "", "panels": [ { - "type": "row", - "title": "Overview", + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, - "collapsed": false, "id": 1, - "panels": [] + "panels": [], + "title": "Overview", + "type": "row" }, { - "type": "stat", - "title": "Total GPUs", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, "gridPos": { "h": 4, "w": 6, @@ -122,36 +102,12 @@ "y": 1 }, "id": 2, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "targets": [ - { - "expr": "cluster:gpu_count:total", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue" - } - ] - } - }, - "overrides": [] - }, "options": { "colorMode": "value", "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -159,35 +115,35 @@ "fields": "", "values": false }, - "textMode": "value" - } - }, - { - "type": "stat", - "title": "Allocated", - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 1 - }, - "id": 3, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" + "showPercentChange": false, + "textMode": "value", + "wideLayout": true }, + "pluginVersion": "11.4.0", "targets": [ { - "expr": "cluster:gpu_count:allocated", - "refId": "A" + "expr": "cluster:gpu_count:total", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + } } ], + "title": "Total GPUs", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + }, "fieldConfig": { "defaults": { - "unit": "short", "color": { "mode": "thresholds" }, + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ @@ -200,13 +156,24 @@ "value": 1 } ] - } + }, + "unit": "short" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 3, "options": { "colorMode": "value", "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -214,37 +181,37 @@ "fields": "", "values": false }, - "textMode": "value" - } + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "cluster:gpu_count:allocated", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + } + } + ], + "title": "Allocated", + "type": "stat" }, { - "type": "stat", - "title": "Average utilization", - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 1 - }, - "id": 4, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, - "targets": [ - { - "expr": "cluster:gpu_util:avg", - "refId": "A" - } - ], "fieldConfig": { "defaults": { - "unit": "percent", - "min": 0, - "max": 100, "color": { "mode": "thresholds" }, + "mappings": [], + "max": 100, + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -261,13 +228,24 @@ "value": 80 } ] - } + }, + "unit": "percent" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 4, "options": { "colorMode": "value", "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -275,12 +253,49 @@ "fields": "", "values": false }, - "textMode": "value" - } + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "cluster:gpu_util:avg", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + } + } + ], + "title": "Average utilization", + "type": "stat" }, { - "type": "stat", - "title": "Power draw", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "watt" + }, + "overrides": [] + }, "gridPos": { "h": 4, "w": 6, @@ -288,37 +303,12 @@ "y": 1 }, "id": 5, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "targets": [ - { - "expr": "cluster:gpu_power_watts:sum", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "unit": "watt", - "decimals": 0, - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, "options": { "colorMode": "value", "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -326,25 +316,100 @@ "fields": "", "values": false }, - "textMode": "value" - } + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "cluster:gpu_power_watts:sum", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + } + } + ], + "title": "Power draw", + "type": "stat" }, { - "type": "row", - "title": "Utilization", + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, - "collapsed": false, "id": 10, - "panels": [] + "panels": [], + "title": "Utilization", + "type": "row" }, { - "type": "timeseries", - "title": "GPU Utilization (NVML)", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, "gridPos": { "h": 8, "w": 12, @@ -352,49 +417,99 @@ "y": 6 }, "id": 11, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, + "pluginVersion": "11.4.0", "targets": [ { "expr": "DCGM_FI_DEV_GPU_UTIL{Hostname=~\"$Hostname\", namespace=~\"$namespace\"}", "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", - "refId": "A" + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + } } ], + "title": "GPU Utilization (NVML)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + }, "fieldConfig": { "defaults": { - "unit": "percent", - "min": 0, - "max": 100, + "color": { + "mode": "palette-classic" + }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", - "lineInterpolation": "linear", "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", - "spanNulls": false - } + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" }, "overrides": [] }, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi" - } - } - }, - { - "type": "timeseries", - "title": "Tensor Pipe Active (realistic load for LLM/AI)", "gridPos": { "h": 8, "w": 12, @@ -402,49 +517,99 @@ "y": 6 }, "id": 12, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, + "pluginVersion": "11.4.0", "targets": [ { "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 100", "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", - "refId": "A" + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + } } ], + "title": "Tensor Pipe Active (realistic load for LLM/AI)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, "fieldConfig": { "defaults": { - "unit": "percent", - "min": 0, - "max": 100, + "color": { + "mode": "palette-classic" + }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", - "lineInterpolation": "linear", "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", - "spanNulls": false - } + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" }, "overrides": [] }, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi" - } - } - }, - { - "type": "timeseries", - "title": "Graphics Engine Active", "gridPos": { "h": 8, "w": 12, @@ -452,49 +617,99 @@ "y": 14 }, "id": 13, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, + "pluginVersion": "11.4.0", "targets": [ { "expr": "DCGM_FI_PROF_GR_ENGINE_ACTIVE{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 100", "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", - "refId": "A" + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + } } ], + "title": "Graphics Engine Active", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + }, "fieldConfig": { "defaults": { - "unit": "percent", - "min": 0, - "max": 100, + "color": { + "mode": "palette-classic" + }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", - "lineInterpolation": "linear", "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", - "spanNulls": false - } + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" }, "overrides": [] }, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi" - } - } - }, - { - "type": "timeseries", - "title": "Memory Copy Utilization", "gridPos": { "h": 8, "w": 12, @@ -502,62 +717,110 @@ "y": 14 }, "id": 14, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, + "pluginVersion": "11.4.0", "targets": [ { "expr": "DCGM_FI_DEV_MEM_COPY_UTIL{Hostname=~\"$Hostname\", namespace=~\"$namespace\"}", "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", - "refId": "A" + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + } } ], - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi" - } - } + "title": "Memory Copy Utilization", + "type": "timeseries" }, { - "type": "row", - "title": "Memory", + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 }, - "collapsed": false, "id": 20, - "panels": [] + "panels": [], + "title": "Memory", + "type": "row" }, { - "type": "timeseries", - "title": "VRAM Used", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, "gridPos": { "h": 8, "w": 12, @@ -565,49 +828,97 @@ "y": 23 }, "id": 21, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, + "pluginVersion": "11.4.0", "targets": [ { "expr": "DCGM_FI_DEV_FB_USED{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 1048576", "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", - "refId": "A" + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + } } ], + "title": "VRAM Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + }, "fieldConfig": { "defaults": { - "unit": "bytes", + "color": { + "mode": "palette-classic" + }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, "lineInterpolation": "linear", - "fillOpacity": 25, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", + "spanNulls": false, "stacking": { + "group": "A", "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } - } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" }, "overrides": [] }, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi" - } - } - }, - { - "type": "timeseries", - "title": "VRAM Free", "gridPos": { "h": 8, "w": 12, @@ -615,59 +926,110 @@ "y": 23 }, "id": 22, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" + "options": { + "legend": { + "calcs": [ + "mean", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, + "pluginVersion": "11.4.0", "targets": [ { "expr": "DCGM_FI_DEV_FB_FREE{Hostname=~\"$Hostname\"} * 1048576", "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + } } ], - "fieldConfig": { - "defaults": { - "unit": "bytes", - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "min" - ] - }, - "tooltip": { - "mode": "multi" - } - } + "title": "VRAM Free", + "type": "timeseries" }, { - "type": "row", - "title": "Power & Temperature", + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 31 }, - "collapsed": false, "id": 30, - "panels": [] + "panels": [], + "title": "Power & Temperature", + "type": "row" }, { - "type": "timeseries", - "title": "Power Usage per GPU", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "watt" + }, + "overrides": [] + }, "gridPos": { "h": 8, "w": 12, @@ -675,80 +1037,88 @@ "y": 32 }, "id": 31, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, + "pluginVersion": "11.4.0", "targets": [ { "expr": "DCGM_FI_DEV_POWER_USAGE{Hostname=~\"$Hostname\"}", "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + } } ], - "fieldConfig": { - "defaults": { - "unit": "watt", - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi" - } - } + "title": "Power Usage per GPU", + "type": "timeseries" }, { - "type": "timeseries", - "title": "GPU Temperature", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 32, "datasource": { "type": "prometheus", - "uid": "$ds_prometheus" + "uid": "${DS_VM-SHORTTERM}" }, - "targets": [ - { - "expr": "DCGM_FI_DEV_GPU_TEMP{Hostname=~\"$Hostname\"}", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" - } - ], "fieldConfig": { "defaults": { - "unit": "celsius", - "min": 20, - "max": 100, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never" + "color": { + "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 20, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "yellow", @@ -759,61 +1129,68 @@ "value": 85 } ] - } + }, + "unit": "celsius" }, "overrides": [] }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 32, "options": { "legend": { - "displayMode": "table", - "placement": "bottom", "calcs": [ "mean", "max" - ] + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true }, "tooltip": { - "mode": "multi" + "mode": "multi", + "sort": "none" } - } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "DCGM_FI_DEV_GPU_TEMP{Hostname=~\"$Hostname\"}", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + } + } + ], + "title": "GPU Temperature", + "type": "timeseries" }, { - "type": "row", - "title": "Health", + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 40 }, - "collapsed": false, "id": 40, - "panels": [] + "panels": [], + "title": "Health", + "type": "row" }, { - "type": "stat", - "title": "XID errors (latest)", - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 41 - }, - "id": 41, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, - "targets": [ - { - "expr": "max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS{Hostname=~\"$Hostname\"})", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" - } - ], "fieldConfig": { "defaults": { - "unit": "short", "color": { "mode": "thresholds" }, @@ -828,10 +1205,18 @@ "value": 1 } ] - } + }, + "unit": "short" }, "overrides": [] }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 41 + }, + "id": 41, "options": { "colorMode": "background", "graphMode": "none", @@ -843,11 +1228,38 @@ "values": false }, "textMode": "value_and_name" - } + }, + "targets": [ + { + "expr": "max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS{Hostname=~\"$Hostname\"})", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + } + } + ], + "title": "XID errors (latest)", + "type": "stat" }, { - "type": "timeseries", - "title": "Power Violation (\u00b5s/s throttled due to power)", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "linear", + "showPoints": "never" + }, + "unit": "µs" + }, + "overrides": [] + }, "gridPos": { "h": 5, "w": 8, @@ -855,29 +1267,6 @@ "y": 41 }, "id": 42, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "targets": [ - { - "expr": "rate(DCGM_FI_DEV_POWER_VIOLATION{Hostname=~\"$Hostname\"}[5m])", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "unit": "\u00b5s", - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [] - }, "options": { "legend": { "displayMode": "list", @@ -886,11 +1275,38 @@ "tooltip": { "mode": "multi" } - } + }, + "targets": [ + { + "expr": "rate(DCGM_FI_DEV_POWER_VIOLATION{Hostname=~\"$Hostname\"}[5m])", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + } + } + ], + "title": "Power Violation (µs/s throttled due to power)", + "type": "timeseries" }, { - "type": "timeseries", - "title": "Thermal Violation (\u00b5s/s throttled due to thermals)", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "linear", + "showPoints": "never" + }, + "unit": "µs" + }, + "overrides": [] + }, "gridPos": { "h": 5, "w": 8, @@ -898,29 +1314,6 @@ "y": 41 }, "id": 43, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "targets": [ - { - "expr": "rate(DCGM_FI_DEV_THERMAL_VIOLATION{Hostname=~\"$Hostname\"}[5m])", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "unit": "\u00b5s", - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [] - }, "options": { "legend": { "displayMode": "list", @@ -929,7 +1322,93 @@ "tooltip": { "mode": "multi" } - } + }, + "targets": [ + { + "expr": "rate(DCGM_FI_DEV_THERMAL_VIOLATION{Hostname=~\"$Hostname\"}[5m])", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + } + } + ], + "title": "Thermal Violation (µs/s throttled due to thermals)", + "type": "timeseries" } - ] -} \ No newline at end of file + ], + "refresh": "", + "schemaVersion": 40, + "tags": [ + "gpu", + "dcgm" + ], + "templating": { + "list": [ + { + "current": {}, + "includeAll": false, + "label": "Prometheus", + "name": "ds_prometheus", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + }, + "definition": "label_values(DCGM_FI_DEV_GPU_UTIL, Hostname)", + "includeAll": true, + "label": "Host", + "multi": true, + "name": "Hostname", + "options": [], + "query": { + "query": "label_values(DCGM_FI_DEV_GPU_UTIL, Hostname)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "definition": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)", + "includeAll": true, + "label": "Namespace", + "multi": true, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "GPU Performance", + "uid": "gpu-performance", + "version": 2, + "weekStart": "" +} diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json new file mode 100644 index 00000000..28f2c892 --- /dev/null +++ b/dashboards/gpu/gpu-quotas.json @@ -0,0 +1,822 @@ +{ + "__inputs": [ + { + "name": "DS_VM-SHORTTERM", + "label": "vm-shortterm", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "bargauge", + "name": "Bar gauge", + "version": "" + }, + { + "type": "panel", + "id": "gauge", + "name": "Gauge", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.4.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Allocation overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"})", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + } + } + ], + "title": "GPU allocatable", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"})", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + } + } + ], + "title": "GPU requested", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "green", + "value": 40 + }, + { + "color": "yellow", + "value": 80 + }, + { + "color": "red", + "value": 95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 4, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}) / sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"}) * 100", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + } + } + ], + "title": "Allocation ratio", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "count((kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"} > 0) * on(namespace, pod) group_left() (kube_pod_status_phase{phase=\"Pending\"} == 1)) or vector(0)", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + } + } + ], + "title": "Pending pods (GPU)", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 10, + "panels": [], + "title": "Per namespace", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 8, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 11, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "sum by (namespace) (kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"})", + "legendFormat": "{{namespace}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + } + } + ], + "title": "GPU requested per namespace", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Allocatable (total)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"})", + "legendFormat": "Allocatable (total)", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + } + }, + { + "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"})", + "legendFormat": "Requested", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + } + } + ], + "title": "GPU allocated over time", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 20, + "panels": [], + "title": "Pods with GPU", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "color-background" + } + }, + { + "id": "mappings", + "value": [ + { + "options": { + "Failed": { + "color": "red", + "index": 2 + }, + "Pending": { + "color": "orange", + "index": 1 + }, + "Running": { + "color": "green", + "index": 0 + } + }, + "type": "value" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 21, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "expr": "kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left(phase) (kube_pod_status_phase{phase=~\"Running|Pending|Failed|Succeeded\"} == 1)", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "requested", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + } + }, + { + "expr": "kube_pod_container_resource_limits{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"}", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "limits", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + } + } + ], + "title": "Pods requesting GPU", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "pod", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "__name__": true, + "__name__ 1": true, + "__name__ 2": true, + "cluster": true, + "cluster 1": true, + "cluster 2": true, + "container 2": true, + "endpoint": true, + "endpoint 1": true, + "endpoint 2": true, + "instance": true, + "instance 1": true, + "instance 2": true, + "job": true, + "job 1": true, + "job 2": true, + "namespace 2": true, + "node 2": true, + "prometheus": true, + "prometheus 1": true, + "prometheus 2": true, + "resource": true, + "resource 1": true, + "resource 2": true, + "service": true, + "service 1": true, + "service 2": true, + "tenant": true, + "tenant 1": true, + "tenant 2": true, + "tier": true, + "tier 1": true, + "tier 2": true, + "uid": true, + "uid 1": true, + "uid 2": true, + "unit": true, + "unit 1": true, + "unit 2": true + }, + "indexByName": { + "Value #limits": 5, + "Value #requested": 4, + "container 1": 2, + "namespace 1": 0, + "node 1": 3, + "phase": 6, + "pod": 1 + }, + "renameByName": { + "Value #limits": "Limit", + "Value #requested": "Req", + "container 1": "Container", + "namespace 1": "Namespace", + "node 1": "Node", + "phase": "Status" + } + } + } + ], + "type": "table" + } + ], + "refresh": "", + "schemaVersion": 40, + "tags": [ + "gpu", + "quotas" + ], + "templating": { + "list": [ + { + "current": {}, + "includeAll": false, + "label": "Prometheus", + "name": "ds_prometheus", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_VM-SHORTTERM}" + }, + "definition": "label_values(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}, namespace)", + "includeAll": true, + "label": "Namespace", + "multi": true, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}, namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "GPU Quotas & Allocation", + "uid": "gpu-quotas", + "version": 4, + "weekStart": "" +} diff --git a/packages/system/monitoring/dashboards-infra.list b/packages/system/monitoring/dashboards-infra.list index 581f9a05..b402fa68 100644 --- a/packages/system/monitoring/dashboards-infra.list +++ b/packages/system/monitoring/dashboards-infra.list @@ -33,3 +33,5 @@ hubble/dns-namespace hubble/l7-http-metrics hubble/network-overview gpu/gpu-performance +gpu/gpu-efficiency +gpu/gpu-quotas From 7e5f3a7f12ae6ff744032c30eb0e7b05071986a2 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Fri, 17 Apr 2026 17:45:06 +0300 Subject: [PATCH 098/250] refactor(monitoring): clean up GPU dashboards Strip Grafana export boilerplate (__inputs, __elements, __requires, default annotations, embedded datasource inputs) and tighten panel layouts across the three GPU dashboards. All three continue to use the $ds_prometheus template variable. Assisted-By: Claude Signed-off-by: Arsolitt --- dashboards/gpu/gpu-efficiency.json | 1186 ++++++++------------ dashboards/gpu/gpu-performance.json | 1574 ++++++++++----------------- dashboards/gpu/gpu-quotas.json | 876 ++++++--------- 3 files changed, 1358 insertions(+), 2278 deletions(-) diff --git a/dashboards/gpu/gpu-efficiency.json b/dashboards/gpu/gpu-efficiency.json index cc2bca6c..1a305b4b 100644 --- a/dashboards/gpu/gpu-efficiency.json +++ b/dashboards/gpu/gpu-efficiency.json @@ -1,72 +1,26 @@ { - "__inputs": [ - { - "name": "DS_VM-SHORTTERM", - "label": "vm-shortterm", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } + "uid": "gpu-efficiency", + "title": "GPU Efficiency Score", + "description": "Tensor saturation, util/watt and throttling — reveals inefficient GPU workloads", + "tags": [ + "gpu", + "efficiency", + "finops" ], - "__elements": {}, - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "11.4.0" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "stat", - "name": "Stat", - "version": "" - }, - { - "type": "panel", - "id": "table", - "name": "Table", - "version": "" - }, - { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "Tensor saturation, util/watt and throttling \u2014 reveals inefficient GPU workloads", + "timezone": "browser", "editable": true, - "fiscalYearStartMonth": 0, "graphTooltip": 1, - "id": null, - "links": [], + "time": { + "from": "now-1h", + "to": "now" + }, + "fiscalYearStartMonth": 0, + "schemaVersion": 42, "panels": [ { + "type": "row", "collapsed": false, + "title": "Overall efficiency metrics", "gridPos": { "h": 1, "w": 24, @@ -74,48 +28,23 @@ "y": 0 }, "id": 1, - "panels": [], - "title": "\u041e\u0431\u0449\u0438\u0435 \u043c\u0435\u0442\u0440\u0438\u043a\u0438 \u044d\u0444\u0444\u0435\u043a\u0442\u0438\u0432\u043d\u043e\u0441\u0442\u0438", - "type": "row" + "panels": [] }, { + "type": "stat", + "id": 2, + "targets": [ + { + "expr": "avg(pod:tensor_saturation:avg5m)", + "refId": "A" + } + ], + "title": "Cluster Tensor Saturation", + "description": "Mean tensor core saturation. \u003c10% means GPUs are used inefficiently (tenants could move to CPU or optimize their code).", + "transparent": false, "datasource": { "type": "prometheus", - "uid": "${ds_prometheus}" - }, - "description": "\u0421\u0440\u0435\u0434\u043d\u044f\u044f \u0437\u0430\u0433\u0440\u0443\u0437\u043a\u0430 \u0442\u0435\u043d\u0437\u043e\u0440\u043d\u044b\u0445 \u044f\u0434\u0435\u0440. <10% \u2014 GPU \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u044e\u0442\u0441\u044f \u043d\u0435\u044d\u0444\u0444\u0435\u043a\u0442\u0438\u0432\u043d\u043e (\u043a\u043b\u0438\u0435\u043d\u0442\u044b \u043c\u043e\u0433\u043b\u0438 \u0431\u044b \u043f\u0435\u0440\u0435\u0435\u0445\u0430\u0442\u044c \u043d\u0430 CPU \u0438\u043b\u0438 \u043e\u043f\u0442\u0438\u043c\u0438\u0437\u0438\u0440\u043e\u0432\u0430\u0442\u044c \u043a\u043e\u0434).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "orange", - "value": 10 - }, - { - "color": "yellow", - "value": 30 - }, - { - "color": "green", - "value": 60 - } - ] - }, - "unit": "percent" - }, - "overrides": [] + "uid": "$ds_prometheus" }, "gridPos": { "h": 5, @@ -123,71 +52,67 @@ "x": 0, "y": 1 }, - "id": 2, + "repeatDirection": "h", "options": { - "colorMode": "background", "graphMode": "area", + "colorMode": "background", "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" - ], - "fields": "", - "values": false + ] }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": true + "percentChangeColorMode": "standard", + "orientation": "" }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "avg(pod:tensor_saturation:avg5m)", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${ds_prometheus}" - } - } - ], - "title": "Cluster Tensor Saturation", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - }, - "description": "NVML utilization % \u043d\u0430 \u043a\u0430\u0436\u0434\u044b\u0439 \u0432\u0430\u0442\u0442. \u0412\u044b\u0441\u043e\u043a\u043e\u0435 \u0437\u043d\u0430\u0447\u0435\u043d\u0438\u0435 = \u044d\u0444\u0444\u0435\u043a\u0442\u0438\u0432\u043d\u0430\u044f \u043d\u0430\u0433\u0440\u0443\u0437\u043a\u0430.", "fieldConfig": { "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 3, - "mappings": [], + "unit": "percent", + "min": 0, + "max": 100, "thresholds": { "mode": "absolute", "steps": [ { - "color": "red", - "value": null + "value": null, + "color": "red" }, { - "color": "yellow", - "value": 0.5 + "value": 10, + "color": "orange" }, { - "color": "green", - "value": 1 + "value": 30, + "color": "yellow" + }, + { + "value": 60, + "color": "green" } ] - }, - "unit": "none" + } }, "overrides": [] + } + }, + { + "type": "stat", + "id": 3, + "targets": [ + { + "expr": "avg(pod:util_per_watt:avg5m)", + "refId": "A" + } + ], + "title": "Avg Utilization per Watt", + "description": "NVML utilization % per watt. Higher value = more efficient workload.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 5, @@ -195,73 +120,62 @@ "x": 8, "y": 1 }, - "id": 3, + "repeatDirection": "h", "options": { - "colorMode": "background", "graphMode": "area", + "colorMode": "background", "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" - ], - "fields": "", - "values": false + ] }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": true + "percentChangeColorMode": "standard", + "orientation": "" }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "avg(pod:util_per_watt:avg5m)", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - } - } - ], - "title": "Avg Utilization per Watt", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${ds_prometheus}" - }, - "description": "\u0414\u043e\u043b\u044f \u0432\u0440\u0435\u043c\u0435\u043d\u0438, \u043a\u043e\u0433\u0434\u0430 GPU \u0443\u043f\u0438\u0440\u0430\u044e\u0442\u0441\u044f \u0432 TDP \u0438 \u0442\u0435\u0440\u044f\u044e\u0442 \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c. >5% \u2014 \u043a\u043b\u0438\u0435\u043d\u0442\u044b \u043d\u0435\u0434\u043e\u043f\u043e\u043b\u0443\u0447\u0430\u044e\u0442 FLOPS.", "fieldConfig": { "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 2, - "mappings": [], - "max": 100, - "min": 0, + "unit": "none", + "decimals": 3, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "value": null, + "color": "red" }, { - "color": "yellow", - "value": 5 + "value": 0.5, + "color": "yellow" }, { - "color": "red", - "value": 20 + "value": 1, + "color": "green" } ] - }, - "unit": "percent" + } }, "overrides": [] + } + }, + { + "type": "stat", + "id": 4, + "targets": [ + { + "expr": "avg(gpu:power_throttle_fraction:rate5m)", + "refId": "A" + } + ], + "title": "Avg Power Throttling", + "description": "Fraction of time GPUs hit the TDP cap and lose performance. \u003e5% means tenants underutilize billed FLOPS.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 5, @@ -269,40 +183,51 @@ "x": 16, "y": 1 }, - "id": 4, + "repeatDirection": "h", "options": { - "colorMode": "background", "graphMode": "area", + "colorMode": "background", "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" - ], - "fields": "", - "values": false + ] }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": true + "percentChangeColorMode": "standard", + "orientation": "" }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "avg(gpu:power_throttle_fraction:rate5m) * 100", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${ds_prometheus}" + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 0.05, + "color": "yellow" + }, + { + "value": 0.2, + "color": "red" + } + ] } - } - ], - "title": "Avg Power Throttling", - "type": "stat" + }, + "overrides": [] + } }, { + "type": "row", "collapsed": false, + "title": "NVML vs Tensor (mismatch detector)", "gridPos": { "h": 1, "w": 24, @@ -310,73 +235,24 @@ "y": 6 }, "id": 10, - "panels": [], - "title": "NVML vs Tensor (\u043a\u0442\u043e \u043e\u0431\u043c\u0430\u043d\u044b\u0432\u0430\u0435\u0442\u0441\u044f)", - "type": "row" + "panels": [] }, { + "type": "timeseries", + "id": 11, + "targets": [ + { + "expr": "DCGM_FI_DEV_GPU_UTIL{namespace=~\"$namespace\", namespace!=\"\"}", + "legendFormat": "{{namespace}}/{{pod}}", + "refId": "A" + } + ], + "title": "NVML GPU Utilization", + "description": "Classic utilization metric. Shows activity of any engine (SM, copy, encoder).", + "transparent": false, "datasource": { "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - }, - "description": "\u041a\u043b\u0430\u0441\u0441\u0438\u0447\u0435\u0441\u043a\u0430\u044f \u043c\u0435\u0442\u0440\u0438\u043a\u0430 \u0443\u0442\u0438\u043b\u0438\u0437\u0430\u0446\u0438\u0438. \u041f\u043e\u043a\u0430\u0437\u044b\u0432\u0430\u0435\u0442 \u0437\u0430\u043d\u044f\u0442\u043e\u0441\u0442\u044c \u043b\u044e\u0431\u043e\u0433\u043e \u0434\u0432\u0438\u0436\u043a\u0430 (SM, copy, encoder).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] + "uid": "$ds_prometheus" }, "gridPos": { "h": 8, @@ -384,100 +260,53 @@ "x": 0, "y": 7 }, - "id": 11, + "repeatDirection": "h", "options": { "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, "calcs": [ "mean", "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + ] }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "" } }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "DCGM_FI_DEV_GPU_UTIL{namespace=~\"$namespace\", namespace!=\"\"}", - "legendFormat": "{{namespace}}/{{pod}}", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - } - } - ], - "title": "NVML GPU Utilization", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${ds_prometheus}" - }, - "description": "\u0420\u0435\u0430\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u0433\u0440\u0443\u0437\u043a\u0430 \u0442\u0435\u043d\u0437\u043e\u0440\u043d\u044b\u0445 \u044f\u0434\u0435\u0440 (HMMA). \u0414\u043b\u044f AI/LLM \u0438\u043d\u0444\u0435\u0440\u0435\u043d\u0441\u0430 \u0434\u043e\u043b\u0436\u043d\u0430 \u0431\u044b\u0442\u044c \u226520%, \u0438\u043d\u0430\u0447\u0435 workload \u043d\u0435 \u043e\u043f\u0442\u0438\u043c\u0438\u0437\u0438\u0440\u043e\u0432\u0430\u043d.", "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 100, + "unit": "percent", "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" + "max": 100, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never" + } }, "overrides": [] + } + }, + { + "type": "timeseries", + "id": 12, + "targets": [ + { + "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace=~\"$namespace\", namespace!=\"\"} * 100", + "legendFormat": "{{namespace}}/{{pod}}", + "refId": "A" + } + ], + "title": "Tensor Pipe Active", + "description": "Real tensor core load (HMMA). For AI/LLM inference it should be ≥20%, otherwise the workload is unoptimized.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 8, @@ -485,39 +314,41 @@ "x": 12, "y": 7 }, - "id": 12, + "repeatDirection": "h", "options": { "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, "calcs": [ "mean", "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + ] }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "" } }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace=~\"$namespace\", namespace!=\"\"} * 100", - "legendFormat": "{{namespace}}/{{pod}}", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${ds_prometheus}" + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never" } - } - ], - "title": "Tensor Pipe Active", - "type": "timeseries" + }, + "overrides": [] + } }, { + "type": "row", "collapsed": false, + "title": "Per-pod ranking", "gridPos": { "h": 1, "w": 24, @@ -525,40 +356,92 @@ "y": 15 }, "id": 20, - "panels": [], - "title": "Per-pod \u0440\u0435\u0439\u0442\u0438\u043d\u0433", - "type": "row" + "panels": [] }, { + "type": "table", + "id": 21, + "targets": [ + { + "expr": "topk(20, pod:tensor_saturation:avg5m)", + "instant": true, + "range": false, + "format": "table", + "refId": "A" + } + ], + "title": "Tensor Saturation per pod (5m avg)", + "description": "Who is exercising tensor cores and who is not. Sorted descending.", + "transparent": false, "datasource": { "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" + "uid": "$ds_prometheus" }, - "description": "\u041a\u0442\u043e \u0436\u043c\u0451\u0442 \u0442\u0435\u043d\u0437\u043e\u0440\u043d\u044b\u0435 \u044f\u0434\u0440\u0430, \u0430 \u043a\u0442\u043e \u043d\u0435\u0442. \u0421\u043e\u0440\u0442\u0438\u0440\u043e\u0432\u043a\u0430 \u043e\u0442 \u043c\u0430\u043a\u0441\u0438\u043c\u0443\u043c\u0430.", - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "repeatDirection": "h", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "DCGM_FI_DRIVER_VERSION": true, + "Hostname": true, + "Time": true, + "UUID": true, + "__name__": true, + "cluster": true, + "container": true, + "device": true, + "endpoint": true, + "gpu": true, + "gpu_driver_version": true, + "instance": true, + "job": true, + "modelName": true, + "pci_bus_id": true, + "prometheus": true, + "service": true, + "tenant": true, + "tier": true, + "uid": true, + "unit": true }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "indexByName": { + "Value": 2, + "namespace": 0, + "pod": 1 + }, + "renameByName": { + "Value": "Saturation", + "namespace": "Namespace", + "pod": "Pod" + } } + } + ], + "options": { + "frameIndex": 0, + "showHeader": true, + "showTypeIcons": false, + "sortBy": [ + { + "displayName": "Saturation", + "desc": true + } + ], + "footer": { + "show": false, + "reducer": [] }, + "cellHeight": "sm" + }, + "fieldConfig": { + "defaults": {}, "overrides": [ { "matcher": { @@ -591,8 +474,7 @@ "mode": "absolute", "steps": [ { - "color": "red", - "value": null + "color": "red" }, { "color": "orange", @@ -612,62 +494,54 @@ ] } ] + } + }, + { + "type": "table", + "id": 22, + "targets": [ + { + "expr": "topk(20, pod:util_per_watt:avg5m)", + "instant": true, + "range": false, + "format": "table", + "refId": "A" + } + ], + "title": "Utilization / Watt per pod (5m avg)", + "description": "How efficiently each pod spends watts. Low value = poor optimization.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 8, "w": 12, - "x": 0, + "x": 12, "y": 16 }, - "id": 21, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Saturation" - } - ] - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "topk(20, pod:tensor_saturation:avg5m)", - "format": "table", - "instant": true, - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - } - } - ], - "title": "Tensor Saturation per pod (5m avg)", + "repeatDirection": "h", "transformations": [ { "id": "organize", "options": { "excludeByName": { + "DCGM_FI_DRIVER_VERSION": true, "Hostname": true, "Time": true, "UUID": true, "__name__": true, "cluster": true, "container": true, + "device": true, "endpoint": true, "gpu": true, + "gpu_driver_version": true, "instance": true, "job": true, "modelName": true, + "pci_bus_id": true, "prometheus": true, "service": true, "tenant": true, @@ -681,45 +555,31 @@ "pod": 1 }, "renameByName": { - "Value": "Saturation", + "Value": "Util/Watt", "namespace": "Namespace", "pod": "Pod" } } } ], - "type": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${ds_prometheus}" - }, - "description": "\u041d\u0430\u0441\u043a\u043e\u043b\u044c\u043a\u043e \u044d\u0444\u0444\u0435\u043a\u0442\u0438\u0432\u043d\u043e \u043a\u0430\u0436\u0434\u044b\u0439 \u043f\u043e\u0434 \u0442\u0440\u0430\u0442\u0438\u0442 \u0432\u0430\u0442\u0442\u044b. \u041d\u0438\u0437\u043a\u043e\u0435 \u0437\u043d\u0430\u0447\u0435\u043d\u0438\u0435 = \u043f\u043b\u043e\u0445\u0430\u044f \u043e\u043f\u0442\u0438\u043c\u0438\u0437\u0430\u0446\u0438\u044f.", - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "options": { + "frameIndex": 0, + "showHeader": true, + "showTypeIcons": false, + "sortBy": [ + { + "displayName": "Util/Watt", + "desc": true } + ], + "footer": { + "show": false, + "reducer": [] }, + "cellHeight": "sm" + }, + "fieldConfig": { + "defaults": {}, "overrides": [ { "matcher": { @@ -756,8 +616,7 @@ "mode": "absolute", "steps": [ { - "color": "red", - "value": null + "color": "red" }, { "color": "yellow", @@ -773,86 +632,12 @@ ] } ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 22, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Util/Watt" - } - ] - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "topk(20, pod:util_per_watt:avg5m)", - "format": "table", - "instant": true, - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${ds_prometheus}" - } - } - ], - "title": "Utilization / Watt per pod (5m avg)", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "Hostname": true, - "Time": true, - "UUID": true, - "__name__": true, - "cluster": true, - "container": true, - "endpoint": true, - "gpu": true, - "instance": true, - "job": true, - "modelName": true, - "prometheus": true, - "service": true, - "tenant": true, - "tier": true, - "uid": true, - "unit": true - }, - "indexByName": { - "Value": 2, - "namespace": 0, - "pod": 1 - }, - "renameByName": { - "Value": "Util/Watt", - "namespace": "Namespace", - "pod": "Pod" - } - } - } - ], - "type": "table" + } }, { + "type": "row", "collapsed": false, + "title": "Throttling", "gridPos": { "h": 1, "w": 24, @@ -860,77 +645,24 @@ "y": 24 }, "id": 30, - "panels": [], - "title": "Throttling", - "type": "row" + "panels": [] }, { + "type": "timeseries", + "id": 31, + "targets": [ + { + "expr": "gpu:power_throttle_fraction:rate5m", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A" + } + ], + "title": "Power throttle fraction per GPU", + "description": "Fraction of time the GPU was power-throttled. 1.0 = always throttled.", + "transparent": false, "datasource": { "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - }, - "description": "\u0414\u043e\u043b\u044f \u0432\u0440\u0435\u043c\u0435\u043d\u0438, \u043a\u043e\u0433\u0434\u0430 GPU \u0434\u0440\u043e\u0441\u0441\u0435\u043b\u0438\u0440\u043e\u0432\u0430\u043d\u0430 \u043f\u043e \u043f\u0438\u0442\u0430\u043d\u0438\u044e. 1.0 = \u043f\u043e\u0441\u0442\u043e\u044f\u043d\u043d\u043e.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 25, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.05 - }, - { - "color": "red", - "value": 0.2 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] + "uid": "$ds_prometheus" }, "gridPos": { "h": 8, @@ -938,104 +670,70 @@ "x": 0, "y": 25 }, - "id": 31, + "repeatDirection": "h", "options": { "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, "calcs": [ "mean", "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + ] }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "" } }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "gpu:power_throttle_fraction:rate5m", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - } - } - ], - "title": "Power throttle fraction per GPU", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${ds_prometheus}" - }, - "description": "\u0414\u043e\u043b\u044f \u0432\u0440\u0435\u043c\u0435\u043d\u0438, \u043a\u043e\u0433\u0434\u0430 GPU \u0434\u0440\u043e\u0441\u0441\u0435\u043b\u0438\u0440\u043e\u0432\u0430\u043d\u0430 \u043f\u043e \u0442\u0435\u043c\u043f\u0435\u0440\u0430\u0442\u0443\u0440\u0435.", "fieldConfig": { "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 25, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 1, + "unit": "percentunit", "min": 0, + "max": 1, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "value": null, + "color": "green" }, { - "color": "yellow", - "value": 0.05 + "value": 0.05, + "color": "yellow" }, { - "color": "red", - "value": 0.2 + "value": 0.2, + "color": "red" } ] }, - "unit": "percentunit" + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 25, + "showPoints": "never" + } }, "overrides": [] + } + }, + { + "type": "timeseries", + "id": 32, + "targets": [ + { + "expr": "gpu:thermal_throttle_fraction:rate5m", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A" + } + ], + "title": "Thermal throttle fraction per GPU", + "description": "Fraction of time the GPU was thermal-throttled.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 8, @@ -1043,89 +741,101 @@ "x": 12, "y": 25 }, - "id": 32, + "repeatDirection": "h", "options": { "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, "calcs": [ "mean", "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + ] }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "" } }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "gpu:thermal_throttle_fraction:rate5m", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${ds_prometheus}" + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 0.05, + "color": "yellow" + }, + { + "value": 0.2, + "color": "red" + } + ] + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 25, + "showPoints": "never" } - } - ], - "title": "Thermal throttle fraction per GPU", - "type": "timeseries" + }, + "overrides": [] + } } ], - "refresh": "", - "schemaVersion": 40, - "tags": [ - "gpu", - "efficiency", - "finops" - ], "templating": { "list": [ { - "current": {}, - "includeAll": false, - "label": "Datasource", + "type": "datasource", "name": "ds_prometheus", - "options": [], + "label": "Prometheus", + "skipUrlSync": false, "query": "prometheus", - "refresh": 1, - "regex": "vm-.*", - "type": "datasource" + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "multi": false, + "allowCustomValue": true, + "includeAll": false, + "regex": "", + "auto": false, + "auto_min": "10s", + "auto_count": 30 }, { - "current": {}, + "type": "query", + "name": "namespace", + "label": "Namespace", + "skipUrlSync": false, + "query": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)", "datasource": { "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" + "uid": "$ds_prometheus" + }, + "current": { + "selected": true, + "text": "All", + "value": "$__all" }, - "definition": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)", - "includeAll": true, - "label": "Namespace", "multi": true, - "name": "namespace", - "options": [], - "query": { - "query": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)", - "refId": "StandardVariableQuery" - }, + "allowCustomValue": true, "refresh": 2, - "regex": "", "sort": 1, - "type": "query" + "includeAll": true, + "auto": false, + "auto_min": "10s", + "auto_count": 30 } ] }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "GPU Efficiency Score", - "uid": "gpu-efficiency", - "version": 2, - "weekStart": "" + "annotations": {} } \ No newline at end of file diff --git a/dashboards/gpu/gpu-performance.json b/dashboards/gpu/gpu-performance.json index 906d6a11..b0f26958 100644 --- a/dashboards/gpu/gpu-performance.json +++ b/dashboards/gpu/gpu-performance.json @@ -1,65 +1,24 @@ { - "__inputs": [ - { - "name": "DS_VM-SHORTTERM", - "label": "vm-shortterm", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } + "uid": "gpu-performance", + "title": "GPU Performance", + "tags": [ + "gpu", + "dcgm" ], - "__elements": {}, - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "11.4.0" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "stat", - "name": "Stat", - "version": "" - }, - { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, + "timezone": "browser", "editable": true, - "fiscalYearStartMonth": 0, "graphTooltip": 1, - "id": null, - "links": [], + "time": { + "from": "now-1h", + "to": "now" + }, + "fiscalYearStartMonth": 0, + "schemaVersion": 42, "panels": [ { + "type": "row", "collapsed": false, + "title": "Overview", "gridPos": { "h": 1, "w": 24, @@ -67,99 +26,75 @@ "y": 0 }, "id": 1, - "panels": [], - "title": "Overview", - "type": "row" + "panels": [] }, { + "type": "stat", + "id": 2, + "targets": [ + { + "expr": "cluster:gpu_count:total", + "refId": "A" + } + ], + "title": "Total GPUs", + "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, - "id": 2, + "repeatDirection": "h", "options": { - "colorMode": "value", "graphMode": "none", + "colorMode": "value", "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" - ], - "fields": "", - "values": false + ] }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "cluster:gpu_count:total", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - } - } - ], - "title": "Total GPUs", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" + "percentChangeColorMode": "standard", + "orientation": "" }, "fieldConfig": { "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], + "unit": "short", "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 + "value": null, + "color": "blue" } ] - }, - "unit": "short" + } }, "overrides": [] + } + }, + { + "type": "stat", + "id": 3, + "targets": [ + { + "expr": "cluster:gpu_count:allocated", + "refId": "A" + } + ], + "title": "Allocated", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 4, @@ -167,71 +102,56 @@ "x": 6, "y": 1 }, - "id": 3, + "repeatDirection": "h", "options": { - "colorMode": "value", "graphMode": "none", + "colorMode": "value", "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" - ], - "fields": "", - "values": false + ] }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "cluster:gpu_count:allocated", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - } - } - ], - "title": "Allocated", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" + "percentChangeColorMode": "standard", + "orientation": "" }, "fieldConfig": { "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "max": 100, - "min": 0, + "unit": "short", "thresholds": { "mode": "absolute", "steps": [ { - "color": "blue", - "value": null + "value": null, + "color": "green" }, { - "color": "green", - "value": 30 - }, - { - "color": "orange", - "value": 80 + "value": 1, + "color": "yellow" } ] - }, - "unit": "percent" + } }, "overrides": [] + } + }, + { + "type": "stat", + "id": 4, + "targets": [ + { + "expr": "cluster:gpu_util:avg", + "refId": "A" + } + ], + "title": "Average utilization", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 4, @@ -239,62 +159,62 @@ "x": 12, "y": 1 }, - "id": 4, + "repeatDirection": "h", "options": { - "colorMode": "value", "graphMode": "area", + "colorMode": "value", "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" - ], - "fields": "", - "values": false + ] }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "cluster:gpu_util:avg", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - } - } - ], - "title": "Average utilization", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" + "percentChangeColorMode": "standard", + "orientation": "" }, "fieldConfig": { "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 0, - "mappings": [], + "unit": "percent", + "min": 0, + "max": 100, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "value": null, + "color": "blue" + }, + { + "value": 30, + "color": "green" + }, + { + "value": 80, + "color": "orange" } ] - }, - "unit": "watt" + } }, "overrides": [] + } + }, + { + "type": "stat", + "id": 5, + "targets": [ + { + "expr": "cluster:gpu_power_watts:sum", + "refId": "A" + } + ], + "title": "Power draw", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 4, @@ -302,40 +222,43 @@ "x": 18, "y": 1 }, - "id": 5, + "repeatDirection": "h", "options": { - "colorMode": "value", "graphMode": "area", + "colorMode": "value", "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" - ], - "fields": "", - "values": false + ] }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": true + "percentChangeColorMode": "standard", + "orientation": "" }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "cluster:gpu_power_watts:sum", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" + "fieldConfig": { + "defaults": { + "unit": "watt", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + } + ] } - } - ], - "title": "Power draw", - "type": "stat" + }, + "overrides": [] + } }, { + "type": "row", "collapsed": false, + "title": "Utilization", "gridPos": { "h": 1, "w": 24, @@ -343,172 +266,77 @@ "y": 5 }, "id": 10, - "panels": [], - "title": "Utilization", - "type": "row" + "panels": [] }, { - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 6 - }, + "type": "timeseries", "id": 11, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", "targets": [ { "expr": "DCGM_FI_DEV_GPU_UTIL{Hostname=~\"$Hostname\", namespace=~\"$namespace\"}", "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - } + "refId": "A" } ], "title": "GPU Utilization (NVML)", - "type": "timeseries" - }, - { + "transparent": false, "datasource": { "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "repeatDirection": "h", + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "" + } }, "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 100, + "unit": "percent", "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" + "max": 100, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false + } }, "overrides": [] + } + }, + { + "type": "timeseries", + "id": 12, + "targets": [ + { + "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 100", + "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", + "refId": "A" + } + ], + "title": "Tensor Pipe Active (realistic load for LLM/AI)", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 8, @@ -516,99 +344,53 @@ "x": 12, "y": 6 }, - "id": 12, + "repeatDirection": "h", "options": { "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, "calcs": [ "mean", "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + ] }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "" } }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 100", - "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - } - } - ], - "title": "Tensor Pipe Active (realistic load for LLM/AI)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 100, + "unit": "percent", "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" + "max": 100, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false + } }, "overrides": [] + } + }, + { + "type": "timeseries", + "id": 13, + "targets": [ + { + "expr": "DCGM_FI_PROF_GR_ENGINE_ACTIVE{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 100", + "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", + "refId": "A" + } + ], + "title": "Graphics Engine Active", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 8, @@ -616,99 +398,53 @@ "x": 0, "y": 14 }, - "id": 13, + "repeatDirection": "h", "options": { "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, "calcs": [ "mean", "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + ] }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "" } }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "DCGM_FI_PROF_GR_ENGINE_ACTIVE{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 100", - "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - } - } - ], - "title": "Graphics Engine Active", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - }, "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 100, + "unit": "percent", "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" + "max": 100, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false + } }, "overrides": [] + } + }, + { + "type": "timeseries", + "id": 14, + "targets": [ + { + "expr": "DCGM_FI_DEV_MEM_COPY_UTIL{Hostname=~\"$Hostname\", namespace=~\"$namespace\"}", + "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", + "refId": "A" + } + ], + "title": "Memory Copy Utilization", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 8, @@ -716,39 +452,42 @@ "x": 12, "y": 14 }, - "id": 14, + "repeatDirection": "h", "options": { "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, "calcs": [ "mean", "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + ] }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "" } }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "DCGM_FI_DEV_MEM_COPY_UTIL{Hostname=~\"$Hostname\", namespace=~\"$namespace\"}", - "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false } - } - ], - "title": "Memory Copy Utilization", - "type": "timeseries" + }, + "overrides": [] + } }, { + "type": "row", "collapsed": false, + "title": "Memory", "gridPos": { "h": 1, "w": 24, @@ -756,168 +495,74 @@ "y": 22 }, "id": 20, - "panels": [], - "title": "Memory", - "type": "row" + "panels": [] }, { + "type": "timeseries", + "id": 21, + "targets": [ + { + "expr": "DCGM_FI_DEV_FB_USED{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 1048576", + "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", + "refId": "A" + } + ], + "title": "VRAM Used", + "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 25, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 23 }, - "id": 21, + "repeatDirection": "h", "options": { "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, "calcs": [ "mean", "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + ] }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "" } }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "DCGM_FI_DEV_FB_USED{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 1048576", - "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - } - } - ], - "title": "VRAM Used", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - }, "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, + "unit": "bytes", "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" + "fillOpacity": 25, + "showPoints": "never" + } }, "overrides": [] + } + }, + { + "type": "timeseries", + "id": 22, + "targets": [ + { + "expr": "DCGM_FI_DEV_FB_FREE{Hostname=~\"$Hostname\"} * 1048576", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A" + } + ], + "title": "VRAM Free", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 8, @@ -925,39 +570,39 @@ "x": 12, "y": 23 }, - "id": 22, + "repeatDirection": "h", "options": { "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, "calcs": [ "mean", "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + ] }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "" } }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "DCGM_FI_DEV_FB_FREE{Hostname=~\"$Hostname\"} * 1048576", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never" } - } - ], - "title": "VRAM Free", - "type": "timeseries" + }, + "overrides": [] + } }, { + "type": "row", "collapsed": false, + "title": "Power \u0026 Temperature", "gridPos": { "h": 1, "w": 24, @@ -965,174 +610,74 @@ "y": 31 }, "id": 30, - "panels": [], - "title": "Power & Temperature", - "type": "row" + "panels": [] }, { + "type": "timeseries", + "id": 31, + "targets": [ + { + "expr": "DCGM_FI_DEV_POWER_USAGE{Hostname=~\"$Hostname\"}", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A" + } + ], + "title": "Power Usage per GPU", + "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "watt" - }, - "overrides": [] - }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 }, - "id": 31, + "repeatDirection": "h", "options": { "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, "calcs": [ "mean", "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + ] }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "" } }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "DCGM_FI_DEV_POWER_USAGE{Hostname=~\"$Hostname\"}", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - } - } - ], - "title": "Power Usage per GPU", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - }, "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, + "unit": "watt", "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 100, - "min": 20, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 75 - }, - { - "color": "red", - "value": 85 - } - ] - }, - "unit": "celsius" + "fillOpacity": 10, + "showPoints": "never" + } }, "overrides": [] + } + }, + { + "type": "timeseries", + "id": 32, + "targets": [ + { + "expr": "DCGM_FI_DEV_GPU_TEMP{Hostname=~\"$Hostname\"}", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A" + } + ], + "title": "GPU Temperature", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 8, @@ -1140,39 +685,58 @@ "x": 12, "y": 32 }, - "id": 32, + "repeatDirection": "h", "options": { "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, "calcs": [ "mean", "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + ] }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "" } }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "DCGM_FI_DEV_GPU_TEMP{Hostname=~\"$Hostname\"}", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" + "fieldConfig": { + "defaults": { + "unit": "celsius", + "min": 20, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 75, + "color": "yellow" + }, + { + "value": 85, + "color": "red" + } + ] + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never" } - } - ], - "title": "GPU Temperature", - "type": "timeseries" + }, + "overrides": [] + } }, { + "type": "row", "collapsed": false, + "title": "Health", "gridPos": { "h": 1, "w": 24, @@ -1180,85 +744,81 @@ "y": 40 }, "id": 40, - "panels": [], - "title": "Health", - "type": "row" + "panels": [] }, { + "type": "stat", + "id": 41, + "targets": [ + { + "expr": "max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS{Hostname=~\"$Hostname\"})", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A" + } + ], + "title": "XID errors (latest)", + "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 1 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, "gridPos": { "h": 5, "w": 8, "x": 0, "y": 41 }, - "id": 41, + "repeatDirection": "h", "options": { - "colorMode": "background", "graphMode": "none", + "colorMode": "background", + "justifyMode": "auto", + "textMode": "value_and_name", + "wideLayout": true, + "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" - ], - "fields": "", - "values": false + ] }, - "textMode": "value_and_name" - }, - "targets": [ - { - "expr": "max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS{Hostname=~\"$Hostname\"})", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - } - } - ], - "title": "XID errors (latest)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" + "percentChangeColorMode": "standard", + "orientation": "" }, "fieldConfig": { "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineInterpolation": "linear", - "showPoints": "never" - }, - "unit": "µs" + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 1, + "color": "red" + } + ] + } }, "overrides": [] + } + }, + { + "type": "timeseries", + "id": 42, + "targets": [ + { + "expr": "rate(DCGM_FI_DEV_POWER_VIOLATION{Hostname=~\"$Hostname\"}[5m])", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A" + } + ], + "title": "Power Violation (µs/s throttled due to power)", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 5, @@ -1266,46 +826,47 @@ "x": 8, "y": 41 }, - "id": 42, + "repeatDirection": "h", "options": { "legend": { "displayMode": "list", - "placement": "bottom" + "placement": "bottom", + "showLegend": false, + "calcs": [] }, "tooltip": { - "mode": "multi" + "mode": "multi", + "sort": "" } }, - "targets": [ - { - "expr": "rate(DCGM_FI_DEV_POWER_VIOLATION{Hostname=~\"$Hostname\"}[5m])", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - } - } - ], - "title": "Power Violation (µs/s throttled due to power)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, "fieldConfig": { "defaults": { + "unit": "µs", "custom": { "drawStyle": "line", - "fillOpacity": 10, "lineInterpolation": "linear", + "fillOpacity": 10, "showPoints": "never" - }, - "unit": "µs" + } }, "overrides": [] + } + }, + { + "type": "timeseries", + "id": 43, + "targets": [ + { + "expr": "rate(DCGM_FI_DEV_THERMAL_VIOLATION{Hostname=~\"$Hostname\"}[5m])", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A" + } + ], + "title": "Thermal Violation (µs/s throttled due to thermals)", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 5, @@ -1313,102 +874,103 @@ "x": 16, "y": 41 }, - "id": 43, + "repeatDirection": "h", "options": { "legend": { "displayMode": "list", - "placement": "bottom" + "placement": "bottom", + "showLegend": false, + "calcs": [] }, "tooltip": { - "mode": "multi" + "mode": "multi", + "sort": "" } }, - "targets": [ - { - "expr": "rate(DCGM_FI_DEV_THERMAL_VIOLATION{Hostname=~\"$Hostname\"}[5m])", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" + "fieldConfig": { + "defaults": { + "unit": "µs", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never" } - } - ], - "title": "Thermal Violation (µs/s throttled due to thermals)", - "type": "timeseries" + }, + "overrides": [] + } } ], - "refresh": "", - "schemaVersion": 40, - "tags": [ - "gpu", - "dcgm" - ], "templating": { "list": [ { - "current": {}, - "includeAll": false, - "label": "Prometheus", + "type": "datasource", "name": "ds_prometheus", - "options": [], + "label": "Prometheus", + "skipUrlSync": false, "query": "prometheus", - "refresh": 1, + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "multi": false, + "allowCustomValue": true, + "includeAll": false, "regex": "", - "type": "datasource" + "auto": false, + "auto_min": "10s", + "auto_count": 30 }, { - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - }, - "definition": "label_values(DCGM_FI_DEV_GPU_UTIL, Hostname)", - "includeAll": true, - "label": "Host", - "multi": true, + "type": "query", "name": "Hostname", - "options": [], - "query": { - "query": "label_values(DCGM_FI_DEV_GPU_UTIL, Hostname)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": {}, + "label": "Host", + "skipUrlSync": false, + "query": "label_values(DCGM_FI_DEV_GPU_UTIL, Hostname)", "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, - "definition": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)", - "includeAll": true, - "label": "Namespace", - "multi": true, - "name": "namespace", - "options": [], - "query": { - "query": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)", - "refId": "StandardVariableQuery" + "current": { + "selected": true, + "text": "All", + "value": "$__all" }, + "multi": true, + "allowCustomValue": true, "refresh": 2, - "regex": "", "sort": 1, - "type": "query" + "includeAll": true, + "auto": false, + "auto_min": "10s", + "auto_count": 30 + }, + { + "type": "query", + "name": "namespace", + "label": "Namespace", + "skipUrlSync": false, + "query": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "multi": true, + "allowCustomValue": true, + "refresh": 2, + "sort": 1, + "includeAll": true, + "auto": false, + "auto_min": "10s", + "auto_count": 30 } ] }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "GPU Performance", - "uid": "gpu-performance", - "version": 2, - "weekStart": "" + "annotations": {} } diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json index 28f2c892..840065f3 100644 --- a/dashboards/gpu/gpu-quotas.json +++ b/dashboards/gpu/gpu-quotas.json @@ -1,83 +1,24 @@ { - "__inputs": [ - { - "name": "DS_VM-SHORTTERM", - "label": "vm-shortterm", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } + "uid": "gpu-quotas", + "title": "GPU Quotas \u0026 Allocation", + "tags": [ + "gpu", + "quotas" ], - "__elements": {}, - "__requires": [ - { - "type": "panel", - "id": "bargauge", - "name": "Bar gauge", - "version": "" - }, - { - "type": "panel", - "id": "gauge", - "name": "Gauge", - "version": "" - }, - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "11.4.0" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "stat", - "name": "Stat", - "version": "" - }, - { - "type": "panel", - "id": "table", - "name": "Table", - "version": "" - }, - { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, + "timezone": "browser", "editable": true, - "fiscalYearStartMonth": 0, "graphTooltip": 1, - "id": null, - "links": [], + "time": { + "from": "now-6h", + "to": "now" + }, + "fiscalYearStartMonth": 0, + "schemaVersion": 42, "panels": [ { + "type": "row", "collapsed": false, + "title": "Allocation overview", "gridPos": { "h": 1, "w": 24, @@ -85,99 +26,75 @@ "y": 0 }, "id": 1, - "panels": [], - "title": "Allocation overview", - "type": "row" + "panels": [] }, { + "type": "stat", + "id": 2, + "targets": [ + { + "expr": "sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"})", + "refId": "A" + } + ], + "title": "GPU allocatable", + "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, - "id": 2, + "repeatDirection": "h", "options": { - "colorMode": "value", "graphMode": "none", + "colorMode": "value", "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" - ], - "fields": "", - "values": false + ] }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"})", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - } - } - ], - "title": "GPU allocatable", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" + "percentChangeColorMode": "standard", + "orientation": "" }, "fieldConfig": { "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], + "unit": "short", "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 + "value": null, + "color": "blue" } ] - }, - "unit": "short" + } }, "overrides": [] + } + }, + { + "type": "stat", + "id": 3, + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"})", + "refId": "A" + } + ], + "title": "GPU requested", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 4, @@ -185,75 +102,56 @@ "x": 6, "y": 1 }, - "id": 3, + "repeatDirection": "h", "options": { - "colorMode": "value", "graphMode": "none", + "colorMode": "value", "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" - ], - "fields": "", - "values": false + ] }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"})", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - } - } - ], - "title": "GPU requested", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" + "percentChangeColorMode": "standard", + "orientation": "" }, "fieldConfig": { "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "max": 100, - "min": 0, + "unit": "short", "thresholds": { "mode": "absolute", "steps": [ { - "color": "blue", - "value": null + "value": null, + "color": "green" }, { - "color": "green", - "value": 40 - }, - { - "color": "yellow", - "value": 80 - }, - { - "color": "red", - "value": 95 + "value": 1, + "color": "yellow" } ] - }, - "unit": "percent" + } }, "overrides": [] + } + }, + { + "type": "gauge", + "id": 4, + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}) / sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"}) * 100", + "refId": "A" + } + ], + "title": "Allocation ratio", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 4, @@ -261,63 +159,64 @@ "x": 12, "y": 1 }, - "id": 4, + "repeatDirection": "h", "options": { - "minVizHeight": 75, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", "minVizWidth": 75, - "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" - ], - "fields": "", - "values": false + ] }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}) / sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"}) * 100", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - } - } - ], - "title": "Allocation ratio", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" + "minVizHeight": 75, + "orientation": "" }, "fieldConfig": { "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], + "unit": "percent", + "min": 0, + "max": 100, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "value": null, + "color": "blue" }, { - "color": "red", - "value": 1 + "value": 40, + "color": "green" + }, + { + "value": 80, + "color": "yellow" + }, + { + "value": 95, + "color": "red" } ] - }, - "unit": "short" + } }, "overrides": [] + } + }, + { + "type": "stat", + "id": 5, + "targets": [ + { + "expr": "count((kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"} \u003e 0) * on(namespace, pod) group_left() (kube_pod_status_phase{phase=\"Pending\"} == 1)) or vector(0)", + "refId": "A" + } + ], + "title": "Pending pods (GPU)", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" }, "gridPos": { "h": 4, @@ -325,40 +224,46 @@ "x": 18, "y": 1 }, - "id": 5, + "repeatDirection": "h", "options": { - "colorMode": "background", "graphMode": "none", + "colorMode": "background", "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" - ], - "fields": "", - "values": false + ] }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": true + "percentChangeColorMode": "standard", + "orientation": "" }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "count((kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"} > 0) * on(namespace, pod) group_left() (kube_pod_status_phase{phase=\"Pending\"} == 1)) or vector(0)", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 1, + "color": "red" + } + ] } - } - ], - "title": "Pending pods (GPU)", - "type": "stat" + }, + "overrides": [] + } }, { + "type": "row", "collapsed": false, + "title": "Per namespace", "gridPos": { "h": 1, "w": 24, @@ -366,140 +271,120 @@ "y": 5 }, "id": 10, - "panels": [], - "title": "Per namespace", - "type": "row" + "panels": [] }, { + "type": "bargauge", + "id": 11, + "targets": [ + { + "expr": "sum by (namespace) (kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"})", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "title": "GPU requested per namespace", + "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "max": 8, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, - "id": 11, + "repeatDirection": "h", "options": { "displayMode": "gradient", + "valueMode": "color", + "namePlacement": "auto", + "showUnfilled": true, + "sizing": "auto", + "minVizWidth": 8, + "minVizHeight": 16, "legend": { - "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": false + "showLegend": false, + "calcs": [] }, - "maxVizHeight": 300, - "minVizHeight": 16, - "minVizWidth": 8, - "namePlacement": "auto", - "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" - ], - "fields": "", - "values": false + ] }, - "showUnfilled": true, - "sizing": "auto", - "valueMode": "color" - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "sum by (namespace) (kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"})", - "legendFormat": "{{namespace}}", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - } - } - ], - "title": "GPU requested per namespace", - "type": "bargauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" + "maxVizHeight": 300, + "orientation": "horizontal" }, "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "stepAfter", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], + "unit": "short", + "min": 0, + "max": 8, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 + "value": null, + "color": "green" } ] - }, - "unit": "short" + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "id": 12, + "targets": [ + { + "expr": "sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"})", + "legendFormat": "Allocatable (total)", + "refId": "A" + }, + { + "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"})", + "legendFormat": "Requested", + "refId": "B" + } + ], + "title": "GPU allocated over time", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "repeatDirection": "h", + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineInterpolation": "stepAfter", + "fillOpacity": 10, + "showPoints": "never" + } }, "overrides": [ { @@ -518,52 +403,12 @@ ] } ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 6 - }, - "id": 12, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"})", - "legendFormat": "Allocatable (total)", - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - } - }, - { - "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"})", - "legendFormat": "Requested", - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" - } - } - ], - "title": "GPU allocated over time", - "type": "timeseries" + } }, { + "type": "row", "collapsed": false, + "title": "Pods with GPU", "gridPos": { "h": 1, "w": 24, @@ -571,124 +416,40 @@ "y": 14 }, "id": 20, - "panels": [], - "title": "Pods with GPU", - "type": "row" + "panels": [] }, { + "type": "table", + "id": 21, + "targets": [ + { + "expr": "kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left(phase) (kube_pod_status_phase{phase=~\"Running|Pending|Failed|Succeeded\"} == 1)", + "instant": true, + "range": false, + "format": "table", + "refId": "requested" + }, + { + "expr": "kube_pod_container_resource_limits{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"}", + "instant": true, + "range": false, + "format": "table", + "refId": "limits" + } + ], + "title": "Pods requesting GPU", + "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Status" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "mode": "basic", - "type": "color-background" - } - }, - { - "id": "mappings", - "value": [ - { - "options": { - "Failed": { - "color": "red", - "index": 2 - }, - "Pending": { - "color": "orange", - "index": 1 - }, - "Running": { - "color": "green", - "index": 0 - } - }, - "type": "value" - } - ] - } - ] - } - ] - }, "gridPos": { "h": 10, "w": 24, "x": 0, "y": 15 }, - "id": 21, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "expr": "kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left(phase) (kube_pod_status_phase{phase=~\"Running|Pending|Failed|Succeeded\"} == 1)", - "format": "table", - "instant": true, - "legendFormat": "", - "refId": "requested", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - } - }, - { - "expr": "kube_pod_container_resource_limits{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"}", - "format": "table", - "instant": true, - "legendFormat": "", - "refId": "limits", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - } - } - ], - "title": "Pods requesting GPU", + "repeatDirection": "h", "transformations": [ { "id": "joinByField", @@ -764,59 +525,106 @@ } } ], - "type": "table" + "options": { + "frameIndex": 0, + "showHeader": true, + "showTypeIcons": false, + "footer": { + "show": false, + "reducer": [] + }, + "cellHeight": "sm" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "color-background" + } + }, + { + "id": "mappings", + "value": [ + { + "options": { + "Failed": { + "color": "red", + "index": 2 + }, + "Pending": { + "color": "orange", + "index": 1 + }, + "Running": { + "color": "green", + "index": 0 + } + }, + "type": "value" + } + ] + } + ] + } + ] + } } ], - "refresh": "", - "schemaVersion": 40, - "tags": [ - "gpu", - "quotas" - ], "templating": { "list": [ { - "current": {}, - "includeAll": false, - "label": "Prometheus", + "type": "datasource", "name": "ds_prometheus", - "options": [], + "label": "Prometheus", + "skipUrlSync": false, "query": "prometheus", - "refresh": 1, + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "multi": false, + "allowCustomValue": true, + "includeAll": false, "regex": "", - "type": "datasource" + "auto": false, + "auto_min": "10s", + "auto_count": 30 }, { - "current": {}, + "type": "query", + "name": "namespace", + "label": "Namespace", + "skipUrlSync": false, + "query": "label_values(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}, namespace)", "datasource": { "type": "prometheus", - "uid": "${DS_VM-SHORTTERM}" + "uid": "$ds_prometheus" + }, + "current": { + "selected": true, + "text": "All", + "value": "$__all" }, - "definition": "label_values(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}, namespace)", - "includeAll": true, - "label": "Namespace", "multi": true, - "name": "namespace", - "options": [], - "query": { - "query": "label_values(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}, namespace)", - "refId": "StandardVariableQuery" - }, + "allowCustomValue": true, "refresh": 2, - "regex": "", "sort": 1, - "type": "query" + "includeAll": true, + "auto": false, + "auto_min": "10s", + "auto_count": 30 } ] }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "GPU Quotas & Allocation", - "uid": "gpu-quotas", - "version": 4, - "weekStart": "" -} + "annotations": {} +} \ No newline at end of file From 2461c239b03840cba10111029fb79e8b8195527e Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Fri, 17 Apr 2026 11:32:44 +0500 Subject: [PATCH 099/250] fix(linstor): restrict linstor-gui to cozystack-cluster-admin group MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before this change oauth2-proxy fronting linstor-gui only enforced that the user could authenticate against the `cozy` Keycloak realm (`--email-domain=*`, no group restriction). Any realm user could reach the UI and, through it, the LINSTOR controller REST API — which the gatekeeper proxies with a static mTLS client cert and no per-user RBAC. Add `--allowed-group=cozystack-cluster-admin` and include `groups` in the OIDC scope so the claim is present at validation time. The `cozystack-cluster-admin` KeycloakRealmGroup and the `groups` client scope are already provisioned by keycloak-configure, so no cluster-wide changes are needed. Assisted-By: Claude Signed-off-by: Myasnikov Daniil (cherry picked from commit 9b54e4672364a0de7859c5273dd2b280374083ea) --- packages/system/linstor-gui/README.md | 7 +++++-- packages/system/linstor-gui/templates/gatekeeper.yaml | 8 +++++++- packages/system/linstor-gui/tests/ingress_auth_test.yaml | 6 ++++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/packages/system/linstor-gui/README.md b/packages/system/linstor-gui/README.md index 2c9ac5d5..8f8053ab 100644 --- a/packages/system/linstor-gui/README.md +++ b/packages/system/linstor-gui/README.md @@ -13,8 +13,11 @@ using mTLS with the `linstor-client-tls` secret created by the `linstor` package The chart ships an `oauth2-proxy` based gatekeeper plus a `KeycloakClient` CRD so the UI can be published on `linstor-gui.` behind the cluster -Keycloak realm. Access is granted to any user who can authenticate against the -`cozy` realm. +Keycloak realm. Access is restricted to members of the +`cozystack-cluster-admin` Keycloak group — the same group that grants +cluster-admin RBAC on the host cluster. Authenticating against the `cozy` +realm alone is not sufficient; users outside that group receive a 403 from +oauth2-proxy before any request reaches the UI or the LINSTOR controller. To turn it on, add `linstor-gui` to `publishing.exposedServices` in the core `cozystack` values (same list that controls `dashboard`). OIDC must be diff --git a/packages/system/linstor-gui/templates/gatekeeper.yaml b/packages/system/linstor-gui/templates/gatekeeper.yaml index 0830a5fa..81451f41 100644 --- a/packages/system/linstor-gui/templates/gatekeeper.yaml +++ b/packages/system/linstor-gui/templates/gatekeeper.yaml @@ -13,6 +13,11 @@ Only rendered when oidc-enabled=true. If the cluster has OIDC disabled we deliberately do not ship an unauthenticated token-proxy fallback (unlike dashboard): linstor-gui surfaces raw storage state and should not be reachable via Ingress without Keycloak login. + +Access is further restricted to the cozystack-cluster-admin group +(--allowed-group below). The proxy authenticates with a static mTLS client +cert and LINSTOR itself has no per-user RBAC, so group membership is the +only boundary between a realm user and raw storage state. */ -}} {{- if eq $oidcEnabled "true" }} apiVersion: apps/v1 @@ -84,7 +89,8 @@ spec: - --cookie-secure=true - --cookie-secret=$(OAUTH2_PROXY_COOKIE_SECRET) - --skip-provider-button - - --scope=openid email profile offline_access + - --scope=openid email profile groups offline_access + - --allowed-group=cozystack-cluster-admin {{- if eq $oidcInsecureSkipVerify "true" }} - --ssl-insecure-skip-verify=true {{- end }} diff --git a/packages/system/linstor-gui/tests/ingress_auth_test.yaml b/packages/system/linstor-gui/tests/ingress_auth_test.yaml index fe735f50..2f0b1d73 100644 --- a/packages/system/linstor-gui/tests/ingress_auth_test.yaml +++ b/packages/system/linstor-gui/tests/ingress_auth_test.yaml @@ -109,6 +109,12 @@ tests: - contains: path: spec.template.spec.containers[0].args content: --oidc-issuer-url=https://keycloak.dev10.example.org/realms/cozy + - contains: + path: spec.template.spec.containers[0].args + content: --scope=openid email profile groups offline_access + - contains: + path: spec.template.spec.containers[0].args + content: --allowed-group=cozystack-cluster-admin - contains: path: spec.template.spec.containers[0].env content: From 11f7d3589b3284fca96d0df21a02a5e967dadce7 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 05:47:33 +0300 Subject: [PATCH 100/250] chore: ignore CLAUDE.local.md Signed-off-by: Arsolitt --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 61003de5..64470222 100644 --- a/.gitignore +++ b/.gitignore @@ -84,3 +84,4 @@ tmp/ # build revision marker (generated by make image-packages) packages/core/platform/.build-revision .claude/ +CLAUDE.local.md From 4f8cef47bf307d7e23eac6536a1e7153a940d283 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 05:54:21 +0300 Subject: [PATCH 101/250] fix(monitoring): restore trailing newline in GPU dashboards Signed-off-by: Arsolitt --- dashboards/gpu/gpu-efficiency.json | 2 +- dashboards/gpu/gpu-quotas.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboards/gpu/gpu-efficiency.json b/dashboards/gpu/gpu-efficiency.json index 1a305b4b..596961e0 100644 --- a/dashboards/gpu/gpu-efficiency.json +++ b/dashboards/gpu/gpu-efficiency.json @@ -838,4 +838,4 @@ ] }, "annotations": {} -} \ No newline at end of file +} diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json index 840065f3..60c628f6 100644 --- a/dashboards/gpu/gpu-quotas.json +++ b/dashboards/gpu/gpu-quotas.json @@ -627,4 +627,4 @@ ] }, "annotations": {} -} \ No newline at end of file +} From 4e8731b58882edab23d79c99b5b6b49dda96c572 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 06:02:17 +0300 Subject: [PATCH 102/250] refactor(monitoring): rework GPU recording rules - Drop gpu.recording.30s group: per-GPU 30s aggregates had no consumers in tracked dashboards, only burned cardinality. - Drop namespace:gpu_allocated_count:gauge: identical expression to namespace:gpu_count:sum under a different name. - Reground :allocated on kube_pod_container_resource_requests so it reflects what tenants requested (Pending+Running) rather than what DCGM currently sees. namespace:gpu_count:sum stays DCGM-based and represents actually-running pods; the gap between the two is the signal admins want. - Add namespace:gpu_count:allocated as the per-namespace counterpart. Signed-off-by: Arsolitt --- .../alerts/gpu-recording.rules.yaml | 82 +++++++++++++------ 1 file changed, 59 insertions(+), 23 deletions(-) diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml index 871a46fc..a54860b7 100644 --- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml +++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml @@ -4,33 +4,17 @@ metadata: name: alerts-gpu-recording.rules spec: groups: - - name: gpu.recording.30s - interval: 30s - rules: - - record: gpu:util:avg - expr: avg by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_DEV_GPU_UTIL) - - record: gpu:mem_copy_util:avg - expr: avg by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_DEV_MEM_COPY_UTIL) - - record: gpu:fb_used_bytes:max - expr: max by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_DEV_FB_USED) * 1048576 - - record: gpu:fb_free_bytes:max - expr: max by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_DEV_FB_FREE) * 1048576 - - record: gpu:power_watts:avg - expr: avg by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_DEV_POWER_USAGE) - - record: gpu:temp_celsius:max - expr: max by (Hostname, gpu, UUID, modelName) (DCGM_FI_DEV_GPU_TEMP) - - record: gpu:tensor_active:avg - expr: avg by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_PROF_PIPE_TENSOR_ACTIVE) - - record: gpu:gr_engine_active:avg - expr: avg by (Hostname, gpu, UUID, modelName, namespace, pod) (DCGM_FI_PROF_GR_ENGINE_ACTIVE) - - name: gpu.recording.cluster.1m interval: 1m rules: - record: cluster:gpu_count:total expr: count(group by (UUID) (DCGM_FI_DEV_GPU_UTIL)) + # Kube-allocated GPU count: GPUs requested by *all* pods regardless of + # phase (Pending+Running). Source of truth for "what tenants asked for" + # — used for capacity planning and billing. Includes system pods so it + # stays consistent with cluster:gpu_count:total when computing :free. - record: cluster:gpu_count:allocated - expr: count(group by (UUID) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"})) + expr: sum(kube_pod_container_resource_requests{resource="nvidia_com_gpu"}) - record: cluster:gpu_count:free expr: cluster:gpu_count:total - (cluster:gpu_count:allocated or vector(0)) - record: cluster:gpu_util:avg @@ -41,8 +25,16 @@ spec: - name: gpu.recording.namespace.1m interval: 1m rules: + # DCGM-visible GPU count per namespace — counts GPUs that are actually + # running a tenant pod right now (driver loaded, scheduler placed it). + # Differs from :allocated when pods are Pending or stuck. - record: namespace:gpu_count:sum expr: count by (namespace) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}) + # Kube-requested GPU count per namespace — billable view, includes + # Pending pods. Use this for GPU-hour reporting via + # sum_over_time(...[1h:1m])/60. + - record: namespace:gpu_count:allocated + expr: sum by (namespace) (kube_pod_container_resource_requests{resource="nvidia_com_gpu"}) - record: namespace:gpu_util:avg expr: avg by (namespace) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}) - record: namespace:tensor_active:avg @@ -53,5 +45,49 @@ spec: expr: sum by (namespace) (DCGM_FI_DEV_POWER_USAGE{namespace!="", namespace!~"cozy-.*|kube-.*"}) - record: namespace:energy_joules:sum expr: sum by (namespace) (DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION{namespace!="", namespace!~"cozy-.*|kube-.*"}) / 1000 - - record: namespace:gpu_allocated_count:gauge - expr: count by (namespace) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}) + + - name: gpu.recording.efficiency.1m + interval: 1m + rules: + # Tensor hardware saturation — the honest "am I using the GPU" metric + # for AI/LLM workloads. Unlike NVML, idle tensor cores are visible. + - record: pod:tensor_saturation:avg5m + expr: | + avg by (Hostname, gpu, UUID, namespace, pod) ( + avg_over_time(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]) + ) * 100 + + # NVML vs tensor gap — ratio <10% means NVML lies: user thinks GPU + # is busy but specialized hardware is idle (cheap tenant signal). + - record: pod:tensor_to_nvml_ratio:avg5m + expr: | + ( + avg_over_time(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]) * 100 + ) + / + clamp_min( + avg_over_time(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]), + 1 + ) + + # Power efficiency — utilization per watt, reveals unoptimized clients. + - record: pod:util_per_watt:avg5m + expr: | + avg_over_time(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]) + / + clamp_min( + avg_over_time(DCGM_FI_DEV_POWER_USAGE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]), + 1 + ) + + # Fraction of time power-throttled (TDP cap) — 1.0 = fully throttled. + # DCGM_FI_DEV_*_VIOLATION is documented as µs but on A10/DCGM 3.x the + # counter grows in nanoseconds in practice — divide by 1e9 to get a + # 0..1 fraction (verified empirically when /1e6 yielded >100× reality). + # clamp_max protects against rate() artefacts at counter resets. + - record: gpu:power_throttle_fraction:rate5m + expr: clamp_max(rate(DCGM_FI_DEV_POWER_VIOLATION[5m]) / 1e9, 1) + + # Fraction of time thermal-throttled. + - record: gpu:thermal_throttle_fraction:rate5m + expr: clamp_max(rate(DCGM_FI_DEV_THERMAL_VIOLATION[5m]) / 1e9, 1) From 0e20159bd999795fd92f9b2099529bbe9b4de040 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 06:09:44 +0300 Subject: [PATCH 103/250] fix(gpu-operator): scope compat DaemonSet to GPU nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restrict the nvidia-driver-compat DaemonSet to nodes labelled nvidia.com/gpu.present=true (NFD/GPU Operator label). Without the nodeSelector it was scheduling onto every node — control-plane and CPU-only workers included — burning a privileged pod slot per host for no benefit. Add resource requests and limits to the init and pause containers so the DaemonSet stays within control-plane budget on small clusters. Signed-off-by: Arsolitt --- .../examples/nvidia-driver-compat.yaml | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml b/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml index e4718e0a..63e07c4f 100644 --- a/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml +++ b/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml @@ -27,6 +27,14 @@ spec: spec: hostPID: true priorityClassName: system-node-critical + # Restrict to GPU nodes only. Without this the DaemonSet schedules onto + # every node (including control-plane and CPU-only workers) and burns + # resources on hosts where the compat tree is meaningless. + # The label is published by Node Feature Discovery / GPU Operator's NFD + # subchart; if NFD is disabled, replace this with whatever label your + # cluster uses to mark GPU hosts. + nodeSelector: + nvidia.com/gpu.present: "true" tolerations: - operator: Exists initContainers: @@ -60,12 +68,26 @@ spec: echo "Done" securityContext: privileged: true + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 100m + memory: 64Mi volumeMounts: - name: host-root mountPath: /host containers: - name: pause image: registry.k8s.io/pause:3.10 + resources: + requests: + cpu: 10m + memory: 8Mi + limits: + cpu: 50m + memory: 16Mi volumes: - name: host-root hostPath: From 38c8a37cb585ff235c7d90d0824a457e7da93095 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 06:15:58 +0300 Subject: [PATCH 104/250] docs(gpu-operator): clarify minimum required DCGM metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous wording implied that the entire custom DCGM CSV was required by the recording rules. In fact only the profiling counters (DCGM_FI_PROF_*) need to be added on top of the upstream defaults — everything else the rules consume is already in default-counters.csv. Add a Verification status block flagging that the minimum-set claim is derived from the DCGM Exporter version pinned in the currently shipped gpu-operator package and must be re-checked when that package moves to a newer release. Signed-off-by: Arsolitt --- .../system/gpu-operator/examples/README.md | 30 +++++++++++++++++-- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/packages/system/gpu-operator/examples/README.md b/packages/system/gpu-operator/examples/README.md index dac3703e..11c2bb09 100644 --- a/packages/system/gpu-operator/examples/README.md +++ b/packages/system/gpu-operator/examples/README.md @@ -21,9 +21,16 @@ Talos. ConfigMap. - [`dcgm-custom-metrics.yaml`](./dcgm-custom-metrics.yaml) — `ConfigMap` with a DCGM metrics CSV that adds profiling, ECC, throttling and - energy counters on top of the upstream defaults. Required by the - recording rules in `packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml` - and by several panels in the `gpu/gpu-performance` dashboard. + energy counters on top of the upstream defaults. The CSV is the + superset needed for full dashboard coverage; the **recording rules + themselves** only require the profiling subset + (`DCGM_FI_PROF_PIPE_TENSOR_ACTIVE`, `DCGM_FI_PROF_GR_ENGINE_ACTIVE`) + on top of the upstream `default-counters.csv` — every other DCGM + series the rules consume (utilization, FB used/free, power, + temperature, energy) is already in the default set. The + `gpu/gpu-performance` dashboard additionally needs the throttle + counters (`DCGM_FI_DEV_POWER_VIOLATION`, + `DCGM_FI_DEV_THERMAL_VIOLATION`), which are not in the default set. - [`nvidia-driver-compat.yaml`](./nvidia-driver-compat.yaml) — DaemonSet that stages `libnvidia-ml.so.1` and `nvidia-smi` from the Talos glibc tree into a path where the NVIDIA GPU Operator validator expects @@ -55,6 +62,23 @@ files into a directory the validator does inspect and creates the [1]: https://github.com/NVIDIA/gpu-operator/issues/1687 +## Verification status + +> **Pending verification on an updated GPU Operator release.** +> +> The minimum-CSV claim above (only `DCGM_FI_PROF_*` is needed beyond +> the upstream default counters) is derived by cross-referencing +> `gpu-recording.rules.yaml` against the DCGM Exporter +> [`default-counters.csv`][default-csv] for the version pinned in the +> currently shipped `gpu-operator` package. The package in this branch +> is **not** the latest GPU Operator release; once we move to a newer +> version, the claim must be re-checked because the upstream default +> set occasionally adds or removes counters between releases. Until +> then, treat the CSV in `dcgm-custom-metrics.yaml` as a known-good +> superset rather than a minimal config. + +[default-csv]: https://github.com/NVIDIA/dcgm-exporter/blob/main/etc/default-counters.csv + ## How the dashboard and recording rules fit in - `dashboards/gpu/gpu-performance.json` expects `DCGM_FI_*` metrics, From 49c1d7e7abeddebf6047a52b6de7e13f53c60ba5 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 06:22:31 +0300 Subject: [PATCH 105/250] test(monitoring): cross-check GPU dashboards against recording rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Catch dangling references at PR time: every recording-rule name used inside a tracked GPU dashboard must exist in packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml. The first iteration of gpu-efficiency.json shipped panels keyed on pod:tensor_saturation:avg5m without the rule defined; the test fails on exactly that class of bug. Scoped to dashboards listed under gpu/* in dashboards-infra.list, so untracked drafts stay out of scope until they are registered. Reverse direction (rule defined but unused) is intentionally NOT enforced — some rules exist for ad-hoc PromQL or upcoming dashboards. Auto-discovered by make bats-unit-tests via hack/cozytest.sh. Signed-off-by: Arsolitt --- hack/check-gpu-recording-rules.bats | 114 ++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 hack/check-gpu-recording-rules.bats diff --git a/hack/check-gpu-recording-rules.bats b/hack/check-gpu-recording-rules.bats new file mode 100644 index 00000000..87e7c8f1 --- /dev/null +++ b/hack/check-gpu-recording-rules.bats @@ -0,0 +1,114 @@ +#!/usr/bin/env bats +# ----------------------------------------------------------------------------- +# Cross-validation between GPU recording rules and the dashboards that consume +# them. Catches: +# +# 1. dangling references — a dashboard query mentions a recording rule name +# that doesn't exist in gpu-recording.rules.yaml. This is the bug the +# pre-merge review caught: gpu-efficiency.json shipped panels keyed on +# pod:tensor_saturation:avg5m without the rule being defined, so the +# panel showed "No data" everywhere. +# +# 2. typos in rule names — same bug class, manifested as a single-character +# difference between rule and reference. +# +# Scope: only dashboards listed in packages/system/monitoring/dashboards-infra.list +# under the "gpu/" prefix (i.e. shipped to production), not every JSON file in +# dashboards/gpu/. Untracked drafts stay out of scope on purpose — adding one +# to dashboards-infra.list is what brings it under the test. +# +# Reverse direction (rule defined but never consumed) is intentionally NOT +# enforced: some rules exist for ad-hoc PromQL or upcoming dashboards. Treat +# unused rules as an editorial concern, not a regression. +# +# Title syntax constraints from cozytest.sh's awk parser: +# - Titles delimited by ASCII double quotes; embedded quotes truncate. +# - Only [A-Za-z0-9] from the title survives into the function name; titles +# differing only in punctuation collapse to the same function. +# +# Run with: hack/cozytest.sh hack/check-gpu-recording-rules.bats +# ----------------------------------------------------------------------------- + +REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" +RULES_FILE="$REPO_ROOT/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml" +DASHBOARDS_LIST="$REPO_ROOT/packages/system/monitoring/dashboards-infra.list" +DASHBOARDS_DIR="$REPO_ROOT/dashboards" + +# Extract the set of "- record: NAME" entries from the rules YAML. +# Outputs one rule name per line, sorted and deduplicated. +extract_rules() { + awk '/^[[:space:]]*-[[:space:]]*record:[[:space:]]/ { + sub(/^[[:space:]]*-[[:space:]]*record:[[:space:]]*/, "") + sub(/[[:space:]]*$/, "") + print + }' "$RULES_FILE" | sort -u +} + +# Extract the set of recording-rule references from a dashboard JSON. +# A recording-rule reference is matched by the pattern +# :(:)+ +# where each is [a-z0-9_]. Raw DCGM metrics (DCGM_FI_*), +# kube-state-metrics (kube_*) and similar uppercase / single-word metric +# names do not match because the leading segment must be lowercase and the +# whole expression must contain at least two ':' characters. +extract_refs() { + json_file=$1 + grep -hoE '[a-z][a-z0-9_]*:[a-z0-9_]+:[a-z0-9_]+' "$json_file" | sort -u +} + +# Resolve "gpu/foo" -> "$DASHBOARDS_DIR/gpu/foo.json" +list_tracked_gpu_dashboards() { + awk '/^gpu\// { print $0 ".json" }' "$DASHBOARDS_LIST" +} + +@test "every recording rule reference in tracked GPU dashboards has a matching record" { + TMP=$(mktemp -d) + trap 'rm -rf "$TMP"' EXIT + + extract_rules > "$TMP/rules.txt" + [ -s "$TMP/rules.txt" ] || { echo "no recording rules extracted from $RULES_FILE" >&2; exit 1; } + + list_tracked_gpu_dashboards > "$TMP/dashboards.txt" + [ -s "$TMP/dashboards.txt" ] || { echo "no gpu/* dashboards listed in $DASHBOARDS_LIST" >&2; exit 1; } + + failed=0 + while IFS= read -r dashboard_rel; do + dashboard="$DASHBOARDS_DIR/$dashboard_rel" + if [ ! -f "$dashboard" ]; then + echo "ERROR: dashboard listed but file missing: $dashboard" >&2 + failed=1 + continue + fi + + extract_refs "$dashboard" > "$TMP/refs.txt" + # comm -23: lines unique to refs.txt (referenced but not defined) + # Both inputs must be sorted; extract_* helpers already sort. + comm -23 "$TMP/refs.txt" "$TMP/rules.txt" > "$TMP/missing.txt" + if [ -s "$TMP/missing.txt" ]; then + echo "ERROR: $dashboard_rel references undefined recording rules:" >&2 + sed 's/^/ - /' "$TMP/missing.txt" >&2 + failed=1 + fi + done < "$TMP/dashboards.txt" + + [ "$failed" -eq 0 ] +} + +@test "every tracked GPU dashboard listed in dashboards-infra.list exists on disk" { + TMP=$(mktemp -d) + trap 'rm -rf "$TMP"' EXIT + + list_tracked_gpu_dashboards > "$TMP/dashboards.txt" + [ -s "$TMP/dashboards.txt" ] || { echo "no gpu/* dashboards listed in $DASHBOARDS_LIST" >&2; exit 1; } + + failed=0 + while IFS= read -r dashboard_rel; do + dashboard="$DASHBOARDS_DIR/$dashboard_rel" + if [ ! -f "$dashboard" ]; then + echo "ERROR: $dashboard_rel listed in dashboards-infra.list but $dashboard does not exist" >&2 + failed=1 + fi + done < "$TMP/dashboards.txt" + + [ "$failed" -eq 0 ] +} From dbed4992b08392c28bac8f0b5f917619b88cec65 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 06:28:49 +0300 Subject: [PATCH 106/250] fix(monitoring): exclude system namespaces from namespace:gpu_count:allocated MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Align namespace:gpu_count:allocated with every other namespace:* rule by filtering out cozy-*/kube-*. All other per-namespace rules (gpu_util, tensor_active, fb_used_bytes, power_watts) already exclude system namespaces, so the label set produced by :allocated diverged from them — any dashboard variable or join that reads across these rules could end up with a different namespace list depending on which rule supplied the :allocated column. Trade-off: per-namespace GPU accounting for system workloads is no longer available through this rule. If it's ever needed, add a dedicated system:gpu_count:allocated rather than widening this one — the "billable tenant view" invariant is what the filter is protecting. Cluster-level cluster:gpu_count:allocated intentionally keeps system pods so it stays aligned with cluster:gpu_count:total and cluster:gpu_count:free remains meaningful. As a consequence, sum(namespace:gpu_count:allocated) no longer equals cluster:gpu_count:allocated; the delta is system-pod GPU usage, which is fine for the cluster-admin view. Signed-off-by: Arsolitt --- .../alerts/gpu-recording.rules.yaml | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml index a54860b7..3a31f5a2 100644 --- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml +++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml @@ -33,8 +33,27 @@ spec: # Kube-requested GPU count per namespace — billable view, includes # Pending pods. Use this for GPU-hour reporting via # sum_over_time(...[1h:1m])/60. + # + # Filters cozy-*/kube-* so the namespace set here matches every + # other namespace:* rule below. This keeps the $namespace variable + # in the tenant dashboard consistent with panels that read filtered + # rules — otherwise picking a system ns from the dropdown leaves + # half the panels empty while others show values. + # + # Trade-off: per-namespace GPU accounting for system workloads is + # no longer available here. If it's ever needed (e.g. operator + # pods pinning tenant GPUs), add a separate rule like + # system:gpu_count:allocated rather than widening this one — the + # "billable tenant view" invariant is relied on by tenants.json + # and keeping it narrow protects consumers from the asymmetry + # that triggered this filter in the first place. + # + # Note: sum(namespace:gpu_count:allocated) no longer equals + # cluster:gpu_count:allocated — the cluster-level rule intentionally + # keeps system pods to stay aligned with cluster:gpu_count:total + # when computing :free. - record: namespace:gpu_count:allocated - expr: sum by (namespace) (kube_pod_container_resource_requests{resource="nvidia_com_gpu"}) + expr: sum by (namespace) (kube_pod_container_resource_requests{resource="nvidia_com_gpu", namespace!="", namespace!~"cozy-.*|kube-.*"}) - record: namespace:gpu_util:avg expr: avg by (namespace) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}) - record: namespace:tensor_active:avg From 6d9066f0748744d78f823e3137f72b5e912069e9 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 06:35:12 +0300 Subject: [PATCH 107/250] feat(monitoring): add GPU fleet and tenants dashboards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - gpu-fleet: cluster-wide admin view — inventory, capacity (total / allocated / free), per-node utilization and power, throttling, temperatures, XID errors. - gpu-tenants: per-namespace view — live allocation, utilization, tensor saturation, power, and 24h GPU-hours / kWh integrations for billing inputs. Register both under gpu/* in dashboards-infra.list so they ship as GrafanaDashboard CRs and fall under the bats cross-check introduced earlier on this branch. Update examples/README to spell out which DCGM counters each of the five gpu/* dashboards actually needs on top of the upstream default CSV — gpu-performance needs profiling and throttling counters, gpu-efficiency needs profiling, gpu-tenants needs only DCGM_FI_PROF_PIPE_TENSOR_ACTIVE for its tensor panel, and gpu-fleet and gpu-quotas work on the default counter set alone. Signed-off-by: Arsolitt --- dashboards/gpu/gpu-fleet.json | 1157 +++++++++++++++ dashboards/gpu/gpu-tenants.json | 1236 +++++++++++++++++ .../system/gpu-operator/examples/README.md | 74 +- .../system/monitoring/dashboards-infra.list | 2 + 4 files changed, 2436 insertions(+), 33 deletions(-) create mode 100644 dashboards/gpu/gpu-fleet.json create mode 100644 dashboards/gpu/gpu-tenants.json diff --git a/dashboards/gpu/gpu-fleet.json b/dashboards/gpu/gpu-fleet.json new file mode 100644 index 00000000..af5ad2fa --- /dev/null +++ b/dashboards/gpu/gpu-fleet.json @@ -0,0 +1,1157 @@ +{ + "uid": "gpu-fleet", + "title": "GPU Fleet Overview", + "description": "Cluster-wide GPU inventory, capacity, utilization and health — admin view across the whole fleet.", + "tags": [ + "gpu", + "fleet", + "admin" + ], + "timezone": "browser", + "editable": true, + "graphTooltip": 1, + "time": { + "from": "now-6h", + "to": "now" + }, + "fiscalYearStartMonth": 0, + "schemaVersion": 42, + "panels": [ + { + "type": "row", + "collapsed": false, + "title": "Cluster snapshot", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [] + }, + { + "type": "stat", + "id": 2, + "targets": [ + { + "expr": "cluster:gpu_count:total", + "refId": "A" + } + ], + "title": "Total GPUs", + "description": "Physical GPUs discovered by DCGM across all nodes.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "repeatDirection": "h", + "options": { + "graphMode": "none", + "colorMode": "value", + "justifyMode": "auto", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "percentChangeColorMode": "standard", + "orientation": "" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "blue" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "id": 3, + "targets": [ + { + "expr": "cluster:gpu_count:allocated", + "refId": "A" + } + ], + "title": "Allocated", + "description": "Sum of kube GPU requests across all pods. Billable view — includes Pending pods.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "repeatDirection": "h", + "options": { + "graphMode": "none", + "colorMode": "background", + "justifyMode": "auto", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "percentChangeColorMode": "standard", + "orientation": "" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "blue" + }, + { + "value": 1, + "color": "green" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "id": 4, + "targets": [ + { + "expr": "cluster:gpu_count:free", + "refId": "A" + } + ], + "title": "Free", + "description": "Unallocated capacity available for new tenants.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, + "repeatDirection": "h", + "options": { + "graphMode": "none", + "colorMode": "background", + "justifyMode": "auto", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "percentChangeColorMode": "standard", + "orientation": "" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "red" + }, + { + "value": 1, + "color": "yellow" + }, + { + "value": 2, + "color": "green" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "id": 5, + "targets": [ + { + "expr": "count(kube_node_labels{label_nvidia_com_gpu_present=\"true\"})", + "refId": "A" + } + ], + "title": "Nodes with GPU", + "description": "Kubernetes nodes advertising nvidia.com/gpu.present=true.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 1 + }, + "repeatDirection": "h", + "options": { + "graphMode": "none", + "colorMode": "value", + "justifyMode": "auto", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "percentChangeColorMode": "standard", + "orientation": "" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "blue" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "id": 6, + "targets": [ + { + "expr": "cluster:gpu_util:avg", + "refId": "A" + } + ], + "title": "Average utilization", + "description": "NVML utilization across all tenant GPUs (engine-active %).", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 + }, + "repeatDirection": "h", + "options": { + "graphMode": "area", + "colorMode": "value", + "justifyMode": "auto", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "percentChangeColorMode": "standard", + "orientation": "" + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "blue" + }, + { + "value": 30, + "color": "green" + }, + { + "value": 80, + "color": "orange" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "id": 7, + "targets": [ + { + "expr": "cluster:gpu_power_watts:sum", + "refId": "A" + } + ], + "title": "Total power", + "description": "Live power draw summed across the fleet.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 1 + }, + "repeatDirection": "h", + "options": { + "graphMode": "area", + "colorMode": "value", + "justifyMode": "auto", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "percentChangeColorMode": "standard", + "orientation": "" + }, + "fieldConfig": { + "defaults": { + "unit": "watt", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "row", + "collapsed": false, + "title": "Per-node GPU activity", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 10, + "panels": [] + }, + { + "type": "bargauge", + "id": 11, + "targets": [ + { + "expr": "avg by (Hostname) (DCGM_FI_DEV_GPU_UTIL)", + "legendFormat": "{{Hostname}}", + "refId": "A" + } + ], + "title": "GPU utilization per node", + "description": "Average NVML utilization per node — spot idle or saturated hosts at a glance.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "repeatDirection": "h", + "options": { + "displayMode": "gradient", + "valueMode": "color", + "namePlacement": "auto", + "showUnfilled": true, + "sizing": "auto", + "minVizWidth": 8, + "minVizHeight": 16, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false, + "calcs": [] + }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "maxVizHeight": 300, + "orientation": "horizontal" + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "blue" + }, + { + "value": 30, + "color": "green" + }, + { + "value": 80, + "color": "orange" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "bargauge", + "id": 12, + "targets": [ + { + "expr": "sum by (Hostname) (DCGM_FI_DEV_POWER_USAGE)", + "legendFormat": "{{Hostname}}", + "refId": "A" + } + ], + "title": "Power draw per node", + "description": "Sum of GPU power usage per node in watts.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "repeatDirection": "h", + "options": { + "displayMode": "gradient", + "valueMode": "color", + "namePlacement": "auto", + "showUnfilled": true, + "sizing": "auto", + "minVizWidth": 8, + "minVizHeight": 16, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false, + "calcs": [] + }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "maxVizHeight": 300, + "orientation": "horizontal" + }, + "fieldConfig": { + "defaults": { + "unit": "watt", + "min": 0, + "max": 200, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 120, + "color": "yellow" + }, + { + "value": 160, + "color": "orange" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "row", + "collapsed": false, + "title": "Usage trends", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 20, + "panels": [] + }, + { + "type": "timeseries", + "id": 21, + "targets": [ + { + "expr": "avg by (Hostname) (DCGM_FI_DEV_GPU_UTIL)", + "legendFormat": "{{Hostname}}", + "refId": "A" + } + ], + "title": "GPU utilization per node (trend)", + "description": "Per-node NVML utilization over time — shows shifting workload across hosts.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "repeatDirection": "h", + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "" + } + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 20, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "id": 22, + "targets": [ + { + "expr": "sum by (Hostname) (DCGM_FI_DEV_POWER_USAGE)", + "legendFormat": "{{Hostname}}", + "refId": "A" + } + ], + "title": "Power draw per node (trend)", + "description": "Per-node GPU power consumption over time.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "repeatDirection": "h", + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "" + } + }, + "fieldConfig": { + "defaults": { + "unit": "watt", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 20, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "row", + "collapsed": false, + "title": "Health", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 30, + "panels": [] + }, + { + "type": "stat", + "id": 31, + "targets": [ + { + "expr": "count((max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS)) \u003e 0) or vector(0)", + "refId": "A" + } + ], + "title": "GPUs with XID errors", + "description": "Count of GPUs reporting a non-zero XID code. XID = NVIDIA hardware/driver error — any non-zero value needs investigation.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 24 + }, + "repeatDirection": "h", + "options": { + "graphMode": "none", + "colorMode": "background", + "justifyMode": "auto", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "percentChangeColorMode": "standard", + "orientation": "" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 1, + "color": "red" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "id": 32, + "targets": [ + { + "expr": "avg(gpu:power_throttle_fraction:rate5m)", + "refId": "A" + } + ], + "title": "Power throttling (cluster avg)", + "description": "Fraction of time the fleet spent power-throttled. \u003e5% = billed FLOPS are being underdelivered.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 24 + }, + "repeatDirection": "h", + "options": { + "graphMode": "none", + "colorMode": "background", + "justifyMode": "auto", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "percentChangeColorMode": "standard", + "orientation": "" + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 0.05, + "color": "yellow" + }, + { + "value": 0.2, + "color": "red" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "id": 33, + "targets": [ + { + "expr": "avg(gpu:thermal_throttle_fraction:rate5m)", + "refId": "A" + } + ], + "title": "Thermal throttling (cluster avg)", + "description": "Fraction of time the fleet spent thermal-throttled. Indicates cooling/datacenter issues.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 24 + }, + "repeatDirection": "h", + "options": { + "graphMode": "none", + "colorMode": "background", + "justifyMode": "auto", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "percentChangeColorMode": "standard", + "orientation": "" + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 0.05, + "color": "yellow" + }, + { + "value": 0.2, + "color": "red" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "id": 34, + "targets": [ + { + "expr": "max(DCGM_FI_DEV_GPU_TEMP)", + "refId": "A" + } + ], + "title": "Max GPU temperature", + "description": "Hottest GPU in the fleet right now. A10 sustained target \u003c83°C.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 24 + }, + "repeatDirection": "h", + "options": { + "graphMode": "none", + "colorMode": "background", + "justifyMode": "auto", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "percentChangeColorMode": "standard", + "orientation": "" + }, + "fieldConfig": { + "defaults": { + "unit": "celsius", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 75, + "color": "yellow" + }, + { + "value": 85, + "color": "red" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "id": 35, + "targets": [ + { + "expr": "DCGM_FI_DEV_GPU_TEMP", + "legendFormat": "{{Hostname}}/{{gpu}}", + "refId": "A" + } + ], + "title": "Temperature per GPU", + "description": "Per-GPU temperature trace. Sustained red zone = cooling issue or aggressive workload.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 29 + }, + "repeatDirection": "h", + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "" + } + }, + "fieldConfig": { + "defaults": { + "unit": "celsius", + "min": 20, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 75, + "color": "yellow" + }, + { + "value": 85, + "color": "red" + } + ] + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "id": 36, + "targets": [ + { + "expr": "gpu:power_throttle_fraction:rate5m", + "legendFormat": "power {{Hostname}}/{{gpu}}", + "refId": "A" + }, + { + "expr": "gpu:thermal_throttle_fraction:rate5m", + "legendFormat": "thermal {{Hostname}}/{{gpu}}", + "refId": "B" + } + ], + "title": "Throttle fraction per GPU", + "description": "Red = power throttle, blue = thermal throttle. Both on same chart to compare causes.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 29 + }, + "repeatDirection": "h", + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "" + } + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 25, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "row", + "collapsed": false, + "title": "Hardware inventory", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 40, + "panels": [] + }, + { + "type": "table", + "id": 41, + "targets": [ + { + "expr": "group by (Hostname, modelName, gpu, UUID) (DCGM_FI_DEV_GPU_UTIL)", + "instant": true, + "range": false, + "format": "table", + "refId": "A" + } + ], + "title": "GPU inventory", + "description": "One row per physical GPU. Model / index / UUID sourced from DCGM labels.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 38 + }, + "repeatDirection": "h", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "__name__": true, + "cluster": true, + "container": true, + "device": true, + "endpoint": true, + "instance": true, + "job": true, + "pci_bus_id": true, + "prometheus": true, + "service": true, + "tenant": true, + "tier": true, + "uid": true, + "unit": true + }, + "indexByName": { + "Hostname": 0, + "UUID": 3, + "gpu": 2, + "modelName": 1 + }, + "renameByName": { + "Hostname": "Node", + "UUID": "UUID", + "gpu": "Index", + "modelName": "GPU Model" + } + } + } + ], + "options": { + "frameIndex": 0, + "showHeader": true, + "showTypeIcons": false, + "footer": { + "show": false, + "reducer": [] + }, + "cellHeight": "sm" + } + } + ], + "templating": { + "list": [ + { + "type": "datasource", + "name": "ds_prometheus", + "label": "Prometheus", + "skipUrlSync": false, + "query": "prometheus", + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "multi": false, + "allowCustomValue": true, + "includeAll": false, + "regex": "", + "auto": false, + "auto_min": "10s", + "auto_count": 30 + } + ] + }, + "annotations": {} +} diff --git a/dashboards/gpu/gpu-tenants.json b/dashboards/gpu/gpu-tenants.json new file mode 100644 index 00000000..d0265c0d --- /dev/null +++ b/dashboards/gpu/gpu-tenants.json @@ -0,0 +1,1236 @@ +{ + "uid": "gpu-tenants", + "title": "GPU Tenant Usage", + "description": "Per-tenant GPU consumption: live allocation, utilization, tensor saturation, power, and 24h GPU-hours / kWh — admin view across all namespaces.", + "tags": [ + "gpu", + "tenants", + "admin" + ], + "timezone": "browser", + "editable": true, + "graphTooltip": 1, + "time": { + "from": "now-24h", + "to": "now" + }, + "fiscalYearStartMonth": 0, + "schemaVersion": 42, + "panels": [ + { + "type": "row", + "collapsed": false, + "title": "Tenant fleet snapshot", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [] + }, + { + "type": "stat", + "id": 2, + "targets": [ + { + "expr": "count(namespace:gpu_count:allocated{namespace=~\"$namespace\"})", + "refId": "A" + } + ], + "title": "Active tenants", + "description": "Namespaces with at least one GPU request (Pending + Running).", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "repeatDirection": "h", + "options": { + "graphMode": "none", + "colorMode": "value", + "justifyMode": "auto", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "percentChangeColorMode": "standard", + "orientation": "" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "blue" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "id": 3, + "targets": [ + { + "expr": "sum(namespace:gpu_count:allocated{namespace=~\"$namespace\"})", + "refId": "A" + } + ], + "title": "Allocated GPUs", + "description": "Sum of kube GPU requests across tenants. Billable view — includes Pending pods.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "repeatDirection": "h", + "options": { + "graphMode": "none", + "colorMode": "background", + "justifyMode": "auto", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "percentChangeColorMode": "standard", + "orientation": "" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "blue" + }, + { + "value": 1, + "color": "green" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "id": 4, + "targets": [ + { + "expr": "avg(namespace:gpu_util:avg{namespace=~\"$namespace\"})", + "refId": "A" + } + ], + "title": "Avg tenant util", + "description": "Average NVML utilization across all tenant GPUs.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, + "repeatDirection": "h", + "options": { + "graphMode": "area", + "colorMode": "value", + "justifyMode": "auto", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "percentChangeColorMode": "standard", + "orientation": "" + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "blue" + }, + { + "value": 30, + "color": "green" + }, + { + "value": 80, + "color": "orange" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "id": 5, + "targets": [ + { + "expr": "sum(namespace:power_watts:sum{namespace=~\"$namespace\"})", + "refId": "A" + } + ], + "title": "Total tenant power", + "description": "Live power draw summed across tenant workloads.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 1 + }, + "repeatDirection": "h", + "options": { + "graphMode": "area", + "colorMode": "value", + "justifyMode": "auto", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "percentChangeColorMode": "standard", + "orientation": "" + }, + "fieldConfig": { + "defaults": { + "unit": "watt", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "id": 6, + "targets": [ + { + "expr": "sum(sum_over_time(namespace:power_watts:sum{namespace=~\"$namespace\"}[24h:1m])) / 60 / 1000", + "refId": "A" + } + ], + "title": "Energy (last 24h)", + "description": "Integrated power draw over 24h. Feed into billing/chargeback as kWh.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 + }, + "repeatDirection": "h", + "options": { + "graphMode": "none", + "colorMode": "value", + "justifyMode": "auto", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "percentChangeColorMode": "standard", + "orientation": "" + }, + "fieldConfig": { + "defaults": { + "unit": "kwatth", + "decimals": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "id": 7, + "targets": [ + { + "expr": "sum(sum_over_time(namespace:gpu_count:allocated{namespace=~\"$namespace\"}[24h:1m])) / 60", + "refId": "A" + } + ], + "title": "GPU-hours (last 24h)", + "description": "Sum of (GPU count × time requested) across all tenants over the last 24h. Integrated from kube requests, so Pending pods count.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 1 + }, + "repeatDirection": "h", + "options": { + "graphMode": "none", + "colorMode": "value", + "justifyMode": "auto", + "textMode": "value", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "percentChangeColorMode": "standard", + "orientation": "" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "blue" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "row", + "collapsed": false, + "title": "Current allocation", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 10, + "panels": [] + }, + { + "type": "bargauge", + "id": 11, + "targets": [ + { + "expr": "namespace:gpu_count:allocated{namespace=~\"$namespace\"}", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "title": "GPUs per namespace", + "description": "Kube GPU requests per tenant (billable). Includes Pending pods.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "repeatDirection": "h", + "options": { + "displayMode": "gradient", + "valueMode": "color", + "namePlacement": "auto", + "showUnfilled": true, + "sizing": "auto", + "minVizWidth": 8, + "minVizHeight": 16, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false, + "calcs": [] + }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "maxVizHeight": 300, + "orientation": "horizontal" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "bargauge", + "id": 12, + "targets": [ + { + "expr": "namespace:fb_used_bytes:sum{namespace=~\"$namespace\"}", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "title": "VRAM used per namespace", + "description": "FB memory actively used by tenant workloads.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "repeatDirection": "h", + "options": { + "displayMode": "gradient", + "valueMode": "color", + "namePlacement": "auto", + "showUnfilled": true, + "sizing": "auto", + "minVizWidth": 8, + "minVizHeight": 16, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false, + "calcs": [] + }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "maxVizHeight": 300, + "orientation": "horizontal" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "row", + "collapsed": false, + "title": "Utilization", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 20, + "panels": [] + }, + { + "type": "timeseries", + "id": 21, + "targets": [ + { + "expr": "namespace:gpu_util:avg{namespace=~\"$namespace\"}", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "title": "NVML utilization per namespace", + "description": "Per-tenant NVML utilization. Stacked to show fleet-wide pressure over time.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "repeatDirection": "h", + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "" + } + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 40, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "id": 22, + "targets": [ + { + "expr": "namespace:tensor_active:avg{namespace=~\"$namespace\"} * 100", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "title": "Tensor saturation per namespace", + "description": "Real tensor core load per tenant. Useful to find AI tenants that claim GPUs but leave tensor cores idle.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "repeatDirection": "h", + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "" + } + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 40, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + } + }, + { + "type": "row", + "collapsed": false, + "title": "Power \u0026 allocation trend", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 30, + "panels": [] + }, + { + "type": "timeseries", + "id": 31, + "targets": [ + { + "expr": "namespace:power_watts:sum{namespace=~\"$namespace\"}", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "title": "Power draw per namespace", + "description": "Per-tenant power draw. Stacked to see cluster-wide energy burn over time.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "repeatDirection": "h", + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "" + } + }, + "fieldConfig": { + "defaults": { + "unit": "watt", + "min": 0, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 40, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "id": 32, + "targets": [ + { + "expr": "namespace:gpu_count:allocated{namespace=~\"$namespace\"}", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "title": "Allocated GPUs per namespace", + "description": "Kube-requested GPU count per tenant over time. Step-after interpolation reflects discrete allocation changes.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "repeatDirection": "h", + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "custom": { + "drawStyle": "line", + "lineInterpolation": "stepAfter", + "fillOpacity": 40, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + } + }, + { + "type": "row", + "collapsed": false, + "title": "Top consumers (live)", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 40, + "panels": [] + }, + { + "type": "table", + "id": 41, + "targets": [ + { + "expr": "namespace:gpu_count:allocated{namespace=~\"$namespace\"}", + "instant": true, + "range": false, + "format": "table", + "refId": "gpus" + }, + { + "expr": "namespace:gpu_util:avg{namespace=~\"$namespace\"}", + "instant": true, + "range": false, + "format": "table", + "refId": "util" + }, + { + "expr": "namespace:tensor_active:avg{namespace=~\"$namespace\"} * 100", + "instant": true, + "range": false, + "format": "table", + "refId": "tensor" + }, + { + "expr": "namespace:power_watts:sum{namespace=~\"$namespace\"}", + "instant": true, + "range": false, + "format": "table", + "refId": "power" + }, + { + "expr": "namespace:fb_used_bytes:sum{namespace=~\"$namespace\"}", + "instant": true, + "range": false, + "format": "table", + "refId": "vram" + } + ], + "title": "Top consumers", + "description": "Live ranking of tenants by power draw. Columns joined from namespace recording rules.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 33 + }, + "repeatDirection": "h", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "namespace", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Time 5": true, + "__name__": true, + "__name__ 1": true, + "__name__ 2": true, + "__name__ 3": true, + "__name__ 4": true, + "__name__ 5": true, + "cluster": true, + "cluster 1": true, + "cluster 2": true, + "cluster 3": true, + "cluster 4": true, + "cluster 5": true, + "endpoint": true, + "endpoint 1": true, + "endpoint 2": true, + "endpoint 3": true, + "endpoint 4": true, + "endpoint 5": true, + "instance": true, + "instance 1": true, + "instance 2": true, + "instance 3": true, + "instance 4": true, + "instance 5": true, + "job": true, + "job 1": true, + "job 2": true, + "job 3": true, + "job 4": true, + "job 5": true, + "prometheus": true, + "prometheus 1": true, + "prometheus 2": true, + "prometheus 3": true, + "prometheus 4": true, + "prometheus 5": true, + "service": true, + "service 1": true, + "service 2": true, + "service 3": true, + "service 4": true, + "service 5": true, + "tenant": true, + "tenant 1": true, + "tenant 2": true, + "tenant 3": true, + "tenant 4": true, + "tenant 5": true, + "tier": true, + "tier 1": true, + "tier 2": true, + "tier 3": true, + "tier 4": true, + "tier 5": true, + "uid": true, + "uid 1": true, + "uid 2": true, + "uid 3": true, + "uid 4": true, + "uid 5": true, + "unit": true, + "unit 1": true, + "unit 2": true, + "unit 3": true, + "unit 4": true, + "unit 5": true + }, + "indexByName": { + "Value #gpus": 1, + "Value #power": 4, + "Value #tensor": 3, + "Value #util": 2, + "Value #vram": 5, + "namespace": 0 + }, + "renameByName": { + "Value #gpus": "GPUs", + "Value #power": "Power", + "Value #tensor": "Tensor %", + "Value #util": "Util %", + "Value #vram": "VRAM", + "namespace": "Namespace" + } + } + } + ], + "options": { + "frameIndex": 0, + "showHeader": true, + "showTypeIcons": false, + "sortBy": [ + { + "displayName": "Power", + "desc": true + } + ], + "footer": { + "show": false, + "reducer": [] + }, + "cellHeight": "sm" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Util %" + }, + "properties": [ + { + "id": "unit", + "value": "percent" + }, + { + "id": "min", + "value": 0 + }, + { + "id": "max", + "value": 100 + }, + { + "id": "decimals", + "value": 1 + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Tensor %" + }, + "properties": [ + { + "id": "unit", + "value": "percent" + }, + { + "id": "min", + "value": 0 + }, + { + "id": "max", + "value": 100 + }, + { + "id": "decimals", + "value": 1 + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Power" + }, + "properties": [ + { + "id": "unit", + "value": "watt" + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VRAM" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + } + }, + { + "type": "row", + "collapsed": false, + "title": "Historical usage (last 24h)", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 43 + }, + "id": 50, + "panels": [] + }, + { + "type": "bargauge", + "id": 51, + "targets": [ + { + "expr": "sum_over_time(namespace:gpu_count:allocated{namespace=~\"$namespace\"}[24h:1m]) / 60", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "title": "GPU-hours per namespace (last 24h)", + "description": "Billable GPU-hours per tenant over the last 24h window.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 44 + }, + "repeatDirection": "h", + "options": { + "displayMode": "gradient", + "valueMode": "color", + "namePlacement": "auto", + "showUnfilled": true, + "sizing": "auto", + "minVizWidth": 8, + "minVizHeight": 16, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false, + "calcs": [] + }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "maxVizHeight": 300, + "orientation": "horizontal" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "blue" + } + ] + } + }, + "overrides": [] + } + }, + { + "type": "bargauge", + "id": 52, + "targets": [ + { + "expr": "sum_over_time(namespace:power_watts:sum{namespace=~\"$namespace\"}[24h:1m]) / 60 / 1000", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "title": "Energy per namespace (last 24h)", + "description": "kWh consumed per tenant over the last 24h. Integrated from 1-minute power samples.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 44 + }, + "repeatDirection": "h", + "options": { + "displayMode": "gradient", + "valueMode": "color", + "namePlacement": "auto", + "showUnfilled": true, + "sizing": "auto", + "minVizWidth": 8, + "minVizHeight": 16, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false, + "calcs": [] + }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "maxVizHeight": 300, + "orientation": "horizontal" + }, + "fieldConfig": { + "defaults": { + "unit": "kwatth", + "decimals": 2, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + } + ] + } + }, + "overrides": [] + } + } + ], + "templating": { + "list": [ + { + "type": "datasource", + "name": "ds_prometheus", + "label": "Prometheus", + "skipUrlSync": false, + "query": "prometheus", + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "multi": false, + "allowCustomValue": true, + "includeAll": false, + "regex": "", + "auto": false, + "auto_min": "10s", + "auto_count": 30 + }, + { + "type": "query", + "name": "namespace", + "label": "Namespace", + "skipUrlSync": false, + "query": "label_values(namespace:gpu_count:allocated, namespace)", + "datasource": { + "type": "prometheus", + "uid": "$ds_prometheus" + }, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "multi": true, + "allowCustomValue": true, + "refresh": 2, + "sort": 1, + "includeAll": true, + "auto": false, + "auto_min": "10s", + "auto_count": 30 + } + ] + }, + "annotations": {} +} diff --git a/packages/system/gpu-operator/examples/README.md b/packages/system/gpu-operator/examples/README.md index 11c2bb09..5374bffc 100644 --- a/packages/system/gpu-operator/examples/README.md +++ b/packages/system/gpu-operator/examples/README.md @@ -21,16 +21,10 @@ Talos. ConfigMap. - [`dcgm-custom-metrics.yaml`](./dcgm-custom-metrics.yaml) — `ConfigMap` with a DCGM metrics CSV that adds profiling, ECC, throttling and - energy counters on top of the upstream defaults. The CSV is the - superset needed for full dashboard coverage; the **recording rules - themselves** only require the profiling subset - (`DCGM_FI_PROF_PIPE_TENSOR_ACTIVE`, `DCGM_FI_PROF_GR_ENGINE_ACTIVE`) - on top of the upstream `default-counters.csv` — every other DCGM - series the rules consume (utilization, FB used/free, power, - temperature, energy) is already in the default set. The - `gpu/gpu-performance` dashboard additionally needs the throttle - counters (`DCGM_FI_DEV_POWER_VIOLATION`, - `DCGM_FI_DEV_THERMAL_VIOLATION`), which are not in the default set. + energy counters on top of the upstream defaults. The CSV is a + superset needed for full coverage of the `gpu/gpu-performance` + dashboard. Which parts are actually required depends on which + dashboards you ship — see the table below. - [`nvidia-driver-compat.yaml`](./nvidia-driver-compat.yaml) — DaemonSet that stages `libnvidia-ml.so.1` and `nvidia-smi` from the Talos glibc tree into a path where the NVIDIA GPU Operator validator expects @@ -62,32 +56,46 @@ files into a directory the validator does inspect and creates the [1]: https://github.com/NVIDIA/gpu-operator/issues/1687 +## Dashboards and what DCGM metrics they need + +Five GPU dashboards live under `gpu/*` in +`packages/system/monitoring/dashboards-infra.list`. All of them share +`packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml` as +their source of aggregated series. The recording rules are safe to +ship on any cluster — they evaluate to empty series when DCGM is not +scraped, or when optional counters are missing. + +What each dashboard needs on top of the upstream DCGM Exporter +[`default-counters.csv`][default-csv]: + +| Dashboard | Scope | Needs beyond defaults | +| ----------------- | ---------------------------------- | -------------------------------------------------------------------------------------------- | +| `gpu-performance` | Per-node, per-GPU deep dive | `DCGM_FI_PROF_PIPE_TENSOR_ACTIVE`, `DCGM_FI_PROF_GR_ENGINE_ACTIVE`, `DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION` | +| `gpu-efficiency` | Per-workload util vs tensor active | `DCGM_FI_PROF_PIPE_TENSOR_ACTIVE` | +| `gpu-fleet` | Cluster-wide admin inventory | nothing (works on default counters) | +| `gpu-quotas` | Kube-quota vs live usage | nothing (kube-state-metrics + default counters) | +| `gpu-tenants` | Per-namespace tenant view | `DCGM_FI_PROF_PIPE_TENSOR_ACTIVE` for the tensor-saturation panel; other panels work on defaults | + +The throttling counters (`DCGM_FI_DEV_POWER_VIOLATION`, +`DCGM_FI_DEV_THERMAL_VIOLATION`) are only required by `gpu-performance`. +The profiling counters (`DCGM_FI_PROF_*`) are required by +`gpu-performance` and `gpu-efficiency`, and unlock the tensor panel in +`gpu-tenants`. Everything else the recording rules consume — +utilization, FB used/free, power, temperature, energy — is already in +the default counter set. + ## Verification status > **Pending verification on an updated GPU Operator release.** > -> The minimum-CSV claim above (only `DCGM_FI_PROF_*` is needed beyond -> the upstream default counters) is derived by cross-referencing -> `gpu-recording.rules.yaml` against the DCGM Exporter -> [`default-counters.csv`][default-csv] for the version pinned in the -> currently shipped `gpu-operator` package. The package in this branch -> is **not** the latest GPU Operator release; once we move to a newer -> version, the claim must be re-checked because the upstream default -> set occasionally adds or removes counters between releases. Until -> then, treat the CSV in `dcgm-custom-metrics.yaml` as a known-good -> superset rather than a minimal config. +> The minimum-CSV claims above are derived by cross-referencing +> `gpu-recording.rules.yaml` and each dashboard against the DCGM +> Exporter [`default-counters.csv`][default-csv] for the version pinned +> in the currently shipped `gpu-operator` package. The package in this +> branch is **not** the latest GPU Operator release; once we move to a +> newer version, the claims must be re-checked because the upstream +> default set occasionally adds or removes counters between releases. +> Until then, treat the CSV in `dcgm-custom-metrics.yaml` as a +> known-good superset rather than a minimal config. [default-csv]: https://github.com/NVIDIA/dcgm-exporter/blob/main/etc/default-counters.csv - -## How the dashboard and recording rules fit in - -- `dashboards/gpu/gpu-performance.json` expects `DCGM_FI_*` metrics, - including profiling series (`DCGM_FI_PROF_PIPE_TENSOR_ACTIVE`, - `DCGM_FI_PROF_GR_ENGINE_ACTIVE`) and throttling counters - (`DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION`). - These are only emitted when DCGM Exporter is started with the custom - CSV in `dcgm-custom-metrics.yaml`. -- `packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml` - precomputes cluster-wide and per-namespace aggregations used by the - overview panels of the dashboard. The rules are safe to ship on any - cluster — they evaluate to empty series when DCGM is not scraped. diff --git a/packages/system/monitoring/dashboards-infra.list b/packages/system/monitoring/dashboards-infra.list index b402fa68..4c7e494d 100644 --- a/packages/system/monitoring/dashboards-infra.list +++ b/packages/system/monitoring/dashboards-infra.list @@ -35,3 +35,5 @@ hubble/network-overview gpu/gpu-performance gpu/gpu-efficiency gpu/gpu-quotas +gpu/gpu-fleet +gpu/gpu-tenants From 7eb9fe8ade53e450962e71548129d776c8fa7e6d Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 06:41:45 +0300 Subject: [PATCH 108/250] refactor(monitoring): drop unused GPU recording rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - namespace:gpu_count:sum — never consumed by any tracked dashboard; the billable view is already covered by namespace:gpu_count:allocated, and the admin view by cluster:gpu_count:allocated. - namespace:energy_joules:sum — no panel integrates joules; kWh readings on the tenant dashboard compute their own integrations from namespace:power_watts:sum. - pod:tensor_to_nvml_ratio:avg5m — interesting tenant signal in theory, but not wired into any panel and carrying it just burns cardinality on large fleets. Signed-off-by: Arsolitt --- .../alerts/gpu-recording.rules.yaml | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml index 3a31f5a2..28d6a45e 100644 --- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml +++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml @@ -25,11 +25,6 @@ spec: - name: gpu.recording.namespace.1m interval: 1m rules: - # DCGM-visible GPU count per namespace — counts GPUs that are actually - # running a tenant pod right now (driver loaded, scheduler placed it). - # Differs from :allocated when pods are Pending or stuck. - - record: namespace:gpu_count:sum - expr: count by (namespace) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}) # Kube-requested GPU count per namespace — billable view, includes # Pending pods. Use this for GPU-hour reporting via # sum_over_time(...[1h:1m])/60. @@ -62,8 +57,6 @@ spec: expr: sum by (namespace) (DCGM_FI_DEV_FB_USED{namespace!="", namespace!~"cozy-.*|kube-.*"}) * 1048576 - record: namespace:power_watts:sum expr: sum by (namespace) (DCGM_FI_DEV_POWER_USAGE{namespace!="", namespace!~"cozy-.*|kube-.*"}) - - record: namespace:energy_joules:sum - expr: sum by (namespace) (DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION{namespace!="", namespace!~"cozy-.*|kube-.*"}) / 1000 - name: gpu.recording.efficiency.1m interval: 1m @@ -76,19 +69,6 @@ spec: avg_over_time(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]) ) * 100 - # NVML vs tensor gap — ratio <10% means NVML lies: user thinks GPU - # is busy but specialized hardware is idle (cheap tenant signal). - - record: pod:tensor_to_nvml_ratio:avg5m - expr: | - ( - avg_over_time(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]) * 100 - ) - / - clamp_min( - avg_over_time(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]), - 1 - ) - # Power efficiency — utilization per watt, reveals unoptimized clients. - record: pod:util_per_watt:avg5m expr: | From 2fa4b3e31cd9777bdc88743272b064f076429c87 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 06:48:33 +0300 Subject: [PATCH 109/250] refactor(monitoring): tighten GPU dashboard queries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - gpu-efficiency: scope Tensor Saturation, Util-per-Watt and Power Throttle stats to the $namespace selector. Cluster-wide means were misleading when a user had narrowed the dashboard to specific tenants — the headline numbers lied relative to the panels below. - gpu-fleet: show per-node power draw as % of combined TDP cap (DCGM_FI_DEV_POWER_MGMT_LIMIT) instead of raw watts. Thresholds (60 / 80 %) generalize across GPU SKUs without per-model tuning. - gpu-quotas: read cluster:gpu_count:allocated from the recording rules instead of recomputing sum(kube_pod_container_resource_requests) inline. Keeps the dashboard aligned with the canonical definition in gpu-recording.rules.yaml so the two can't drift. Signed-off-by: Arsolitt --- dashboards/gpu/gpu-efficiency.json | 12 ++++++------ dashboards/gpu/gpu-fleet.json | 14 +++++++------- dashboards/gpu/gpu-quotas.json | 4 ++-- .../alerts/gpu-recording.rules.yaml | 1 + 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/dashboards/gpu/gpu-efficiency.json b/dashboards/gpu/gpu-efficiency.json index 596961e0..07000f71 100644 --- a/dashboards/gpu/gpu-efficiency.json +++ b/dashboards/gpu/gpu-efficiency.json @@ -35,12 +35,12 @@ "id": 2, "targets": [ { - "expr": "avg(pod:tensor_saturation:avg5m)", + "expr": "avg(pod:tensor_saturation:avg5m{namespace=~\"$namespace\"})", "refId": "A" } ], - "title": "Cluster Tensor Saturation", - "description": "Mean tensor core saturation. \u003c10% means GPUs are used inefficiently (tenants could move to CPU or optimize their code).", + "title": "Avg Tensor Saturation", + "description": "Mean tensor core saturation across selected namespaces. \u003c10% means GPUs are used inefficiently (tenants could move to CPU or optimize their code).", "transparent": false, "datasource": { "type": "prometheus", @@ -103,12 +103,12 @@ "id": 3, "targets": [ { - "expr": "avg(pod:util_per_watt:avg5m)", + "expr": "avg(pod:util_per_watt:avg5m{namespace=~\"$namespace\"})", "refId": "A" } ], "title": "Avg Utilization per Watt", - "description": "NVML utilization % per watt. Higher value = more efficient workload.", + "description": "NVML utilization % per watt across selected namespaces. Higher value = more efficient workload.", "transparent": false, "datasource": { "type": "prometheus", @@ -166,7 +166,7 @@ "id": 4, "targets": [ { - "expr": "avg(gpu:power_throttle_fraction:rate5m)", + "expr": "avg(gpu:power_throttle_fraction:rate5m{namespace=~\"$namespace\"})", "refId": "A" } ], diff --git a/dashboards/gpu/gpu-fleet.json b/dashboards/gpu/gpu-fleet.json index af5ad2fa..4cd5b4b5 100644 --- a/dashboards/gpu/gpu-fleet.json +++ b/dashboards/gpu/gpu-fleet.json @@ -467,13 +467,13 @@ "id": 12, "targets": [ { - "expr": "sum by (Hostname) (DCGM_FI_DEV_POWER_USAGE)", + "expr": "sum by (Hostname) (DCGM_FI_DEV_POWER_USAGE) / sum by (Hostname) (DCGM_FI_DEV_POWER_MGMT_LIMIT) * 100", "legendFormat": "{{Hostname}}", "refId": "A" } ], - "title": "Power draw per node", - "description": "Sum of GPU power usage per node in watts.", + "title": "Power draw per node (% of TDP)", + "description": "GPU power usage as a fraction of the node's combined TDP cap. Works across GPU SKUs without per-model thresholds.", "transparent": false, "datasource": { "type": "prometheus", @@ -510,9 +510,9 @@ }, "fieldConfig": { "defaults": { - "unit": "watt", + "unit": "percent", "min": 0, - "max": 200, + "max": 100, "thresholds": { "mode": "absolute", "steps": [ @@ -521,11 +521,11 @@ "color": "green" }, { - "value": 120, + "value": 60, "color": "yellow" }, { - "value": 160, + "value": 80, "color": "orange" } ] diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json index 60c628f6..3bc01a83 100644 --- a/dashboards/gpu/gpu-quotas.json +++ b/dashboards/gpu/gpu-quotas.json @@ -86,7 +86,7 @@ "id": 3, "targets": [ { - "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"})", + "expr": "cluster:gpu_count:allocated", "refId": "A" } ], @@ -143,7 +143,7 @@ "id": 4, "targets": [ { - "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}) / sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"}) * 100", + "expr": "cluster:gpu_count:allocated / sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"}) * 100", "refId": "A" } ], diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml index 28d6a45e..a24951f5 100644 --- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml +++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml @@ -2,6 +2,7 @@ apiVersion: operator.victoriametrics.com/v1beta1 kind: VMRule metadata: name: alerts-gpu-recording.rules + namespace: cozy-monitoring spec: groups: - name: gpu.recording.cluster.1m From ccfec2ef62684b854508294ab82e379bffcad771 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 06:55:17 +0300 Subject: [PATCH 110/250] fix(monitoring): close DCGM coverage gap for gpu-fleet TDP panel gpu-fleet.json references DCGM_FI_DEV_POWER_MGMT_LIMIT for its "TDP vs draw" panel, but the custom DCGM Exporter CSV did not declare it, so the panel silently rendered "No data" on clusters using that config. Declare the counter, fix the dashboards table in the gpu-operator examples README, and add a bats test that cross-checks every DCGM_FI_* reference in tracked dashboards and recording rules against the union of the upstream default set (snapshotted under hack/) and the project's custom CSV. Signed-off-by: Arsolitt --- hack/check-gpu-recording-rules.bats | 92 +++++++++++++++- hack/dcgm-default-counters.csv | 104 ++++++++++++++++++ .../system/gpu-operator/examples/README.md | 50 ++++----- .../examples/dcgm-custom-metrics.yaml | 1 + 4 files changed, 220 insertions(+), 27 deletions(-) create mode 100644 hack/dcgm-default-counters.csv diff --git a/hack/check-gpu-recording-rules.bats b/hack/check-gpu-recording-rules.bats index 87e7c8f1..4981959a 100644 --- a/hack/check-gpu-recording-rules.bats +++ b/hack/check-gpu-recording-rules.bats @@ -1,7 +1,7 @@ #!/usr/bin/env bats # ----------------------------------------------------------------------------- -# Cross-validation between GPU recording rules and the dashboards that consume -# them. Catches: +# Cross-validation between GPU recording rules, the dashboards that consume +# them, and the DCGM Exporter metric set the cluster actually scrapes. Catches: # # 1. dangling references — a dashboard query mentions a recording rule name # that doesn't exist in gpu-recording.rules.yaml. This is the bug the @@ -12,6 +12,13 @@ # 2. typos in rule names — same bug class, manifested as a single-character # difference between rule and reference. # +# 3. undeclared DCGM metrics — a dashboard query or recording rule mentions +# a DCGM_FI_* metric that is neither in the upstream default CSV nor in +# the project's custom CSV (dcgm-custom-metrics.yaml), meaning DCGM +# Exporter would never emit it and the panel silently shows "No data". +# Example regression: gpu-fleet.json shipped a TDP panel referencing +# DCGM_FI_DEV_POWER_MGMT_LIMIT before the custom CSV declared it. +# # Scope: only dashboards listed in packages/system/monitoring/dashboards-infra.list # under the "gpu/" prefix (i.e. shipped to production), not every JSON file in # dashboards/gpu/. Untracked drafts stay out of scope on purpose — adding one @@ -33,6 +40,8 @@ REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" RULES_FILE="$REPO_ROOT/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml" DASHBOARDS_LIST="$REPO_ROOT/packages/system/monitoring/dashboards-infra.list" DASHBOARDS_DIR="$REPO_ROOT/dashboards" +DCGM_DEFAULT_CSV="$REPO_ROOT/hack/dcgm-default-counters.csv" +DCGM_CUSTOM_CSV="$REPO_ROOT/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml" # Extract the set of "- record: NAME" entries from the rules YAML. # Outputs one rule name per line, sorted and deduplicated. @@ -61,6 +70,37 @@ list_tracked_gpu_dashboards() { awk '/^gpu\// { print $0 ".json" }' "$DASHBOARDS_LIST" } +# Extract the set of DCGM_FI_* metric names declared in a CSV file. Handles +# both the upstream-style default CSV (unindented) and the ConfigMap-style +# custom CSV (YAML-indented). A declaration line starts — after any leading +# whitespace — with "DCGM_FI_," ; comment lines begin with "#" and are +# skipped. Uses POSIX awk's match()+RSTART/RLENGTH so no GNU extensions +# are required. +extract_csv_metrics() { + file=$1 + awk ' + { + line = $0 + sub(/^[[:space:]]+/, "", line) + if (line ~ /^#/) next + if (match(line, /^DCGM_FI_[A-Z0-9_]+/)) { + print substr(line, RSTART, RLENGTH) + } + } + ' "$file" | sort -u +} + +# Extract the set of DCGM_FI_* metric references from a text file (dashboard +# JSON or rules YAML). A DCGM metric name has at least two underscore-delimited +# segments after the "DCGM_FI_" prefix (e.g. DCGM_FI_DEV_GPU_UTIL, DCGM_FI_PROF_ +# PIPE_TENSOR_ACTIVE, DCGM_FI_DRIVER_VERSION). Requiring two segments keeps +# the matcher from latching onto glob stubs like "DCGM_FI_DEV_*_VIOLATION" that +# appear in comments. +extract_dcgm_refs() { + file=$1 + grep -hoE 'DCGM_FI_[A-Z0-9]+(_[A-Z0-9]+)+' "$file" | sort -u +} + @test "every recording rule reference in tracked GPU dashboards has a matching record" { TMP=$(mktemp -d) trap 'rm -rf "$TMP"' EXIT @@ -94,6 +134,54 @@ list_tracked_gpu_dashboards() { [ "$failed" -eq 0 ] } +@test "every DCGM metric referenced in tracked dashboards and rules is declared" { + TMP=$(mktemp -d) + trap 'rm -rf "$TMP"' EXIT + + [ -f "$DCGM_DEFAULT_CSV" ] || { echo "missing $DCGM_DEFAULT_CSV" >&2; exit 1; } + [ -f "$DCGM_CUSTOM_CSV" ] || { echo "missing $DCGM_CUSTOM_CSV" >&2; exit 1; } + + { + extract_csv_metrics "$DCGM_DEFAULT_CSV" + extract_csv_metrics "$DCGM_CUSTOM_CSV" + } | sort -u > "$TMP/declared.txt" + [ -s "$TMP/declared.txt" ] || { echo "no DCGM metrics extracted from CSVs" >&2; exit 1; } + + list_tracked_gpu_dashboards > "$TMP/dashboards.txt" + [ -s "$TMP/dashboards.txt" ] || { echo "no gpu/* dashboards listed in $DASHBOARDS_LIST" >&2; exit 1; } + + failed=0 + + # Dashboard coverage — every dashboard's DCGM references must resolve. + while IFS= read -r dashboard_rel; do + dashboard="$DASHBOARDS_DIR/$dashboard_rel" + [ -f "$dashboard" ] || continue # handled by the existence test + extract_dcgm_refs "$dashboard" > "$TMP/refs.txt" + [ -s "$TMP/refs.txt" ] || continue # dashboard relies entirely on recording rules + comm -23 "$TMP/refs.txt" "$TMP/declared.txt" > "$TMP/missing.txt" + if [ -s "$TMP/missing.txt" ]; then + echo "ERROR: $dashboard_rel references DCGM metrics not declared in any CSV:" >&2 + sed 's/^/ - /' "$TMP/missing.txt" >&2 + failed=1 + fi + done < "$TMP/dashboards.txt" + + # Rules coverage — recording rules consume DCGM directly, so their set + # must be declared too, otherwise derived series on every dashboard + # collapse to empty. + extract_dcgm_refs "$RULES_FILE" > "$TMP/rule-refs.txt" + if [ -s "$TMP/rule-refs.txt" ]; then + comm -23 "$TMP/rule-refs.txt" "$TMP/declared.txt" > "$TMP/rule-missing.txt" + if [ -s "$TMP/rule-missing.txt" ]; then + echo "ERROR: gpu-recording.rules.yaml references DCGM metrics not declared in any CSV:" >&2 + sed 's/^/ - /' "$TMP/rule-missing.txt" >&2 + failed=1 + fi + fi + + [ "$failed" -eq 0 ] +} + @test "every tracked GPU dashboard listed in dashboards-infra.list exists on disk" { TMP=$(mktemp -d) trap 'rm -rf "$TMP"' EXIT diff --git a/hack/dcgm-default-counters.csv b/hack/dcgm-default-counters.csv new file mode 100644 index 00000000..b5e94540 --- /dev/null +++ b/hack/dcgm-default-counters.csv @@ -0,0 +1,104 @@ +# Snapshot of the upstream DCGM Exporter default-counters.csv used as a +# fixture by hack/check-gpu-recording-rules.bats. The test verifies that +# every DCGM_FI_* metric referenced by a tracked GPU dashboard is either +# declared here (upstream defaults) or in +# packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml +# (the project's custom CSV). +# +# Source: https://github.com/NVIDIA/dcgm-exporter/blob/4.1.1-4.0.4/etc/default-counters.csv +# Pinned to the DCGM Exporter image tag shipped by the gpu-operator +# chart under packages/system/gpu-operator/charts/gpu-operator/values.yaml +# (dcgmExporter.version = 4.1.1-4.0.4-ubuntu22.04). When that image is +# bumped, refresh this file from the matching tag in the NVIDIA/dcgm-exporter +# repo and re-run `./hack/cozytest.sh hack/check-gpu-recording-rules.bats`. + +# Format +# If line starts with a '#' it is considered a comment +# DCGM FIELD, Prometheus metric type, help message + +# Clocks +DCGM_FI_DEV_SM_CLOCK, gauge, SM clock frequency (in MHz). +DCGM_FI_DEV_MEM_CLOCK, gauge, Memory clock frequency (in MHz). + +# Temperature +DCGM_FI_DEV_MEMORY_TEMP, gauge, Memory temperature (in C). +DCGM_FI_DEV_GPU_TEMP, gauge, GPU temperature (in C). + +# Power +DCGM_FI_DEV_POWER_USAGE, gauge, Power draw (in W). +DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION, counter, Total energy consumption since boot (in mJ). + +# PCIE +# DCGM_FI_PROF_PCIE_TX_BYTES, counter, Total number of bytes transmitted through PCIe TX via NVML. +# DCGM_FI_PROF_PCIE_RX_BYTES, counter, Total number of bytes received through PCIe RX via NVML. +DCGM_FI_DEV_PCIE_REPLAY_COUNTER, counter, Total number of PCIe retries. + +# Utilization (the sample period varies depending on the product) +DCGM_FI_DEV_GPU_UTIL, gauge, GPU utilization (in %). +DCGM_FI_DEV_MEM_COPY_UTIL, gauge, Memory utilization (in %). +DCGM_FI_DEV_ENC_UTIL, gauge, Encoder utilization (in %). +DCGM_FI_DEV_DEC_UTIL , gauge, Decoder utilization (in %). + +# Errors and violations +DCGM_FI_DEV_XID_ERRORS, gauge, Value of the last XID error encountered. +# DCGM_FI_DEV_POWER_VIOLATION, counter, Throttling duration due to power constraints (in us). +# DCGM_FI_DEV_THERMAL_VIOLATION, counter, Throttling duration due to thermal constraints (in us). +# DCGM_FI_DEV_SYNC_BOOST_VIOLATION, counter, Throttling duration due to sync-boost constraints (in us). +# DCGM_FI_DEV_BOARD_LIMIT_VIOLATION, counter, Throttling duration due to board limit constraints (in us). +# DCGM_FI_DEV_LOW_UTIL_VIOLATION, counter, Throttling duration due to low utilization (in us). +# DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (in us). + +# Memory usage +DCGM_FI_DEV_FB_FREE, gauge, Framebuffer memory free (in MiB). +DCGM_FI_DEV_FB_USED, gauge, Framebuffer memory used (in MiB). + +# ECC +# DCGM_FI_DEV_ECC_SBE_VOL_TOTAL, counter, Total number of single-bit volatile ECC errors. +# DCGM_FI_DEV_ECC_DBE_VOL_TOTAL, counter, Total number of double-bit volatile ECC errors. +# DCGM_FI_DEV_ECC_SBE_AGG_TOTAL, counter, Total number of single-bit persistent ECC errors. +# DCGM_FI_DEV_ECC_DBE_AGG_TOTAL, counter, Total number of double-bit persistent ECC errors. + +# Retired pages +# DCGM_FI_DEV_RETIRED_SBE, counter, Total number of retired pages due to single-bit errors. +# DCGM_FI_DEV_RETIRED_DBE, counter, Total number of retired pages due to double-bit errors. +# DCGM_FI_DEV_RETIRED_PENDING, counter, Total number of pages pending retirement. + +# NVLink +# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_TOTAL, counter, Total number of NVLink flow-control CRC errors. +# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_TOTAL, counter, Total number of NVLink data CRC errors. +# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_TOTAL, counter, Total number of NVLink retries. +# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_TOTAL, counter, Total number of NVLink recovery errors. +DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL, counter, Total number of NVLink bandwidth counters for all lanes. +# DCGM_FI_DEV_NVLINK_BANDWIDTH_L0, counter, The number of bytes of active NVLink rx or tx data including both header and payload. + +# VGPU License status +DCGM_FI_DEV_VGPU_LICENSE_STATUS, gauge, vGPU License status + +# Remapped rows +DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for uncorrectable errors +DCGM_FI_DEV_CORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for correctable errors +DCGM_FI_DEV_ROW_REMAP_FAILURE, gauge, Whether remapping of rows has failed + +# Static configuration information. These appear as labels on the other metrics +DCGM_FI_DRIVER_VERSION, label, Driver Version +# DCGM_FI_NVML_VERSION, label, NVML Version +# DCGM_FI_DEV_BRAND, label, Device Brand +# DCGM_FI_DEV_SERIAL, label, Device Serial Number +# DCGM_FI_DEV_OEM_INFOROM_VER, label, OEM inforom version +# DCGM_FI_DEV_ECC_INFOROM_VER, label, ECC inforom version +# DCGM_FI_DEV_POWER_INFOROM_VER, label, Power management object inforom version +# DCGM_FI_DEV_INFOROM_IMAGE_VER, label, Inforom image version +# DCGM_FI_DEV_VBIOS_VERSION, label, VBIOS version of the device + +# Datacenter Profiling (DCP) metrics +# NOTE: supported on Nvidia datacenter Volta GPUs and newer +DCGM_FI_PROF_GR_ENGINE_ACTIVE, gauge, Ratio of time the graphics engine is active. +# DCGM_FI_PROF_SM_ACTIVE, gauge, The ratio of cycles an SM has at least 1 warp assigned. +# DCGM_FI_PROF_SM_OCCUPANCY, gauge, The ratio of number of warps resident on an SM. +DCGM_FI_PROF_PIPE_TENSOR_ACTIVE, gauge, Ratio of cycles the tensor (HMMA) pipe is active. +DCGM_FI_PROF_DRAM_ACTIVE, gauge, Ratio of cycles the device memory interface is active sending or receiving data. +# DCGM_FI_PROF_PIPE_FP64_ACTIVE, gauge, Ratio of cycles the fp64 pipes are active. +# DCGM_FI_PROF_PIPE_FP32_ACTIVE, gauge, Ratio of cycles the fp32 pipes are active. +# DCGM_FI_PROF_PIPE_FP16_ACTIVE, gauge, Ratio of cycles the fp16 pipes are active. +DCGM_FI_PROF_PCIE_TX_BYTES, gauge, The rate of data transmitted over the PCIe bus - including both protocol headers and data payloads - in bytes per second. +DCGM_FI_PROF_PCIE_RX_BYTES, gauge, The rate of data received over the PCIe bus - including both protocol headers and data payloads - in bytes per second. diff --git a/packages/system/gpu-operator/examples/README.md b/packages/system/gpu-operator/examples/README.md index 5374bffc..9599acb7 100644 --- a/packages/system/gpu-operator/examples/README.md +++ b/packages/system/gpu-operator/examples/README.md @@ -68,34 +68,34 @@ scraped, or when optional counters are missing. What each dashboard needs on top of the upstream DCGM Exporter [`default-counters.csv`][default-csv]: -| Dashboard | Scope | Needs beyond defaults | -| ----------------- | ---------------------------------- | -------------------------------------------------------------------------------------------- | -| `gpu-performance` | Per-node, per-GPU deep dive | `DCGM_FI_PROF_PIPE_TENSOR_ACTIVE`, `DCGM_FI_PROF_GR_ENGINE_ACTIVE`, `DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION` | -| `gpu-efficiency` | Per-workload util vs tensor active | `DCGM_FI_PROF_PIPE_TENSOR_ACTIVE` | -| `gpu-fleet` | Cluster-wide admin inventory | nothing (works on default counters) | -| `gpu-quotas` | Kube-quota vs live usage | nothing (kube-state-metrics + default counters) | -| `gpu-tenants` | Per-namespace tenant view | `DCGM_FI_PROF_PIPE_TENSOR_ACTIVE` for the tensor-saturation panel; other panels work on defaults | +| Dashboard | Scope | Needs beyond defaults | +| ----------------- | ---------------------------------- | ----------------------------------------------------------------------- | +| `gpu-performance` | Per-node, per-GPU deep dive | `DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION` | +| `gpu-efficiency` | Per-workload util vs tensor active | `DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION` (via `gpu:*_throttle_fraction:rate5m` recording rules) | +| `gpu-fleet` | Cluster-wide admin inventory | `DCGM_FI_DEV_POWER_MGMT_LIMIT` (for the TDP vs draw panel) | +| `gpu-quotas` | Kube-quota vs live usage | nothing (kube-state-metrics + default counters) | +| `gpu-tenants` | Per-namespace tenant view | nothing (works on default counters) | -The throttling counters (`DCGM_FI_DEV_POWER_VIOLATION`, -`DCGM_FI_DEV_THERMAL_VIOLATION`) are only required by `gpu-performance`. -The profiling counters (`DCGM_FI_PROF_*`) are required by -`gpu-performance` and `gpu-efficiency`, and unlock the tensor panel in -`gpu-tenants`. Everything else the recording rules consume — -utilization, FB used/free, power, temperature, energy — is already in -the default counter set. +`DCGM_FI_PROF_PIPE_TENSOR_ACTIVE` and `DCGM_FI_PROF_GR_ENGINE_ACTIVE` +are already in the upstream default set for the pinned DCGM Exporter +version, so the tensor-saturation and engine-active panels work without +any CSV override. The three counters listed in the table — throttling +violations and the power management limit — are the only extras the +tracked dashboards need. The recording rules in +`gpu-recording.rules.yaml` consume utilization, FB used, power, +temperature and the tensor-active profiling counter, all of which are +in the default set. ## Verification status -> **Pending verification on an updated GPU Operator release.** -> -> The minimum-CSV claims above are derived by cross-referencing -> `gpu-recording.rules.yaml` and each dashboard against the DCGM -> Exporter [`default-counters.csv`][default-csv] for the version pinned -> in the currently shipped `gpu-operator` package. The package in this -> branch is **not** the latest GPU Operator release; once we move to a -> newer version, the claims must be re-checked because the upstream -> default set occasionally adds or removes counters between releases. -> Until then, treat the CSV in `dcgm-custom-metrics.yaml` as a -> known-good superset rather than a minimal config. +The minimum-CSV claims above are verified by +`hack/check-gpu-recording-rules.bats`, which cross-checks every +`DCGM_FI_*` reference in the tracked GPU dashboards and recording rules +against the union of the upstream default set (snapshotted at +`hack/dcgm-default-counters.csv` for the pinned DCGM Exporter version) +and the custom CSV in `dcgm-custom-metrics.yaml`. When the DCGM +Exporter image in `packages/system/gpu-operator/charts/gpu-operator/values.yaml` +is bumped, refresh the snapshot from the matching tag of the +[`NVIDIA/dcgm-exporter`][default-csv] repository and rerun the test. [default-csv]: https://github.com/NVIDIA/dcgm-exporter/blob/main/etc/default-counters.csv diff --git a/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml b/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml index 14b89b5c..92064701 100644 --- a/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml +++ b/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml @@ -26,6 +26,7 @@ data: # Power DCGM_FI_DEV_POWER_USAGE, gauge, Power draw (in W). + DCGM_FI_DEV_POWER_MGMT_LIMIT, gauge, Current power management limit (in W). DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION, counter, Total energy consumption since boot (in mJ). # PCIE From 2518e09d6775ae9916c52dba16b52d88a0e7beaa Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 07:03:42 +0300 Subject: [PATCH 111/250] refactor(monitoring): store pod:tensor_saturation as unitless ratio Align pod:tensor_saturation:avg5m with namespace:tensor_active:avg and DCGM's native 0..1 range by dropping the * 100 from the recording rule and multiplying at display time in gpu-efficiency.json. Also scope pod:util_per_watt:avg5m with avg by (Hostname, gpu, UUID, namespace, pod) so the series mirrors pod:tensor_saturation's grouping and stays usable in topk queries. Signed-off-by: Arsolitt --- dashboards/gpu/gpu-efficiency.json | 4 ++-- .../alerts/gpu-recording.rules.yaml | 16 ++++++++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/dashboards/gpu/gpu-efficiency.json b/dashboards/gpu/gpu-efficiency.json index 07000f71..dddb2001 100644 --- a/dashboards/gpu/gpu-efficiency.json +++ b/dashboards/gpu/gpu-efficiency.json @@ -35,7 +35,7 @@ "id": 2, "targets": [ { - "expr": "avg(pod:tensor_saturation:avg5m{namespace=~\"$namespace\"})", + "expr": "avg(pod:tensor_saturation:avg5m{namespace=~\"$namespace\"}) * 100", "refId": "A" } ], @@ -363,7 +363,7 @@ "id": 21, "targets": [ { - "expr": "topk(20, pod:tensor_saturation:avg5m)", + "expr": "topk(20, pod:tensor_saturation:avg5m * 100)", "instant": true, "range": false, "format": "table", diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml index a24951f5..a6ecb20a 100644 --- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml +++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml @@ -64,20 +64,24 @@ spec: rules: # Tensor hardware saturation — the honest "am I using the GPU" metric # for AI/LLM workloads. Unlike NVML, idle tensor cores are visible. + # Stored as a 0..1 ratio to stay consistent with namespace:tensor_active:avg + # and DCGM's native units. Consumers multiply by 100 at display time. - record: pod:tensor_saturation:avg5m expr: | avg by (Hostname, gpu, UUID, namespace, pod) ( avg_over_time(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]) - ) * 100 + ) # Power efficiency — utilization per watt, reveals unoptimized clients. - record: pod:util_per_watt:avg5m expr: | - avg_over_time(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]) - / - clamp_min( - avg_over_time(DCGM_FI_DEV_POWER_USAGE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]), - 1 + avg by (Hostname, gpu, UUID, namespace, pod) ( + avg_over_time(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]) + / + clamp_min( + avg_over_time(DCGM_FI_DEV_POWER_USAGE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]), + 1 + ) ) # Fraction of time power-throttled (TDP cap) — 1.0 = fully throttled. From 5b210ac7fd29e15804fe787f6de8f29455eeeab4 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 07:11:28 +0300 Subject: [PATCH 112/250] docs(monitoring): mark gpu-fleet average utilization as legacy NVML Clarify that the "Average utilization" panel on gpu-fleet reflects the legacy NVML view (DCGM_FI_DEV_GPU_UTIL) rather than engine-active profiling. For AI/LLM workloads the NVML number is optimistic; the gpu-efficiency dashboard carries the profiling-based view. Signed-off-by: Arsolitt --- dashboards/gpu/gpu-fleet.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboards/gpu/gpu-fleet.json b/dashboards/gpu/gpu-fleet.json index 4cd5b4b5..ea169433 100644 --- a/dashboards/gpu/gpu-fleet.json +++ b/dashboards/gpu/gpu-fleet.json @@ -268,7 +268,7 @@ } ], "title": "Average utilization", - "description": "NVML utilization across all tenant GPUs (engine-active %).", + "description": "Legacy NVML utilization across all tenant GPUs.", "transparent": false, "datasource": { "type": "prometheus", From 4e37f64553728bab5b59ae7d7e8a50ee91c23232 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 07:19:05 +0300 Subject: [PATCH 113/250] docs(gpu-operator): document tolerate-all on compat DaemonSet Explain why tolerations: [{operator: Exists}] is safe on the driver compat DaemonSet: the nodeSelector already confines scheduling to GPU nodes, so the blanket toleration only kicks in when those nodes carry the dedicated=gpu / nvidia.com/gpu taints that the GPU Operator's default policy and many deployments apply. Signed-off-by: Arsolitt --- .../system/gpu-operator/examples/nvidia-driver-compat.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml b/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml index 63e07c4f..bad8609e 100644 --- a/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml +++ b/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml @@ -35,6 +35,13 @@ spec: # cluster uses to mark GPU hosts. nodeSelector: nvidia.com/gpu.present: "true" + # Tolerate-all is intentionally broad: the nodeSelector above already + # confines scheduling to GPU nodes, so the "Exists" toleration only + # matters when those nodes carry extra taints (dedicated=gpu:NoSchedule, + # nvidia.com/gpu:NoSchedule from the GPU Operator's default policy, + # etc.). This mirrors the stance of the upstream nvidia-driver-daemonset + # and nvidia-device-plugin DaemonSets — blanket tolerations plus a + # narrow nodeSelector. tolerations: - operator: Exists initContainers: From 5db6ec3e1fc8b2d30d3aa017f0ff7bc4dd943000 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 07:27:33 +0300 Subject: [PATCH 114/250] fix(dashboard): scope GPU panels to selected namespace Pod-level panels on the efficiency dashboard and DCGM-level panels on the performance dashboard ignored the $namespace template variable, so changing it left the visualizations unchanged. Add the filter to each query. Performance-side queries use the `$namespace|` empty-tolerant form so host-level DCGM series without a namespace label remain visible when a specific namespace is selected. Signed-off-by: Arsolitt --- dashboards/gpu/gpu-efficiency.json | 4 ++-- dashboards/gpu/gpu-performance.json | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dashboards/gpu/gpu-efficiency.json b/dashboards/gpu/gpu-efficiency.json index dddb2001..e2e8c105 100644 --- a/dashboards/gpu/gpu-efficiency.json +++ b/dashboards/gpu/gpu-efficiency.json @@ -363,7 +363,7 @@ "id": 21, "targets": [ { - "expr": "topk(20, pod:tensor_saturation:avg5m * 100)", + "expr": "topk(20, pod:tensor_saturation:avg5m{namespace=~\"$namespace\"} * 100)", "instant": true, "range": false, "format": "table", @@ -501,7 +501,7 @@ "id": 22, "targets": [ { - "expr": "topk(20, pod:util_per_watt:avg5m)", + "expr": "topk(20, pod:util_per_watt:avg5m{namespace=~\"$namespace\"})", "instant": true, "range": false, "format": "table", diff --git a/dashboards/gpu/gpu-performance.json b/dashboards/gpu/gpu-performance.json index b0f26958..e16f8c47 100644 --- a/dashboards/gpu/gpu-performance.json +++ b/dashboards/gpu/gpu-performance.json @@ -553,7 +553,7 @@ "id": 22, "targets": [ { - "expr": "DCGM_FI_DEV_FB_FREE{Hostname=~\"$Hostname\"} * 1048576", + "expr": "DCGM_FI_DEV_FB_FREE{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"} * 1048576", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } @@ -617,7 +617,7 @@ "id": 31, "targets": [ { - "expr": "DCGM_FI_DEV_POWER_USAGE{Hostname=~\"$Hostname\"}", + "expr": "DCGM_FI_DEV_POWER_USAGE{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"}", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } @@ -668,7 +668,7 @@ "id": 32, "targets": [ { - "expr": "DCGM_FI_DEV_GPU_TEMP{Hostname=~\"$Hostname\"}", + "expr": "DCGM_FI_DEV_GPU_TEMP{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"}", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } @@ -751,7 +751,7 @@ "id": 41, "targets": [ { - "expr": "max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS{Hostname=~\"$Hostname\"})", + "expr": "max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"})", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } @@ -809,7 +809,7 @@ "id": 42, "targets": [ { - "expr": "rate(DCGM_FI_DEV_POWER_VIOLATION{Hostname=~\"$Hostname\"}[5m])", + "expr": "rate(DCGM_FI_DEV_POWER_VIOLATION{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"}[5m])", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } @@ -857,7 +857,7 @@ "id": 43, "targets": [ { - "expr": "rate(DCGM_FI_DEV_THERMAL_VIOLATION{Hostname=~\"$Hostname\"}[5m])", + "expr": "rate(DCGM_FI_DEV_THERMAL_VIOLATION{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"}[5m])", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } From b64bfcc414ec4faddecf49ff9ece8ef169896bc0 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 07:36:14 +0300 Subject: [PATCH 115/250] fix(dashboard): deduplicate pending GPU pods by (namespace, pod) The Pending GPU pods counter on gpu-quotas joined raw kube_pod_container_resource_requests (per-container series) against kube_pod_status_phase (per-pod series). Multi-container pods were counted once per requesting container instead of once per pod, so the widget over-reported whenever a Pending pod had more than one GPU container. Collapse the requests to pod level before the join. Signed-off-by: Arsolitt --- dashboards/gpu/gpu-quotas.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json index 3bc01a83..624f072b 100644 --- a/dashboards/gpu/gpu-quotas.json +++ b/dashboards/gpu/gpu-quotas.json @@ -208,7 +208,7 @@ "id": 5, "targets": [ { - "expr": "count((kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"} \u003e 0) * on(namespace, pod) group_left() (kube_pod_status_phase{phase=\"Pending\"} == 1)) or vector(0)", + "expr": "count((sum by (namespace, pod) (kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}) \u003e 0) * on(namespace, pod) group_left() (kube_pod_status_phase{phase=\"Pending\"} == 1)) or vector(0)", "refId": "A" } ], From 51b0dedd0820c2a23978880baac0b8a6a3cbca94 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 07:45:02 +0300 Subject: [PATCH 116/250] chore(monitoring): tidy GPU VMRule top-level structure Drop the hardcoded metadata.namespace so the rule inherits the chart's release namespace, and add an explicit empty params field on every group for schema consistency. No behavior change. Signed-off-by: Arsolitt --- .../system/monitoring-agents/alerts/gpu-recording.rules.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml index a6ecb20a..22b3edf0 100644 --- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml +++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml @@ -2,11 +2,11 @@ apiVersion: operator.victoriametrics.com/v1beta1 kind: VMRule metadata: name: alerts-gpu-recording.rules - namespace: cozy-monitoring spec: groups: - name: gpu.recording.cluster.1m interval: 1m + params: {} rules: - record: cluster:gpu_count:total expr: count(group by (UUID) (DCGM_FI_DEV_GPU_UTIL)) @@ -25,6 +25,7 @@ spec: - name: gpu.recording.namespace.1m interval: 1m + params: {} rules: # Kube-requested GPU count per namespace — billable view, includes # Pending pods. Use this for GPU-hour reporting via @@ -61,6 +62,7 @@ spec: - name: gpu.recording.efficiency.1m interval: 1m + params: {} rules: # Tensor hardware saturation — the honest "am I using the GPU" metric # for AI/LLM workloads. Unlike NVML, idle tensor cores are visible. From 0634654d63d0a9b6ef17074161fce84443e3ba95 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 07:53:48 +0300 Subject: [PATCH 117/250] fix(monitoring): exclude terminated pods from GPU allocation count MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit kube-state-metrics keeps kube_pod_container_resource_requests series for Failed/Succeeded pods until the apiserver garbage-collects them, which could inflate :allocated beyond what tenants actually hold and drive cluster:gpu_count:free negative. Join the request metric against kube_pod_status_phase filtered to Pending|Running — the canonical pattern from Kubernetes' own container_resource recording rules — on both the cluster and namespace aggregates. Add clamp_min(..., 0) on cluster:gpu_count:free as a second line of defence against transient label drift between kube-state-metrics and DCGM. Signed-off-by: Arsolitt --- .../alerts/gpu-recording.rules.yaml | 35 +++++++++++++++---- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml index 22b3edf0..de81b8cc 100644 --- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml +++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml @@ -10,14 +10,29 @@ spec: rules: - record: cluster:gpu_count:total expr: count(group by (UUID) (DCGM_FI_DEV_GPU_UTIL)) - # Kube-allocated GPU count: GPUs requested by *all* pods regardless of - # phase (Pending+Running). Source of truth for "what tenants asked for" - # — used for capacity planning and billing. Includes system pods so it - # stays consistent with cluster:gpu_count:total when computing :free. + # Kube-allocated GPU count: GPUs requested by *active* pods — Pending + # and Running only. Source of truth for "what tenants asked for" — used + # for capacity planning and billing. Includes system pods so it stays + # consistent with cluster:gpu_count:total when computing :free. + # + # Phase filter via group_left() on kube_pod_status_phase matches the + # canonical pattern from k8s.rules.container_resource.yaml. Without it + # kube-state-metrics keeps series for Failed/Succeeded pods until the + # apiserver garbage-collects them, which inflates billing hours and + # can push :free negative. - record: cluster:gpu_count:allocated - expr: sum(kube_pod_container_resource_requests{resource="nvidia_com_gpu"}) + expr: | + sum( + kube_pod_container_resource_requests{resource="nvidia_com_gpu"} + * on (namespace, pod) group_left() max by (namespace, pod) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + # clamp_min guards against transients at DCGM/kube-state-metrics + # restart where cluster:gpu_count:total dips before :allocated catches + # up, or rare label drift between the two sources. - record: cluster:gpu_count:free - expr: cluster:gpu_count:total - (cluster:gpu_count:allocated or vector(0)) + expr: clamp_min(cluster:gpu_count:total - (cluster:gpu_count:allocated or vector(0)), 0) - record: cluster:gpu_util:avg expr: avg(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}) - record: cluster:gpu_power_watts:sum @@ -50,7 +65,13 @@ spec: # keeps system pods to stay aligned with cluster:gpu_count:total # when computing :free. - record: namespace:gpu_count:allocated - expr: sum by (namespace) (kube_pod_container_resource_requests{resource="nvidia_com_gpu", namespace!="", namespace!~"cozy-.*|kube-.*"}) + expr: | + sum by (namespace) ( + kube_pod_container_resource_requests{resource="nvidia_com_gpu", namespace!="", namespace!~"cozy-.*|kube-.*"} + * on (namespace, pod) group_left() max by (namespace, pod) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) - record: namespace:gpu_util:avg expr: avg by (namespace) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}) - record: namespace:tensor_active:avg From 605bcd338c744905097dc5c34a9b26f20b999661 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 08:03:21 +0300 Subject: [PATCH 118/250] fix(monitoring): pin label matching on GPU efficiency and throttle rules pod:util_per_watt:avg5m divided two DCGM metrics without an explicit on(...) clause, so the match used the intersection of their label sets. If dcgm-exporter relabeling ever diverges between DCGM_FI_DEV_GPU_UTIL and DCGM_FI_DEV_POWER_USAGE (e.g. a pod-mapping label appears on one but not the other after a config change), the entire result drops to empty silently. Pin the match to the labels we group by so divergence becomes a missing side, not a missing rule. Throttle fractions had a related shape problem: dcgm-exporter emits one series per GPU for each pod-mapping combination. On a shared GPU (restart races, MIG/MPS) the same physical counter appears under multiple pod labels and downstream avg(...) panels get diluted by the pod count. Fold duplicates with max by (Hostname, gpu, UUID) before clamp_max so the fraction is tied to the physical GPU. Signed-off-by: Arsolitt --- .../alerts/gpu-recording.rules.yaml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml index de81b8cc..c56ea851 100644 --- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml +++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml @@ -96,11 +96,14 @@ spec: ) # Power efficiency — utilization per watt, reveals unoptimized clients. + # Explicit on(...) pins the bigop matching set to the labels we + # group by — protects against empty results if dcgm-exporter + # relabeling ever diverges between the two metrics. - record: pod:util_per_watt:avg5m expr: | avg by (Hostname, gpu, UUID, namespace, pod) ( avg_over_time(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]) - / + / on (Hostname, gpu, UUID, namespace, pod) clamp_min( avg_over_time(DCGM_FI_DEV_POWER_USAGE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]), 1 @@ -112,9 +115,16 @@ spec: # counter grows in nanoseconds in practice — divide by 1e9 to get a # 0..1 fraction (verified empirically when /1e6 yielded >100× reality). # clamp_max protects against rate() artefacts at counter resets. + # + # max by (Hostname, gpu, UUID) collapses duplicate series from + # dcgm-exporter's pod-mapping labels (one physical GPU may appear + # under multiple pod/container labels during restarts or under + # MIG/MPS). Throttling is a physical-GPU property; taking max + # keeps consumer avg(...) honest — a naive avg() would dilute the + # signal by the number of pods sharing the GPU. - record: gpu:power_throttle_fraction:rate5m - expr: clamp_max(rate(DCGM_FI_DEV_POWER_VIOLATION[5m]) / 1e9, 1) + expr: clamp_max(max by (Hostname, gpu, UUID) (rate(DCGM_FI_DEV_POWER_VIOLATION[5m]) / 1e9), 1) # Fraction of time thermal-throttled. - record: gpu:thermal_throttle_fraction:rate5m - expr: clamp_max(rate(DCGM_FI_DEV_THERMAL_VIOLATION[5m]) / 1e9, 1) + expr: clamp_max(max by (Hostname, gpu, UUID) (rate(DCGM_FI_DEV_THERMAL_VIOLATION[5m]) / 1e9), 1) From 165e175d70566b6890abea3efd01c18784f9b9ff Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 08:11:57 +0300 Subject: [PATCH 119/250] feat(monitoring): alert on DCGM throttle divisor drift MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The /1e9 divisor in gpu:{power,thermal}_throttle_fraction:rate5m was derived empirically against DCGM 3.x on A10 — the counter documents itself as microseconds but ticks in nanoseconds in practice. If a future exporter release honors the documented units, pre-clamp values would exceed 1.0 while clamp_max(..., 1) silently masks the drift, plateauing every throttle fraction at 100% and making the panels lie in unison. Add a validation group that fires when the raw max/1e9 value exceeds 1.0 for 15m, so we notice and rescale to /1e6 before dashboards silently mislead operators. Signed-off-by: Arsolitt --- .../alerts/gpu-recording.rules.yaml | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml index c56ea851..7589f894 100644 --- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml +++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml @@ -128,3 +128,33 @@ spec: # Fraction of time thermal-throttled. - record: gpu:thermal_throttle_fraction:rate5m expr: clamp_max(max by (Hostname, gpu, UUID) (rate(DCGM_FI_DEV_THERMAL_VIOLATION[5m]) / 1e9), 1) + + # Regression watch — the /1e9 divisor on *_VIOLATION was derived + # empirically against DCGM 3.x on A10. If a future DCGM version + # restores the documented µs units, pre-clamp values would exceed 1.0 + # while clamp_max(..., 1) silently masks the drift — recorded + # throttle fractions would plateau at 1.0 and consumers would think + # every GPU is always throttled. This alert fires on that condition + # so we can rescale to /1e6 before dashboards start lying. + - name: gpu.recording.throttle.validation.5m + interval: 5m + params: {} + rules: + - alert: GPUThrottleFractionOverOne + expr: | + max by (Hostname, gpu, UUID) (rate(DCGM_FI_DEV_POWER_VIOLATION[5m]) / 1e9) > 1 + or + max by (Hostname, gpu, UUID) (rate(DCGM_FI_DEV_THERMAL_VIOLATION[5m]) / 1e9) > 1 + for: 15m + labels: + severity: warning + component: gpu-monitoring + annotations: + summary: "DCGM throttle fraction exceeds 1.0 pre-clamp on {{ $labels.Hostname }}/{{ $labels.gpu }}" + description: | + gpu:{power,thermal}_throttle_fraction:rate5m clamps to 1.0, but the raw + rate(DCGM_FI_DEV_*_VIOLATION)/1e9 value is already >1 on this GPU — the + empirical nanosecond assumption is broken. Likely the DCGM exporter + version changed and the counter now grows in its documented µs units. + Re-verify on the exporter Pod and adjust the divisor (/1e9 → /1e6) in + gpu.recording.efficiency.1m before dashboards plateau at 100% throttle. From 549b341675ea09ce2d61de9c931267adb55801ea Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sun, 19 Apr 2026 10:59:42 +0300 Subject: [PATCH 120/250] fix(efficiency): drop namespace filter on cluster-level throttle metrics Signed-off-by: Arsolitt --- dashboards/gpu/gpu-efficiency.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dashboards/gpu/gpu-efficiency.json b/dashboards/gpu/gpu-efficiency.json index e2e8c105..3bd59bc0 100644 --- a/dashboards/gpu/gpu-efficiency.json +++ b/dashboards/gpu/gpu-efficiency.json @@ -166,12 +166,12 @@ "id": 4, "targets": [ { - "expr": "avg(gpu:power_throttle_fraction:rate5m{namespace=~\"$namespace\"})", + "expr": "avg(gpu:power_throttle_fraction:rate5m)", "refId": "A" } ], "title": "Avg Power Throttling", - "description": "Fraction of time GPUs hit the TDP cap and lose performance. \u003e5% means tenants underutilize billed FLOPS.", + "description": "Fraction of time GPUs hit the TDP cap and lose performance. \u003e5% means tenants underutilize billed FLOPS. Cluster-wide — rule aggregates by (Hostname, gpu, UUID) and drops namespace label.", "transparent": false, "datasource": { "type": "prometheus", @@ -658,7 +658,7 @@ } ], "title": "Power throttle fraction per GPU", - "description": "Fraction of time the GPU was power-throttled. 1.0 = always throttled.", + "description": "Fraction of time the GPU was power-throttled. 1.0 = always throttled. Cluster-wide — rule aggregates by (Hostname, gpu, UUID) and drops namespace label.", "transparent": false, "datasource": { "type": "prometheus", @@ -729,7 +729,7 @@ } ], "title": "Thermal throttle fraction per GPU", - "description": "Fraction of time the GPU was thermal-throttled.", + "description": "Fraction of time the GPU was thermal-throttled. Cluster-wide — rule aggregates by (Hostname, gpu, UUID) and drops namespace label.", "transparent": false, "datasource": { "type": "prometheus", From 5e070840d66aa9a5da934db330a8ec16d5448e26 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sun, 19 Apr 2026 11:00:02 +0300 Subject: [PATCH 121/250] fix(fleet): count GPU nodes via DCGM instead of kube_node_labels Signed-off-by: Arsolitt --- dashboards/gpu/gpu-fleet.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboards/gpu/gpu-fleet.json b/dashboards/gpu/gpu-fleet.json index ea169433..8139549e 100644 --- a/dashboards/gpu/gpu-fleet.json +++ b/dashboards/gpu/gpu-fleet.json @@ -209,12 +209,12 @@ "id": 5, "targets": [ { - "expr": "count(kube_node_labels{label_nvidia_com_gpu_present=\"true\"})", + "expr": "count(count by (Hostname) (DCGM_FI_DEV_GPU_UTIL))", "refId": "A" } ], "title": "Nodes with GPU", - "description": "Kubernetes nodes advertising nvidia.com/gpu.present=true.", + "description": "Nodes reporting at least one GPU to DCGM. Counted via DCGM_FI_DEV_GPU_UTIL because kube-state-metrics does not expose node labels by default.", "transparent": false, "datasource": { "type": "prometheus", From eefb3651a0b8c16ce4b6aa812fefed3b58019e15 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sun, 19 Apr 2026 11:00:53 +0300 Subject: [PATCH 122/250] fix(performance): drop namespace filter on per-GPU physical metrics Signed-off-by: Arsolitt --- dashboards/gpu/gpu-performance.json | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/dashboards/gpu/gpu-performance.json b/dashboards/gpu/gpu-performance.json index e16f8c47..e581a76b 100644 --- a/dashboards/gpu/gpu-performance.json +++ b/dashboards/gpu/gpu-performance.json @@ -553,7 +553,7 @@ "id": 22, "targets": [ { - "expr": "DCGM_FI_DEV_FB_FREE{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"} * 1048576", + "expr": "DCGM_FI_DEV_FB_FREE{Hostname=~\"$Hostname\", namespace=~\"$namespace\"} * 1048576", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } @@ -617,12 +617,13 @@ "id": 31, "targets": [ { - "expr": "DCGM_FI_DEV_POWER_USAGE{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"}", + "expr": "DCGM_FI_DEV_POWER_USAGE{Hostname=~\"$Hostname\"}", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } ], "title": "Power Usage per GPU", + "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.", "transparent": false, "datasource": { "type": "prometheus", @@ -668,12 +669,13 @@ "id": 32, "targets": [ { - "expr": "DCGM_FI_DEV_GPU_TEMP{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"}", + "expr": "DCGM_FI_DEV_GPU_TEMP{Hostname=~\"$Hostname\"}", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } ], "title": "GPU Temperature", + "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.", "transparent": false, "datasource": { "type": "prometheus", @@ -751,12 +753,13 @@ "id": 41, "targets": [ { - "expr": "max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"})", + "expr": "max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS{Hostname=~\"$Hostname\"})", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } ], "title": "XID errors (latest)", + "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.", "transparent": false, "datasource": { "type": "prometheus", @@ -809,12 +812,13 @@ "id": 42, "targets": [ { - "expr": "rate(DCGM_FI_DEV_POWER_VIOLATION{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"}[5m])", + "expr": "rate(DCGM_FI_DEV_POWER_VIOLATION{Hostname=~\"$Hostname\"}[5m])", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } ], "title": "Power Violation (µs/s throttled due to power)", + "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.", "transparent": false, "datasource": { "type": "prometheus", @@ -857,12 +861,13 @@ "id": 43, "targets": [ { - "expr": "rate(DCGM_FI_DEV_THERMAL_VIOLATION{Hostname=~\"$Hostname\", namespace=~\"$namespace|\"}[5m])", + "expr": "rate(DCGM_FI_DEV_THERMAL_VIOLATION{Hostname=~\"$Hostname\"}[5m])", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } ], "title": "Thermal Violation (µs/s throttled due to thermals)", + "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.", "transparent": false, "datasource": { "type": "prometheus", From 14d9188fcda1a9780912f43d3b3ad3fd11f4c6f4 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sun, 19 Apr 2026 11:01:22 +0300 Subject: [PATCH 123/250] fix(quotas): exclude terminated pods from GPU request panel Signed-off-by: Arsolitt --- dashboards/gpu/gpu-quotas.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json index 624f072b..29e727ce 100644 --- a/dashboards/gpu/gpu-quotas.json +++ b/dashboards/gpu/gpu-quotas.json @@ -423,7 +423,7 @@ "id": 21, "targets": [ { - "expr": "kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left(phase) (kube_pod_status_phase{phase=~\"Running|Pending|Failed|Succeeded\"} == 1)", + "expr": "kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left(phase) (kube_pod_status_phase{phase=~\"Running|Pending\"} == 1)", "instant": true, "range": false, "format": "table", From 950c5dd6695872850e0ec87f2c3dbb28f8dbef52 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sun, 19 Apr 2026 11:01:40 +0300 Subject: [PATCH 124/250] fix(fleet): guard TDP division and document DCGM dependency Signed-off-by: Arsolitt --- dashboards/gpu/gpu-fleet.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboards/gpu/gpu-fleet.json b/dashboards/gpu/gpu-fleet.json index 8139549e..a7d3f085 100644 --- a/dashboards/gpu/gpu-fleet.json +++ b/dashboards/gpu/gpu-fleet.json @@ -467,13 +467,13 @@ "id": 12, "targets": [ { - "expr": "sum by (Hostname) (DCGM_FI_DEV_POWER_USAGE) / sum by (Hostname) (DCGM_FI_DEV_POWER_MGMT_LIMIT) * 100", + "expr": "sum by (Hostname) (DCGM_FI_DEV_POWER_USAGE) / clamp_min(sum by (Hostname) (DCGM_FI_DEV_POWER_MGMT_LIMIT), 1) * 100", "legendFormat": "{{Hostname}}", "refId": "A" } ], "title": "Power draw per node (% of TDP)", - "description": "GPU power usage as a fraction of the node's combined TDP cap. Works across GPU SKUs without per-model thresholds.", + "description": "GPU power usage as a fraction of the node's combined TDP cap. Works across GPU SKUs without per-model thresholds. Requires DCGM_FI_DEV_POWER_MGMT_LIMIT from custom DCGM CSV (see packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml).", "transparent": false, "datasource": { "type": "prometheus", From 43fe172d2f08c9bdf669473913cb485b4e6a6b86 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sun, 19 Apr 2026 11:02:05 +0300 Subject: [PATCH 125/250] docs(gpu-operator): document POWER/THERMAL_VIOLATION and PSS requirements Signed-off-by: Arsolitt --- .../system/gpu-operator/examples/README.md | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/packages/system/gpu-operator/examples/README.md b/packages/system/gpu-operator/examples/README.md index 9599acb7..7922d85a 100644 --- a/packages/system/gpu-operator/examples/README.md +++ b/packages/system/gpu-operator/examples/README.md @@ -56,6 +56,14 @@ files into a directory the validator does inspect and creates the [1]: https://github.com/NVIDIA/gpu-operator/issues/1687 +The compat DaemonSet runs privileged and bind-mounts host paths, so +the target namespace must allow privileged pods. On clusters that +enforce the Kubernetes Pod Security Standards at `baseline` or +`restricted`, label the namespace with +`pod-security.kubernetes.io/enforce: privileged` (and the matching +`audit`/`warn` labels if the admission webhook is configured to +surface violations) before applying the manifest. + ## Dashboards and what DCGM metrics they need Five GPU dashboards live under `gpu/*` in @@ -83,8 +91,20 @@ any CSV override. The three counters listed in the table — throttling violations and the power management limit — are the only extras the tracked dashboards need. The recording rules in `gpu-recording.rules.yaml` consume utilization, FB used, power, -temperature and the tensor-active profiling counter, all of which are -in the default set. +temperature and the tensor-active profiling counter from the default +set, plus `DCGM_FI_DEV_POWER_VIOLATION` and +`DCGM_FI_DEV_THERMAL_VIOLATION` — used by the +`gpu.recording.efficiency.1m` group to derive the +`gpu:power_throttle_fraction:rate5m` and +`gpu:thermal_throttle_fraction:rate5m` series consumed by the +throttling panels on the efficiency and fleet dashboards. + +The `gpu.recording.throttle.validation.5m` group additionally ships the +`GPUThrottleFractionOverOne` alert (severity `warning`) as a regression +detector: it fires when either throttle-fraction series exceeds 1.0, +which would indicate that DCGM changed the scale/divisor of the +underlying violation counters and the recording rules need to be +re-derived. ## Verification status From f8b99008737b7f1067d89220a517edbed77ef777 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sun, 19 Apr 2026 11:08:25 +0300 Subject: [PATCH 126/250] fix(quotas): use allocated recording rules to exclude terminated pods Signed-off-by: Arsolitt --- dashboards/gpu/gpu-quotas.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json index 29e727ce..f8ce4ba8 100644 --- a/dashboards/gpu/gpu-quotas.json +++ b/dashboards/gpu/gpu-quotas.json @@ -278,7 +278,7 @@ "id": 11, "targets": [ { - "expr": "sum by (namespace) (kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"})", + "expr": "sum by (namespace) (namespace:gpu_count:allocated{namespace=~\"$namespace\"})", "legendFormat": "{{namespace}}", "refId": "A" } @@ -346,7 +346,7 @@ "refId": "A" }, { - "expr": "sum(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"})", + "expr": "sum(namespace:gpu_count:allocated{namespace=~\"$namespace\"})", "legendFormat": "Requested", "refId": "B" } From a3241bf51bf255e24b77ebc23de89e33064370eb Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sun, 19 Apr 2026 11:08:34 +0300 Subject: [PATCH 127/250] fix(quotas): apply phase join to GPU limits column Signed-off-by: Arsolitt --- dashboards/gpu/gpu-quotas.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json index f8ce4ba8..ce1e387e 100644 --- a/dashboards/gpu/gpu-quotas.json +++ b/dashboards/gpu/gpu-quotas.json @@ -430,7 +430,7 @@ "refId": "requested" }, { - "expr": "kube_pod_container_resource_limits{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"}", + "expr": "kube_pod_container_resource_limits{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left() (kube_pod_status_phase{phase=~\"Running|Pending\"} == 1)", "instant": true, "range": false, "format": "table", From 2cc60f170c3c370a6939e2dfffc20185fb29d0c1 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sun, 19 Apr 2026 11:08:47 +0300 Subject: [PATCH 128/250] docs(gpu-operator): reflect recording-rule dependency for gpu-quotas Signed-off-by: Arsolitt --- packages/system/gpu-operator/examples/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/system/gpu-operator/examples/README.md b/packages/system/gpu-operator/examples/README.md index 7922d85a..f9094549 100644 --- a/packages/system/gpu-operator/examples/README.md +++ b/packages/system/gpu-operator/examples/README.md @@ -81,7 +81,7 @@ What each dashboard needs on top of the upstream DCGM Exporter | `gpu-performance` | Per-node, per-GPU deep dive | `DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION` | | `gpu-efficiency` | Per-workload util vs tensor active | `DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION` (via `gpu:*_throttle_fraction:rate5m` recording rules) | | `gpu-fleet` | Cluster-wide admin inventory | `DCGM_FI_DEV_POWER_MGMT_LIMIT` (for the TDP vs draw panel) | -| `gpu-quotas` | Kube-quota vs live usage | nothing (kube-state-metrics + default counters) | +| `gpu-quotas` | Kube-quota vs live usage | `kube_pod_container_resource_requests`, `kube_pod_status_phase`, `DCGM_FI_DEV_GPU_UTIL` (via `namespace:gpu_count:allocated` / `cluster:gpu_count:*` recording rules) | | `gpu-tenants` | Per-namespace tenant view | nothing (works on default counters) | `DCGM_FI_PROF_PIPE_TENSOR_ACTIVE` and `DCGM_FI_PROF_GR_ENGINE_ACTIVE` From 95ea20119e7fddc601addc9f79777d0e24aadd30 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sun, 19 Apr 2026 11:09:01 +0300 Subject: [PATCH 129/250] fix(gpu-operator): drop unused hostPID on driver-compat DaemonSet Signed-off-by: Arsolitt --- .../system/gpu-operator/examples/nvidia-driver-compat.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml b/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml index bad8609e..7706b221 100644 --- a/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml +++ b/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml @@ -24,8 +24,11 @@ spec: metadata: labels: app: nvidia-driver-compat + # DaemonSet rather than a one-shot Job: staging is idempotent per-node, + # must re-run after every Talos reboot (hostPath contents are wiped on + # reboot when the system extension re-installs), and the pause container + # keeps the pod visible for operator observability (kubectl get pods). spec: - hostPID: true priorityClassName: system-node-critical # Restrict to GPU nodes only. Without this the DaemonSet schedules onto # every node (including control-plane and CPU-only workers) and burns From 5e8194c850a889d03ba60e900c7e91d3f25b8e53 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sun, 19 Apr 2026 11:09:50 +0300 Subject: [PATCH 130/250] docs(monitoring): explain cluster-layer filter asymmetry in GPU rules Signed-off-by: Arsolitt --- .../alerts/gpu-recording.rules.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml index 7589f894..681ac3e8 100644 --- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml +++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml @@ -33,6 +33,24 @@ spec: # up, or rare label drift between the two sources. - record: cluster:gpu_count:free expr: clamp_min(cluster:gpu_count:total - (cluster:gpu_count:allocated or vector(0)), 0) + # Cluster-layer filter asymmetry — intentional, not a bug: + # + # * cluster:gpu_count:total and cluster:gpu_power_watts:sum do NOT + # filter cozy-*/kube-* namespaces because they describe the + # physical hardware fleet. A GPU draws power and occupies a + # slot regardless of which namespace's pod happens to hold it; + # excluding infra pods would under-report raw capacity and + # break capacity-planning math. + # + # * cluster:gpu_util:avg DOES filter infra namespaces because it + # is a tenant-oriented KPI. Admins want to see the mean + # utilization of tenant workloads, not have it diluted by + # GPU Operator DaemonSet pods whose containers hold a GPU + # handle but do no useful compute. + # + # Do not "normalize" by adding the filter to the hardware rules — + # downstream consumers (capacity planning, billing) rely on the + # raw totals. - record: cluster:gpu_util:avg expr: avg(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}) - record: cluster:gpu_power_watts:sum From f5f083e84102d8c7e743766774cd54b226fc6ef1 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sun, 19 Apr 2026 11:09:51 +0300 Subject: [PATCH 131/250] feat(monitoring): annotate GPUThrottleFractionOverOne with verified hardware Signed-off-by: Arsolitt --- .../alerts/gpu-recording.rules.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml index 681ac3e8..093a2939 100644 --- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml +++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml @@ -99,6 +99,16 @@ spec: - record: namespace:power_watts:sum expr: sum by (namespace) (DCGM_FI_DEV_POWER_USAGE{namespace!="", namespace!~"cozy-.*|kube-.*"}) + # Known gap: there is no lower-bound sanity alert for under-reporting. + # If DCGM ever switches POWER/THERMAL_VIOLATION counter units the other + # direction — e.g. from the current nanoseconds back to microseconds, + # making rate()/1e9 produce values around 0.001 instead of real + # fractions — the throttle signal would silently collapse to near-zero + # instead of plateauing at 1.0. GPUThrottleFractionOverOne below catches + # the >1 drift; the <<1 drift would require correlation with independent + # signals (power draw vs TDP, clock dips) that we do not yet wire up. + # Documented here so future maintainers know this is a known gap, not + # an oversight. - name: gpu.recording.efficiency.1m interval: 1m params: {} @@ -176,3 +186,5 @@ spec: version changed and the counter now grows in its documented µs units. Re-verify on the exporter Pod and adjust the divisor (/1e9 → /1e6) in gpu.recording.efficiency.1m before dashboards plateau at 100% throttle. + verified_on: "NVIDIA A10, DCGM 3.x — other GPU families (H100, L40S) may require a different divisor" + runbook: "If this alert fires, DCGM may have changed POWER_VIOLATION/THERMAL_VIOLATION counter units. Recalibrate by comparing raw counter rate against known throttle events; see gpu-recording.rules.yaml comments." From 16b2fd008ba7bc4db6dae122202884791a22887c Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sun, 19 Apr 2026 11:10:43 +0300 Subject: [PATCH 132/250] docs(monitoring): comment bats regex rule-name convention Signed-off-by: Arsolitt --- hack/check-gpu-recording-rules.bats | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hack/check-gpu-recording-rules.bats b/hack/check-gpu-recording-rules.bats index 4981959a..bae6785e 100644 --- a/hack/check-gpu-recording-rules.bats +++ b/hack/check-gpu-recording-rules.bats @@ -62,6 +62,10 @@ extract_rules() { # whole expression must contain at least two ':' characters. extract_refs() { json_file=$1 + # Prometheus convention allows 2-segment rule names (level:metric); this + # regex is tuned to the 3+ segment convention used in this repo + # (level:metric:op — e.g. cluster:gpu_count:total). Update if future + # rules use 2 segments, otherwise they will be silently skipped. grep -hoE '[a-z][a-z0-9_]*:[a-z0-9_]+:[a-z0-9_]+' "$json_file" | sort -u } From b5232bd15c55fb4a1faf25fa3f51e201a3002a60 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sun, 19 Apr 2026 11:10:46 +0300 Subject: [PATCH 133/250] feat(gpu-operator): enable NVLINK bandwidth in default DCGM CSV Signed-off-by: Arsolitt --- .../system/gpu-operator/examples/dcgm-custom-metrics.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml b/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml index 92064701..8e57788b 100644 --- a/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml +++ b/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml @@ -70,8 +70,10 @@ data: DCGM_FI_DEV_LOW_UTIL_VIOLATION, counter, Throttling duration due to low utilization (in us). DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (in us). - # NVLink — enable only for GPUs that actually have NVLink (A10 has none). - # DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL, counter, Total number of NVLink bandwidth counters for all lanes. + # NVLink — DCGM silently drops this metric on GPUs without NVLink, so + # enabling it unconditionally is safe and keeps this file reusable + # across GPU families. + DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL, counter, Total number of NVLink bandwidth counters for all lanes. # DCP (profiling) metrics — supported from Ampere onwards. DCGM_FI_PROF_GR_ENGINE_ACTIVE, gauge, Ratio of time the graphics engine is active. From 4d9a61a0ec49538c70d7b0e3c7d3cd37949781fa Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sun, 19 Apr 2026 11:10:47 +0300 Subject: [PATCH 134/250] docs(gpu-operator): document native-talos service-monitor interval Signed-off-by: Arsolitt --- packages/system/gpu-operator/examples/values-native-talos.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/system/gpu-operator/examples/values-native-talos.yaml b/packages/system/gpu-operator/examples/values-native-talos.yaml index 86e436c4..5a3d7786 100644 --- a/packages/system/gpu-operator/examples/values-native-talos.yaml +++ b/packages/system/gpu-operator/examples/values-native-talos.yaml @@ -39,6 +39,9 @@ spec: dcgmExporter: serviceMonitor: enabled: true + # Matches the 1m evaluation cadence of GPU recording rules with + # 4x oversampling headroom. DCGM_FI_PROF_* counters have + # documented overhead concerns below 1s; 15s is safe. interval: "15s" honorLabels: true config: From e148343fd9bf9440d726bf01ebb07374119927d0 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Sun, 19 Apr 2026 19:01:53 +0300 Subject: [PATCH 135/250] fix(kamaji): increase memory limits and add startup probe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Increase memory limit from 500Mi to 512Mi - Increase memory request from 100Mi to 256Mi - Add startup probe with 60s timeout (12 attempts × 5s) - Increase readiness/liveness initialDelaySeconds from 5/15 to 30s This fixes OOMKilled crashes observed in production where kamaji controller was being killed due to insufficient memory during startup. Signed-off-by: IvanHunters --- packages/system/kamaji/values.yaml | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/packages/system/kamaji/values.yaml b/packages/system/kamaji/values.yaml index b1ca3cb7..e6dbe99b 100644 --- a/packages/system/kamaji/values.yaml +++ b/packages/system/kamaji/values.yaml @@ -8,9 +8,32 @@ kamaji: resources: limits: cpu: 200m - memory: 500Mi + memory: 512Mi requests: cpu: 100m - memory: 100Mi + memory: 256Mi + startupProbe: + httpGet: + path: /healthz + port: healthcheck + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 12 + livenessProbe: + httpGet: + path: /healthz + port: healthcheck + initialDelaySeconds: 30 + periodSeconds: 20 + timeoutSeconds: 1 + readinessProbe: + httpGet: + path: /readyz + port: healthcheck + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 1 extraArgs: - --migrate-image=ghcr.io/cozystack/cozystack/kamaji:v1.3.0-rc.1@sha256:8c9af083b60600c0800eb56a2cda75f26007b7272a1cf019140de003bfce1a4d From d94f937011cca3981e2c8364da533a6c6d249663 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Mon, 20 Apr 2026 13:01:23 +0300 Subject: [PATCH 136/250] fix(linstor): increase satellite startup probe failure threshold MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The default Kubernetes startup probe allows only 30 seconds (3 retries × 10s) for LINSTOR satellites to become ready. This is insufficient on nodes with slow storage initialization, causing unnecessary pod restarts. Raise failureThreshold to 30, giving satellites up to 300 seconds (5 minutes) to complete startup. Signed-off-by: Arsolitt --- packages/system/linstor/templates/satellites-cozy.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/system/linstor/templates/satellites-cozy.yaml b/packages/system/linstor/templates/satellites-cozy.yaml index c621126d..1e8f8f72 100644 --- a/packages/system/linstor/templates/satellites-cozy.yaml +++ b/packages/system/linstor/templates/satellites-cozy.yaml @@ -14,6 +14,10 @@ spec: containers: - name: linstor-satellite image: {{ .Values.piraeusServer.image.repository }}:{{ .Values.piraeusServer.image.tag }} + startupProbe: + tcpSocket: + port: linstor + failureThreshold: 30 securityContext: # real-world installations need some debugging from time to time readOnlyRootFilesystem: false From 8ee6ac96d400e8345141391d45e7fc4a5c28c87b Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Mon, 20 Apr 2026 13:08:44 +0300 Subject: [PATCH 137/250] fix(linstor): explicitly set periodSeconds in satellite startup probe Make the 5-minute timeout self-documenting by setting periodSeconds: 10 explicitly rather than relying on the Kubernetes default. Signed-off-by: Arsolitt --- packages/system/linstor/templates/satellites-cozy.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/system/linstor/templates/satellites-cozy.yaml b/packages/system/linstor/templates/satellites-cozy.yaml index 1e8f8f72..e6f877a0 100644 --- a/packages/system/linstor/templates/satellites-cozy.yaml +++ b/packages/system/linstor/templates/satellites-cozy.yaml @@ -17,6 +17,7 @@ spec: startupProbe: tcpSocket: port: linstor + periodSeconds: 10 failureThreshold: 30 securityContext: # real-world installations need some debugging from time to time From 83ae237000149462829a4841bea2dd52929b5cda Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Mon, 20 Apr 2026 15:48:53 +0300 Subject: [PATCH 138/250] feat(monitoring): upgrade victoria-metrics-operator to v0.68.4 Bumps the vendored victoria-metrics-operator chart from 0.59.1 to 0.61.0 (operator appVersion v0.68.1 to v0.68.4) via `make update`. Picks up upstream bugfixes in the v0.68.x line: correct VMPodScrape port for VMAgent/VLAgent, StatefulSet pod deletion when maxUnavailable=100%, VMDistributed PVC ownership fix, finalizer cleanup, ingest-only mode not mounting scrape config secrets, STS recreation on immutable field changes, and PVC resize completion wait. Chart 0.60.0 is skipped because it introduced a matchLabels change requiring Deployment recreation that was reverted in 0.61.0. Local patch disable-ca-key-rotation.patch reapplies cleanly. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- .../victoria-metrics-operator/Chart.lock | 8 ++--- .../victoria-metrics-operator/Chart.yaml | 13 ++++---- .../victoria-metrics-operator/README.md | 3 +- .../victoria-metrics-operator/RELEASE_NOTES | 8 ++--- .../charts/crds/crds/crd.yaml | 29 ++++++++++++++++++ .../charts/crds/files/crd.yaml.bz2 | Bin 27612 -> 27506 bytes .../charts/victoria-metrics-common/Chart.yaml | 7 +++-- .../charts/victoria-metrics-common/README.md | 3 +- .../victoria-metrics-common/RELEASE_NOTES | 6 ++-- .../templates/_helpers.tpl | 24 ++++++++++----- .../templates/_image.tpl | 4 +-- .../templates/_pod.tpl | 8 ++--- .../templates/_service.tpl | 12 ++++---- .../templates/NOTES.txt | 2 +- .../templates/pdb.yaml | 3 ++ .../templates/server.yaml | 5 ++- .../templates/webhook.yaml | 19 +++++++++--- .../victoria-metrics-operator/values.yaml | 19 ++++++++++-- 18 files changed, 121 insertions(+), 52 deletions(-) diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/Chart.lock b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/Chart.lock index 1b0e2a46..2868e66f 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/Chart.lock +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/Chart.lock @@ -1,9 +1,9 @@ dependencies: - name: victoria-metrics-common - repository: https://victoriametrics.github.io/helm-charts - version: 0.0.46 + repository: oci://ghcr.io/victoriametrics/helm-charts + version: 0.3.0 - name: crds repository: "" version: 0.0.* -digest: sha256:43d3d210a9d1a2234e6c56518f8c477125a5ad5e8ed08d46209528f19acd9c89 -generated: "2025-12-23T12:58:01.428960334Z" +digest: sha256:adec954bf2814f744f5b69fcf67c8b246bf21f416adb0ab37d91c9b42f72d353 +generated: "2026-04-16T10:14:10.795552631Z" diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/Chart.yaml b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/Chart.yaml index bb9253b1..030af8f0 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/Chart.yaml +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/Chart.yaml @@ -1,7 +1,7 @@ annotations: artifacthub.io/category: monitoring-logging artifacthub.io/changes: | - - updates operator to [v0.68.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.68.1) version + - revert change in Deployment's matchLabels, that was introduced in release 0.60.0 artifacthub.io/license: Apache-2.0 artifacthub.io/links: | - name: Sources @@ -16,13 +16,14 @@ annotations: artifacthub.io/readme: | # VictoriaMetrics Operator Helm chart - Chart documentation is available [here](https://docs.victoriametrics.com/helm/victoria-metrics-operator/) + Chart documentation is available [here](https://docs.victoriametrics.com/helm/victoria-metrics-operator/). + Changelog is [here](https://docs.victoriametrics.com/helm/victoria-metrics-operator/changelog/). apiVersion: v2 -appVersion: v0.68.1 +appVersion: v0.68.4 dependencies: - name: victoria-metrics-common - repository: https://victoriametrics.github.io/helm-charts - version: 0.0.* + repository: oci://ghcr.io/victoriametrics/helm-charts + version: 0.3.* - condition: crds.plain name: crds repository: "" @@ -46,4 +47,4 @@ sources: - https://github.com/VictoriaMetrics/helm-charts - https://github.com/VictoriaMetrics/operator type: application -version: 0.59.1 +version: 0.61.0 diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/README.md b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/README.md index 49a06de2..6a080424 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/README.md +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/README.md @@ -1,3 +1,4 @@ # VictoriaMetrics Operator Helm chart -Chart documentation is available [here](https://docs.victoriametrics.com/helm/victoria-metrics-operator/) +Chart documentation is available [here](https://docs.victoriametrics.com/helm/victoria-metrics-operator/). +Changelog is [here](https://docs.victoriametrics.com/helm/victoria-metrics-operator/changelog/). diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/RELEASE_NOTES b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/RELEASE_NOTES index bf22fd3f..3529783a 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/RELEASE_NOTES +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/RELEASE_NOTES @@ -1,7 +1,7 @@ -# Release notes for version 0.59.1 +# Release notes for version 0.61.0 -**Release date:** 03 Mar 2026 +**Release date:** 16 Apr 2026 -![Helm: v3](https://img.shields.io/badge/Helm-v3.14%2B-informational?color=informational&logo=helm&link=https%3A%2F%2Fgithub.com%2Fhelm%2Fhelm%2Freleases%2Ftag%2Fv3.14.0) ![AppVersion: v0.68.1](https://img.shields.io/badge/v0.68.1-success?logo=VictoriaMetrics&labelColor=gray&link=https%3A%2F%2Fdocs.victoriametrics.com%2Foperator%2Fchangelog%2F%23v0681) +![Helm: v3](https://img.shields.io/badge/Helm-v3.14%2B-informational?color=informational&logo=helm&link=https%3A%2F%2Fgithub.com%2Fhelm%2Fhelm%2Freleases%2Ftag%2Fv3.14.0) ![AppVersion: v0.68.4](https://img.shields.io/badge/v0.68.4-success?logo=VictoriaMetrics&labelColor=gray&link=https%3A%2F%2Fdocs.victoriametrics.com%2Foperator%2Fchangelog%2F%23v0684) -- updates operator to [v0.68.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.68.1) version +- revert change in Deployment's matchLabels, that was introduced in release 0.60.0 diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/crds/crds/crd.yaml b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/crds/crds/crd.yaml index c4138be8..bdf7e8d0 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/crds/crds/crd.yaml +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/crds/crds/crd.yaml @@ -6079,6 +6079,7 @@ spec: - spec type: object shardCount: + format: int32 type: integer startupProbe: type: object @@ -6087,6 +6088,14 @@ spec: type: boolean statefulRollingUpdateStrategy: type: string + statefulRollingUpdateStrategyBehavior: + properties: + maxUnavailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + type: object statefulStorage: properties: emptyDir: @@ -12326,6 +12335,7 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true shardCount: + format: int32 type: integer startupProbe: type: object @@ -16808,6 +16818,9 @@ spec: type: boolean vmauth: properties: + enabled: + default: true + type: boolean name: type: string spec: @@ -24727,6 +24740,14 @@ spec: type: boolean statefulRollingUpdateStrategy: type: string + statefulRollingUpdateStrategyBehavior: + properties: + maxUnavailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + type: object statefulStorage: properties: emptyDir: @@ -28628,6 +28649,14 @@ spec: type: boolean statefulRollingUpdateStrategy: type: string + statefulRollingUpdateStrategyBehavior: + properties: + maxUnavailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + type: object statefulStorage: properties: emptyDir: diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/crds/files/crd.yaml.bz2 b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/crds/files/crd.yaml.bz2 index 7f518a91e11a4af367dd2ff450bc953ddc650964..ab7c4d95d1fa275eed967afd607309920c09a05d 100644 GIT binary patch literal 27506 zcmV)MK)Am`T4*^jL0KkKSq!65qyn;j|A19hRaI60|M0)>|Np=L|NdaJ{^7_4J@aTc zU=&j!q2BLXJ@>J8r`&w@o~NVNyUyrD8%V8H%QtR{6SlqV;3(|so3yLm!R@aAVAcb$ zaqk>=buM!mo4MOG(C{7SxHKbebRvh3C)YB7)`WZD1bsNZjqU>?sP*p3L=+UDbnEDK zK~zB&_VjsuzE1aUH@n?md+zd{-(J@A>+gi# zm2bDRdEP36opqF*U0+>#U3VXDShMK*6#Bisd$WD@-+SiK(aRi zRZ$o`4_nEH-#+lt-p;}gZ$95Q$I#ziUspKl6W6_>_2%(<-MzP+Ev>fAcDlJITHS9x zKKQSEU=A;>?+eYiI`QfC$U9Qo4~?+8`SZJVzGW*~0)PTe?01R{2d3|K)3|M&-hD3b zfJxW2zCK)!w>-S&_UEU*x8DzUUc?HLB%QXQYTHe3$o`?e@O&T-->H|OwgFp(Jq)1a!CzOq*Q%umCL8r^pn)odYjZw8X8F2hJ#|Bqe1QURIgZ}b1c|G&WGaEVDJg%WjB^(sYAOZ<>Lif~lgdbtS+i$;29^D1LVFa|%$ zh4_fb$jC2MHF6;lIx;!{VAcu$7(x(|C8~K9C_vOC0@Tb!JvDS(&BaSpyqU>**>{?` zbaK37f}|n5Rr6|0g^#nPo34aZmm~;sWN2_P2un$DEG#_QHAyT&A#b(N$Tk}!B#2cw zLrD2-NGZs2<&29Et{b;v8w??G-M8NCnJco$oQI$|VJuddmtcCB1<7&syJp17WjD|= z38!YwBaz4j2+1NwJi?YPgK~w7n1@v-nodq>L%33K&cY$vp``3=Bf-m@%abm!(n44S zV;Q_+SOp=Sg}O6DCGG@r*^NpE0NGCBNN0C8bG>;TYqjIaOUoiwWQoY)n~Q`Obgik| z10;xax=51ha_V+&oNOJP_)|rw)2u1Ty&m}x5y2Rx|ansC0a4C+1 z5rKm2h&xZM8U`Hd7l%kBvH<;B~HX*oE+$npjrF%yOHx!>1n*!WkWO=l)&><~otw*J2fL zC)TrWFS#Y zxiWO06wtl@`+026si>Q*iwvX2j0$*sO3A`ZB|eElK_-QPK`vEp8;E@1fyv$GNu}#{ z8j5o{r5zCNs~mJgKtM8RzX4?dik@s16uDG;8vk~SP>67p8hXSq9yIfhtX?@Y5D83` z8KEYx7b9qFtcancB7xl5K$i5xF;m1Kwm{j@XHnpRRw2^%we4;&1F=&8AhLvNri29Y zAaI(l=HaCGrcO=iaENvQ=1_xL3_=7#jUtTkj<|*1t=}5OOs-tA+162&m6MdoWo2`` zlVvZ0+fb2D=Bfn#5YyvjY%c6iv~Y!|-X&R1)n+RTwTY}tzZ_s$z-vjMU{DxCSaq!& z*cT<_C>X&?d@?G$nO*A2<5SMXUNZ-q3*fc!V6~3)`R$uK^JSEN*<)D;CLCc`Zc|cs zwVpYN}5+m|#MQrU%Ou=JN2HW_cQq zG@Tykp|>HXQ&9$EwX4Jf5L$rul`%q*e9XUeZ__01} znRM)M-ly->jp3IR%e!52Y(a=six0zsGoLB5LZC0gNeHU7-KgBzGFeSzY{YYulC^N+ zDwYdSh(Qgf5T3YYyqmyD%)TL`=4=TdNQ8+5;oFfs+v`e6AxKqn}NoHVFDX+wBw7jWr^lNMKYGsfk6ZeJ~=KRJky*v}!di7^b7QxFrJkQ2hzk1nFnXvs`{BJ??Y{y2V6;JORsabQQ;X<_?c62IVWH_ zn8<}AE&4p!iF7`K(FDlm+8>sqB9s}MY~JZJ+db_7=trEfQ$c{ExC&&lZ6*}@Cdq{? zliMO>g}4(E1p|=@htd$>8^{V_Vb1w?B+cZ~pl)X^vAYbFS)7r)&Ii7j$sWRT&R;jV zA{^~AxaH0u4bEUID07HGb8O>9S*oid78?zee9dOLA-}5&#F7}0zRY|!B+kVC*46F; zxB$~rl~5J&F2ch31Z+PJwC;m3GER#OOaP+{yTmavbMOQ>OOE0zE>Zq;ktm3CC_~JIfHRhXYt+xa11W z${gc)9zldiLPB9gC_pZ7T{|A=>2&sX*1@9*ldE$&#%lp_ptxO!xuSzmOa^Hm4WYF> zlZ_?8jbdz=S~u1m?)bqya;CBux-nZBQp^T)(r=Ypfq1YQijpWJzL1>ac)PNvaRNZN zGZn!b1b7-i9td1Qtq90*2-E29^9 zqH=xaTo8~|fRG{)D+(ll!O~*|69zsgqX#I2w{X^2Yu2Ou>;6tqcG04x4H_UhAy>O3*0NB z^OnuR+jk;#ds}$D@+^;zC3C{?!n=I?>=W8ml=x(n%q{}>o(=k>_J9`rn)k(>R7k)u zBhv9j_RC}vNWngH?E5d-57_l>JZ4<^Y{b8=RVpcQ1wEis0^^nk;vxJ*^b&PRvM~vG z-N+yLQ}l#;!T9K;>J$)P0_Z!pxl#Inz3P#BgU~q|XefnU*CX`w^Ar+ZWFV=m$JUBuozc3!X@~#X9!xHH)66h3Qvk$>=zWoI*lk zr;OT(`GKT`Lt|7X;j&o%XjjUPs8BuF?k1$6@w6p^ZczCW!3{&GU6~MFm9*h3O+M+E zgj1Rn%K#hZ;n_&DT56hHhm(?G7Iem;Ifco+VvES2nG=)?!UZ23JTTrB4f8kN9FaKD zwG8D$c1T48hapLUXxi35gxYs#s&Y$CIObeEWr!UToK9)5%^|r?GeJiPcg;0+-oV{T z4iGsc#7v|XhSj0DBrb<+Ol}y2C#@+-$HqCiq>$zZDI`sV+$-6$(aKKvK5iK(L!>2a zc!lN-LOGXrP|AiCF$#GxO_-Wc%*|#SvO9IG97B-K~ zp5v9qY6Ot-W{g7wTRR^olUcK=P#fGlk2@9gVdaFru7<2I4izya&u1r+0C|gv=Cw{| zqb0*RqU^y|{h1ipofLOGxyjJHV_J{N=cLHU$bxDo*oT;l>TVf^eW@vm--MzZq!UC5 zFYlW_74XjS)H)cUAOYB|@VtRJcAygrXA2;D@~Nf2HX4_*{@bEOn7X+2LGaZIDtAgR#u*0sCGWV8#}_ zZ3JJG6-_V*{EtoftXWfaX-Nte1(HCtB!rjP6d~wKQ;;!OCmcN#!wB~A!MYWKX{#Xv zP|+=DhR0)~#0<#^M;n_gswAlvr!IMSHZ^QJ!{d>rQL9;Y+M8r;7{=?2k3S9D9#5}0 zy8b_s_mI|W)+>~Rmo<-W^n{3DaEU)%m@8MEndB&`dyY)KVf*{wyp&di>cq2i7cgZh zw-m6F5}XgB9!Oko6_BCgVj=m8A2A=|AG#1^Q1b!IKEIB_&ojyr!X98EZV!luU?~GL zvq!;>&CA&shZX@(3q$HEOdQ1Eh)nF$9Ajz|O+WT^y^We;9rjz>x*l)_*yrK8w$L^lwS z(gZkE1UGYcMq9`nvZOia!-13~<;KE_5IoYmN?EZA@Rwj|(kBqjnp1fPVGzh%)G^TM zOv7eE^8-d2z}?}Y!Q~l7XL?{YT^hUs4%!kVAtVQCzm*~PGIMrjt zUh(Im-;A4@;gRK-GIjaatQ2I1Wtu~};j26v_cvB2FwirUG{HLi4E+cX;fj(fgjNCp z&m;*D4a=*Rl@1jNPw2(~^a^-B4?(q4q5dK5CX6!b{qdVkZkNHAGaAdQ5Z#H*kU7+C zHS$&jeHkc4_9tN~1u+NkWGmhRFtr+Jw0Fi>G#|d*LA{BMprWixR0YKv*6E48zmEkFTFgOu8H3$W8oDK$nLA8OkF(7tv zrkM2U2xGwvj2(n9g~Fc#9L^Btu8eayXzvHOI3_V86p)c1LTgJiI(KAiN#aw+XtdL% z*_d^r%P4_%!a=ct2Fra}g9ZW?o1%+y6a^1Pj;+IkvdSU6A#UM`>j7E8RdbNIRQvYz z06xe;5=4MRdw{;X?)$n9q6I{DffarroPsq6?+*|~Je&v2F0NWsJQYHH)FKfb^sf7g zj&hFXGQ)by8q{0h?UM41qssJaadzWWWX`*0lQ4R0YAe4@8nCMvP8v8dHc^%ww$99> zt|TnoU8h3qbDKCP%AtoLK_ck@P@$&O(20U1R>KHh6dwlA*oJh&1Ez_g*Hec^Ch4hx zC_&~%aORU2uzN#;yr2TB8JVH*|<=*WC}XB1Ir5!d~S~gvG7cm z1(wm#*SXj};{{+56p50i#No!1$zo)=m387*DvgPd+b;Ykc{Wb=#B#@0K+2Hb(QL?x zVazy^w&4{Efefw>LFz`ZlR7}8q?1Y|Nf3ODR1Aydh8?9j4Z~5G!=234 zOO=(}w9-LHp{leX$j}ys4vyhjN$ZG0l5c1afFh}+PY;1I;b|lx!UMtr9|}qp3}H~j zr1qj68%A}cgwQL5cyUvBS{NB(4o+1Kv7jZ4Q$4kj>#c~mPM(C}f}?^f1~aL+od;Le zgmODh7pUNi4MC(y)`s&85FBEK1N`MrrA_=puKTV}i+bK|b8~bT_{I6DzieDWuMucH z(Q1);1*Ia1KI)>1Rw%Rxz<{hQRZ7CjlM6&NERis@lvr35>4hs&9ki=K1#&S`iwjb$ zqbjB*P-Rn6($H3glv=W(t5rT)6VLUtzsoZ)G`#zLVR1d*(4Y7&yN)k=V?ciLrzm7H=_VP>U7v^jD_t8%Pat2IpM%Hmqf zR;I3tE;tKCoKuTvb~QN7nzYMX6JtxGO&nFq(!|bOGHRT$vRYQfiE7ARd#2cC<{PCp zZ2?$WwNn-r!IqkCwsPpIWa3$2sPw=MsRAhQhB|5ym!0C%L`abYAxZpxr*BnrJ^xHBp}2v zcw>>YY)GRSu_@l2CDue#>B*ZZmdx8S!c^3;s|4}Z8JnXASw@POWmLpgLb9eARWMkA zl?#J)#=~)zI17n6Cap&)%M{L5R;*fPZ$#I!aukBzrH?xNPn<)hV`qoH;f^_ z@QJZDZ+HU&V8$@;H%$dfLqcfQtZfVC?$ElVD`8~V8WPO!IdYjYCUrK4Dv+$3c9p$z z056DnY8O>MUgfsiZMNHOw%cvC+ikYnZMNHOx6&Sj@b%30J_m$U>A&jdd_%?6Z^PJh zCqtj{A!v2X)j5Tfa6BL4>`uh+dh59L{oz_X7I+T^F6yd^D64LPE<*e7DXzMypp-=v z7GfT!x!>*ln_HF}7~oHgz5a2`#*R9+?YWB9ar4(!6R`>qvxm_->!(rNns;*biNw2d zL$wZNU7&PS=qcJx(sqgKPg*bHp3!?p_@Ho8`xl7uuBx>ibVEQi(9kn&Ov2MQVN-lC zz{28Sn4<~ytqM@4jAY3QSRkaOIVTP-S0^LI)1ofwI@Mm%?vdDTRJdD^Zbsw`lrkK| z%nZQXGs_#Ll-XmBG@?-C6b)%hPD7KBhDJykA-ZlmY&_xW7tqprEYdy$D<-Ahd|mA($C~JoE2%1$3C0;Zmmvs+wpfnrTS3(7Dyueeo?T zHn|I^C@3$$%Qjii8U|r?*2+?-nrW)C%k?U%t!>ti;5Pxf7g?2+VyS>&1i__MROaJl zROM<+tjY6Li_fDuy7~8V6epq9khD79a(WL(qtSAus+QWgiEy_!0^AMDb9L$#g5*bJ zyCbr@QtYeB4*HjTchoNg&>kxAc${DM{yQja3$P)uj=~!YvW~*LAR8(>A+n*cCcx~j z&c@E$a1IX+8|kn>n(CUNpqdVWpl~=>$GZFHlO{~0Uwe6ZMHJ~Krl&+RLpu3kN5lkI zs-MiK#Gqfnf0%zg5q=dvJwbk`FaHXJX-HqT3kp=qg32P^pub>_5pNk8VQ68MiKSxO zWMyHQg|(SjhD6O1Oj(tiRwCLgvddBu6|*Z!NVT*i2~x_Eu{D;9M4BsTtxFAKw3=I6Qfoz%CWyAE)sn3iB}S68(rmOWu~90uV2vb7NQts0NVbbul4zwRw9+Q2 zq_)(w*)+;a6q8yMtSmO!YZ)-73RCZ@e--}!gjb>$NRZ-rA$1oA`T8m47fF5>L2{R% zSr_NJ7xLoiQp)r@+46AJ4Xk2SV+2b9P@s_$*cVC4uSfHAbbZg&3$&k?uwR95#qAdg z{t^GJvU~L*qn9EFaKC`NxzeXK75BalOk*oEGHfR1A+=B7YNi+VRN5IR+{%(~Cw)xo zQwG&Mk@Mwy9XM4V!>RjqnPz6X3*V(C7an{2Mb%5vEJ@pikQo)waEb}#$Q*L?sgc?$ z+55$%J4Mu7U*RSnT@_Q}ioIgO`6tNeE`sP@sazIa1@jBAU5LL~acZ{G%$nyFiH1?~ zkKjKgo^{-9t|N^jOk<@;->G>?Ep4iq?P6vgmiOa$#@udKVQ1~ocbZ~wuP8V=h4QNL z0`ia3Z%?)|$6@A*5(jDh{z7RJEWqpz&NZze)^KCoDG8kNR01K`d5QHTi*$Rz!WRR` zRpJ4_|P|0tk-8-q-IBRnn zX)?qq5#W&eiU?c>6{~dw9?*(5!~!^ngQEHIuapkoOb>4lv6g_)lpP0euRR58cK5Ku zJdxvpC_CSsw(%h1Fy1w-_wTO_Cxmg&y!163rPOm~2lV>0%kG6oC}{PUTFxW(LY9QE zq>VR?uRd+}y&OjZsdN_W3kOX$;nT~JXu*u0hB{1Pdy`c) zj%Y;#)%e*X=O~l9P|$=I7eJnZe5vA;JVis%H{<#(3xVWGNW-9kokzJHqg3+q4M&!5 zhmLUVySDyn)L?Kp7IJp=QuC(7mD39&e#}-Sk8{_7GEE zxknqC^;R_<6`J*IRijq7_8{;jWfXEw-)$p$U3o8S!Pm?X@!xg zpQU;#(=fb!SWsvR4IdfaWMW;hy#I9eA_PPq2aE4$r^q z@8s@xKM--wb&l_Ex)(#OP_54{*u@tY1H zBwX+>-_Uh{CNT`k-1en{?lB9bP5bt11}$FFAi`My5VkM69w`k3Pag(@u*2mbebn%$ z5cPv_F&b0hIfXEXwd_m{M65C=W9ow-kwcMBf#nCNIu|Sb*+L#484NM9d;<#%l-QZ1 z@MR8!C&^+~QGg+~30eznOl=x1j5{YqbxpLZ6O5MJi{4My23estCU_m0PSA*2qIJs^ z!XyUIUrEl2L*!qTMo!wL3J;M8F9RaGWji~~4H-(ptYmla-)fjmu`ouMr^G^TfO>%3 zK{-5pQUdZSwN>r&Lswrx-;nkXam*&&Z{KxP=1Up~2H<1fs!Fe3OAPw~txTLlhm?~y z=f?Q)^4r&33xVUvD?Cf#y73KunB$F~9a&&zQ3huhWU7$h^`xoQP=|k<33;mC`Hr(v zUNdX@R?fM~^Zok2Q_r6lpMJaQA){K&-##4Pkk}te6TSkx!Se}aGii29%k!JxG;<~p z@)<)WAWo3TLv&d&@9mDeP#y1lG@F7H-oE`GWF;Xyk7DGiXpoxKTGcwnt|qj3^e-Vo z6Rxkur4RUFK@mD)hx3{!LqtIcKr}JHx>G|57~v!|wm}3KFojTK{C2kZe){jKkcdhC zO@6-Eo;75UXGfY!Rs9>IiRD&kF%x^5pLYw8f!n5^kFUdW7)WmvG$5XF>nE6A2x{0@ zq9OZFK@j#5FKkDAcv+Zm446D63cZ1|t&S2Lh|+z8$6K$3J>Ox&;jen}6nml??)rV+ z+xwo78j=ezqX$qEs2q#X!9Fbr%7DZMD>+o4N5^v-z{W4MNvZ%nh>UYmZmAtQqM`^K zi_WDdmHturZ}4_fc^DKhhp02JC746AB`&^4RnzR`4??H6MzG4sah=#C}7?eqx6~~O+?~%sDCa!x7^691- zM@sF&>7a6K{rT(PQ_n1n;)k(JD{WyRR5%=r`LiNUA&O7i@>KIcWek<~Xue^=N(nJZ z^xg`NJUBidO>P0RHKQly(}kXLd6(qw-(o^xs{QAfB|D#s+vCmQowO`isj53QCeMyFmMHAW8nGW$c-yF&sjM#k*? zj)TIjkWrrsj8isJyH~-&m9o?z0YV2nR$;qqB1x^%*cQkf_XfO5-)D3z9&9Q6AqX!Dpc{;JzF} z9?nc)(3uj)$?j8=2Zq&x!0*FsdUqvx6P0o>?kuK-UfQDA!KED!k`yX?arL_kb_drg z!(tHmH)Qacu{+pOctBzEzhI0ayDEkEAoN`)uzM6}aCVolQ^HY8F_D8oJQJQA3{@hX zjvfm72O^vT1EC0lU?L+}4b?a8<91-oH)R)HBvZ9JFdHKHB?m!hbQmiEkg%Q(Am2=O zgW9-EFg64eBiU49kxc{uEY%2LZi6ev(UA^`P&#@GF99GVu&}9(2-uqth6&JzPFE~& z2SjNE!2!e!A*UoF5(I&^j?3&0ilCS_AW-nLAarCE?@PxmaROyvaRUHt#m5W~vJV>` zwx!`y0i-tBR??d$O&Ez*CQ~wyyqnPBGCGy58rBCe2hyjdos|*ET>@(Hi&K@YL77%% zD`GRqE~(}hZPF^oMsuVtij{O>bb_Il7a+1^D9Aw_^)+o90-B=Bo(*kAR!uaE zTE_9r+?yteHt4a#Yc`^7Sy-r8wKkz-wwozwIjH2)YQ?mzSX(MxN@AzO=WVjtZML?_ zB$T$>EtcDDw%cvC+ikYnZM}M_`u{iU6Z`_Of{b8BN>zP(eT(UA+x-;ysph-9sXx_I z6aIpf8Xrm34#af6k1D8?rEJqpG%^RJDxV+w5=lRKRKRS5hDdhQ_*lOi^!b=}RE z$giF^M=~%jACtl^E^$1Y1t~=69V>D{9=_+}~o` z+k>F$9Omz;QwmczGG>h&9u8(qg5P|OsQ2O5l$+!4X^h5KPyW6lRm7ghwNuWc#4evQ zV+;(x66QbB!Oo-dBhsB@>gdNW{qK`9Ne{@TvU-e+oNf{UcKiHYw|8yc3of-X z__nO_?iO_9k7_tgQ$|zD9$tIS+&v}1jOTebiD@wq%4n3OlSB7{E206PRaS?uRHZGo zt!YY9l=Pow4rG90Egs43jUb=h(D!M@Ctp; zZhU^K?%mpKHg4DNMnXgA51-k-e8k8GA%N`bfua!zJKkt%E>Pf1i=f;L!I?p@mW9Z3 z=dQBo(i6Hm?-}^!m#hslG)_^5XPniWk+7W|sNN}D$Bseqd3r~~d4y*Zw#=q}o$&6UrGY|U$0 z>R%mS_FqVP6tEFr$gESZw&+rK3SkLGG{!Q>2$8Ea?7oL7Znf zp}gpgQA8q?d~sLY$GP&D`goYJDNSgRIp>@45f5#Wm8~gCG^Ip5Ku|gJH{jfIIY>aW zFPf*QRPI6Ssq<9x>y5p{3hR<4AY%j}3Pf-T+IrJdJz`BHH9~OL1i>&aaE&zF(2J;S zW4ddgVSr*7CYn;3Iy**377SF*rq^sDOB*E3Z12+k~1_j zGb-TSJKX8sHK8&X6vaWv%qSbXfcF4id3X=Y*OmI^mU1ur&#$oU(4T=1>s00^;v#+| z^ZN4|n$29htAG{}sKOr zL=j#kg6W!faRdKALxFU z_ReF8UG+T4!s1~hQKb`&5|pudf*Pt+1=4)<+X~@RZp<+Gy^-qHqjD=X184MOF|z*< z({fe)q^lBDvlNwo?munvQXGISQXZsLcI}_(vL*buO&cP949n%!d4cxRM1&+n0n`q_ zQ!rVl`JxaFuWnnJX{3x`x)}tD$`2`17fyP5S%vrCRo_}h=%>WO8_E*a@<9b5L#wD% zqQ)c`(Ss$QF!9}gL7P5P;t-4t2zL}WIX!bWztgNdW-Yhm!EeEtcVcn&0r_2Z!8$C&2m za5qfWkqp>u)dEo5vn;^Y8riVUw?c;G5J9NyXkk-Q6wC&TP!=_W)$3kX5hDr590FYe z{duz)DX<$O;fO>l*e2*FC_h1SFw`*;aInP8q?K-)A9ydYSRjMe*!HgtvJK}e3wajX zgIp^vR?TLb+D{5OO7+U%T!wU}u5v4jm3_0~hnhF_tI#acnANO_xkkSzs=_;voWf6x!+2lG6SsDHuq_t0je z>xLnS2PHQJsI5@++i7TfO}3ZgZ^otgJQpuvazp(a)BMU!145T?)#2m}B0?~Ny`%)R zR6^^nZ9n9HNtHjSE@*&t6zYJPl=PH#lmApZ(CZ$k@P~w-CGtT3ihfo8fPK4KcW810 zkq9uVvxF*ronCRyJ^ufH?>=fX2h}U2e5|7-h9|>~Zd@GU!Xhd;wnc&fh*bgzHK*u6 z{MGihPCBn+o)h@hT7Ftz)aX&S5%=0fYC6f^e>Zn^y|U+bHOt$Ru`;jWjulvH8$O_d zSSmr1Sc06&Ln?ug$SPQ>n5ov0M+Vq3UX(--{nL6ia=%2k92X`_D-$l>#t++Wh(wVB z9}Q>tcm+W&|3lJ%peUk(iRbt<6c!!*^#4smb{G-^Neo0lrb7LSm->K8Q5Wo5j5RE@ zD{9Kp*%j`(XTNRlm~ZD>O0!Z;scQ+lwd3bYgdH<(8RW4G+NYeIWp&2|;JHx~f^8Bbj$`wofX+H({ z)Z>Qojzqt0FpOghAs>-E2y;3fYjp04h$&^DSRhj+WTO)D^F=X7mlS=-KiEuLT0-cS&bx8fx!9NTx3J(v9R@FIKWXYPgDv`Y!7@E;%x6WEe zvX-gr$PO6ebg;K8iT@bPj54YCl-|bAVv{XNm1HUvV=GPHif}2uI$L%7|>F*6jDOmk`+G)0&U3lDB_u-z z(eF=Ci*fze8kP6)0oZWpPR2T{_XLZGNB2C9mNspvBxg06IZaDC`Mq)7-~UsF+0 za8MLaInx#{g<|+W?E0h9n$w78pWUV+&B3 zl%=I}GL9LS2Qa|0kjUm%+Tx)QOmV6Nh*?dMfqyNvR&6Q0x~(I4Xo*50(q%~nOq2=H zkNLhD^ZAyqAnpz-apF`wX$nG+X|$-ev95AzY6I2weVOYJCXk@#J#L#iSdo%$meC@Y zH@uFD0py4QL;(wmnBs)AB`%Vbso^yhZ`l*DgepR$05v8klsROHh8!mvAPvs)Wpa&H z1%(n)Gi!}KfuL)Ik+@5LJa}-%r}zecNA7&p9go620P(^a8@AAxW|^4@hH0w*ZlOEr z>B1))k9rX%2LP%#!^kaD#S8fb=1&L?%#NKWaYUI4 zkgGiTDoz&$wevR31YZ0V0~IR+00P4d64eV`_o|cDoS8ll_Y-FNc659p@`ty#Ft-9g zkRk#hh@7)MA~aOKoYKjeNC;DB5aK${&pH!~^UrJ&^dk8xH@m&f z-S-OfPcwc=Rpg;iV*UV@TMG1lsyx>&B$y5?Oh7IQ< zgtbJP1|g?eGT|VC0E9tL!@u=Bg(Du+m+Sa6G-OVky80iD)NI=Iff8_#tw{K)m2qhRTS0q@Zrsy-@V=V4#LBpmH&YH z`~(A^oD`gQ<>09U;KHOpgdqY!01rh6B<|6cOduB95iu}38k-$e3sK+N4gd}ISF)6$ zf*4_p(?mc-GXNvI4XA0n=mRhs2$Yl<*037iG=@k8 z()CiL3n2(WiD74Ovv(5$w|BTg8VSg@S`%vAs91I_8QoAnbk%FMo0K4gBa}&YPC7%$ zHSlK zLm=XigO!8=xG122QA2T(Ou2ii+t2r0Qk#3?r%g9heiTL0=61ImS(Z9zJ&}{q)%q#3{h3s0XX&OrCI> zUV;QXQr3W?1Wtm2?cZWx%rJtO-ikud(NA*B<0iYte2kT8n})&ra6WGjAi>l$Qs9HA_i)EfDB4ok{;8y*(VY{n!bZb32>O2~;3}Skg2S}VDMCjBCR)m#;S)n|F9b7`E2pu{!MX#>!codINi$f^_ z4TXW~92_VP-0t&b4oDgyh-v|rI}#;2psP;6c>s)daAjjTK(N4rtqjKK+}jhdSr#a% z9Yxz$NSNNErzPCYBY?P*k=EHRm55s)ND={FqT?n9NS!KddUBuG2(OKyqz!v+1PDa) z5Ym~mhJ+|+ZSc^AHxvpJU`Zf$4P^GJl6xMvIpx=GAVeTZWRP}y(l2kS=&C5FP^C&E zQIMhpAUWaUsLy9c(Y7hyC?p93d2SX$iVJaowFgAFghER^kRbvQN^s7~Y*)^+NVK8v z;BqP8JCi(unyQj{s(_edNI=|^6A-y31c@Yo!Ip{8ckR%wK-w7?nZ+lBwJp}gau+vg z*5a21PSNNN1%}hbhY)&7q#ijm8KPOcJ=>)$7oqxtz#U261jkHKGx?H6G$ zpXK6H@0!ZSCRno!u*n_K(&^yqTT}Vdg1|ILqTXAtW)7L&n6_IiTOmzS+i43@*)>B^ zX|+=d*;`i3jbg=(w%h;0^fD?;)PpjhvtSDs?&7gE65x|-e9haaCpBsgLdsf|-i~XN zxO?q-Dru;q?HG!uMIjc#zy-StHX(c4Z|>Ty=_i|)8v)CfZ?!?e3$B+$5=2DPO~D1F zqAGP$_9zgcQyPFAe&SjtnVv1&=5hr{1GKZ>Q0KOp>})9KXfP^yY}dX*rP^HlOpLwvx-lM%A{ zX|T4`3_^+NdF0(B)>a~98KKpI_|Ir|AP7ME>+Mok;S3?L%nO!PP5qEN|FBfReM&z) z)c(06m(ea&J{ul*#KYQ*XgB)5MgSbZ(d7tXSS}+~9E*$rZn^!aK5XWeJu^O@r#gRx zUy%>055y@@c>s8?anVg zSMTNh@E%N>o`9pyafg&>I zY_TBFN@$EC1Tt>Xf61mq2_bC zQ?l+a@Eyz-gnBaAwU)KA#A&9Qb@mP9Rf{O2kuN%^M@ZrMe!IV$o@*qj^RICaxT{wfJFYwG@2{I83p%BX~fqAmqgCaP9KO(kTCWY#rr2Jl2{OerJn{2OTE@W09 z6e`eesHy|5J=9Nv0(Pn&CTxk4%q)w#{yx^Vs@KL3(&|an{bq|K5>CNT5WVQ#)RSPj zRN#jru16Y1y!l6$H8*oY9gssgIAsaJ3c?tm#hpJWWzNK2k=YQkz((;lp`0qDhzizg zg%~V0gcVwBl&C|()Dq0n+mTvfD?*L^K4w426 zpb*SQM>8^TmQ6Hi0WE_d08CPfVj#?OhB}BKZYNshfOQofgotF*!U$kG;iN|dMMFgt zb~8hglm`aNM6Pverl5u$+Gw)~fg&V3(+~rO!-0x{g(`*GsP4KLh%luDFmS4gNX)|p zNf3HtuAO60bhgEWHHa*grrlw1w&~ezu`hh3JJQiO(q9hD+Ljhl$)#GdtXZZ@A}p%X zWr(Vhj<(sXc0$?iW?X0-Y|^Z;u}|PxZatZf%^`L}%Ps-1wmXeCsJvr(PD^p6ipZf* zkwV2v{-^7314aT99C1umVi<)`#KAC3HwOn4C{V39aoR1G+B17x9wjTLOxmqk4M~+4 zUV+=PrbUd5EsbQ>YX_R_{jVpPI(SwsF8~a4({D|-tO?9|wV6yxS$KZ*N=%c*`21Bb zLrlazWAXSQ@WcmT97DKG6bqg|WsQ%$4`81W{UQ28N!UIFpmlK09Y_`{~kjweIHs3`1)ZY*qm)E7a7e*+sNk$~9?P$hC>AscRuN zlC7f62CQ1eHKnaVn>MYXt7wgzP^}h5d=2`?;LW~9&Tq@HHmJpTM=4yUlM_nQisW%z zaGbcg4ht%@!f55ia5->SCZ(yVlMR~7R-ybvIwDiv+~(+pdITLckJXz7YCa&?Q` zizWAq^|lw|U6J8e)l3`;&)YAj z-ThH`V8d4U`=wo12{pA>KvXJB1>kCEbq?e8A=HkQ?5BM62ZeW)-$Cb33Xty=IDq7i zNsxZ<>i7nr2B>N(YpNQ6YC5T?hNhtE4xz50s%ygz@7%(Bf47{aDN0mnq@^Wu7cqMw z?X(`pW5ras77*BdYZ;qt;#rmT$Zj-q7TL(=$Q@111=l(CxZO&OOJ7==MpD@ks~R-Y z7>pU&nUNdQ2*QM3v-Q4r6-YrgY?7fYwQZ=T!`Fy)GDx;$(+ag253;8(9%FMX94yKS ztti$KTEtb8wtpb+TMT5#WXw@Fq2tQ)^Od=H`1tSL76lYhL$tJe&hmNZXD28GegYPI zTk1!N=Zg9C$fvp$Dq`57&7}#3k=(?EgMu)jaS{?B!|aU$24!vqT`&|(K47@8{H^x3 zHk*ED-`!?Dgb%*jGfKsljbu{^F)YT&W}{N7EPz@cRcnzV)}t63{7>=~-7H4KD#KfmF%;pr+GqrWTN6zn0r_huBfCr|J2o zn*VWh1EkY4Y^c<0@*0FZfjoix0Q&&R3()0%uG7?7RE0=YRYmo+m6%O~F@gC+FGBP$ zzWKHg&_a34OUfwg6UsIY4(yWqkiLBCIVv*?+#!)dAJVKVq1}=ciNJUV0HN8T3B;Xz z4}t9O&cW<-9&bws*F^v1m49o6#rJ!}+Q!yOpU5o2L6PwWr*aVro@#~__!UQ5LnWRa+BAwlTg(mQl*ro=Bce2x8Jh9MDqJ5f$8CRXdgAA8qu{#QL4ftvaFK zF8M_MH2aaZ*ChMbzBTlHr;ANvFCV97*4WOHuAM(nA4N}nIIy!C)uyW(7-eB)m8qmz z1lBCIw9%x~D${06#XfXnQS$rK)+|nK_%kg^;sj0(C}Jn5<)C;6xvRQza&oY1Lb~aP z;0wY^&}&N*EXy`Vs=wbahAT2ANq8ajmHSUQx7YKU#x@3GhD|KK%V9f=D*C*S_d!1e zRaHq}6-^wL7tQ+2X5k2@MhRAzbH4Y_%P z<`BTz6JLE{LZtfw!W1D#KInZ!@<8X#q0)-KXq5+8VRFyYjGc%YP1TKUv5xfa#0p%-eEYs6{6!ld(=&3&5 zBi|N=W^Jl-3zenImCR{#4rtts%&{YDBt)}GW`%{LDT!GvDw5J-ES5%PR;*fT%NrYN z#>&9VrG9GMSv-<>qs1Q=i{9zxcZ%!7oX23`21*C`4o&gf_b7b+Yw~@9=*dI?e9m7K zB&7f?WGKT28~DZ;cOdzlsdbL43|PLh*3@Rm+n>Mj)aId0T50Jk^CjKh*ULPK^kqKD z{7Py~@8r*#zUvR^6uztN5Pozo$|dVnJ{3pfQSgJ}3islCLa*gZ_SIj6AGJsHh5A$X z)cxuE=lj?Ez7M{>=gOt|6@~nNlG73R>M3zLM@e6k>jQqJ#AIZ1&3>_?Td>w`sjG=| z6Q>66PNy#PZWHu&&8E`2s{bOV3hY}$wfEL*bTL}I!7BT)Qmbf+HriPFzUR}Qj#K`n zk&4K};}+c!?8bW?t+9F%tXr%y!p_;lL)pvoKQHswj-QvENb6TR!94}%MCpYp>r>dD zoduw!334kQf&mw*o`NapDQYL83FxPygXkxq51>5|Jrwj`LG<7ohT!kHv9`mundfkp zS6snxT%{Kjl-xH{4cA?08=t+ht{-v$$N~OpoWij$GtCUZv@;6^?rw?9G&#O%pX90D z2Vq|`$ny~q{F!|!eehHSKHq+x%Kuq{M1L_T+i79$hGUvzuGE9I2khTzL22so1zk z;o;#&hk@Q5?|JHcKh?Yg4AkF3tsVty)J?Ly8R}M^o5JPvHR<<(<5%_<;8pPchwOaQ z3sHP&e3XR4Gy;qf<(Os}_3Cw*Q<9}2NKH*8Gb&Q4{%_hjsC0!gL#QmbHGIZ+0ALV5 zTVJtL6%>Di>xcFDKuA)^!}qO>9iIoDJHQGfhLtf#Dj_1ER+M@WBlpX&7!d<#ZVr>m@U$)ytI8_J!=E$Od%j&!`yRpG zA=L3yBXd{So&oj>ATy8AQ_Erb2Q$1*^`Q>PcEX$IeD0JzKDmOsuw(K`J=&$Ex-^7> z5*9+7Cr<_u#~&Qbu{HvvA6Zov8JiD?c!nx8|26M(0?|1Xa-5umAW6yR-Mj6(e3;F> z@)?oZq40WHl~?fAm`PzkUvHa+v-Huoe`A@Ij}{WeIRe^2gODizkyOQ62^hkeBdoqb zYY>y1XiZu9rb4H3A>Vw=erMNHr7n~`)cp0M3NVB)2WAG@*bwp%zu2T%r{Va}A-lVx zsmaKP)6x(<3ZJa`pwu9N1d5Cx5&%t=NcI%St4y5NW*Zz$G^D~7Qv(1Gr_%_tdNit( zNLs0w5pWKQnV|oqATk1wkWP_2;PV;Kl5|JEzrUitl@zJyfIjq4N88-M9#rfA3VI9Z zUWM=|EWEGHRiJO>QIi>gB z8Q?Ak2;iBfW1dDPCNpq3g!!SB4Kqt}38u#kHB{<(cl6k)WgRx(@8uJIH@9iNX98eD zO?9WPTKrwES_aizEb=BfG>EjWP<;kq>w7w%wm z$C9MV^Jzj&)3*fjn%iAPNv~(~MG#5P$d_1!)s1LZy%fe-eJ}vfy22bE*!DxTH@@z| z#O7@NU{)0=wp82IE;S8KMY^J`~Pqj9M|b88F;rHgimqWgOMUT+uY$nib6HXq#!J4P-&L*p5{3 z`gFY+IQiB0_XB&E_$$Ua(7W)h10*a?O(Fc^)#%iY%Tz6^_N=e- zD0e%vsjDd78$n@&ZMn8hslgbHNwQ?snub(0n4zshQyjFm%V2BT}of#`w?6iL~v>jYkGYXkx24qu{C}R<8A#z$uR(v9@ky}W% z6yG1XoN9b|>m4#*j#a}uGOd@%CD61leupbY?Aw*kkfo-D^GNt4T^iaimaCHwI-Idj z##Q*}eTg4MIK}x<_QUQUl>EQGq5jI9^*tCb&F{o}u^Y1(%svouLzf;lG!_)L4pov1&*(K%ZCRJ96pG?C~=?G^{+0|uzb z46xpJXj;UzopU*frdn=uG+~A$eiYe^rK=N+#YYQh-F!4zl4h!Bc+I+HYSHp8GwW+@ zB3eT`+hoK>Nb1byELgEsYe^-(9k#+h6LLkl&x7dse7Gr6UKLsS%u4M^SYC-+nvX_H zu(D4qYAmu6pE2R$$>=g96HVG$YZ~^ZX!m%7Y92P(OJ!a&*+0#< z6DW@n)B|8snyO>#tkyoFB0Ui98x?v5fVqXWTPmN(OX&FOe2JiO0P%=M3|Qi)-9=@{ z=%ycPu2ez*inLI)go^VFVT^)8G7B8Eg{3S81d9?iT!O2j$!$|2?9^Qf#JMQbn;Qo? z&9lL6Eh5y~H7y$3ZEG6aItSu*6*)BVk~LW=Hou1BD_A zbX|Ra5Ay_8&}m21D*UsacvR2N_IB!9Xj=R~L-$`RuUg@pPwugqQu;5*?fxs1 z`2@Zc3LlTXrqxRoJKB)Gzy?UbpSe($C>h_1vHDIA=+!Y;TF!v8*1x%{B5n3!oVPJ; zM<(W)X=9D^g;k}4@Nsvo3S|5)G_=O|667$Cbg#)C5)p^~GhvQXGkPPRzlJGK|4yDQj&NtQV)MF4EDpX02)R zGsa=+$Kc=E)u9m)Vw9Dv)=Y}BWV?m=itL5mCEZK2*ZeQnhGnU|q)tM*YE28IJ^L|X zpEfzRK(@)5qH5KZHYR4$Qwp)z!oBg7xBWlOvV)(sV^X9r>zA_>qx0{b<9FA%-cKch z(Pfsbn6+tUNs`2|EHO!CmNkt=RhqW7ys_%_h10E_LE^$j)Y>hJLs~MMQ&+}K zXw|&y?u=UuQF5d71Ln#z1ln#>0_ZQeoz;gDzwxQkB%HcscU4XCd`)6Zh}tG;Fs4kl zMT;#Iwyj=^ZV{_3rK?iaiF0d{jLnYsVo%2FHvE2A%;Q)eYAb!*MUT1mpD-7pcwB8X zj#hg9sE^l-9?H5<)>V!T%-g^>yPigU?W9{#Nmg3OnwF_gMdLkD=C!grzddC1JNq&v zjv!a`B$YO*{$U`XAOQQ6*kqr7TUPUF=(Vh!={&;yH9jU!b&fe{oMPE6lG;Zgtk@Bs z!(=x^GZg;?Tg?!|5Z2HG3Mb&haT|v?ZfaqHiT8rE)eB1CS!LK-Q~*Db3L107hQ&i> z$s?Cac6}0Bj}Yr9yf+}}Q0P~s=0GjF@uwk&r#J&fe!saGgjw1jl@R@g5vg;P`#u-_ zi$)jc!Q!Lzefw5%!{|N*2uZG{pNjNSn*-`LzfrXNZn?E>62P%6Y#S_qQpU@lQ&7|t ztAhPh`m1XFK>Dxee>;VRwd|zC`0GBp=QUrjDjmu$t}7?n1Z@ry-CL`Wau-@&qTI@Igjo`%%|40#pjJ(l zg>6e_!B@&GQBpWm`F?5h;xbzmsb-CvS+%u=m7N_xuOz(>b>}XS-aZCX?XYYXQEG~y zITwK-OrM{SR2gNFl~1rVWZc(=nog(&Fsb~Or327b$PoAp49W>6B>weDIR*LOFc}`w ziYU^5Zz1G|YWJN9(4Kkjpl0JcZcoWo{`eY9N0ILXnjmqm0J9BX6mb(ZJiyV@9Wp}G zBrOHpM|BOr_krn_HdmeH^OuuKT#;l_Qgsu^VeBhGK|$n+xiD+5af^dPHb`(1MrHD) zytJ4P!9fSOT!aUZZih59n)uHLGXmQfDW;Z^k^*Qmq6+ED{8c~lihF?fqp%EcB{~{V=EX|D>3!lZ02L#EX`J;;{%#gm6V!ls>@2b zsIFRDDCMYgE1E7%6EZZct6G*-nOS9wWUMNfS{5dj)@;#bwn%$tJ=?^0d{~ZV)Ob8@ zy2posDy~J_Otg?lLzqJ_2^(m^flHL5i5YQ*ViAJCBEW>gIJS;lxh-W|DB3$k#D1n5 zrfUM5OE!x(v3uJf;>9pwOMhaMpZvnp{rUY)bS`MJ+D%=mcXgq%ZskheU%HUXZSbsA z<&t+Sg*k>sVyrAOVPhsTq4-paeAlO^r!VDGsXTEZlEAYhVTg^S>unwwLPcal6oMpE zF3*HvTCs>|;i?{sDRFGH<9R<6ekAXA(cgF=>Ivd^_fpVMeu;Sr*$*1-oe9vMdD_d0 zIU;)yu~NlMl`smze}BC9_aGmrfEq8hGea|oxFNtzOm%lS6#@#UF$g;_!))o6RaJB8OS#Y>GHEg!hi^d<44MbU-Y)hMQzne7h;Ir+Ww# z6;v#csktmNI)mX3A@9y?(E-yPA)z#L*w0_y_(#B*Xrb(OHVe^<(Y=*{dRq0kxa)a& z^3z8Tp#2&8l6oax>JM5`?U3gcdeS{;Z2y)7ih`|cYC&bohXwwBr!JR2()npq`o*lxWc$@T6Z9gSzdI?cLjb6MMpY8p5we1HDy9874vW7Q;8m!4Ri|xes!Pe_(w1R{ z2S^D(0SI#mjvYApcBSt>6zEAy?xDRxSo00$;s!PoSd7=XUOs} zLgYSwFVFW%^*!oP_E`$NUiRB=V!5v!(m82uE@mP{Fb~1_?74-%k7#6RCiMNI+&!?u zszxe)E9&y^sk2z&)k*n{_~UxgP;O)0UOMbkNNG?!?(g_5odBK);H&d~ZI z6WtGTDPZY56(Qfw5c$6-UPz%Tr%^N8_|WE|qtOT)zL1SI314iZ{tr(LvzxgNH*o(@ z(Tn05J{3E%45VGKt5Z_5BO@!)oE=i;hlga&%!_4nvbj-tyyHxaNA)b6%#UxUSk0*E zw4JEs-7iSO6Inwd6t8gjLyh7@DrE~k)unKR5&rBzDhYaN`R0DdjZGF`lP(bE3EO8E zrkXg0s9v=*WuR_LSYj?>-t%-W+%j01T==Ryklt zDdd{*cg1FQ7Bu)F<_f+dgjO*F-yfR*^x}k&GO)1mF{-A(bGprB>buO1iky~YA6!aoL(h|i z@O$@rya7X)l>bpP5J1r&x`;}oJEY&9c-cx4AVu@xS`jF#xjGDItaKS3C>1RDcXj}G=sMcYauZMqWiT# z%c3C)DM&|UFDyi%I)%xxbuCcn(E^YmgbV0Q2Ed3TYY4-T?5kG9=<3jQVsCYITOCYE zx~!dDZIx_}r&(w=bu}z?tEsc8fpH`loM;k_q*r!CrkXap3@B_Oj!+~Gz-b;FN_b)7 zTCF0HIB;rc!~;Z$k;s9HkwOUu%VUDhr77Aj&gosn#9se$xE<0IDJq(hf2<+&@AWVV z_*B{_&g79}g8NhL#=sEL6sJf`eg?b(ev+t$#UFR9_81lLB6twJ0Tm8w=V{X(e45=K z$Wf}Fv65kn43DzElKTE%H{s}F1N1}pzY5=<#^0S3G+>3OPd#6Xr41N`O_n$z^b_bm zV18hoQpf5gLpKq6d!yTs>j-A$b8~GC4HPsqau=g*(R4~B3Mc0<9{0UR&YyMlYggK9 z49#K7dXHa^rnRfh_&A`j3pPW&3oNoylVM>1^$vm1IJPMWSOgDv0cjz(s!488we58` z(bn0?=V>V>deF$AhbqvZOe`t__NA>0i@`8}dm(xmU3K^BfK-J@RaH$U2FN^ndu1s~ zQv4>Ex=MMC23w{V(ccy$-f)M3?RT@ za~j(0ol)Ye-k^izso*MHmRMIc*#<$lIAn1exFseilMPg~vms{z=sYq8+xS&a8bQel zqy&(aT6aFb0YjJ&5hx7|$J~VjUx*E7wzZfbd)!M2B#AxIuQ5hrt;L~crd0HONM;IN z@j0J)f0q7rO*GP!vsC=TKYSoDlkBNWM%s$ffc{lVATugQ21*kmzeHIg=)Hpg+sh-m z(Y;o)d>;CP$T)_h>~XaonZ#Lwh3J0(AyNwzSg0CM-TBG|LgF7WMcTHHuK7-HM#L!ayHh-$;HC{CNGGo=eolRaI40YwhQEpz&?!z^ei$ zMM~6f3$6@+xiD-xbbZgS;;G*80^-urM2R23bvTETy@XEm0P4;i;H1hn8lDHT-&Cb3 zN>Z0z%+WMG;p|k0#5)ITt~(ui+StRDRZ3Enr726C-1j)5c_LPQJ>F+ltzEgSN>IOr zL1}d(@n&K%jEKf6W(((}EKZB^`aTUIp(!b)q@>qnOe~q%tqI1V_MPjsMO8{t<4TH8 zJo}s4=##-HF8?8I7US7^r@c~95fR=!lyYW3_>SHI+lsq2MLd(LXhqKwMlp<~TSl8U zk|_Z~Nw(Qb6%f#YFsdVVsv&~~i~$KI+hxep5dStJFBn+1wV2m@*x5B{BW_%Gz>Z81 z+%S-F3qP{pb^;hc2S_5)A~H_fWfVvO7!ZJQqVgHMEwi&tHQt@sncCYOl2{tHtSYx# zIOMM7QwpX@Nwu}H66a*9eM1S#?&Fp)wKyNbJVMMVr-E=pqj!+fCU^$Jh-5HBnxu0n ztA(dn3nFCST1Yb*LnV-1Ga=DO4z!#E=t?UzJc$UWSQN8j+8M(-4yo5%G^y9|#~VAU z@05jN)2w$57z~K)O2R{g~SOK8A2&(xF%4wVx&|&u(MR8WgP^7Q+5q%sEM6IxcOisLw`un3?MDvt!8+5)Xn4UqH`kvq@y}b|Ru7J}%{1xi}$!nGV6QU=LOX z^-e?rfea;na7tw#x%z7)fMk#dTeL7yc_t}c>5UBo@ku4%j{*bxD53!XyUec(6jW3B zGaynlXP$er@5nirT0Ux*Nkc&x9AH)>#wU@q(<*%=r0_E^>V_y`+!%K|A+YQ-lO5!V z5I|VBh@_%?&Vpy$O8lV+Z_jsK%GxQU+Odq7 zv|6^BmuBM{YSu$yrS~MSH%COUX{umC0V5L!OF%!`@bdIX8j0ZGA#8M|F-9t}zK8Kr5d21=Hu5h&qAFu|r6QiNDTKp2&X4JQjHONFMbEo9+42*T1? zH=+$BMuC+ujYbm?p+quJxaEnM(jj13NEZEc1{V9BVZ@q6iAiN>#ZYV-fwDVzmiS>z z(GZk03pEb`@EpBWnk(%vn5zCjrvXm5YQsnrr4Tvu>*E3$gJ3=!A?`Pm$k|d9A)XW3 zg@|coIH>N5Aeka$;tk4SZ!ee0e9@F4xhKOl(+?_#Cfd@&3hBCE{S!&B;Ml#Da zWY$_~iWxC$9=+?;nOhpsHKRET-?o{9Y9bGM@4s@b$E3_752pKxU&#r~A zzFR>=vQw-gHQ2x~`$&yopAs|IGE^cKAuTD$4*`foDLzfnc#j0)5eSB9DKm?>j}g$| zVbXAq8^CyO5mu~rytOEpg>46&+OUQv|Vt4V1` zkCkS{G3AqDnl&Ru;X}by;8p%)Uz%UEU$$S=zp8$(Y9%icr;1APhm0OEu{CK=5N#{Q z4Hl)XSBxyPR!NbSgi4q)|ZK`UMle~ykChi_?0hz)>F<` zu9fR#Xssp@>YlSBNoq?G>rY#9I6ZbtD{!2JYFDjTV#F|`*0oZ?y<;kBH6FGmsx*3; zS|+bk8KT1~^_gDBvy9igs(s@BSykg!v__T|e z?4vNrYihC!*h7*xu&Rt;rj<-r!L zr?I7NX0a?J#E>jj!NFW9G?_h=T1jE5Se3Zi4|r&7m1eEAGg?ffZNr!{Z8rCrkYv#r z2m^s^EEd~kl(K6yl)~F#?=xweBD9vW+FDGc#L-^LF)LeEm@!jJRI3Qcz6mgf7+RVx z708)dr?MtW)|xRaNNS7^gfOOLjT2RfM)!@TYR3)CG+6huEe=)0ta}bZAyeQorN5_5RJAr0TzYE9`kMm)a<UaZ z8Cn`!WY&ohx7*-st)CyG-2d5~ovK-hm?@%$*l&L`{txzc)lL-(b+5r&JgNEx`@J8t zoN+BVYgNf!wLMvkV;C+&(Gs8CaYJNR;6_9;MZX9>^R2%vz1^E_7_L%Ose~{DKMrvm zJBNtOhtuct{K^PCYlL}H$oL;E#Fx1J1m(v1OV8(5pTV?k_Y=)6=KKE4dak3z89i!u z>QztG`zM8NL#Z|Nm-{JPDeAvVO(X7+{@8p_hxXy{MPVPG*r9p8YHqcri}UmKy+iZt zq#M|6o7UPji!Bt$V=QE+*WNs1x4Jfsw(YmW%a<=56$njLQ-sx^`Dbmf-Jt!?+KJ3c1N8oOxc4UAYl{p77b*)Wm%|=(`a(O>0=2ZT7#7$EM zWMeR~4X(Wx7+g|s9T)PSSKM7MWUu+u$bFxw!FW;GVlpwaX7xp1or`xQ_~m?5d^GrD z;jXeiVdf8-A9$a5`@`=Gt}eN>w74lm^%Xwq!m*_*3h*CWh=wL87@_y4-amgI&&5F- z_Hp$r+l%01nTP2Eqi!WBD3qnP-HR#Y#``ZFym-v)`dd$JPpg#Tk1>JZtb@1{<9TP_&;u&m0gZyPQB(Uj%Fc_W)ElKw$%9tka*ES z;CJ`dA7E8!U4CT8nK}|-ovFMp3h1D`zWfldObak?FfLM;0HiRbp)d>|HymFws;Y)j zEReknugZDQpOC3NdD!YO!^%DjBO@~y{|2o^h94Fu%eiymJH#>WGZ?H=RD;Fh&ZeQL za@Js&CYq#8rMK~PZ>ol)f!t4ef{%m^UgZ?}Dqvb*+U@Jnd@#!+mi`~yJ=A&?=0|^u zSr1L@wVK=IoTzQ z*jUPAddn-qv5Datu)GUPN4aAQOr8x4wPf(L*2UqbZ7arcf(|@RCP9#6edYh$a8!5- zJOoYh05X0ahOl0JZbO#@h_3Wk2EB&l@@g#kZs zL)pN=$t2b*Y9^w#TCrO}^CbM8h8wKV)+TIF%`meFG?tI2idQnV45h0@S`4XZSs8Br za!zoi+0ZydAL^A_s70Fqkf0ic1uHTEG{HjwNI-=mn~E(&V5BCYq{^*KRh48|w6TSz zmel9dFKyEN&eh(QS1U}i{g}T`L-m?+j@cYXYMC7k<9)Ytg1WIS3qw&uHXItfh!9PQ zEFsa5bS8mFCPO=p62i05Wy~VAP=w{tsb&dwT*op-x{VhN&eu&Y z2NZQ*E`RNess9+kWMNxnZ7gMK+Nj#ZShH5iDWgJImLo$-G@C7!u|-Jz{vJhD<;trS z`O!6X57;7C1Kc1x4|lWbcMZ!rr^x=G!{VtA@Tm<*eHY<9a``9ek9Lkr`Ks}s;xGS; Nxgwk>NCr`;(g31i;nDyA literal 27612 zcmV)9K*hg8T4*^jL0KkKSzvNV*8;Kh|A19hRaI60|M0)>|Np=L|NdaJ-O%RJnz!4# zT6Y}+hjX|M&8Yg`W7lHYXja{6$7w0sTaHlEv6r?$71oTREYDQcDniwUxipt;)iQR> zO&Pnc(bbu=dv{-bb_cHG z1p?aM_k+&beE?_@P^A>LK0bHdwcTxf=J#i}z2|*y_Yb$nM!wIyJL28MOxP&TW=y zx6hvU)O>9~_V<0Xd^+`g?RUT(0(7;VlUX@^@$a#S^{%uD00^C{cgG>=rkmS)&B||W z=np&sOClEkbbCW&19gwV()o(c^#3=n8OuMiOP1XfT`e{=N@{T$EVnW!{TNY(`-WHJUxA@?^v z<{aO<{qOg79J31qt0M%tDt#)Fr$zhV=PAIcygFv0i%WxaZ~H1~!WagB>V|fR(@hJ> zQ){US6Ohxe14E1_@ra12G^yGuV1ddg4JpKmcR8#sYT~7;UX5tI@VnJqx;n2p(5VP> zDS4EV;Zg0vrelE>C0v0GON|DDAX2GZ28}z^jD@X%U@xPgsB$+^QW&QMY@T_NLYzk# ztea%88HVe&2Ga^Sy1u7v%FV5?~}I>9dt>XrvihL@`vS;AwS$1Xc5rKM;QjAt_?fKnRZ zTeDO`Um!18kZg)Ab{6np>M zPl6piw?Sc-h;D@6UuJ~j$O_#FQOb=tx@9uj2e(dV1KD9gUjT3|y@L5(A=XSd-en~* zK@c_C2zQ|>JD2i9{KN-9KU>6kPQ*@JoKVUCYnU#fL#3C1v@Zco$!ex4@{Fj}Bs^6V zsS5uh9+fTP9H0h7i|r{WH(^0_Z6k4y%U7*+tTDok79?@`oP(9SG-Z(v7Qtg0h-nNd z-h)#ng|IIOVGN5oz==*gdRUp3(FjH6A$*i@qcMr9LTSU6hFDQ<0}+8@OuO0}2t= zv?R;nq&SA;;Fe+$4kHlM5j2Dnu1i+Xp)TCzZEr7dHbllm$QB)$0c;v$q#&9@c)`Ge z8!txMK-|!6ru5`OxJDm`yA5jqjTF~n6K@J&lK`Cva2RndMp1o9?@fo!nLAICXkQQg zy*AG2sGKZ|3`emURGxlKS)47xr@owUY5#(Sq#_z614mYcgQkvY>n}Q*hymG zJW+E~HdT>2V7I*(wP><1$kNj(XED7H7TR}%AyAc;NhzeDkdTqUWJw7O=m>|zR7ENC zW&(c%Hu>RP3%e8X975CbiB>7)Sp~xJu{DW#W1I^)4QVu3XmJMd-u89CE=$l*F@lu& zWI#KJbeS5kysV+{SykheK1_U=En~?(t8KiqYbgF{V_64A9N|}wNvS-V&t_tJN2cyG zR|vp2Ez)y|whc0+m6YAkErt2}I`;LsK0aH#zTcB)B4doBp1)4-mF?y6;3qlX85s+V z_er9!;5d^~mi48D4xgpKt!s~Op7*1(=Ct64v8Y1*6PFb~vr;&Q zT+=TNcgeCh8my@IXh!3@-LS+3?5ZVI!>>=FUC0>fOa$RT)$dg77ZC0n5bWgj z$r%O(2V&|m170E8+&9gI8=0_&s?~l(uM>sCUttPQl4(h;Aq~{xO@{kkF^Rm?X*@gE zSxl4c{c4z@_s7d1>=_QLoS<`5iBeHLU$IjV%TNwO?qNJ-4}G{Ur6Kma2UXgsYF(oA!H^( zLz;kgiSEqWJca7z(T#KrsYMb!cvfl1s)(Na-_t#C%)~tBW?7{476Me zkf4Ff0^#+-oCA46Lk67dw{qOxT^I)DacVV?xk}5_jcW8BFt$kU6IC@_&D@A z0={L~SYJ>^!}NEzlp2Ymc35P<6k&CmL#A+bp_om|!Z!02a7|Ga5R^j-2KAwq;^oWC zyS>J9h3hB@#Be!9@`*JKYh|gkH&L*T(4;Vp)Tcu7YnutYYzAm>Hd7|#$_%@Ral@QX z5X6xbS};Qb0C}JfPQmlXYqz(@TMZaYo!lATIoAgA(Y$YGqoYSmP7ZZ18*$d2NzRhw z#<4b-S~%7oPbk4X`BPa7=ZGzYRAc}rR3=4S052>CqNIul^Ry>jC!4z}c!&}O!I@nM z*)GROF38?Sw;xy4!*SyuSJfPG^wm$Izgd1D3t&LrQWvTc1QhxZ$^c!FkIN|eB^Faq zK=_F=WfqFx(W2x4(f2GjH!y~P7}+Hlm%SEVtgR}Ek+O<_PCBx}8VE**OS5{`@VZ{R zhh$Kz0zn}rF@jJqaF)SDvC>9`vvFvcRbr?Rr=6QP>$Yr28_=m>R2*QKom4u*3emU# zGl;t^%; z%{1wY3`3dIkYrtOU8L&lB+RgZ3}F}nFy0;e*OU-6wEYRwpuKEpL|~f9X%% z2>8SF?4<4#5MLtfJX_*ae()E@=@-Nv!O+t|L@Mr^kK<=8K_zPL17+#G!E2S2${B0} z47S?cZ)h!Ug-B34?IVFkdtYe-7*g+XGAua{3$7+lLEY;U4Kk~V zN=^j^W4g68nByBBP~C|kArpwjl$r>LXgMek#Qba)M^VaQ?t_E<-}rP@yvLHKPh=ia4MPX1n@>raNJ8Z zO1Bx{ihJZ~_mZbClZ`3|I75(vKpvn2$cQ-wJM-Ww7d_33+OH3>>^enGkdT>E&aFiK zplu;g;MEDpaF#!{BKi^CA_vPJM8uRHs)AG@xQDc@AZ!kxJ95C2YZT#I4L$~B3n9$} z#(+n?!>X1`u{J|TA<3yN3Y#sYCakNOwkW!a32`)l@Su~qhh`hWfxYJXuPuinMS+}P zZre*BLY$&p1tP{;6uO;iNg>Nr#eV#wE37p$XO$Q8e+5<>4eXVX7pFkeOvWnL0H@-(>G_%7r>9SlP%A zFpLW`9o>P78Zk769PKH)NwWh}GLT6_B{U{n)C0-#oRbz3W$DkD$s3AT7D7O=w0_on zj+Zr1B!{C^WH5_of%JMet2>nezC+L-Oc(h;`GA+`>}taR(}db?+n1IsIn^496p z_Gq|kR9%==zlKH(6QM)F^TbZY?2T$Z|6L|VMn)%0p7uPlSDfTE4SZ5l6~8G&I;0b1 z2`}>1pQU{>#&-_JC`hZ}7W5~0_D|HxK@TKZd`4VYdj}Y@R~8f;%*_re+3d_Ch^ZM+ zd>3tztrP?*#Zuwz91{q%PRR6Kn)Yjv)78b=@a8)p{BkcCys>>>Jttlab!J?qj6&0?qFhii3iNqRK38v19EV4VTHZ_4v6Kz=t z8-~kEgmWHa$k2hsMTQP3B~V0EK{REYw+am?9iSf+h7yJ~A2y}LEeJygQ4g;z+a9Oi z)?I%O<9yI>RqrdRLQ9*^j~hZnFmV!oyfCfaxNE3UQu!Skd&lzm!t_yF6T1@5Ou2@V zn{t*CLQ{k6hoTo7%Fs~snGpS=hm4Q9J~Sbv4%i+e)9&jH9k*&0f_Y&lTwXaI0+@50 zV!l|97~@CbAZ}FWBD9pWfzf;*eFvG{tUC)p_qEvW6qmhBkb%a`q9T}%o1TCu)5|&VP%Tc)DLR6y3V7)C4;j9qL zfoY`*a9|KOb9YHslsMKD=Y|{%LcDS`Ndkv@Gf^A112|V;VX`KWyqHm%gOG?^D)9?& zHc@tDxK^w%VT=sT7#bZ=${BmvVB)(RUI2%Afk_ciJk@=wN37ZFk2=afU)w{dKQME3 zo@)7?yDj?B$-a#bR$-&R+kWt)L^Ukh9xJ1Ec{lQMtWL3@ZkRa2b$W*O5I#()N}($P z0n@aCkPddUR+A&4kz?#;0qO-i4+Ge7RP29*J|x+Uy1$&3m4ZL19t0{~*oO>Gtbx|0 zxzpO$33<6`7reT{$T5(7jsraqBN>R{vD2xDsc_VyESmz23}D8aMH)C~3=?~^27_rp zg^h`w34mKEBr-NiyOxMji^FWn+Av!`i53V~Q6zw_#+ZR_YDjrIqKP3o+K=h*e zbf|F@JP?cwSi-3XARb~rC|Vyc+^h@7ol%oJ?=+c%-Lp|%T4>dUSi*AA$%(ZYVXUda zGY#pC87=1wY-a;nrjyT6w>Z%?}4G2wW+{3C%c5d64L08ZtGZ7EBBnm~J`4(s`LjS65E1xX`y`3Olz4s|ydD7PD<`Ds82Rh$DsB+#S^B(&Lf_CM9xY)@DWIvZ2bntT6GE=r|1_ zAp?&Bl^wBs4K$EaC~mC?GBgFDhi8aZl6&b0QciIJ@&r{jiRJPpoLWf;bU<`KL(LL` zkhT|ofo*_78`>|lf8}V7(jC>D)~j&E*@OD`Z_hoJi9rrS!*?$3;IRwrhZXr z3cH1ndBv3?>J~vNB>Rd~Qm~@P5rBZK78O#kSg9~s5Xe}OFj*xQ3jtnOQnHZWO0ozm zQH4rOsf02sAX!Tyt01u~mO)ty6j)%ZO$*`GQ`h#h%hNM3=tm;?;}p{ZWrj*X43HrR z?gYU?8B!)BE@h3DF)=jLIW%Gukd)&&IE96cn5l>e7eW~&Nf~O=45^k7q{UilDGb0R zCJBINOok#vkU+ttlN?n@QgH_og?7N>G{Fp!N;I&VVTHsD2uVh8#gTw;mXRSP2?&}< z4B**HlS-Rekun0XSj?bhB}!G27;Xu}NCP8?l1ji3!~&@b1u9IiagMHTrywbM?#5FL z9P+7gQ$%rzm=u+Q1)69{7)YoTWCCD01{ffOvsz$G0jYwB=6W5Ww`eIKOo0;6PXmr2 zN`wqdOra!0A)(Q2#;9r}QH;od^o@z7L{J@I%^=B4)Xaeu4H6BIW1Y$+vJKH$L1kG2 z%29t)-Az7%pjor{9GU*p+!582Z(*D(XtGm0>F3eQDr&iC5h)zxie^HhevzU3UML;qBVRN+z{l?)I0v=KV(N~>a3vVfQhe-%%s@ZT)RT*kgf%jfNV=O&1uqT z(3;%h9aSM(IPoi+>cA_Aa;XZ%Pq(^!>wOulY~_5zv%17 zNO`)hd3*a--LFE3w0(cY2@4pf6{!v;ymU#~*UDZ_-QC>X3JfZ8? zqkVPF=!_vW#aD-m#n*@!vP|;VoNFwN_ren$_3a<EO zqNAc3DjEW44vOgLXz8ZGv$ophpg>IXOwh7 z2xOQh2!;w+DWXUsNs5VLVrrovsbUzGS|OMugoX$us2WJAf2DWDQVx#-VxqiP$*WW{pCn!=+*$vdiT zXgvR?cJJSRo)+?ct-|~(J3K|Aznp*XGZWEL85*?^I1BlUhPaD_QR8>p?4nwv&eowOdF}_gWWHQ(}7ez?x6h{O;M#!Xn>5Vr%nFhb18X6 zgq!j+DVR6c8wy=6YPh+a;^v$XSpNZJrtxzNOZ=q4?4qiAVy@9(y;IcKTMJ_IO3<-v zFN|A-xfk#(ND73R7|cRcf>8IN_W|>_g7t>8ggC(r4vLUJhUF0xD^w?FK@d8R?E%sp zbCSFTpS!T}s$z6kq#Qei^;LNS^pD-Qx8pR&fe0iH)BF5{(k7f8f$zZ@-jMHDXUI|$ zTIy5+A>chk`^h5RA24wXf#@po0nl`-rITE?LgE~RF$hg6O$R7E1!2v$Nvc1^9CxRC zsKAeY@NP0uKy+~I@A=`aL_21f1Ue+7JoU@$EOHC8mnjY+Ct_MzELK@k%_gPljM6K| z71}agQIs-Rn@3t=qUD_(CY#5D4Tvo(t>ie`F62wg<w;t+|nCD8RcYpE{N+H9tR`z`wlK6Ic}Z-Xnmre z5DL*H=$QCJ?e65?z7$w&NDrNZ#y)tUa~)g7Aqj3rZnfBiIXqo|shXAqQy17;D`GmjzFnA8L^Y#8G}=P1Md+GOK!T zmv2smqXsg2h;)#E{$f(4Lx@EK-TC1o>qwKuC~QItmq4DveU$S`9%7;Fob&z5M)2}X z)Un*do{{sT#x+k*P}F?Pf$=nX!_IvRNI>X140=@NkhYNDCW+Lo>`%B!Q@8s+BrM_v zyBES8q3Zh=uNT=6@8CgA``Q}7ufVXRIu=*uM@(QFOtqYmxkpTwW1ut!A&Sn=DAPR7 zTjt{pvJUJjWYUTruz{y8Ju3AXq~bNdPJ;ts(igk5IN>7}(e3gTlvEE(@}-1@b_FuT zE2Nsa@d~Vvwx;m{#49!5g{5$=AEX@XuJgt5`9kP-x(}2tbQYjho;&{?=o}A`z6Y0r z1rFSPs|31!;rUUDewwk`6Dyv^{XEY61jZqmTpu!6F9Q&}MB~eCaN~7&Op6(|1a8Xo zj~z!tCy%S4;Bk3OK5BT=v_0X-85&dRI>j*$i{O|w60wmy51x)mly=H`uG&2DbXPV0 ztkOJh9!@N_&r-(DTDF|(c)5<~&ueL}Wr8}`n&@ubF}Tre!^Jx*xh{oZnuy-P{RI7R z(>5mr*FoUYc#MtIo%?N95@BuU?wz}}9@TqlS)LZPMvrPSJj{ym%6Pnu4GBeoSdi`D z-m=9?->3xVg*D?H2S<#~qxJn7CKUD;r2Q3h)+$yFiH z_q3_qP=}A(1ie*roo9NfFEwlV*ADvXy?=hM-1Y0t>*v26#5QkwtmmC($Yc+~6MO}A zgzpN}%|~r``>%cZ(dP|;><&QTv@d9AA<8V7^YNYdP#$xg8%{wAZ-19(;WJU5OTk>V zt4vg}fs7X_!x4yH*i`CJNV>YU*~-EFv4J9WGKawAV22_B2!Y|fhh}PIyZf;Z9=r&S z=0lu!|G(AO+djTseOW|>e|IlF{oS0jR5P>HB`W@H(b9TVn+%E0ZoelNK?B3KpRd2@ z+=e7Mlr$iov)+$Tydk?`UdV^;p28vUBwq|iojn$69RZ^cDMGJ+ICsIshfx|Q>PvB~ zcv0c(Z1axx=@fT@80LF^Uk(08v<9St%n^r_3DpNldWI9V#%w@j23F$~pd-gF+F)B3 z;w0Sw9|T4^sX0`R-O*444x;tkr1ZbYe$D=ef|Jm|p@cobo%$@qJQ^u?_8r%6!=OEi zpBV%l87(9l$@ub?Ym`={06lqu3SatoTe7Jb_``j#Z^~-AqdI zkZd7|*n1cpH7G#plfB{cn&+iF0eOcaiE>ei!d6H`IU?buh{)@gY31dqvcq~4QBPi) z)IIa&Ul@i`9U*o=Di(*rbcKiS%*e|(k;G%M-qK!7tLbNh1n9vIZFSB8koHJ=WYOMn zj-`A2we9W0%c%(|mZKawWp)tdO?O!&24aJiH(@+2ra}xUP`rx)?mG%8G@K>)VZs;4 z8pG%LhWijX14VvK7uGmZf=s0Qw}nSMxP3m_*r2u;n?lB)9%wM_r|)qHrwB`(H|)t5CWn*K07bl0FZFu!^!ETVMaPI z4BHz*&rABhm8PGFd$)*SixX+K*P?H(LA8-H$|e^!)`tT5Iw@N%LI4yXb>_7WTUruL z@|M81htjw>)s9qzLhNv09?psp*#V3f5SYT4i1U<#@<96U(V_LRHGOd03p*&FPO??r zu3f{~9R!|*lwcjSo}f_2u#4>WEa6Veh1BqB{2ohhkPSff@O7v@;B{5|pq^rMLAR8Z zIBlYcrq9z4haS+;K_LQpot_1^zO?g8PpN2e4CY+;uI#C@Y?HL1ET0F1m+xD$9KO8h zNC0Y_iaiWF*-rjXvF-LGaKSOeWjt737)M!7o-KI8cUniG;cO%C1~o&24}+qGS04kr zx$-{-gR`Se1NC)gG$9)hu%;Y|1F~2$)#zZJk3xitQp?Pe#ga6+Xrl@#L?KoIV<_K#yA<5)e9p z^muuZ*L4_o#B)02(Ckc$RXihWDuA9OqJR{Ipd)QOu z4@Dn-jlk%F=#U^3N5km)EcPaOFQcR(@aW7sCn>e$^lIh9&vk;p^22!dcqMuQZFUem z0~FY+;i@f+8d2E#Aws9ePrg`Nz&`y|hXf(^Ib`yhu{>}oJfJZ9bKoNguL_}lNIjQH z@E-*m96TlP6!Mf(%+SH0o=Ml21}d_i3yZR^GUZOg3!*Y721siIl}-G4<(O(5vWu?L zDdA5B2ME4NLD*Uyh82OJu%1qkZ?-%^@UBiQI*d+NgIaN^PKE<+nnw!gG`%*Ah;~YW z+t^un5&}yL3YoDR6M_)Jb|KrP(;Ne`G=gCP$QXv*5Qs<;2O2yV;10^5nCga?X55bM zO1wH(Ynxr6oYoznU>tJej1aO9O^>^$x5%o zf2DoRxBo#-BAAVMDu1e_=lw+{GCne@9SCx^Zq!YgoqE=_I1k6kQ^Wq{Rq@%3ub%S+ zd;~oFGBPtcM(N>rb!GLH^+yQmMgV3!E%rB76l3RBB4Tnus~K8X^0|9tKew!_IBbJh9Q}qqtBwJ zo~8F}4Ev~l78-lH;U~#MXih0;9^0HQc})0ptd6<8j^;hGdCB2RpOJkAQXDvk2w{j~ z6;%5i5ZEE?6+WnIPP5gPUC&r%ZcnQ!c~bWIJd+2Ad=Z3&pR1ZABtIcOgg*!4kARO8 z0h|!iB}&PG74HER$;JBehptkHLXEf3P$}(HyFaB*i^!foZ?j`Y z81h|fgFKGsrMlqI(A){1EeWAE2RXaqCJ33imo$jj=x**@6EY##hv@^oqGUhOD1?Sr zZ~neQRm`5(%&F^9>-ehuyv^W(e9^sPNSh5zvTPMsy(8dxI3nr z=;|m9Y5DrQZtmN>EPGUs?k#7}fEm`rKOqeX6(SH*1UqT&-VHNUPN>FfuD548foELN&TQTVa#YzT zMcD#2VGfMcMv8F50OQ4a_VjNUDCAz1NR%Q<5)#NTwr+%lXjkrN_bO=Ds-?nqd8t(S z;^|Fe%{q&HQOgyt49sH~#yO|fKjB_+=BC7}=v9>O@9UzMTw@Vo7(`4&V7c-pyFEp} z4g=lP51t2U1cw-%9cP&8ZUSVcl){!>mRVuT`nK-vySr=`wg`A(OjOz~`^pUcV9G#h zNdcV*Ptg1Jr;*v(F?3<5?;4Gxdrj?|*4e{)Yp0wuhdyJO)##mdhMZG_!h&I_ZkuF< z3>fHm4F2H@J3q8+HJa9_Z(Ac&VG@{lbX5H4cXkMV1R)6sqK1?yLXU8tfLI=zYB7k2 zkr9Ln5EKcoC9$yHZ-{|b9|cblQ;H8oPlBh1JL}>=S3D6f0{9R@3PKwMto-Mp&V;%{ z?uG_7C5d5IqBLQbffo?S$1ukLvcR+~OfaI192}vc%pIYgv*XAK=@744b$X^^gyHjg zJ5!N$Q&jVx8lE!I(HWWk9Qsmk&B56CS$caOW#g}LE|F(mKL zL7|*Y!XEWXWrpY?ZiGF4y=Lb1S1#(|a-~HI6ZAqM5Qu3CszbwJZ|p#+nA>g)Z3{wy zGYNx(f)ungEulf>uZY+{?{t@tIQux#oK6vzOMF&xJRX2MLG}6gdS5{K!1u^fd{8}B zMof_e6A=>-MEKG}XXGBZL(}r13+rLoX;)g0nvZT#$*+GS zUacY!5=dqtswVRjHNcz`B}ghb0x<#=YYRMU9|3_eB$^R#w!(*CQLd-pD)NvQ(_f+nKb0aW}i={r)W#t!JBz4@`W?9lJ$ecy>Hg3GJiuQY7(7&udtV*y- zzu=$pR>Z4l4Zq8b#@G9UP6=1`lB`HJvLrv0epjlH)B$Bkd67UTgdqOo0zcujkkJ$7 z$i2C*06wYah=fW5+y}f4S`8eZnnDAb^}IT`am1N`vy31l6^stio)$`Z`PFr{+eM!# z#>h|9x(%es*7leY$T)VpLlXs=A)FX%FM#8i|7^UT=hPt>8xZjnIXa&D)vxz=k4(#L z`xsmFUDjkxJ%LN>*Y=9J&B&m8HZB=}teJfCuK)O^MA z`Dvyx2^a3am>-RF2AK67U5-Z+*199s%fGrB%f`1`F2g#s8CJ+BYCwSlG0@1mhJ+`O z43>Z@dJF6C*Ux05-09AMmq5RNR%VK92FUqh5eoPT$_zArQRT7V$xpt|gXq~^KX1YM zfqsEvBhV;yTndl_&M5+U1g-$s6s83+FvSS^0dNrYO3+${Hl|j!6{V`Z^S2`+5)xpN z7>Jc5Sx|}sMnsi_m0?Lj5@3{Kq$OdRND3J$mI;yopaYT_z?Y>8LZ&5Bh9EhD_do=| z5I^qse~v%I54;b&{0HX`?|+x+;P6rSPGy*0l=*_xR;YWoxoCXG_b;_9(MRNT3;byErB~Fa)d}jSx>4gN{-}8& z-aY8K0_0AE(F6WT`q%pb^=VnhBX}FgAi-5-2r7H|v>`$pw{HEx$wCkVvMnfdV4))f zC(wgAU>Sjd1Q1jiQw)OxBUCXKtv?uk8oTRRoXx|2HsNRb=xX+p`z|zA+{*E4QKfUG z()aXqS>r8m)>^!AHzq3ncxa%Nfm(V(36QA`SsPnB8`ny87ren z5&RA))zbb6a5*lFmR2TSKGqNBxJF5)Bldf^{hw*n*V*~MK2A<~M9|l%`*&)o*ynB`&5uE2{^VKM$*vW z+VKBQ7phpLN|4IdLy-&}$eqE|5bgTfes~v8%$M0dD7m?<$gT9BX=N)I!pKkLpKC8y z();Vji`!Bu-@tb&-=U&rzE4^%wEZ$$=f0QOnVE%odI+~p*p5!j64ZX&Qgc;~sZ^dO z|DvRSMKDj@g~>tZ@@m|tCS;OnLa2f0FbG31N!yt*2ZatP;|i|?aj_Ws=WJNtQpOa0jjF^Lm0LYy_GGdYA?PU#dpH5V$KqU}=oy{}GgYm%%PNisg| zN|vw^%%L+COvsbtR)?{HZIZ0QvnJ=uvu4aKw;PPxc`aL7j9DZ6n+I6jX-F_Z#b7X2 z2BD~0rWYK?Dgg=tfS?*EC>kZ0#DM~i6MU|HXg`3F5VW?2f`i25=Z+swvWu};zAyY) z20n^er@l0(dbmBYafJqrYYeFwlK_StXvqp>DKchBOetDqqQJR`iZe43VVL1F3lKVT>~wtK%epJ^teUWlCpAgKjN05m1&lQ>|Bb=yiaKn+gfMR0~|0brs=CM___(isMK zMH>W>;Y&$LEb#vEhnFKL#H?{fpc-x?qTcPs*7t5Ux4ysa_Z(lJ=kdq7dtKhW-;B*P zQy9kpH=tf`U&K`2Uhgzakn$4|#5)CQbwb_YX@j%_qZ5}YThSy!geb=y>XL>r5Hx6>Sh+yVtfmBaur}6=32H7iE# zma!4Kra}*hR*ABO5jq4O99y6S3rxzf@Ug*4JqQ%_z~<uQz_5a*YM3Jw zkKN_`Z$U`M%#!`T7KV(8+oykV_PdR@d-@C2UtG9Dv`)ltDS4GXB^MIAC0_zBkX_Jy znGcCR&>nZ+vpo&d(A>pURT$@_@bL4piX#}-6$F99DnFq5JH!KrgbB^G-`=SK?_jer z85o$rzM3yd^m}c3SqF!PX2}82%+ThjR*t?v+yL8EIw?9AA$8Z?$pT?aLLpg~asCff z(^}v0nyne&)oqd$!Hgpf-seFR#6%E)A$l}UY1|#}bME(hs~z4jb3( zc}ScYbLYc98Wrfl5I8RJq=7--RF49)Bcup3<0dkhSa~hwF_Da=_beyfrD!nAR+gf_ zM)d}=EXB4gV#YFvwtck_P%N^kN3~qNJk>c5phydEk?M?L2uO!9SMnu|KZjK`tt}=w(rIwDuf0B#Q32W!QIsUxt!U5*sGHsi z)`)W5fD!>zHDu~!bAgv#ooMNAo`KF5j2I!kb)m8EN@burJxK->8%k2b!J~S?ySWSo zJH25_XhW7ps9j`nfYYlYOV-E-B5jw#vQZUg0$cL1i02MTaM+z|E&PW|h zf>N78+XBH8Y1SpMGf56&X-L5s6x@ZCDTJE_28f7E156pL6M7;w4wQ(P%1{}iAwWc# zWw!EidY8H9)isNQqlmf1x)t^mrxyz3-B!%TTxd2Ww1-*Y3d!Y1oiI2M|9!dzBWwnd z=d8RHaX9lv!eXLgEtA zZSzthdg0d4q=FbAciDP8zIM0kqn7w|Hvu{qP#Quh5J?Oeq~q9Ewg(WJ_Jz~1l!O}) zi^Wn3>VRoN;2aDys9-CIQGuRbwi-NZzg^!0i4MwpDtE&HQu$j>-OpE05*!MaKuWf? z;ST%5xHE4@3Ey~BO@>5m@+FtDiV=WCEYRLUoE0|!_kC#5>l16(K!=)I*c3pC*ic=( z@Fr>)f|<>fg`u*a63u3f_|1ABRAh8Dsyt5pVc!E@D&|Q+a_%m>v_Ee5!w-GE{G6n6$7YfY@i-G?0xCWT;Nd|O+r&K2>7FoB1tg`U|rUe{mF4EajMI2O!q(Kl7bgQ7+rZ}kW(RedR zaU1b|n#xuXYYovLtqbf4h_SI-dI2L3q$1R{EfZB1Fx(HdG?ylo>^ z@`jVB%*;FsT)rx#p9j89y*u#a2!shNk`E7>MezHs#Sx-NH7snjN)(|v;poL%;Z!R$wls@!8rg z^ID6wq}rYd(%2uq^z;UoX|?DOh)wGI+VP9*%3C~VFRS)wIRCw|ScDP@2uc*pC`3dL zf&u&(U}y&bwx(>_F&dM*n4~-TN$Dkh+sL`9>d_y9yDj3}OXT>pl>6o+B!rk~CJ7Mh zL!;BU>Rl=O(~`g}F@|B6OMb9zII$9vk`j;

~qO%jmQ13@sgLIo*HN+b+~goRaq z^;5ZlKt^E&S&$V;kN^n_^&v<@K^6fKS$l39f^}A)=q#nFUP$V?T@`mV*e&g091{>+ z`c@lO)k@yYn<0D~%NHMu`m{7J~C4vXlR&VkrS`3Sb_?M6SK8P z;0mkkeaK9tFqF!Z3`j{dl>|u=#DoyQM1>_n5|@6_(kDRn^(RtU7Qme<;Zx_pxu>$G zV8noAF;Yj_?Dfiz(Nd@nxIb8O`13i?Dx#ln;ZqWRRXr0><1jI##BF_gNCPBLVFeS; z)3anlIMj)NVuv;Z?Uu0UKnNgwv+`0l-3AC~G6La+5`L%-zr<9yd{lY02hkA;`;nbf ze1VJz2>{eX8V){x$6)7Y3!NC-Rx5JVOL<$cP8VOr2h*KW$GbD`?scd5#rL4>fN&I` zI{X72qmUznQi8-J5D>Rt+h63V#Y|D%SMT)x{0~Nr z^$L-^?7e?BdeGh19BVR@BMhXbCWvpisq0hCr%5#^Nt1J?{UOjqAV`e4t1L(~5}G3j zfejqAU-Tk$nWJ70KhGm0GuWf0+{?jUef0hpW2qZK-rgnZqy$c?s2GI}Dp)mx`TlqP;11O^2pbOK8lejM z+8;a=!>keD#BG zgbWG+9rbnWo#fcsIihtjbhgq~&fSHGhJG@opbUxB>?vcfPCxxTyA-9J*bYjt>2uB)DX#=+jRHn;jIOIB=v& z>$_Dq1Tp2N%P|p|COUL(2QD$mx`x_njd7|+F~GDzg%k)_*-D8-W!YlP$hM8`xU&On zsK{WlFak*kYk^Y~t!%1MJ*o&j5g|tyMEL2N1eGKbMi3y7kdcImAc9J!AtWHGh@jG_ z$ZQ}bJCQ6H0i`TemNrxV(#wJE%y_hg;)kYOgJRj>X~jkCjww1W29zv@1|dorDr5A% zPJlKrA!)YMwPm4bQiY|7Vp*0pHlaa*js^xBAe5DX9;^e1qO>+h!%S7AuvSyDtCuKP zrVA@t^#P(QUDP!`v^^qwNAM58J;a2BpO@uPc}oRD#Z&6~n|W@+ zEC>mRk`>P?5{ZUpA%bCj ztP+tZF$oYV%`_A%1qv+-KvM*eC(1uyeW<^bQyl~fE=X!TMp%nd0xPqd!>um4%@ozzdepTT}xTE>#ZUQ*aIl;Ft&I z9NzHE0L=`=O!Gr9497Jy5X{UR!OSzvGfeiokMLlgUv~(`F^p!*D8?${E+X_p%CtHi z4yw-*v4V%{#S*EeGEC3N!4gnlSs>74C;^m=pcfbM5@s_p32X3ENXlD8YQ~K;fdnAq zZHPXT-1nqk;h)3SKMIiq{5@ExVofVSLV@T)15l9=RG6j}YBC?Dr%z{5!IlnIVhM~z zVfsT{$*fBDG4S9NGL#vLL@;*1aNUNQD_Xn_G6BummEfLh=;Wx^5l^YL^zNRJgF$&A(fydP!&uc$cwjcIc_wjIC+~kb1SwcyB&`< znrrVffXX@uF0J%)h}z@35jf$swZEkk|2iq8_VgeXFaC@rO=q`FI)3y;;Uh4?gWdGome`}S^`u+lL#_W|p$SnzeYM)sxH7|Av{bG&QN{up+VzERB z#6bfZ21kfAo47@RZ0-R%9Q<%iAJj~l_GQGA%%)V4QM3Ql9Y5)$S#QMSF!P*L-afsdUr$0 zUQ&tv6#Z!4?uq*sa<%IAS6!=FW3S`d*|u554m^Ka9-(&!3=GQbHQLJNN%uvatm+7`AfrVeA(EhXm zKH93Ph^nfps;a80%=~ZeFUgpmf#Qnt3$y0=ex#)f#hwe@s)E;Zg|ulNwls_*GuGs9 z(9f1za1_@RS>-K9YNQdj7Y_ zs;l)^-93GW>3u-}BP5W3kzzQc z2_>bhGP5fKGL`up3n!vaVm*lY7n_Q4JB4%Ggoi-b3=|LOI5xwl@KE@C7u@?J%+V^R zuJ8JlqUNaMVKJ9?Pj+>A{G$8M8eQYM!xk^VwYoEC-(QE?E)o>V$&<=gxh@=8d{do0 zc&FMvXpuyf_rsF<*ayr79?zr%1I$8uYA)4N!BOo|;15Vwq9>$P_LJ=g*sA+tzZ$3M zm+u$vm+xP~f4{!*uetFleCok{KRuHO_S90*ZCjQ1{s3{@xr~gCxv$zZYlc>{Z5dii zSWUDuigOycqcl(3yl7}cgb%F&#SNtj@_(ErEiO@qilje&P=hd(R%T*lexKid82e>E z>RA}9j6P9svOT!ZYm;o9l!i^yD;Z4XoYt!IZ~5rZ}O96^0>D*}=&;21f_MQ~Xss zfaojWc02@=9*y@n`q?Z`md#FPVUgB`wCmt)(@_Mb!xQmH;7C$T5v8TY zg~-7JjR}e*gwd=p72?e$1jHq)avED=i4ItR$uQNZ0CgFW4ghNjm{o)@%QG_!Wd}0)$Tf z{`tH6&fxBQj~BoFTSL#*mVNNWdQp#UEX$9it|^tltISgclN zW@nb^62v=%^!j}=nh1lxz#xh?8cui`5~Ig$qH`1Nq?-W2~ABv}K2izPE;X9UOIw8{O-e)qE13wgi3tRw!;2@htMk6SMP>4Yw zu@u@=?_iDh^~A#SKqx`<6ircyp!E)+dW`?%z0M$3Ch~3al8{0IY~J>4`X0}D7QB%H zknB_sGt$zl_ht}8NE84Uumkba2(vaiYNnGK z)lKZm*yyRg3;awh$}CxKS-W*T6LPe?6V>YVcn`G~2Y>Ohi9{rmf>cRf7&wklaiZMXaVB6H6;@j2G94IpWz ziRTu3CiFLZ_`-XD`28f1uz(n;fM!#8n#nL^HHD#y)=JX~V=0CVVfR=DWJVfP(30Jh zmWCU#Qq2uC&Owr{NVv#pfE^GRhOtIwRJ55S+Xj{qqb(S%FwvDsw2jG&GN98_snqY= zuJ81hL439YofbM24QC=X5?0B2eNgxf3RXAn63-`CTz3*4>d;?(RRYu&n zi#ivyB?Sjl!W7Xk;y|3rUWiZ($_0pAC8b>2%gNB3Ny+DDCpRKhzz2dzhkJB46VTi- zs-6)0- z6oya>A|;`3DNW+iOjd5zk&4D*5_MupI7({JY0SC676#2AZlL5-WE>&L)DjU=$N_LJ z7EQozHzuK^pm-JlkPskO8e??ij7Fr|WYwC6R5h5PtwU2CG`E)FKV>kjhQ*8y%50gm z-sx!>V#G|uNJbKZ5b$nTxH@D)CD|kTZj?K@s%8~3$PCD*Mo`8oVWJfeUW1GvrY*VyE1d@=9*TVxMSfWnNKf zihZSDZ4c6s_EVf+sUM6#Sp7%p{>q2*)bgqBV81VhBj?0$%wsS&p}h_Jw^YnjPSiWn zgrj{X?Z(Es@GUJcsP1=nd>)*+IdbLpMf%UU55WJ7ALWm{cH2tfiw8j?+|Rn23q`0-+y6fAye$YMk!lcaLa8T6h=db)#J9c!v~syfS5 zzXrR^Ar7OM27sppRKL+loIXM#Jdol!#ndbU)(bMM#Z&eY_53wHM0N)V2R1@smUOA& zirJ~yrjL5Au@IC6Tof}1rFhY%G|-VWEbB0g!wi9fmPy5RDw~-mV6sz;DY`O~NSJT| zKr=B@fR-j8nid9$mMWqpik&L@U8n~LnlS*8NZKq;kdTCs<*dLxy9;mS5G}tNs|5|wbrZ4c2*y_(EmnM!mwFmU|8*6dlEql^ub0> zAuR?&j4;H850Dj85&`z%ZI}uYKcvBov5qcf&|x1uuf9CwdPREXSWuvH01oJOhco#) zOX2##CXl8m5+MjsJJ8cLL`R6D+Vl_af|20`5IF)pf_4-HG@(E=C`CeqN~ebirzIh1 zA`|3Ogh1;Y!b_l>uHW`QW>#ZMawRSG9CVhd zaSx*;WGBsrs$epvB(TH~NDU~=(1ALZJEugtNjQW$s3+lo;B=gWtB z?jA=XKqd(!nn*%vi4q|qLL?$ULWq({29^boCJG|!7ssRj31~Wn5jue8iO=eg6A+M) zs;W`x@)orsoY&vu)8nfBs3?KldxSmN$btw@ehyT+1^fl$cjcc|?e(Qs;xCj-8Bo^$k}4uOg;X#N0wIYwZcoD{&jlj)_KjIzsO|# z-)GlpuV=7!{x+vu!mIbDIWQm7PkKpzKP*|xCsmeMoKkWZ?y2cwdF5)>GE5MZiAa_p z$O4f;8}(%yqzuUq-eh^C3^E4A05C{=_-;ZtbB5-_3=D_L3{8*}W(F8$211|#{7|8% zT_iXv95_iGJ5#~;B()xp??}BIkanncE86v-3vRsa(Z$~9fzi*q`m)H%&j|V{A0r6V zxFhTDeiK!c`ZBJ#udBynIk~S@<8v!cJQ@9~$w}%Dkk zeovLcaINF0_m{kM9{RoK-uvBL_eUKRke2-A&pc$K_XPu~HI%}VT_7V^Od}|=i#P<} zH7Mu^Xocz;0ibBVYpJXIf8U5;Y8r$IF@zxvccZ-s%eSBxXLL6~j3ed`&ZMM>5J?17 zpvTA`*s6q$Au1FJO)7!%sz@RNE+`Mn^AF&ZSrU{WNrk3~qcXu_v9p*8=a*B?=`E?# zJ}b$)x3n}0rmO`}91B2*CJ)1~P#9r>g%6}MY01Z{&F0)Hy6|6_z^mwoK1%VKHc&{g zi}|WWz%9nv0KoDlnrx-=-aF9gPO_mY63Z=<44b8E!S^b^$^$ug?>%5}L=Duy3a-uo zqj;F9;08`t<$@KMAy_V=I;d&~tPeads+?^tl(dpD-is1)bC{im4?$UpiH^vck_LJ4 zFK%XJ!3~6oiEL+WDZBai4{!m|L)?eB58T^jkXuMZ zwBYC_1n=dU53fAP!1N}4ADjjHV175>g8>+(En06^>2Q7yVBr|yL_fWl#w@}R3DKte zMCd0%Oi8q!e7?bPAqBV`3CX=0LF0}}9K5*NpkHpjZPo59qGkwYMr4o4r8OhxGD{38 zt17KxEf^(>Dy*1>u%k#yL2!XwkjPU86lO()BnW{d36PRWWswLF22_PfMiPmpM3Pb< zJ3~E}5N`UAgAr&r4jpL@4g|$rL#kqykfr7n%mUb{DJUa|lM-yx8qyOB0W1+{oJtoA zH7v1KVvNY%EhF(_n@Ng-g^3mk7DDq<0KtVIA|~%NONadGn!kpBWx%PzQyV3xBAz=? zEKD|{;y=`)1aBe?6*rQ2EQLCTMq;ciGGSvTSStGjqVRThc5nAZ&r7c)s}lmuk%l5S zCwp<{h|H@pG^8R*Yd3aGV~a8y7%@QfjH5Fk#F|u22yhb8)eBpMIf8XNxX#Q>e%SUC zq8(|nR3$=LWoaxX-ihf#rAn1BRJbV#eSWjo#Q^+B0L^W1#SBft+=jwtUz@41Kp?7c z5P{Ky1`HTr;)wwh(7YfF4k=_bvMByU5KKkM3|wCp97&yoRE zgc<9JMX~6nf}P9|CMcj-p+jO+U~>ky|ui=U|2{Sa$L!qEr zj4ejiH3g|#&cef;rKh_ro4zCQQ{p0KB76l`X%**6xF~j#yGbtEfYDG51QSF=&_Q$r zG!c6W>Vv3<#1Duc7UtI1nkaXuAodh;GAB9d1PV|!ot%)@T|+?8&o8>V%IgZ9nyS}* z1CdF2!-5)4r|(qmLS$w&3DY3snwFK8mYHo;*I{ujhp|2|ISgPTHx4Am0fB)jDJgDV zND3GPg;5BV-6WV`OIb=O3T~weRnfzz=AI?-Qm6NeEMqKJj4r8tQj34lVQVE>0xJEv zD3)XunH5W&S|Ga-`#Amzodmvc=PLh+epeB2Z`P8+cvFhFt92*c)@rTcKO9xYbmZLv zSwqUKu7*$~@81zJ%CfAFrww2rhX9b0m6Qty;;Z*!K7IB1 zCGM5(3T2c`iz+5bh?Wv!qbnd}tVNPwRf1&7uqly}vM~z^wG1SJEToV!2*Q$JvW1x< z$jM_Q8Ce)GgA*7rNXX1G2177HAq=QUzyz{{vJk*iW|3dsCgeWK|pezc$p znM^VaIMlQpuvm;B3>a3!0wDmv5+=YL0|^W<3l&uXmJq`zB0>=ukRYQ58zO=~Vy9df zUk{EaK6&%?G@hbaWo0ELmR3?yPSQP&#mHEPYWsbH89M?B`I0IJlkBRpP{{JNI?OXL z&1)eGA%J}ULtMdrjYBL;ccbY&LFtAJQZZBcY5r<@u`yDs{u(Jv2=q%lU!DePN&;3< zso9|NZ4AHV-KcDHz8v$hEWWJoqo^6pV^P46H3_-s97p>OkLcPIBlaqPWM9=&%g>)3 zx2^j(qj_nbDL0E)EK0O2Mf$38Z?4M~o<)3-O9yG>sSh7)L+yT$y%9oErqM6I_|VoN zgRux2u8@rwgs;Yt|DW5@#&ye(>AQ#eX3V~j+4QO6$T2}piiQ>umP*N4979Cpwa~>58+Z+(7Hm;z%n7J)MU4-lTUrX zw-_#Pp{Yda0zCr|4!KZnMF5T!i}x4(6kH2ERw3cg^3t1O47A4bE@ z<4KrvSlR3`s;0o}cbduFc^MljbXk$^?KN#4y&V_O`1yQZ0Yj*i|3NhXK-nO>h)Se9 zQgia7?>nslwM<9hNbqj9bV(=xJ)v<5fSD-|aQ03R)4M6<>p-5jKJlqoVs}LzH1*H$ zXhHSlrVxkQ@9nT7>nvnLPr=`cr?Aqlg*^cm4dB`;ciD2p4G_$;(~A2c)KDJHg%Nzu zQ1UY1m{k;f(YS!xoM0Xa3Bk`c0{Q{r29)`lbddIi@>K;6;n-o8e8m9og%El*O&3b# z4*b4zNICN1f}3LGHP4;*7!36MK4#i|CKkqA*rLOdyYu@Z#t7bgYWwL`OH z3P6S+Ut(d@WJ`67vD+REZmp+tyFuNF#ogU(?qW_AWbW^6R?*z=Ee74qOC9R&ZQQ`Q znTscDL{Y?w^2n6aM;C#L90;SN2?LNCN28>tqaKpgX%va0LrH@W4T&bFQU;YJ2uMjO zaZ@g6i=|FFqpdY5^mUHMhNDVZj#c?YJug2qV4g~x(L8z8EbP}>p8nRtI!3A9GmnDT zi9USRnChkJch`WyUm_=w3)m4*>wj)1Q2NHfZbRZH#C_rR1ge!kAt)I)3(M6HEC)Cti-nDi zb2Bqc%+0)BO?E=aMiGLE`wNG;?nF6>>w88w;%f}eVbh9_f6LviYW05SG&_#|`1>Qj ze)`s~?(8q9`cI_zUFnhBV*3H$h}X>`wkTHgvU_IdT%@wIlaf5#xab3a!6Si1$CGWMGVPkX+{Z3Q;7@R z1klkn-Duc_byaI31O<@wyH3Nf_+Shpr|nVn3^Q1JUaRtsRXspy zJgHd%1Ki$JMIuGx4EPE%9PBF!voN2N;)WoluH%9CmiE`ei6oLSjw+raJOTjCdE#N1 zIhcy!fIm8>2o9AmV-*PzUy3YIa$bROY1;#hV`{A?w>{JcVB8sxrM|RzV+d7pg{Xf} zAgKXhqh&*-9e3r-HI3wZa+TFvkHY%rQ|@dK=r}JBvriSlx;=S*)(!Gv41Brt3 zzce=?y3Q=cq?5u}#xaa%(%K}PI<@j(D!@eKsY;EZWr2VeCCx`JPpR~KRXd%)SXS_9 zrk~)w&w=_X`qF)nKC6T5DY{h|o#&>%7{)P-V;+4OAtWB)^r}JX9RsD79S(f!YGC0- z5sYIP#xdb14Nb_MktsUYh*U+3Wdj97P#?Gu2M4GS_SB5cBQmC-zPm!i?7vU5=-L|- zX7=pa_xvh*sz*kYB^n3F?p-J-qA`x?)KXbx)Yi8omIg;>?-#6I!|3f)`c*Qj`3GRQ zGXkfi-=H_)R_v&!Jm!|FQ-(1mXh8i@8h7c%Kyv(r3Q4NF^j8Pok3L&9|m;@@` zW?8AjNPgWSFAPXaG9cT<1tF#)1yvFB~612HrU*Vn5Q z3D}fYsCtPAr+6u61++Dc>^rA@^lekW;nSQvRe0)CtHV2081znjT<$`#OAtt-E$GL4`>{D9T=g%Eh)|MErOm7gw#WUmtAaeS7zNN z0(8MNrUFnl;@oD{gIY}t8Q=;$L}Hm$BNt92@SNFd+1xJXm+-r}z9De}MTU@yT5<`b zEm)})4@@lGDH%s$AQat$TdE-hy`oOrHUiO9xH=*jVp7|(f*WWA#9m+kRGqv;(DVz3 z89D*pks;#{aYg3BrU8Id)X8P3GffPKgMArcfwXTZ%n6Nx9B2xTC{ngG^#u*DWpvJ} zbqt`)s$G*l@vYR4e12)X;lqCrmWIIvykIzgGSMYqh+oAAy)0HE&e3f`X5^UWPE9#P z!3a$!S!JI(Dhh+IUs>p`yQ;WYAzNBZOiW!wt%2G@Rz3=rVuy%~J@Bo3 z*6{UfN}rPy+&X8{h3H+xF5>8hgP_VtcM&2)?5crjm@qM!=Dm|}p6H<-^?aEc!|2;z zIdl=oib@mU*d_fXKH&&w`Ns|+AfUoD63Zhg3PD2xZpJcYWr$`KFS#Xh*|IWetuP?~ zk%@h?$*iR9=Iwl|iTOa?bsQw_1is4du3GX+*MMKZ!GETBCL-lx_oSU^Sv z2%IqEDbiP#3CYz8N0Q5sX?+LZdoQ_s)joQgu*Nrl#bj?K*Wt~9nlUyMmatf1V3P!a zh7oPHnliDD0?Df(X`GoYwMbxui5yYdiz-4SCe#o#ks1b6!ZjF7jT%N&9GKW%MwV^LnOlpPzeZ3hoST#G8C~2F$*{< zdrBFVXsXgDyej#l@ltzy{*ID-?%ok@7-a7YKMEGb_3o63WT$yVwQ-gPk9i}ECuuiM zHDVGOA!ZnH1Hdr|g%jIYE|JihLLm^mB@*8%5!ERqHV7#eIg8@zUG zO)_=kSGh39yb9tiohW8rTNRavJyC!J z@V^yPI3grCJy2qC>LgrU;A8=k6(AkDqt$wyf0Ck?h71-UFlk+!(_8m^kB;xrKRZ^P1R{FDE4|*Y^FUjY^IG;u}?%iimxG8 z*o)mS$X}GdwSR2MI|LGTTfVvFG!QHh4& zwIqBb&A`yFf-xpljjEaw^H@VBEy<-Xgg<^&A zj8QN`1tTE{!~+UJ2@NaUjM76%V0>j5RLEdpWs1brp%zvt-pNKLD;W%8F^mRAWd?z& zC5dFdB{3w*uuP;{RtLle0Z6GDR)l6DD1mS?35vyhQeaG2VK86;fD(Zq5~@i=kjTVP zfeL}WqfN&ni>{a@fcQ1Pm5AAtvoGNWnLB* zHp4%ccl?96cz93le*Jwl$M zh?v229_UJcb3PO9{^N+0>H>g zFi9Xp#1T&$K)nk#LPR8z$v5FV_rw{Hl8Pk>DM9Uxb-!*y_P=TNmRSg0wG6m2Ot`5! zT?|IxxNAP#N;5~;YNR?bKn?1f!+?_n0f7LR7DMqkJzrvEfc>wzMihXgF_>6}?){fI zT+(qJm+>E?^_Oeg7x=139}mcI4{(QI!V(ch7l@^OY%JiD;g#v5!k!V~T+({$*H2hJ z@jmhOhu#)gSz}pQU`~nRQ2MC~Ms%bpxP1-c7nh)3hu)uf{rkUW%))hk`CjM5J|Xn;hk4!INPEJH z%4zTik43MirsVH^;(3@yT0 zP?vbAEjZNXFktUPc)%+9dp=K9l!``hcRQI>%rg!7$PxsS zP>G~;{T!QUnV{I>OKO6TdAu zWO^A{Q^^@vFCk>II_xl6F+7ZfkVyInVwChlG$G_ZQ664Lk?0Hp`)w`#=E{!3j>0C; zRUj&xEDIJQK}!X8QqXob8`AK<)Q_N8BLpBoLlF>rz&~RE423d?Yw4=Q&7p{33H1nd zIt>o0qEMh!g&C6!qcK(YNq$^NTb2tW5{$ttLnJX)B*lIjh}K3GWK1zb6%%<@i%k{A%lKbjw}png!JXnBZf4=RxkgAmKZh*x$cp>fkF=VxWD@`fi= z&6w`wx+g%S6G5&=62i6FWz0otp$XN|xn>gVxub=)>{XB(_DTsubzDtuQ5=T{9hMlZivrly*lScSDsOsrXet0%q!U$!6I5BHXov`ZrQrR@ z=_hs-{&|Y;)(r1;3sQG(@gTFCx;pcZ_PX;697jdsPb)B7sPL1qakbcD~v1 zK*~RoPj&7;u@aaM5dqM6`hG_dv1L^EAMpfNmyYQW|^-tX%dNnWeRoy?J Pzy2=dig2MIz~qsxrm1(z diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/Chart.yaml b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/Chart.yaml index 04aba91f..e8f67b95 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/Chart.yaml +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/Chart.yaml @@ -1,7 +1,7 @@ annotations: artifacthub.io/category: monitoring-logging artifacthub.io/changes: | - - allow setting global labels for each resource using `.Values.global.extraLabels`. See [#2607](https://github.com/VictoriaMetrics/helm-charts/issues/2607). + - reverted usage of `app` label in `vm.selectorLabels` artifacthub.io/license: Apache-2.0 artifacthub.io/links: | - name: Sources @@ -11,7 +11,8 @@ annotations: artifacthub.io/readme: | # VictoriaMetrics Common Helm chart - Chart documentation is available [here](https://docs.victoriametrics.com/helm/victoria-metrics-common/) + Chart documentation is available [here](https://docs.victoriametrics.com/helm/victoria-metrics-common/). + Changelog is [here](https://docs.victoriametrics.com/helm/victoria-metrics-common/changelog/). apiVersion: v2 description: VictoriaMetrics Common - contains shared templates for all Victoria Metrics helm charts @@ -29,4 +30,4 @@ name: victoria-metrics-common sources: - https://github.com/VictoriaMetrics/helm-charts type: library -version: 0.0.46 +version: 0.3.0 diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/README.md b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/README.md index 9ba0b4f5..c8ee7051 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/README.md +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/README.md @@ -1,3 +1,4 @@ # VictoriaMetrics Common Helm chart -Chart documentation is available [here](https://docs.victoriametrics.com/helm/victoria-metrics-common/) +Chart documentation is available [here](https://docs.victoriametrics.com/helm/victoria-metrics-common/). +Changelog is [here](https://docs.victoriametrics.com/helm/victoria-metrics-common/changelog/). diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/RELEASE_NOTES b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/RELEASE_NOTES index 055cd4c5..ad0cdfa9 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/RELEASE_NOTES +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/RELEASE_NOTES @@ -1,7 +1,7 @@ -# Release notes for version 0.0.46 +# Release notes for version 0.3.0 -**Release date:** 23 Dec 2025 +**Release date:** 16 Apr 2026 ![Helm: v3](https://img.shields.io/badge/Helm-v3.14%2B-informational?color=informational&logo=helm&link=https%3A%2F%2Fgithub.com%2Fhelm%2Fhelm%2Freleases%2Ftag%2Fv3.14.0) -- allow setting global labels for each resource using `.Values.global.extraLabels`. See [#2607](https://github.com/VictoriaMetrics/helm-charts/issues/2607). +- reverted usage of `app` label in `vm.selectorLabels` diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/templates/_helpers.tpl b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/templates/_helpers.tpl index 99ac30d5..bd945678 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/templates/_helpers.tpl +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/templates/_helpers.tpl @@ -125,8 +125,8 @@ If release name contains chart name it will be used as a full name. {{- $ctx := . -}} {{- $values := $Values -}} {{- range $ak := $appKey }} - {{- $values = ternary (default dict) (index $values $ak | default dict) (empty $values) -}} - {{- $ctx = ternary (default dict) (index $ctx $ak | default dict) (empty $ctx) -}} + {{- $values = ternary (dict) (index $values $ak | default dict) (empty $values) -}} + {{- $ctx = ternary (dict) (index $ctx $ak | default dict) (empty $ctx) -}} {{- if and (empty $values) (empty $ctx) -}} {{- fail (printf "No data for appKey %s" (join "->" $appKey)) -}} {{- end -}} @@ -187,6 +187,9 @@ If release name contains chart name it will be used as a full name. {{- include "vm.validate.args" . -}} {{- $Release := (.helm).Release | default .Release -}} {{- $labels := fromYaml (include "vm.selectorLabels" .) -}} + {{- with $labels.app -}} + {{- $_ := set $labels "app.kubernetes.io/component" . -}} + {{- end -}} {{- $labels = mergeOverwrite $labels (.extraLabels | default dict) -}} {{- $_ := set $labels "app.kubernetes.io/managed-by" $Release.Service -}} {{- toYaml $labels -}} @@ -197,7 +200,7 @@ If release name contains chart name it will be used as a full name. {{- include "vm.validate.args" . -}} {{- $Values := (.helm).Values | default .Values -}} {{- $globalLabels := deepCopy (($Values.global).extraLabels | default dict) -}} - {{- $labels := fromYaml (include "vm.selectorLabels" .) -}} + {{- $labels := fromYaml (include "vm.commonLabels" .) -}} {{- $labels = mergeOverwrite $globalLabels $labels (fromYaml (include "vm.metaLabels" .)) -}} {{- with (include "vm.image.tag" .) }} {{- $_ := set $labels "app.kubernetes.io/version" (regexReplaceAll "(.*)(@sha.*)" . "${1}") -}} @@ -231,11 +234,16 @@ If release name contains chart name it will be used as a full name. {{- $_ := set $labels "app.kubernetes.io/name" (include "vm.name" .) -}} {{- $_ := set $labels "app.kubernetes.io/instance" (include "vm.release" .) -}} {{- with (include "vm.app.name" .) -}} - {{- if eq $.style "managed" -}} - {{- $_ := set $labels "app.kubernetes.io/component" (printf "%s-%s" (include "vm.name" $) .) -}} - {{- else -}} - {{- $_ := set $labels "app" . -}} - {{- end -}} + {{- $_ := set $labels "app" . -}} {{- end -}} {{- toYaml $labels -}} {{- end }} + +{{- define "vm.commonLabels" -}} + {{- $labels := fromYaml (include "vm.selectorLabels" . ) -}} + {{- with $labels.app -}} + {{- $_ := set $labels "app.kubernetes.io/component" . -}} + {{- $_ := unset $labels "app" -}} + {{- end -}} + {{- toYaml $labels -}} +{{- end -}} diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/templates/_image.tpl b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/templates/_image.tpl index 6d30bd03..66d019e3 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/templates/_image.tpl +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/templates/_image.tpl @@ -42,8 +42,8 @@ Victoria Metrics Image {{- with .appKey -}} {{- $appKey := ternary (list .) . (kindIs "string" .) -}} {{- range $ak := $appKey -}} - {{- $values = ternary (default dict) (index $values $ak | default dict) (empty $values) -}} - {{- $ctx = ternary (default dict) (index $ctx $ak | default dict) (empty $ctx) -}} + {{- $values = ternary (dict) (index $values $ak | default dict) (empty $values) -}} + {{- $ctx = ternary (dict) (index $ctx $ak | default dict) (empty $ctx) -}} {{- if and (empty $values) (empty $ctx) -}} {{- fail (printf "No data for appKey %s" (join "->" $appKey)) -}} {{- end -}} diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/templates/_pod.tpl b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/templates/_pod.tpl index 7534ae2d..62981dfb 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/templates/_pod.tpl +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/templates/_pod.tpl @@ -39,9 +39,9 @@ Render probe {{- define "vm.probe" -}} {{- /* undefined value */ -}} {{- $null := (fromYaml "value: null").value -}} - {{- $probe := dig .type (default dict) .app.probe -}} + {{- $probe := dig .type (dict) .app.probe -}} {{- $probeType := "" -}} - {{- $defaultProbe := default dict -}} + {{- $defaultProbe := dict -}} {{- if ne (dig "httpGet" $null $probe) $null -}} {{- /* httpGet probe */ -}} {{- $defaultProbe = dict "path" (include "vm.probe.http.path" .) "scheme" (include "vm.probe.http.scheme" .) "port" (include "vm.probe.port" .) -}} @@ -51,7 +51,7 @@ Render probe {{- $defaultProbe = dict "port" (include "vm.probe.port" .) -}} {{- $probeType = "tcpSocket" -}} {{- end -}} - {{- $defaultProbe = ternary (default dict) (dict $probeType $defaultProbe) (empty $probeType) -}} + {{- $defaultProbe = ternary (dict) (dict $probeType $defaultProbe) (empty $probeType) -}} {{- $probe = mergeOverwrite $defaultProbe $probe -}} {{- range $key, $value := $probe -}} {{- if and (has (kindOf $value) (list "object" "map")) (ne $key $probeType) -}} @@ -100,7 +100,7 @@ Net probe port command line arguments */ -}} {{- define "vm.args" -}} - {{- $args := default list -}} + {{- $args := list -}} {{- range $key, $value := . -}} {{- if not $key -}} {{- fail "Empty key in command line args is not allowed" -}} diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/templates/_service.tpl b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/templates/_service.tpl index 77a1365f..77cdeee7 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/templates/_service.tpl +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/templates/_service.tpl @@ -37,10 +37,10 @@ {{- $values := $Values -}} {{- $ctx := . -}} {{- range $ak := $appKey -}} - {{- $values = ternary (default dict) (index $values $ak | default dict) (empty $values) -}} - {{- $ctx = ternary (default dict) (index $ctx $ak | default dict) (empty $ctx) -}} + {{- $values = ternary (dict) (index $values $ak | default dict) (empty $values) -}} + {{- $ctx = ternary (dict) (index $ctx $ak | default dict) (empty $ctx) -}} {{- end -}} - {{- $spec := default dict -}} + {{- $spec := dict -}} {{- if $ctx -}} {{- $spec = $ctx -}} {{- else if $values -}} @@ -69,10 +69,10 @@ {{- $values := $Values -}} {{- $ctx := . -}} {{- range $ak := $appKey -}} - {{- $values = ternary (default dict) (index $values $ak | default dict) (empty $values) -}} - {{- $ctx = ternary (default dict) (index $ctx $ak | default dict) (empty $ctx) -}} + {{- $values = ternary (dict) (index $values $ak | default dict) (empty $values) -}} + {{- $ctx = ternary (dict) (index $ctx $ak | default dict) (empty $ctx) -}} {{- end -}} - {{- $spec := default dict -}} + {{- $spec := dict -}} {{- if $values -}} {{- $spec = $values -}} {{- else if $ctx -}} diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/templates/NOTES.txt b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/templates/NOTES.txt index 7fb3fbd9..41f39937 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/templates/NOTES.txt +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/templates/NOTES.txt @@ -1,5 +1,5 @@ {{ include "vm.name" . }} has been installed. Check its status by running: - kubectl --namespace {{ include "vm.namespace" . }} get pods -l "app.kubernetes.io/instance={{ $.Release.Name }}" + kubectl --namespace {{ include "vm.namespace" . }} get pods -l "app.kubernetes.io/name={{ include "vm.name" . }}" -l "app.kubernetes.io/instance={{ include "vm.release" . }}" Get more information on https://github.com/VictoriaMetrics/helm-charts/tree/master/charts/victoria-metrics-operator. See "Getting started guide for VM Operator" on https://docs.victoriametrics.com/guides/getting-started-with-vm-operator diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/templates/pdb.yaml b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/templates/pdb.yaml index 8abd9dea..9ec3c2c2 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/templates/pdb.yaml +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/templates/pdb.yaml @@ -18,6 +18,9 @@ spec: {{- with $pdb.maxUnavailable }} maxUnavailable: {{ . }} {{- end }} + {{- with $pdb.unhealthyPodEvictionPolicy }} + unhealthyPodEvictionPolicy: {{ . }} + {{- end }} selector: matchLabels: {{ include "vm.selectorLabels" $ctx | nindent 6 }} {{- end }} diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/templates/server.yaml b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/templates/server.yaml index e224c074..9d7fd7ed 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/templates/server.yaml +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/templates/server.yaml @@ -18,9 +18,12 @@ metadata: annotations: {{ toYaml . | nindent 4 }} {{- end }} spec: - replicas: {{.Values.replicaCount }} + replicas: {{ .Values.replicaCount }} selector: matchLabels: {{ include "vm.selectorLabels" $ctx | nindent 6 }} + {{- with .Values.strategy }} + strategy: {{ toYaml . | nindent 4 }} + {{- end }} template: metadata: {{- with .Values.annotations }} diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/templates/webhook.yaml b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/templates/webhook.yaml index 51039cae..e38f7448 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/templates/webhook.yaml +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/templates/webhook.yaml @@ -1,22 +1,31 @@ -{{- if .Values.admissionWebhooks.enabled }} +{{- $webhooks := .Values.admissionWebhooks }} +{{- if $webhooks.enabled }} {{- $ctx := dict "helm" . "extraLabels" .Values.extraLabels }} {{- $tls := fromYaml (include "vm-operator.certs" $ctx) }} {{- $fullname := include "vm.plain.fullname" $ctx }} {{- $domain := ((.Values.global).cluster).dnsDomain }} {{- $ns := include "vm.namespace" $ctx }} -{{- $certManager := .Values.admissionWebhooks.certManager }} +{{- $certManager := $webhooks.certManager }} {{- $files := .Files }} {{- $crds := $files.Get "crd.yaml" | splitList "---" }} -{{- $disabledFor := .Values.admissionWebhooks.disabledFor | default list }} +{{- $disabledFor := $webhooks.disabledFor | default list }} --- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: name: {{ $fullname }}-admission - {{- if $certManager.enabled }} + {{- if or $certManager.enabled $webhooks.annotations }} annotations: + {{- if $certManager.enabled }} certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-validation" $ns $fullname | quote }} cert-manager.io/inject-ca-from: {{ printf "%s/%s-validation" $ns $fullname | quote }} + {{- end }} + {{- with $certManager.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $webhooks.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} labels: {{ include "vm.labels" $ctx | nindent 4 }} webhooks: @@ -34,7 +43,7 @@ webhooks: {{- if not $certManager.enabled }} caBundle: {{ $tls.caCert }} {{- end }} - failurePolicy: {{ $.Values.admissionWebhooks.policy }} + failurePolicy: {{ $webhooks.policy }} name: '{{ $crd.metadata.name }}' admissionReviewVersions: - v1 diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/values.yaml b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/values.yaml index 279ceee9..3a1027f2 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/values.yaml +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/values.yaml @@ -125,7 +125,7 @@ crds: # whenUnsatisfiable: DoNotSchedule # labelSelector: # matchLabels: - # app: alertmanager + # app.kubernetes.io/component: alertmanager # -- Labels to add to the upgrade job labels: {} @@ -276,8 +276,12 @@ service: # -- See `kubectl explain poddisruptionbudget.spec` for more or check [these docs](https://kubernetes.io/docs/tasks/run-application/configure-pdb/) podDisruptionBudget: enabled: false - # minAvailable: 1 - # maxUnavailable: 1 + # -- min number or percentage of pods that can be unavailable + minAvailable: 0 + # -- max number or percentage of pods that can be unavailable + maxUnavailable: 0 + # -- Defines criteria when unhealthy pods should be considered for eviction + unhealthyPodEvictionPolicy: labels: {} # -- Graceful pod termination timeout. See [this article](https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#hook-handler-execution) for details. @@ -299,6 +303,13 @@ resources: # -- Pod's node selector. Details are [here](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) nodeSelector: {} +# -- Deployment strategy, set to standard k8s default +strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + # -- Name of Priority Class priorityClassName: "" @@ -364,6 +375,8 @@ shareProcessNamespace: false admissionWebhooks: # -- Enables validation webhook. enabled: true + # -- Annotations for webhook. Can be used to define Helm or ArgoCD annotations. + annotations: {} # -- List of CRD names to disable validation for disabledFor: [] # - vmagent From e8806fcc6fdec075325352c4dfe7fde7ddb0552a Mon Sep 17 00:00:00 2001 From: "cozystack-ci[bot]" <274107086+cozystack-ci[bot]@users.noreply.github.com> Date: Tue, 21 Apr 2026 01:42:32 +0000 Subject: [PATCH 139/250] docs: add changelog for v1.2.3 Signed-off-by: cozystack-ci[bot] <274107086+cozystack-ci[bot]@users.noreply.github.com> --- docs/changelogs/v1.2.3.md | 58 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 docs/changelogs/v1.2.3.md diff --git a/docs/changelogs/v1.2.3.md b/docs/changelogs/v1.2.3.md new file mode 100644 index 00000000..6d1b3b74 --- /dev/null +++ b/docs/changelogs/v1.2.3.md @@ -0,0 +1,58 @@ + + +# v1.2.3 (2026-04-20) + +A patch release with bug fixes and documentation updates. + +## Features and Improvements + +_No notable features in this patch release._ + +## Fixes + +* **fix(kubernetes): set explicit ephemeral-storage on virt-launcher pods**: Prevents VM crashes caused by ephemeral-storage eviction by setting explicit `domain.resources` ephemeral-storage on the VirtualMachine spec. Uses sanitized limits and requests so virt-launcher pods do not inherit too-small namespace defaults. ([**@kvaps**](https://github.com/kvaps) in #2317, backport #2423). + +## Documentation + +* **[website] feat: add Telemetry page under OSS Health section**: Add Telemetry page and initial data seeding to OSS Health docs ([**@tym83**](https://github.com/tym83) in cozystack/website#471). +* **[website] Refactor docs versions to major.minor variants**: Move docs to major.minor versioning for v1.x series ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#477). +* **[website] docs(tenants): document namespace layout and parent/child derivation** ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#479). +* **[website] docs(tenants): document the checkbox-then-edit-CR customization pattern** ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#485). +* **[website] docs: fix 14 broken links and stale talm anchor across v1 docs** ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#486). +* **[website] fix(og): update social badge image and title** ([**@tym83**](https://github.com/tym83) in cozystack/website#487). +* **[website] docs(external-apps): rewrite guide for ApplicationDefinition API** ([**@kitsunoff**](https://github.com/kitsunoff) in cozystack/website#488). +* **[website] docs: add CLAUDE.md for AI agent guidance** ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#489). +* **[website] fix: update /docs/v1/ redirect to latest v1.2** ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#492). +* **[website] fix(ci): add OpenAPI spec download to GitHub Pages build** ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#494). +* **[website] feat(blog): add managed PostgreSQL with synchronous replication post** ([**@tym83**](https://github.com/tym83) in cozystack/website#497). +* **[website] chore(blog): add images frontmatter for social preview on existing posts** ([**@tym83**](https://github.com/tym83) in cozystack/website#498). +* **[website] feat(blog): taxonomies and client-side filter UI** ([**@tym83**](https://github.com/tym83) in cozystack/website#499). +* **[website] style(oss-health): add breathing room between navbar and hero** ([**@tym83**](https://github.com/tym83) in cozystack/website#500). + +## Other repositories + +* **[talm] feat(config): migrate to Talos v1.12 multi-document config format**: Upgrade Talos config format and modernize configuration handling ([**@lexfrei**](https://github.com/lexfrei) in cozystack/talm#116). +* **[talm] chore(deps): bump dependencies and modernize codebase** ([**@lexfrei**](https://github.com/lexfrei) in cozystack/talm#124). +* **[external-apps-example] feat: replace MongoDB example with Minecraft apps from cozylex** ([**@lexfrei**](https://github.com/lexfrei) in cozystack/external-apps-example#2). +* **[ansible-cozystack] fix(examples): add v prefix to collection version in requirements.yml** ([**@lexfrei**](https://github.com/lexfrei) in cozystack/ansible-cozystack#23). +* **[ansible-cozystack] fix(plugins): replace ansible.utils.ipaddr with stdlib-based test plugin** ([**@lexfrei**](https://github.com/lexfrei) in cozystack/ansible-cozystack#24). +* **[ansible-cozystack] feat(examples): comprehensive node prerequisites audit (fixes #19)** ([**@lexfrei**](https://github.com/lexfrei) in cozystack/ansible-cozystack#27). +* **[ansible-cozystack] chore(deps): update dependency cozystack.installer to v1.2.3** ([**@app/renovate**](https://github.com/apps/renovate) in cozystack/ansible-cozystack#29). +* **[ansible-cozystack] feat(role): expose publishing.externalIPs and tenant-root ingress via role variables** ([**@lexfrei**](https://github.com/lexfrei) in cozystack/ansible-cozystack#30). + +## Contributors + +Thanks to everyone who contributed to this patch release: + +* [**@app/github-actions**](https://github.com/apps/github-actions) +* [**@app/renovate**](https://github.com/apps/renovate) +* [**@kitsunoff**](https://github.com/kitsunoff) +* [**@kvaps**](https://github.com/kvaps) +* [**@lexfrei**](https://github.com/lexfrei) +* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) +* [**@tym83**](https://github.com/tym83) + + +**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.2.2...v1.2.3 From ac14df54d4cdaa75842d4f1e058a10f3ed754229 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Tue, 21 Apr 2026 08:54:41 +0500 Subject: [PATCH 140/250] ci(docs): promote next/ trunk on new minor/major releases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The website repo is switching to a permanent `content/en/docs/next/` trunk (cozystack/website#495). Released version directories are no longer pre-created — the upstream release workflow is now responsible for explicitly promoting `next/` → `vX.Y/` on new minor/major releases via `make release-next`. Update the `update-website-docs` job to match that contract: - New `Determine if this release promotes next/` step parses the tag and only enables promotion for final `vX.Y.Z` tags where `content/en/docs/vX.Y/` does not already exist. Prereleases and patch releases keep the old behaviour (target routing is handled by the website Makefile on its own). - New `Promote next/ to released version` step runs `make release-next RELEASE_TAG=...` before `make update-all`, so the subsequent `update-all` routes into the freshly-created `vX.Y/` directory and `next/` stays untouched as the trunk for the following release. - `git add content hugo.yaml` to pick up the version registration that `release-next` writes via `register_version.sh`. - Convert the existing `update-all` step to use `env:` vars, matching the new step and addressing the workflow-injection hardening guidance. Signed-off-by: Myasnikov Daniil --- .github/workflows/tags.yaml | 45 +++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tags.yaml b/.github/workflows/tags.yaml index d698333e..b10897cf 100644 --- a/.github/workflows/tags.yaml +++ b/.github/workflows/tags.yaml @@ -441,17 +441,58 @@ jobs: token: ${{ steps.app-token.outputs.token }} ref: main + # Decide whether this release promotes the `next/` trunk to a new released + # version directory. Per the website repo's contract (see website#495): + # - `make release-next` runs only for new minor/major final releases + # (tag matches `vX.Y.Z` with no prerelease suffix AND `vX.Y/` does + # not yet exist on disk). + # - Prereleases and patch releases skip this step and let + # `make update-all` handle routing on its own. + - name: Determine if this release promotes next/ + id: promote + env: + TAG: ${{ steps.tag.outputs.tag }} + run: | + if [[ ! "$TAG" =~ ^v([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then + echo "promote=false" >> "$GITHUB_OUTPUT" + echo "Prerelease tag '$TAG' — skipping release-next." + exit 0 + fi + MAJOR="${BASH_REMATCH[1]}" + MINOR="${BASH_REMATCH[2]}" + if [[ "$MAJOR" == "0" ]]; then + DOC_VERSION="v0" + else + DOC_VERSION="v${MAJOR}.${MINOR}" + fi + if [[ -d "content/en/docs/${DOC_VERSION}" ]]; then + echo "promote=false" >> "$GITHUB_OUTPUT" + echo "content/en/docs/${DOC_VERSION}/ already exists — patch release, skipping release-next." + else + echo "promote=true" >> "$GITHUB_OUTPUT" + echo "New minor/major release ${DOC_VERSION} — will promote next/ → ${DOC_VERSION}/." + fi + + - name: Promote next/ to released version + if: steps.promote.outputs.promote == 'true' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + TAG: ${{ steps.tag.outputs.tag }} + run: make release-next RELEASE_TAG="$TAG" + - name: Update docs from release branch env: GH_TOKEN: ${{ steps.app-token.outputs.token }} - run: make update-all BRANCH=release-${{ steps.tag.outputs.version }} RELEASE_TAG=${{ steps.tag.outputs.tag }} + TAG: ${{ steps.tag.outputs.tag }} + VERSION: ${{ steps.tag.outputs.version }} + run: make update-all BRANCH="release-$VERSION" RELEASE_TAG="$TAG" - name: Commit and push id: commit run: | git config user.name "cozystack-ci[bot]" git config user.email "274107086+cozystack-ci[bot]@users.noreply.github.com" - git add content + git add content hugo.yaml if git diff --cached --quiet; then echo "No changes to commit" echo "changed=false" >> $GITHUB_OUTPUT From a133fa30ddb8ead7d8ad7491dce997a818909289 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Tue, 21 Apr 2026 14:31:07 +0500 Subject: [PATCH 141/250] chore(maintenance): add @myasnikovdaniil to CODEOWNERS Assisted-By: Claude Signed-off-by: Myasnikov Daniil --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2045ec5a..2482712d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @kvaps @lllamnyp @lexfrei @androndo @IvanHunters @sircthulhu +* @kvaps @lllamnyp @lexfrei @androndo @IvanHunters @sircthulhu @myasnikovdaniil From 531bc00524be0789ba99e2082234a1104e010b51 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Wed, 8 Apr 2026 22:17:34 +0300 Subject: [PATCH 142/250] feat(postgres): add serverName parameter for backup recovery Add serverName field to bootstrap configuration to explicitly specify Barman server name from backup.info. This fixes "no target backup found" errors when server_name in backup.info differs from Kubernetes cluster name. Signed-off-by: IvanHunters --- api/apps/v1alpha1/postgresql/types.go | 5 ++++- packages/apps/postgres/README.md | 13 +++++++------ packages/apps/postgres/templates/db.yaml | 3 +++ packages/apps/postgres/values.schema.json | 7 ++++++- packages/apps/postgres/values.yaml | 4 +++- packages/system/postgres-rd/cozyrds/postgres.yaml | 4 ++-- 6 files changed, 25 insertions(+), 11 deletions(-) diff --git a/api/apps/v1alpha1/postgresql/types.go b/api/apps/v1alpha1/postgresql/types.go index fa85d51f..56580018 100644 --- a/api/apps/v1alpha1/postgresql/types.go +++ b/api/apps/v1alpha1/postgresql/types.go @@ -86,12 +86,15 @@ type Bootstrap struct { // Whether to restore from a backup. // +kubebuilder:default:=false Enabled bool `json:"enabled"` - // Previous cluster name before deletion. + // Previous cluster name before deletion (matches serverName in backup.info). // +kubebuilder:default:="" OldName string `json:"oldName"` // Timestamp (RFC3339) for point-in-time recovery; empty means latest. // +kubebuilder:default:="" RecoveryTime string `json:"recoveryTime,omitempty"` + // Barman server name from the old cluster's backup.info. Use when the original cluster used a different serverName than its Kubernetes cluster name. + // +kubebuilder:default:="" + ServerName string `json:"serverName,omitempty"` } type Database struct { diff --git a/packages/apps/postgres/README.md b/packages/apps/postgres/README.md index 5a550f7a..648b3d71 100644 --- a/packages/apps/postgres/README.md +++ b/packages/apps/postgres/README.md @@ -133,12 +133,13 @@ See: ### Bootstrap (recovery) parameters -| Name | Description | Type | Value | -| ------------------------ | ------------------------------------------------------------------- | -------- | ------- | -| `bootstrap` | Bootstrap configuration. | `object` | `{}` | -| `bootstrap.enabled` | Whether to restore from a backup. | `bool` | `false` | -| `bootstrap.recoveryTime` | Timestamp (RFC3339) for point-in-time recovery; empty means latest. | `string` | `""` | -| `bootstrap.oldName` | Previous cluster name before deletion. | `string` | `""` | +| Name | Description | Type | Value | +| ------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------- | +| `bootstrap` | Bootstrap configuration. | `object` | `{}` | +| `bootstrap.enabled` | Whether to restore from a backup. | `bool` | `false` | +| `bootstrap.recoveryTime` | Timestamp (RFC3339) for point-in-time recovery; empty means latest. | `string` | `""` | +| `bootstrap.oldName` | Previous cluster name before deletion (matches serverName in backup.info). | `string` | `""` | +| `bootstrap.serverName` | Barman server name from the old cluster's backup.info. Use when the original cluster used a different serverName than its Kubernetes cluster name. | `string` | `""` | ## Parameter examples and reference diff --git a/packages/apps/postgres/templates/db.yaml b/packages/apps/postgres/templates/db.yaml index 5c40b2c7..2cdaec1d 100644 --- a/packages/apps/postgres/templates/db.yaml +++ b/packages/apps/postgres/templates/db.yaml @@ -32,6 +32,9 @@ spec: - name: {{ .Values.bootstrap.oldName }} barmanObjectStore: destinationPath: {{ .Values.backup.destinationPath }} + {{- if .Values.bootstrap.serverName }} + serverName: {{ .Values.bootstrap.serverName }} + {{- end }} endpointURL: {{ .Values.backup.endpointURL }} s3Credentials: accessKeyId: diff --git a/packages/apps/postgres/values.schema.json b/packages/apps/postgres/values.schema.json index b2a4aeba..28acc5a3 100644 --- a/packages/apps/postgres/values.schema.json +++ b/packages/apps/postgres/values.schema.json @@ -246,7 +246,7 @@ "default": false }, "oldName": { - "description": "Previous cluster name before deletion.", + "description": "Previous cluster name before deletion (matches serverName in backup.info).", "type": "string", "default": "" }, @@ -254,6 +254,11 @@ "description": "Timestamp (RFC3339) for point-in-time recovery; empty means latest.", "type": "string", "default": "" + }, + "serverName": { + "description": "Barman server name from the old cluster's backup.info. Use when the original cluster used a different serverName than its Kubernetes cluster name.", + "type": "string", + "default": "" } } } diff --git a/packages/apps/postgres/values.yaml b/packages/apps/postgres/values.yaml index b8f07f63..a4a4bf86 100644 --- a/packages/apps/postgres/values.yaml +++ b/packages/apps/postgres/values.yaml @@ -153,7 +153,8 @@ backup: ## @typedef {struct} Bootstrap - Bootstrap configuration for restoring a database cluster from a backup. ## @field {bool} enabled - Whether to restore from a backup. ## @field {string} [recoveryTime] - Timestamp (RFC3339) for point-in-time recovery; empty means latest. -## @field {string} oldName - Previous cluster name before deletion. +## @field {string} oldName - Previous cluster name before deletion (matches serverName in backup.info). +## @field {string} [serverName] - Barman server name from the old cluster's backup.info. Use when the original cluster used a different serverName than its Kubernetes cluster name. ## @param {Bootstrap} bootstrap - Bootstrap configuration. bootstrap: @@ -161,3 +162,4 @@ bootstrap: # example: 2020-11-26 15:22:00.00000+00 recoveryTime: "" oldName: "" + serverName: "" diff --git a/packages/system/postgres-rd/cozyrds/postgres.yaml b/packages/system/postgres-rd/cozyrds/postgres.yaml index c74c5783..41457db4 100644 --- a/packages/system/postgres-rd/cozyrds/postgres.yaml +++ b/packages/system/postgres-rd/cozyrds/postgres.yaml @@ -8,7 +8,7 @@ spec: singular: postgres plural: postgreses openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Postgres replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each PostgreSQL replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"version":{"description":"PostgreSQL major version to deploy","type":"string","default":"v18","enum":["v18","v17","v16","v15","v14","v13"]},"postgresql":{"description":"PostgreSQL server configuration.","type":"object","default":{},"properties":{"parameters":{"description":"PostgreSQL server parameters.","type":"object","default":{},"properties":{"max_connections":{"description":"Maximum number of concurrent connections to the database server.","type":"integer","default":100}}}}},"quorum":{"description":"Quorum configuration for synchronous replication.","type":"object","default":{},"required":["maxSyncReplicas","minSyncReplicas"],"properties":{"maxSyncReplicas":{"description":"Maximum number of synchronous replicas allowed (must be less than total replicas).","type":"integer","default":0},"minSyncReplicas":{"description":"Minimum number of synchronous replicas required for commit.","type":"integer","default":0}}},"users":{"description":"Users configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"password":{"description":"Password for the user.","type":"string"},"replication":{"description":"Whether the user has replication privileges.","type":"boolean"}}}},"databases":{"description":"Databases configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"extensions":{"description":"List of enabled PostgreSQL extensions.","type":"array","items":{"type":"string"}},"roles":{"description":"Roles assigned to users.","type":"object","properties":{"admin":{"description":"List of users with admin privileges.","type":"array","items":{"type":"string"}},"readonly":{"description":"List of users with read-only privileges.","type":"array","items":{"type":"string"}}}}}}},"backup":{"description":"Backup configuration.","type":"object","default":{},"required":["enabled"],"properties":{"destinationPath":{"description":"Destination path for backups (e.g. s3://bucket/path/).","type":"string","default":"s3://bucket/path/to/folder/"},"enabled":{"description":"Enable regular backups.","type":"boolean","default":false},"endpointURL":{"description":"S3 endpoint URL for uploads.","type":"string","default":"http://minio-gateway-service:9000"},"retentionPolicy":{"description":"Retention policy (e.g. \"30d\").","type":"string","default":"30d"},"s3AccessKey":{"description":"Access key for S3 authentication.","type":"string","default":""},"s3SecretKey":{"description":"Secret key for S3 authentication.","type":"string","default":""},"schedule":{"description":"Cron schedule for automated backups.","type":"string","default":"0 2 * * * *"}}},"bootstrap":{"description":"Bootstrap configuration.","type":"object","default":{},"required":["enabled","oldName"],"properties":{"enabled":{"description":"Whether to restore from a backup.","type":"boolean","default":false},"oldName":{"description":"Previous cluster name before deletion.","type":"string","default":""},"recoveryTime":{"description":"Timestamp (RFC3339) for point-in-time recovery; empty means latest.","type":"string","default":""}}}}} + {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Postgres replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each PostgreSQL replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"version":{"description":"PostgreSQL major version to deploy","type":"string","default":"v18","enum":["v18","v17","v16","v15","v14","v13"]},"postgresql":{"description":"PostgreSQL server configuration.","type":"object","default":{},"properties":{"parameters":{"description":"PostgreSQL server parameters.","type":"object","default":{},"properties":{"max_connections":{"description":"Maximum number of concurrent connections to the database server.","type":"integer","default":100}}}}},"quorum":{"description":"Quorum configuration for synchronous replication.","type":"object","default":{},"required":["maxSyncReplicas","minSyncReplicas"],"properties":{"maxSyncReplicas":{"description":"Maximum number of synchronous replicas allowed (must be less than total replicas).","type":"integer","default":0},"minSyncReplicas":{"description":"Minimum number of synchronous replicas required for commit.","type":"integer","default":0}}},"users":{"description":"Users configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"password":{"description":"Password for the user.","type":"string"},"replication":{"description":"Whether the user has replication privileges.","type":"boolean"}}}},"databases":{"description":"Databases configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"extensions":{"description":"List of enabled PostgreSQL extensions.","type":"array","items":{"type":"string"}},"roles":{"description":"Roles assigned to users.","type":"object","properties":{"admin":{"description":"List of users with admin privileges.","type":"array","items":{"type":"string"}},"readonly":{"description":"List of users with read-only privileges.","type":"array","items":{"type":"string"}}}}}}},"backup":{"description":"Backup configuration.","type":"object","default":{},"required":["enabled"],"properties":{"destinationPath":{"description":"Destination path for backups (e.g. s3://bucket/path/).","type":"string","default":"s3://bucket/path/to/folder/"},"enabled":{"description":"Enable regular backups.","type":"boolean","default":false},"endpointURL":{"description":"S3 endpoint URL for uploads.","type":"string","default":"http://minio-gateway-service:9000"},"retentionPolicy":{"description":"Retention policy (e.g. \"30d\").","type":"string","default":"30d"},"s3AccessKey":{"description":"Access key for S3 authentication.","type":"string","default":""},"s3SecretKey":{"description":"Secret key for S3 authentication.","type":"string","default":""},"schedule":{"description":"Cron schedule for automated backups.","type":"string","default":"0 2 * * * *"}}},"bootstrap":{"description":"Bootstrap configuration.","type":"object","default":{},"required":["enabled","oldName"],"properties":{"enabled":{"description":"Whether to restore from a backup.","type":"boolean","default":false},"oldName":{"description":"Previous cluster name before deletion (matches serverName in backup.info).","type":"string","default":""},"recoveryTime":{"description":"Timestamp (RFC3339) for point-in-time recovery; empty means latest.","type":"string","default":""},"serverName":{"description":"Barman server name from the old cluster's backup.info. Use when the original cluster used a different serverName than its Kubernetes cluster name.","type":"string","default":""}}}}} release: prefix: postgres- labels: @@ -33,7 +33,7 @@ spec: # labelSelector: # helm.toolkit.fluxcd.io/name: "{reqs[0]['metadata','name']}" - keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "replicas"], ["spec", "resources"], ["spec", "resourcesPreset"], ["spec", "size"], ["spec", "storageClass"], ["spec", "external"], ["spec", "version"], ["spec", "postgresql"], ["spec", "postgresql", "parameters"], ["spec", "postgresql", "parameters", "max_connections"], ["spec", "quorum"], ["spec", "quorum", "minSyncReplicas"], ["spec", "quorum", "maxSyncReplicas"], ["spec", "users"], ["spec", "databases"], ["spec", "backup"], ["spec", "backup", "enabled"], ["spec", "backup", "retentionPolicy"], ["spec", "backup", "destinationPath"], ["spec", "backup", "endpointURL"], ["spec", "backup", "schedule"], ["spec", "backup", "s3AccessKey"], ["spec", "backup", "s3SecretKey"], ["spec", "bootstrap"], ["spec", "bootstrap", "enabled"], ["spec", "bootstrap", "recoveryTime"], ["spec", "bootstrap", "oldName"]] + keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "replicas"], ["spec", "resources"], ["spec", "resourcesPreset"], ["spec", "size"], ["spec", "storageClass"], ["spec", "external"], ["spec", "version"], ["spec", "postgresql"], ["spec", "postgresql", "parameters"], ["spec", "postgresql", "parameters", "max_connections"], ["spec", "quorum"], ["spec", "quorum", "minSyncReplicas"], ["spec", "quorum", "maxSyncReplicas"], ["spec", "users"], ["spec", "databases"], ["spec", "backup"], ["spec", "backup", "enabled"], ["spec", "backup", "retentionPolicy"], ["spec", "backup", "destinationPath"], ["spec", "backup", "endpointURL"], ["spec", "backup", "schedule"], ["spec", "backup", "s3AccessKey"], ["spec", "backup", "s3SecretKey"], ["spec", "bootstrap"], ["spec", "bootstrap", "enabled"], ["spec", "bootstrap", "recoveryTime"], ["spec", "bootstrap", "oldName"], ["spec", "bootstrap", "serverName"]] secrets: exclude: [] include: From 9f41dc3228b18308f9cd6819a8b2470778f27f0c Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Tue, 21 Apr 2026 13:17:32 +0300 Subject: [PATCH 143/250] docs(postgres): clarify bootstrap field descriptions Update oldName and serverName field descriptions based on code review feedback to avoid confusion about their actual roles: - oldName: Remove misleading "(matches serverName in backup.info)" text. This field represents the Kubernetes cluster resource name, not the Barman server name. - serverName: Provide clearer explanation that it's the S3 path prefix (barmanObjectStore.serverName) used by the original cluster, and should only be set when it differs from the Kubernetes resource name. Updated in: - values.yaml (source of truth for field documentation) - types.go (Go API type comments) - values.schema.json (JSON schema for validation) - postgres.yaml (CRD with embedded OpenAPI schema) Signed-off-by: IvanHunters --- api/apps/v1alpha1/postgresql/types.go | 4 ++-- api/backups/v1alpha1/zz_generated.deepcopy.go | 5 +++++ packages/apps/postgres/README.md | 14 +++++++------- packages/apps/postgres/values.schema.json | 4 ++-- packages/apps/postgres/values.yaml | 4 ++-- packages/system/postgres-rd/cozyrds/postgres.yaml | 2 +- 6 files changed, 19 insertions(+), 14 deletions(-) diff --git a/api/apps/v1alpha1/postgresql/types.go b/api/apps/v1alpha1/postgresql/types.go index 56580018..a2ff77a8 100644 --- a/api/apps/v1alpha1/postgresql/types.go +++ b/api/apps/v1alpha1/postgresql/types.go @@ -86,13 +86,13 @@ type Bootstrap struct { // Whether to restore from a backup. // +kubebuilder:default:=false Enabled bool `json:"enabled"` - // Previous cluster name before deletion (matches serverName in backup.info). + // Previous cluster name before deletion. // +kubebuilder:default:="" OldName string `json:"oldName"` // Timestamp (RFC3339) for point-in-time recovery; empty means latest. // +kubebuilder:default:="" RecoveryTime string `json:"recoveryTime,omitempty"` - // Barman server name from the old cluster's backup.info. Use when the original cluster used a different serverName than its Kubernetes cluster name. + // Barman server name (S3 path prefix) used by the original cluster when writing backups. Set this only when the original cluster had an explicit barmanObjectStore.serverName that differed from its Kubernetes resource name. // +kubebuilder:default:="" ServerName string `json:"serverName,omitempty"` } diff --git a/api/backups/v1alpha1/zz_generated.deepcopy.go b/api/backups/v1alpha1/zz_generated.deepcopy.go index 89f6171f..61b8f839 100644 --- a/api/backups/v1alpha1/zz_generated.deepcopy.go +++ b/api/backups/v1alpha1/zz_generated.deepcopy.go @@ -620,6 +620,11 @@ func (in *RestoreJobSpec) DeepCopyInto(out *RestoreJobSpec) { *out = new(v1.TypedLocalObjectReference) (*in).DeepCopyInto(*out) } + if in.Options != nil { + in, out := &in.Options, &out.Options + *out = new(runtime.RawExtension) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestoreJobSpec. diff --git a/packages/apps/postgres/README.md b/packages/apps/postgres/README.md index 648b3d71..4cda7284 100644 --- a/packages/apps/postgres/README.md +++ b/packages/apps/postgres/README.md @@ -133,13 +133,13 @@ See: ### Bootstrap (recovery) parameters -| Name | Description | Type | Value | -| ------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------- | -| `bootstrap` | Bootstrap configuration. | `object` | `{}` | -| `bootstrap.enabled` | Whether to restore from a backup. | `bool` | `false` | -| `bootstrap.recoveryTime` | Timestamp (RFC3339) for point-in-time recovery; empty means latest. | `string` | `""` | -| `bootstrap.oldName` | Previous cluster name before deletion (matches serverName in backup.info). | `string` | `""` | -| `bootstrap.serverName` | Barman server name from the old cluster's backup.info. Use when the original cluster used a different serverName than its Kubernetes cluster name. | `string` | `""` | +| Name | Description | Type | Value | +| ------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------- | +| `bootstrap` | Bootstrap configuration. | `object` | `{}` | +| `bootstrap.enabled` | Whether to restore from a backup. | `bool` | `false` | +| `bootstrap.recoveryTime` | Timestamp (RFC3339) for point-in-time recovery; empty means latest. | `string` | `""` | +| `bootstrap.oldName` | Previous cluster name before deletion. | `string` | `""` | +| `bootstrap.serverName` | Barman server name (S3 path prefix) used by the original cluster when writing backups. Set this only when the original cluster had an explicit barmanObjectStore.serverName that differed from its Kubernetes resource name. | `string` | `""` | ## Parameter examples and reference diff --git a/packages/apps/postgres/values.schema.json b/packages/apps/postgres/values.schema.json index 28acc5a3..98e29822 100644 --- a/packages/apps/postgres/values.schema.json +++ b/packages/apps/postgres/values.schema.json @@ -246,7 +246,7 @@ "default": false }, "oldName": { - "description": "Previous cluster name before deletion (matches serverName in backup.info).", + "description": "Previous cluster name before deletion.", "type": "string", "default": "" }, @@ -256,7 +256,7 @@ "default": "" }, "serverName": { - "description": "Barman server name from the old cluster's backup.info. Use when the original cluster used a different serverName than its Kubernetes cluster name.", + "description": "Barman server name (S3 path prefix) used by the original cluster when writing backups. Set this only when the original cluster had an explicit barmanObjectStore.serverName that differed from its Kubernetes resource name.", "type": "string", "default": "" } diff --git a/packages/apps/postgres/values.yaml b/packages/apps/postgres/values.yaml index a4a4bf86..2ceaa9cf 100644 --- a/packages/apps/postgres/values.yaml +++ b/packages/apps/postgres/values.yaml @@ -153,8 +153,8 @@ backup: ## @typedef {struct} Bootstrap - Bootstrap configuration for restoring a database cluster from a backup. ## @field {bool} enabled - Whether to restore from a backup. ## @field {string} [recoveryTime] - Timestamp (RFC3339) for point-in-time recovery; empty means latest. -## @field {string} oldName - Previous cluster name before deletion (matches serverName in backup.info). -## @field {string} [serverName] - Barman server name from the old cluster's backup.info. Use when the original cluster used a different serverName than its Kubernetes cluster name. +## @field {string} oldName - Previous cluster name before deletion. +## @field {string} [serverName] - Barman server name (S3 path prefix) used by the original cluster when writing backups. Set this only when the original cluster had an explicit barmanObjectStore.serverName that differed from its Kubernetes resource name. ## @param {Bootstrap} bootstrap - Bootstrap configuration. bootstrap: diff --git a/packages/system/postgres-rd/cozyrds/postgres.yaml b/packages/system/postgres-rd/cozyrds/postgres.yaml index 41457db4..72d013f1 100644 --- a/packages/system/postgres-rd/cozyrds/postgres.yaml +++ b/packages/system/postgres-rd/cozyrds/postgres.yaml @@ -8,7 +8,7 @@ spec: singular: postgres plural: postgreses openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Postgres replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each PostgreSQL replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"version":{"description":"PostgreSQL major version to deploy","type":"string","default":"v18","enum":["v18","v17","v16","v15","v14","v13"]},"postgresql":{"description":"PostgreSQL server configuration.","type":"object","default":{},"properties":{"parameters":{"description":"PostgreSQL server parameters.","type":"object","default":{},"properties":{"max_connections":{"description":"Maximum number of concurrent connections to the database server.","type":"integer","default":100}}}}},"quorum":{"description":"Quorum configuration for synchronous replication.","type":"object","default":{},"required":["maxSyncReplicas","minSyncReplicas"],"properties":{"maxSyncReplicas":{"description":"Maximum number of synchronous replicas allowed (must be less than total replicas).","type":"integer","default":0},"minSyncReplicas":{"description":"Minimum number of synchronous replicas required for commit.","type":"integer","default":0}}},"users":{"description":"Users configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"password":{"description":"Password for the user.","type":"string"},"replication":{"description":"Whether the user has replication privileges.","type":"boolean"}}}},"databases":{"description":"Databases configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"extensions":{"description":"List of enabled PostgreSQL extensions.","type":"array","items":{"type":"string"}},"roles":{"description":"Roles assigned to users.","type":"object","properties":{"admin":{"description":"List of users with admin privileges.","type":"array","items":{"type":"string"}},"readonly":{"description":"List of users with read-only privileges.","type":"array","items":{"type":"string"}}}}}}},"backup":{"description":"Backup configuration.","type":"object","default":{},"required":["enabled"],"properties":{"destinationPath":{"description":"Destination path for backups (e.g. s3://bucket/path/).","type":"string","default":"s3://bucket/path/to/folder/"},"enabled":{"description":"Enable regular backups.","type":"boolean","default":false},"endpointURL":{"description":"S3 endpoint URL for uploads.","type":"string","default":"http://minio-gateway-service:9000"},"retentionPolicy":{"description":"Retention policy (e.g. \"30d\").","type":"string","default":"30d"},"s3AccessKey":{"description":"Access key for S3 authentication.","type":"string","default":""},"s3SecretKey":{"description":"Secret key for S3 authentication.","type":"string","default":""},"schedule":{"description":"Cron schedule for automated backups.","type":"string","default":"0 2 * * * *"}}},"bootstrap":{"description":"Bootstrap configuration.","type":"object","default":{},"required":["enabled","oldName"],"properties":{"enabled":{"description":"Whether to restore from a backup.","type":"boolean","default":false},"oldName":{"description":"Previous cluster name before deletion (matches serverName in backup.info).","type":"string","default":""},"recoveryTime":{"description":"Timestamp (RFC3339) for point-in-time recovery; empty means latest.","type":"string","default":""},"serverName":{"description":"Barman server name from the old cluster's backup.info. Use when the original cluster used a different serverName than its Kubernetes cluster name.","type":"string","default":""}}}}} + {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Postgres replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each PostgreSQL replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"version":{"description":"PostgreSQL major version to deploy","type":"string","default":"v18","enum":["v18","v17","v16","v15","v14","v13"]},"postgresql":{"description":"PostgreSQL server configuration.","type":"object","default":{},"properties":{"parameters":{"description":"PostgreSQL server parameters.","type":"object","default":{},"properties":{"max_connections":{"description":"Maximum number of concurrent connections to the database server.","type":"integer","default":100}}}}},"quorum":{"description":"Quorum configuration for synchronous replication.","type":"object","default":{},"required":["maxSyncReplicas","minSyncReplicas"],"properties":{"maxSyncReplicas":{"description":"Maximum number of synchronous replicas allowed (must be less than total replicas).","type":"integer","default":0},"minSyncReplicas":{"description":"Minimum number of synchronous replicas required for commit.","type":"integer","default":0}}},"users":{"description":"Users configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"password":{"description":"Password for the user.","type":"string"},"replication":{"description":"Whether the user has replication privileges.","type":"boolean"}}}},"databases":{"description":"Databases configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"extensions":{"description":"List of enabled PostgreSQL extensions.","type":"array","items":{"type":"string"}},"roles":{"description":"Roles assigned to users.","type":"object","properties":{"admin":{"description":"List of users with admin privileges.","type":"array","items":{"type":"string"}},"readonly":{"description":"List of users with read-only privileges.","type":"array","items":{"type":"string"}}}}}}},"backup":{"description":"Backup configuration.","type":"object","default":{},"required":["enabled"],"properties":{"destinationPath":{"description":"Destination path for backups (e.g. s3://bucket/path/).","type":"string","default":"s3://bucket/path/to/folder/"},"enabled":{"description":"Enable regular backups.","type":"boolean","default":false},"endpointURL":{"description":"S3 endpoint URL for uploads.","type":"string","default":"http://minio-gateway-service:9000"},"retentionPolicy":{"description":"Retention policy (e.g. \"30d\").","type":"string","default":"30d"},"s3AccessKey":{"description":"Access key for S3 authentication.","type":"string","default":""},"s3SecretKey":{"description":"Secret key for S3 authentication.","type":"string","default":""},"schedule":{"description":"Cron schedule for automated backups.","type":"string","default":"0 2 * * * *"}}},"bootstrap":{"description":"Bootstrap configuration.","type":"object","default":{},"required":["enabled","oldName"],"properties":{"enabled":{"description":"Whether to restore from a backup.","type":"boolean","default":false},"oldName":{"description":"Previous cluster name before deletion.","type":"string","default":""},"recoveryTime":{"description":"Timestamp (RFC3339) for point-in-time recovery; empty means latest.","type":"string","default":""},"serverName":{"description":"Barman server name (S3 path prefix) used by the original cluster when writing backups. Set this only when the original cluster had an explicit barmanObjectStore.serverName that differed from its Kubernetes resource name.","type":"string","default":""}}}}} release: prefix: postgres- labels: From 2b6e20cc3f96505ee8abdb40be64f1234a40b29d Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Tue, 21 Apr 2026 17:00:31 +0500 Subject: [PATCH 144/250] fix(platform): migrate ACME HTTP-01 to ingressClassName API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ClusterIssuer solver referenced IngressClass "nginx" which does not exist on cozystack clusters — real classes are named after tenant namespaces (e.g. tenant-root). Cert issuance only worked because every requesting Ingress overrode the ClusterIssuer via the legacy acme.cert-manager.io/http01-ingress-class annotation. Switch both sides to the modern cert-manager API (available since cert-manager 1.12; cozystack ships 1.19.3): - ClusterIssuer: http01.ingress.ingressClassName, value parameterized from _cluster.expose-ingress (default "tenant-root") - Ingress annotation: http01-ingress-ingressclassname These must migrate together — mixing ingressClassName (ClusterIssuer) with the old http01-ingress-class annotation triggers cert-manager's "fields ingressClassName and class cannot be set at the same time" validation and breaks issuance. Assisted-By: Claude Signed-off-by: Myasnikov Daniil --- packages/apps/harbor/templates/ingress.yaml | 2 +- .../extra/bootbox/templates/matchbox/ingress.yaml | 2 +- packages/extra/seaweedfs/templates/seaweedfs.yaml | 2 +- packages/system/bucket/templates/ingress.yaml | 2 +- .../templates/cluster-issuers.yaml | 15 ++++++++------- packages/system/dashboard/templates/ingress.yaml | 2 +- packages/system/keycloak/templates/ingress.yaml | 2 +- .../system/linstor-gui/templates/ingress.yaml | 2 +- .../monitoring/templates/alerta/alerta.yaml | 2 +- .../monitoring/templates/grafana/grafana.yaml | 2 +- packages/system/seaweedfs/values.yaml | 2 +- 11 files changed, 18 insertions(+), 17 deletions(-) diff --git a/packages/apps/harbor/templates/ingress.yaml b/packages/apps/harbor/templates/ingress.yaml index bb6ef07c..70933f5f 100644 --- a/packages/apps/harbor/templates/ingress.yaml +++ b/packages/apps/harbor/templates/ingress.yaml @@ -15,7 +15,7 @@ metadata: nginx.ingress.kubernetes.io/ssl-redirect: "true" nginx.ingress.kubernetes.io/backend-protocol: "HTTP" {{- if eq $solver "http01" }} - acme.cert-manager.io/http01-ingress-class: {{ $ingress }} + acme.cert-manager.io/http01-ingress-ingressclassname: {{ $ingress }} {{- end }} cert-manager.io/cluster-issuer: {{ $clusterIssuer }} spec: diff --git a/packages/extra/bootbox/templates/matchbox/ingress.yaml b/packages/extra/bootbox/templates/matchbox/ingress.yaml index 4262cd10..5eef3489 100644 --- a/packages/extra/bootbox/templates/matchbox/ingress.yaml +++ b/packages/extra/bootbox/templates/matchbox/ingress.yaml @@ -10,7 +10,7 @@ metadata: app: bootbox annotations: {{- if eq $solver "http01" }} - acme.cert-manager.io/http01-ingress-class: {{ $ingress }} + acme.cert-manager.io/http01-ingress-ingressclassname: {{ $ingress }} {{- end }} cert-manager.io/cluster-issuer: {{ $clusterIssuer }} {{- if .Values.whitelistHTTP }} diff --git a/packages/extra/seaweedfs/templates/seaweedfs.yaml b/packages/extra/seaweedfs/templates/seaweedfs.yaml index 42441b26..2f5720ca 100644 --- a/packages/extra/seaweedfs/templates/seaweedfs.yaml +++ b/packages/extra/seaweedfs/templates/seaweedfs.yaml @@ -243,7 +243,7 @@ spec: nginx.ingress.kubernetes.io/proxy-body-size: "0" nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" {{- if eq $solver "http01" }} - acme.cert-manager.io/http01-ingress-class: {{ $ingress }} + acme.cert-manager.io/http01-ingress-ingressclassname: {{ $ingress }} {{- end }} cert-manager.io/cluster-issuer: {{ $clusterIssuer }} tls: diff --git a/packages/system/bucket/templates/ingress.yaml b/packages/system/bucket/templates/ingress.yaml index 6e5028fc..b7ffaf8b 100644 --- a/packages/system/bucket/templates/ingress.yaml +++ b/packages/system/bucket/templates/ingress.yaml @@ -12,7 +12,7 @@ metadata: nginx.ingress.kubernetes.io/proxy-read-timeout: "99999" nginx.ingress.kubernetes.io/proxy-send-timeout: "99999" {{- if eq $solver "http01" }} - acme.cert-manager.io/http01-ingress-class: {{ $ingress }} + acme.cert-manager.io/http01-ingress-ingressclassname: {{ $ingress }} {{- end }} cert-manager.io/cluster-issuer: {{ $clusterIssuer }} spec: diff --git a/packages/system/cert-manager-issuers/templates/cluster-issuers.yaml b/packages/system/cert-manager-issuers/templates/cluster-issuers.yaml index 442f1fb3..3b582082 100644 --- a/packages/system/cert-manager-issuers/templates/cluster-issuers.yaml +++ b/packages/system/cert-manager-issuers/templates/cluster-issuers.yaml @@ -1,6 +1,7 @@ {{- $solver := (index .Values._cluster "solver") | default "http01" }} +{{- $exposeIngress := (index .Values._cluster "expose-ingress") | default "tenant-root" }} -apiVersion: cert-manager.io/v1 +apiVersion: cert-manager.io/v1 kind: ClusterIssuer metadata: name: letsencrypt-prod @@ -17,9 +18,9 @@ spec: name: cloudflare-api-token-secret key: api-token {{- else }} - http01: - ingress: - class: nginx + http01: + ingress: + ingressClassName: {{ $exposeIngress }} {{- end }} --- @@ -41,9 +42,9 @@ spec: name: cloudflare-api-token-secret key: api-token {{- else }} - http01: - ingress: - class: nginx + http01: + ingress: + ingressClassName: {{ $exposeIngress }} {{- end }} --- diff --git a/packages/system/dashboard/templates/ingress.yaml b/packages/system/dashboard/templates/ingress.yaml index a10797fb..6cf01490 100644 --- a/packages/system/dashboard/templates/ingress.yaml +++ b/packages/system/dashboard/templates/ingress.yaml @@ -11,7 +11,7 @@ metadata: annotations: cert-manager.io/cluster-issuer: {{ $clusterIssuer }} {{- if eq $solver "http01" }} - acme.cert-manager.io/http01-ingress-class: {{ $exposeIngress }} + acme.cert-manager.io/http01-ingress-ingressclassname: {{ $exposeIngress }} {{- end }} nginx.ingress.kubernetes.io/rewrite-target: / nginx.ingress.kubernetes.io/client-max-body-size: 100m diff --git a/packages/system/keycloak/templates/ingress.yaml b/packages/system/keycloak/templates/ingress.yaml index aa282f71..7f9fd476 100644 --- a/packages/system/keycloak/templates/ingress.yaml +++ b/packages/system/keycloak/templates/ingress.yaml @@ -11,7 +11,7 @@ metadata: {{- with .Values.ingress.annotations }} annotations: {{- if eq $solver "http01" }} - acme.cert-manager.io/http01-ingress-class: {{ $exposeIngress }} + acme.cert-manager.io/http01-ingress-ingressclassname: {{ $exposeIngress }} {{- end }} cert-manager.io/cluster-issuer: {{ $clusterIssuer }} {{- toYaml . | nindent 4 }} diff --git a/packages/system/linstor-gui/templates/ingress.yaml b/packages/system/linstor-gui/templates/ingress.yaml index bf950eab..c50b0f86 100644 --- a/packages/system/linstor-gui/templates/ingress.yaml +++ b/packages/system/linstor-gui/templates/ingress.yaml @@ -22,7 +22,7 @@ metadata: annotations: cert-manager.io/cluster-issuer: {{ $clusterIssuer }} {{- if eq $solver "http01" }} - acme.cert-manager.io/http01-ingress-class: {{ $exposeIngress }} + acme.cert-manager.io/http01-ingress-ingressclassname: {{ $exposeIngress }} {{- end }} nginx.ingress.kubernetes.io/proxy-body-size: 100m # Keycloak access+refresh+id tokens make the oauth2-proxy session diff --git a/packages/system/monitoring/templates/alerta/alerta.yaml b/packages/system/monitoring/templates/alerta/alerta.yaml index 44046d1f..d30e698d 100644 --- a/packages/system/monitoring/templates/alerta/alerta.yaml +++ b/packages/system/monitoring/templates/alerta/alerta.yaml @@ -181,7 +181,7 @@ metadata: app: alerta annotations: {{- if eq $solver "http01" }} - acme.cert-manager.io/http01-ingress-class: {{ $ingress }} + acme.cert-manager.io/http01-ingress-ingressclassname: {{ $ingress }} {{- end }} cert-manager.io/cluster-issuer: {{ $clusterIssuer }} spec: diff --git a/packages/system/monitoring/templates/grafana/grafana.yaml b/packages/system/monitoring/templates/grafana/grafana.yaml index d7f51e31..d65a7dc4 100644 --- a/packages/system/monitoring/templates/grafana/grafana.yaml +++ b/packages/system/monitoring/templates/grafana/grafana.yaml @@ -74,7 +74,7 @@ spec: metadata: annotations: {{- if eq $solver "http01" }} - acme.cert-manager.io/http01-ingress-class: "{{ $ingress }}" + acme.cert-manager.io/http01-ingress-ingressclassname: "{{ $ingress }}" {{- end }} cert-manager.io/cluster-issuer: {{ $clusterIssuer }} spec: diff --git a/packages/system/seaweedfs/values.yaml b/packages/system/seaweedfs/values.yaml index daf2c055..57843ff5 100644 --- a/packages/system/seaweedfs/values.yaml +++ b/packages/system/seaweedfs/values.yaml @@ -111,7 +111,7 @@ seaweedfs: nginx.ingress.kubernetes.io/client-body-timeout: "3600" nginx.ingress.kubernetes.io/client-header-timeout: "120" nginx.ingress.kubernetes.io/service-upstream: "true" - acme.cert-manager.io/http01-ingress-class: tenant-root + acme.cert-manager.io/http01-ingress-ingressclassname: tenant-root cert-manager.io/cluster-issuer: letsencrypt-prod tls: - hosts: From 310b0ece6b51572311b00c0b18f30af6afc4889e Mon Sep 17 00:00:00 2001 From: Andrey Kolkov Date: Mon, 13 Apr 2026 13:48:49 +0400 Subject: [PATCH 145/250] feat(dashboard): add restorejob list and add page and sidebar link Signed-off-by: Andrey Kolkov --- internal/controller/dashboard/sidebar.go | 9 + .../controller/dashboard/static_refactored.go | 222 +++++++++++++++++- 2 files changed, 228 insertions(+), 3 deletions(-) diff --git a/internal/controller/dashboard/sidebar.go b/internal/controller/dashboard/sidebar.go index 90721b3d..c9619f3a 100644 --- a/internal/controller/dashboard/sidebar.go +++ b/internal/controller/dashboard/sidebar.go @@ -144,6 +144,9 @@ func (m *Manager) ensureSidebar(ctx context.Context, crd *cozyv1alpha1.Applicati // Add sidebar for backups.cozystack.io Backup resource keysAndTags["backups"] = []any{"backup-sidebar"} + // Add sidebar for backups.cozystack.io RestoreJob resource + keysAndTags["restorejobs"] = []any{"restorejob-sidebar"} + // 3) Sort items within each category by Weight (desc), then Label (A→Z) for cat := range categories { sort.Slice(categories[cat], func(i, j int) bool { @@ -215,6 +218,11 @@ func (m *Manager) ensureSidebar(ctx context.Context, crd *cozyv1alpha1.Applicati "label": "Backups", "link": "/openapi-ui/{cluster}/{namespace}/api-table/backups.cozystack.io/v1alpha1/backups", }, + map[string]any{ + "key": "restorejobs", + "label": "RestoreJobs", + "link": "/openapi-ui/{cluster}/{namespace}/api-table/backups.cozystack.io/v1alpha1/restorejobs", + }, }, }) @@ -258,6 +266,7 @@ func (m *Manager) ensureSidebar(ctx context.Context, crd *cozyv1alpha1.Applicati "stock-project-factory-plan-details", "stock-project-factory-backupjob-details", "stock-project-factory-backup-details", + "stock-project-factory-restorejob-details", "stock-project-factory-external-ips", "stock-project-api-form", "stock-project-api-table", diff --git a/internal/controller/dashboard/static_refactored.go b/internal/controller/dashboard/static_refactored.go index d96bbaaa..a180f66d 100644 --- a/internal/controller/dashboard/static_refactored.go +++ b/internal/controller/dashboard/static_refactored.go @@ -425,6 +425,14 @@ func CreateAllCustomColumnsOverrides() []*dashboardv1alpha1.CustomColumnsOverrid createTimestampColumn("Taken At", ".spec.takenAt"), createTimestampColumn("Created", ".metadata.creationTimestamp"), }), + + // Stock namespace backups cozystack io v1alpha1 restorejobs + createCustomColumnsOverride("stock-namespace-/backups.cozystack.io/v1alpha1/restorejobs", []any{ + createCustomColumnWithJsonPath("Name", ".metadata.name", "RestoreJob", "", "/openapi-ui/{2}/{reqsJsonPath[0]['.metadata.namespace']['-']}/factory/restorejob-details/{reqsJsonPath[0]['.metadata.name']['-']}"), + createStringColumn("Phase", ".status.phase"), + createStringColumn("Backup", ".spec.backupRef.name"), + createTimestampColumn("Created", ".metadata.creationTimestamp"), + }), } } @@ -547,6 +555,31 @@ func CreateAllCustomFormsOverrides() []*dashboardv1alpha1.CustomFormsOverride { "backupClassName": listInputScemaItemBackupClass(), }), }), + + // RestoreJobs form override - backups.cozystack.io/v1alpha1 + createCustomFormsOverride("default-/backups.cozystack.io/v1alpha1/restorejobs", map[string]any{ + "formItems": []any{ + createFormItem("metadata.name", "Name", "text"), + createFormItem("metadata.namespace", "Namespace", "text"), + createFormItem("spec.backupRef.name", "Backup", "text"), + // Target application: leave empty to restore into the same application + // as referenced by the selected Backup. Fill all three only to rename + // or restore into a different application. + createFormItem("spec.targetApplicationRef.apiGroup", "Target Application API Group (optional, only for rename)", "text"), + createFormItem("spec.targetApplicationRef.kind", "Target Application Kind (optional, only for rename)", "text"), + createFormItem("spec.targetApplicationRef.name", "Target Application Name (optional, only for rename)", "text"), + // Driver-specific options (key-value editor). Known keys for VMInstance: + // targetNamespace, failIfTargetExists, keepOriginalPVC, keepOriginalIpAndMac. + createFormItem("spec.options", "Options (VMInstance keys: targetNamespace, failIfTargetExists, keepOriginalPVC, keepOriginalIpAndMac)", "object"), + }, + "schema": createSchema(map[string]any{ + "backupRef": map[string]any{ + "properties": map[string]any{ + "name": listInputSchemaItemBackup(), + }, + }, + }), + }), } } @@ -1937,6 +1970,174 @@ func CreateAllFactories() []*dashboardv1alpha1.Factory { } backupSpec := createUnifiedFactory(backupConfig, backupTabs, []any{"/api/clusters/{2}/k8s/apis/backups.cozystack.io/v1alpha1/namespaces/{3}/backups/{6}"}) + // RestoreJob details factory using unified approach + restoreJobConfig := UnifiedResourceConfig{ + Name: "restorejob-details", + ResourceType: "factory", + Kind: "RestoreJob", + Plural: "restorejobs", + Title: "restorejob", + } + restoreJobTabs := []any{ + map[string]any{ + "key": "details", + "label": "Details", + "children": []any{ + contentCard("details-card", map[string]any{ + "marginBottom": "24px", + }, []any{ + antdText("details-title", true, "RestoreJob details", map[string]any{ + "fontSize": 20, + "marginBottom": "12px", + }), + spacer("details-spacer", 16), + antdRow("details-grid", []any{48, 12}, []any{ + antdCol("col-left", 12, []any{ + antdFlexVertical("col-left-stack", 24, []any{ + antdFlexVertical("meta-name-block", 4, []any{ + antdText("meta-name-label", true, "Name", nil), + parsedText("meta-name-value", "{reqsJsonPath[0]['.metadata.name']['-']}", nil), + }), + antdFlexVertical("meta-namespace-block", 8, []any{ + antdText("meta-namespace-label", true, "Namespace", nil), + antdFlex("header-row", 6, []any{ + map[string]any{ + "type": "antdText", + "data": map[string]any{ + "id": "header-badge", + "text": "NS", + "title": "namespace", + "style": map[string]any{ + "backgroundColor": "#a25792ff", + "borderRadius": "20px", + "color": "#fff", + "display": "inline-block", + "fontFamily": "RedHatDisplay, Overpass, overpass, helvetica, arial, sans-serif", + "fontSize": "15px", + "fontWeight": 400, + "lineHeight": "24px", + "minWidth": 24, + "padding": "0 9px", + "textAlign": "center", + "whiteSpace": "nowrap", + }, + }, + }, + map[string]any{ + "type": "antdLink", + "data": map[string]any{ + "id": "namespace-link", + "text": "{reqsJsonPath[0]['.metadata.namespace']['-']}", + "href": "/openapi-ui/{2}/{reqsJsonPath[0]['.metadata.namespace']['-']}/factory/marketplace", + }, + }, + }), + }), + antdFlexVertical("meta-created-block", 4, []any{ + antdText("time-label", true, "Created", nil), + antdFlex("time-block", 6, []any{ + map[string]any{ + "type": "antdText", + "data": map[string]any{ + "id": "time-icon", + "text": "🌐", + }, + }, + map[string]any{ + "type": "parsedText", + "data": map[string]any{ + "formatter": "timestamp", + "id": "time-value", + "text": "{reqsJsonPath[0]['.metadata.creationTimestamp']['-']}", + }, + }, + }), + }), + }), + }), + antdCol("col-right", 12, []any{ + antdFlexVertical("col-right-stack", 24, []any{ + antdFlexVertical("status-phase-block", 4, []any{ + antdText("phase-label", true, "Phase", nil), + parsedText("phase-value", "{reqsJsonPath[0]['.status.phase']['-']}", nil), + }), + antdFlexVertical("spec-backup-ref-block", 4, []any{ + antdText("backup-ref-label", true, "Backup Ref", nil), + parsedText("backup-ref-value", "{reqsJsonPath[0]['.spec.backupRef.name']['-']}", nil), + }), + antdFlexVertical("spec-target-application-ref-block", 4, []any{ + antdText("target-application-ref-label", true, "Target Application", nil), + parsedText("target-application-ref-value", "{reqsJsonPath[0]['.spec.targetApplicationRef.kind']['-']}.{reqsJsonPath[0]['.spec.targetApplicationRef.apiGroup']['-']}/{reqsJsonPath[0]['.spec.targetApplicationRef.name']['-']}", nil), + }), + antdFlexVertical("spec-options-target-namespace-block", 4, []any{ + antdText("options-target-namespace-label", true, "Target Namespace", nil), + parsedText("options-target-namespace-value", "{reqsJsonPath[0]['.spec.options.targetNamespace']['-']}", nil), + }), + antdFlexVertical("spec-options-fail-if-target-exists-block", 4, []any{ + antdText("options-fail-if-target-exists-label", true, "Fail If Target Exists", nil), + parsedText("options-fail-if-target-exists-value", "{reqsJsonPath[0]['.spec.options.failIfTargetExists']['-']}", nil), + }), + antdFlexVertical("spec-options-keep-original-pvc-block", 4, []any{ + antdText("options-keep-original-pvc-label", true, "Keep Original PVC", nil), + parsedText("options-keep-original-pvc-value", "{reqsJsonPath[0]['.spec.options.keepOriginalPVC']['-']}", nil), + }), + antdFlexVertical("spec-options-keep-original-ip-mac-block", 4, []any{ + antdText("options-keep-original-ip-mac-label", true, "Keep Original IP/MAC", nil), + parsedText("options-keep-original-ip-mac-value", "{reqsJsonPath[0]['.spec.options.keepOriginalIpAndMac']['-']}", nil), + }), + antdFlexVertical("status-started-at-block", 4, []any{ + antdText("started-at-label", true, "Started At", nil), + antdFlex("time-block", 6, []any{ + map[string]any{ + "type": "antdText", + "data": map[string]any{ + "id": "time-icon", + "text": "🌐", + }, + }, + map[string]any{ + "type": "parsedText", + "data": map[string]any{ + "formatter": "timestamp", + "id": "time-value", + "text": "{reqsJsonPath[0]['.status.startedAt']['-']}", + }, + }, + }), + }), + antdFlexVertical("status-completed-at-block", 4, []any{ + antdText("completed-at-label", true, "Completed At", nil), + antdFlex("time-block", 6, []any{ + map[string]any{ + "type": "antdText", + "data": map[string]any{ + "id": "time-icon", + "text": "🌐", + }, + }, + map[string]any{ + "type": "parsedText", + "data": map[string]any{ + "formatter": "timestamp", + "id": "time-value", + "text": "{reqsJsonPath[0]['.status.completedAt']['-']}", + }, + }, + }), + }), + antdFlexVertical("status-message-block", 4, []any{ + antdText("message-label", true, "Message", nil), + parsedText("message-value", "{reqsJsonPath[0]['.status.message']['-']}", nil), + }), + }), + }), + }), + }), + }, + }, + } + restoreJobSpec := createUnifiedFactory(restoreJobConfig, restoreJobTabs, []any{"/api/clusters/{2}/k8s/apis/backups.cozystack.io/v1alpha1/namespaces/{3}/restorejobs/{6}"}) + // External IPs factory (filtered services) externalIPsTabs := []any{ map[string]any{ @@ -1989,6 +2190,7 @@ func CreateAllFactories() []*dashboardv1alpha1.Factory { createFactory("plan-details", planSpec), createFactory("backupjob-details", backupJobSpec), createFactory("backup-details", backupSpec), + createFactory("restorejob-details", restoreJobSpec), createFactory("external-ips", externalIPsSpec), } } @@ -2008,9 +2210,10 @@ func CreateAllNavigations() []*dashboardv1alpha1.Navigation { "base-factory-namespaced-api-networking.k8s.io-v1-ingresses": "kube-ingress-details", "base-factory-namespaced-api-cozystack.io-v1alpha1-workloadmonitors": "workloadmonitor-details", // Backup resources (not ApplicationDefinitions, so ensureNavigation doesn't cover them) - "base-factory-namespaced-api-backups.cozystack.io-v1alpha1-plans": "plan-details", - "base-factory-namespaced-api-backups.cozystack.io-v1alpha1-backupjobs": "backupjob-details", - "base-factory-namespaced-api-backups.cozystack.io-v1alpha1-backups": "backup-details", + "base-factory-namespaced-api-backups.cozystack.io-v1alpha1-plans": "plan-details", + "base-factory-namespaced-api-backups.cozystack.io-v1alpha1-backupjobs": "backupjob-details", + "base-factory-namespaced-api-backups.cozystack.io-v1alpha1-backups": "backup-details", + "base-factory-namespaced-api-backups.cozystack.io-v1alpha1-restorejobs": "restorejob-details", } return []*dashboardv1alpha1.Navigation{ @@ -2135,6 +2338,19 @@ func listInputScemaItemBackupClass() map[string]any { } } +// listInputSchemaItemBackup returns a listInput schema overlay for selecting a Backup +// from the current namespace (used by RestoreJob form for spec.backupRef.name). +func listInputSchemaItemBackup() map[string]any { + return map[string]any{ + "type": "listInput", + "customProps": map[string]any{ + "valueUri": "/api/clusters/{cluster}/k8s/apis/backups.cozystack.io/v1alpha1/namespaces/{namespace}/backups", + "keysToValue": []any{"metadata", "name"}, + "keysToLabel": []any{"metadata", "name"}, + }, + } +} + // backupClassSchema returns the schema for spec.backupClassName as listInput (BackupJob/Plan). func createSchema(customProps map[string]any) map[string]any { return map[string]any{ From 61058828564c84e19c86607e8cfaa08056a5310c Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Tue, 21 Apr 2026 17:27:30 +0500 Subject: [PATCH 146/250] fix(dashboard): address review feedback on RestoreJob views - Replace rename-only wording on spec.targetApplicationRef form labels so users discover that the field also supports restoring into a different application, not just renaming. - Drop VMInstance-specific keys from the spec.options form label; the restore options are driver-specific, so hardcoding one driver's keys is misleading once other drivers land. - Render a single "Same as backup" fallback for an omitted spec.targetApplicationRef on the details page instead of the noisy "-.-/-" produced by concatenating three missing path fallbacks. - De-duplicate the time-block / time-icon / time-value component IDs within the restorejob-details factory by prefixing them with created-, started-at-, and completed-at-. - Mark non-CRD-backed stock-project-factory-*-details sidebars (kube-*, plan, backupjob, backup, restorejob) as static in upsertMultipleSidebars so they pick up consistent managed-by labels instead of being left unmanaged. Signed-off-by: Myasnikov Daniil --- internal/controller/dashboard/sidebar.go | 20 +++++++-- .../controller/dashboard/static_refactored.go | 41 ++++++++++--------- 2 files changed, 38 insertions(+), 23 deletions(-) diff --git a/internal/controller/dashboard/sidebar.go b/internal/controller/dashboard/sidebar.go index c9619f3a..69c933fb 100644 --- a/internal/controller/dashboard/sidebar.go +++ b/internal/controller/dashboard/sidebar.go @@ -281,7 +281,12 @@ func (m *Manager) ensureSidebar(ctx context.Context, crd *cozyv1alpha1.Applicati "stock-instance-builtin-table", } - // Add details sidebars for all CRDs with dashboard config + // Add details sidebars for all CRDs with dashboard config, and collect + // the set of IDs that are genuinely CRD-backed (dynamic). The hardcoded + // `-details` IDs above (e.g. kube-* and backup/backupjob/plan/restorejob) + // are not tied to an ApplicationDefinition and must be treated as static + // so they receive consistent labels via upsertMultipleSidebars(). + dynamicDetailsIDs := map[string]bool{} for i := range all { def := &all[i] if def.Spec.Dashboard == nil { @@ -291,17 +296,22 @@ func (m *Manager) ensureSidebar(ctx context.Context, crd *cozyv1alpha1.Applicati lowerKind := strings.ToLower(kind) detailsID := fmt.Sprintf("stock-project-factory-%s-details", lowerKind) targetIDs = append(targetIDs, detailsID) + dynamicDetailsIDs[detailsID] = true } // 7) Upsert all target sidebars with identical menuItems and keysAndTags - return m.upsertMultipleSidebars(ctx, crd, targetIDs, keysAndTags, menuItems) + return m.upsertMultipleSidebars(ctx, crd, targetIDs, dynamicDetailsIDs, keysAndTags, menuItems) } // upsertMultipleSidebars creates/updates several Sidebar resources with the same menu spec. +// dynamicDetailsIDs identifies `stock-project-factory--details` sidebars that are +// backed by an ApplicationDefinition and should therefore be owned by that CRD. +// Any other ID is treated as a static sidebar (managed-by labels, no owner ref). func (m *Manager) upsertMultipleSidebars( ctx context.Context, crd *cozyv1alpha1.ApplicationDefinition, ids []string, + dynamicDetailsIDs map[string]bool, keysAndTags map[string]any, menuItems []any, ) error { @@ -317,8 +327,10 @@ func (m *Manager) upsertMultipleSidebars( if _, err := controllerutil.CreateOrUpdate(ctx, m.Client, obj, func() error { // Only set owner reference for dynamic sidebars (stock-project-factory-{kind}-details) - // Static sidebars (stock-instance-*, stock-project-*) should not have owner references - if strings.HasPrefix(id, "stock-project-factory-") && strings.HasSuffix(id, "-details") { + // that are actually backed by an ApplicationDefinition. Static sidebars — including + // hardcoded details sidebars for built-in/backup resources — must fall through to the + // static-label branch so they're managed consistently. + if strings.HasPrefix(id, "stock-project-factory-") && strings.HasSuffix(id, "-details") && dynamicDetailsIDs[id] { // This is a dynamic sidebar, set owner reference only if it matches the current CRD _, _, kind := pickGVK(crd) lowerKind := strings.ToLower(kind) diff --git a/internal/controller/dashboard/static_refactored.go b/internal/controller/dashboard/static_refactored.go index a180f66d..9f28c0fb 100644 --- a/internal/controller/dashboard/static_refactored.go +++ b/internal/controller/dashboard/static_refactored.go @@ -563,14 +563,14 @@ func CreateAllCustomFormsOverrides() []*dashboardv1alpha1.CustomFormsOverride { createFormItem("metadata.namespace", "Namespace", "text"), createFormItem("spec.backupRef.name", "Backup", "text"), // Target application: leave empty to restore into the same application - // as referenced by the selected Backup. Fill all three only to rename - // or restore into a different application. - createFormItem("spec.targetApplicationRef.apiGroup", "Target Application API Group (optional, only for rename)", "text"), - createFormItem("spec.targetApplicationRef.kind", "Target Application Kind (optional, only for rename)", "text"), - createFormItem("spec.targetApplicationRef.name", "Target Application Name (optional, only for rename)", "text"), - // Driver-specific options (key-value editor). Known keys for VMInstance: - // targetNamespace, failIfTargetExists, keepOriginalPVC, keepOriginalIpAndMac. - createFormItem("spec.options", "Options (VMInstance keys: targetNamespace, failIfTargetExists, keepOriginalPVC, keepOriginalIpAndMac)", "object"), + // as referenced by the selected Backup. Fill all three to restore into + // a different application (e.g. rename, or restore into a new target). + createFormItem("spec.targetApplicationRef.apiGroup", "Target Application API Group (optional, used to restore into a different application)", "text"), + createFormItem("spec.targetApplicationRef.kind", "Target Application Kind (optional, used to restore into a different application)", "text"), + createFormItem("spec.targetApplicationRef.name", "Target Application Name (optional, used to restore into a different application)", "text"), + // Driver-specific options (key-value editor). Refer to the backup driver + // documentation for supported keys. + createFormItem("spec.options", "Options (driver-specific key/value pairs)", "object"), }, "schema": createSchema(map[string]any{ "backupRef": map[string]any{ @@ -2034,12 +2034,12 @@ func CreateAllFactories() []*dashboardv1alpha1.Factory { }), }), antdFlexVertical("meta-created-block", 4, []any{ - antdText("time-label", true, "Created", nil), - antdFlex("time-block", 6, []any{ + antdText("created-time-label", true, "Created", nil), + antdFlex("created-time-block", 6, []any{ map[string]any{ "type": "antdText", "data": map[string]any{ - "id": "time-icon", + "id": "created-time-icon", "text": "🌐", }, }, @@ -2047,7 +2047,7 @@ func CreateAllFactories() []*dashboardv1alpha1.Factory { "type": "parsedText", "data": map[string]any{ "formatter": "timestamp", - "id": "time-value", + "id": "created-time-value", "text": "{reqsJsonPath[0]['.metadata.creationTimestamp']['-']}", }, }, @@ -2067,7 +2067,10 @@ func CreateAllFactories() []*dashboardv1alpha1.Factory { }), antdFlexVertical("spec-target-application-ref-block", 4, []any{ antdText("target-application-ref-label", true, "Target Application", nil), - parsedText("target-application-ref-value", "{reqsJsonPath[0]['.spec.targetApplicationRef.kind']['-']}.{reqsJsonPath[0]['.spec.targetApplicationRef.apiGroup']['-']}/{reqsJsonPath[0]['.spec.targetApplicationRef.name']['-']}", nil), + // targetApplicationRef is optional — when absent, the restore targets + // the same application as the selected Backup. Show a single friendly + // fallback in that case instead of rendering "-.-/-". + parsedText("target-application-ref-value", "{reqsJsonPath[0]['.spec.targetApplicationRef.name']['Same as backup']}", nil), }), antdFlexVertical("spec-options-target-namespace-block", 4, []any{ antdText("options-target-namespace-label", true, "Target Namespace", nil), @@ -2087,11 +2090,11 @@ func CreateAllFactories() []*dashboardv1alpha1.Factory { }), antdFlexVertical("status-started-at-block", 4, []any{ antdText("started-at-label", true, "Started At", nil), - antdFlex("time-block", 6, []any{ + antdFlex("started-at-time-block", 6, []any{ map[string]any{ "type": "antdText", "data": map[string]any{ - "id": "time-icon", + "id": "started-at-time-icon", "text": "🌐", }, }, @@ -2099,7 +2102,7 @@ func CreateAllFactories() []*dashboardv1alpha1.Factory { "type": "parsedText", "data": map[string]any{ "formatter": "timestamp", - "id": "time-value", + "id": "started-at-time-value", "text": "{reqsJsonPath[0]['.status.startedAt']['-']}", }, }, @@ -2107,11 +2110,11 @@ func CreateAllFactories() []*dashboardv1alpha1.Factory { }), antdFlexVertical("status-completed-at-block", 4, []any{ antdText("completed-at-label", true, "Completed At", nil), - antdFlex("time-block", 6, []any{ + antdFlex("completed-at-time-block", 6, []any{ map[string]any{ "type": "antdText", "data": map[string]any{ - "id": "time-icon", + "id": "completed-at-time-icon", "text": "🌐", }, }, @@ -2119,7 +2122,7 @@ func CreateAllFactories() []*dashboardv1alpha1.Factory { "type": "parsedText", "data": map[string]any{ "formatter": "timestamp", - "id": "time-value", + "id": "completed-at-time-value", "text": "{reqsJsonPath[0]['.status.completedAt']['-']}", }, }, From 04cc1633be6910e3c0d06f9621ab281f0c8dea02 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Tue, 21 Apr 2026 17:13:27 +0200 Subject: [PATCH 147/250] fix(kube-ovn): scope kubeovn-plunger cache and RBAC to its namespace The kubeovn-plunger controller-runtime cache attempted cluster-wide list/watch on Deployments and Pods, which the namespace-scoped Role cannot satisfy. Additionally, the deployments rule relied on resourceNames, which does not restrict list/watch verbs and left the permission effectively unusable. Scope the manager cache to the kube-ovn namespace so list/watch hit the namespaced API, and drop resourceNames from the deployments rule. Wire --kube-ovn-namespace and --ovn-central-name through the Deployment so values are actually consumed. Co-Authored-By: Claude Signed-off-by: Andrei Kvapil --- cmd/kubeovn-plunger/main.go | 6 ++++++ packages/system/kubeovn-plunger/templates/deployment.yaml | 2 ++ packages/system/kubeovn-plunger/templates/rbac.yaml | 2 -- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/cmd/kubeovn-plunger/main.go b/cmd/kubeovn-plunger/main.go index 210405ce..611d030b 100644 --- a/cmd/kubeovn-plunger/main.go +++ b/cmd/kubeovn-plunger/main.go @@ -29,6 +29,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics" @@ -131,6 +132,11 @@ func main() { HealthProbeBindAddress: probeAddr, LeaderElection: enableLeaderElection, LeaderElectionID: "29a0338b.cozystack.io", + Cache: cache.Options{ + DefaultNamespaces: map[string]cache.Config{ + kubeOVNNamespace: {}, + }, + }, // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily // when the Manager ends. This requires the binary to immediately end when the // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly diff --git a/packages/system/kubeovn-plunger/templates/deployment.yaml b/packages/system/kubeovn-plunger/templates/deployment.yaml index 183e8c5e..37b49620 100644 --- a/packages/system/kubeovn-plunger/templates/deployment.yaml +++ b/packages/system/kubeovn-plunger/templates/deployment.yaml @@ -29,6 +29,8 @@ spec: {{- end }} - --metrics-bind-address=:8080 - --metrics-secure=false + - --kube-ovn-namespace={{ .Release.Namespace }} + - --ovn-central-name={{ .Values.ovnCentralName }} ports: - name: metrics containerPort: 8080 diff --git a/packages/system/kubeovn-plunger/templates/rbac.yaml b/packages/system/kubeovn-plunger/templates/rbac.yaml index a0c5527b..3ca22740 100644 --- a/packages/system/kubeovn-plunger/templates/rbac.yaml +++ b/packages/system/kubeovn-plunger/templates/rbac.yaml @@ -22,8 +22,6 @@ rules: - get - list - watch - resourceNames: - - {{ .Values.ovnCentralName }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding From bb4be57774ef105c1e52fbbfab8bee0dcf52fd06 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Tue, 21 Apr 2026 20:09:19 +0200 Subject: [PATCH 148/250] fix(kube-ovn): bump kube-ovn to v1.15.10 with port-group regression fix Pulls cozystack/kubeovn-chart v1.15.10-cozy.1, which bumps upstream kube-ovn from v1.15.3 to v1.15.10 and carries a patch over pkg/controller/pod.go that preserves a VM LSP's port-group memberships when kubernetes GCs a completed virt-launcher pod while another virt-launcher pod of the same VM is still running. Without the patch, the destination pod of a successful live migration loses its security groups, network policies and node-scoped routing after kubernetes cleans up the migration source pod, and only recovers after a kube-ovn-controller restart. Upstream issue: kubeovn/kube-ovn#6665 Upstream fix PR: kubeovn/kube-ovn#6666 Chart carry: cozystack/kubeovn-chart#4 Co-Authored-By: Claude Signed-off-by: Andrei Kvapil --- .../system/kubeovn/charts/kube-ovn/Chart.yaml | 4 +- .../kube-ovn/templates/kube-ovn-crd.yaml | 71 ++++++++++ .../charts/kube-ovn/templates/ovncni-ds.yaml | 3 + .../templates/pre-upgrade-ovs-ovn.yaml | 129 ++++++++++++++++++ .../kubeovn/charts/kube-ovn/values.yaml | 4 +- packages/system/kubeovn/values.yaml | 2 +- 6 files changed, 208 insertions(+), 5 deletions(-) create mode 100644 packages/system/kubeovn/charts/kube-ovn/templates/pre-upgrade-ovs-ovn.yaml diff --git a/packages/system/kubeovn/charts/kube-ovn/Chart.yaml b/packages/system/kubeovn/charts/kube-ovn/Chart.yaml index f0295b87..9c01c123 100644 --- a/packages/system/kubeovn/charts/kube-ovn/Chart.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/Chart.yaml @@ -15,12 +15,12 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: v1.15.3 +version: v1.15.10 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "1.15.3" +appVersion: "1.15.10" kubeVersion: ">= 1.29.0-0" diff --git a/packages/system/kubeovn/charts/kube-ovn/templates/kube-ovn-crd.yaml b/packages/system/kubeovn/charts/kube-ovn/templates/kube-ovn-crd.yaml index 5b093be6..62ae328e 100644 --- a/packages/system/kubeovn/charts/kube-ovn/templates/kube-ovn-crd.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/templates/kube-ovn-crd.yaml @@ -1353,6 +1353,77 @@ spec: type: string type: object type: array + resources: + description: |- + `resources` are the compute resources required by this container. + For more information, see https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. + + This field depends on the DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + bandwidth: + type: object + description: Optional bandwidth limit for each egress gateway instance in both ingress and egress directions. + properties: + ingress: + type: integer + format: int64 + description: ingress bandwidth limit in Mbps + egress: + type: integer + format: int64 + description: egress bandwidth limit in Mbps --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition diff --git a/packages/system/kubeovn/charts/kube-ovn/templates/ovncni-ds.yaml b/packages/system/kubeovn/charts/kube-ovn/templates/ovncni-ds.yaml index 8be8eba6..06971d01 100644 --- a/packages/system/kubeovn/charts/kube-ovn/templates/ovncni-ds.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/templates/ovncni-ds.yaml @@ -126,6 +126,9 @@ spec: - --enable-ovn-ipsec={{- .Values.func.ENABLE_OVN_IPSEC }} - --set-vxlan-tx-off={{- .Values.func.SET_VXLAN_TX_OFF }} - --non-primary-cni-mode={{- .Values.cni_conf.NON_PRIMARY_CNI }} + {{- with .Values.mtu }} + - --mtu={{ . }} + {{- end }} securityContext: runAsUser: 0 privileged: false diff --git a/packages/system/kubeovn/charts/kube-ovn/templates/pre-upgrade-ovs-ovn.yaml b/packages/system/kubeovn/charts/kube-ovn/templates/pre-upgrade-ovs-ovn.yaml new file mode 100644 index 00000000..a18a4111 --- /dev/null +++ b/packages/system/kubeovn/charts/kube-ovn/templates/pre-upgrade-ovs-ovn.yaml @@ -0,0 +1,129 @@ +{{- if include "kubeovn.ovn.versionCompatibility" . -}} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ovs-ovn-pre-upgrade + namespace: {{ .Values.namespace }} + annotations: + "helm.sh/hook": pre-upgrade + "helm.sh/hook-weight": "1" + "helm.sh/hook-delete-policy": hook-succeeded +automountServiceAccountToken: false +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.k8s.io/system-only: "true" + "helm.sh/hook": pre-upgrade + "helm.sh/hook-weight": "2" + "helm.sh/hook-delete-policy": hook-succeeded + name: system:ovs-ovn-pre-upgrade +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - list + - get + - apiGroups: + - "" + resources: + - pods/exec + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ovs-ovn-pre-upgrade + annotations: + "helm.sh/hook": pre-upgrade + "helm.sh/hook-weight": "3" + "helm.sh/hook-delete-policy": hook-succeeded +roleRef: + name: system:ovs-ovn-pre-upgrade + kind: ClusterRole + apiGroup: rbac.authorization.k8s.io +subjects: + - kind: ServiceAccount + name: ovs-ovn-pre-upgrade + namespace: {{ .Values.namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: "{{ .Chart.Name }}-pre-upgrade-hook" + namespace: {{ .Values.namespace }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + annotations: + "helm.sh/hook": pre-upgrade + "helm.sh/hook-weight": "4" + "helm.sh/hook-delete-policy": hook-succeeded +spec: + completions: 1 + template: + metadata: + name: "{{ .Release.Name }}" + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + app: pre-upgrade + component: job + spec: + tolerations: + - key: "" + operator: "Exists" + effect: "NoSchedule" + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - pre-upgrade + - key: component + operator: In + values: + - job + restartPolicy: Never + hostNetwork: true + nodeSelector: + kubernetes.io/os: "linux" + serviceAccountName: ovs-ovn-pre-upgrade + automountServiceAccountToken: true + securityContext: + seccompProfile: + type: RuntimeDefault + containers: + - name: ovs-ovn-pre-upgrade + image: "{{ .Values.global.registry.address}}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }}" + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + command: + - bash + - -eo + - pipefail + - -c + - bash /kube-ovn/pre-upgrade-ovs.sh 2>&1 | tee -a /var/log/kube-ovn/pre-upgrade-ovs.log + volumeMounts: + - mountPath: /var/log/kube-ovn + name: kube-ovn-log + volumes: + - name: kube-ovn-log + hostPath: + path: {{ .Values.log_conf.LOG_DIR }}/kube-ovn +{{- end -}} diff --git a/packages/system/kubeovn/charts/kube-ovn/values.yaml b/packages/system/kubeovn/charts/kube-ovn/values.yaml index 3ffb7750..fd3472f6 100644 --- a/packages/system/kubeovn/charts/kube-ovn/values.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/values.yaml @@ -8,11 +8,11 @@ global: images: kubeovn: repository: kube-ovn - tag: v1.15.3 + tag: v1.15.10 natgateway: repository: vpc-nat-gateway # Falls back to the same tag as kubeovn if empty - tag: v1.15.3 + tag: v1.15.10 image: pullPolicy: IfNotPresent diff --git a/packages/system/kubeovn/values.yaml b/packages/system/kubeovn/values.yaml index 6911f8aa..e9d2d10f 100644 --- a/packages/system/kubeovn/values.yaml +++ b/packages/system/kubeovn/values.yaml @@ -65,4 +65,4 @@ global: images: kubeovn: repository: kubeovn - tag: v1.15.3@sha256:fa53d5f254f640cb626329ad35d9e7aad647dd8e1e645e68f3f13c3659472a30 + tag: v1.15.10@sha256:741299cbd0081a786a6b60c460fa3156b3a42a38141c559dd8ac031f50c5504f From 68a624dccbe0a50c3731111ba2761f47467c8157 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Tue, 21 Apr 2026 20:26:13 +0200 Subject: [PATCH 149/250] fix(harbor): remove incorrect tenant module flags Harbor is a PaaS service, not a tenant module. It is not deployed automatically into tenant namespaces (no manifest in packages/apps/tenant/templates/). Remove the misplaced `dashboard.module: true` flag and `internal.cozystack.io/tenantmodule: "true"` release label so Harbor appears under the PaaS category in the dashboard and is not treated as a tenant module by the controllers. Co-Authored-By: Claude Signed-off-by: Andrei Kvapil --- packages/system/harbor-rd/cozyrds/harbor.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/system/harbor-rd/cozyrds/harbor.yaml b/packages/system/harbor-rd/cozyrds/harbor.yaml index 5abc725f..5682c053 100644 --- a/packages/system/harbor-rd/cozyrds/harbor.yaml +++ b/packages/system/harbor-rd/cozyrds/harbor.yaml @@ -13,7 +13,6 @@ spec: prefix: "harbor-" labels: sharding.fluxcd.io/key: tenants - internal.cozystack.io/tenantmodule: "true" chartRef: kind: ExternalArtifact name: cozystack-harbor-application-default-harbor @@ -24,7 +23,6 @@ spec: plural: Harbor name: harbor description: Managed Harbor container registry - module: true tags: - registry - container From 0fdb25df724ba5d44c794271cc0661d97b1d9b13 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 22 Apr 2026 11:59:50 +0500 Subject: [PATCH 150/250] ci(api): add codegen drift check Run root 'make generate' as a pre-commit hook and as a dedicated CI workflow so missed codegen updates (CRDs, deepcopy, clients, RBAC) are caught instead of merging stale generated files. Pre-commit hook is scoped to files that actually affect codegen (api/, pkg/apis/, hack/update-codegen.sh, hack/boilerplate.go.txt) so unrelated commits are not slowed down. CI job sets up Go from go.mod, runs make generate, and fails on drift with a pointer to the local fix. Assisted-By: Claude Signed-off-by: Myasnikov Daniil --- .github/workflows/codegen-drift.yml | 42 +++++++++++++++++++++++++++++ .pre-commit-config.yaml | 11 ++++++++ 2 files changed, 53 insertions(+) create mode 100644 .github/workflows/codegen-drift.yml diff --git a/.github/workflows/codegen-drift.yml b/.github/workflows/codegen-drift.yml new file mode 100644 index 00000000..bd7927f7 --- /dev/null +++ b/.github/workflows/codegen-drift.yml @@ -0,0 +1,42 @@ +name: Codegen Drift Check + +on: + pull_request: + types: [opened, synchronize, reopened] + paths: + - 'api/**' + - 'pkg/apis/**' + - 'hack/update-codegen.sh' + - 'hack/boilerplate.go.txt' + - 'go.mod' + - 'go.sum' + - '.github/workflows/codegen-drift.yml' + +concurrency: + group: codegen-drift-${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + codegen-drift: + name: Verify generated code is up to date + runs-on: ubuntu-22.04 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Run make generate + run: make generate + + - name: Fail on drift + run: | + if ! git diff --exit-code; then + echo "::error::'make generate' produced changes. Run 'make generate' locally and commit the result." + git status --short + exit 1 + fi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ac0f7e30..836f79c5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,17 @@ repos: - repo: local hooks: + - id: run-make-generate-root + name: Run 'make generate' at repo root + entry: | + flock -x .git/pre-commit.lock sh -c ' + echo "Running make generate at repo root" + make generate || exit $? + git diff --color=always | cat + ' + language: system + files: ^(api/|pkg/apis/|hack/update-codegen\.sh$|hack/boilerplate\.go\.txt$) + pass_filenames: false - id: run-make-generate name: Run 'make generate' in all app directories entry: | From 860f431187c3f577bfd2b7ab60201583bdef9384 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 22 Apr 2026 12:00:22 +0500 Subject: [PATCH 151/250] chore(api): regenerate deepcopy for RestoreJobSpec.Options The Options field was added to RestoreJobSpec without re-running 'make generate', leaving zz_generated.deepcopy.go out of sync. Regenerated to include the missing DeepCopyInto handling for the runtime.RawExtension pointer. Assisted-By: Claude Signed-off-by: Myasnikov Daniil --- api/backups/v1alpha1/zz_generated.deepcopy.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/api/backups/v1alpha1/zz_generated.deepcopy.go b/api/backups/v1alpha1/zz_generated.deepcopy.go index 89f6171f..61b8f839 100644 --- a/api/backups/v1alpha1/zz_generated.deepcopy.go +++ b/api/backups/v1alpha1/zz_generated.deepcopy.go @@ -620,6 +620,11 @@ func (in *RestoreJobSpec) DeepCopyInto(out *RestoreJobSpec) { *out = new(v1.TypedLocalObjectReference) (*in).DeepCopyInto(*out) } + if in.Options != nil { + in, out := &in.Options, &out.Options + *out = new(runtime.RawExtension) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestoreJobSpec. From b52e2801b460e5ba00a864dd5adc29fd4f20fae8 Mon Sep 17 00:00:00 2001 From: "cozystack-ci[bot]" <274107086+cozystack-ci[bot]@users.noreply.github.com> Date: Wed, 22 Apr 2026 07:28:09 +0000 Subject: [PATCH 152/250] Prepare release v1.3.0 Signed-off-by: cozystack-ci[bot] <274107086+cozystack-ci[bot]@users.noreply.github.com> --- packages/apps/kubernetes/images/kubevirt-csi-driver.tag | 2 +- packages/apps/mariadb/images/mariadb-backup.tag | 2 +- packages/core/installer/values.yaml | 4 ++-- packages/core/platform/values.yaml | 2 +- packages/core/testing/values.yaml | 2 +- packages/extra/bootbox/images/matchbox.tag | 2 +- packages/extra/seaweedfs/images/objectstorage-sidecar.tag | 2 +- packages/system/backup-controller/values.yaml | 2 +- packages/system/backupstrategy-controller/values.yaml | 2 +- packages/system/bucket/images/s3manager.tag | 2 +- packages/system/cozystack-api/values.yaml | 2 +- packages/system/cozystack-controller/values.yaml | 2 +- packages/system/dashboard/templates/configmap.yaml | 2 +- packages/system/dashboard/values.yaml | 6 +++--- .../system/grafana-operator/images/grafana-dashboards.tag | 2 +- packages/system/kamaji/values.yaml | 4 ++-- packages/system/kubeovn-plunger/values.yaml | 2 +- packages/system/kubeovn-webhook/values.yaml | 2 +- packages/system/kubevirt-csi-node/values.yaml | 2 +- packages/system/lineage-controller-webhook/values.yaml | 2 +- packages/system/linstor/values.yaml | 4 ++-- packages/system/multus/templates/multus-daemonset-thick.yml | 4 ++-- packages/system/objectstorage-controller/values.yaml | 2 +- packages/system/seaweedfs/values.yaml | 2 +- 24 files changed, 30 insertions(+), 30 deletions(-) diff --git a/packages/apps/kubernetes/images/kubevirt-csi-driver.tag b/packages/apps/kubernetes/images/kubevirt-csi-driver.tag index ea4a21b3..00aa2caa 100644 --- a/packages/apps/kubernetes/images/kubevirt-csi-driver.tag +++ b/packages/apps/kubernetes/images/kubevirt-csi-driver.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/kubevirt-csi-driver:0.0.0@sha256:36e26a6b9063761ca3f5597bbd6272651d8a502abbc1c4e3a20e6b7e45a2875a +ghcr.io/cozystack/cozystack/kubevirt-csi-driver:0.0.0@sha256:72154a97054e16cdf3dea6129d962b8d7e86b55cf9386095e8ac2ce7c8b69172 diff --git a/packages/apps/mariadb/images/mariadb-backup.tag b/packages/apps/mariadb/images/mariadb-backup.tag index 1e381661..6c830892 100644 --- a/packages/apps/mariadb/images/mariadb-backup.tag +++ b/packages/apps/mariadb/images/mariadb-backup.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/mariadb-backup:0.0.0@sha256:0ddbbec0568dcb9fbc317cd9cc654e826dbe88ba3f184fa9b6b58aacb93b4570 +ghcr.io/cozystack/cozystack/mariadb-backup:0.0.0@sha256:3841eb171416711977dea0cf8cd45d32344caac9727af760c37d5e1dd41ee4bb diff --git a/packages/core/installer/values.yaml b/packages/core/installer/values.yaml index f0a363e1..eef691ea 100644 --- a/packages/core/installer/values.yaml +++ b/packages/core/installer/values.yaml @@ -1,9 +1,9 @@ cozystackOperator: # Deployment variant: talos, generic, hosted variant: talos - image: ghcr.io/cozystack/cozystack/cozystack-operator:v1.3.0-rc.1@sha256:d833abf3eac990732440d7d04e2df62ef9cd46704e0637e63e86983132c6d958 + image: ghcr.io/cozystack/cozystack/cozystack-operator:v1.3.0@sha256:62574f12486bb40c901cf5ed484cca264405ce5810196d86555cbb27cce1ba48 platformSourceUrl: 'oci://ghcr.io/cozystack/cozystack/cozystack-packages' - platformSourceRef: 'digest=sha256:db28afd18635f6295342f58cd90ac7015c8f59d46ae9704fffeb2c654c3c8a0e' + platformSourceRef: 'digest=sha256:a0b9ef938446b3132d3d22ad2262beb1027c48c9037b6c2346fdc2f19acd3036' # Generic variant configuration (only used when cozystackOperator.variant=generic) cozystack: # Kubernetes API server host (IP only, no protocol/port) diff --git a/packages/core/platform/values.yaml b/packages/core/platform/values.yaml index 1d9a51c5..f33926db 100644 --- a/packages/core/platform/values.yaml +++ b/packages/core/platform/values.yaml @@ -5,7 +5,7 @@ sourceRef: path: / migrations: enabled: false - image: ghcr.io/cozystack/cozystack/platform-migrations:v1.3.0-rc.1@sha256:555e4b76421361805a84bc9088b01b23a9c4a9430bd8ebd2db82ef9677d7008c + image: ghcr.io/cozystack/cozystack/platform-migrations:v1.3.0@sha256:555e4b76421361805a84bc9088b01b23a9c4a9430bd8ebd2db82ef9677d7008c targetVersion: 39 # Bundle deployment configuration bundles: diff --git a/packages/core/testing/values.yaml b/packages/core/testing/values.yaml index fe76a451..9d4bdd10 100644 --- a/packages/core/testing/values.yaml +++ b/packages/core/testing/values.yaml @@ -1,2 +1,2 @@ e2e: - image: ghcr.io/cozystack/cozystack/e2e-sandbox:v1.3.0-rc.1@sha256:0367a03b981df2a3ea13f411d4cb7869c2bf2c89c07d3d5c8971b9a28921ccef + image: ghcr.io/cozystack/cozystack/e2e-sandbox:v1.3.0@sha256:0367a03b981df2a3ea13f411d4cb7869c2bf2c89c07d3d5c8971b9a28921ccef diff --git a/packages/extra/bootbox/images/matchbox.tag b/packages/extra/bootbox/images/matchbox.tag index 214064c3..f51c7675 100644 --- a/packages/extra/bootbox/images/matchbox.tag +++ b/packages/extra/bootbox/images/matchbox.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/matchbox:v1.3.0-rc.1@sha256:27959a7e36fb5594049cf88e7b196e661c6ad161dc89330b2c0ef543d1d48367 +ghcr.io/cozystack/cozystack/matchbox:v1.3.0@sha256:85b8e04bf6f0690612dd63e80475df269f4a436d16680f8a40f2860cf16e2f74 diff --git a/packages/extra/seaweedfs/images/objectstorage-sidecar.tag b/packages/extra/seaweedfs/images/objectstorage-sidecar.tag index 6d85d153..9b413da5 100644 --- a/packages/extra/seaweedfs/images/objectstorage-sidecar.tag +++ b/packages/extra/seaweedfs/images/objectstorage-sidecar.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.3.0-rc.1@sha256:5bcccbdb13979a16cee535eb5fbcdf0d87973689010a10b7b25e55c5be3edaa6 +ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.3.0@sha256:5bcccbdb13979a16cee535eb5fbcdf0d87973689010a10b7b25e55c5be3edaa6 diff --git a/packages/system/backup-controller/values.yaml b/packages/system/backup-controller/values.yaml index 3c8834dc..34e033ce 100644 --- a/packages/system/backup-controller/values.yaml +++ b/packages/system/backup-controller/values.yaml @@ -1,5 +1,5 @@ backupController: - image: "ghcr.io/cozystack/cozystack/backup-controller:v1.3.0-rc.1@sha256:9d1727b80b436387141b43fd8c12ec36d75220e1d6014ebb58e2e8df3caba5e8" + image: "ghcr.io/cozystack/cozystack/backup-controller:v1.3.0@sha256:e1a083dc92f26dfef004f47c1cd20a6357174aad835004f58e751c494b76649a" replicas: 2 debug: false metrics: diff --git a/packages/system/backupstrategy-controller/values.yaml b/packages/system/backupstrategy-controller/values.yaml index 326dd7c2..d6453124 100644 --- a/packages/system/backupstrategy-controller/values.yaml +++ b/packages/system/backupstrategy-controller/values.yaml @@ -1,5 +1,5 @@ backupStrategyController: - image: "ghcr.io/cozystack/cozystack/backupstrategy-controller:v1.3.0-rc.1@sha256:aebdf354e15bff73d4e12d3f6117494002e8a4e586d1c46450cd0a3da493fbb5" + image: "ghcr.io/cozystack/cozystack/backupstrategy-controller:v1.3.0@sha256:be0a9ec1f4307064b16388a24628aee46e06252738338add80b99ea1e04e62bf" replicas: 2 debug: false metrics: diff --git a/packages/system/bucket/images/s3manager.tag b/packages/system/bucket/images/s3manager.tag index 30b082df..889db0af 100644 --- a/packages/system/bucket/images/s3manager.tag +++ b/packages/system/bucket/images/s3manager.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/s3manager:v0.5.0@sha256:cfffa1267ad72d138c6e9eed6fa282f4fa23698bef2180be98c0be39c073ec28 +ghcr.io/cozystack/cozystack/s3manager:v0.5.0@sha256:fb65e734bbdbdaef2238769cc2ecfb54dbddaf1e0952ad438a9b3e26b9dbb4b5 diff --git a/packages/system/cozystack-api/values.yaml b/packages/system/cozystack-api/values.yaml index 2ccdb706..dddabf54 100644 --- a/packages/system/cozystack-api/values.yaml +++ b/packages/system/cozystack-api/values.yaml @@ -1,3 +1,3 @@ cozystackAPI: - image: ghcr.io/cozystack/cozystack/cozystack-api:v1.3.0-rc.1@sha256:c57f2e62a547e4d62ec1c88708bbce146653bc68e1e312ab16f97a9206bd6750 + image: ghcr.io/cozystack/cozystack/cozystack-api:v1.3.0@sha256:5fa8648821cf1e9e08cf7c2899c4b1c4226bb74c6773327141456c7d3f2b9b7e replicas: 2 diff --git a/packages/system/cozystack-controller/values.yaml b/packages/system/cozystack-controller/values.yaml index 6addae2b..4d0d0fde 100644 --- a/packages/system/cozystack-controller/values.yaml +++ b/packages/system/cozystack-controller/values.yaml @@ -1,4 +1,4 @@ cozystackController: - image: ghcr.io/cozystack/cozystack/cozystack-controller:v1.3.0-rc.1@sha256:5ab50893e9d0237d26f366c9d647da6337ca9b97bae764430571d4fb080f6200 + image: ghcr.io/cozystack/cozystack/cozystack-controller:v1.3.0@sha256:d03d19b78c4c98f970ac549a68b01ef6bd1ad755f5e0dcb9e08503511cdf2fdc debug: false disableTelemetry: false diff --git a/packages/system/dashboard/templates/configmap.yaml b/packages/system/dashboard/templates/configmap.yaml index c2be8d46..a2b7684c 100644 --- a/packages/system/dashboard/templates/configmap.yaml +++ b/packages/system/dashboard/templates/configmap.yaml @@ -1,6 +1,6 @@ {{- $brandingConfig := .Values._cluster.branding | default dict }} -{{- $tenantText := "v1.3.0-rc.1" }} +{{- $tenantText := "v1.3.0" }} {{- $footerText := "Cozystack" }} {{- $titleText := "Cozystack Dashboard" }} {{- $logoText := "" }} diff --git a/packages/system/dashboard/values.yaml b/packages/system/dashboard/values.yaml index 63305af8..a5188825 100644 --- a/packages/system/dashboard/values.yaml +++ b/packages/system/dashboard/values.yaml @@ -1,6 +1,6 @@ openapiUI: - image: ghcr.io/cozystack/cozystack/openapi-ui:v1.3.0-rc.1@sha256:69dcecbea0cfff681043c75e3508c84f03a01b7be3e3a425deb740d629d3ab04 + image: ghcr.io/cozystack/cozystack/openapi-ui:v1.3.0@sha256:0fa79c373a62840a617ff1ca1b0e31931c13a6cf7b0bb0ff0dc191f047a465a3 openapiUIK8sBff: - image: ghcr.io/cozystack/cozystack/openapi-ui-k8s-bff:v1.3.0-rc.1@sha256:873cf834deddaa3954b6d9c5d520a8e03714b77178eff44826d93540561f599e + image: ghcr.io/cozystack/cozystack/openapi-ui-k8s-bff:v1.3.0@sha256:1b3ea6d4c7dbbe6a8def3b2807fffdfab2ac4afc39d7a846e57dd491fa168f92 tokenProxy: - image: ghcr.io/cozystack/cozystack/token-proxy:v1.3.0-rc.1@sha256:2e280991e07853ea48f97b0a42946afffa10d03d6a83d41099ed83e6ffc94fdc + image: ghcr.io/cozystack/cozystack/token-proxy:v1.3.0@sha256:2e280991e07853ea48f97b0a42946afffa10d03d6a83d41099ed83e6ffc94fdc diff --git a/packages/system/grafana-operator/images/grafana-dashboards.tag b/packages/system/grafana-operator/images/grafana-dashboards.tag index 033a6742..cde83a70 100644 --- a/packages/system/grafana-operator/images/grafana-dashboards.tag +++ b/packages/system/grafana-operator/images/grafana-dashboards.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/grafana-dashboards:v1.3.0-rc.1@sha256:2c9aa0b48e2bf6167db198f4d15882bfe51700108edf2e9f6d0942940a2c1204 +ghcr.io/cozystack/cozystack/grafana-dashboards:v1.3.0@sha256:2c9aa0b48e2bf6167db198f4d15882bfe51700108edf2e9f6d0942940a2c1204 diff --git a/packages/system/kamaji/values.yaml b/packages/system/kamaji/values.yaml index b1ca3cb7..4de74113 100644 --- a/packages/system/kamaji/values.yaml +++ b/packages/system/kamaji/values.yaml @@ -3,7 +3,7 @@ kamaji: deploy: false image: pullPolicy: IfNotPresent - tag: v1.3.0-rc.1@sha256:8c9af083b60600c0800eb56a2cda75f26007b7272a1cf019140de003bfce1a4d + tag: v1.3.0@sha256:8c9af083b60600c0800eb56a2cda75f26007b7272a1cf019140de003bfce1a4d repository: ghcr.io/cozystack/cozystack/kamaji resources: limits: @@ -13,4 +13,4 @@ kamaji: cpu: 100m memory: 100Mi extraArgs: - - --migrate-image=ghcr.io/cozystack/cozystack/kamaji:v1.3.0-rc.1@sha256:8c9af083b60600c0800eb56a2cda75f26007b7272a1cf019140de003bfce1a4d + - --migrate-image=ghcr.io/cozystack/cozystack/kamaji:v1.3.0@sha256:8c9af083b60600c0800eb56a2cda75f26007b7272a1cf019140de003bfce1a4d diff --git a/packages/system/kubeovn-plunger/values.yaml b/packages/system/kubeovn-plunger/values.yaml index a6946967..02620593 100644 --- a/packages/system/kubeovn-plunger/values.yaml +++ b/packages/system/kubeovn-plunger/values.yaml @@ -1,4 +1,4 @@ portSecurity: true routes: "" -image: ghcr.io/cozystack/cozystack/kubeovn-plunger:v1.3.0-rc.1@sha256:4819095c5237239b8e2282ed1710141a327b3e97773f598f983a22ec0f5add05 +image: ghcr.io/cozystack/cozystack/kubeovn-plunger:v1.3.0@sha256:b75a0facb99c3b0fe8090414b3425f5c4858fff632d1de122ce9944c4daa89c0 ovnCentralName: ovn-central diff --git a/packages/system/kubeovn-webhook/values.yaml b/packages/system/kubeovn-webhook/values.yaml index 24d75d95..0c6ce81a 100644 --- a/packages/system/kubeovn-webhook/values.yaml +++ b/packages/system/kubeovn-webhook/values.yaml @@ -1,3 +1,3 @@ portSecurity: true routes: "" -image: ghcr.io/cozystack/cozystack/kubeovn-webhook:v1.3.0-rc.1@sha256:e18f9fd679e38f65362a8d0042f25468272f6d081136ad47027168d8e7e07a4a +image: ghcr.io/cozystack/cozystack/kubeovn-webhook:v1.3.0@sha256:e18f9fd679e38f65362a8d0042f25468272f6d081136ad47027168d8e7e07a4a diff --git a/packages/system/kubevirt-csi-node/values.yaml b/packages/system/kubevirt-csi-node/values.yaml index a4d51b79..56e28543 100644 --- a/packages/system/kubevirt-csi-node/values.yaml +++ b/packages/system/kubevirt-csi-node/values.yaml @@ -1,3 +1,3 @@ storageClass: replicated csiDriver: - image: ghcr.io/cozystack/cozystack/kubevirt-csi-driver:0.0.0@sha256:36e26a6b9063761ca3f5597bbd6272651d8a502abbc1c4e3a20e6b7e45a2875a + image: ghcr.io/cozystack/cozystack/kubevirt-csi-driver:0.0.0@sha256:72154a97054e16cdf3dea6129d962b8d7e86b55cf9386095e8ac2ce7c8b69172 diff --git a/packages/system/lineage-controller-webhook/values.yaml b/packages/system/lineage-controller-webhook/values.yaml index 32439941..c5671548 100644 --- a/packages/system/lineage-controller-webhook/values.yaml +++ b/packages/system/lineage-controller-webhook/values.yaml @@ -1,5 +1,5 @@ lineageControllerWebhook: - image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.3.0-rc.1@sha256:7e443b9252f0477fe07471f76365edfa28550add51fe8fd4d11312d60a81850e + image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.3.0@sha256:e8984709686a5eaf19b89da378d7b8c688ea5607e0783a88d9c9e4ccfca96fb0 debug: false localK8sAPIEndpoint: enabled: true diff --git a/packages/system/linstor/values.yaml b/packages/system/linstor/values.yaml index 57f5f331..62dd2cac 100644 --- a/packages/system/linstor/values.yaml +++ b/packages/system/linstor/values.yaml @@ -1,7 +1,7 @@ piraeusServer: image: repository: ghcr.io/cozystack/cozystack/piraeus-server - tag: 1.33.2@sha256:e748ea7b2adb285b0387fde4453320155fc03e4470db7d9af1d5447087bf18b4 + tag: 1.33.2@sha256:553f313ab35dc2e345ef3683156d29e75c23177e2750e9af3a83aa9e23941cbb # Talos-specific workarounds (disable for generic Linux like Ubuntu/Debian) talos: enabled: true @@ -13,4 +13,4 @@ linstor: linstorCSI: image: repository: ghcr.io/cozystack/cozystack/linstor-csi - tag: v1.10.5@sha256:a80d39f96a988085e84be2d6e896f4b4a008fccd5bbd9eaa2b8534b076429942 + tag: v1.10.5@sha256:b8f59b5659fb1791cb764d3f37df4cf29920aadcc10637231ba7d857233f377d diff --git a/packages/system/multus/templates/multus-daemonset-thick.yml b/packages/system/multus/templates/multus-daemonset-thick.yml index 2ddc6362..cad330ea 100644 --- a/packages/system/multus/templates/multus-daemonset-thick.yml +++ b/packages/system/multus/templates/multus-daemonset-thick.yml @@ -155,7 +155,7 @@ spec: serviceAccountName: multus containers: - name: kube-multus - image: ghcr.io/cozystack/cozystack/multus-cni:v1.3.0-rc.1@sha256:d76cc81c9ef1521e1317aacdc98360325f4fbecdfc687b97852e2f5c275551b1 + image: ghcr.io/cozystack/cozystack/multus-cni:v1.3.0@sha256:6735ffc12e5e660951f5a42943b38c308f33774a55836e94c191001405b58ec0 command: [ "/usr/src/multus-cni/bin/multus-daemon" ] resources: requests: @@ -201,7 +201,7 @@ spec: fieldPath: spec.nodeName initContainers: - name: install-multus-binary - image: ghcr.io/cozystack/cozystack/multus-cni:v1.3.0-rc.1@sha256:d76cc81c9ef1521e1317aacdc98360325f4fbecdfc687b97852e2f5c275551b1 + image: ghcr.io/cozystack/cozystack/multus-cni:v1.3.0@sha256:6735ffc12e5e660951f5a42943b38c308f33774a55836e94c191001405b58ec0 command: - "/usr/src/multus-cni/bin/install_multus" - "-d" diff --git a/packages/system/objectstorage-controller/values.yaml b/packages/system/objectstorage-controller/values.yaml index 4def5b26..a3859194 100644 --- a/packages/system/objectstorage-controller/values.yaml +++ b/packages/system/objectstorage-controller/values.yaml @@ -1,3 +1,3 @@ objectstorage: controller: - image: "ghcr.io/cozystack/cozystack/objectstorage-controller:v1.3.0-rc.1@sha256:50b7eb360538dec3db747c9b28792b207068defc1f630d4bd965015d934d5279" + image: "ghcr.io/cozystack/cozystack/objectstorage-controller:v1.3.0@sha256:7a9e4bf9c3f95f364756396815bb51b6c0f58f85db653460574d94b96cd3c7d5" diff --git a/packages/system/seaweedfs/values.yaml b/packages/system/seaweedfs/values.yaml index 57843ff5..3542a77f 100644 --- a/packages/system/seaweedfs/values.yaml +++ b/packages/system/seaweedfs/values.yaml @@ -177,7 +177,7 @@ seaweedfs: bucketClassName: "seaweedfs" region: "" sidecar: - image: "ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.3.0-rc.1@sha256:5bcccbdb13979a16cee535eb5fbcdf0d87973689010a10b7b25e55c5be3edaa6" + image: "ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.3.0@sha256:5bcccbdb13979a16cee535eb5fbcdf0d87973689010a10b7b25e55c5be3edaa6" certificates: commonName: "SeaweedFS CA" ipAddresses: [] From 51c2ec0ad41722f34d6cb8a5d111ceac021c4464 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Wed, 22 Apr 2026 13:25:16 +0300 Subject: [PATCH 153/250] feat(hami): add HAMi GPU virtualization system chart Add HAMi v2.8.1 (CNCF Sandbox) as a new system package for fractional GPU sharing in tenant Kubernetes clusters. The chart enables workloads to request specific amounts of GPU memory and compute cores instead of claiming entire GPUs. Vendored upstream chart includes device plugin, scheduler extender, mutating webhook, and DRA subchart. Wrapper values configure nvidia RuntimeClass and clear hardcoded node config from upstream defaults. Assisted-By: Claude Signed-off-by: Arsolitt --- packages/system/hami/Chart.yaml | 3 + packages/system/hami/Makefile | 11 + packages/system/hami/charts/hami/Chart.lock | 6 + packages/system/hami/charts/hami/Chart.yaml | 22 + packages/system/hami/charts/hami/README.md | 237 +++++++++ .../charts/hami/charts/hami-dra/.helmignore | 24 + .../charts/hami/charts/hami-dra/Chart.yaml | 18 + .../charts/hami-dra/templates/_commons.tpl | 84 ++++ .../charts/hami-dra/templates/_helpers.tpl | 73 +++ .../hami-dra-driver/deviceclass.yaml | 11 + .../nvidia-dra-driver-daemonset.yaml | 145 ++++++ .../templates/hami-dra-driver/rbac.yaml | 110 ++++ .../monitor/monitor-clusterrole.yaml | 11 + .../monitor/monitor-clusterrolebinding.yaml | 15 + .../templates/monitor/monitor-deployment.yaml | 62 +++ .../templates/monitor/monitor-service.yaml | 26 + .../monitor/monitor-serviceaccount.yaml | 8 + .../templates/webhook/cert-manager.yaml | 29 ++ .../templates/webhook/cert-secret.yaml | 13 + .../templates/webhook/device-config.yaml | 394 +++++++++++++++ .../webhook/mutatingwebhookconfiguration.yaml | 28 ++ .../validatingwebhookconfiguration.yaml | 29 ++ .../webhook/webhook-clusterrole.yaml | 14 + .../webhook/webhook-clusterrolebinding.yaml | 12 + .../templates/webhook/webhook-deployment.yaml | 59 +++ .../templates/webhook/webhook-service.yaml | 15 + .../webhook/webhook-serviceaccount.yaml | 5 + .../charts/hami/charts/hami-dra/values.yaml | 166 ++++++ .../hami/charts/hami/templates/NOTES.txt | 3 + .../hami/charts/hami/templates/_commons.tpl | 49 ++ .../hami/charts/hami/templates/_helpers.tpl | 163 ++++++ .../templates/device-plugin/configmap.yaml | 13 + .../device-plugin/daemonsetmock.yaml | 55 ++ .../device-plugin/daemonsetnvidia.yaml | 262 ++++++++++ .../templates/device-plugin/monitorrole.yaml | 29 ++ .../device-plugin/monitorrolebinding.yaml | 17 + .../device-plugin/monitorservice.yaml | 29 ++ .../device-plugin/monitorserviceaccount.yaml | 10 + .../device-plugin/runtime-class.yaml | 11 + .../hami/templates/scheduler/certmanager.yaml | 31 ++ .../hami/templates/scheduler/clusterrole.yaml | 36 ++ .../scheduler/clusterrolebinding.yaml | 49 ++ .../hami/templates/scheduler/configmap.yaml | 142 ++++++ .../templates/scheduler/configmapnew.yaml | 102 ++++ .../hami/templates/scheduler/deployment.yaml | 227 +++++++++ .../templates/scheduler/device-configmap.yaml | 410 +++++++++++++++ .../scheduler/job-patch/clusterrole.yaml | 30 ++ .../job-patch/clusterrolebinding.yaml | 22 + .../scheduler/job-patch/job-createSecret.yaml | 70 +++ .../scheduler/job-patch/job-patchWebhook.yaml | 65 +++ .../templates/scheduler/job-patch/psp.yaml | 40 ++ .../templates/scheduler/job-patch/role.yaml | 23 + .../scheduler/job-patch/rolebinding.yaml | 23 + .../scheduler/job-patch/serviceaccount.yaml | 15 + .../charts/hami/templates/scheduler/role.yaml | 14 + .../hami/templates/scheduler/rolebinding.yaml | 33 ++ .../hami/templates/scheduler/service.yaml | 36 ++ .../templates/scheduler/serviceaccount.yaml | 18 + .../hami/templates/scheduler/webhook.yaml | 57 +++ packages/system/hami/charts/hami/values.yaml | 476 ++++++++++++++++++ packages/system/hami/values.yaml | 8 + 61 files changed, 4198 insertions(+) create mode 100644 packages/system/hami/Chart.yaml create mode 100644 packages/system/hami/Makefile create mode 100644 packages/system/hami/charts/hami/Chart.lock create mode 100644 packages/system/hami/charts/hami/Chart.yaml create mode 100644 packages/system/hami/charts/hami/README.md create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/.helmignore create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/_commons.tpl create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/_helpers.tpl create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/deviceclass.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/nvidia-dra-driver-daemonset.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/rbac.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrole.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrolebinding.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-deployment.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-service.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-serviceaccount.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-manager.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-secret.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/device-config.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/mutatingwebhookconfiguration.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/validatingwebhookconfiguration.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrole.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrolebinding.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-deployment.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-service.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-serviceaccount.yaml create mode 100644 packages/system/hami/charts/hami/charts/hami-dra/values.yaml create mode 100644 packages/system/hami/charts/hami/templates/NOTES.txt create mode 100644 packages/system/hami/charts/hami/templates/_commons.tpl create mode 100644 packages/system/hami/charts/hami/templates/_helpers.tpl create mode 100644 packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml create mode 100644 packages/system/hami/charts/hami/templates/device-plugin/daemonsetmock.yaml create mode 100644 packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml create mode 100644 packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml create mode 100644 packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml create mode 100644 packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml create mode 100644 packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml create mode 100644 packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/configmap.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/deployment.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/role.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/service.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml create mode 100644 packages/system/hami/charts/hami/templates/scheduler/webhook.yaml create mode 100644 packages/system/hami/charts/hami/values.yaml create mode 100644 packages/system/hami/values.yaml diff --git a/packages/system/hami/Chart.yaml b/packages/system/hami/Chart.yaml new file mode 100644 index 00000000..3fcf4c5d --- /dev/null +++ b/packages/system/hami/Chart.yaml @@ -0,0 +1,3 @@ +apiVersion: v2 +name: cozy-hami +version: 0.0.0 # Placeholder, the actual version will be automatically set during the build process diff --git a/packages/system/hami/Makefile b/packages/system/hami/Makefile new file mode 100644 index 00000000..b7a169c2 --- /dev/null +++ b/packages/system/hami/Makefile @@ -0,0 +1,11 @@ +export NAME=hami +export NAMESPACE=cozy-$(NAME) + +include ../../../hack/common-envs.mk +include ../../../hack/package.mk + +update: + rm -rf charts + helm repo add hami-charts https://project-hami.github.io/HAMi/ + helm repo update hami-charts + helm pull hami-charts/hami --untar --untardir charts diff --git a/packages/system/hami/charts/hami/Chart.lock b/packages/system/hami/charts/hami/Chart.lock new file mode 100644 index 00000000..25856c07 --- /dev/null +++ b/packages/system/hami/charts/hami/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: hami-dra + repository: https://project-hami.github.io/HAMi-DRA/ + version: 0.1.0 +digest: sha256:374551539570bcee82c1c4dea93eac59e6f410b6d5967d188efd630e203d43bb +generated: "2026-04-17T04:06:47.689117678Z" diff --git a/packages/system/hami/charts/hami/Chart.yaml b/packages/system/hami/charts/hami/Chart.yaml new file mode 100644 index 00000000..aba7e95c --- /dev/null +++ b/packages/system/hami/charts/hami/Chart.yaml @@ -0,0 +1,22 @@ +apiVersion: v2 +appVersion: 2.8.1 +dependencies: +- condition: dra.enabled + name: hami-dra + repository: https://project-hami.github.io/HAMi-DRA/ + version: 0.1.0 +description: Heterogeneous AI Computing Virtualization Middleware +keywords: +- vgpu +- gpu +kubeVersion: '>= 1.18.0-0' +maintainers: +- email: archlitchi@gmail.com + name: limengxuan +- email: xiaozhang0210@hotmail.com + name: zhangxiao +name: hami +sources: +- https://github.com/Project-HAMi/HAMi +type: application +version: 2.8.1 diff --git a/packages/system/hami/charts/hami/README.md b/packages/system/hami/charts/hami/README.md new file mode 100644 index 00000000..e8217290 --- /dev/null +++ b/packages/system/hami/charts/hami/README.md @@ -0,0 +1,237 @@ +# HAMi Helm Chart Values Documentation + +This document provides detailed descriptions of all configurable values parameters for the HAMi Helm Chart. + +## Global Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `global.imageRegistry` | Global Docker image registry | `""` | +| `global.imagePullSecrets` | Global Docker image pull secrets | `[]` | +| `global.imageTag` | Image tag | `"v2.8.1"` | +| `global.gpuHookPath` | GPU Hook path | `/usr/local` | +| `global.labels` | Global labels | `{}` | +| `global.annotations` | Global annotations | `{}` | +| `global.managedNodeSelectorEnable` | Whether to enable managed node selector | `false` | +| `global.managedNodeSelector.usage` | Managed node selector usage | `"gpu"` | +| `nameOverride` | Name override | `""` | +| `fullnameOverride` | Full name override | `""` | +| `namespaceOverride` | Namespace override | `""` | + +## Resource Name Configuration + +### NVIDIA GPU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `resourceName` | GPU resource name | `"nvidia.com/gpu"` | +| `resourceMem` | GPU memory resource name | `"nvidia.com/gpumem"` | +| `resourceMemPercentage` | GPU memory percentage resource name | `"nvidia.com/gpumem-percentage"` | +| `resourceCores` | GPU core resource name | `"nvidia.com/gpucores"` | +| `resourcePriority` | GPU priority resource name | `"nvidia.com/priority"` | + +### Cambricon MLU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `mluResourceName` | MLU resource name | `"cambricon.com/vmlu"` | +| `mluResourceMem` | MLU memory resource name | `"cambricon.com/mlu.smlu.vmemory"` | +| `mluResourceCores` | MLU core resource name | `"cambricon.com/mlu.smlu.vcore"` | + +### Hygon DCU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `dcuResourceName` | DCU resource name | `"hygon.com/dcunum"` | +| `dcuResourceMem` | DCU memory resource name | `"hygon.com/dcumem"` | +| `dcuResourceCores` | DCU core resource name | `"hygon.com/dcucores"` | + +### Metax GPU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `metaxResourceName` | GPU resource name | `"metax-tech.com/sgpu"` | +| `metaxResourceCore` | GPU core resource name | `"metax-tech.com/vcore"` | +| `metaxResourceMem` | GPU memory resource name | `"metax-tech.com/vmemory"` | +| `metaxsGPUTopologyAware` | GPU topology awareness | `"false"` | + +### Enflame GCU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `enflameResourceNameVGCU` | vGCU resource name | `"enflame.com/vgcu"` | +| `enflameResourceNameVGCUPercentage` | vGCU percentage resource name | `"enflame.com/vgcu-percentage"` | + +### Kunlunxin XPU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `kunlunResourceName` | XPU resource name | `"kunlunxin.com/xpu"` | + +## Scheduler Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `schedulerName` | Scheduler name | `"hami-scheduler"` | +| `scheduler.nodeName` | Define node name, scheduler will schedule to this node | `""` | +| `scheduler.overwriteEnv` | Whether to overwrite environment variables | `"false"` | +| `scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy` | Node scheduler policy | `binpack` | +| `scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy` | GPU scheduler policy | `spread` | +| `scheduler.metricsBindAddress` | Metrics bind address | `":9395"` | +| `scheduler.forceOverwriteDefaultScheduler` | Whether to force overwrite default scheduler | `true` | +| `scheduler.livenessProbe` | Whether to enable liveness probe | `false` | +| `scheduler.leaderElect` | Whether to enable leader election | `true` | +| `scheduler.replicas` | Number of replicas | `1` | + +### Kube Scheduler Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `scheduler.kubeScheduler.enabled` | Whether to run kube-scheduler container in scheduler pod | `true` | +| `scheduler.kubeScheduler.image.registry` | Kube scheduler image registry | `"registry.cn-hangzhou.aliyuncs.com"` | +| `scheduler.kubeScheduler.image.repository` | Kube scheduler image repository | `"google_containers/kube-scheduler"` | +| `scheduler.kubeScheduler.image.tag` | Kube scheduler image tag | `""` | +| `scheduler.kubeScheduler.image.pullPolicy` | Kube scheduler image pull policy | `IfNotPresent` | +| `scheduler.kubeScheduler.image.pullSecrets` | Kube scheduler image pull secrets | `[]` | +| `scheduler.kubeScheduler.extraNewArgs` | Extra new arguments | `["--config=/config/config.yaml", "-v=4"]` | +| `scheduler.kubeScheduler.extraArgs` | Extra arguments | `["--policy-config-file=/config/config.json", "-v=4"]` | + +### Extender Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `scheduler.extender.image.registry` | Scheduler extender image registry | `"docker.io"` | +| `scheduler.extender.image.repository` | Scheduler extender image repository | `"projecthami/hami"` | +| `scheduler.extender.image.tag` | Scheduler extender image tag | `""` | +| `scheduler.extender.image.pullPolicy` | Scheduler extender image pull policy | `IfNotPresent` | +| `scheduler.extender.image.pullSecrets` | Scheduler extender image pull secrets | `[]` | +| `scheduler.extender.extraArgs` | Scheduler extender extra arguments | `["--debug", "-v=4"]` | + +### Admission Webhook Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `scheduler.admissionWebhook.enabled` | Whether to enable admission webhook | `true` | +| `scheduler.admissionWebhook.customURL.enabled` | Whether to enable custom URL | `false` | +| `scheduler.admissionWebhook.customURL.host` | Custom URL host | `127.0.0.1` | +| `scheduler.admissionWebhook.customURL.port` | Custom URL port | `31998` | +| `scheduler.admissionWebhook.customURL.path` | Custom URL path | `/webhook` | +| `scheduler.admissionWebhook.reinvocationPolicy` | Reinvocation policy | `Never` | +| `scheduler.admissionWebhook.failurePolicy` | Failure policy | `Ignore` | + +### TLS Certificate Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `scheduler.certManager.enabled` | Whether to use cert-manager to generate self-signed certificates | `false` | +| `scheduler.patch.enabled` | Whether to use kube-webhook-certgen to generate self-signed certificates | `true` | +| `scheduler.patch.image.registry` | Certgen image registry | `"docker.io"` | +| `scheduler.patch.image.repository` | Certgen image repository | `"jettech/kube-webhook-certgen"` | +| `scheduler.patch.image.tag` | Certgen image tag | `"v1.5.2"` | +| `scheduler.patch.image.pullPolicy` | Certgen image pull policy | `IfNotPresent` | +| `scheduler.patch.imageNew.registry` | New certgen image registry | `"docker.io"` | +| `scheduler.patch.imageNew.repository` | New certgen image repository | `"liangjw/kube-webhook-certgen"` | +| `scheduler.patch.imageNew.tag` | New certgen image tag | `"v1.1.1"` | + +### Scheduler Service Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `scheduler.service.type` | Service type | `NodePort` | +| `scheduler.service.httpPort` | HTTP port | `443` | +| `scheduler.service.schedulerPort` | Scheduler NodePort | `31998` | +| `scheduler.service.monitorPort` | Monitor port | `31993` | +| `scheduler.service.monitorTargetPort` | Monitor target port | `9395` | + +## Device Plugin Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devicePlugin.image.registry` | Device plugin image registry | `"docker.io"` | +| `devicePlugin.image.repository` | Device plugin image repository | `"projecthami/hami"` | +| `devicePlugin.image.tag` | Device plugin image tag | `""` | +| `devicePlugin.image.pullPolicy` | Device plugin image pull policy | `IfNotPresent` | +| `devicePlugin.image.pullSecrets` | Device plugin image pull secrets | `[]` | + +### Monitor Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devicePlugin.monitor.image.registry` | Monitor image registry | `"docker.io"` | +| `devicePlugin.monitor.image.repository` | Monitor image repository | `"projecthami/hami"` | +| `devicePlugin.monitor.image.tag` | Monitor image tag | `""` | +| `devicePlugin.monitor.image.pullPolicy` | Monitor image pull policy | `IfNotPresent` | +| `devicePlugin.monitor.image.pullSecrets` | Monitor image pull secrets | `[]` | +| `devicePlugin.monitor.ctrPath` | Container path | `/usr/local/vgpu/containers` | +| `devicePlugin.monitor.extraArgs` | Monitor extra arguments | `["-v=4"]` | +| `devicePlugin.monitor.extraEnvs` | Monitor extra environments | `{}` | + +### Device Plugin Other Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devicePlugin.deviceSplitCount` | Integer type, default value: 10. Maximum number of tasks assigned to a single GPU device | `10` | +| `devicePlugin.deviceMemoryScaling` | Device memory scaling ratio | `1` | +| `devicePlugin.deviceCoreScaling` | Device core scaling ratio | `1` | +| `devicePlugin.runtimeClassName` | Runtime class name | `""` | +| `devicePlugin.createRuntimeClass` | Whether to create runtime class | `false` | +| `devicePlugin.migStrategy` | String type, "none" means ignore MIG functionality, "mixed" means allocate MIG devices through independent resources | `"none"` | +| `devicePlugin.disablecorelimit` | String type, "true" means disable core limit, "false" means enable core limit | `"false"` | +| `devicePlugin.passDeviceSpecsEnabled` | Whether to enable passing device specs | `false` | +| `devicePlugin.extraArgs` | Device plugin extra arguments | `["-v=4"]` | +| `devicePlugin.nodeConfiguration.config` | Node configuration for device plugin by json | An example of default configuration. | +| `devicePlugin.nodeConfiguration.externalConfigName` | Node configuration for device plugin by external congimap | `""` | +| `devicePlugin.extraEnvs` | Device plugin extra environments | `{}` | + +### Device Plugin Service Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devicePlugin.service.type` | Service type | `NodePort` | +| `devicePlugin.service.httpPort` | HTTP port | `31992` | + +### Device Plugin Deployment Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devicePlugin.pluginPath` | Plugin path | `/var/lib/kubelet/device-plugins` | +| `devicePlugin.libPath` | Library path | `/usr/local/vgpu` | +| `devicePlugin.nvidiaNodeSelector` | NVIDIA node selector | `{"gpu": "on"}` | +| `devicePlugin.updateStrategy.type` | Update strategy type | `RollingUpdate` | +| `devicePlugin.updateStrategy.rollingUpdate.maxUnavailable` | Maximum unavailable count | `1` | + +## Device Configuration + +### AWS Neuron +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.awsneuron.customresources` | Custom resources | `["aws.amazon.com/neuron", "aws.amazon.com/neuroncore"]` | + +### Kunlunxin +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.kunlun.enabled` | Whether to enable | `true` | +| `devices.kunlun.customresources` | Custom resources | `["kunlunxin.com/xpu"]` | + +### Mthreads +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.mthreads.enabled` | Whether to enable | `true` | +| `devices.mthreads.customresources` | Custom resources | `["mthreads.com/vgpu"]` | + +### NVIDIA +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.nvidia.gpuCorePolicy` | GPU core policy | `default` | +| `devices.nvidia.libCudaLogLevel` | CUDA library log level | `1` | + +### Huawei Ascend +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.ascend.enabled` | Whether to enable | `false` | +| `devices.ascend.image` | Image | `""` | +| `devices.ascend.imagePullPolicy` | Image pull policy | `IfNotPresent` | +| `devices.ascend.extraArgs` | Extra arguments | `[]` | +| `devices.ascend.nodeSelector` | Node selector | `{"ascend": "on"}` | +| `devices.ascend.tolerations` | Tolerations | `[]` | +| `devices.ascend.customresources` | Custom resources | `["huawei.com/Ascend910A", "huawei.com/Ascend910A-memory", ...]` | + +### Iluvatar +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.iluvatar.enabled` | Whether to enable | `false` | +| `devices.iluvatar.customresources` | Custom resources | `["iluvatar.ai/BI-V150-vgpu", "iluvatar.ai/BI-V150.vMem","iluvatar.ai/BI-V150.vCore", ...]` | diff --git a/packages/system/hami/charts/hami/charts/hami-dra/.helmignore b/packages/system/hami/charts/hami/charts/hami-dra/.helmignore new file mode 100644 index 00000000..898df488 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/.helmignore @@ -0,0 +1,24 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ + diff --git a/packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml b/packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml new file mode 100644 index 00000000..d01f678f --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml @@ -0,0 +1,18 @@ +apiVersion: v2 +appVersion: 0.1.0 +description: A Helm chart for HAMi DRA +home: https://github.com/Project-HAMi/HAMi-DRA +keywords: +- webhook +- kubernetes +- admission-controller +- hami +- dra +maintainers: +- name: hami-dra + url: https://github.com/Project-HAMi/HAMi-DRA +name: hami-dra +sources: +- https://github.com/Project-HAMi/HAMi-DRA +type: application +version: 0.1.0 diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/_commons.tpl b/packages/system/hami/charts/hami/charts/hami-dra/templates/_commons.tpl new file mode 100644 index 00000000..02fbed20 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/_commons.tpl @@ -0,0 +1,84 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* +Return the proper image name +{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" $) }} +*/}} +{{- define "common.images.image" -}} +{{- $registryName := .imageRoot.registry -}} +{{- $repositoryName := .imageRoot.repository -}} +{{- $tag := .imageRoot.tag | toString -}} +{{- if .global }} + {{- if .global.imageRegistry }} + {{- $registryName = .global.imageRegistry -}} + {{- end -}} +{{- end -}} +{{- if .tag }} + {{- if .tag.imageTag }} + {{- $tag = .tag.imageTag -}} + {{- end -}} +{{- end -}} +{{- if $registryName }} +{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- else -}} +{{- printf "%s:%s" $repositoryName $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead) +{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }} +*/}} +{{- define "common.images.pullSecrets" -}} + {{- $pullSecrets := list }} + + {{- if .global }} + {{- range .global.imagePullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- range .images -}} + {{- range .pullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- if (not (empty $pullSecrets)) }} +imagePullSecrets: + {{- range $pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end -}} + +{{/* +Renders a value that contains template. +Usage: +{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $) }} +*/}} +{{- define "common.tplvalues.render" -}} + {{- if typeIs "string" .value }} + {{- tpl .value .context }} + {{- else }} + {{- tpl (.value | toYaml) .context }} + {{- end }} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "common.names.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/_helpers.tpl b/packages/system/hami/charts/hami/charts/hami-dra/templates/_helpers.tpl new file mode 100644 index 00000000..99beec77 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/_helpers.tpl @@ -0,0 +1,73 @@ +{{- define "hami.dra.webhook.fullname" -}} +{{- printf "%s-%s" (include "common.names.fullname" .) "webhook" | trunc 63 | trimSuffix "-" -}} +{{- end }} + +{{- define "hami.dra.webhook.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.webhook.image "global" .Values.global) }} +{{- end -}} + +{{- define "hami.dra.webhook.imagePullSecrets" -}} +{{ include "common.images.pullSecrets" (dict "images" (list .Values.webhook.image) "global" .Values.global) }} +{{- end -}} + +{{- define "hami.dra.driver.nvidia.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.drivers.nvidia.image "global" .Values.global) }} +{{- end -}} + +{{- define "hami.dra.driver.nvidia.imagePullSecrets" -}} +{{ include "common.images.pullSecrets" (dict "images" (list .Values.drivers.nvidia.image) "global" .Values.global) }} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "hami-dra-webhook.labels" -}} +helm.sh/chart: {{ .Chart.Name }} +{{ include "hami-dra-webhook.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "hami-dra-webhook.selectorLabels" -}} +app.kubernetes.io/name: {{ .Release.Name }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: webhook +{{- end }} + +{{- define "hami.dra.monitor.fullname" -}} +{{- printf "%s-%s" (include "common.names.fullname" .) "monitor" | trunc 63 | trimSuffix "-" -}} +{{- end }} + +{{- define "hami.dra.monitor.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.monitor.image "global" .Values.global) }} +{{- end -}} + +{{- define "hami.dra.monitor.imagePullSecrets" -}} +{{ include "common.images.pullSecrets" (dict "images" (list .Values.monitor.image) "global" .Values.global) }} +{{- end -}} + +{{/* +Common labels for monitor +*/}} +{{- define "hami-dra-monitor.labels" -}} +helm.sh/chart: {{ .Chart.Name }} +{{ include "hami-dra-monitor.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels for monitor +*/}} +{{- define "hami-dra-monitor.selectorLabels" -}} +app.kubernetes.io/name: {{ .Release.Name }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: monitor +{{- end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/deviceclass.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/deviceclass.yaml new file mode 100644 index 00000000..bcb6d2c5 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/deviceclass.yaml @@ -0,0 +1,11 @@ +{{- if .Values.drivers.nvidia.enabled }} +apiVersion: resource.k8s.io/v1 +kind: DeviceClass +metadata: + name: hami-core-gpu.project-hami.io +spec: + selectors: + - cel: + expression: |- + device.driver == "hami-core-gpu.project-hami.io" && device.attributes["hami-core-gpu.project-hami.io"].type == "hami-gpu" +{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/nvidia-dra-driver-daemonset.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/nvidia-dra-driver-daemonset.yaml new file mode 100644 index 00000000..a3fb0d01 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/nvidia-dra-driver-daemonset.yaml @@ -0,0 +1,145 @@ +{{- if .Values.drivers.nvidia.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: hami-dra-driver-kubelet-plugin + namespace: {{ .Release.Namespace }} +spec: + revisionHistoryLimit: 10 + selector: + matchLabels: + hami-dra-driver-component: kubelet-plugin + template: + metadata: + labels: + app.kubernetes.io/instance: hami-dra-driver + app.kubernetes.io/name: hami-dra-driver + hami-dra-driver-component: kubelet-plugin + spec: + {{- include "hami.dra.driver.nvidia.imagePullSecrets" . | nindent 6 }} + initContainers: + - name: init-container + image: {{ include "hami.dra.driver.nvidia.image" . }} + securityContext: + privileged: true + command: [bash, /usr/bin/kubelet-plugin-prestart.sh] + env: + - name: NVIDIA_DRIVER_ROOT + value: {{ if .Values.drivers.nvidia.containerDriver }}/run/nvidia/driver{{ else }}/{{ end }} + - name: NVIDIA_VISIBLE_DEVICES + value: void + - name: KUBELET_REGISTRAR_DIRECTORY_PATH + value: /var/lib/kubelet/plugins_registry + - name: KUBELET_PLUGINS_DIRECTORY_PATH + value: /var/lib/kubelet/plugins + volumeMounts: + - name: driver-root-parent + mountPath: /driver-root-parent + readOnly: true + containers: + - args: + - |- + # Conditionally mask the params file to prevent this container from + # recreating any missing GPU device nodes. This is necessary, for + # example, when running under nvkind to limit the set GPUs governed + # by the plugin even though it has cgroup access to all of them. + if [ "${MASK_NVIDIA_DRIVER_PARAMS}" = "true" ]; then + cp /proc/driver/nvidia/params root/gpu-params + sed -i 's/^ModifyDeviceFiles: 1$/ModifyDeviceFiles: 0/' root/gpu-params + mount --bind root/gpu-params /proc/driver/nvidia/params + fi + hami-kubelet-plugin -v 6 + command: + - bash + - -c + env: + - name: MASK_NVIDIA_DRIVER_PARAMS + value: "" + - name: NVIDIA_DRIVER_ROOT + value: {{ if .Values.drivers.nvidia.containerDriver }}/run/nvidia/driver{{ else }}/{{ end }} + - name: NVIDIA_VISIBLE_DEVICES + value: void + - name: CDI_ROOT + value: /var/run/cdi + - name: NVIDIA_MIG_CONFIG_DEVICES + value: all + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: KUBELET_REGISTRAR_DIRECTORY_PATH + value: /var/lib/kubelet/plugins_registry + - name: KUBELET_PLUGINS_DIRECTORY_PATH + value: /var/lib/kubelet/plugins + - name: IMAGE_NAME + value: {{ include "hami.dra.driver.nvidia.image" . }} + lifecycle: + postStart: + exec: + command: ["/bin/bash", "-c", "/usr/bin/vgpu-init.sh /usr/local/vgpu/"] + image: {{ include "hami.dra.driver.nvidia.image" . }} + imagePullPolicy: {{ .Values.drivers.nvidia.image.pullPolicy | default "IfNotPresent" }} + name: gpus + securityContext: + privileged: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /var/lib/kubelet/plugins_registry + name: plugins-registry + - mountPath: /var/lib/kubelet/plugins + mountPropagation: Bidirectional + name: plugins + - mountPath: /var/run/cdi + name: cdi + - mountPath: /driver-root + mountPropagation: HostToContainer + name: driver-root + readOnly: true + - mountPath: /usr/local/vgpu + name: host-vgpu + - mountPath: /tmp + name: host-tmp + dnsPolicy: ClusterFirst + serviceAccount: hami-dra-driver-service-account + serviceAccountName: hami-dra-driver-service-account + volumes: + - hostPath: + path: /var/lib/kubelet/plugins_registry + type: "" + name: plugins-registry + - hostPath: + path: /var/lib/kubelet/plugins + type: "" + name: plugins + - hostPath: + path: /var/run/cdi + type: "" + name: cdi + - hostPath: + path: {{if .Values.drivers.nvidia.containerDriver }}/run/nvidia{{ else }}/{{ end }} + type: DirectoryOrCreate + name: driver-root-parent + - hostPath: + path: {{if .Values.drivers.nvidia.containerDriver }}/run/nvidia/driver{{ else }}/{{ end }} + type: DirectoryOrCreate + name: driver-root + - hostPath: + path: /dev + type: "" + name: host-dev + - hostPath: + path: /usr/local/vgpu + type: DirectoryOrCreate + name: host-vgpu + - hostPath: + path: /tmp + type: "" + name: host-tmp +{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/rbac.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/rbac.yaml new file mode 100644 index 00000000..435c37cc --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/rbac.yaml @@ -0,0 +1,110 @@ +{{- if .Values.drivers.nvidia.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hami-dra-driver-service-account + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +items: +- apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: hami-dra-driver-role + namespace: {{ .Release.Namespace }} + rules: + - apiGroups: + - apps + resources: + - daemonsets + - deployments + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +kind: List +metadata: + resourceVersion: "" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: hami-dra-driver-role-binding + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: hami-dra-driver-role +subjects: +- kind: ServiceAccount + name: hami-dra-driver-service-account + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: hami-dra-driver-clusterrole +rules: +- apiGroups: + - resource.k8s.io + resources: + - resourceclaims + verbs: + - get + - list + - watch +- apiGroups: + - resource.k8s.io + resources: + - resourceclaimtemplates + verbs: + - get + - list + - watch + - create + - update + - delete +- apiGroups: + - resource.k8s.io + resources: + - resourceslices + verbs: + - get + - list + - watch + - create + - update + - delete +- apiGroups: + - resource.k8s.io + resources: + - resourceclaims/status + verbs: + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: hami-dra-driver-clusterrole-binding-hami-dra-driver +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hami-dra-driver-clusterrole +subjects: +- kind: ServiceAccount + name: hami-dra-driver-service-account + namespace: {{ .Release.Namespace }} +{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrole.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrole.yaml new file mode 100644 index 00000000..45c24863 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrole.yaml @@ -0,0 +1,11 @@ +{{- if .Values.monitor.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "hami.dra.monitor.fullname" . }} +rules: +- apiGroups: ["resource.k8s.io"] + resources: ["resourceslices", "resourceclaims"] + verbs: ["get", "list", "watch"] +{{- end }} + diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrolebinding.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrolebinding.yaml new file mode 100644 index 00000000..766d3098 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrolebinding.yaml @@ -0,0 +1,15 @@ +{{- if .Values.monitor.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami.dra.monitor.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "hami.dra.monitor.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "hami.dra.monitor.fullname" . }} + namespace: {{ .Release.Namespace }} +{{- end }} + diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-deployment.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-deployment.yaml new file mode 100644 index 00000000..b544b376 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-deployment.yaml @@ -0,0 +1,62 @@ +{{- if .Values.monitor.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "hami.dra.monitor.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "hami.dra.monitor.fullname" . }} + {{- include "hami-dra-monitor.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.monitor.replicas | default 1 }} + selector: + matchLabels: + {{- include "hami-dra-monitor.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "hami-dra-monitor.selectorLabels" . | nindent 8 }} + spec: + {{- include "hami.dra.monitor.imagePullSecrets" . | nindent 6 }} + serviceAccountName: {{ include "hami.dra.monitor.fullname" . }} + containers: + - name: monitor + image: {{ include "hami.dra.monitor.image" . }} + imagePullPolicy: {{ .Values.monitor.image.pullPolicy | default "IfNotPresent" }} + command: + - /bin/monitor + - --v={{ .Values.monitor.logLevel | default 2 }} + - --metrics-bind-address={{ .Values.monitor.metricsBindAddress | default ":8080" }} + - --health-probe-bind-address={{ .Values.monitor.healthProbeBindAddress | default ":8000" }} + - --kube-api-qps={{ .Values.monitor.kubeAPIQPS | default 40.0 }} + - --kube-api-burst={{ .Values.monitor.kubeAPIBurst | default 60 }} + - --collect-interval={{ .Values.monitor.collectInterval | default "30s" }} + ports: + - containerPort: 8080 + name: metrics + protocol: TCP + - containerPort: 8000 + name: health + protocol: TCP + {{- if .Values.monitor.resources }} + resources: + {{- toYaml .Values.monitor.resources | nindent 12 }} + {{- end }} + livenessProbe: + httpGet: + path: /healthz + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8000 + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 +{{- end }} + diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-service.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-service.yaml new file mode 100644 index 00000000..d740cc38 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-service.yaml @@ -0,0 +1,26 @@ +{{- if .Values.monitor.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "hami.dra.monitor.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "hami-dra-monitor.selectorLabels" . | nindent 4 }} +spec: + type: {{ .Values.monitor.service.type | default "ClusterIP" }} + selector: + {{- include "hami-dra-monitor.selectorLabels" . | nindent 4 }} + ports: + - port: 8080 + targetPort: 8080 + name: metrics + protocol: TCP + {{- if and (eq (.Values.monitor.service.type | default "ClusterIP") "NodePort") .Values.monitor.service.nodePort.metrics }} + nodePort: {{ .Values.monitor.service.nodePort.metrics }} + {{- end }} + - port: 8000 + targetPort: 8000 + name: health + protocol: TCP +{{- end }} + diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-serviceaccount.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-serviceaccount.yaml new file mode 100644 index 00000000..c61ad454 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-serviceaccount.yaml @@ -0,0 +1,8 @@ +{{- if .Values.monitor.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "hami.dra.monitor.fullname" . }} + namespace: {{ .Release.Namespace }} +{{- end }} + diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-manager.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-manager.yaml new file mode 100644 index 00000000..dd31c71d --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-manager.yaml @@ -0,0 +1,29 @@ +{{- if .Values.certs.certManager.enabled }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ .Release.Name }}-dra-webhook-serving-cert + namespace: {{ .Release.Namespace }} + labels: + {{- include "hami-dra-webhook.labels" . | nindent 4 }} +spec: + dnsNames: + - {{ .Release.Name }}-dra-webhook.{{ .Release.Namespace }}.svc + - {{ .Release.Name }}-dra-webhook.{{ .Release.Namespace }}.svc.cluster.local + issuerRef: + kind: Issuer + name: {{ .Release.Name }}-dra-webhook-selfsigned-issuer + secretName: {{ .Release.Name }}-dra-webhook-tls + privateKey: + rotationPolicy: {{ .Values.certs.certManager.privateKeyRotationPolicy | default "Never" }} +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ .Release.Name }}-dra-webhook-selfsigned-issuer + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/component: hami-dra-webhook +spec: + selfSigned: {} +{{- end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-secret.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-secret.yaml new file mode 100644 index 00000000..eadbb517 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-secret.yaml @@ -0,0 +1,13 @@ +{{- if eq .Values.certs.mode "custom" }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "hami-dra-webhook.tlsSecretName" . }} + namespace: {{ .Release.Namespace }} +type: kubernetes.io/tls +data: + tls.crt: | + {{- .Values.certs.custom.crt | b64enc | indent 4 }} + tls.key: | + {{- .Values.certs.custom.key | b64enc | indent 4 }} +{{- end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/device-config.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/device-config.yaml new file mode 100644 index 00000000..97283f3e --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/device-config.yaml @@ -0,0 +1,394 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "hami.dra.webhook.fullname" . }}-device-config + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-dra-webhook.selectorLabels" . | nindent 4 }} +data: + device-config.yaml: |- + {{- if .Files.Glob "files/device-config.yaml" }} + {{- .Files.Get "files/device-config.yaml" | nindent 4}} + {{- else }} + nvidia: + resourceCountName: {{ .Values.resourceName }} + resourceMemoryName: {{ .Values.resourceMem }} + resourceMemoryPercentageName: {{ .Values.resourceMemPercentage }} + resourceCoreName: {{ .Values.resourceCores }} + resourcePriorityName: {{ .Values.resourcePriority }} + overwriteEnv: false + defaultMemory: 0 + defaultCores: 0 + defaultGPUNum: 1 + knownMigGeometries: + - models: [ "A30" ] + allowedGeometries: + - + - name: 1g.6gb + core: 25 + memory: 6144 + count: 4 + - + - name: 2g.12gb + core: 50 + memory: 12288 + count: 2 + - + - name: 4g.24gb + core: 100 + memory: 24576 + count: 1 + - models: [ "A100-SXM4-40GB", "A100-40GB-PCIe", "A100-PCIE-40GB"] + allowedGeometries: + - + - name: 1g.5gb + core: 14 + memory: 5120 + count: 7 + - + - name: 1g.5gb + core: 14 + memory: 5120 + count: 1 + - name: 2g.10gb + core: 28 + memory: 10240 + count: 3 + - + - name: 3g.20gb + core: 42 + memory: 20480 + count: 2 + - + - name: 7g.40gb + core: 100 + memory: 40960 + count: 1 + - models: [ "A100-SXM4-80GB", "A100-80GB-PCIe", "A100-PCIE-80GB"] + allowedGeometries: + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 7 + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 1 + - name: 2g.20gb + core: 28 + memory: 20480 + count: 3 + - + - name: 3g.40gb + core: 42 + memory: 40960 + count: 2 + - + - name: 7g.79gb + core: 100 + memory: 80896 + count: 1 + - models: [ "H100-PCIE-80GB", "H100-SXM5-80GB"] + allowedGeometries: + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 7 + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 1 + - name: 2g.20gb + core: 28 + memory: 20480 + count: 3 + - + - name: 3g.40gb + core: 42 + memory: 40960 + count: 2 + - + - name: 7g.80gb + core: 100 + memory: 81920 + count: 1 + - models: [ "H100-PCIE-94GB", "H100-SXM5-94GB"] + allowedGeometries: + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 7 + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 1 + - name: 2g.24gb + core: 28 + memory: 24576 + count: 3 + - + - name: 3g.47gb + core: 42 + memory: 48128 + count: 2 + - + - name: 7g.94gb + core: 100 + memory: 96256 + count: 1 + - models: [ "H20", "H100 on GH200"] + allowedGeometries: + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 7 + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 1 + - name: 2g.24gb + core: 28 + memory: 24576 + count: 3 + - + - name: 3g.48gb + core: 42 + memory: 49152 + count: 2 + - + - name: 7g.96gb + core: 100 + memory: 98304 + count: 1 + - models: [ "H200 NVL", "H200-SXM5"] + allowedGeometries: + - + - name: 1g.18gb + core: 14 + memory: 18432 + count: 7 + - + - name: 1g.18gb + core: 14 + memory: 18432 + count: 1 + - name: 2g.35gb + core: 28 + memory: 35840 + count: 3 + - + - name: 3g.71gb + core: 42 + memory: 72704 + count: 2 + - + - name: 7g.141gb + core: 100 + memory: 144384 + count: 1 + - models: [ "B200" ] + allowedGeometries: + - + - name: 1g.23gb + core: 14 + memory: 23552 + count: 7 + - + - name: 1g.23gb + core: 14 + memory: 23552 + count: 1 + - name: 2g.45gb + core: 28 + memory: 46080 + count: 3 + - + - name: 3g.90gb + core: 42 + memory: 92160 + count: 2 + - + - name: 7g.180gb + core: 100 + memory: 184320 + count: 1 + cambricon: + resourceCountName: {{ .Values.mluResourceName }} + resourceMemoryName: {{ .Values.mluResourceMem }} + resourceCoreName: {{ .Values.mluResourceCores }} + hygon: + resourceCountName: {{ .Values.dcuResourceName }} + resourceMemoryName: {{ .Values.dcuResourceMem }} + resourceCoreName: {{ .Values.dcuResourceCores }} + metax: + resourceCountName: "metax-tech.com/gpu" + resourceVCountName: {{ .Values.metaxResourceName }} + resourceVMemoryName: {{ .Values.metaxResourceMem }} + resourceVCoreName: {{ .Values.metaxResourceCore }} + sgpuTopologyAware: {{ .Values.metaxsGPUTopologyAware }} + enflame: + resourceNameGCU: "enflame.com/gcu" + resourceNameVGCU: {{ .Values.enflameResourceNameVGCU }} + resourceNameVGCUPercentage: {{ .Values.enflameResourceNameVGCUPercentage }} + mthreads: + resourceCountName: "mthreads.com/vgpu" + resourceMemoryName: "mthreads.com/sgpu-memory" + resourceCoreName: "mthreads.com/sgpu-core" + iluvatars: + - chipName: MR-V100 + commonWord: MR-V100 + resourceCountName: iluvatar.ai/MR-V100-vgpu + resourceMemoryName: iluvatar.ai/MR-V100.vMem + resourceCoreName: iluvatar.ai/MR-V100.vCore + - chipName: MR-V50 + commonWord: MR-V50 + resourceCountName: iluvatar.ai/MR-V50-vgpu + resourceMemoryName: iluvatar.ai/MR-V50.vMem + resourceCoreName: iluvatar.ai/MR-V50.vCore + - chipName: BI-V150 + commonWord: BI-V150 + resourceCountName: iluvatar.ai/BI-V150-vgpu + resourceMemoryName: iluvatar.ai/BI-V150.vMem + resourceCoreName: iluvatar.ai/BI-V150.vCore + - chipName: BI-V100 + commonWord: BI-V100 + resourceCountName: iluvatar.ai/BI-V100-vgpu + resourceMemoryName: iluvatar.ai/BI-V100.vMem + resourceCoreName: iluvatar.ai/BI-V100.vCore + kunlun: + resourceCountName: {{ .Values.kunlunResourceName }} + resourceVCountName: {{ .Values.kunlunResourceVCountName }} + resourceVMemoryName: {{ .Values.kunlunResourceVMemoryName }} + awsneuron: + resourceCountName: "aws.amazon.com/neuron" + resourceCoreName: "aws.amazon.com/neuroncore" + amd: + resourceCountName: "amd.com/gpu" + vnpus: + - chipName: 910A + commonWord: Ascend910A + resourceName: huawei.com/Ascend910A + resourceMemoryName: huawei.com/Ascend910A-memory + memoryAllocatable: 32768 + memoryCapacity: 32768 + aiCore: 30 + templates: + - name: vir02 + memory: 2184 + aiCore: 2 + - name: vir04 + memory: 4369 + aiCore: 4 + - name: vir08 + memory: 8738 + aiCore: 8 + - name: vir16 + memory: 17476 + aiCore: 16 + - chipName: 910B2 + commonWord: Ascend910B2 + resourceName: huawei.com/Ascend910B2 + resourceMemoryName: huawei.com/Ascend910B2-memory + memoryAllocatable: 65536 + memoryCapacity: 65536 + aiCore: 24 + aiCPU: 6 + templates: + - name: vir03_1c_8g + memory: 8192 + aiCore: 3 + aiCPU: 1 + - name: vir06_1c_16g + memory: 16384 + aiCore: 6 + aiCPU: 1 + - name: vir12_3c_32g + memory: 32768 + aiCore: 12 + aiCPU: 3 + - chipName: 910B3 + commonWord: Ascend910B3 + resourceName: huawei.com/Ascend910B3 + resourceMemoryName: huawei.com/Ascend910B3-memory + memoryAllocatable: 65536 + memoryCapacity: 65536 + aiCore: 20 + aiCPU: 7 + templates: + - name: vir05_1c_16g + memory: 16384 + aiCore: 5 + aiCPU: 1 + - name: vir10_3c_32g + memory: 32768 + aiCore: 10 + aiCPU: 3 + - chipName: 910B4-1 + commonWord: Ascend910B4-1 + resourceName: huawei.com/Ascend910B4-1 + resourceMemoryName: huawei.com/Ascend910B4-1-memory + memoryAllocatable: 65536 + memoryCapacity: 65536 + aiCore: 20 + aiCPU: 7 + templates: + # NOTE: Names of vnpu template for 910B4-1 are fixed by Ascend runtime and must not be changed. + # The memory is used for scheduling so the correct values must be set. + # Template vir05_1c_8g actually provides 16GB memory, + - name: vir05_1c_8g + memory: 16384 + aiCore: 5 + aiCPU: 1 + # Template vir10_3c_16g actually provides 32GB memory + - name: vir10_3c_16g + memory: 32768 + aiCore: 10 + aiCPU: 3 + - chipName: 910B4 + commonWord: Ascend910B4 + resourceName: huawei.com/Ascend910B4 + resourceMemoryName: huawei.com/Ascend910B4-memory + memoryAllocatable: 32768 + memoryCapacity: 32768 + aiCore: 20 + aiCPU: 7 + templates: + - name: vir05_1c_8g + memory: 8192 + aiCore: 5 + aiCPU: 1 + - name: vir10_3c_16g + memory: 16384 + aiCore: 10 + aiCPU: 3 + - chipName: 310P3 + commonWord: Ascend310P + resourceName: huawei.com/Ascend310P + resourceMemoryName: huawei.com/Ascend310P-memory + memoryAllocatable: 21527 + memoryCapacity: 24576 + aiCore: 8 + aiCPU: 7 + templates: + - name: vir01 + memory: 3072 + aiCore: 1 + aiCPU: 1 + - name: vir02 + memory: 6144 + aiCore: 2 + aiCPU: 2 + - name: vir04 + memory: 12288 + aiCore: 4 + aiCPU: 4 + {{ end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/mutatingwebhookconfiguration.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/mutatingwebhookconfiguration.yaml new file mode 100644 index 00000000..c763b6aa --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/mutatingwebhookconfiguration.yaml @@ -0,0 +1,28 @@ +{{- if .Values.webhook.config.mutating.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: {{ .Release.Name }}-mutatingwebhookconfiguration + annotations: + cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ .Release.Name }}-dra-webhook-serving-cert +webhooks: + - name: mutate.hami.io + admissionReviewVersions: ["v1"] + clientConfig: + service: + name: {{ .Release.Name }}-dra-webhook + namespace: {{ .Release.Namespace }} + path: /mutate + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - pods + scope: '*' + sideEffects: None + timeoutSeconds: 10 +{{- end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/validatingwebhookconfiguration.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/validatingwebhookconfiguration.yaml new file mode 100644 index 00000000..acdcbb96 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/validatingwebhookconfiguration.yaml @@ -0,0 +1,29 @@ +{{- if .Values.webhook.config.validating.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: {{ .Release.Name }}-validatingwebhookconfiguration + annotations: + cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ .Release.Name }}-dra-webhook-serving-cert +webhooks: + - name: validate.hami.io + admissionReviewVersions: ["v1"] + clientConfig: + service: + name: {{ .Release.Name }}-dra-webhook + namespace: {{ .Release.Namespace }} + path: /validate + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - DELETE + resources: + - pods + scope: '*' + sideEffects: None + timeoutSeconds: 10 + failurePolicy: Ignore +{{- end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrole.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrole.yaml new file mode 100644 index 00000000..28916a36 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrole.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "hami.dra.webhook.fullname" . }} +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] +- apiGroups: ["admissionregistration.k8s.io"] + resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"] + verbs: ["get", "list", "watch"] +- apiGroups: ["resource.k8s.io"] + resources: ["resourceclaims"] + verbs: ["*"] diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrolebinding.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrolebinding.yaml new file mode 100644 index 00000000..daad52f2 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrolebinding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami.dra.webhook.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "hami.dra.webhook.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "hami.dra.webhook.fullname" . }} + namespace: {{ .Release.Namespace }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-deployment.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-deployment.yaml new file mode 100644 index 00000000..8e3cf90c --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-deployment.yaml @@ -0,0 +1,59 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "hami.dra.webhook.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "hami.dra.webhook.fullname" . }} + {{- include "hami-dra-webhook.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "hami-dra-webhook.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "hami-dra-webhook.selectorLabels" . | nindent 8 }} + spec: + {{- include "hami.dra.webhook.imagePullSecrets" . | nindent 6 }} + serviceAccountName: {{ include "hami.dra.webhook.fullname" . }} + containers: + - name: webhook + image: {{ include "hami.dra.webhook.image" . }} + imagePullPolicy: {{ .Values.webhook.image.pullPolicy | default "IfNotPresent" }} + command: + - /bin/webhook + - --v=5 + - --bind-address=0.0.0.0 + - --secure-port=8443 + - --cert-dir=/tls + - --tls-cert-file-name=tls.crt + - --tls-private-key-file-name=tls.key + - --metrics-bind-address=:8080 + - --health-probe-bind-address=:8000 + - --device-config-file=/device-config.yaml + ports: + - containerPort: 8443 + name: webhook + protocol: TCP + - containerPort: 8080 + name: metrics + protocol: TCP + - containerPort: 8000 + name: health + protocol: TCP + volumeMounts: + - name: device-config + mountPath: /device-config.yaml + subPath: device-config.yaml + - name: tls-config + mountPath: /tls + readOnly: true + volumes: + - name: device-config + configMap: + name: {{ include "hami.dra.webhook.fullname" . }}-device-config + - name: tls-config + secret: + secretName: {{ if .Values.certs.certManager.enabled }}{{ .Release.Name }}-dra-webhook-tls{{ else }}{{ .Release.Name }}-tls{{ end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-service.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-service.yaml new file mode 100644 index 00000000..290174ec --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-dra-webhook + namespace: {{ .Release.Namespace }} + labels: + {{- include "hami-dra-webhook.selectorLabels" . | nindent 4 }} +spec: + type: "ClusterIP" + selector: + {{- include "hami-dra-webhook.selectorLabels" . | nindent 4 }} + ports: + - port: 443 + targetPort: 8443 + name: webhook diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-serviceaccount.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-serviceaccount.yaml new file mode 100644 index 00000000..b0dc2fb2 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-serviceaccount.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "hami.dra.webhook.fullname" . }} + namespace: {{ .Release.Namespace }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/values.yaml b/packages/system/hami/charts/hami/charts/hami-dra/values.yaml new file mode 100644 index 00000000..ebbbfed7 --- /dev/null +++ b/packages/system/hami/charts/hami/charts/hami-dra/values.yaml @@ -0,0 +1,166 @@ +## @section Global configuration +## @param global.imageRegistry Global Docker image registry +## @param global.imagePullSecrets Global Docker image pull secrets +global: + ## @param global.imageRegistry Global Docker image registry + imageRegistry: "" + ## E.g. + ## imagePullSecrets: + ## - myRegistryKeySecretName + ## @param global.imagePullSecrets Global Docker image pull secrets + imagePullSecrets: [] + +#Nvidia GPU Parameters +resourceName: "nvidia.com/gpu" +resourceMem: "nvidia.com/gpumem" +resourceMemPercentage: "nvidia.com/gpumem-percentage" +resourceCores: "nvidia.com/gpucores" +resourcePriority: "nvidia.com/priority" + +#MLU Parameters +mluResourceName: "cambricon.com/vmlu" +mluResourceMem: "cambricon.com/mlu.smlu.vmemory" +mluResourceCores: "cambricon.com/mlu.smlu.vcore" + +#Hygon DCU Parameters +dcuResourceName: "hygon.com/dcunum" +dcuResourceMem: "hygon.com/dcumem" +dcuResourceCores: "hygon.com/dcucores" + +#Metax sGPU Parameters +metaxResourceName: "metax-tech.com/sgpu" +metaxResourceCore: "metax-tech.com/vcore" +metaxResourceMem: "metax-tech.com/vmemory" +metaxsGPUTopologyAware: "false" + +#Enflame VGCU Parameters +enflameResourceNameVGCU: "enflame.com/vgcu" +enflameResourceNameVGCUPercentage: "enflame.com/vgcu-percentage" + +#Kunlun XPU Parameters +kunlunResourceName: "kunlunxin.com/xpu" +kunlunResourceVCountName: "kunlunxin.com/vxpu" +kunlunResourceVMemoryName: "kunlunxin.com/vxpu-memory" + +# Webhook deployment configuration +webhook: + image: + registry: ghcr.io + repository: project-hami/hami-dra-webhook + tag: "v0.1.0" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## @param resources controllerManager resource requests and limits + resources: + # If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 128Mi + ## @param nodeSelector controllerManager node labels for pod assignment + args: + webhookName: "" # Default: {{ .Release.Name }}-hami-dra-webhook + secretName: "" # Default: {{ .Release.Name }}-hami-dra-webhook-tls + # Webhook configuration + config: + mutating: + enabled: true + validating: + enabled: true + +# Monitor deployment configuration +monitor: + enabled: true + replicas: 1 + image: + registry: ghcr.io + repository: project-hami/hami-dra-monitor + tag: "v0.1.0" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## @param resources monitor resource requests and limits + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 128Mi + ## Monitor configuration + logLevel: 2 + metricsBindAddress: ":8080" + healthProbeBindAddress: ":8000" + kubeAPIQPS: 40.0 + kubeAPIBurst: 60 + collectInterval: "30s" + ## Monitor service configuration + service: + ## Service type: ClusterIP, NodePort, or LoadBalancer + ## Defaults to ClusterIP + type: ClusterIP + ## NodePort configuration (only used when type is NodePort) + nodePort: + ## Metrics port (NodePort). If not specified, Kubernetes will assign a random port + metrics: "" + +# Certificate configuration +certs: + # Custom certificate (used when mode is "custom") + custom: + crt: "" + key: "" + # Cert-manager configuration (used when mode is "cert-manager") + certManager: + enabled: true + # Private key rotation policy: "Never" or "Always" + # In cert-manager >= v1.18.0, the default changed from "Never" to "Always" + # Setting to "Never" avoids frequent certificate rotation for webhooks + privateKeyRotationPolicy: "Never" + issuer: + clusterIssuerName: "" # Required when type is "clusterIssuer" + +# DRA Drivers configuration +drivers: + nvidia: + enabled: true + # If you are using gpu driver on host, you need to set this to false + containerDriver: true + image: + registry: ghcr.io + repository: projecthami/k8s-dra-driver + tag: "v0.0.1-dev" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] diff --git a/packages/system/hami/charts/hami/templates/NOTES.txt b/packages/system/hami/charts/hami/templates/NOTES.txt new file mode 100644 index 00000000..15bb1218 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/NOTES.txt @@ -0,0 +1,3 @@ +** Please be patient while the chart is being deployed ** +Resource name: {{ .Values.resourceName }} + diff --git a/packages/system/hami/charts/hami/templates/_commons.tpl b/packages/system/hami/charts/hami/templates/_commons.tpl new file mode 100644 index 00000000..78a262c9 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/_commons.tpl @@ -0,0 +1,49 @@ +{{/* +Return the proper image name +{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" $) }} +*/}} +{{- define "common.images.image" -}} +{{- $registryName := .imageRoot.registry -}} +{{- $repositoryName := .imageRoot.repository -}} +{{- $tag := .imageRoot.tag | toString -}} +{{- if .global }} + {{- if .global.imageRegistry }} + {{- $registryName = .global.imageRegistry -}} + {{- end -}} +{{- end -}} +{{- if .tag }} + {{- $tag = .tag | toString -}} +{{- end -}} +{{- if $registryName }} +{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- else -}} +{{- printf "%s:%s" $repositoryName $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead) +{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }} +*/}} +{{- define "common.images.pullSecrets" -}} + {{- $pullSecrets := list }} + + {{- if .global }} + {{- range .global.imagePullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- range .images -}} + {{- range .pullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- if (not (empty $pullSecrets)) }} +imagePullSecrets: + {{- range $pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end -}} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/_helpers.tpl b/packages/system/hami/charts/hami/templates/_helpers.tpl new file mode 100644 index 00000000..ffcc61da --- /dev/null +++ b/packages/system/hami/charts/hami/templates/_helpers.tpl @@ -0,0 +1,163 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "hami-vgpu.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "hami-vgpu.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "hami-vgpu.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +The app name for Scheduler +*/}} +{{- define "hami-vgpu.scheduler" -}} +{{- printf "%s-scheduler" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +The app name for DevicePlugin +*/}} +{{- define "hami-vgpu.device-plugin" -}} +{{- printf "%s-device-plugin" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* + The app name for MockDevicePlugin + */}} +{{- define "hami-vgpu.mock-device-plugin" -}} +{{- printf "%s-mock-device-plugin" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +The tls secret name for Scheduler +*/}} +{{- define "hami-vgpu.scheduler.tls" -}} +{{- printf "%s-scheduler-tls" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +The webhook name +*/}} +{{- define "hami-vgpu.scheduler.webhook" -}} +{{- printf "%s-webhook" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "hami-vgpu.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "hami-vgpu.labels" -}} +helm.sh/chart: {{ include "hami-vgpu.chart" . }} +{{ include "hami-vgpu.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "hami-vgpu.selectorLabels" -}} +app.kubernetes.io/name: {{ include "hami-vgpu.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + + +{{/* + Resolve the tag for kubeScheduler. +*/}} +{{- define "resolvedKubeSchedulerTag" -}} +{{- if .Values.scheduler.kubeScheduler.image.tag }} +{{- .Values.scheduler.kubeScheduler.image.tag | trim -}} +{{- else }} +{{- include "strippedKubeVersion" . | trim -}} +{{- end }} +{{- end }} + +{{/* + Return the stripped Kubernetes version string by removing extra parts after semantic version number. + v1.31.1+k3s1 -> v1.31.1 + v1.30.8-eks-2d5f260 -> v1.30.8 + v1.31.1 -> v1.31.1 +*/}} +{{- define "strippedKubeVersion" -}} +{{ regexReplaceAll "^(v[0-9]+\\.[0-9]+\\.[0-9]+)(.*)$" .Capabilities.KubeVersion.Version "$1" }} +{{- end -}} + +{{- define "hami.scheduler.kubeScheduler.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.kubeScheduler.image "global" .Values.global "tag" (include "resolvedKubeSchedulerTag" .)) }} +{{- end -}} + +{{- define "hami.scheduler.extender.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.extender.image "global" .Values.global "tag" .Values.global.imageTag) }} +{{- end -}} + +{{- define "hami.devicePlugin.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.devicePlugin.image "global" .Values.global "tag" .Values.global.imageTag) }} +{{- end -}} + +{{- define "hami.mockDevicePlugin.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.mockDevicePlugin.image "global" .Values.global "tag" .Values.mockDevicePlugin.tag) }} +{{- end -}} + +{{- define "hami.devicePlugin.monitor.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.devicePlugin.monitor.image "global" .Values.global "tag" .Values.global.imageTag) }} +{{- end -}} + +{{- define "hami.scheduler.patch.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.patch.image "global" .Values.global) }} +{{- end -}} + +{{- define "hami.scheduler.patch.new.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.patch.imageNew "global" .Values.global) }} +{{- end -}} + +{{- define "hami.scheduler.extender.imagePullSecrets" -}} +{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.extender.image) "global" .Values.global) }} +{{- end -}} + +{{- define "hami.devicePlugin.imagePullSecrets" -}} +{{ include "common.images.pullSecrets" (dict "images" (list .Values.devicePlugin.image) "global" .Values.global) }} +{{- end -}} + +{{- define "hami.scheduler.patch.imagePullSecrets" -}} +{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.patch.image) "global" .Values.global) }} +{{- end -}} + +{{- define "hami.scheduler.patch.new.imagePullSecrets" -}} +{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.patch.imageNew) "global" .Values.global) }} +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml b/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml new file mode 100644 index 00000000..b36347f2 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml @@ -0,0 +1,13 @@ +{{- if and .Values.devicePlugin.enabled (not .Values.devicePlugin.nodeConfiguration.externalConfigName) (not .Values.dra.enabled) -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "hami-vgpu.device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-device-plugin + {{- include "hami-vgpu.labels" . | nindent 4 }} +data: + config.json: | +{{- .Values.devicePlugin.nodeConfiguration.config | nindent 4 }} +{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/device-plugin/daemonsetmock.yaml b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetmock.yaml new file mode 100644 index 00000000..a0dc4ff3 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetmock.yaml @@ -0,0 +1,55 @@ +{{- if .Values.mockDevicePlugin.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "hami-vgpu.mock-device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +spec: + selector: + matchLabels: + app.kubernetes.io/component: hami-mock-device-plugin + {{- include "hami-vgpu.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" + labels: + app.kubernetes.io/component: hami-mock-device-plugin + {{- include "hami-vgpu.selectorLabels" . | nindent 8 }} + spec: + serviceAccountName: {{ include "hami-vgpu.mock-device-plugin" . }} + tolerations: + - key: CriticalAddonsOnly + operator: Exists + containers: + - image: {{ include "hami.mockDevicePlugin.image" . }} + imagePullPolicy: {{ .Values.mockDevicePlugin.image.pullPolicy }} + name: hami-mock-dp-cntr + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + command: + - ./k8s-device-plugin + - -v=5 + - --device-config-file=/device-config.yaml + volumeMounts: + - name: dp + mountPath: /var/lib/kubelet/device-plugins + - name: sys + mountPath: /sys + - name: device-config + mountPath: /device-config.yaml + subPath: device-config.yaml + volumes: + - name: dp + hostPath: + path: /var/lib/kubelet/device-plugins + - name: sys + hostPath: + path: /sys + - name: device-config + configMap: + name: {{ include "hami-vgpu.scheduler" . }}-device +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml new file mode 100644 index 00000000..577dc0f9 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml @@ -0,0 +1,262 @@ +{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "hami-vgpu.device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-device-plugin + {{- include "hami-vgpu.labels" . | nindent 4 }} + {{- with .Values.global.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.global.annotations }} + annotations: {{ toYaml .Values.global.annotations | nindent 4}} + {{- end }} +spec: + updateStrategy: + {{- with .Values.devicePlugin.updateStrategy }} + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/component: hami-device-plugin + {{- include "hami-vgpu.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app.kubernetes.io/component: hami-device-plugin + hami.io/webhook: ignore + {{- include "hami-vgpu.selectorLabels" . | nindent 8 }} + annotations: + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + checksum/hami-scheduler-newversion-config: {{ include (print $.Template.BasePath "/scheduler/configmapnew.yaml") . | sha256sum }} + {{- else }} + checksum/hami-scheduler-config: {{ include (print $.Template.BasePath "/scheduler/configmap.yaml") . | sha256sum }} + {{- end }} + checksum/hami-scheduler-device-config: {{ include (print $.Template.BasePath "/scheduler/device-configmap.yaml") . | sha256sum }} + {{- if .Values.devicePlugin.podAnnotations }} + {{- toYaml .Values.devicePlugin.podAnnotations | nindent 8 }} + {{- end }} + spec: + {{- if .Values.devicePlugin.runtimeClassName }} + runtimeClassName: {{ .Values.devicePlugin.runtimeClassName }} + {{- end }} + serviceAccountName: {{ include "hami-vgpu.device-plugin" . }} + priorityClassName: system-node-critical + hostPID: true + hostNetwork: true + {{- include "hami.devicePlugin.imagePullSecrets" . | nindent 6 }} + {{- if .Values.devicePlugin.gpuOperatorToolkitReady.enabled }} + initContainers: + - name: toolkit-validation + image: {{ include "hami.devicePlugin.image" . }} + imagePullPolicy: {{ .Values.devicePlugin.image.pullPolicy }} + securityContext: + privileged: true + runAsUser: 0 + command: ["sh", "-c"] + args: + - | + echo "Waiting for NVIDIA Toolkit to be ready..." + until [ -f {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }}/toolkit-ready ]; do + echo "Waiting for {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }}/toolkit-ready..." + sleep 5 + done + echo "NVIDIA Toolkit is ready!" + volumeMounts: + - name: nvidia-validations + mountPath: {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath | quote }} + mountPropagation: HostToContainer + readOnly: true + {{- end }} + containers: + - name: device-plugin + image: {{ include "hami.devicePlugin.image" . }} + imagePullPolicy: {{ .Values.devicePlugin.image.pullPolicy }} + lifecycle: + postStart: + exec: + command: ["/bin/sh","-c", {{ printf "/k8s-vgpu/bin/vgpu-init.sh %s/vgpu/" .Values.global.gpuHookPath | quote }}] + command: + - nvidia-device-plugin + - --config-file=/device-config.yaml + - --mig-strategy={{ .Values.devicePlugin.migStrategy }} + - --disable-core-limit={{ .Values.devicePlugin.disablecorelimit }} + {{- range .Values.devicePlugin.extraArgs }} + - {{ . }} + {{- end }} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: NVIDIA_MIG_MONITOR_DEVICES + value: all + - name: DEVICE_LIST_STRATEGY + value: {{ .Values.devicePlugin.deviceListStrategy }} + - name: HOOK_PATH + value: {{ .Values.global.gpuHookPath }} + {{- if typeIs "bool" .Values.devicePlugin.passDeviceSpecsEnabled }} + - name: PASS_DEVICE_SPECS + value: {{ .Values.devicePlugin.passDeviceSpecsEnabled | quote }} + {{- end }} + {{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }} + - name: NVIDIA_DRIVER_ROOT + value: {{ .Values.devicePlugin.nvidiaDriverRoot }} + {{- end }} + {{- if typeIs "string" .Values.devicePlugin.nvidiaHookPath }} + - name: NVIDIA_CDI_HOOK_PATH + value: {{ .Values.devicePlugin.nvidiaHookPath }} + {{- end }} + {{- if typeIs "bool" .Values.devicePlugin.gdrcopyEnabled }} + - name: GDRCOPY_ENABLED + value: {{ .Values.devicePlugin.gdrcopyEnabled | quote }} + {{- end }} + {{- if typeIs "bool" .Values.devicePlugin.gdsEnabled }} + - name: GDS_ENABLED + value: {{ .Values.devicePlugin.gdsEnabled | quote }} + {{- end }} + {{- if typeIs "bool" .Values.devicePlugin.mofedEnabled }} + - name: MOFED_ENABLED + value: {{ .Values.devicePlugin.mofedEnabled | quote }} + {{- end }} + {{- if eq (.Values.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy | default "spread") "topology-aware" }} + - name: ENABLE_TOPOLOGY_SCORE + value: "true" + {{- end }} + {{- with .Values.devicePlugin.extraEnvs }} + {{- . | toYaml | nindent 12 }} + {{- end }} + securityContext: + privileged: true + allowPrivilegeEscalation: true + capabilities: + drop: ["ALL"] + add: ["SYS_ADMIN"] + resources: + {{- toYaml .Values.devicePlugin.resources | nindent 12 }} + volumeMounts: + - name: device-plugin + mountPath: /var/lib/kubelet/device-plugins + - name: lib + mountPath: {{ printf "%s%s" .Values.global.gpuHookPath "/vgpu" }} + - name: usrbin + mountPath: /usrbin + - name: deviceconfig + mountPath: /config + - name: hosttmp + mountPath: /tmp + - name: device-config + mountPath: /device-config.yaml + subPath: device-config.yaml + - name: cdi-root + mountPath: /var/run/cdi + {{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }} + # We always mount the driver root at /driver-root in the container. + # This is required for CDI detection to work correctly. + - name: driver-root + mountPath: /driver-root + readOnly: true + {{- end }} + - name: vgpu-monitor + image: {{ include "hami.devicePlugin.monitor.image" . }} + imagePullPolicy: {{ .Values.devicePlugin.monitor.image.pullPolicy }} + command: + - "vGPUmonitor" + {{- range .Values.devicePlugin.monitor.extraArgs }} + - {{ . }} + {{- end }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + add: ["SYS_ADMIN"] + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: NVIDIA_VISIBLE_DEVICES + value: "all" + - name: NVIDIA_MIG_MONITOR_DEVICES + value: "all" + - name: HOOK_PATH + value: "{{ .Values.global.gpuHookPath }}/vgpu" + - name: HAMI_RESYNC_INTERVAL + value: {{ .Values.devicePlugin.monitor.resyncInterval | default "5m" | quote }} + {{- with .Values.devicePlugin.monitor.extraEnvs }} + {{- . | toYaml | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.devicePlugin.monitor.resources | nindent 12 }} + volumeMounts: + - name: ctrs + mountPath: {{ .Values.devicePlugin.monitor.ctrPath }} + - name: dockers + mountPath: /run/docker + - name: containerds + mountPath: /run/containerd + - name: sysinfo + mountPath: /sysinfo + - name: hostvar + mountPath: /hostvar + - name: hosttmp + mountPath: /tmp + volumes: + - name: ctrs + hostPath: + path: {{ .Values.devicePlugin.monitor.ctrPath }} + - name: hosttmp + hostPath: + path: /tmp + - name: dockers + hostPath: + path: /run/docker + - name: containerds + hostPath: + path: /run/containerd + - name: device-plugin + hostPath: + path: {{ .Values.devicePlugin.pluginPath }} + - name: lib + hostPath: + path: {{ .Values.devicePlugin.libPath }} + {{- if .Values.devicePlugin.gpuOperatorToolkitReady.enabled }} + - name: nvidia-validations + hostPath: + path: {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }} + type: DirectoryOrCreate + {{- end }} + {{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }} + - name: driver-root + hostPath: + path: {{ .Values.devicePlugin.nvidiaDriverRoot }} + type: Directory + {{- end }} + - name: cdi-root + hostPath: + path: /var/run/cdi + type: DirectoryOrCreate + - name: usrbin + hostPath: + path: /usr/bin + - name: sysinfo + hostPath: + path: /sys + - name: hostvar + hostPath: + path: /var + - name: deviceconfig + configMap: + name: {{ .Values.devicePlugin.nodeConfiguration.externalConfigName | default (include "hami-vgpu.device-plugin" .) }} + - name: device-config + configMap: + name: {{ include "hami-vgpu.scheduler" . }}-device + {{- if .Values.devicePlugin.nvidiaNodeSelector }} + nodeSelector: {{ toYaml .Values.devicePlugin.nvidiaNodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.devicePlugin.tolerations }} + tolerations: {{ toYaml .Values.devicePlugin.tolerations | nindent 8 }} + {{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml new file mode 100644 index 00000000..c51ff6c6 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "hami-vgpu.device-plugin" . }}-monitor +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - create + - watch + - list + - update + - patch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - update + - list + - patch +{{- end -}} + + diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml new file mode 100644 index 00000000..93a118dd --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.device-plugin" . }} + labels: + app.kubernetes.io/component: "hami-device-plugin" + {{- include "hami-vgpu.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "hami-vgpu.device-plugin" . }}-monitor +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml new file mode 100644 index 00000000..24daca9b --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "hami-vgpu.device-plugin" . }}-monitor + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-device-plugin + {{- include "hami-vgpu.labels" . | nindent 4 }} + {{- if .Values.devicePlugin.service.labels }} # Use devicePlugin instead of scheduler + {{ toYaml .Values.devicePlugin.service.labels | indent 4 }} + {{- end }} + {{- if .Values.devicePlugin.service.annotations }} # Use devicePlugin instead of scheduler + annotations: {{ toYaml .Values.devicePlugin.service.annotations | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.devicePlugin.service.type | default "NodePort" }} # Default type is NodePort + ports: + - name: monitorport + port: {{ .Values.devicePlugin.service.httpPort | default 31992 }} # Default HTTP port is 31992 + targetPort: 9394 + {{- if eq (.Values.devicePlugin.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort + nodePort: {{ .Values.devicePlugin.service.httpPort | default 31992 }} + {{- end }} + protocol: TCP + selector: + app.kubernetes.io/component: hami-device-plugin + {{- include "hami-vgpu.selectorLabels" . | nindent 4 }} +{{- end -}} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml new file mode 100644 index 00000000..b10d2cb9 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml @@ -0,0 +1,10 @@ +{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "hami-vgpu.device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: "hami-device-plugin" + {{- include "hami-vgpu.labels" . | nindent 4 }} +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml b/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml new file mode 100644 index 00000000..cd3d14cc --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml @@ -0,0 +1,11 @@ +{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}} +{{- if and .Values.devicePlugin.createRuntimeClass .Values.devicePlugin.runtimeClassName }} +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: {{ .Values.devicePlugin.runtimeClassName }} + annotations: + helm.sh/hook: pre-install,pre-upgrade +handler: nvidia +{{- end }} +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml b/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml new file mode 100644 index 00000000..b6edb605 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml @@ -0,0 +1,31 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} +{{- if .Values.scheduler.certManager.enabled }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-serving-cert + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} +spec: + dnsNames: + - {{ include "hami-vgpu.scheduler" . }}.{{ include "hami-vgpu.namespace" . }}.svc + - {{ include "hami-vgpu.scheduler" . }}.{{ include "hami-vgpu.namespace" . }}.svc.cluster.local + issuerRef: + kind: Issuer + name: {{ include "hami-vgpu.scheduler" . }}-selfsigned-issuer + secretName: {{ include "hami-vgpu.scheduler.tls" . }} +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-selfsigned-issuer + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} +spec: + selfSigned: {} +{{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml b/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml new file mode 100644 index 00000000..9c510c00 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml @@ -0,0 +1,36 @@ +{{- if not .Values.dra.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["pods", "configmaps"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: [""] + resources: ["pods/binding"] + verbs: ["create"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "patch", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "get", "list"] + - apiGroups: [""] + resources: ["resourcequotas"] + verbs: ["get", "list", "watch"] +--- +{{- if .Values.mockDevicePlugin.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "hami-vgpu.mock-device-plugin" . }} +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "update", "list", "patch"] +{{- end -}} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml new file mode 100644 index 00000000..69d75986 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml @@ -0,0 +1,49 @@ +{{- if not .Values.dra.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-kube + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-scheduler +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-volume + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:volume-scheduler +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "hami-vgpu.scheduler" . }} +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml b/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml new file mode 100644 index 00000000..75889146 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml @@ -0,0 +1,142 @@ +{{- if and .Values.scheduler.kubeScheduler.enabled (not .Values.dra.enabled) -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} +data: + config.json: | + { + "kind": "Policy", + "apiVersion": "v1", + "extenders": [ + { + {{- if .Values.scheduler.admissionWebhook.enabled }} + "urlPrefix": "https://127.0.0.1:443", + "enableHttps": true, + "tlsConfig": { + "insecure": true + }, + {{- else }} + "urlPrefix": "http://127.0.0.1:80", + "enableHttps": false, + {{- end }} + "filterVerb": "filter", + "bindVerb": "bind", + "weight": 1, + "nodeCacheCapable": true, + "httpTimeout": 30000000000, + "managedResources": [ + {{- range .Values.devices.amd.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- if .Values.devices.ascend.enabled }} + {{- range .Values.devices.ascend.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- end }} + {{- if .Values.devices.mthreads.enabled }} + {{- range .Values.devices.mthreads.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- end }} + {{- if .Values.devices.enflame.enabled }} + {{- range .Values.devices.enflame.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- end }} + {{- if .Values.devices.kunlun.enabled }} + {{- range .Values.devices.kunlun.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- end }} + {{- range .Values.devices.awsneuron.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- if .Values.devices.iluvatar.enabled }} + {{- range .Values.devices.iluvatar.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- end }} + { + "name": "{{ .Values.resourceName }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.resourceMem }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.resourceCores }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.resourceMemPercentage }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.resourcePriority }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.mluResourceName }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.dcuResourceName }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.dcuResourceMem }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.dcuResourceCores }}", + "ignoredByScheduler": true + }, + { + "name": "metax-tech.com/gpu", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.metaxResourceName }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.metaxResourceCore }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.metaxResourceMem }}", + "ignoredByScheduler": true + } + ], + "ignoreable": false + } + ] + } +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml b/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml new file mode 100644 index 00000000..e2a91d8b --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml @@ -0,0 +1,102 @@ +{{- if and .Values.scheduler.kubeScheduler.enabled (not .Values.dra.enabled) -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-newversion + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} +data: + config.yaml: | + {{- if gt (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 25}} + apiVersion: kubescheduler.config.k8s.io/v1 + {{- else }} + apiVersion: kubescheduler.config.k8s.io/v1beta2 + {{- end }} + kind: KubeSchedulerConfiguration + leaderElection: + leaderElect: false + profiles: + - schedulerName: {{ .Values.schedulerName }} + extenders: + {{- if .Values.scheduler.admissionWebhook.enabled }} + - urlPrefix: "https://127.0.0.1:443" + enableHTTPS: true + tlsConfig: + insecure: true + {{- else }} + - urlPrefix: "http://127.0.0.1:80" + enableHTTPS: false + {{- end }} + filterVerb: filter + bindVerb: bind + nodeCacheCapable: true + weight: 1 + httpTimeout: 30s + managedResources: + - name: {{ .Values.resourceName }} + ignoredByScheduler: true + - name: {{ .Values.resourceMem }} + ignoredByScheduler: true + - name: {{ .Values.resourceCores }} + ignoredByScheduler: true + - name: {{ .Values.resourceMemPercentage }} + ignoredByScheduler: true + - name: {{ .Values.resourcePriority }} + ignoredByScheduler: true + - name: {{ .Values.mluResourceName }} + ignoredByScheduler: true + - name: {{ .Values.dcuResourceName }} + ignoredByScheduler: true + - name: {{ .Values.dcuResourceMem }} + ignoredByScheduler: true + - name: {{ .Values.dcuResourceCores }} + ignoredByScheduler: true + - name: "metax-tech.com/gpu" + ignoredByScheduler: true + - name: {{ .Values.metaxResourceName }} + ignoredByScheduler: true + - name: {{ .Values.metaxResourceCore }} + ignoredByScheduler: true + - name: {{ .Values.metaxResourceMem }} + ignoredByScheduler: true + {{- if .Values.devices.ascend.enabled }} + {{- range .Values.devices.ascend.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- end }} + {{- if .Values.devices.mthreads.enabled }} + {{- range .Values.devices.mthreads.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- end }} + {{- if .Values.devices.enflame.enabled }} + {{- range .Values.devices.enflame.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- end }} + {{- if .Values.devices.kunlun.enabled }} + {{- range .Values.devices.kunlun.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- end }} + {{- range .Values.devices.awsneuron.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- if .Values.devices.iluvatar.enabled }} + {{- range .Values.devices.iluvatar.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- end }} + {{- range .Values.devices.amd.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml b/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml new file mode 100644 index 00000000..6364cae3 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml @@ -0,0 +1,227 @@ +{{- if not .Values.dra.enabled -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} + {{- with .Values.global.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.global.annotations }} + annotations: {{ toYaml .Values.global.annotations | nindent 4}} + {{- end }} +spec: + {{- if .Values.scheduler.leaderElect }} + replicas: {{ .Values.scheduler.replicas }} + {{- else }} + replicas: 1 + {{- end }} + selector: + matchLabels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.selectorLabels" . | nindent 8 }} + hami.io/webhook: ignore + annotations: + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + checksum/hami-scheduler-newversion-config: {{ include (print $.Template.BasePath "/scheduler/configmapnew.yaml") . | sha256sum }} + {{- else }} + checksum/hami-scheduler-config: {{ include (print $.Template.BasePath "/scheduler/configmap.yaml") . | sha256sum }} + {{- end }} + checksum/hami-scheduler-device-config: {{ include (print $.Template.BasePath "/scheduler/device-configmap.yaml") . | sha256sum }} + {{- if .Values.scheduler.podAnnotations }} + {{- toYaml .Values.scheduler.podAnnotations | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "hami-vgpu.scheduler" . }} + priorityClassName: system-node-critical + {{- include "hami.scheduler.extender.imagePullSecrets" . | nindent 6 }} + containers: + {{- if .Values.scheduler.kubeScheduler.enabled }} + - name: kube-scheduler + image: {{ include "hami.scheduler.kubeScheduler.image" . }} + imagePullPolicy: {{ .Values.scheduler.kubeScheduler.image.pullPolicy }} + command: + - kube-scheduler + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + {{- range .Values.scheduler.kubeScheduler.extraNewArgs }} + - {{ . }} + {{- end }} + {{- else }} + - --scheduler-name={{ .Values.schedulerName }} + {{- range .Values.scheduler.kubeScheduler.extraArgs }} + - {{ . }} + {{- end }} + {{- end }} + - --leader-elect={{ .Values.scheduler.leaderElect }} + - --leader-elect-resource-name={{ .Values.schedulerName }} + - --leader-elect-resource-namespace={{ include "hami-vgpu.namespace" . }} + resources: + {{- toYaml .Values.scheduler.kubeScheduler.resources | nindent 12 }} + volumeMounts: + - name: scheduler-config + mountPath: /config + {{- end }} + {{- if .Values.scheduler.livenessProbe }} + livenessProbe: + failureThreshold: 8 + httpGet: + path: /healthz + port: 10259 + scheme: HTTPS + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 15 + {{- end }} + - name: vgpu-scheduler-extender + image: {{ include "hami.scheduler.extender.image" . }} + imagePullPolicy: {{ .Values.scheduler.extender.image.pullPolicy }} + env: + {{- if .Values.scheduler.nodeLockExpire }} + - name: HAMI_NODELOCK_EXPIRE + value: "{{ .Values.scheduler.nodeLockExpire }}" + {{- end }} + {{- if .Values.global.managedNodeSelectorEnable }} + {{- range $key, $value := .Values.global.managedNodeSelector }} + - name: NODE_SELECTOR_{{ $key | upper | replace "-" "_" }} + value: "{{ $value }}" + {{- end }} + {{- end }} + command: + - scheduler + {{- if .Values.scheduler.admissionWebhook.enabled }} + - --http_bind=0.0.0.0:443 + - --cert_file=/tls/tls.crt + - --key_file=/tls/tls.key + {{- else }} + - --http_bind=0.0.0.0:80 + {{- end }} + - --scheduler-name={{ .Values.schedulerName }} + - --metrics-bind-address={{ .Values.scheduler.metricsBindAddress }} + - --node-scheduler-policy={{ .Values.scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy }} + - --gpu-scheduler-policy={{ .Values.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy }} + - --force-overwrite-default-scheduler={{ .Values.scheduler.forceOverwriteDefaultScheduler}} + - --device-config-file=/device-config.yaml + - --leader-elect={{ .Values.scheduler.leaderElect }} + - --leader-elect-resource-name={{ .Values.schedulerName }} + - --leader-elect-resource-namespace={{ include "hami-vgpu.namespace" . }} + {{- if .Values.devices.ascend.enabled }} + - --enable-ascend=true + {{- end }} + {{- if .Values.devices.iluvatar.enabled }} + - --enable-iluvatar=true + {{- end }} + {{- if .Values.scheduler.nodeLabelSelector }} + - --node-label-selector={{- $first := true -}} + {{- range $key, $value := .Values.scheduler.nodeLabelSelector -}} + {{- if not $first }},{{ end -}} + {{- $key }}={{ $value -}} + {{- $first = false -}} + {{- end -}} + {{- end }} + {{- range .Values.scheduler.extender.extraArgs }} + - {{ . }} + {{- end }} + ports: + {{- if .Values.scheduler.admissionWebhook.enabled }} + - name: https + containerPort: 443 + protocol: TCP + {{- else }} + - name: http + containerPort: 80 + protocol: TCP + {{- end }} + - name: metrics + containerPort: {{ last (splitList ":" .Values.scheduler.metricsBindAddress) | int }} + protocol: TCP + resources: + {{- toYaml .Values.scheduler.extender.resources | nindent 12 }} + volumeMounts: + - name: device-config + mountPath: /device-config.yaml + subPath: device-config.yaml + {{- if .Values.scheduler.admissionWebhook.enabled }} + - name: tls-config + mountPath: /tls + {{- end }} + {{- if .Values.scheduler.livenessProbe }} + livenessProbe: + httpGet: + path: /healthz + {{- if .Values.scheduler.admissionWebhook.enabled }} + port: https + scheme: HTTPS + {{- else }} + port: http + scheme: HTTP + {{- end }} + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 3 + timeoutSeconds: 5 + {{- end }} + {{- if .Values.scheduler.leaderElect }} + readinessProbe: + httpGet: + path: /readyz + {{- if .Values.scheduler.admissionWebhook.enabled }} + port: https + scheme: HTTPS + {{- else }} + port: http + scheme: HTTP + {{- end }} + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 3 + timeoutSeconds: 5 + {{- end }} + {{- if .Values.scheduler.leaderElect }} + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app.kubernetes.io/component + operator: In + values: + - hami-scheduler + topologyKey: "kubernetes.io/hostname" + {{- end }} + volumes: + {{- if .Values.scheduler.admissionWebhook.enabled }} + - name: tls-config + secret: + secretName: {{ template "hami-vgpu.scheduler.tls" . }} + {{- end }} + {{- if .Values.scheduler.kubeScheduler.enabled }} + - name: scheduler-config + configMap: + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + name: {{ template "hami-vgpu.scheduler" . }}-newversion + {{- else }} + name: {{ template "hami-vgpu.scheduler" . }} + {{- end }} + {{- end }} + - name: device-config + configMap: + name: {{ include "hami-vgpu.scheduler" . }}-device + {{- if .Values.scheduler.nodeSelector }} + nodeSelector: {{ toYaml .Values.scheduler.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.scheduler.tolerations }} + tolerations: {{ toYaml .Values.scheduler.tolerations | nindent 8 }} + {{- end }} + {{- if .Values.scheduler.nodeName }} + nodeName: {{ .Values.scheduler.nodeName }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml b/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml new file mode 100644 index 00000000..ccb945ac --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml @@ -0,0 +1,410 @@ +{{- if not .Values.dra.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-device + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} +data: + device-config.yaml: |- + {{- if .Files.Glob "files/device-config.yaml" }} + {{- .Files.Get "files/device-config.yaml" | nindent 4}} + {{- else }} + nvidia: + resourceCountName: {{ .Values.resourceName }} + resourceMemoryName: {{ .Values.resourceMem }} + resourceMemoryPercentageName: {{ .Values.resourceMemPercentage }} + resourceCoreName: {{ .Values.resourceCores }} + resourcePriorityName: {{ .Values.resourcePriority }} + overwriteEnv: false + defaultMemory: 0 + defaultCores: 0 + defaultGPUNum: 1 + memoryFactor: 1 + deviceSplitCount: {{ .Values.devicePlugin.deviceSplitCount }} + deviceMemoryScaling: {{ .Values.devicePlugin.deviceMemoryScaling }} + deviceCoreScaling: {{ .Values.devicePlugin.deviceCoreScaling }} + gpuCorePolicy: {{ .Values.devices.nvidia.gpuCorePolicy }} + libCudaLogLevel: {{ .Values.devices.nvidia.libCudaLogLevel }} + runtimeClassName: "{{ .Values.devicePlugin.runtimeClassName }}" + knownMigGeometries: + - models: [ "A30" ] + allowedGeometries: + - + - name: 1g.6gb + core: 25 + memory: 6144 + count: 4 + - + - name: 2g.12gb + core: 50 + memory: 12288 + count: 2 + - + - name: 4g.24gb + core: 100 + memory: 24576 + count: 1 + - models: [ "A100-SXM4-40GB", "A100-40GB-PCIe", "A100-PCIE-40GB"] + allowedGeometries: + - + - name: 1g.5gb + core: 14 + memory: 5120 + count: 7 + - + - name: 1g.5gb + core: 14 + memory: 5120 + count: 1 + - name: 2g.10gb + core: 28 + memory: 10240 + count: 3 + - + - name: 3g.20gb + core: 42 + memory: 20480 + count: 2 + - + - name: 7g.40gb + core: 100 + memory: 40960 + count: 1 + - models: [ "A100-SXM4-80GB", "A100-80GB-PCIe", "A100-PCIE-80GB"] + allowedGeometries: + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 7 + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 1 + - name: 2g.20gb + core: 28 + memory: 20480 + count: 3 + - + - name: 3g.40gb + core: 42 + memory: 40960 + count: 2 + - + - name: 7g.79gb + core: 100 + memory: 80896 + count: 1 + - models: [ "H100-PCIE-80GB", "H100-SXM5-80GB"] + allowedGeometries: + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 7 + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 1 + - name: 2g.20gb + core: 28 + memory: 20480 + count: 3 + - + - name: 3g.40gb + core: 42 + memory: 40960 + count: 2 + - + - name: 7g.80gb + core: 100 + memory: 81920 + count: 1 + - models: [ "H100-PCIE-94GB", "H100-SXM5-94GB"] + allowedGeometries: + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 7 + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 1 + - name: 2g.24gb + core: 28 + memory: 24576 + count: 3 + - + - name: 3g.47gb + core: 42 + memory: 48128 + count: 2 + - + - name: 7g.94gb + core: 100 + memory: 96256 + count: 1 + - models: [ "H20", "H100 on GH200"] + allowedGeometries: + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 7 + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 1 + - name: 2g.24gb + core: 28 + memory: 24576 + count: 3 + - + - name: 3g.48gb + core: 42 + memory: 49152 + count: 2 + - + - name: 7g.96gb + core: 100 + memory: 98304 + count: 1 + - models: [ "H200 NVL", "H200-SXM5"] + allowedGeometries: + - + - name: 1g.18gb + core: 14 + memory: 18432 + count: 7 + - + - name: 1g.18gb + core: 14 + memory: 18432 + count: 1 + - name: 2g.35gb + core: 28 + memory: 35840 + count: 3 + - + - name: 3g.71gb + core: 42 + memory: 72704 + count: 2 + - + - name: 7g.141gb + core: 100 + memory: 144384 + count: 1 + - models: [ "B200" ] + allowedGeometries: + - + - name: 1g.23gb + core: 14 + memory: 23552 + count: 7 + - + - name: 1g.23gb + core: 14 + memory: 23552 + count: 1 + - name: 2g.45gb + core: 28 + memory: 46080 + count: 3 + - + - name: 3g.90gb + core: 42 + memory: 92160 + count: 2 + - + - name: 7g.180gb + core: 100 + memory: 184320 + count: 1 + cambricon: + resourceCountName: {{ .Values.mluResourceName }} + resourceMemoryName: {{ .Values.mluResourceMem }} + resourceCoreName: {{ .Values.mluResourceCores }} + hygon: + resourceCountName: {{ .Values.dcuResourceName }} + resourceMemoryName: {{ .Values.dcuResourceMem }} + resourceCoreName: {{ .Values.dcuResourceCores }} + memoryFactor: 1 + metax: + resourceCountName: "metax-tech.com/gpu" + resourceVCountName: {{ .Values.metaxResourceName }} + resourceVMemoryName: {{ .Values.metaxResourceMem }} + resourceVCoreName: {{ .Values.metaxResourceCore }} + sgpuTopologyAware: {{ .Values.metaxsGPUTopologyAware }} + enflame: + resourceNameGCU: "enflame.com/gcu" + resourceNameVGCU: {{ .Values.enflameResourceNameVGCU }} + resourceNameVGCUPercentage: {{ .Values.enflameResourceNameVGCUPercentage }} + mthreads: + resourceCountName: "mthreads.com/vgpu" + resourceMemoryName: "mthreads.com/sgpu-memory" + resourceCoreName: "mthreads.com/sgpu-core" + iluvatars: + - chipName: MR-V100 + commonWord: MR-V100 + resourceCountName: iluvatar.ai/MR-V100-vgpu + resourceMemoryName: iluvatar.ai/MR-V100.vMem + resourceCoreName: iluvatar.ai/MR-V100.vCore + - chipName: MR-V50 + commonWord: MR-V50 + resourceCountName: iluvatar.ai/MR-V50-vgpu + resourceMemoryName: iluvatar.ai/MR-V50.vMem + resourceCoreName: iluvatar.ai/MR-V50.vCore + - chipName: BI-V150 + commonWord: BI-V150 + resourceCountName: iluvatar.ai/BI-V150-vgpu + resourceMemoryName: iluvatar.ai/BI-V150.vMem + resourceCoreName: iluvatar.ai/BI-V150.vCore + - chipName: BI-V100 + commonWord: BI-V100 + resourceCountName: iluvatar.ai/BI-V100-vgpu + resourceMemoryName: iluvatar.ai/BI-V100.vMem + resourceCoreName: iluvatar.ai/BI-V100.vCore + kunlun: + resourceCountName: {{ .Values.kunlunResourceName }} + resourceVCountName: {{ .Values.kunlunResourceVCountName }} + resourceVMemoryName: {{ .Values.kunlunResourceVMemoryName }} + awsneuron: + resourceCountName: "aws.amazon.com/neuron" + resourceCoreName: "aws.amazon.com/neuroncore" + amd: + resourceCountName: "amd.com/gpu" + vnpus: + - chipName: 910A + commonWord: Ascend910A + resourceName: huawei.com/Ascend910A + resourceMemoryName: huawei.com/Ascend910A-memory + memoryAllocatable: 32768 + memoryCapacity: 32768 + memoryFactor: 1 + aiCore: 30 + templates: + - name: vir02 + memory: 2184 + aiCore: 2 + - name: vir04 + memory: 4369 + aiCore: 4 + - name: vir08 + memory: 8738 + aiCore: 8 + - name: vir16 + memory: 17476 + aiCore: 16 + - chipName: 910B2 + commonWord: Ascend910B2 + resourceName: huawei.com/Ascend910B2 + resourceMemoryName: huawei.com/Ascend910B2-memory + memoryAllocatable: 65536 + memoryCapacity: 65536 + memoryFactor: 1 + aiCore: 24 + aiCPU: 6 + templates: + - name: vir03_1c_8g + memory: 8192 + aiCore: 3 + aiCPU: 1 + - name: vir06_1c_16g + memory: 16384 + aiCore: 6 + aiCPU: 1 + - name: vir12_3c_32g + memory: 32768 + aiCore: 12 + aiCPU: 3 + - chipName: 910B3 + commonWord: Ascend910B3 + resourceName: huawei.com/Ascend910B3 + resourceMemoryName: huawei.com/Ascend910B3-memory + memoryAllocatable: 65536 + memoryCapacity: 65536 + memoryFactor: 1 + aiCore: 20 + aiCPU: 7 + templates: + - name: vir05_1c_16g + memory: 16384 + aiCore: 5 + aiCPU: 1 + - name: vir10_3c_32g + memory: 32768 + aiCore: 10 + aiCPU: 3 + - chipName: 910B4-1 + commonWord: Ascend910B4-1 + resourceName: huawei.com/Ascend910B4-1 + resourceMemoryName: huawei.com/Ascend910B4-1-memory + memoryAllocatable: 65536 + memoryCapacity: 65536 + memoryFactor: 1 + aiCore: 20 + aiCPU: 7 + templates: + # NOTE: Names of vnpu template for 910B4-1 are fixed by Ascend runtime and must not be changed. + # The memory is used for scheduling so the correct values must be set. + # Template vir05_1c_8g actually provides 16GB memory, + - name: vir05_1c_8g + memory: 16384 + aiCore: 5 + aiCPU: 1 + # Template vir10_3c_16g actually provides 32GB memory + - name: vir10_3c_16g + memory: 32768 + aiCore: 10 + aiCPU: 3 + - chipName: 910B4 + commonWord: Ascend910B4 + resourceName: huawei.com/Ascend910B4 + resourceMemoryName: huawei.com/Ascend910B4-memory + memoryAllocatable: 32768 + memoryCapacity: 32768 + memoryFactor: 1 + aiCore: 20 + aiCPU: 7 + templates: + - name: vir05_1c_8g + memory: 8192 + aiCore: 5 + aiCPU: 1 + - name: vir10_3c_16g + memory: 16384 + aiCore: 10 + aiCPU: 3 + - chipName: 310P3 + commonWord: Ascend310P + resourceName: huawei.com/Ascend310P + resourceMemoryName: huawei.com/Ascend310P-memory + memoryAllocatable: 21527 + memoryCapacity: 24576 + memoryFactor: 1 + aiCore: 8 + aiCPU: 7 + templates: + - name: vir01 + memory: 3072 + aiCore: 1 + aiCPU: 1 + - name: vir02 + memory: 6144 + aiCore: 2 + aiCPU: 2 + - name: vir04 + memory: 12288 + aiCore: 4 + aiCPU: 4 + {{ end }} +{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml new file mode 100644 index 00000000..b089ae82 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml @@ -0,0 +1,30 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} +{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +rules: + - apiGroups: + - admissionregistration.k8s.io + resources: + #- validatingwebhookconfigurations + - mutatingwebhookconfigurations + verbs: + - get + - update +{{- if .Values.podSecurityPolicy.enabled }} + - apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ include "hami-vgpu.fullname" . }}-admission +{{- end }} +{{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml new file mode 100644 index 00000000..64f01ecf --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} +{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "hami-vgpu.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.fullname" . }}-admission + namespace: {{ include "hami-vgpu.namespace" . }} +{{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml new file mode 100644 index 00000000..400168a7 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml @@ -0,0 +1,70 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} +{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission-create + namespace: {{ include "hami-vgpu.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission-create + {{- if .Values.scheduler.patch.podAnnotations }} + annotations: {{ toYaml .Values.scheduler.patch.podAnnotations | nindent 8 }} + {{- end }} + labels: + {{- include "hami-vgpu.labels" . | nindent 8 }} + app.kubernetes.io/component: admission-webhook + hami.io/webhook: ignore + spec: + {{- if .Values.scheduler.patch.priorityClassName }} + priorityClassName: {{ .Values.scheduler.patch.priorityClassName }} + {{- end }} + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + {{- include "hami.scheduler.patch.new.imagePullSecrets" . | nindent 6 }} + {{- else }} + {{- include "hami.scheduler.patch.imagePullSecrets" . | nindent 6 }} + {{- end }} + containers: + - name: create + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + image: {{ include "hami.scheduler.patch.new.image" . }} + imagePullPolicy: {{ .Values.scheduler.patch.imageNew.pullPolicy }} + {{- else }} + image: {{ include "hami.scheduler.patch.image" . }} + imagePullPolicy: {{ .Values.scheduler.patch.image.pullPolicy }} + {{- end }} + args: + - create + - --cert-name=tls.crt + - --key-name=tls.key + {{- if .Values.scheduler.admissionWebhook.customURL.enabled }} + - --host={{ printf "%s.%s.svc,127.0.0.1,%s" (include "hami-vgpu.scheduler" .) (include "hami-vgpu.namespace" .) .Values.scheduler.admissionWebhook.customURL.host}} + {{- else }} + - --host={{ printf "%s.%s.svc,127.0.0.1" (include "hami-vgpu.scheduler" .) (include "hami-vgpu.namespace" .) }} + {{- end }} + - --namespace={{ include "hami-vgpu.namespace" . }} + - --secret-name={{ include "hami-vgpu.scheduler.tls" . }} + restartPolicy: OnFailure + serviceAccountName: {{ include "hami-vgpu.fullname" . }}-admission + {{- if .Values.scheduler.patch.nodeSelector }} + nodeSelector: {{ toYaml .Values.scheduler.patch.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.scheduler.patch.tolerations }} + tolerations: {{ toYaml .Values.scheduler.patch.tolerations | nindent 8 }} + {{- end }} + securityContext: + runAsNonRoot: true + runAsUser: {{ .Values.scheduler.patch.runAsUser }} +{{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml new file mode 100644 index 00000000..71e97f6b --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml @@ -0,0 +1,65 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} +{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission-patch + namespace: {{ include "hami-vgpu.namespace" . }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission-patch + {{- if .Values.scheduler.patch.podAnnotations }} + annotations: {{ toYaml .Values.scheduler.patch.podAnnotations | nindent 8 }} + {{- end }} + labels: + {{- include "hami-vgpu.labels" . | nindent 8 }} + app.kubernetes.io/component: admission-webhook + hami.io/webhook: ignore + spec: + {{- if .Values.scheduler.patch.priorityClassName }} + priorityClassName: {{ .Values.scheduler.patch.priorityClassName }} + {{- end }} + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + {{- include "hami.scheduler.patch.new.imagePullSecrets" . | nindent 6 }} + {{- else }} + {{- include "hami.scheduler.patch.imagePullSecrets" . | nindent 6 }} + {{- end }} + containers: + - name: patch + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + image: {{ include "hami.scheduler.patch.new.image" . }} + imagePullPolicy: {{ .Values.scheduler.patch.imageNew.pullPolicy }} + {{- else }} + image: {{ include "hami.scheduler.patch.image" . }} + imagePullPolicy: {{ .Values.scheduler.patch.image.pullPolicy }} + {{- end }} + args: + - patch + - --webhook-name={{ include "hami-vgpu.scheduler.webhook" . }} + - --namespace={{ include "hami-vgpu.namespace" . }} + - --patch-validating=false + - --secret-name={{ include "hami-vgpu.scheduler.tls" . }} + restartPolicy: OnFailure + serviceAccountName: {{ include "hami-vgpu.fullname" . }}-admission + {{- if .Values.scheduler.patch.nodeSelector }} + nodeSelector: {{ toYaml .Values.scheduler.patch.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.scheduler.patch.tolerations }} + tolerations: {{ toYaml .Values.scheduler.patch.tolerations | nindent 8 }} + {{- end }} + securityContext: + runAsNonRoot: true + runAsUser: {{ .Values.scheduler.patch.runAsUser }} +{{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml new file mode 100644 index 00000000..a48270e7 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml @@ -0,0 +1,40 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} +{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +{{- if .Values.podSecurityPolicy.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +spec: + allowPrivilegeEscalation: false + fsGroup: + ranges: + - max: 65535 + min: 1 + rule: MustRunAs + requiredDropCapabilities: + - ALL + runAsUser: + rule: MustRunAsNonRoot + seLinux: + rule: RunAsAny + supplementalGroups: + ranges: + - max: 65535 + min: 1 + rule: MustRunAs + volumes: + - configMap + - emptyDir + - projected + - secret + - downwardAPI +{{- end }} +{{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml new file mode 100644 index 00000000..74e45681 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml @@ -0,0 +1,23 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} +{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission + namespace: {{ include "hami-vgpu.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create +{{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml new file mode 100644 index 00000000..32da5ba4 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml @@ -0,0 +1,23 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} +{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission + namespace: {{ include "hami-vgpu.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "hami-vgpu.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.fullname" . }}-admission + namespace: {{ include "hami-vgpu.namespace" . }} +{{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml new file mode 100644 index 00000000..d2eb41de --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) }} +{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission + namespace: {{ include "hami-vgpu.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +{{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/role.yaml b/packages/system/hami/charts/hami/templates/scheduler/role.yaml new file mode 100644 index 00000000..6e681967 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/role.yaml @@ -0,0 +1,14 @@ +{{- if not .Values.dra.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create", "list", "watch", "get", "update", "patch"] +{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml new file mode 100644 index 00000000..934f8223 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml @@ -0,0 +1,33 @@ +{{- if not .Values.dra.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "hami-vgpu.scheduler" . }} +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +--- +{{- if .Values.mockDevicePlugin.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.mock-device-plugin" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "hami-vgpu.mock-device-plugin" . }} +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.mock-device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +{{- end -}} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/service.yaml b/packages/system/hami/charts/hami/templates/scheduler/service.yaml new file mode 100644 index 00000000..3e18bc98 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/service.yaml @@ -0,0 +1,36 @@ +{{- if not .Values.dra.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} + {{- if .Values.scheduler.service.labels }} + {{ toYaml .Values.scheduler.service.labels | indent 4 }} + {{- end }} + {{- if .Values.scheduler.service.annotations }} + annotations: {{ toYaml .Values.scheduler.service.annotations | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.scheduler.service.type | default "NodePort" }} # Default type is NodePort + ports: + - name: http + port: {{ .Values.scheduler.service.httpPort | default 443 }} # Default HTTP port is 443 + targetPort: {{ .Values.scheduler.service.httpTargetPort | default 443 }} + {{- if eq (.Values.scheduler.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort + nodePort: {{ .Values.scheduler.service.schedulerPort | default 31998 }} + {{- end }} + protocol: TCP + - name: monitor + port: {{ .Values.scheduler.service.monitorPort | default 31993 }} # Default monitoring port is 31993 + targetPort: {{ .Values.scheduler.service.monitorTargetPort | default 9395 }} + {{- if eq (.Values.scheduler.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort + nodePort: {{ .Values.scheduler.service.monitorPort | default 31993 }} + {{- end }} + protocol: TCP + selector: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.selectorLabels" . | nindent 4 }} +{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml b/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml new file mode 100644 index 00000000..929bf59a --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml @@ -0,0 +1,18 @@ +{{- if not .Values.dra.enabled -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +--- +{{- if .Values.mockDevicePlugin.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "hami-vgpu.mock-device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +{{- end -}} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml b/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml new file mode 100644 index 00000000..148feb5c --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml @@ -0,0 +1,57 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + {{- if .Values.scheduler.certManager.enabled }} + annotations: + cert-manager.io/inject-ca-from: {{ include "hami-vgpu.namespace" . }}/{{ include "hami-vgpu.scheduler" . }}-serving-cert + {{- end }} + name: {{ include "hami-vgpu.scheduler.webhook" . }} +webhooks: + - admissionReviewVersions: + - v1beta1 + clientConfig: + {{- if .Values.scheduler.admissionWebhook.customURL.enabled }} + url: https://{{ .Values.scheduler.admissionWebhook.customURL.host}}:{{.Values.scheduler.admissionWebhook.customURL.port}}{{.Values.scheduler.admissionWebhook.customURL.path}} + {{- else }} + service: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + path: /webhook + port: {{ .Values.scheduler.service.httpPort }} + {{- end }} + failurePolicy: {{ .Values.scheduler.admissionWebhook.failurePolicy }} + matchPolicy: Equivalent + name: vgpu.hami.io + namespaceSelector: + matchExpressions: + - key: hami.io/webhook + operator: NotIn + values: + - ignore + {{- if .Values.scheduler.admissionWebhook.whitelistNamespaces }} + - key: kubernetes.io/metadata.name + operator: NotIn + values: + {{- toYaml .Values.scheduler.admissionWebhook.whitelistNamespaces | nindent 10 }} + {{- end }} + objectSelector: + matchExpressions: + - key: hami.io/webhook + operator: NotIn + values: + - ignore + reinvocationPolicy: {{ .Values.scheduler.admissionWebhook.reinvocationPolicy }} + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - pods + scope: '*' + sideEffects: None + timeoutSeconds: 10 +{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/values.yaml b/packages/system/hami/charts/hami/values.yaml new file mode 100644 index 00000000..9fbeca70 --- /dev/null +++ b/packages/system/hami/charts/hami/values.yaml @@ -0,0 +1,476 @@ +## @section Global configuration +## @param global.imageRegistry Global Docker image registry +## @param global.imagePullSecrets Global Docker image pull secrets +global: + ## @param global.imageRegistry Global Docker image registry + imageRegistry: "" + ## E.g. + ## imagePullSecrets: + ## - myRegistryKeySecretName + ## @param global.imagePullSecrets Global Docker image pull secrets + imagePullSecrets: [] + imageTag: "v2.8.1" + gpuHookPath: /usr/local + labels: {} + annotations: {} + managedNodeSelectorEnable: false + managedNodeSelector: + usage: "gpu" + +nameOverride: "" +fullnameOverride: "" +namespaceOverride: "" + +#Nvidia GPU Parameters +resourceName: "nvidia.com/gpu" +resourceMem: "nvidia.com/gpumem" +resourceMemPercentage: "nvidia.com/gpumem-percentage" +resourceCores: "nvidia.com/gpucores" +resourcePriority: "nvidia.com/priority" + +#MLU Parameters +mluResourceName: "cambricon.com/vmlu" +mluResourceMem: "cambricon.com/mlu.smlu.vmemory" +mluResourceCores: "cambricon.com/mlu.smlu.vcore" + +#Hygon DCU Parameters +dcuResourceName: "hygon.com/dcunum" +dcuResourceMem: "hygon.com/dcumem" +dcuResourceCores: "hygon.com/dcucores" + +#Metax sGPU Parameters +metaxResourceName: "metax-tech.com/sgpu" +metaxResourceCore: "metax-tech.com/vcore" +metaxResourceMem: "metax-tech.com/vmemory" +metaxsGPUTopologyAware: "false" + +#Enflame VGCU Parameters +enflameResourceNameVGCU: "enflame.com/vgcu" +enflameResourceNameVGCUPercentage: "enflame.com/vgcu-percentage" + +#Kunlun XPU Parameters +kunlunResourceName: "kunlunxin.com/xpu" +kunlunResourceVCountName: "kunlunxin.com/vxpu" +kunlunResourceVMemoryName: "kunlunxin.com/vxpu-memory" + +schedulerName: "hami-scheduler" + +podSecurityPolicy: + enabled: false + +scheduler: + # @param nodeName defines the node name and the nvidia-vgpu-scheduler-scheduler will schedule to the node. + # if we install the nvidia-vgpu-scheduler-scheduler as default scheduler, we need to remove the k8s default + # scheduler pod from the cluster first, we must specify node name to skip the schedule workflow. + nodeName: "" + #nodeLabelSelector: + # "gpu": "on" + overwriteEnv: "false" + defaultSchedulerPolicy: + nodeSchedulerPolicy: binpack + gpuSchedulerPolicy: spread + metricsBindAddress: ":9395" + # If set to false, When Pod.Spec.SchedulerName equals to the const DefaultSchedulerName in k8s.io/api/core/v1 package, webhook will not overwrite it + forceOverwriteDefaultScheduler: true + livenessProbe: false + leaderElect: true + # when leaderElect is true, replicas is available, otherwise replicas is 1. + replicas: 1 + kubeScheduler: + # @param enabled indicate whether to run kube-scheduler container in the scheduler pod, it's true by default. + enabled: true + ## @param image.registry kube scheduler image registry + ## @param image.repository kube scheduler image repository + ## @param image.tag kube scheduler image tag (immutable tags are recommended) + ## @param image.pullPolicy kube scheduler image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + image: + registry: "registry.cn-hangzhou.aliyuncs.com" + repository: "google_containers/kube-scheduler" + tag: "" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + resources: {} + # If you do want to specify resources, uncomment the following lines, adjust them as necessary. + # and remove the curly braces after 'resources:'. +# limits: +# cpu: 1000m +# memory: 1000Mi +# requests: +# cpu: 100m +# memory: 100Mi + extraNewArgs: + - --config=/config/config.yaml + - -v=4 + extraArgs: + - --policy-config-file=/config/config.json + - -v=4 + extender: + ## @param image.registry scheduler extender image registry + ## @param image.repository scheduler extender image repository + ## @param image.tag scheduler extender image tag (immutable tags are recommended) + ## @param image.pullPolicy scheduler extender image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + image: + registry: "docker.io" + repository: "projecthami/hami" + tag: "" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + resources: {} + # If you do want to specify resources, uncomment the following lines, adjust them as necessary, + # and remove the curly braces after 'resources:'. +# limits: +# cpu: 1000m +# memory: 1000Mi +# requests: +# cpu: 100m +# memory: 100Mi + extraArgs: + - --debug + - -v=4 + nodeLockExpire: "5m" + podAnnotations: {} + tolerations: [] + #serviceAccountName: "hami-vgpu-scheduler-sa" + admissionWebhook: + # If set to false, the admission webhook is not installed and any pods that should use HAMi must be + # configured to use the right 'schedulerName' and other device-specific configurations. + enabled: true + customURL: + enabled: false + # must be an endpoint using https. + # should generate host certs here + host: 127.0.0.1 # hostname or ip, can be your node'IP if you want to use https://:/ + port: 31998 + path: /webhook + whitelistNamespaces: + # Specify the namespaces that the webhook will not be applied to. + # - default + # - kube-system + # - istio-system + reinvocationPolicy: Never + failurePolicy: Ignore + ## TLS Certificate Option 1: Use cert-manager to generate self-signed certificate. + ## If enabled, always takes precedence over options 2. + certManager: + enabled: false + ## TLS Certificate Option 2: Use kube-webhook-certgen to generate self-signed certificate. + ## If true and certManager.enabled is false, Helm will automatically create a self-signed cert and secret for you. + patch: + enabled: true + ## @param image.registry scheduler extender image registry + ## @param image.repository scheduler extender image repository + ## @param image.tag scheduler extender image tag (immutable tags are recommended) + ## @param image.pullPolicy scheduler extender image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + image: + registry: "docker.io" + repository: "jettech/kube-webhook-certgen" + tag: "v1.5.2" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## @param image.registry scheduler extender image registry + ## @param image.repository scheduler extender image repository + ## @param image.tag scheduler extender image tag (immutable tags are recommended) + ## @param image.pullPolicy scheduler extender image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + imageNew: + registry: "docker.io" + repository: "liangjw/kube-webhook-certgen" + tag: "v1.1.1" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + priorityClassName: "" + podAnnotations: {} + nodeSelector: {} + tolerations: [] + runAsUser: 2000 + service: + type: NodePort # Default type is NodePort, can be changed to ClusterIP + httpPort: 443 # HTTP port + schedulerPort: 31998 # NodePort for HTTP + monitorPort: 31993 # Monitoring port + monitorTargetPort: 9395 + httpTargetPort: 443 + labels: {} + annotations: {} + +devicePlugin: + enabled: true + gpuOperatorToolkitReady: + enabled: false + hostPath: "/run/nvidia/validations" + ## @param image.registry devicePlugin image registry + ## @param image.repository devicePlugin image repository + ## @param image.tag devicePlugin image tag (immutable tags are recommended) + ## @param image.pullPolicy devicePlugin image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + image: + registry: "docker.io" + repository: "projecthami/hami" + tag: "" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + monitor: + ## @param image.registry monitor image registry + ## @param image.repository monitor image repository + ## @param image.tag monitor image tag (immutable tags are recommended) + ## @param image.pullPolicy monitor image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + image: + registry: "docker.io" + repository: "projecthami/hami" + tag: "" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ctrPath: /usr/local/vgpu/containers + resyncInterval: "5m" + extraArgs: + - -v=4 + extraEnvs: {} + resources: {} + # If you do want to specify resources, uncomment the following lines, adjust them as necessary. + # and remove the curly braces after 'resources:'. +# limits: +# cpu: 1000m +# memory: 1000Mi +# requests: +# cpu: 100m +# memory: 100Mi + + deviceSplitCount: 10 + deviceMemoryScaling: 1 + deviceCoreScaling: 1 + # Node configuration for device plugin, Priority: externalConfigName > config > default config + nodeConfiguration: + # If you want to use a custom config.json, you can set the content here. + # If this is set, it will override the default config.json(An example is as follows). + config: | + { + "nodeconfig": [ + { + "name": "your-node-name", + "operatingmode": "hami-core", + "devicememoryscaling": 1, + "devicesplitcount": 10, + "migstrategy": "none", + "filterdevices": { + "uuid": [], + "index": [] + } + } + ] + } + # If you want to use an existing ConfigMap, you can set the name here. + # If this is set, the chart will not create the ConfigMap and will use the existing one. + externalConfigName: "" + # The runtime class name to be used by the device plugin, and added to the pod.spec.runtimeClassName of applications utilizing NVIDIA GPUs + runtimeClassName: "" + # Whether to create runtime class, name comes from runtimeClassName when it is set + createRuntimeClass: false + migStrategy: "none" + disablecorelimit: "false" + passDeviceSpecsEnabled: false + deviceListStrategy: "envvar" + nvidiaHookPath: null + nvidiaDriverRoot: null + gdrcopyEnabled: null + gdsEnabled: null + mofedEnabled: null + + extraArgs: + - -v=4 + extraEnvs: {} + service: + type: NodePort # Default type is NodePort, can be changed to ClusterIP + httpPort: 31992 + labels: {} + annotations: {} + + pluginPath: /var/lib/kubelet/device-plugins + libPath: /usr/local/vgpu + + podAnnotations: {} + nvidiaNodeSelector: + gpu: "on" + tolerations: [] + # The updateStrategy for DevicePlugin DaemonSet. + # If you want to update the DaemonSet by manual, set type as "OnDelete". + # We recommend use OnDelete update strategy because DevicePlugin pod restart will cause business pod restart, this behavior is destructive. + # Otherwise, you can use RollingUpdate update strategy to rolling update DevicePlugin pod. + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + + resources: {} + # If you do want to specify resources, uncomment the following lines, adjust them as necessary. + # and remove the curly braces after 'resources:'. +# limits: +# cpu: 1000m +# memory: 1000Mi +# requests: +# cpu: 100m +# memory: 100Mi + +mockDevicePlugin: + enabled: false + image: + registry: "docker.io" + repository: "projecthami/mock-device-plugin" + tag: "1.0.1" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] +# If this option is enabled, the DRA will be installed and the scheduler extender will not be installed. +dra: + enabled: false + +# Configuration for the hami-dra subchart, which includes DRA drivers. +# This is part of config for DRA, please refer to the DRA documentation for more details. +hami-dra: + monitor: + enabled: true + drivers: + nvidia: + enabled: true + # If you are using gpu driver on host, you need to set this to false + containerDriver: true + image: + repository: projecthami/k8s-dra-driver + tag: "v0.0.1-dev" + +devices: + amd: + customresources: + - amd.com/gpu + - amd.com/gpu-memory + awsneuron: + customresources: + - aws.amazon.com/neuron + - aws.amazon.com/neuroncore + kunlun: + enabled: true + customresources: + - kunlunxin.com/xpu + - kunlunxin.com/vxpu + - kunlunxin.com/vxpu-memory + enflame: + enabled: true + customresources: + - enflame.com/vgcu + - enflame.com/vgcu-percentage + - enflame.com/gcu + mthreads: + enabled: true + customresources: + - mthreads.com/vgpu + nvidia: + gpuCorePolicy: default + libCudaLogLevel: 1 + ascend: + enabled: false + image: "" + imagePullPolicy: IfNotPresent + extraArgs: [] + nodeSelector: + ascend: "on" + tolerations: [] + customresources: + - huawei.com/Ascend910A + - huawei.com/Ascend910A-memory + - huawei.com/Ascend910B2 + - huawei.com/Ascend910B2-memory + - huawei.com/Ascend910B3 + - huawei.com/Ascend910B3-memory + - huawei.com/Ascend910B4 + - huawei.com/Ascend910B4-memory + - huawei.com/Ascend910B4-1 + - huawei.com/Ascend910B4-1-memory + - huawei.com/Ascend310P + - huawei.com/Ascend310P-memory + iluvatar: + enabled: false + customresources: + - iluvatar.ai/BI-V100-vgpu + - iluvatar.ai/BI-V100.vCore + - iluvatar.ai/BI-V100.vMem + - iluvatar.ai/BI-V150-vgpu + - iluvatar.ai/BI-V150.vCore + - iluvatar.ai/BI-V150.vMem + - iluvatar.ai/MR-V100-vgpu + - iluvatar.ai/MR-V100.vCore + - iluvatar.ai/MR-V100.vMem + - iluvatar.ai/MR-V50-vgpu + - iluvatar.ai/MR-V50.vCore + - iluvatar.ai/MR-V50.vMem diff --git a/packages/system/hami/values.yaml b/packages/system/hami/values.yaml new file mode 100644 index 00000000..1155e2cb --- /dev/null +++ b/packages/system/hami/values.yaml @@ -0,0 +1,8 @@ +hami: + devicePlugin: + runtimeClassName: nvidia + nodeConfiguration: + config: | + { + "nodeconfig": [] + } From a3e2fbd742d0c9f8bf6a255836913f080f42b826 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Wed, 22 Apr 2026 13:25:23 +0300 Subject: [PATCH 154/250] feat(kubernetes): integrate HAMi as optional addon Add HAMi HelmRelease to the kubernetes app as a toggleable addon. When enabled, GPU Operator's built-in device plugin is automatically disabled via a values merge to avoid conflicts with HAMi's own device plugin. The HelmRelease fails fast if GPU Operator is not enabled, since HAMi depends on it for driver management and container toolkit. Assisted-By: Claude Signed-off-by: Arsolitt --- .../templates/helmreleases/gpu-operator.yaml | 13 ++++- .../templates/helmreleases/hami.yaml | 49 +++++++++++++++++++ packages/apps/kubernetes/values.schema.json | 23 +++++++++ packages/apps/kubernetes/values.yaml | 8 +++ 4 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 packages/apps/kubernetes/templates/helmreleases/hami.yaml diff --git a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml index 5ef48912..995aa430 100644 --- a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml @@ -1,3 +1,11 @@ +{{- define "cozystack.defaultGpuOperatorValues" -}} +{{- if and (hasKey .Values.addons "hami") .Values.addons.hami.enabled }} +gpu-operator: + devicePlugin: + enabled: false +{{- end }} +{{- end }} + {{- if .Values.addons.gpuOperator.enabled }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease @@ -29,7 +37,10 @@ spec: force: true remediation: retries: -1 - {{- with .Values.addons.gpuOperator.valuesOverride }} + {{- $defaults := fromYaml (include "cozystack.defaultGpuOperatorValues" .) }} + {{- $overrides := deepCopy (default (dict) .Values.addons.gpuOperator.valuesOverride) }} + {{- $merged := mergeOverwrite (default (dict) $defaults) $overrides }} + {{- with $merged }} values: {{- toYaml . | nindent 4 }} {{- end }} diff --git a/packages/apps/kubernetes/templates/helmreleases/hami.yaml b/packages/apps/kubernetes/templates/helmreleases/hami.yaml new file mode 100644 index 00000000..1f7a5358 --- /dev/null +++ b/packages/apps/kubernetes/templates/helmreleases/hami.yaml @@ -0,0 +1,49 @@ +{{- if .Values.addons.hami.enabled }} +{{- if not .Values.addons.gpuOperator.enabled }} +{{- fail "addons.hami requires addons.gpuOperator to be enabled" }} +{{- end }} +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: {{ .Release.Name }}-hami + labels: + cozystack.io/repository: system + cozystack.io/target-cluster-name: {{ .Release.Name }} + sharding.fluxcd.io/key: tenants +spec: + releaseName: hami + chartRef: + kind: ExternalArtifact + name: cozystack-kubernetes-application-kubevirt-kubernetes-hami + namespace: cozy-system + kubeConfig: + secretRef: + name: {{ .Release.Name }}-admin-kubeconfig + key: super-admin.svc + targetNamespace: cozy-hami + storageNamespace: cozy-hami + interval: 5m + timeout: 10m + install: + createNamespace: true + remediation: + retries: -1 + upgrade: + force: true + remediation: + retries: -1 + {{- with .Values.addons.hami.valuesOverride }} + values: + {{- toYaml . | nindent 4 }} + {{- end }} + + dependsOn: + {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }} + - name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} + {{- end }} + - name: {{ .Release.Name }}-cilium + namespace: {{ .Release.Namespace }} + - name: {{ .Release.Name }}-gpu-operator + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/packages/apps/kubernetes/values.schema.json b/packages/apps/kubernetes/values.schema.json index 1beeff9c..3d7a17dd 100644 --- a/packages/apps/kubernetes/values.schema.json +++ b/packages/apps/kubernetes/values.schema.json @@ -149,6 +149,7 @@ "fluxcd", "gatewayAPI", "gpuOperator", + "hami", "ingressNginx", "monitoringAgents", "velero", @@ -268,6 +269,28 @@ } } }, + "hami": { + "description": "HAMi GPU virtualization middleware. Enables fractional GPU sharing (memory and compute isolation) for workloads. Requires GPU Operator. Note: workload containers must use glibc < 2.34 (not musl/Alpine) for full compute isolation.", + "type": "object", + "default": {}, + "required": [ + "enabled", + "valuesOverride" + ], + "properties": { + "enabled": { + "description": "Enable HAMi.", + "type": "boolean", + "default": false + }, + "valuesOverride": { + "description": "Custom Helm values overrides.", + "type": "object", + "default": {}, + "x-kubernetes-preserve-unknown-fields": true + } + } + }, "ingressNginx": { "description": "Ingress-NGINX controller.", "type": "object", diff --git a/packages/apps/kubernetes/values.yaml b/packages/apps/kubernetes/values.yaml index a67b5d69..f601b979 100644 --- a/packages/apps/kubernetes/values.yaml +++ b/packages/apps/kubernetes/values.yaml @@ -94,6 +94,10 @@ host: "" ## @field {bool} enabled - Enable FluxCD. ## @field {object} valuesOverride - Custom Helm values overrides. +## @typedef {struct} HAMiAddon - HAMi GPU virtualization middleware. +## @field {bool} enabled - Enable HAMi (requires GPU Operator). +## @field {object} valuesOverride - Custom Helm values overrides. + ## @typedef {struct} MonitoringAgentsAddon - Monitoring agents (Fluent Bit, VMAgents). ## @field {bool} enabled - Enable monitoring agents. ## @field {object} valuesOverride - Custom Helm values overrides. @@ -114,6 +118,7 @@ host: "" ## @field {GatewayAPIAddon} gatewayAPI - Gateway API addon. ## @field {IngressNginxAddon} ingressNginx - Ingress-NGINX controller. ## @field {GPUOperatorAddon} gpuOperator - NVIDIA GPU Operator. +## @field {HAMiAddon} hami - HAMi GPU virtualization middleware. ## @field {FluxCDAddon} fluxcd - FluxCD GitOps operator. ## @field {MonitoringAgentsAddon} monitoringAgents - Monitoring agents. ## @field {VerticalPodAutoscalerAddon} verticalPodAutoscaler - Vertical Pod Autoscaler. @@ -137,6 +142,9 @@ addons: gpuOperator: enabled: false valuesOverride: {} + hami: + enabled: false + valuesOverride: {} fluxcd: enabled: false valuesOverride: {} From 273eb7811a36fb69141d80360705f081a3276194 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Wed, 22 Apr 2026 13:25:29 +0300 Subject: [PATCH 155/250] docs(hami): document glibc < 2.34 limitation and upstream issues HAMi-core relies on _dl_sym (private glibc symbol removed in 2.34) for CUDA interception. This breaks compute isolation on modern container images using Ubuntu 22.04+ and makes Alpine/musl completely incompatible. Include upstream issue references and a compatibility matrix so users can make informed decisions about base image selection. Assisted-By: Claude Signed-off-by: Arsolitt --- packages/system/hami/README.md | 81 ++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 packages/system/hami/README.md diff --git a/packages/system/hami/README.md b/packages/system/hami/README.md new file mode 100644 index 00000000..283f9e80 --- /dev/null +++ b/packages/system/hami/README.md @@ -0,0 +1,81 @@ +# HAMi — GPU Virtualization Middleware + +[HAMi](https://github.com/Project-HAMi/HAMi) (Heterogeneous AI Computing Virtualization Middleware) is a CNCF Sandbox project that enables fractional GPU sharing in Kubernetes. It allows workloads to request specific amounts of GPU memory and compute cores instead of claiming entire GPUs. + +## Architecture + +HAMi consists of four components: + +- **MutatingWebhook** — intercepts pod creation, injects `schedulerName: hami-scheduler` +- **Scheduler Extender** — extends kube-scheduler with GPU-aware Filter and Bind logic +- **Device Plugin** (DaemonSet) — registers vGPU resources via the Kubernetes Device Plugin API +- **HAMi-core** (`libvgpu.so`) — `LD_PRELOAD` library injected into workload containers, intercepts CUDA API calls to enforce memory and compute isolation + +## Prerequisites + +- GPU Operator must be enabled (`addons.gpuOperator.enabled: true`) +- NVIDIA driver >= 440 on host nodes +- nvidia-container-toolkit configured as the default container runtime +- GPU nodes labeled with `gpu=on` + +## Known Limitations + +### glibc < 2.34 requirement for workload containers + +HAMi-core uses `LD_PRELOAD` to intercept `dlsym()` for CUDA symbol resolution. The fallback code path relies on `_dl_sym`, a private glibc internal symbol that was removed in glibc 2.34 when libdl and libpthread were merged into libc.so. + +**This limitation affects workload containers only**, not the host OS or HAMi's own components. + +| Distribution | glibc | Result | +| --------------- | ----- | -------------------------------------------- | +| Ubuntu 18.04 | 2.27 | Full isolation (memory + compute) | +| Ubuntu 20.04 | 2.31 | Full isolation (memory + compute) | +| Ubuntu 22.04 | 2.35 | Memory isolation works, compute breaks | +| Ubuntu 24.04 | 2.39 | Both memory and compute isolation break | +| Alpine (musl) | N/A | Completely incompatible (`dlvsym` absent) | + +Most modern ML/AI base images (CUDA 12.x, PyTorch 2.x, TensorFlow 2.x) use Ubuntu 22.04+ with glibc >= 2.35, which means compute isolation will not work with these images until the upstream fix is merged. + +**Upstream tracking issues:** + +- [HAMi-core#174](https://github.com/Project-HAMi/HAMi-core/issues/174) — `_dl_sym` removal breaks HAMi-core on glibc >= 2.34 +- [HAMi#1190](https://github.com/Project-HAMi/HAMi/issues/1190) — degraded isolation across glibc versions +- [HAMi#173](https://github.com/Project-HAMi/HAMi/issues/173) — documentation incorrectly states glibc < 2.30 (actual boundary is 2.34) + +### musl libc (Alpine) incompatibility + +HAMi-core is completely incompatible with musl libc. The `dlvsym()` function used by HAMi-core is a glibc extension not available in musl. Only glibc-based container images (Debian, Ubuntu, RHEL, etc.) can use HAMi GPU isolation. + +## Usage + +Enable HAMi in your tenant Kubernetes cluster values: + +```yaml +addons: + gpuOperator: + enabled: true + hami: + enabled: true +``` + +When HAMi is enabled, GPU Operator's built-in device plugin is automatically disabled to avoid conflicts. + +### Requesting fractional GPU resources + +```yaml +resources: + limits: + nvidia.com/gpu: 1 + nvidia.com/gpumem: 3000 # 3000 MB of GPU memory + nvidia.com/gpucores: 30 # 30% of GPU compute cores +``` + +## Parameters + +| Name | Description | Default | +| --- | --- | --- | +| `hami.devicePlugin.runtimeClassName` | RuntimeClass for device plugin pods | `nvidia` | +| `hami.devicePlugin.deviceSplitCount` | Max virtual GPUs per physical GPU | `10` | +| `hami.devicePlugin.deviceMemoryScaling` | Memory overcommit factor (> 1.0 enables overcommit) | `1` | +| `hami.scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy` | Node packing strategy (`binpack` or `spread`) | `binpack` | +| `hami.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy` | GPU packing strategy (`binpack` or `spread`) | `spread` | From 43035562efff34f18c3c439026fe2a233dc8d17b Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Wed, 22 Apr 2026 13:25:35 +0300 Subject: [PATCH 156/250] test(kubernetes): add helm-unittest tests for HAMi integration Cover HelmRelease rendering, gpuOperator dependency validation, ExternalArtifact chartRef, namespace targeting, dependency chain, valuesOverride passthrough, and automatic devicePlugin disable in GPU Operator when HAMi is active. Assisted-By: Claude Signed-off-by: Arsolitt --- .../tests/gpu_operator_hami_test.yaml | 80 +++++++++++ packages/apps/kubernetes/tests/hami_test.yaml | 136 ++++++++++++++++++ 2 files changed, 216 insertions(+) create mode 100644 packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml create mode 100644 packages/apps/kubernetes/tests/hami_test.yaml diff --git a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml new file mode 100644 index 00000000..33256ae6 --- /dev/null +++ b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml @@ -0,0 +1,80 @@ +suite: GPU Operator HelmRelease HAMi integration tests +templates: + - templates/helmreleases/gpu-operator.yaml +tests: + - it: should disable devicePlugin when hami is enabled + set: + addons: + gpuOperator: + enabled: true + valuesOverride: {} + hami: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.values.gpu-operator.devicePlugin.enabled + value: false + + - it: should not have values when hami is disabled and no overrides + set: + addons: + gpuOperator: + enabled: true + valuesOverride: {} + hami: + enabled: false + valuesOverride: {} + asserts: + - notExists: + path: spec.values + + - it: should allow user overrides to merge with hami defaults + set: + addons: + gpuOperator: + enabled: true + valuesOverride: + gpu-operator: + driver: + enabled: false + hami: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.values.gpu-operator.devicePlugin.enabled + value: false + - equal: + path: spec.values.gpu-operator.driver.enabled + value: false + + - it: should let user override devicePlugin back to true if needed + set: + addons: + gpuOperator: + enabled: true + valuesOverride: + gpu-operator: + devicePlugin: + enabled: true + hami: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.values.gpu-operator.devicePlugin.enabled + value: true + + - it: should not render when gpuOperator is disabled + set: + addons: + gpuOperator: + enabled: false + valuesOverride: {} + hami: + enabled: false + valuesOverride: {} + asserts: + - hasDocuments: + count: 0 diff --git a/packages/apps/kubernetes/tests/hami_test.yaml b/packages/apps/kubernetes/tests/hami_test.yaml new file mode 100644 index 00000000..e9d32236 --- /dev/null +++ b/packages/apps/kubernetes/tests/hami_test.yaml @@ -0,0 +1,136 @@ +suite: HAMi HelmRelease tests +templates: + - templates/helmreleases/hami.yaml +tests: + - it: should not render when hami is disabled + set: + addons: + hami: + enabled: false + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - hasDocuments: + count: 0 + + - it: should render HelmRelease when hami is enabled + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - hasDocuments: + count: 1 + - isKind: + of: HelmRelease + + - it: should fail when gpuOperator is not enabled + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: false + valuesOverride: {} + asserts: + - failedTemplate: + errorMessage: "addons.hami requires addons.gpuOperator to be enabled" + + - it: should have correct metadata labels + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: metadata.labels["cozystack.io/repository"] + value: system + - equal: + path: metadata.labels["sharding.fluxcd.io/key"] + value: tenants + + - it: should use ExternalArtifact chartRef + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.chartRef.kind + value: ExternalArtifact + - equal: + path: spec.chartRef.namespace + value: cozy-system + + - it: should target cozy-hami namespace + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.targetNamespace + value: cozy-hami + - equal: + path: spec.storageNamespace + value: cozy-hami + + - it: should depend on gpu-operator and cilium + release: + name: test + namespace: test-ns + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - contains: + path: spec.dependsOn + content: + name: test-cilium + namespace: test-ns + - contains: + path: spec.dependsOn + content: + name: test-gpu-operator + namespace: test-ns + + - it: should pass through valuesOverride + set: + addons: + hami: + enabled: true + valuesOverride: + hami: + devicePlugin: + deviceSplitCount: 5 + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.values.hami.devicePlugin.deviceSplitCount + value: 5 From 1eeeb2652aaaf6f92e655f010fb3655e865f6abc Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 22 Apr 2026 15:51:08 +0500 Subject: [PATCH 157/250] docs: add changelog for v1.3.0 Co-Authored-By: Claude Opus 4.7 Signed-off-by: Myasnikov Daniil --- docs/changelogs/v1.3.0.md | 242 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 242 insertions(+) create mode 100644 docs/changelogs/v1.3.0.md diff --git a/docs/changelogs/v1.3.0.md b/docs/changelogs/v1.3.0.md new file mode 100644 index 00000000..af7bb30e --- /dev/null +++ b/docs/changelogs/v1.3.0.md @@ -0,0 +1,242 @@ + + +# Cozystack v1.3.0 + +Cozystack v1.3.0 brings **storage-aware pod scheduling** via a LINSTOR scheduler extender, a managed **LINSTOR GUI** web console with Keycloak SSO, a curated **VM Default Images** catalog for out-of-the-box virtual-machine provisioning, a new **WorkloadsReady / Events** observability surface with S3 bucket metering, and **cross-namespace VMInstance backup restore** with a full **RestoreJob dashboard** flow. The release also ships stricter tenant-name validation, VMInstance network-selector improvements, Keycloak theme injection and SMTP configuration, a host-runtime preflight check, and rolls up every fix from the v1.2.1 → v1.2.4 patch line. + +> **Note:** Items marked *(backported to v1.2.x)* were also shipped in v1.2.1, v1.2.2, v1.2.3, or v1.2.4 patch releases. + +## Feature Highlights + +### Storage-Aware Scheduling via the LINSTOR Extender + +The `cozystack-scheduler` now calls a **LINSTOR scheduler extender** for storage-locality-aware pod placement. When a pod declares both a `SchedulingClass` and LINSTOR-backed PVCs, the scheduler consults LINSTOR to prefer nodes where volume replicas already exist — reducing cross-node replication traffic and improving I/O latency for storage-heavy workloads such as databases, object stores, and VMs. + +The integration builds on the existing `SchedulingClass` tenant workload placement system introduced in v1.2.0 and requires no tenant-side configuration — workloads simply benefit once a SchedulingClass is assigned. Administrators can mix storage locality with the existing data-center / hardware-generation constraints defined on SchedulingClass CRs ([**@lllamnyp**](https://github.com/lllamnyp) in #2330). + +### LINSTOR GUI: Managed Web Console for Storage Administration + +A new opt-in `linstor-gui` system package deploys **LINBIT's linstor-gui web UI** alongside the LINSTOR controller with mTLS client authentication, non-root security context, and a ClusterIP-only service by default. When OIDC is configured on the platform, an optional **Keycloak-protected Ingress** (via oauth2-proxy) exposes the UI for browser access. Access is restricted to members of the `cozystack-cluster-admin` Keycloak group, consistent with host-cluster admin RBAC, and the gatekeeper blocks in-app LINSTOR authentication setup at the nginx proxy layer so the managed configuration cannot be subverted through the UI. + +Operators who prefer CLI access keep the existing `linstor` command; the GUI is strictly additive and stays disabled by default ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2382, #2390, #2415, #2419). + +### VM Default Images: Out-of-the-Box VM Provisioning + +The new `vm-default-images` package provides a curated set of **cluster-wide virtual-machine images** (Ubuntu, Debian, CentOS Stream, and others) as pre-populated DataVolumes, so tenants can provision VMs against well-known base images without first having to upload them. The package is opt-in via the `iaas` bundle and defaults to replicated storage for high availability. Migration 38 renames legacy `vm-image-*` DataVolumes to the new `vm-default-images-*` naming scheme, and the `vm-disk` chart gains a new "disk" source type for cloning from existing vm-disks in the same namespace ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2258). + +### Application Observability: WorkloadsReady, Events, and S3 Bucket Metering + +Applications now expose a **WorkloadsReady** condition on their status by querying associated WorkloadMonitor resources, giving operators a single place to check whether all underlying workloads (Deployments, StatefulSets, DaemonSets, PVCs) are healthy. The dashboard gains a new **Events tab** showing namespace-scoped Kubernetes events per application, with fallback to `.firstTimestamp` when `.eventTime` is absent. A long-standing bug where WorkloadMonitor's `Operational` status was never persisted is fixed in the same change ([**@lexfrei**](https://github.com/lexfrei) in #2356). + +The WorkloadMonitor reconciler is extended to track **COSI BucketClaim** objects as first-class Workloads, and the bucket controller now queries SeaweedFS logical and physical bucket-size metrics from VictoriaMetrics via a namespace-scoped monitoring endpoint, enabling S3 billing integration on par with Pods and PVCs ([**@kitsunoff**](https://github.com/kitsunoff) in #2391). Workloads are also enriched with `workloads.cozystack.io/resource-preset` and source-object labels so downstream billing pipelines can correlate monitors with the tenant preset that produced them ([**@androndo**](https://github.com/androndo) in #2416). + +### Cross-Namespace VM Backup Restore and RestoreJob Dashboard + +The backup system now supports **restoring VMInstance backups into a different namespace** (cross-namespace copy restores) with IP/MAC preservation and safe rename semantics. In-place backup and restore flows for VMDisk and VMInstance are improved: HelmReleases and DataVolumes are properly handled, and Velero failure messages are propagated to the Application status. The backup status structure has been refactored to store underlying resources as a generic opaque JSON object, enabling arbitrary application-specific metadata without status-schema churn ([**@androndo**](https://github.com/androndo) in #2251, #2319, #2329). + +The dashboard now ships a complete **RestoreJob experience**: list view, details page, create form, and sidebar entry, with a "Same as backup" fallback rendering when `spec.targetApplicationRef` is omitted. Non-CRD-backed sidebar factories (`kube-*`, `plan`, `backupjob`, `backup`, `restorejob`) are marked static so they pick up consistent managed-by labels across reconciles ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2437). + +## Major Features and Improvements + +* **[api] Reject tenant names with dashes at Create time**: Enforces alphanumeric-only naming for Tenants at the API level, preventing names with hyphens that would silently fail during Helm reconciliation. A corresponding regex tightening and regression test suite hardens the validation ([**@lexfrei**](https://github.com/lexfrei) in #2380). + +* **[platform] Validate computed tenant namespace length**: Rejects Tenant creation when the computed ancestor-chain namespace would exceed the 63-character Kubernetes namespace limit, preventing opaque HelmRelease reconcile errors downstream ([**@lexfrei**](https://github.com/lexfrei) in #2376). + +* **[vm-instance] Rename subnets to networks and add dropdown selector**: Renames the misleading `subnets` field to `networks` in VMInstance for clarity, adds a dropdown selector for available networks in the dashboard form, and includes migration 36 to copy existing `subnets` values. The old field remains supported for backward compatibility ([**@sircthulhu**](https://github.com/sircthulhu) in #2263). + +* **[keycloak] Enable injecting themes**: Cozystack administrators can now inject custom Keycloak themes via `initContainers` for UI white-labeling and customization ([**@lllamnyp**](https://github.com/lllamnyp) in #2142). + +* **[keycloak-configure] Add email verification and SMTP configuration**: Adds configurable Keycloak settings for user self-registration, email verification, and SMTP server configuration, enabling automated user onboarding flows ([**@BROngineer**](https://github.com/BROngineer) in #2318). + +* **[postgres] Hardcode PostgreSQL 17 for monitoring databases**: Pins PostgreSQL 17.7 images for system databases (Grafana, Alerta, Harbor, Keycloak, SeaweedFS) and adds migration 37 to backfill `spec.version=v17` for existing PostgreSQL resources, preventing CNPG from defaulting to PostgreSQL 18 *(backported to v1.2.1)* ([**@IvanHunters**](https://github.com/IvanHunters) in #2304). + +* **[platform] Prevent installed packages deletion**: Adds the `helm.sh/resource-policy: keep` annotation to platform packages so disabling a package no longer triggers automatic Helm deletion, restoring the documented behavior where operators must explicitly delete a package *(backported to v1.2.1)* ([**@kvaps**](https://github.com/kvaps) in #2273). + +* **[mariadb] Always enable replication for consistent service naming**: MariaDB now always enables replication, creating `-primary`/`-secondary` services even for single-replica instances. This fixes dashboard visibility and backup functionality for single-replica setups ([**@sircthulhu**](https://github.com/sircthulhu) in #2279). + +* **[hack] Add host runtime preflight check**: New `check-host-runtime.sh` script and `make preflight` target that warns operators when a standalone containerd or docker runtime is running alongside the embedded k3s runtime, helping diagnose container-runtime conflicts early in an installation ([**@lexfrei**](https://github.com/lexfrei) in #2371). + +* **[hack] Add check-readiness.sh diagnostic script**: A new diagnostic script for tracking platform reconciliation by checking readiness of Packages, ArtifactGenerators, ExternalArtifacts, and HelmReleases, with support for watch mode and continuous monitoring ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2294). + +* **[platform] Add resourcePreset labels to WorkloadMonitor labels**: WorkloadMonitor labels with the `workloads.cozystack.io/` prefix are now propagated onto created Workloads; created Workloads always include the reserved `workloads.cozystack.io/monitor` label, and Helm app charts add `workloads.cozystack.io/resource-preset` metadata to WorkloadMonitor manifests, enabling downstream billing pipelines to correlate monitors with the tenant preset that produced them ([**@androndo**](https://github.com/androndo) in #2416). + +## Bug Fixes + +* **[platform] Migrate ACME HTTP-01 to ingressClassName API**: Switches ACME HTTP-01 issuance from the deprecated `acme.cert-manager.io/http01-ingress-class` annotation to the modern `ingressClassName` field on `ClusterIssuer` and solver pods. Previously, ClusterIssuers referenced a non-existent `nginx` class while each Ingress individually overrode it via annotation — producing `ingressClassName and class cannot be set at the same time` errors when tenants attempted to migrate to the modern field. The migration is atomic: both the ClusterIssuer and consuming Ingresses are updated together *(backported to v1.2.4)* ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2436). + +* **[harbor] Remove incorrect tenant module flags**: Harbor is a PaaS service, not a tenant module. Incorrect `spec.dashboard.module: true` and `internal.cozystack.io/tenantmodule` flags caused Harbor to appear in the sidebar "Modules" section and be misclassified by controllers handling tenant modules. The flags are now removed so Harbor is displayed in its proper PaaS category and is no longer treated as a tenant-scoped HelmRelease ([**@kvaps**](https://github.com/kvaps) in #2444). + +* **[kube-ovn] Resolve kubeovn-plunger RBAC forbidden on deployments**: Grants `kube-ovn-plunger` the RBAC needed to list Deployments so it can reconcile `ovn-central`, fixing `deployments.apps is forbidden` errors in `cozy-kubeovn` ([**@kvaps**](https://github.com/kvaps) in #2441). + +* **[cilium] Opt-out of cri-containerd.apparmor.d for nsenter init containers**: Opts cilium-agent init containers out of the `cri-containerd.apparmor.d` AppArmor profile on non-Talos variants (`cilium-generic`, `kubeovn-cilium-generic`), fixing `Init:CrashLoopBackOff` on Ubuntu 22.04+ and Debian where the profile denies `nsenter` namespace entry. Talos variants are untouched as Talos does not load the AppArmor LSM *(backported to v1.2.2)* ([**@lexfrei**](https://github.com/lexfrei) in #2370). + +* **[virtual-machine] Exclude external VM services from Cilium BPF LB**: Adds the `service.kubernetes.io/service-proxy-name: cozy-proxy` label to VM LoadBalancer services with `external: true`, telling Cilium to skip BPF processing entirely. Fixes inter-tenant connectivity via public LB IPs (Cilium's DNAT caused cross-tenant pod-to-pod flow classification, triggering CiliumClusterwideNetworkPolicy blocks) and restores WholeIP behavior on Cilium 1.19+ where wildcard service drop entries previously blocked traffic to LB IPs on undeclared ports *(backported to v1.2.2)* ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #2357). + +* **[monitoring] Fix infra dashboards missing in default variant**: Includes the `cozy-monitoring` namespace in the dashboard rendering condition, fixing infrastructure Grafana dashboards not rendering in the default platform variant (only the `tenant-root` namespace was previously checked) *(backported to v1.2.2)* ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #2365). + +* **[postgres] Fix system PostgreSQL images to 17.7-standard-trixie**: Normalizes system PostgreSQL image tags to the `17.7-standard-trixie` variant with migration logic for existing CNPG clusters, ensuring system databases use the correct image variant consistent with the monitoring stack requirements introduced in v1.2.1 *(backported to v1.2.2)* ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2364). + +* **[build] Filter git describe to match only v* tags**: Adds `--match 'v*'` to all `git describe` calls in `hack/common-envs.mk`, preventing the `api/apps/v1alpha1/vX.Y.Z` subtag from being picked up instead of the release tag and producing invalid Docker image tags *(backported to v1.2.2)* ([**@kvaps**](https://github.com/kvaps) in #2386). + +* **[platform] Fix resource allocation ratios not propagated to packages**: Restores propagation of `cpuAllocationRatio`, `memoryAllocationRatio`, and `ephemeralStorageAllocationRatio` from `platform/values.yaml` to the `cozystack-values` Secret that managed applications and KubeVirt read, fixing a regression introduced in the bundle restructure that silently ignored operator-configured ratios *(backported to v1.2.1)* ([**@sircthulhu**](https://github.com/sircthulhu) in #2296). + +* **[kubernetes] Set explicit ephemeral-storage on virt-launcher pods**: Sets explicit `domain.resources` ephemeral-storage on the VirtualMachine spec to prevent virt-launcher pods from being evicted because LimitRange defaults were too small for the actual emptyDisk capacity *(backported to v1.2.3)* ([**@kvaps**](https://github.com/kvaps) in #2317). + +* **[multus] Pin master CNI to 05-cilium.conflist**: Prevents a boot-time race where multus could auto-detect kube-ovn's conflist instead of Cilium's, which would cause pods to bypass the Cilium chain entirely and lose their endpoint *(backported to v1.2.1)* ([**@kvaps**](https://github.com/kvaps) in #2315). + +* **[multus] Build custom image with DEL cache fix**: Fixes sandbox cleanup deadlock when CNI ADD never completes, preventing stale sandbox name reservations from permanently blocking pod creation after a node disruption *(backported to v1.2.1)* ([**@kvaps**](https://github.com/kvaps) in #2313). + +* **[linstor] Set verify-alg to crc32c**: Prevents DRBD connection failures on kernels where `crct10dif` is unavailable (e.g., Talos v1.12.6 with kernel 6.18.18) by setting the LINSTOR verify-alg controller default to `crc32c` *(backported to v1.2.1)* ([**@kvaps**](https://github.com/kvaps) in #2303). + +* **[linstor] Preserve TCP ports during toggle-disk operations**: Saves existing TCP ports into the `LayerPayload` before `removeLayerData()` deletes them, preventing DRBD resources from entering StandAlone state when a satellite misses the resulting update *(backported to v1.2.1)* ([**@kvaps**](https://github.com/kvaps) in #2292). + +* **[linstor] Increase satellite startup probe failure threshold**: Raises the LINSTOR satellite `startupProbe` `failureThreshold` from 3 to 30 (30s → 300s) in the `LinstorSatelliteConfiguration` pod template, giving satellites with slow storage initialization enough time to come up without being killed and restarted ([**@Arsolitt**](https://github.com/Arsolitt) in #2425). + +## Security + +* **docs: add SECURITY.md**: Adds vulnerability reporting procedures, disclosure expectations, and supported release lines ([**@kvaps**](https://github.com/kvaps) in #2230). + +* **docs: add OpenSSF Best Practices badge to README**: Adds the OpenSSF Best Practices passing badge to the project README ([**@lexfrei**](https://github.com/lexfrei) in #2320). + +* **[linstor-gui] Restrict to cozystack-cluster-admin group**: Tightens access control on the `linstor-gui` Ingress so the UI and its underlying LINSTOR controller REST API are reachable only by members of the `cozystack-cluster-admin` Keycloak group. Previously, the oauth2-proxy gatekeeper enforced only realm membership (`--email-domain=*`), allowing any tenant-scoped account to reach the gatekeeper's static mTLS client cert *(backported to release-1.3 via #2419)* ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2415, #2419). + +## Dependencies & Version Updates + +* **[kube-ovn] Bump kube-ovn to v1.15.10 with port-group regression fix**: Updates `packages/system/kubeovn` to upstream v1.15.10 (from v1.15.3) and carries a patch for `pkg/controller/pod.go` that preserves a VM LSP's port-group memberships when Kubernetes GCs a completed virt-launcher pod while another virt-launcher pod of the same VM is still running. Without the patch, the destination pod of a successful live migration lost its security groups, network policies, and node-scoped routing until `kube-ovn-controller` was restarted ([**@kvaps**](https://github.com/kvaps) in #2443). + +* **[monitoring] Upgrade victoria-metrics-operator to v0.68.4**: Bumps the vendored `victoria-metrics-operator` Helm chart from 0.59.1 to 0.61.0 (operator appVersion v0.68.1 → v0.68.4), picking up upstream fixes for `VMPodScrape` port routing on VMAgent/VLAgent and `StatefulSet` pod deletion (not eviction) when `maxUnavailable=100%` ([**@lexfrei**](https://github.com/lexfrei) in #2426). + +* **[linstor] Update piraeus-server to v1.33.2 with selected backports**: Bumps LINSTOR server from v1.33.1 to v1.33.2 with backported patches for stale bitmap adjust retry, LUKS2 header sizing, optimal I/O size detection, and the maintainer implementation. All patches verified against upstream v1.33.2 with `git apply --check` and `gradlew compileJava` *(backported to v1.2.2)* ([**@kvaps**](https://github.com/kvaps) in #2331). + +* **[kamaji] Update to 26.3.5-edge, drop upstreamed patches**: Updates Kamaji from edge-26.2.4 to 26.3.5-edge and removes two patches accepted upstream. Adds configurable probe tuning and DataStore readiness conditions ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2260). + +* **[talm] Release v0.23.0, v0.23.1, v0.24.0** (github.com/cozystack/talm): Migrates to the Talos v1.12 multi-document machine config format ([**@lexfrei**](https://github.com/lexfrei) in cozystack/talm#116); renders templates online in `apply` to resolve lookups ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/talm#119); bumps dependencies and modernizes the codebase ([**@lexfrei**](https://github.com/lexfrei) in cozystack/talm#124). + +* **[ansible-cozystack] Release v1.2.1, v1.2.2, v1.2.4** (github.com/cozystack/ansible-cozystack): Exposes `publishing.externalIPs` and tenant-root ingress via role variables ([**@lexfrei**](https://github.com/lexfrei) in cozystack/ansible-cozystack#30); adds a comprehensive node prerequisites audit ([**@lexfrei**](https://github.com/lexfrei) in cozystack/ansible-cozystack#27); replaces `ansible.utils.ipaddr` with a stdlib-based test plugin ([**@lexfrei**](https://github.com/lexfrei) in cozystack/ansible-cozystack#24); adds `v` prefix to collection version in requirements.yml examples ([**@lexfrei**](https://github.com/lexfrei) in cozystack/ansible-cozystack#23); tracks installer releases v1.2.1 through v1.2.4 ([**@app/renovate**](https://github.com/apps/renovate) in cozystack/ansible-cozystack#20, #22, #29, #31, #32). + +## Development, Testing, and CI/CD + +* **[ci] Replace cozystack-bot PAT with cozystack-ci GitHub App**: Replaces the long-lived `cozystack-bot` personal access token with short-lived, scoped tokens from the `cozystack-ci` GitHub App across all release workflows (`tags.yaml`, `auto-release.yaml`, `pull-requests-release.yaml`), improving security and auditability of CI operations ([**@tym83**](https://github.com/tym83) in #2351; [**@kvaps**](https://github.com/kvaps) in #2383, #2392). + +* **[ci] Add Gemini Code Assist and CodeRabbit configuration**: Adds repository-level configuration for AI code reviewers with ignore patterns for vendored/generated code and incremental review settings ([**@lexfrei**](https://github.com/lexfrei) in #2385). + +* **[ci] Promote next/ trunk on new minor/major releases**: Updates `update-website-docs` in `tags.yaml` to match the new docs-versioning contract — the website repo replaces the old "pre-create `vX.Y/` draft directory" scheme with a permanent `content/en/docs/next/` trunk, and released version directories are promoted explicitly by the release workflow ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2433). + +* **[tests] Fix Kafka E2E test timeout and retry race condition**: Increases Kafka E2E test timeout from 60s to 300s and fixes a retry race where `kubectl apply` could hit a still-deleting resource ([**@lexfrei**](https://github.com/lexfrei) in #2358). + +* **docs: adopt Conventional Commits for commit and PR titles**: Standardizes commit and PR title format to `type(scope): description` across all contributing docs and the PR template ([**@lexfrei**](https://github.com/lexfrei) in #2395). + +* **docs(ci): require screenshots for UI changes in PR template**: Adds a mandatory screenshots section to the PR template for UI-related changes ([**@kitsunoff**](https://github.com/kitsunoff) in #2407). + +* **chore(maintenance): add @myasnikovdaniil to CODEOWNERS**: Adds @myasnikovdaniil to the default owners in `.github/CODEOWNERS` for automatic review requests ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2434). + +## Documentation + +* **[website] Add ApplicationDefinition naming convention reference**: Documents how `cozystack-api` resolves kinds to their backing definitions ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#478). + +* **[website] Document Talos / talosctl / Cozystack version pairing**: Adds a version compatibility matrix for installation ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#484). + +* **[website] Document namespace layout and parent/child derivation**: Explains tenant namespace hierarchy and parent/child namespace derivation rules ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#479). + +* **[website] Document the checkbox-then-edit-CR customization pattern for tenants**: Describes the workflow for customizing tenant settings via the CR after initial checkbox-based creation ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#485). + +* **[website] Add custom Keycloak themes documentation**: Covers the theme image contract, configuration, `imagePullSecrets`, and theme activation in the Keycloak admin console ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#463). + +* **[website] Add bonding (LACP) configuration how-to guide**: Covers network bonding configuration for Cozystack installations ([**@sircthulhu**](https://github.com/sircthulhu) in cozystack/website#459). + +* **[website] Improve registry mirrors for tenant Kubernetes in air-gapped guide**: Improves documentation for configuring registry mirrors in air-gapped environments ([**@sircthulhu**](https://github.com/sircthulhu) in cozystack/website#461). + +* **[website] Rewrite guide for ApplicationDefinition API (external-apps)**: Comprehensive rewrite of the external apps guide using the ApplicationDefinition API with Minecraft server examples ([**@kitsunoff**](https://github.com/kitsunoff) in cozystack/website#488). + +* **[website] Add documentation for Go types usage**: Guide for using generated Go types for Cozystack managed applications as a Go module ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#465). + +* **[website] Update backup/restore documentation for VMI/VMDisk**: Updates backup documentation with VM instance and VM disk restore improvements ([**@androndo**](https://github.com/androndo) in cozystack/website#466). + +* **[website] Refactor docs versions to major.minor variants**: Moves docs to major.minor versioning for the v1.x series ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#477). + +* **[website] Trunk-based versioning with permanent next/ directory**: Replaces the old "pre-create `vX.Y/` draft directory" scheme with a permanent `content/en/docs/next/` trunk; released version directories are promoted explicitly by `hack/release_next.sh` on new minor/major releases, and routing between `next/` and `vX.Y/` is Makefile-driven ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#495). + +* **[website] Add updated OpenAPI spec**: Updates the OpenAPI specification for managed applications reference ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#469). + +* **[website] Add OpenAPI spec download to GitHub Pages build**: Fixes the GitHub Pages build to include the OpenAPI spec download ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#494). + +* **[website] Add OSS Health pages and OpenSSF badge**: Adds OSS Health section with OpenSSF Scorecard and Best Practices badges to the website ([**@tym83**](https://github.com/tym83) in cozystack/website#470). + +* **[website] Add Telemetry page under OSS Health section**: Adds the Telemetry page with initial data seeding to the OSS Health docs ([**@tym83**](https://github.com/tym83) in cozystack/website#471, cozystack/website#504). + +* **[website] Blog: OSS Health section launch announcement**: Publishes the announcement blog post for the OSS Health section ([**@tym83**](https://github.com/tym83) in cozystack/website#474). + +* **[website] Fix OpenSSF canonical status URL**: Changes the OpenSSF canonical status URL from pt-BR to en ([**@tym83**](https://github.com/tym83) in cozystack/website#475). + +* **[website] Add CozySummit Virtual 2026 program announcement**: Publishes the CozySummit Virtual 2026 program announcement blog post ([**@tym83**](https://github.com/tym83) in cozystack/website#472). + +* **[website] Add missing release announcements for v0.1–v0.41**: Backfills missing release announcement blog posts for historical Cozystack versions ([**@tym83**](https://github.com/tym83) in cozystack/website#468). + +* **[website] Blog: managed PostgreSQL with synchronous replication**: Adds a post covering the managed PostgreSQL synchronous-replication feature ([**@tym83**](https://github.com/tym83) in cozystack/website#497). + +* **[website] Blog taxonomies and client-side filter UI**: Registers article-type and topic taxonomies and adds a client-side filter on the blog list page ([**@tym83**](https://github.com/tym83) in cozystack/website#499). + +* **[website] Add images frontmatter for social preview on existing posts**: Adds images frontmatter for social preview on existing blog posts ([**@tym83**](https://github.com/tym83) in cozystack/website#498). + +* **[website] Fix broken links and stale anchors across v1 docs**: Fixes 14 broken links and stale talm anchors ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#486). + +* **[website] Prefix bundle package names with cozystack. in v1 examples**: Corrects package naming in documentation examples ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#482). + +* **[website] Finish isolated-field removal and document opt-in policy labels**: Removes the obsolete `isolated` field from tenant documentation and documents the new opt-in policy labels approach ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#481). + +* **[website] Add --take-ownership flag and describe networking.* fields**: Documents the `--take-ownership` flag and `networking.*` fields in the installation guide ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#480). + +* **[website] Fix KubeOVN MASTER_NODES example path and key in troubleshooting**: Corrects the MASTER_NODES example path and key ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#483). + +* **[website] Add CLAUDE.md for AI agent guidance**: Adds a CLAUDE.md file describing the trunk-based docs architecture for AI agent guidance ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#489). + +* **[website] Update /docs/v1/ redirect to latest v1.2**: Updates the `/docs/v1/` redirect target to point to the latest v1.2 docs on GitHub Pages ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#492). + +* **[website] Remove nbykov from CODEOWNERS and CLAUDE.md**: Cleans up CODEOWNERS and CLAUDE.md entries ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#491). + +* **[website] Add Ahrefs Analytics tracker**: Adds the Ahrefs Analytics tracker to the website ([**@tym83**](https://github.com/tym83) in cozystack/website#503). + +* **[website] Add breathing room between navbar and hero on OSS Health**: Minor styling fix for the OSS Health section ([**@tym83**](https://github.com/tym83) in cozystack/website#500). + +* **[website] Fix og social badge image and title**: Updates the social badge image and title ([**@tym83**](https://github.com/tym83) in cozystack/website#487). + +* **[website] Update managed apps reference for v1.2.1**: Automated managed-apps reference update ([**@cozystack-bot**](https://github.com/cozystack-bot) in cozystack/website#464). + +* **[external-apps-example] Replace MongoDB example with Minecraft apps**: Refactors the external apps example to use the ApplicationDefinition API with Minecraft server applications ([**@lexfrei**](https://github.com/lexfrei) in cozystack/external-apps-example#2). + +* **docs: update README introductory description**: Refines the platform positioning and improves clarity on core capabilities in the main README ([**@tym83**](https://github.com/tym83) in #2409). + +## Governance + +* **Add Mattia Eleuteri ([@mattia-eleuteri](https://github.com/mattia-eleuteri)) as Maintainer**: CSI, Storage, Networking & Security ([**@tym83**](https://github.com/tym83) in #2345). + +* **Add Matthieu Robin ([@matthieu-robin](https://github.com/matthieu-robin)) as Maintainer**: Managed applications, platform quality, and benchmarking ([**@tym83**](https://github.com/tym83) in #2346). + +## Contributors + +We'd like to thank all contributors who made this release possible: + +* [**@androndo**](https://github.com/androndo) +* [**@Arsolitt**](https://github.com/Arsolitt) +* [**@BROngineer**](https://github.com/BROngineer) +* [**@IvanHunters**](https://github.com/IvanHunters) +* [**@kitsunoff**](https://github.com/kitsunoff) +* [**@kvaps**](https://github.com/kvaps) +* [**@lexfrei**](https://github.com/lexfrei) +* [**@lllamnyp**](https://github.com/lllamnyp) +* [**@mattia-eleuteri**](https://github.com/mattia-eleuteri) +* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) +* [**@sircthulhu**](https://github.com/sircthulhu) +* [**@tym83**](https://github.com/tym83) + +### New Contributors + +We're excited to welcome our first-time contributors: + +* [**@Arsolitt**](https://github.com/Arsolitt) — First contribution! + +--- + +**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.2.0...v1.3.0 From 44bc79cef115c02e0e586955c29ee184557c52f3 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Thu, 23 Apr 2026 10:17:30 +0500 Subject: [PATCH 158/250] docs(changelog): correct v1.3.0 postgres and linstor-gui entries Post-release cleanup of docs/changelogs/v1.3.0.md so the notes match what users actually experience in the released v1.3.0: - Rewrite the postgres major-features entry so author (myasnikovdaniil), PR (#2369), and description all match the 17.7-standard-trixie pin + migration-37 imageName rewrite that actually shipped. The previous entry credited #2304 (superseded spec.version=v17 backfill approach). - Remove the duplicate #2364 postgres bug-fix entry; the same work is now folded into the single major-features entry above, with backport references to #2309 (v1.2.1) and #2364 (v1.2.2). - Remove the [linstor-gui] Restrict to cozystack-cluster-admin group security entry. The vulnerable state never shipped in a tagged release, so there is nothing user-facing to announce; the restriction is already described in the linstor-gui Feature Highlights section as part of the feature's day-one behavior. Signed-off-by: Myasnikov Daniil --- docs/changelogs/v1.3.0.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docs/changelogs/v1.3.0.md b/docs/changelogs/v1.3.0.md index af7bb30e..60c800a8 100644 --- a/docs/changelogs/v1.3.0.md +++ b/docs/changelogs/v1.3.0.md @@ -50,7 +50,7 @@ The dashboard now ships a complete **RestoreJob experience**: list view, details * **[keycloak-configure] Add email verification and SMTP configuration**: Adds configurable Keycloak settings for user self-registration, email verification, and SMTP server configuration, enabling automated user onboarding flows ([**@BROngineer**](https://github.com/BROngineer) in #2318). -* **[postgres] Hardcode PostgreSQL 17 for monitoring databases**: Pins PostgreSQL 17.7 images for system databases (Grafana, Alerta, Harbor, Keycloak, SeaweedFS) and adds migration 37 to backfill `spec.version=v17` for existing PostgreSQL resources, preventing CNPG from defaulting to PostgreSQL 18 *(backported to v1.2.1)* ([**@IvanHunters**](https://github.com/IvanHunters) in #2304). +* **[postgres] Pin system PostgreSQL to 17.7-standard-trixie**: Pins the PostgreSQL image for system databases (Grafana, Alerta, Harbor, Keycloak, SeaweedFS) to `17.7-standard-trixie` across chart templates and `values.yaml`, and ships migration 37 to patch existing CNPG Cluster `imageName` fields to the same variant (handling unset, any PG 17 tag, and bare-version tags). This prevents CNPG from defaulting to PostgreSQL 18 and locks system databases to the trixie variant consistent with the monitoring stack requirements *(related backports shipped in v1.2.1 via #2309 and v1.2.2 via #2364)* ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2369). * **[platform] Prevent installed packages deletion**: Adds the `helm.sh/resource-policy: keep` annotation to platform packages so disabling a package no longer triggers automatic Helm deletion, restoring the documented behavior where operators must explicitly delete a package *(backported to v1.2.1)* ([**@kvaps**](https://github.com/kvaps) in #2273). @@ -76,8 +76,6 @@ The dashboard now ships a complete **RestoreJob experience**: list view, details * **[monitoring] Fix infra dashboards missing in default variant**: Includes the `cozy-monitoring` namespace in the dashboard rendering condition, fixing infrastructure Grafana dashboards not rendering in the default platform variant (only the `tenant-root` namespace was previously checked) *(backported to v1.2.2)* ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #2365). -* **[postgres] Fix system PostgreSQL images to 17.7-standard-trixie**: Normalizes system PostgreSQL image tags to the `17.7-standard-trixie` variant with migration logic for existing CNPG clusters, ensuring system databases use the correct image variant consistent with the monitoring stack requirements introduced in v1.2.1 *(backported to v1.2.2)* ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2364). - * **[build] Filter git describe to match only v* tags**: Adds `--match 'v*'` to all `git describe` calls in `hack/common-envs.mk`, preventing the `api/apps/v1alpha1/vX.Y.Z` subtag from being picked up instead of the release tag and producing invalid Docker image tags *(backported to v1.2.2)* ([**@kvaps**](https://github.com/kvaps) in #2386). * **[platform] Fix resource allocation ratios not propagated to packages**: Restores propagation of `cpuAllocationRatio`, `memoryAllocationRatio`, and `ephemeralStorageAllocationRatio` from `platform/values.yaml` to the `cozystack-values` Secret that managed applications and KubeVirt read, fixing a regression introduced in the bundle restructure that silently ignored operator-configured ratios *(backported to v1.2.1)* ([**@sircthulhu**](https://github.com/sircthulhu) in #2296). @@ -100,8 +98,6 @@ The dashboard now ships a complete **RestoreJob experience**: list view, details * **docs: add OpenSSF Best Practices badge to README**: Adds the OpenSSF Best Practices passing badge to the project README ([**@lexfrei**](https://github.com/lexfrei) in #2320). -* **[linstor-gui] Restrict to cozystack-cluster-admin group**: Tightens access control on the `linstor-gui` Ingress so the UI and its underlying LINSTOR controller REST API are reachable only by members of the `cozystack-cluster-admin` Keycloak group. Previously, the oauth2-proxy gatekeeper enforced only realm membership (`--email-domain=*`), allowing any tenant-scoped account to reach the gatekeeper's static mTLS client cert *(backported to release-1.3 via #2419)* ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2415, #2419). - ## Dependencies & Version Updates * **[kube-ovn] Bump kube-ovn to v1.15.10 with port-group regression fix**: Updates `packages/system/kubeovn` to upstream v1.15.10 (from v1.15.3) and carries a patch for `pkg/controller/pod.go` that preserves a VM LSP's port-group memberships when Kubernetes GCs a completed virt-launcher pod while another virt-launcher pod of the same VM is still running. Without the patch, the destination pod of a successful live migration lost its security groups, network policies, and node-scoped routing until `kube-ovn-controller` was restarted ([**@kvaps**](https://github.com/kvaps) in #2443). From e1c6f9c0299c36aae80e71ea7ca9da0eef050a58 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Thu, 23 Apr 2026 10:22:09 +0500 Subject: [PATCH 159/250] docs(agents): scope changelog.md to a file-only deliverable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The v1.3.0 release pipeline broke because Copilot, invoked by .github/workflows/tags.yaml with --allow-all-tools, committed the generated changelog onto HEAD of main on its own. The workflow's next step — `git checkout -b ... origin/main` — then wiped the file, and `git add` failed with a pathspec error. The root cause is in this document. The checklist ends with "Save the changelog", which an agent with broad tool access can reasonably interpret as "also commit it, push it, and open a PR". There was no explicit boundary. Add a "Scope and boundaries" section at the top and an explicit "then exit" at the end of Step 9: - The single deliverable is docs/changelogs/v.md. - Forbidden by default: git commit / push / checkout (to switch branches) / branch / tag / reset / merge / rebase; PR creation; GitHub API writes (POST/PATCH/DELETE); modifying any file other than the changelog. - Read-only analysis (git log/show/fetch/diff, gh pr view, gh api GET) remains expected. - Auxiliary repo clones under _repos/ remain allowed for cross-repo analysis per Step 6. - Scoped "unless the caller explicitly instructs otherwise" so interactive use with an IDE remains flexible. With the rules in the doc, CI and interactive callers share the same boundary; the workflow can invoke the doc with a one-line prompt instead of re-stating the constraints every time. Signed-off-by: Myasnikov Daniil --- docs/agents/changelog.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/agents/changelog.md b/docs/agents/changelog.md index 35d59a00..13a674a2 100644 --- a/docs/agents/changelog.md +++ b/docs/agents/changelog.md @@ -6,6 +6,20 @@ This file contains detailed instructions for AI-powered IDE on how to generate c Follow these instructions when the user explicitly asks to generate a changelog. +## Scope and boundaries + +**Your single deliverable is the file `docs/changelogs/v.md`.** Write the complete, verified changelog to that path. That is the entire task. Exit as soon as the file is written and verified against the checklist in Step 9. + +Unless the caller explicitly instructs otherwise: + +- **Do not** run `git commit`, `git push`, `git checkout` (to switch branches), `git branch`, `git tag`, `git reset`, `git merge`, `git rebase`, or any other command that writes to refs, HEAD, or remotes. +- **Do not** create pull requests, push branches, or issue GitHub API write calls (POST / PATCH / DELETE). +- In the cozystack working tree, the **only** file you create or modify is `docs/changelogs/v.md`. Cloning auxiliary repositories under `_repos/` for cross-repo analysis (see Step 6) is fine — that directory is outside the cozystack tree. + +The caller — a GitHub Actions workflow in CI, or a developer running you interactively — owns branching, committing, pushing, and PR creation. They will perform those actions after you exit. Do not pre-empt them even if the working tree looks ready. + +Read-only analysis is expected and encouraged: `git log`, `git show`, `git fetch`, `git diff`, `gh pr view`, `gh api` GET requests, and reading any file in the repository. + ## Required Tools Before generating changelogs, ensure you have access to `gh` (GitHub CLI) tool, which is used to fetch commit and PR author information. The GitHub CLI is used to correctly identify PR authors from commits and pull requests. @@ -608,6 +622,8 @@ Create a new changelog file in the format matching previous versions: **Save the changelog:** Save the changelog to file `docs/changelogs/v.md` according to the version for which the changelog is being generated. +**Then exit.** Do not commit, push, create a branch, or open a pull request — the caller handles all git and GitHub operations after you return. See the "Scope and boundaries" section at the top of this document. + ### Important notes - **After fetch with --force** local tags are up-to-date, use them for work From 3720f0f3f2306db9faa4bf1a68bf928e319a04a3 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Thu, 23 Apr 2026 10:22:26 +0500 Subject: [PATCH 160/250] fix(ci): harden tags.yaml changelog job against agent misbehavior Three changes to the generate-changelog job to fix the v1.3.0 release pipeline failure (run 24765377017) and make the job robust to whatever state the Copilot step leaves behind. 1. Add `timeout-minutes: 30` to the Generate changelog using AI step. On the v1.3.0 re-run the step hung silently for 10+ minutes; with no timeout a hung Copilot would hold a self-hosted runner for up to 6 hours (job default). The previous successful run took ~26 minutes, so 30 is a reasonable ceiling. 2. Replace the terse, ambiguous Copilot prompt with a one-liner that invokes docs/agents/changelog.md directly. The "Scope and boundaries" section added to that doc in the previous commit is now the single source of truth for what the agent may and may not do, so the workflow only needs to pass the version and point at the relevant doc. VERSION is moved to step env: to match GitHub's workflow-injection hardening guidance. 3. Rewrite the Create changelog branch and commit step: - add `set -euo pipefail` so any failure is visible - validate the file exists up front and fail loud with `::error::` if not - copy the file to a tempfile BEFORE `git checkout -b`, so the checkout to `origin/main` cannot remove it (this is the fix for the original pathspec error the v1.3.0 run hit) - use `trap` to clean up the tempfile on any exit path - move VERSION to env - drop the dead "no changes to commit" branch: the check_changelog step earlier in the job gates this step on the file being absent from origin/main, so `git add` + `git commit` must produce a diff. If they don't (e.g. Copilot emitted an empty file), fail loud instead of pushing an empty branch. Signed-off-by: Myasnikov Daniil --- .github/workflows/tags.yaml | 67 ++++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 31 deletions(-) diff --git a/.github/workflows/tags.yaml b/.github/workflows/tags.yaml index b10897cf..0ec0217d 100644 --- a/.github/workflows/tags.yaml +++ b/.github/workflows/tags.yaml @@ -303,51 +303,56 @@ jobs: - name: Generate changelog using AI if: steps.check_changelog.outputs.exists == 'false' + timeout-minutes: 30 env: COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} GH_TOKEN: ${{ steps.app-token.outputs.token }} + VERSION: ${{ steps.tag.outputs.version }} run: | - copilot --prompt "prepare changelog file for tagged release v${{ steps.tag.outputs.version }}, use @docs/agents/changelog.md for it. Create the changelog file at docs/changelogs/v${{ steps.tag.outputs.version }}.md" \ + copilot \ + --prompt "Generate the release changelog for tag v${VERSION}. Follow the instructions in @docs/agents/changelog.md exactly, including the 'Scope and boundaries' section at the top. Your deliverable is the single file docs/changelogs/v${VERSION}.md — write it and exit; this workflow handles branching, committing, pushing, and opening the PR." \ --allow-all-tools --allow-all-paths < /dev/null - name: Create changelog branch and commit if: steps.check_changelog.outputs.exists == 'false' env: APP_TOKEN: ${{ steps.app-token.outputs.token }} + VERSION: ${{ steps.tag.outputs.version }} run: | - git config user.name "cozystack-ci[bot]" - git config user.email "274107086+cozystack-ci[bot]@users.noreply.github.com" - git remote set-url origin https://x-access-token:${APP_TOKEN}@github.com/${GITHUB_REPOSITORY} - - CHANGELOG_FILE="docs/changelogs/v${{ steps.tag.outputs.version }}.md" - CHANGELOG_BRANCH="changelog-v${{ steps.tag.outputs.version }}" - - if [ -f "$CHANGELOG_FILE" ]; then - # Fetch latest main branch - git fetch origin main - - # Delete local branch if it exists - git branch -D "$CHANGELOG_BRANCH" 2>/dev/null || true - - # Create and checkout new branch from main - git checkout -b "$CHANGELOG_BRANCH" origin/main - - # Add and commit changelog - git add "$CHANGELOG_FILE" - if git diff --staged --quiet; then - echo "⚠️ No changes to commit (file may already be committed)" - else - git commit -m "docs: add changelog for v${{ steps.tag.outputs.version }}" -s - echo "✅ Changelog committed to branch $CHANGELOG_BRANCH" - fi - - # Push the branch (force push to update if it exists) - git push -f origin "$CHANGELOG_BRANCH" - else - echo "⚠️ Changelog file was not generated" + set -euo pipefail + + CHANGELOG_FILE="docs/changelogs/v${VERSION}.md" + CHANGELOG_BRANCH="changelog-v${VERSION}" + + if [ ! -f "$CHANGELOG_FILE" ]; then + echo "::error::Changelog file $CHANGELOG_FILE was not produced by the Generate changelog using AI step" exit 1 fi + # Snapshot the file across the branch switch — the checkout below + # resets tracked files to match origin/main. + TEMP_FILE="$(mktemp)" + trap 'rm -f "$TEMP_FILE"' EXIT + cp "$CHANGELOG_FILE" "$TEMP_FILE" + + git config user.name "cozystack-ci[bot]" + git config user.email "274107086+cozystack-ci[bot]@users.noreply.github.com" + git remote set-url origin "https://x-access-token:${APP_TOKEN}@github.com/${GITHUB_REPOSITORY}" + + git fetch origin main + git branch -D "$CHANGELOG_BRANCH" 2>/dev/null || true + git checkout -b "$CHANGELOG_BRANCH" origin/main + + mkdir -p "$(dirname "$CHANGELOG_FILE")" + cp "$TEMP_FILE" "$CHANGELOG_FILE" + + # The `check_changelog` step gated this job on the file being absent + # from origin/main, so `git add` + `git commit` must produce a diff. + # If they don't, something is wrong (e.g. empty file) — fail loud. + git add "$CHANGELOG_FILE" + git commit -m "docs: add changelog for v${VERSION}" -s + git push -f origin "$CHANGELOG_BRANCH" + - name: Create PR for changelog if: steps.check_changelog.outputs.exists == 'false' uses: actions/github-script@v7 From c4477259c7e914accaf45238a2b2b591451672a9 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Thu, 23 Apr 2026 10:41:41 +0500 Subject: [PATCH 161/250] fix(backups): move velero-configmap Role to velero chart The backupstrategy-controller chart declared a Role/RoleBinding in the cozy-velero namespace for ResourceModifier ConfigMap management. Because velero is an optional package, that namespace does not exist in bundles without velero, so Helm install aborted with "namespaces \"cozy-velero\" not found" and blocked the default install of backupstrategy-controller. Move the Role and RoleBinding into the velero chart so they are created only when velero is actually installed. The RoleBinding subject points to the backupstrategy-controller ServiceAccount in its fixed namespace (cozy-backup-controller). Assisted-By: Claude Signed-off-by: Myasnikov Daniil --- .../templates/rbac-bind.yaml | 14 ---------- .../templates/rbac.yaml | 15 +++-------- .../backupstrategy-controller-rbac.yaml | 26 +++++++++++++++++++ 3 files changed, 29 insertions(+), 26 deletions(-) create mode 100644 packages/system/velero/templates/backupstrategy-controller-rbac.yaml diff --git a/packages/system/backupstrategy-controller/templates/rbac-bind.yaml b/packages/system/backupstrategy-controller/templates/rbac-bind.yaml index 4ecacce0..03578cc2 100644 --- a/packages/system/backupstrategy-controller/templates/rbac-bind.yaml +++ b/packages/system/backupstrategy-controller/templates/rbac-bind.yaml @@ -10,17 +10,3 @@ subjects: - kind: ServiceAccount name: backupstrategy-controller namespace: {{ .Release.Namespace }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: backups.cozystack.io:strategy-controller:velero-configmaps - namespace: cozy-velero -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: backups.cozystack.io:strategy-controller:velero-configmaps -subjects: -- kind: ServiceAccount - name: backupstrategy-controller - namespace: {{ .Release.Namespace }} diff --git a/packages/system/backupstrategy-controller/templates/rbac.yaml b/packages/system/backupstrategy-controller/templates/rbac.yaml index bbdb74f8..634ea88b 100644 --- a/packages/system/backupstrategy-controller/templates/rbac.yaml +++ b/packages/system/backupstrategy-controller/templates/rbac.yaml @@ -27,7 +27,9 @@ rules: resources: ["pods"] verbs: ["get", "list", "watch"] # ConfigMaps: controller-runtime cache requires cluster-scoped list/watch; -# create/update/delete is scoped to cozy-velero via the Role below. +# create/update/delete is scoped to cozy-velero via a Role shipped by the +# velero chart (packages/system/velero/templates/backupstrategy-controller-rbac.yaml) +# so it is only created when velero is installed. - apiGroups: [""] resources: ["configmaps"] verbs: ["get", "list", "watch"] @@ -78,14 +80,3 @@ rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "list", "watch", "create", "update", "patch"] ---- -# To create ResourceModifiers in ConfigMaps for Restore in Velero install namespace. -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: backups.cozystack.io:strategy-controller:velero-configmaps - namespace: cozy-velero -rules: -- apiGroups: [""] - resources: ["configmaps"] - verbs: ["create", "get", "list", "delete", "deletecollection", "patch", "update"] diff --git a/packages/system/velero/templates/backupstrategy-controller-rbac.yaml b/packages/system/velero/templates/backupstrategy-controller-rbac.yaml new file mode 100644 index 00000000..43c51cd7 --- /dev/null +++ b/packages/system/velero/templates/backupstrategy-controller-rbac.yaml @@ -0,0 +1,26 @@ +# Grants the backupstrategy-controller permission to manage ResourceModifier +# ConfigMaps in the Velero install namespace. Lives here (not in the +# backupstrategy-controller chart) so that the Role is only created when +# velero is actually installed — otherwise the chart would try to create it +# in a namespace that does not exist. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: backups.cozystack.io:strategy-controller:velero-configmaps +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["create", "get", "list", "delete", "deletecollection", "patch", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: backups.cozystack.io:strategy-controller:velero-configmaps +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: backups.cozystack.io:strategy-controller:velero-configmaps +subjects: +- kind: ServiceAccount + name: backupstrategy-controller + namespace: cozy-backup-controller From 3c95f3052161372cc0c53c4b7119b56118d19846 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Thu, 23 Apr 2026 12:25:19 +0500 Subject: [PATCH 162/250] docs(agents): scope git-write ban to cozystack working tree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback on PR #2460: the previous "do not write to refs, HEAD, or remotes" wording contradicted the explicit allowance of `git fetch` (which updates remote-tracking refs) and the mandatory cross-repo checks in Step 6, which `cd` into `_repos/` and run `git checkout`, `git pull`, etc. Sharpen the scope paragraph: - The git-write ban is now explicitly scoped to the cozystack working tree — it bans writing to local branches, tags, or HEAD in that repo, not "refs/HEAD/remotes" globally. - `git fetch` is called out as expected. - Local git operations inside disposable `_repos/` clones (`git checkout`, `git pull`, etc.) are explicitly allowed, with the remaining rules (no push, no PR creation, no API writes) applying to any repository. Signed-off-by: Myasnikov Daniil --- docs/agents/changelog.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/agents/changelog.md b/docs/agents/changelog.md index 13a674a2..2fde1224 100644 --- a/docs/agents/changelog.md +++ b/docs/agents/changelog.md @@ -12,9 +12,9 @@ Follow these instructions when the user explicitly asks to generate a changelog. Unless the caller explicitly instructs otherwise: -- **Do not** run `git commit`, `git push`, `git checkout` (to switch branches), `git branch`, `git tag`, `git reset`, `git merge`, `git rebase`, or any other command that writes to refs, HEAD, or remotes. -- **Do not** create pull requests, push branches, or issue GitHub API write calls (POST / PATCH / DELETE). -- In the cozystack working tree, the **only** file you create or modify is `docs/changelogs/v.md`. Cloning auxiliary repositories under `_repos/` for cross-repo analysis (see Step 6) is fine — that directory is outside the cozystack tree. +- **In the cozystack working tree**, do not run `git commit`, `git push`, `git checkout` (to switch branches), `git branch`, `git tag`, `git reset`, `git merge`, or `git rebase`. Do not write to local branches, tags, or HEAD. `git fetch` is expected and fine (see the read-only analysis list below). +- **Do not** push to any remote, open pull requests, or issue GitHub API write calls (POST / PATCH / DELETE) for any repository. +- In the cozystack working tree, the **only** file you create or modify is `docs/changelogs/v.md`. Cloning auxiliary repositories under `_repos/` for cross-repo analysis (see Step 6) is fine; local git operations inside those disposable clones (`git checkout`, `git pull`, etc.) are allowed — just never push from them or open PRs against them. The caller — a GitHub Actions workflow in CI, or a developer running you interactively — owns branching, committing, pushing, and PR creation. They will perform those actions after you exit. Do not pre-empt them even if the working tree looks ready. From e7e83b0d0b299f99c4f8881db5e8474dd0afd25e Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Thu, 23 Apr 2026 12:25:30 +0500 Subject: [PATCH 163/250] fix(ci): reject empty changelog file before commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback on PR #2460: the existing `[ -f ]` check catches missing files but a zero-byte `docs/changelogs/v${VERSION}.md` would still be staged and committed — `git add` + `git commit -s` on a new empty file succeeds and produces a real commit, leaving the downstream PR with no actual changelog content. Add a `[ -s ]` guard after the existence check: if the Generate changelog using AI step produces an empty file, emit a matching `::error::` annotation and exit 1 before snapshotting. Signed-off-by: Myasnikov Daniil --- .github/workflows/tags.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/tags.yaml b/.github/workflows/tags.yaml index 0ec0217d..ae77d015 100644 --- a/.github/workflows/tags.yaml +++ b/.github/workflows/tags.yaml @@ -328,6 +328,10 @@ jobs: echo "::error::Changelog file $CHANGELOG_FILE was not produced by the Generate changelog using AI step" exit 1 fi + if [ ! -s "$CHANGELOG_FILE" ]; then + echo "::error::Changelog file $CHANGELOG_FILE is empty" + exit 1 + fi # Snapshot the file across the branch switch — the checkout below # resets tracked files to match origin/main. From 76c4eabdff8df513cca04c2313928c707df1c6ca Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Thu, 23 Apr 2026 12:26:14 +0500 Subject: [PATCH 164/250] fix(ci): use a read-only app token for the Copilot step MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback on PR #2460: the Generate changelog using AI step ran Copilot with --allow-all-tools and GH_TOKEN set to the write-capable installation token issued to the job (contents: write, pull-requests: write on all cozystack/* repos). The scope rules in docs/agents/changelog.md and the step prompt tell the agent not to use those permissions, but nothing at the token layer prevented it. Mint a second, read-only installation token from the same app (same COZYSTACK_CI_APP_ID / COZYSTACK_CI_PRIVATE_KEY, scoped to contents/pull-requests/metadata read) and pass that one to the AI step instead. The write-capable token is still used by the checkout, commit/push, and PR-creation steps that actually need it. This is defense in depth: even if a future prompt change or agent misbehavior ignored the scope rules, the token itself has no write capability on any repository in the cozystack org. No new secret, no new GitHub App install, no admin-side change — the RO token is minted in the same workflow from the same app credentials. Signed-off-by: Myasnikov Daniil --- .github/workflows/tags.yaml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tags.yaml b/.github/workflows/tags.yaml index ae77d015..90f04829 100644 --- a/.github/workflows/tags.yaml +++ b/.github/workflows/tags.yaml @@ -255,6 +255,21 @@ jobs: private-key: ${{ secrets.COZYSTACK_CI_PRIVATE_KEY }} owner: cozystack + # Read-only token for the AI step. Minting a separate scoped token + # means the Generate changelog using AI step cannot push branches, + # open PRs, or mutate any repository even with --allow-all-tools, + # regardless of whether the agent follows the prompt's instructions. + - name: Generate read-only GitHub App token + id: app-token-read + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ secrets.COZYSTACK_CI_APP_ID }} + private-key: ${{ secrets.COZYSTACK_CI_PRIVATE_KEY }} + owner: cozystack + permission-contents: read + permission-pull-requests: read + permission-metadata: read + - name: Parse tag id: tag uses: actions/github-script@v7 @@ -306,7 +321,7 @@ jobs: timeout-minutes: 30 env: COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} - GH_TOKEN: ${{ steps.app-token.outputs.token }} + GH_TOKEN: ${{ steps.app-token-read.outputs.token }} VERSION: ${{ steps.tag.outputs.version }} run: | copilot \ From c1508940bda27b2979639981fd25461c9d4824b3 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Thu, 23 Apr 2026 13:34:21 +0500 Subject: [PATCH 165/250] fix(etcd): remove destructive post-upgrade cert-regeneration hook The etcd chart shipped a `post-upgrade` Helm hook that `kubectl delete`d the etcd TLS chain (`etcd-{ca,peer-ca,client,peer,server}-tls`) and then deleted etcd pods on every chart upgrade, gated by a semver compare of an `etcd-deployed-version` ConfigMap against `2.6.1`. The hook was added as a one-shot migration for the chart `2.6.0 -> 2.6.1` transition. Since commit f871fbdb ("Remove versions_map logic") all chart versions are stamped as `0.0.0+`, which per semver is always `< 2.6.1`. The gate therefore always resolves to "update certs", firing the destructive hook on every etcd upgrade. On clusters running Kamaji-managed tenant control planes this wipes the etcd CA, cert-manager re-issues it, and tenant kube-apiservers hit `x509: certificate signed by unknown authority` against `etcd..svc:2379` until each tenant DataStore is manually re-reconciled. Commit 47d81f70 ("Disabled private key rotation in CA certs") already fixed the underlying `rotationPolicy: Always` issue the migration was papering over, so the hook has no remaining use. Remove the hook Job, its RBAC, the version ConfigMap it read, and add a helm-unittest suite under `packages/extra/etcd/tests/` that guards against re-introducing the hook or the version ConfigMap. Assisted-By: Claude Signed-off-by: Myasnikov Daniil --- packages/extra/etcd/Makefile | 3 ++ packages/extra/etcd/templates/hook/job.yaml | 39 ------------------- packages/extra/etcd/templates/hook/role.yaml | 26 ------------- .../etcd/templates/hook/rolebinding.yaml | 15 ------- .../etcd/templates/hook/serviceaccount.yaml | 7 ---- packages/extra/etcd/templates/version.yaml | 6 --- .../etcd/tests/no-post-upgrade-hook_test.yaml | 34 ++++++++++++++++ 7 files changed, 37 insertions(+), 93 deletions(-) delete mode 100644 packages/extra/etcd/templates/hook/job.yaml delete mode 100644 packages/extra/etcd/templates/hook/role.yaml delete mode 100644 packages/extra/etcd/templates/hook/rolebinding.yaml delete mode 100644 packages/extra/etcd/templates/hook/serviceaccount.yaml delete mode 100644 packages/extra/etcd/templates/version.yaml create mode 100644 packages/extra/etcd/tests/no-post-upgrade-hook_test.yaml diff --git a/packages/extra/etcd/Makefile b/packages/extra/etcd/Makefile index f37d6e1e..2b3ed61e 100644 --- a/packages/extra/etcd/Makefile +++ b/packages/extra/etcd/Makefile @@ -5,3 +5,6 @@ include ../../../hack/package.mk generate: cozyvalues-gen -v values.yaml -s values.schema.json -r README.md ../../../hack/update-crd.sh + +test: + helm unittest . diff --git a/packages/extra/etcd/templates/hook/job.yaml b/packages/extra/etcd/templates/hook/job.yaml deleted file mode 100644 index 3bf2f84b..00000000 --- a/packages/extra/etcd/templates/hook/job.yaml +++ /dev/null @@ -1,39 +0,0 @@ -{{- $shouldUpdateCerts := true }} -{{- $configMap := lookup "v1" "ConfigMap" .Release.Namespace "etcd-deployed-version" }} -{{- if $configMap }} - {{- $deployedVersion := index $configMap "data" "version" }} - {{- if $deployedVersion | semverCompare ">= 2.6.1" }} - {{- $shouldUpdateCerts = false }} - {{- end }} -{{- end }} - -{{- if $shouldUpdateCerts }} ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: etcd-hook - annotations: - helm.sh/hook: post-upgrade - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded -spec: - template: - metadata: - labels: - policy.cozystack.io/allow-to-apiserver: "true" - spec: - serviceAccountName: etcd-hook - containers: - - name: kubectl - image: docker.io/alpine/k8s:1.33.4 - command: - - sh - args: - - -exc - - |- - kubectl --namespace={{ .Release.Namespace }} delete secrets etcd-ca-tls etcd-peer-ca-tls - sleep 10 - kubectl --namespace={{ .Release.Namespace }} delete secrets etcd-client-tls etcd-peer-tls etcd-server-tls - kubectl --namespace={{ .Release.Namespace }} delete pods --selector=app.kubernetes.io/instance=etcd,app.kubernetes.io/managed-by=etcd-operator,app.kubernetes.io/name=etcd,cozystack.io/service=etcd - restartPolicy: Never -{{- end }} diff --git a/packages/extra/etcd/templates/hook/role.yaml b/packages/extra/etcd/templates/hook/role.yaml deleted file mode 100644 index 327eeadb..00000000 --- a/packages/extra/etcd/templates/hook/role.yaml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - annotations: - helm.sh/hook: post-upgrade - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded - name: etcd-hook -rules: -- apiGroups: - - "" - resources: - - secrets - - pods - verbs: - - get - - list - - watch - - delete -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch diff --git a/packages/extra/etcd/templates/hook/rolebinding.yaml b/packages/extra/etcd/templates/hook/rolebinding.yaml deleted file mode 100644 index 0ee0ffd1..00000000 --- a/packages/extra/etcd/templates/hook/rolebinding.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: etcd-hook - annotations: - helm.sh/hook: post-upgrade - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: etcd-hook -subjects: - - kind: ServiceAccount - name: etcd-hook - namespace: {{ .Release.Namespace | quote }} diff --git a/packages/extra/etcd/templates/hook/serviceaccount.yaml b/packages/extra/etcd/templates/hook/serviceaccount.yaml deleted file mode 100644 index 552fb5fc..00000000 --- a/packages/extra/etcd/templates/hook/serviceaccount.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: etcd-hook - annotations: - helm.sh/hook: post-upgrade - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded diff --git a/packages/extra/etcd/templates/version.yaml b/packages/extra/etcd/templates/version.yaml deleted file mode 100644 index cc9375bb..00000000 --- a/packages/extra/etcd/templates/version.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: etcd-deployed-version -data: - version: {{ .Chart.Version }} diff --git a/packages/extra/etcd/tests/no-post-upgrade-hook_test.yaml b/packages/extra/etcd/tests/no-post-upgrade-hook_test.yaml new file mode 100644 index 00000000..0e4f5aa3 --- /dev/null +++ b/packages/extra/etcd/tests/no-post-upgrade-hook_test.yaml @@ -0,0 +1,34 @@ +suite: etcd chart does not ship a destructive post-upgrade cert-regeneration hook + +release: + name: etcd + namespace: tenant-root + +templates: + - templates/check-release-name.yaml + - templates/dashboard-resourcemap.yaml + - templates/datastore.yaml + - templates/etcd-defrag.yaml + - templates/hook/job.yaml + - templates/podscrape.yaml + - templates/prometheus-rules.yaml + - templates/version.yaml + +tests: + - it: renders no Job named etcd-hook + documentSelector: + path: metadata.name + value: etcd-hook + skipEmptyTemplates: true + asserts: + - hasDocuments: + count: 0 + + - it: renders no ConfigMap named etcd-deployed-version + documentSelector: + path: metadata.name + value: etcd-deployed-version + skipEmptyTemplates: true + asserts: + - hasDocuments: + count: 0 From 9222b6feda4d97eda7481569d5d9a466b69fec94 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Thu, 23 Apr 2026 14:32:20 +0500 Subject: [PATCH 166/250] ci(api): pre-fetch k8s.io/code-generator in codegen drift job hack/update-codegen.sh sources kube_codegen.sh from the Go module cache at ~/go/pkg/mod/k8s.io/code-generator@vX.Y.Z/, but the module is not declared in go.mod so a fresh runner has nothing to source from. Add a workflow step that parses the pinned version out of the script and pulls the module into the cache before running make generate. Assisted-By: Claude Signed-off-by: Myasnikov Daniil --- .github/workflows/codegen-drift.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/codegen-drift.yml b/.github/workflows/codegen-drift.yml index bd7927f7..b08f5d7e 100644 --- a/.github/workflows/codegen-drift.yml +++ b/.github/workflows/codegen-drift.yml @@ -30,6 +30,17 @@ jobs: go-version-file: go.mod cache: true + - name: Pre-fetch k8s.io/code-generator module + # hack/update-codegen.sh sources kube_codegen.sh from the Go module cache. + # The module is not declared in go.mod, so fetch it explicitly at the + # version pinned in the script. + run: | + version=$(grep -oP 'code-generator@\Kv[0-9.]+' hack/update-codegen.sh) + tmpdir=$(mktemp -d) + cd "$tmpdir" + go mod init codegen-fetch + go get "k8s.io/code-generator@${version}" + - name: Run make generate run: make generate From 0baa93006fcec09fb6245a4ccd8ec8a047448587 Mon Sep 17 00:00:00 2001 From: Kirill Ilin Date: Thu, 23 Apr 2026 16:57:23 +0500 Subject: [PATCH 167/250] chore(hetzner-robotlb): update robotlb chart to appVersion 0.0.6 Pulls the latest robotlb chart (0.1.3) which ships robotlb 0.0.6. The new appVersion adds RBAC permissions for discovery.k8s.io/endpointslices needed to support EndpointSlice-based services such as KubeVirt. Assisted-By: Claude AI Signed-off-by: Kirill Ilin --- packages/system/hetzner-robotlb/charts/robotlb/Chart.yaml | 2 +- .../hetzner-robotlb/charts/robotlb/templates/deployment.yaml | 2 +- .../system/hetzner-robotlb/charts/robotlb/templates/role.yaml | 3 ++- packages/system/hetzner-robotlb/charts/robotlb/values.yaml | 3 +++ 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/system/hetzner-robotlb/charts/robotlb/Chart.yaml b/packages/system/hetzner-robotlb/charts/robotlb/Chart.yaml index 743f255d..192470dd 100644 --- a/packages/system/hetzner-robotlb/charts/robotlb/Chart.yaml +++ b/packages/system/hetzner-robotlb/charts/robotlb/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 0.0.5 +appVersion: 0.0.6 description: A Helm chart for robotlb (loadbalancer on hetzner cloud). name: robotlb type: application diff --git a/packages/system/hetzner-robotlb/charts/robotlb/templates/deployment.yaml b/packages/system/hetzner-robotlb/charts/robotlb/templates/deployment.yaml index 4fd71366..41b4661a 100644 --- a/packages/system/hetzner-robotlb/charts/robotlb/templates/deployment.yaml +++ b/packages/system/hetzner-robotlb/charts/robotlb/templates/deployment.yaml @@ -5,7 +5,7 @@ metadata: labels: {{- include "robotlb.labels" . | nindent 4 }} spec: - replicas: {{ .Values.replicas }} + replicas: 1 selector: matchLabels: {{- include "robotlb.selectorLabels" . | nindent 6 }} diff --git a/packages/system/hetzner-robotlb/charts/robotlb/templates/role.yaml b/packages/system/hetzner-robotlb/charts/robotlb/templates/role.yaml index 3a7b9334..76bac249 100644 --- a/packages/system/hetzner-robotlb/charts/robotlb/templates/role.yaml +++ b/packages/system/hetzner-robotlb/charts/robotlb/templates/role.yaml @@ -3,7 +3,8 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: {{ include "robotlb.fullname" . }}-cr -rules: {{- toYaml .Values.serviceAccount.permissions | nindent 2 }} +rules: + {{- toYaml .Values.serviceAccount.permissions | nindent 2 }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/packages/system/hetzner-robotlb/charts/robotlb/values.yaml b/packages/system/hetzner-robotlb/charts/robotlb/values.yaml index 739c7b73..4365f677 100644 --- a/packages/system/hetzner-robotlb/charts/robotlb/values.yaml +++ b/packages/system/hetzner-robotlb/charts/robotlb/values.yaml @@ -36,6 +36,9 @@ serviceAccount: - apiGroups: [""] resources: [nodes, pods] verbs: [get, list, watch] + - apiGroups: [discovery.k8s.io] + resources: [endpointslices] + verbs: [get, list, watch] podAnnotations: {} podLabels: {} From ecd2ead5defaf25001cbd56e5878b9f4d2c2d27b Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 23 Apr 2026 15:55:49 +0300 Subject: [PATCH 168/250] docs(agents): document make generate requirement before committing Pre-commit CI runs make generate in every package and fails with exit 123 on any uncommitted generator output. Add explicit guidance so agents stage regenerated README.md, values.schema.json and packages/system/-rd artifacts alongside the hand edits. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- docs/agents/contributing.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/agents/contributing.md b/docs/agents/contributing.md index 68a584ee..9626e81e 100644 --- a/docs/agents/contributing.md +++ b/docs/agents/contributing.md @@ -8,6 +8,30 @@ Project-side conventions for commits, branches, and pull requests in Cozystack. - [ ] Commit is signed off with `--signoff` - [ ] Branch is rebased on `upstream/main` (no extra commits) - [ ] PR body includes description and release note +- [ ] Ran `make generate` in every package whose `values.yaml`, `values.schema.json`, `Chart.yaml`, or `README.md` was touched, and committed the regenerated files + +## Regenerate Artifacts Before Committing + +Several files in each package are produced by `make generate` from `values.yaml` + `values.schema.json` and must stay in sync with the hand-edited sources: + +- `packages/(apps|extra)//README.md` — regenerated by `cozyvalues-gen` (parameter table, formatting). +- `packages/(apps|extra)//values.schema.json` — `cozyvalues-gen` rewrites ordering and derived fields. +- `packages/system/-rd/cozyrds/.yaml` — produced by `hack/update-crd.sh`, which `make generate` invokes. + +**Before committing edits to any of those sources**, run `make generate` inside the package and stage the full diff: + +```bash +make -C packages/extra/ generate +git add packages/extra// packages/system/-rd/ +``` + +The repo's pre-commit CI job runs `make generate` in every package and then `git diff --exit-code`. Any unstaged generator output fails the job with exit code 123 and blocks the PR. Also rerun `make generate` after a `git commit --amend` if the amended change touched any of the sources above. + +To locate packages a WIP branch is likely to need regenerated: + +```bash +git diff --name-only | xargs -n1 dirname | sort -u | grep ^packages/ +``` ## Commit Format From 0e4b66a70f036d446c5d61dfc92174ff6eae34ac Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 23 Apr 2026 13:36:04 +0300 Subject: [PATCH 169/250] chore(cilium): bump to v1.19.3 Vendored chart refreshed via make update in packages/system/cilium. Motivation: v1.19.2 fixes a cert-manager HTTP-01 bug on hostnames with both HTTP and HTTPS listeners (cilium#44492, backport PR #44517). This is a prerequisite for upcoming Gateway API work. v1.19.3 is the latest stable release in the v1.19.x line (15 Apr 2026). Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- .../system/cilium/charts/cilium/Chart.yaml | 4 +- .../system/cilium/charts/cilium/README.md | 50 +++- .../configmap/bootstrap-config.yaml | 9 + .../templates/cilium-agent/daemonset.yaml | 6 +- .../cilium/templates/cilium-configmap.yaml | 20 +- .../cilium-operator/clusterrole.yaml | 9 + .../cilium/templates/ztunnel/daemonset.yaml | 165 +++++++++++++ .../cilium/templates/ztunnel/secret.yaml | 23 ++ .../templates/ztunnel/serviceaccount.yaml | 21 ++ .../cilium/charts/cilium/values.schema.json | 219 +++++++++++++++++- .../system/cilium/charts/cilium/values.yaml | 147 ++++++++++-- .../cilium/charts/cilium/values.yaml.tmpl | 106 ++++++++- .../system/cilium/images/cilium/Dockerfile | 2 +- 13 files changed, 726 insertions(+), 55 deletions(-) create mode 100644 packages/system/cilium/charts/cilium/templates/ztunnel/daemonset.yaml create mode 100644 packages/system/cilium/charts/cilium/templates/ztunnel/secret.yaml create mode 100644 packages/system/cilium/charts/cilium/templates/ztunnel/serviceaccount.yaml diff --git a/packages/system/cilium/charts/cilium/Chart.yaml b/packages/system/cilium/charts/cilium/Chart.yaml index 0bb34451..5fe2baed 100644 --- a/packages/system/cilium/charts/cilium/Chart.yaml +++ b/packages/system/cilium/charts/cilium/Chart.yaml @@ -76,7 +76,7 @@ annotations: Cilium Gateway Class Config\n description: |\n CiliumGatewayClassConfig defines a configuration for Gateway API GatewayClass.\n" apiVersion: v2 -appVersion: 1.19.1 +appVersion: 1.19.3 description: eBPF-based Networking, Security, and Observability home: https://cilium.io/ icon: https://cdn.jsdelivr.net/gh/cilium/cilium@main/Documentation/images/logo-solo.svg @@ -92,4 +92,4 @@ kubeVersion: '>= 1.21.0-0' name: cilium sources: - https://github.com/cilium/cilium -version: 1.19.1 +version: 1.19.3 diff --git a/packages/system/cilium/charts/cilium/README.md b/packages/system/cilium/charts/cilium/README.md index fe8aea3f..d7697c56 100644 --- a/packages/system/cilium/charts/cilium/README.md +++ b/packages/system/cilium/charts/cilium/README.md @@ -1,6 +1,6 @@ # cilium -![Version: 1.19.1](https://img.shields.io/badge/Version-1.19.1-informational?style=flat-square) ![AppVersion: 1.19.1](https://img.shields.io/badge/AppVersion-1.19.1-informational?style=flat-square) +![Version: 1.19.3](https://img.shields.io/badge/Version-1.19.3-informational?style=flat-square) ![AppVersion: 1.19.3](https://img.shields.io/badge/AppVersion-1.19.3-informational?style=flat-square) Cilium is open source software for providing and transparently securing network connectivity and loadbalancing between application workloads such as @@ -89,7 +89,7 @@ contributors across the globe, there is almost always someone available to help. | authentication.mutual.spire.install.agent.tolerations | list | `[{"effect":"NoSchedule","key":"node.kubernetes.io/not-ready"},{"effect":"NoSchedule","key":"node-role.kubernetes.io/master"},{"effect":"NoSchedule","key":"node-role.kubernetes.io/control-plane"},{"effect":"NoSchedule","key":"node.cloudprovider.kubernetes.io/uninitialized","value":"true"},{"key":"CriticalAddonsOnly","operator":"Exists"}]` | SPIRE agent tolerations configuration By default it follows the same tolerations as the agent itself to allow the Cilium agent on this node to connect to SPIRE. ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ | | authentication.mutual.spire.install.enabled | bool | `true` | Enable SPIRE installation. This will only take effect only if authentication.mutual.spire.enabled is true | | authentication.mutual.spire.install.existingNamespace | bool | `false` | SPIRE namespace already exists. Set to true if Helm should not create, manage, and import the SPIRE namespace. | -| authentication.mutual.spire.install.initImage | object | `{"digest":"sha256:b3255e7dfbcd10cb367af0d409747d511aeb66dfac98cf30e97e87e4207dd76f","override":null,"pullPolicy":"IfNotPresent","repository":"docker.io/library/busybox","tag":"1.37.0","useDigest":true}` | init container image of SPIRE agent and server | +| authentication.mutual.spire.install.initImage | object | `{"digest":"sha256:1487d0af5f52b4ba31c7e465126ee2123fe3f2305d638e7827681e7cf6c83d5e","override":null,"pullPolicy":"IfNotPresent","repository":"docker.io/library/busybox","tag":"1.37.0","useDigest":true}` | init container image of SPIRE agent and server | | authentication.mutual.spire.install.namespace | string | `"cilium-spire"` | SPIRE namespace to install into | | authentication.mutual.spire.install.server.affinity | object | `{}` | SPIRE server affinity configuration | | authentication.mutual.spire.install.server.annotations | object | `{}` | SPIRE server annotations | @@ -175,7 +175,7 @@ contributors across the globe, there is almost always someone available to help. | bpf.tproxy | bool | `false` | Configure the eBPF-based TPROXY (beta) to reduce reliance on iptables rules for implementing Layer 7 policy. Note this is incompatible with netkit (`bpf.datapathMode=netkit`, `bpf.datapathMode=netkit-l2`). | | bpf.vlanBypass | list | `[]` | Configure explicitly allowed VLAN id's for bpf logic bypass. [0] will allow all VLAN id's without any filtering. | | bpfClockProbe | bool | `false` | Enable BPF clock source probing for more efficient tick retrieval. | -| certgen | object | `{"affinity":{},"annotations":{"cronJob":{},"job":{}},"cronJob":{"failedJobsHistoryLimit":1,"successfulJobsHistoryLimit":3},"extraVolumeMounts":[],"extraVolumes":[],"generateCA":true,"image":{"digest":"sha256:19921f48ee7e2295ea4dca955878a6cd8d70e6d4219d08f688e866ece9d95d4d","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/certgen","tag":"v0.3.2","useDigest":true},"nodeSelector":{},"podLabels":{},"priorityClassName":"","resources":{},"tolerations":[],"ttlSecondsAfterFinished":null}` | Configure certificate generation for Hubble integration. If hubble.tls.auto.method=cronJob, these values are used for the Kubernetes CronJob which will be scheduled regularly to (re)generate any certificates not provided manually. | +| certgen | object | `{"affinity":{},"annotations":{"cronJob":{},"job":{}},"cronJob":{"failedJobsHistoryLimit":1,"successfulJobsHistoryLimit":3},"extraVolumeMounts":[],"extraVolumes":[],"generateCA":true,"image":{"digest":"sha256:f0c656830e856d26b24b0e144df1f8b327d3b46748d76a630514111fc365b697","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/certgen","tag":"v0.4.1","useDigest":true},"nodeSelector":{},"podLabels":{},"priorityClassName":"","resources":{},"tolerations":[],"ttlSecondsAfterFinished":null}` | Configure certificate generation for Hubble integration. If hubble.tls.auto.method=cronJob, these values are used for the Kubernetes CronJob which will be scheduled regularly to (re)generate any certificates not provided manually. | | certgen.affinity | object | `{}` | Affinity for certgen | | certgen.annotations | object | `{"cronJob":{},"job":{}}` | Annotations to be added to the hubble-certgen initial Job and CronJob | | certgen.cronJob.failedJobsHistoryLimit | int | `1` | The number of failed finished jobs to keep | @@ -214,7 +214,7 @@ contributors across the globe, there is almost always someone available to help. | clustermesh.apiserver.extraVolumeMounts | list | `[]` | Additional clustermesh-apiserver volumeMounts. | | clustermesh.apiserver.extraVolumes | list | `[]` | Additional clustermesh-apiserver volumes. | | clustermesh.apiserver.healthPort | int | `9880` | TCP port for the clustermesh-apiserver health API. | -| clustermesh.apiserver.image | object | `{"digest":"sha256:56d6c3dc13b50126b80ecb571707a0ea97f6db694182b9d61efd386d04e5bb28","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/clustermesh-apiserver","tag":"v1.19.1","useDigest":true}` | Clustermesh API server image. | +| clustermesh.apiserver.image | object | `{"digest":"sha256:a8136a7615d6c6041d3aa6f2674d17beaec238170d669507ccc05328a778e2b7","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/clustermesh-apiserver","tag":"v1.19.3","useDigest":true}` | Clustermesh API server image. | | clustermesh.apiserver.kvstoremesh.enabled | bool | `true` | Enable KVStoreMesh. KVStoreMesh caches the information retrieved from the remote clusters in the local etcd instance (deprecated - KVStoreMesh will always be enabled once the option is removed). | | clustermesh.apiserver.kvstoremesh.extraArgs | list | `[]` | Additional KVStoreMesh arguments. | | clustermesh.apiserver.kvstoremesh.extraEnv | list | `[]` | Additional KVStoreMesh environment variables. | @@ -340,6 +340,10 @@ contributors across the globe, there is almost always someone available to help. | cni.resources | object | `{"limits":{"cpu":1,"memory":"1Gi"},"requests":{"cpu":"100m","memory":"10Mi"}}` | Specifies the resources for the cni initContainer | | cni.uninstall | bool | `false` | Remove the CNI configuration and binary files on agent shutdown. Enable this if you're removing Cilium from the cluster. Disable this to prevent the CNI configuration file from being removed during agent upgrade, which can cause nodes to go unmanageable. | | commonLabels | object | `{}` | commonLabels allows users to add common labels for all Cilium resources. | +| configDriftDetection | object | `{"driftChecker":true,"enabled":true,"ignoredKeys":[]}` | Configuration for the ConfigMap drift detection feature. When enabled, the agent continuously watches the cilium-config ConfigMap and exposes a cilium_drift_checker_config_delta Prometheus metric reporting the number of keys that differ between the ConfigMap and the agent's active settings. A non-zero value indicates that the agent has not yet applied all current ConfigMap changes and needs to be restarted. | +| configDriftDetection.driftChecker | bool | `true` | Enable the drift checker which compares the DynamicConfig table against the agent's active settings and publishes the cilium_drift_checker_config_delta metric. | +| configDriftDetection.enabled | bool | `true` | Enable watching of the cilium-config ConfigMap and reflecting its contents into the agent's internal DynamicConfig table. | +| configDriftDetection.ignoredKeys | list | `[]` | List of config-map keys to ignore when computing the drift delta. | | connectivityProbeFrequencyRatio | float64 | `0.5` | Ratio of the connectivity probe frequency vs resource usage, a float in [0, 1]. 0 will give more frequent probing, 1 will give less frequent probing. Probing frequency is dynamically adjusted based on the cluster size. | | conntrackGCInterval | string | `"0s"` | Configure how frequently garbage collection should occur for the datapath connection tracking table. | | conntrackGCMaxInterval | string | `""` | Configure the maximum frequency for the garbage collection of the connection tracking table. Only affects the automatic computation for the frequency and has no effect when 'conntrackGCInterval' is set. This can be set to more frequently clean up unused identities created from ToFQDN policies. | @@ -380,7 +384,6 @@ contributors across the globe, there is almost always someone available to help. | enableMasqueradeRouteSource | bool | `false` | Enables masquerading to the source of the route for traffic leaving the node from endpoints. | | enableNoServiceEndpointsRoutable | bool | `true` | Enable routing to a service that has zero endpoints | | enableNonDefaultDenyPolicies | bool | `true` | Enable Non-Default-Deny policies | -| enableTunnelBIGTCP | bool | `false` | Enable BIG TCP in tunneling mode and increase maximum GRO/GSO limits for VXLAN/GENEVE tunnels | | enableXTSocketFallback | bool | `true` | Enables the fallback compatibility solution for when the xt_socket kernel module is missing and it is needed for the datapath L7 redirection to work properly. See documentation for details on when this can be disabled: https://docs.cilium.io/en/stable/operations/system_requirements/#linux-kernel. | | encryption.enabled | bool | `false` | Enable transparent network encryption. | | encryption.ipsec.encryptedOverlay | bool | `false` | Enable IPsec encrypted overlay | @@ -401,8 +404,29 @@ contributors across the globe, there is almost always someone available to help. | encryption.strictMode.ingress.enabled | bool | `false` | Enable strict ingress encryption. When enabled, all unencrypted overlay ingress traffic will be dropped. This option is only applicable when WireGuard and tunneling are enabled. | | encryption.type | string | `"ipsec"` | Encryption method. Can be one of ipsec, wireguard or ztunnel. | | encryption.wireguard.persistentKeepalive | string | `"0s"` | Controls WireGuard PersistentKeepalive option. Set 0s to disable. | +| encryption.ztunnel | object | `{"affinity":{},"annotations":{},"caAddress":"https://localhost:15012","extraEnv":[],"extraVolumeMounts":[],"extraVolumes":[],"healthPort":15021,"image":{"digest":null,"override":null,"pullPolicy":"IfNotPresent","repository":"docker.io/istio/ztunnel","tag":"1.28.0-distroless","useDigest":false},"nodeSelector":{"kubernetes.io/os":"linux"},"podAnnotations":{},"podLabels":{},"priorityClassName":null,"readinessProbe":{"failureThreshold":3,"initialDelaySeconds":0,"periodSeconds":10},"resources":{"requests":{"cpu":"200m","memory":"512Mi"}},"secrets":{"bootstrapRootCert":null},"terminationGracePeriodSeconds":30,"tolerations":[{"effect":"NoSchedule","operator":"Exists"},{"key":"CriticalAddonsOnly","operator":"Exists"},{"effect":"NoExecute","operator":"Exists"}],"updateStrategy":{"rollingUpdate":{"maxSurge":1,"maxUnavailable":0},"type":"RollingUpdate"}}` | ztunnel encryption configuration. ztunnel is Istio's purpose-built, per-node proxy for handling L4 traffic in ambient mesh mode. These settings only apply when encryption.type is set to "ztunnel". | +| encryption.ztunnel.affinity | object | `{}` | Affinity for ztunnel pods. | +| encryption.ztunnel.annotations | object | `{}` | Annotations to be added to all ztunnel resources. | +| encryption.ztunnel.caAddress | string | `"https://localhost:15012"` | CA server address for certificate requests. | +| encryption.ztunnel.extraEnv | list | `[]` | Additional ztunnel container environment variables. | +| encryption.ztunnel.extraVolumeMounts | list | `[]` | Additional ztunnel volumeMounts. | +| encryption.ztunnel.extraVolumes | list | `[]` | Additional ztunnel volumes. | +| encryption.ztunnel.healthPort | int | `15021` | TCP port for the health API. | +| encryption.ztunnel.image | object | `{"digest":null,"override":null,"pullPolicy":"IfNotPresent","repository":"docker.io/istio/ztunnel","tag":"1.28.0-distroless","useDigest":false}` | ztunnel container image. | +| encryption.ztunnel.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node selector for ztunnel pods. | +| encryption.ztunnel.podAnnotations | object | `{}` | Annotations to be added to ztunnel pods. | +| encryption.ztunnel.podLabels | object | `{}` | Labels to be added to ztunnel pods. | +| encryption.ztunnel.priorityClassName | string | `nil` | The priority class to use for ztunnel pods. | +| encryption.ztunnel.readinessProbe | object | `{"failureThreshold":3,"initialDelaySeconds":0,"periodSeconds":10}` | Readiness probe configuration. | +| encryption.ztunnel.resources | object | `{"requests":{"cpu":"200m","memory":"512Mi"}}` | ztunnel resource limits & requests. | +| encryption.ztunnel.secrets | object | `{"bootstrapRootCert":null}` | ztunnel secrets configuration. | +| encryption.ztunnel.secrets.bootstrapRootCert | string | `nil` | Base64-encoded bootstrap root certificate content. If not provided, the secret must be created manually before deploying. @schema type: [null, string] @schema | +| encryption.ztunnel.terminationGracePeriodSeconds | int | `30` | Configure termination grace period for ztunnel DaemonSet. | +| encryption.ztunnel.tolerations | list | `[{"effect":"NoSchedule","operator":"Exists"},{"key":"CriticalAddonsOnly","operator":"Exists"},{"effect":"NoExecute","operator":"Exists"}]` | Node tolerations for ztunnel scheduling. | +| encryption.ztunnel.updateStrategy | object | `{"rollingUpdate":{"maxSurge":1,"maxUnavailable":0},"type":"RollingUpdate"}` | ztunnel update strategy. | | endpointHealthChecking.enabled | bool | `true` | Enable connectivity health checking between virtual endpoints. | | endpointLockdownOnMapOverflow | bool | `false` | Enable endpoint lockdown on policy map overflow. | +| endpointPolicyUpdateTimeoutDuration | string | `nil` | Max duration to wait for envoy to respond to configuration changes. Default "10s". | | endpointRoutes.enabled | bool | `false` | Enable use of per endpoint routes instead of routing via the cilium_host interface. | | eni.awsEnablePrefixDelegation | bool | `false` | Enable ENI prefix delegation | | eni.awsReleaseExcessIPs | bool | `false` | Release IPs not used from the ENI | @@ -446,7 +470,7 @@ contributors across the globe, there is almost always someone available to help. | envoy.httpRetryCount | int | `3` | Maximum number of retries for each HTTP request | | envoy.httpUpstreamLingerTimeout | string | `nil` | Time in seconds to block Envoy worker thread while an upstream HTTP connection is closing. If set to 0, the connection is closed immediately (with TCP RST). If set to -1, the connection is closed asynchronously in the background. | | envoy.idleTimeoutDurationSeconds | int | `60` | Set Envoy upstream HTTP idle connection timeout seconds. Does not apply to connections with pending requests. Default 60s | -| envoy.image | object | `{"digest":"sha256:8188114a2768b5f49d6ce58e168b20d765e0fbc64eee0d83241aa2b150ccd788","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium-envoy","tag":"v1.35.9-1770979049-232ed4a26881e4ab4f766f251f258ed424fff663","useDigest":true}` | Envoy container image. | +| envoy.image | object | `{"digest":"sha256:ba0ab8adac082d50d525fd2c5ba096c8facea3a471561b7c61c7a5b9c2e0de0d","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium-envoy","tag":"v1.36.6-1776000132-2437d2edeaf4d9b56ef279bd0d71127440c067aa","useDigest":true}` | Envoy container image. | | envoy.initContainers | list | `[]` | Init containers added to the cilium Envoy DaemonSet. | | envoy.initialFetchTimeoutSeconds | int | `30` | Time in seconds after which the initial fetch on an xDS stream is considered timed out | | envoy.livenessProbe.enabled | bool | `true` | Enable liveness probe for cilium-envoy | @@ -591,7 +615,7 @@ contributors across the globe, there is almost always someone available to help. | hubble.relay.extraVolumes | list | `[]` | Additional hubble-relay volumes. | | hubble.relay.gops.enabled | bool | `true` | Enable gops for hubble-relay | | hubble.relay.gops.port | int | `9893` | Configure gops listen port for hubble-relay | -| hubble.relay.image | object | `{"digest":"sha256:d8c4e13bc36a56179292bb52bc6255379cb94cb873700d316ea3139b1bdb8165","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/hubble-relay","tag":"v1.19.1","useDigest":true}` | Hubble-relay container image. | +| hubble.relay.image | object | `{"digest":"sha256:5ee21d57b6ef2aa6db67e603a735fdceb162454b352b7335b651456e308f681b","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/hubble-relay","tag":"v1.19.3","useDigest":true}` | Hubble-relay container image. | | hubble.relay.listenHost | string | `""` | Host to listen to. Specify an empty string to bind to all the interfaces. | | hubble.relay.listenPort | string | `"4245"` | Port to listen to. | | hubble.relay.logOptions | object | `{"format":null,"level":null}` | Logging configuration for hubble-relay. | @@ -709,7 +733,7 @@ contributors across the globe, there is almost always someone available to help. | identityAllocationMode | string | `"crd"` | Method to use for identity allocation (`crd`, `kvstore` or `doublewrite-readkvstore` / `doublewrite-readcrd` for migrating between identity backends). | | identityChangeGracePeriod | string | `"5s"` | Time to wait before using new identity on endpoint identity change. | | identityManagementMode | string | `"agent"` | Control whether CiliumIdentities are created by the agent ("agent"), the operator ("operator") or both ("both"). "Both" should be used only to migrate between "agent" and "operator". Operator-managed identities is a beta feature. | -| image | object | `{"digest":"sha256:41f1f74a0000de8656f1de4088ea00c8f2d49d6edea579034c73c5fd5fe01792","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium","tag":"v1.19.1","useDigest":true}` | Agent container image. | +| image | object | `{"digest":"sha256:2e61680593cddca8b6c055f6d4c849d87a26a1c91c7e3b8b56c7fb76ab7b7b10","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium","tag":"v1.19.3","useDigest":true}` | Agent container image. | | imagePullSecrets | list | `[]` | Configure image pull secrets for pulling container images | | ingressController.default | bool | `false` | Set cilium ingress controller to be the default ingress controller This will let cilium ingress controller route entries without ingress class set | | ingressController.defaultSecretName | string | `nil` | Default secret name for ingresses without .spec.tls[].secretName set. | @@ -797,12 +821,13 @@ contributors across the globe, there is almost always someone available to help. | livenessProbe.failureThreshold | int | `10` | failure threshold of liveness probe | | livenessProbe.periodSeconds | int | `30` | interval between checks of the liveness probe | | livenessProbe.requireK8sConnectivity | bool | `false` | whether to require k8s connectivity as part of the check. | -| loadBalancer | object | `{"acceleration":"disabled","l7":{"algorithm":"round_robin","backend":"disabled","ports":[]}}` | Configure service load balancing | +| loadBalancer | object | `{"acceleration":"disabled","l7":{"algorithm":"round_robin","backend":"disabled","ports":[]},"serviceTopology":false}` | Configure service load balancing | | loadBalancer.acceleration | string | `"disabled"` | acceleration is the option to accelerate service handling via XDP Applicable values can be: disabled (do not use XDP), native (XDP BPF program is run directly out of the networking driver's early receive path), or best-effort (use native mode XDP acceleration on devices that support it). | | loadBalancer.l7 | object | `{"algorithm":"round_robin","backend":"disabled","ports":[]}` | L7 LoadBalancer | | loadBalancer.l7.algorithm | string | `"round_robin"` | Default LB algorithm The default LB algorithm to be used for services, which can be overridden by the service annotation (e.g. service.cilium.io/lb-l7-algorithm) Applicable values: round_robin, least_request, random | | loadBalancer.l7.backend | string | `"disabled"` | Enable L7 service load balancing via envoy proxy. The request to a k8s service, which has specific annotation e.g. service.cilium.io/lb-l7, will be forwarded to the local backend proxy to be load balanced to the service endpoints. Please refer to docs for supported annotations for more configuration. Applicable values: - envoy: Enable L7 load balancing via envoy proxy. This will automatically set enable-envoy-config as well. - disabled: Disable L7 load balancing by way of service annotation. | | loadBalancer.l7.ports | list | `[]` | List of ports from service to be automatically redirected to above backend. Any service exposing one of these ports will be automatically redirected. Fine-grained control can be achieved by using the service annotation. | +| loadBalancer.serviceTopology | bool | `false` | serviceTopology enables K8s Topology Aware Hints -based service endpoints filtering | | localRedirectPolicies.addressMatcherCIDRs | string | `nil` | Limit the allowed addresses in Address Matcher rule of Local Redirect Policies to the given CIDRs. @schema@ type: [null, array] @schema@ | | localRedirectPolicies.enabled | bool | `false` | Enable local redirect policies. | | localRedirectPolicy | bool | `false` | Enable Local Redirect Policy (deprecated, please use 'localRedirectPolicies.enabled' instead) | @@ -860,7 +885,7 @@ contributors across the globe, there is almost always someone available to help. | operator.hostNetwork | bool | `true` | HostNetwork setting | | operator.identityGCInterval | string | `"15m0s"` | Interval for identity garbage collection. | | operator.identityHeartbeatTimeout | string | `"30m0s"` | Timeout for identity heartbeats. | -| operator.image | object | `{"alibabacloudDigest":"sha256:837b12f4239e88ea5b4b5708ab982c319a94ee05edaecaafe5fd0e5b1962f554","awsDigest":"sha256:18913d05a6c4d205f0b7126c4723bb9ccbd4dc24403da46ed0f9f4bf2a142804","azureDigest":"sha256:82bce78603056e709d4c4e9f9ebb25c222c36d8a07f8c05381c2372d9078eca8","genericDigest":"sha256:e7278d763e448bf6c184b0682cf98cdca078d58a27e1b2f3c906792670aa211a","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/operator","suffix":"","tag":"v1.19.1","useDigest":true}` | cilium-operator image. | +| operator.image | object | `{"alibabacloudDigest":"sha256:176321a65123373ff8c7823b25183102cbad98375e8d6c80b96d68b6e8491103","awsDigest":"sha256:a53dcbfb77282bf2ddd3abbe60f6d49762e7c1389a36cb35b71d504644a56640","azureDigest":"sha256:699c1571a3df1a98882ee13610d47cffb7b34ee7e8d276096db798a5f6c7e4cb","genericDigest":"sha256:205b09b0ed6accbf9fe688d312a9f0fcfc6a316fc081c23fbffb472af5dd62cd","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/operator","suffix":"","tag":"v1.19.3","useDigest":true}` | cilium-operator image. | | operator.nodeGCInterval | string | `"5m0s"` | Interval for cilium node garbage collection. | | operator.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node labels for cilium-operator pod assignment ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector | | operator.podAnnotations | object | `{}` | Annotations to be added to cilium-operator pods | @@ -918,11 +943,11 @@ contributors across the globe, there is almost always someone available to help. | preflight.affinity | object | `{"podAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchLabels":{"k8s-app":"cilium"}},"topologyKey":"kubernetes.io/hostname"}]}}` | Affinity for cilium-preflight | | preflight.annotations | object | `{}` | Annotations to be added to all top-level preflight objects (resources under templates/cilium-preflight) | | preflight.enabled | bool | `false` | Enable Cilium pre-flight resources (required for upgrade) | -| preflight.envoy.image | object | `{"digest":"sha256:8188114a2768b5f49d6ce58e168b20d765e0fbc64eee0d83241aa2b150ccd788","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium-envoy","tag":"v1.35.9-1770979049-232ed4a26881e4ab4f766f251f258ed424fff663","useDigest":true}` | Envoy pre-flight image. | +| preflight.envoy.image | object | `{"digest":"sha256:ba0ab8adac082d50d525fd2c5ba096c8facea3a471561b7c61c7a5b9c2e0de0d","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium-envoy","tag":"v1.36.6-1776000132-2437d2edeaf4d9b56ef279bd0d71127440c067aa","useDigest":true}` | Envoy pre-flight image. | | preflight.extraEnv | list | `[]` | Additional preflight environment variables. | | preflight.extraVolumeMounts | list | `[]` | Additional preflight volumeMounts. | | preflight.extraVolumes | list | `[]` | Additional preflight volumes. | -| preflight.image | object | `{"digest":"sha256:41f1f74a0000de8656f1de4088ea00c8f2d49d6edea579034c73c5fd5fe01792","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium","tag":"v1.19.1","useDigest":true}` | Cilium pre-flight image. | +| preflight.image | object | `{"digest":"sha256:2e61680593cddca8b6c055f6d4c849d87a26a1c91c7e3b8b56c7fb76ab7b7b10","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium","tag":"v1.19.3","useDigest":true}` | Cilium pre-flight image. | | preflight.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node labels for preflight pod assignment ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector | | preflight.podAnnotations | object | `{}` | Annotations to be added to preflight pods | | preflight.podDisruptionBudget.enabled | bool | `false` | enable PodDisruptionBudget ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ | @@ -979,6 +1004,7 @@ contributors across the globe, there is almost always someone available to help. | serviceAccounts.corednsMCSAPI | object | `{"annotations":{},"automount":true,"create":true,"name":"cilium-coredns-mcsapi-autoconfig"}` | CorednsMCSAPI is used if clustermesh.mcsapi.corednsAutoConfigure.enabled=true | | serviceAccounts.hubblecertgen | object | `{"annotations":{},"automount":true,"create":true,"name":"hubble-generate-certs"}` | Hubblecertgen is used if hubble.tls.auto.method=cronJob | | serviceAccounts.nodeinit.enabled | bool | `false` | Enabled is temporary until https://github.com/cilium/cilium-cli/issues/1396 is implemented. Cilium CLI doesn't create the SAs for node-init, thus the workaround. Helm is not affected by this issue. Name and automount can be configured, if enabled is set to true. Otherwise, they are ignored. Enabled can be removed once the issue is fixed. Cilium-nodeinit DS must also be fixed. | +| serviceAccounts.ztunnel | object | `{"annotations":{},"automount":false,"create":true,"name":"ztunnel-cilium"}` | Ztunnel is used if encryption.type=ztunnel | | serviceNoBackendResponse | string | `"reject"` | Configure what the response should be to traffic for a service without backends. Possible values: - reject (default) - drop | | sleepAfterInit | bool | `false` | Do not run Cilium agent when running with clean mode. Useful to completely uninstall Cilium as it will stop Cilium from starting and create artifacts in the node. | | socketLB | object | `{"enabled":false}` | Configure socket LB | diff --git a/packages/system/cilium/charts/cilium/files/cilium-envoy/configmap/bootstrap-config.yaml b/packages/system/cilium/charts/cilium/files/cilium-envoy/configmap/bootstrap-config.yaml index ea1d3bda..90ebf4ac 100644 --- a/packages/system/cilium/charts/cilium/files/cilium-envoy/configmap/bootstrap-config.yaml +++ b/packages/system/cilium/charts/cilium/files/cilium-envoy/configmap/bootstrap-config.yaml @@ -167,6 +167,8 @@ staticResources: circuitBreakers: thresholds: - maxRetries: {{ .Values.envoy.maxConcurrentRetries }} + maxConnections: {{ .Values.envoy.clusterMaxConnections }} + maxRequests: {{ .Values.envoy.clusterMaxRequests }} lbPolicy: "CLUSTER_PROVIDED" typedExtensionProtocolOptions: envoy.extensions.upstreams.http.v3.HttpProtocolOptions: @@ -183,6 +185,8 @@ staticResources: circuitBreakers: thresholds: - maxRetries: {{ .Values.envoy.maxConcurrentRetries }} + maxConnections: {{ .Values.envoy.clusterMaxConnections }} + maxRequests: {{ .Values.envoy.clusterMaxRequests }} lbPolicy: "CLUSTER_PROVIDED" typedExtensionProtocolOptions: envoy.extensions.upstreams.http.v3.HttpProtocolOptions: @@ -204,6 +208,8 @@ staticResources: circuitBreakers: thresholds: - maxRetries: {{ .Values.envoy.maxConcurrentRetries }} + maxConnections: {{ .Values.envoy.clusterMaxConnections }} + maxRequests: {{ .Values.envoy.clusterMaxRequests }} lbPolicy: "CLUSTER_PROVIDED" typedExtensionProtocolOptions: envoy.extensions.upstreams.http.v3.HttpProtocolOptions: @@ -220,6 +226,8 @@ staticResources: circuitBreakers: thresholds: - maxRetries: {{ .Values.envoy.maxConcurrentRetries }} + maxConnections: {{ .Values.envoy.clusterMaxConnections }} + maxRequests: {{ .Values.envoy.clusterMaxRequests }} lbPolicy: "CLUSTER_PROVIDED" typedExtensionProtocolOptions: envoy.extensions.upstreams.http.v3.HttpProtocolOptions: @@ -304,3 +312,4 @@ admin: address: pipe: path: "/var/run/cilium/envoy/sockets/admin.sock" + mode: 0660 diff --git a/packages/system/cilium/charts/cilium/templates/cilium-agent/daemonset.yaml b/packages/system/cilium/charts/cilium/templates/cilium-agent/daemonset.yaml index fa3afc14..fff1b384 100644 --- a/packages/system/cilium/charts/cilium/templates/cilium-agent/daemonset.yaml +++ b/packages/system/cilium/charts/cilium/templates/cilium-agent/daemonset.yaml @@ -546,7 +546,7 @@ spec: {{- toYaml .Values.initResources | trim | nindent 10 }} {{- end }} command: - - sh + - bash - -ec # The statically linked Go program binary is invoked to avoid any # dependency on utilities like sh and mount that can be missing on certain @@ -592,7 +592,7 @@ spec: - name: BIN_PATH value: {{ .Values.cni.binPath }} command: - - sh + - bash - -ec # The statically linked Go program binary is invoked to avoid any # dependency on utilities like sh that can be missing on certain @@ -660,7 +660,7 @@ spec: {{- toYaml . | trim | nindent 10 }} {{- end }} command: - - sh + - bash - -c - | until test -s {{ (print "/tmp/cilium-bootstrap.d/" (.Values.nodeinit.bootstrapFile | base)) | quote }}; do diff --git a/packages/system/cilium/charts/cilium/templates/cilium-configmap.yaml b/packages/system/cilium/charts/cilium/templates/cilium-configmap.yaml index a3a38c09..5d76944d 100644 --- a/packages/system/cilium/charts/cilium/templates/cilium-configmap.yaml +++ b/packages/system/cilium/charts/cilium/templates/cilium-configmap.yaml @@ -464,6 +464,9 @@ data: {{- if has (kindOf .Values.bpf.policyMapPressureMetricsThreshold) (list "int64" "float64") }} bpf-policy-map-pressure-metrics-threshold: {{ .Values.bpf.policyMapPressureMetricsThreshold | quote }} {{- end }} +{{- if .Values.endpointPolicyUpdateTimeoutDuration }} + endpoint-policy-update-timeout: {{ .Values.endpointPolicyUpdateTimeoutDuration | quote }} +{{- end }} {{- if hasKey .Values.bpf "policyStatsMapMax" }} # bpf-policy-stats-map-max specifies the maximum number of entries in global # policy stats map @@ -706,7 +709,6 @@ data: enable-ipv4-big-tcp: {{ .Values.enableIPv4BIGTCP | quote }} enable-ipv6-big-tcp: {{ .Values.enableIPv6BIGTCP | quote }} enable-ipv6-masquerade: {{ .Values.enableIPv6Masquerade | quote }} - enable-tunnel-big-tcp: {{ .Values.enableTunnelBIGTCP | quote }} {{- if hasKey .Values.bpf "enableTCX" }} enable-tcx: {{ .Values.bpf.enableTCX | quote }} @@ -906,9 +908,9 @@ data: {{- end }} {{- if hasKey .Values.loadBalancer "serviceTopology" }} enable-service-topology: {{ .Values.loadBalancer.serviceTopology | quote }} -# {{- end }} - {{- end }} +{{- end }} + {{- if hasKey .Values.maglev "tableSize" }} bpf-lb-maglev-table-size: {{ .Values.maglev.tableSize | quote}} {{- end }} @@ -1380,11 +1382,7 @@ data: {{- if .Values.operator.unmanagedPodWatcher.restart }} {{- $interval := .Values.operator.unmanagedPodWatcher.intervalSeconds }} - {{- if kindIs "float64" $interval }} unmanaged-pod-watcher-interval: {{ printf "%ds" (int $interval) | quote }} - {{- else }} - unmanaged-pod-watcher-interval: {{ $interval | quote }} - {{- end }} {{- else }} unmanaged-pod-watcher-interval: "0" {{- end }} @@ -1517,6 +1515,14 @@ data: connectivity-probe-frequency-ratio: {{ .Values.connectivityProbeFrequencyRatio | quote }} {{- end }} +{{- if hasKey .Values "configDriftDetection" }} + enable-dynamic-config: {{ .Values.configDriftDetection.enabled | quote }} + enable-drift-checker: {{ .Values.configDriftDetection.driftChecker | quote }} + {{- if .Values.configDriftDetection.ignoredKeys }} + ignore-flags-drift-checker: {{ join "," .Values.configDriftDetection.ignoredKeys | quote }} + {{- end }} +{{- end }} + # Extra config allows adding arbitrary properties to the cilium config. # By putting it at the end of the ConfigMap, it's also possible to override existing properties. {{- if .Values.extraConfig }} diff --git a/packages/system/cilium/charts/cilium/templates/cilium-operator/clusterrole.yaml b/packages/system/cilium/charts/cilium/templates/cilium-operator/clusterrole.yaml index 96f72c7f..c147e2b5 100644 --- a/packages/system/cilium/charts/cilium/templates/cilium-operator/clusterrole.yaml +++ b/packages/system/cilium/charts/cilium/templates/cilium-operator/clusterrole.yaml @@ -407,6 +407,15 @@ rules: verbs: - update - patch +- apiGroups: + - multicluster.x-k8s.io + resources: + # The controller needs to be able to set serviceimport finalizers to be able to create a derived Service + # resource that is owned by the ServiceImport and sets blockOwnerDeletion=true in its ownerRef. + # This is required when the admission plugin OwnerReferencesPermissionEnforcement is activated. + - serviceimports/finalizers + verbs: + - update - apiGroups: - multicluster.x-k8s.io resources: diff --git a/packages/system/cilium/charts/cilium/templates/ztunnel/daemonset.yaml b/packages/system/cilium/charts/cilium/templates/ztunnel/daemonset.yaml new file mode 100644 index 00000000..f9363068 --- /dev/null +++ b/packages/system/cilium/charts/cilium/templates/ztunnel/daemonset.yaml @@ -0,0 +1,165 @@ +{{- if and .Values.encryption.enabled (eq .Values.encryption.type "ztunnel") }} +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: ztunnel-cilium + namespace: {{ include "cilium.namespace" . }} + {{- with .Values.encryption.ztunnel.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app: ztunnel-cilium + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: ztunnel-cilium + {{- with .Values.commonLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + revisionHistoryLimit: 10 + selector: + matchLabels: + app: ztunnel-cilium + {{- with .Values.encryption.ztunnel.updateStrategy }} + updateStrategy: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + template: + metadata: + annotations: + sidecar.istio.io/inject: "false" + {{- with .Values.encryption.ztunnel.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + app: ztunnel-cilium + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: ztunnel-cilium + {{- with .Values.commonLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.encryption.ztunnel.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + hostNetwork: true + dnsPolicy: ClusterFirst + containers: + - name: istio-proxy + image: {{ include "cilium.image" .Values.encryption.ztunnel.image | quote }} + imagePullPolicy: {{ .Values.encryption.ztunnel.image.pullPolicy }} + args: + - proxy + - ztunnel + env: + - name: XDS_ADDRESS + value: "https://localhost:15012" + - name: XDS_ROOT_CA + value: "/etc/ztunnel/bootstrap-root.crt" + - name: CA_ROOT_CA + value: "/etc/ztunnel/bootstrap-root.crt" + - name: CA_ADDRESS + value: {{ .Values.encryption.ztunnel.caAddress | quote }} + - name: ISTIO_META_DNS_CAPTURE + value: "false" + - name: INPOD_UDS + value: "/var/run/cilium/ztunnel.sock" + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + - name: INSTANCE_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: ISTIO_META_ENABLE_HBONE + value: "true" + {{- with .Values.encryption.ztunnel.extraEnv }} + {{- toYaml . | trim | nindent 12 }} + {{- end }} + readinessProbe: + httpGet: + path: /healthz/ready + port: {{ .Values.encryption.ztunnel.healthPort }} + host: "127.0.0.1" + scheme: HTTP + initialDelaySeconds: {{ .Values.encryption.ztunnel.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.encryption.ztunnel.readinessProbe.periodSeconds }} + failureThreshold: {{ .Values.encryption.ztunnel.readinessProbe.failureThreshold }} + successThreshold: 1 + timeoutSeconds: 1 + {{- with .Values.encryption.ztunnel.resources }} + resources: + {{- toYaml . | trim | nindent 12 }} + {{- end }} + securityContext: + allowPrivilegeEscalation: true + capabilities: + add: + - NET_ADMIN + - SYS_ADMIN + - NET_RAW + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsGroup: 1337 + runAsNonRoot: false + runAsUser: 0 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /var/run/cilium + name: cilium-dir + readOnly: false + - mountPath: /etc/ztunnel + name: cilium-ztunnel-secrets + readOnly: true + {{- with .Values.encryption.ztunnel.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.encryption.ztunnel.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.encryption.ztunnel.nodeSelector }} + nodeSelector: + {{- toYaml . | trim | nindent 8 }} + {{- end }} + {{- with .Values.encryption.ztunnel.tolerations }} + tolerations: + {{- toYaml . | trim | nindent 8 }} + {{- end }} + priorityClassName: {{ include "cilium.priorityClass" (list $ .Values.encryption.ztunnel.priorityClassName "system-node-critical") }} + restartPolicy: Always + terminationGracePeriodSeconds: {{ .Values.encryption.ztunnel.terminationGracePeriodSeconds }} + {{- if .Values.serviceAccounts.ztunnel.create }} + serviceAccountName: {{ .Values.serviceAccounts.ztunnel.name | quote }} + automountServiceAccountToken: {{ .Values.serviceAccounts.ztunnel.automount }} + {{- else }} + automountServiceAccountToken: false + {{- end }} + volumes: + - name: cilium-dir + hostPath: + path: /var/run/cilium + type: DirectoryOrCreate + - name: cilium-ztunnel-secrets + secret: + secretName: cilium-ztunnel-secrets + defaultMode: 420 + items: + - key: bootstrap-root.crt + path: bootstrap-root.crt + mode: 420 + {{- with .Values.encryption.ztunnel.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/packages/system/cilium/charts/cilium/templates/ztunnel/secret.yaml b/packages/system/cilium/charts/cilium/templates/ztunnel/secret.yaml new file mode 100644 index 00000000..520857dc --- /dev/null +++ b/packages/system/cilium/charts/cilium/templates/ztunnel/secret.yaml @@ -0,0 +1,23 @@ +{{- if and .Values.encryption.enabled (eq .Values.encryption.type "ztunnel") }} +{{- if .Values.encryption.ztunnel.secrets.bootstrapRootCert }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: cilium-ztunnel-secrets + namespace: {{ include "cilium.namespace" . }} + {{- with .Values.encryption.ztunnel.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: ztunnel-cilium + {{- with .Values.commonLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +type: Opaque +data: + bootstrap-root.crt: {{ .Values.encryption.ztunnel.secrets.bootstrapRootCert | b64enc }} +{{- end }} +{{- end }} diff --git a/packages/system/cilium/charts/cilium/templates/ztunnel/serviceaccount.yaml b/packages/system/cilium/charts/cilium/templates/ztunnel/serviceaccount.yaml new file mode 100644 index 00000000..4ef9e2bc --- /dev/null +++ b/packages/system/cilium/charts/cilium/templates/ztunnel/serviceaccount.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.encryption.enabled (eq .Values.encryption.type "ztunnel") .Values.serviceAccounts.ztunnel.create }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccounts.ztunnel.name | quote }} + namespace: {{ include "cilium.namespace" . }} + {{- with .Values.commonLabels }} + labels: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if or .Values.serviceAccounts.ztunnel.annotations .Values.encryption.ztunnel.annotations }} + annotations: + {{- with .Values.encryption.ztunnel.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccounts.ztunnel.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/packages/system/cilium/charts/cilium/values.schema.json b/packages/system/cilium/charts/cilium/values.schema.json index d18566d7..65c84af5 100644 --- a/packages/system/cilium/charts/cilium/values.schema.json +++ b/packages/system/cilium/charts/cilium/values.schema.json @@ -1714,6 +1714,21 @@ "object" ] }, + "configDriftDetection": { + "properties": { + "driftChecker": { + "type": "boolean" + }, + "enabled": { + "type": "boolean" + }, + "ignoredKeys": { + "items": {}, + "type": "array" + } + }, + "type": "object" + }, "connectivityProbeFrequencyRatio": { "type": [ "null", @@ -1898,9 +1913,6 @@ "enableNonDefaultDenyPolicies": { "type": "boolean" }, - "enableTunnelBIGTCP": { - "type": "boolean" - }, "enableXTSocketFallback": { "type": "boolean" }, @@ -1984,6 +1996,184 @@ } }, "type": "object" + }, + "ztunnel": { + "properties": { + "affinity": { + "type": "object" + }, + "annotations": { + "type": "object" + }, + "caAddress": { + "type": "string" + }, + "extraEnv": { + "items": {}, + "type": "array" + }, + "extraVolumeMounts": { + "items": {}, + "type": "array" + }, + "extraVolumes": { + "items": {}, + "type": "array" + }, + "healthPort": { + "type": "integer" + }, + "image": { + "properties": { + "digest": { + "type": [ + "null", + "string" + ] + }, + "override": { + "type": [ + "null", + "string" + ] + }, + "pullPolicy": { + "type": "string" + }, + "repository": { + "type": "string" + }, + "tag": { + "type": "string" + }, + "useDigest": { + "type": "boolean" + } + }, + "type": "object" + }, + "nodeSelector": { + "properties": { + "kubernetes.io/os": { + "type": "string" + } + }, + "type": "object" + }, + "podAnnotations": { + "type": "object" + }, + "podLabels": { + "type": "object" + }, + "priorityClassName": { + "type": [ + "null", + "string" + ] + }, + "readinessProbe": { + "properties": { + "failureThreshold": { + "type": "integer" + }, + "initialDelaySeconds": { + "type": "integer" + }, + "periodSeconds": { + "type": "integer" + } + }, + "type": "object" + }, + "resources": { + "properties": { + "requests": { + "properties": { + "cpu": { + "type": "string" + }, + "memory": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "secrets": { + "properties": { + "bootstrapRootCert": { + "type": [ + "null", + "string" + ] + } + }, + "type": "object" + }, + "terminationGracePeriodSeconds": { + "type": "integer" + }, + "tolerations": { + "items": { + "anyOf": [ + { + "properties": { + "effect": { + "type": "string" + }, + "operator": { + "type": "string" + } + } + }, + { + "properties": { + "key": { + "type": "string" + }, + "operator": { + "type": "string" + } + } + }, + { + "properties": { + "effect": { + "type": "string" + }, + "operator": { + "type": "string" + } + } + } + ] + }, + "type": "array" + }, + "updateStrategy": { + "properties": { + "rollingUpdate": { + "properties": { + "maxSurge": { + "type": "integer" + }, + "maxUnavailable": { + "type": "integer" + } + }, + "type": "object" + }, + "type": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" } }, "type": "object" @@ -1999,6 +2189,9 @@ "endpointLockdownOnMapOverflow": { "type": "boolean" }, + "endpointPolicyUpdateTimeoutDuration": { + "type": "null" + }, "endpointRoutes": { "properties": { "enabled": { @@ -4484,6 +4677,9 @@ } }, "type": "object" + }, + "serviceTopology": { + "type": "boolean" } }, "type": "object" @@ -6029,6 +6225,23 @@ } }, "type": "object" + }, + "ztunnel": { + "properties": { + "annotations": { + "type": "object" + }, + "automount": { + "type": "boolean" + }, + "create": { + "type": "boolean" + }, + "name": { + "type": "string" + } + }, + "type": "object" } }, "type": "object" diff --git a/packages/system/cilium/charts/cilium/values.yaml b/packages/system/cilium/charts/cilium/values.yaml index b9b30830..66f76982 100644 --- a/packages/system/cilium/charts/cilium/values.yaml +++ b/packages/system/cilium/charts/cilium/values.yaml @@ -210,6 +210,12 @@ serviceAccounts: name: cilium-coredns-mcsapi-autoconfig automount: true annotations: {} + # -- Ztunnel is used if encryption.type=ztunnel + ztunnel: + create: true + name: ztunnel-cilium + automount: false + annotations: {} # -- Configure termination grace period for cilium-agent DaemonSet. terminationGracePeriodSeconds: 1 # -- Install the cilium agent resources. @@ -218,6 +224,22 @@ agent: true name: cilium # -- Roll out cilium agent pods automatically when configmap is updated. rollOutCiliumPods: false +# -- Configuration for the ConfigMap drift detection feature. +# When enabled, the agent continuously watches the cilium-config ConfigMap +# and exposes a cilium_drift_checker_config_delta Prometheus metric reporting +# the number of keys that differ between the ConfigMap and the agent's active +# settings. A non-zero value indicates that the agent has not yet applied all +# current ConfigMap changes and needs to be restarted. +configDriftDetection: + # -- Enable watching of the cilium-config ConfigMap and reflecting its + # contents into the agent's internal DynamicConfig table. + enabled: true + # -- Enable the drift checker which compares the DynamicConfig table against + # the agent's active settings and publishes the + # cilium_drift_checker_config_delta metric. + driftChecker: true + # -- List of config-map keys to ignore when computing the drift delta. + ignoredKeys: [] # -- Agent container image. image: # @schema @@ -225,10 +247,10 @@ image: # @schema override: ~ repository: "quay.io/cilium/cilium" - tag: "v1.19.1" + tag: "v1.19.3" pullPolicy: "IfNotPresent" # cilium-digest - digest: sha256:41f1f74a0000de8656f1de4088ea00c8f2d49d6edea579034c73c5fd5fe01792 + digest: sha256:2e61680593cddca8b6c055f6d4c849d87a26a1c91c7e3b8b56c7fb76ab7b7b10 useDigest: true # -- Scheduling configurations for cilium pods scheduling: @@ -1133,9 +1155,89 @@ encryption: wireguard: # -- Controls WireGuard PersistentKeepalive option. Set 0s to disable. persistentKeepalive: 0s + # -- ztunnel encryption configuration. + # ztunnel is Istio's purpose-built, per-node proxy for handling L4 traffic in ambient mesh mode. + # These settings only apply when encryption.type is set to "ztunnel". + ztunnel: + # -- ztunnel container image. + image: + # @schema + # type: [null, string] + # @schema + override: ~ + repository: "docker.io/istio/ztunnel" + tag: "1.28.0-distroless" + pullPolicy: "IfNotPresent" + # @schema + # type: [null, string] + # @schema + digest: ~ + useDigest: false + # -- CA server address for certificate requests. + caAddress: "https://localhost:15012" + # -- TCP port for the health API. + healthPort: 15021 + # -- ztunnel resource limits & requests. + resources: + requests: + cpu: 200m + memory: 512Mi + # -- ztunnel update strategy. + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + # -- Configure termination grace period for ztunnel DaemonSet. + terminationGracePeriodSeconds: 30 + # -- Readiness probe configuration. + readinessProbe: + initialDelaySeconds: 0 + periodSeconds: 10 + failureThreshold: 3 + # -- Node selector for ztunnel pods. + nodeSelector: + kubernetes.io/os: linux + # -- Node tolerations for ztunnel scheduling. + tolerations: + - effect: NoSchedule + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + # -- Affinity for ztunnel pods. + affinity: {} + # @schema + # type: [null, string] + # @schema + # -- The priority class to use for ztunnel pods. + priorityClassName: ~ + # -- Annotations to be added to all ztunnel resources. + annotations: {} + # -- Annotations to be added to ztunnel pods. + podAnnotations: {} + # -- Labels to be added to ztunnel pods. + podLabels: {} + # -- Additional ztunnel container environment variables. + extraEnv: [] + # -- Additional ztunnel volumes. + extraVolumes: [] + # -- Additional ztunnel volumeMounts. + extraVolumeMounts: [] + # -- ztunnel secrets configuration. + secrets: + # -- Base64-encoded bootstrap root certificate content. + # If not provided, the secret must be created manually before deploying. + # @schema + # type: [null, string] + # @schema + bootstrapRootCert: ~ endpointHealthChecking: # -- Enable connectivity health checking between virtual endpoints. enabled: true +# -- Max duration to wait for envoy to respond to configuration changes. Default "10s". +endpointPolicyUpdateTimeoutDuration: null endpointRoutes: # @schema # type: [boolean, string] @@ -1250,8 +1352,8 @@ certgen: # @schema override: ~ repository: "quay.io/cilium/certgen" - tag: "v0.3.2" - digest: "sha256:19921f48ee7e2295ea4dca955878a6cd8d70e6d4219d08f688e866ece9d95d4d" + tag: "v0.4.1" + digest: "sha256:f0c656830e856d26b24b0e144df1f8b327d3b46748d76a630514111fc365b697" useDigest: true pullPolicy: "IfNotPresent" # @schema @@ -1599,9 +1701,9 @@ hubble: # @schema override: ~ repository: "quay.io/cilium/hubble-relay" - tag: "v1.19.1" + tag: "v1.19.3" # hubble-relay-digest - digest: sha256:d8c4e13bc36a56179292bb52bc6255379cb94cb873700d316ea3139b1bdb8165 + digest: sha256:5ee21d57b6ef2aa6db67e603a735fdceb162454b352b7335b651456e308f681b useDigest: true pullPolicy: "IfNotPresent" # -- Specifies the resources for the hubble-relay pods @@ -2313,8 +2415,6 @@ enableMasqueradeRouteSource: false enableIPv4BIGTCP: false # -- Enables IPv6 BIG TCP support which increases maximum IPv6 GSO/GRO limits for nodes and pods enableIPv6BIGTCP: false -# -- Enable BIG TCP in tunneling mode and increase maximum GRO/GSO limits for VXLAN/GENEVE tunnels -enableTunnelBIGTCP: false nat: # -- Number of the top-k SNAT map connections to track in Cilium statedb. mapStatsEntries: 32 @@ -2392,8 +2492,7 @@ loadBalancer: # -- serviceTopology enables K8s Topology Aware Hints -based service # endpoints filtering - # serviceTopology: false - + serviceTopology: false # -- L7 LoadBalancer l7: # -- Enable L7 service load balancing via envoy proxy. @@ -2626,9 +2725,9 @@ envoy: # @schema override: ~ repository: "quay.io/cilium/cilium-envoy" - tag: "v1.35.9-1770979049-232ed4a26881e4ab4f766f251f258ed424fff663" + tag: "v1.36.6-1776000132-2437d2edeaf4d9b56ef279bd0d71127440c067aa" pullPolicy: "IfNotPresent" - digest: "sha256:8188114a2768b5f49d6ce58e168b20d765e0fbc64eee0d83241aa2b150ccd788" + digest: "sha256:ba0ab8adac082d50d525fd2c5ba096c8facea3a471561b7c61c7a5b9c2e0de0d" useDigest: true # -- Init containers added to the cilium Envoy DaemonSet. initContainers: [] @@ -3011,15 +3110,15 @@ operator: # @schema override: ~ repository: "quay.io/cilium/operator" - tag: "v1.19.1" + tag: "v1.19.3" # operator-generic-digest - genericDigest: sha256:e7278d763e448bf6c184b0682cf98cdca078d58a27e1b2f3c906792670aa211a + genericDigest: sha256:205b09b0ed6accbf9fe688d312a9f0fcfc6a316fc081c23fbffb472af5dd62cd # operator-azure-digest - azureDigest: sha256:82bce78603056e709d4c4e9f9ebb25c222c36d8a07f8c05381c2372d9078eca8 + azureDigest: sha256:699c1571a3df1a98882ee13610d47cffb7b34ee7e8d276096db798a5f6c7e4cb # operator-aws-digest - awsDigest: sha256:18913d05a6c4d205f0b7126c4723bb9ccbd4dc24403da46ed0f9f4bf2a142804 + awsDigest: sha256:a53dcbfb77282bf2ddd3abbe60f6d49762e7c1389a36cb35b71d504644a56640 # operator-alibabacloud-digest - alibabacloudDigest: sha256:837b12f4239e88ea5b4b5708ab982c319a94ee05edaecaafe5fd0e5b1962f554 + alibabacloudDigest: sha256:176321a65123373ff8c7823b25183102cbad98375e8d6c80b96d68b6e8491103 useDigest: true pullPolicy: "IfNotPresent" suffix: "" @@ -3344,9 +3443,9 @@ preflight: # @schema override: ~ repository: "quay.io/cilium/cilium" - tag: "v1.19.1" + tag: "v1.19.3" # cilium-digest - digest: sha256:41f1f74a0000de8656f1de4088ea00c8f2d49d6edea579034c73c5fd5fe01792 + digest: sha256:2e61680593cddca8b6c055f6d4c849d87a26a1c91c7e3b8b56c7fb76ab7b7b10 useDigest: true pullPolicy: "IfNotPresent" envoy: @@ -3357,9 +3456,9 @@ preflight: # @schema override: ~ repository: "quay.io/cilium/cilium-envoy" - tag: "v1.35.9-1770979049-232ed4a26881e4ab4f766f251f258ed424fff663" + tag: "v1.36.6-1776000132-2437d2edeaf4d9b56ef279bd0d71127440c067aa" pullPolicy: "IfNotPresent" - digest: "sha256:8188114a2768b5f49d6ce58e168b20d765e0fbc64eee0d83241aa2b150ccd788" + digest: "sha256:ba0ab8adac082d50d525fd2c5ba096c8facea3a471561b7c61c7a5b9c2e0de0d" useDigest: true # -- The priority class to use for the preflight pod. priorityClassName: "" @@ -3603,9 +3702,9 @@ clustermesh: # @schema override: ~ repository: "quay.io/cilium/clustermesh-apiserver" - tag: "v1.19.1" + tag: "v1.19.3" # clustermesh-apiserver-digest - digest: sha256:56d6c3dc13b50126b80ecb571707a0ea97f6db694182b9d61efd386d04e5bb28 + digest: sha256:a8136a7615d6c6041d3aa6f2674d17beaec238170d669507ccc05328a778e2b7 useDigest: true pullPolicy: "IfNotPresent" # -- TCP port for the clustermesh-apiserver health API. @@ -4140,7 +4239,7 @@ authentication: override: ~ repository: "docker.io/library/busybox" tag: "1.37.0" - digest: "sha256:b3255e7dfbcd10cb367af0d409747d511aeb66dfac98cf30e97e87e4207dd76f" + digest: "sha256:1487d0af5f52b4ba31c7e465126ee2123fe3f2305d638e7827681e7cf6c83d5e" useDigest: true pullPolicy: "IfNotPresent" # SPIRE agent configuration diff --git a/packages/system/cilium/charts/cilium/values.yaml.tmpl b/packages/system/cilium/charts/cilium/values.yaml.tmpl index c21f1c26..8039b40c 100644 --- a/packages/system/cilium/charts/cilium/values.yaml.tmpl +++ b/packages/system/cilium/charts/cilium/values.yaml.tmpl @@ -213,6 +213,12 @@ serviceAccounts: name: cilium-coredns-mcsapi-autoconfig automount: true annotations: {} + # -- Ztunnel is used if encryption.type=ztunnel + ztunnel: + create: true + name: ztunnel-cilium + automount: false + annotations: {} # -- Configure termination grace period for cilium-agent DaemonSet. terminationGracePeriodSeconds: 1 # -- Install the cilium agent resources. @@ -221,6 +227,22 @@ agent: true name: cilium # -- Roll out cilium agent pods automatically when configmap is updated. rollOutCiliumPods: false +# -- Configuration for the ConfigMap drift detection feature. +# When enabled, the agent continuously watches the cilium-config ConfigMap +# and exposes a cilium_drift_checker_config_delta Prometheus metric reporting +# the number of keys that differ between the ConfigMap and the agent's active +# settings. A non-zero value indicates that the agent has not yet applied all +# current ConfigMap changes and needs to be restarted. +configDriftDetection: + # -- Enable watching of the cilium-config ConfigMap and reflecting its + # contents into the agent's internal DynamicConfig table. + enabled: true + # -- Enable the drift checker which compares the DynamicConfig table against + # the agent's active settings and publishes the + # cilium_drift_checker_config_delta metric. + driftChecker: true + # -- List of config-map keys to ignore when computing the drift delta. + ignoredKeys: [] # -- Agent container image. image: # @schema @@ -1148,9 +1170,89 @@ encryption: wireguard: # -- Controls WireGuard PersistentKeepalive option. Set 0s to disable. persistentKeepalive: 0s + # -- ztunnel encryption configuration. + # ztunnel is Istio's purpose-built, per-node proxy for handling L4 traffic in ambient mesh mode. + # These settings only apply when encryption.type is set to "ztunnel". + ztunnel: + # -- ztunnel container image. + image: + # @schema + # type: [null, string] + # @schema + override: ~ + repository: "docker.io/istio/ztunnel" + tag: "1.28.0-distroless" + pullPolicy: "IfNotPresent" + # @schema + # type: [null, string] + # @schema + digest: ~ + useDigest: false + # -- CA server address for certificate requests. + caAddress: "https://localhost:15012" + # -- TCP port for the health API. + healthPort: 15021 + # -- ztunnel resource limits & requests. + resources: + requests: + cpu: 200m + memory: 512Mi + # -- ztunnel update strategy. + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + # -- Configure termination grace period for ztunnel DaemonSet. + terminationGracePeriodSeconds: 30 + # -- Readiness probe configuration. + readinessProbe: + initialDelaySeconds: 0 + periodSeconds: 10 + failureThreshold: 3 + # -- Node selector for ztunnel pods. + nodeSelector: + kubernetes.io/os: linux + # -- Node tolerations for ztunnel scheduling. + tolerations: + - effect: NoSchedule + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + # -- Affinity for ztunnel pods. + affinity: {} + # @schema + # type: [null, string] + # @schema + # -- The priority class to use for ztunnel pods. + priorityClassName: ~ + # -- Annotations to be added to all ztunnel resources. + annotations: {} + # -- Annotations to be added to ztunnel pods. + podAnnotations: {} + # -- Labels to be added to ztunnel pods. + podLabels: {} + # -- Additional ztunnel container environment variables. + extraEnv: [] + # -- Additional ztunnel volumes. + extraVolumes: [] + # -- Additional ztunnel volumeMounts. + extraVolumeMounts: [] + # -- ztunnel secrets configuration. + secrets: + # -- Base64-encoded bootstrap root certificate content. + # If not provided, the secret must be created manually before deploying. + # @schema + # type: [null, string] + # @schema + bootstrapRootCert: ~ endpointHealthChecking: # -- Enable connectivity health checking between virtual endpoints. enabled: true +# -- Max duration to wait for envoy to respond to configuration changes. Default "10s". +endpointPolicyUpdateTimeoutDuration: null endpointRoutes: # @schema # type: [boolean, string] @@ -2339,8 +2441,6 @@ enableMasqueradeRouteSource: false enableIPv4BIGTCP: false # -- Enables IPv6 BIG TCP support which increases maximum IPv6 GSO/GRO limits for nodes and pods enableIPv6BIGTCP: false -# -- Enable BIG TCP in tunneling mode and increase maximum GRO/GSO limits for VXLAN/GENEVE tunnels -enableTunnelBIGTCP: false nat: # -- Number of the top-k SNAT map connections to track in Cilium statedb. @@ -2420,7 +2520,7 @@ loadBalancer: # -- serviceTopology enables K8s Topology Aware Hints -based service # endpoints filtering - # serviceTopology: false + serviceTopology: false # -- L7 LoadBalancer l7: diff --git a/packages/system/cilium/images/cilium/Dockerfile b/packages/system/cilium/images/cilium/Dockerfile index b62f9b46..32fc6fb8 100644 --- a/packages/system/cilium/images/cilium/Dockerfile +++ b/packages/system/cilium/images/cilium/Dockerfile @@ -1,2 +1,2 @@ -ARG VERSION=v1.19.1 +ARG VERSION=v1.19.3 FROM quay.io/cilium/cilium:${VERSION} From a78505e932650d6dcdab69feb01f6bb3c24f4807 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 23 Apr 2026 13:50:06 +0300 Subject: [PATCH 170/250] chore(cilium): refresh image digest for v1.19.3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Built ghcr.io/cozystack/cozystack/cilium from the refreshed upstream v1.19.3 base image and updated values.yaml with the new digest. Previously values.yaml still pointed at the v1.19.1 cozystack rebuild by digest while Chart.yaml and the Dockerfile were on v1.19.3 — with chart default useDigest=true that would have silently pulled v1.19.1 until the next release-tag rebuild. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- packages/system/cilium/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/system/cilium/values.yaml b/packages/system/cilium/values.yaml index cfb2cd6b..c1da6c38 100644 --- a/packages/system/cilium/values.yaml +++ b/packages/system/cilium/values.yaml @@ -15,8 +15,8 @@ cilium: mode: "kubernetes" image: repository: ghcr.io/cozystack/cozystack/cilium - tag: 1.19.1 - digest: "sha256:ab3acf270821df4614a8456348a4e0d3098aed72a4b2016a0edfa30d91428c3d" + tag: latest + digest: "sha256:8f5ab52982fc848ee098ff89919e3528aa8f3a553b82106a25149dcd0b87ea7e" envoy: enabled: false rollOutCiliumPods: true From 3f36a1b45b2406bc70ab49b4799d45ee88868e48 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 23 Apr 2026 17:14:23 +0300 Subject: [PATCH 171/250] fix(cilium): rebuild image multi-arch and pin tag to 1.19.3 The previous image digest in values.yaml pointed at a single-arch linux/arm64 manifest because 'make image' was run from an arm64 host with the default buildx platform. Cozystack targets amd64 (Talos build output, E2E runners, most real-world clusters) and also arm64 for hybrid fleets, so Helm install would fail on amd64 nodes with 'no matching manifest for linux/amd64 in the manifest list entries' whenever somebody installed directly from this commit between merge and the next release-tag CI rebuild. Fix: rebuilt the image locally with PLATFORM='linux/amd64,linux/arm64' make image from a buildx docker-container driver, pushed the multi-arch manifest, and refreshed values.yaml with: - digest of the new multi-arch manifest list (verified via 'docker manifest inspect': amd64 sha256:e1977323..., arm64 sha256:8f5ab529...). - tag bumped from 'latest' (emitted by the common-envs.mk settag macro on a non-tagged checkout) to '1.19.3', matching the established convention in every other packages/system/*/values.yaml so reviewers and incident response have a human-readable version anchor independent of digest chasing. The Makefile is left untouched so the CI builder (which only uses the default docker driver) keeps building single-arch for whatever architecture it runs on; multi-arch is a responsibility of the release-tag pipeline or an explicit local rebuild. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- packages/system/cilium/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/system/cilium/values.yaml b/packages/system/cilium/values.yaml index c1da6c38..9aba7cb9 100644 --- a/packages/system/cilium/values.yaml +++ b/packages/system/cilium/values.yaml @@ -15,8 +15,8 @@ cilium: mode: "kubernetes" image: repository: ghcr.io/cozystack/cozystack/cilium - tag: latest - digest: "sha256:8f5ab52982fc848ee098ff89919e3528aa8f3a553b82106a25149dcd0b87ea7e" + tag: 1.19.3 + digest: "sha256:700f06f4803a838a8e830be5ace4650e3ad82bdefabfb2f4d110368d307a5efb" envoy: enabled: false rollOutCiliumPods: true From adc7abe5c16fc81ce1c063d153d9a920c0aac097 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 23 Apr 2026 14:55:21 +0300 Subject: [PATCH 172/250] feat(ingress): add loadBalancer exposure mode via CiliumLoadBalancerIPPool Service.spec.externalIPs is deprecated upstream in Kubernetes v1.36 (KEP-5707, kubernetes#137293). The AllowServiceExternalIPs feature gate is expected to default to off around v1.40 and the implementation to be removed around v1.43. For bare-metal installs that rely on externalIPs today, cozystack needs a migration path. This change adds an opt-in 'loadBalancer' exposure mode for the ingress-nginx Service: - New platform value 'publishing.exposure' (enum: externalIPs | loadBalancer, default externalIPs). Plumbed through cozystack-values into each tenant's ingress HelmRelease via the new 'expose-mode' key. - Unknown values and loadBalancer with an empty externalIPs list fail the chart render with explicit error messages, rather than silently producing a broken Service. - When exposure=loadBalancer and the current namespace matches publishing.ingressName, the Service becomes type: LoadBalancer with externalTrafficPolicy: Local. - A new template renders a CiliumLoadBalancerIPPool whose blocks come from publishing.externalIPs (IPv4 addresses get /32, IPv6 addresses get /128) and whose serviceSelector uses Cilium's synthetic io.kubernetes.service.namespace key combined with the standard app.kubernetes.io/name: ingress-nginx label. No custom label is written to the Service itself, avoiding cross-tenant collisions from user-defined labels. Default behaviour is unchanged: without opting in, the Service is still ClusterIP + spec.externalIPs as today. Scope: only ingress-nginx is migrated by this setting. Other cozystack components that still write Service.spec.externalIPs directly (notably the vpn app) must be migrated separately before the v1.40 feature gate flip. Tests: packages/extra/ingress/tests/exposure_test.yaml adds 13 helm-unittest cases covering both modes, IPv4/IPv6, empty-token filtering, unknown-mode rejection, and the non-matching-namespace fallback. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- packages/core/platform/templates/apps.yaml | 1 + packages/core/platform/values.yaml | 35 +++ packages/extra/ingress/Makefile | 3 + packages/extra/ingress/README.md | 14 + .../ingress/templates/cilium-lb-pool.yaml | 25 ++ .../ingress/templates/nginx-ingress.yaml | 21 +- .../extra/ingress/tests/exposure_test.yaml | 251 ++++++++++++++++++ 7 files changed, 348 insertions(+), 2 deletions(-) create mode 100644 packages/extra/ingress/templates/cilium-lb-pool.yaml create mode 100644 packages/extra/ingress/tests/exposure_test.yaml diff --git a/packages/core/platform/templates/apps.yaml b/packages/core/platform/templates/apps.yaml index 7ee0de52..6e1ae25a 100644 --- a/packages/core/platform/templates/apps.yaml +++ b/packages/core/platform/templates/apps.yaml @@ -30,6 +30,7 @@ stringData: expose-services: {{ .Values.publishing.exposedServices | join "," | quote }} expose-ingress: {{ .Values.publishing.ingressName | quote }} expose-external-ips: {{ .Values.publishing.externalIPs | join "," | quote }} + expose-mode: {{ .Values.publishing.exposure | default "externalIPs" | quote }} cluster-domain: {{ .Values.networking.clusterDomain | quote }} api-server-endpoint: {{ .Values.publishing.apiServerEndpoint | quote }} {{- with .Values.branding }} diff --git a/packages/core/platform/values.yaml b/packages/core/platform/values.yaml index f33926db..605e23bc 100644 --- a/packages/core/platform/values.yaml +++ b/packages/core/platform/values.yaml @@ -45,6 +45,41 @@ publishing: - cdi-uploadproxy apiServerEndpoint: "" # example: "https://api.example.org" externalIPs: [] + # Exposure mode for the ingress-nginx Service. When "externalIPs" (current + # default) is selected, the Service is created as ClusterIP with + # Service.spec.externalIPs set from publishing.externalIPs. When + # "loadBalancer" is selected, the Service is type: LoadBalancer and a + # CiliumLoadBalancerIPPool makes those same addresses allocatable via LB IPAM. + # + # Service.spec.externalIPs is deprecated upstream in Kubernetes v1.36 + # (KEP-5707). The AllowServiceExternalIPs feature gate is expected to default + # to false around v1.40 and the implementation removed around v1.43 — switch + # to "loadBalancer" before upgrading past v1.40. + # + # Caveats for the "loadBalancer" mode: + # - publishing.externalIPs must contain at least one non-empty address, + # otherwise the chart render fails with an explicit error (a LoadBalancer + # Service without a pool would sit in forever). + # - The ingress-nginx Service is created with externalTrafficPolicy: Local + # to preserve the client source IP. Traffic arriving on a node that does + # not host an ingress-nginx pod is dropped, so the external IP must be + # routed to a node that runs the ingress pod (floating IP / keepalived / + # upstream router / podAntiAffinity). + # - Cilium does NOT announce the IP on its own unless L2 announcements or + # BGP are enabled in the Cilium values (disabled by default in Cozystack). + # This mode assumes the operator already routes the externalIPs to a + # cluster node; enabling announcements is out of scope for this setting. + # - Switching this value on a running cluster causes the ingress-nginx + # Service to be recreated (the HelmRelease has upgrade.force: true and + # the Service kind changes between ClusterIP and LoadBalancer). Expect a + # brief interruption of ingress traffic during the flip. + # + # Scope: this setting only controls the ingress-nginx Service. Other + # cozystack components that currently write Service.spec.externalIPs directly + # (e.g. the vpn app at packages/apps/vpn/templates/service.yaml) are NOT + # migrated by flipping this value and must be addressed separately before + # the AllowServiceExternalIPs feature gate flips to off in ~v1.40. + exposure: externalIPs # "externalIPs" or "loadBalancer" certificates: solver: http01 # "http01" or "dns01" issuerName: letsencrypt-prod diff --git a/packages/extra/ingress/Makefile b/packages/extra/ingress/Makefile index 958ce484..65b53c03 100644 --- a/packages/extra/ingress/Makefile +++ b/packages/extra/ingress/Makefile @@ -10,3 +10,6 @@ get-cloudflare-ips: generate: cozyvalues-gen -v values.yaml -s values.schema.json -r README.md ../../../hack/update-crd.sh + +test: + helm unittest . diff --git a/packages/extra/ingress/README.md b/packages/extra/ingress/README.md index 0e786dfb..c541d8e9 100644 --- a/packages/extra/ingress/README.md +++ b/packages/extra/ingress/README.md @@ -14,3 +14,17 @@ | `resources.memory` | Memory (RAM) available to each replica. | `quantity` | `""` | | `resourcesPreset` | Default sizing preset used when `resources` is omitted. | `string` | `micro` | + +## Exposure mode + +The ingress Service type is driven by the cluster-wide `publishing.exposure` value in the platform chart, not by any key in this package. Two modes exist: + +- `externalIPs` (default) has three rendered shapes: + - Release namespace matches `publishing.ingressName` AND `publishing.externalIPs` is non-empty → Service is `ClusterIP` with `Service.spec.externalIPs` set from that list and `externalTrafficPolicy: Cluster`. + - Release namespace matches `publishing.ingressName` but `publishing.externalIPs` is empty → Service falls back to `type: LoadBalancer` with `externalTrafficPolicy: Local`. + - Release namespace does not match `publishing.ingressName` (non-root tenants) → Service is `type: LoadBalancer` with `externalTrafficPolicy: Local`. + `Service.spec.externalIPs` is deprecated upstream in Kubernetes v1.36 (KEP-5707); plan migration before v1.40. +- `loadBalancer` — Service is `type: LoadBalancer` with `externalTrafficPolicy: Local`, and a `CiliumLoadBalancerIPPool` makes the addresses in `publishing.externalIPs` allocatable via Cilium LB IPAM. Requires `publishing.externalIPs` to contain at least one non-empty address (render fails otherwise) and assumes the addresses are already routed to a cluster node (floating IP / upstream router). See the inline comment on `publishing.exposure` in the platform chart for full caveats, including the note that switching the value on a running cluster causes the ingress Service to be recreated. + +This setting only migrates ingress-nginx away from `Service.spec.externalIPs`. Other cozystack components that use the same deprecated field (e.g. the `vpn` app) must be migrated separately before Kubernetes v1.40 flips the `AllowServiceExternalIPs` feature gate off. + diff --git a/packages/extra/ingress/templates/cilium-lb-pool.yaml b/packages/extra/ingress/templates/cilium-lb-pool.yaml new file mode 100644 index 00000000..a383922b --- /dev/null +++ b/packages/extra/ingress/templates/cilium-lb-pool.yaml @@ -0,0 +1,25 @@ +{{- $exposeIngress := (index .Values._cluster "expose-ingress") | default "tenant-root" }} +{{- $exposeMode := (index .Values._cluster "expose-mode") | default "externalIPs" }} +{{- $exposeExternalIPs := (index .Values._cluster "expose-external-ips") | default "" | nospace }} +{{- $exposeIPsList := list }} +{{- range splitList "," $exposeExternalIPs }} + {{- $ip := . | trim }} + {{- if $ip }} + {{- $exposeIPsList = append $exposeIPsList $ip }} + {{- end }} +{{- end }} +{{- if and (eq $exposeMode "loadBalancer") (eq $exposeIngress .Release.Namespace) $exposeIPsList }} +apiVersion: cilium.io/v2 +kind: CiliumLoadBalancerIPPool +metadata: + name: {{ trimPrefix "tenant-" .Release.Namespace }}-ingress +spec: + blocks: + {{- range $exposeIPsList }} + - cidr: {{ . }}/{{ if contains ":" . }}128{{ else }}32{{ end }} + {{- end }} + serviceSelector: + matchLabels: + "io.kubernetes.service.namespace": {{ .Release.Namespace | quote }} + "app.kubernetes.io/name": ingress-nginx +{{- end }} diff --git a/packages/extra/ingress/templates/nginx-ingress.yaml b/packages/extra/ingress/templates/nginx-ingress.yaml index ca50d276..8e1f00fb 100644 --- a/packages/extra/ingress/templates/nginx-ingress.yaml +++ b/packages/extra/ingress/templates/nginx-ingress.yaml @@ -1,5 +1,19 @@ {{- $exposeIngress := (index .Values._cluster "expose-ingress") | default "tenant-root" }} {{- $exposeExternalIPs := (index .Values._cluster "expose-external-ips") | default "" | nospace }} +{{- $exposeMode := (index .Values._cluster "expose-mode") | default "externalIPs" }} +{{- $exposeIPsList := list }} +{{- range splitList "," $exposeExternalIPs }} + {{- $ip := . | trim }} + {{- if $ip }} + {{- $exposeIPsList = append $exposeIPsList $ip }} + {{- end }} +{{- end }} +{{- if not (has $exposeMode (list "externalIPs" "loadBalancer")) }} +{{- fail (printf "unknown publishing.exposure mode %q: must be \"externalIPs\" or \"loadBalancer\"" $exposeMode) }} +{{- end }} +{{- if and (eq $exposeMode "loadBalancer") (eq $exposeIngress .Release.Namespace) (not $exposeIPsList) }} +{{- fail "publishing.exposure=loadBalancer requires publishing.externalIPs to contain at least one non-empty address: the CiliumLoadBalancerIPPool has nothing to advertise and the Service would stay in ." }} +{{- end }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -41,9 +55,12 @@ spec: enabled: false {{- end }} service: - {{- if and (eq $exposeIngress .Release.Namespace) $exposeExternalIPs }} + {{- if and (eq $exposeIngress .Release.Namespace) (eq $exposeMode "loadBalancer") }} + type: LoadBalancer + externalTrafficPolicy: Local + {{- else if and (eq $exposeIngress .Release.Namespace) $exposeIPsList }} externalIPs: - {{- toYaml (splitList "," $exposeExternalIPs) | nindent 12 }} + {{- toYaml $exposeIPsList | nindent 12 }} type: ClusterIP externalTrafficPolicy: Cluster {{- else }} diff --git a/packages/extra/ingress/tests/exposure_test.yaml b/packages/extra/ingress/tests/exposure_test.yaml new file mode 100644 index 00000000..780e0715 --- /dev/null +++ b/packages/extra/ingress/tests/exposure_test.yaml @@ -0,0 +1,251 @@ +suite: ingress exposure modes +templates: + - templates/nginx-ingress.yaml + - templates/cilium-lb-pool.yaml + +release: + name: ingress + namespace: tenant-root + +tests: + - it: default exposure (externalIPs) renders ClusterIP Service with spec.externalIPs and no CiliumLoadBalancerIPPool + set: + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10,192.0.2.11" + asserts: + - template: templates/nginx-ingress.yaml + equal: + path: spec.values.ingress-nginx.controller.service.type + value: ClusterIP + - template: templates/nginx-ingress.yaml + equal: + path: spec.values.ingress-nginx.controller.service.externalTrafficPolicy + value: Cluster + - template: templates/nginx-ingress.yaml + equal: + path: spec.values.ingress-nginx.controller.service.externalIPs + value: + - 192.0.2.10 + - 192.0.2.11 + - template: templates/nginx-ingress.yaml + notExists: + path: spec.values.ingress-nginx.controller.service.labels + - template: templates/cilium-lb-pool.yaml + hasDocuments: + count: 0 + + - it: legacy config without expose-mode falls back to externalIPs behavior + set: + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10" + asserts: + - template: templates/nginx-ingress.yaml + equal: + path: spec.values.ingress-nginx.controller.service.type + value: ClusterIP + - template: templates/cilium-lb-pool.yaml + hasDocuments: + count: 0 + + - it: externalIPs mode in a namespace other than publishing.ingressName renders LoadBalancer fallback without externalIPs + set: + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10" + release: + namespace: tenant-other + asserts: + - template: templates/nginx-ingress.yaml + equal: + path: spec.values.ingress-nginx.controller.service.type + value: LoadBalancer + - template: templates/nginx-ingress.yaml + equal: + path: spec.values.ingress-nginx.controller.service.externalTrafficPolicy + value: Local + - template: templates/nginx-ingress.yaml + notExists: + path: spec.values.ingress-nginx.controller.service.externalIPs + - template: templates/cilium-lb-pool.yaml + hasDocuments: + count: 0 + + - it: loadBalancer mode renders LoadBalancer Service with lb-pool label and a v2 CiliumLoadBalancerIPPool + set: + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10,192.0.2.11" + expose-mode: loadBalancer + asserts: + - template: templates/nginx-ingress.yaml + equal: + path: spec.values.ingress-nginx.controller.service.type + value: LoadBalancer + - template: templates/nginx-ingress.yaml + equal: + path: spec.values.ingress-nginx.controller.service.externalTrafficPolicy + value: Local + - template: templates/nginx-ingress.yaml + notExists: + path: spec.values.ingress-nginx.controller.service.labels + - template: templates/nginx-ingress.yaml + notExists: + path: spec.values.ingress-nginx.controller.service.externalIPs + - template: templates/cilium-lb-pool.yaml + hasDocuments: + count: 1 + - template: templates/cilium-lb-pool.yaml + equal: + path: apiVersion + value: cilium.io/v2 + - template: templates/cilium-lb-pool.yaml + equal: + path: kind + value: CiliumLoadBalancerIPPool + - template: templates/cilium-lb-pool.yaml + equal: + path: metadata.name + value: root-ingress + - template: templates/cilium-lb-pool.yaml + equal: + path: spec.blocks + value: + - cidr: 192.0.2.10/32 + - cidr: 192.0.2.11/32 + - template: templates/cilium-lb-pool.yaml + equal: + path: spec.serviceSelector.matchLabels["io.kubernetes.service.namespace"] + value: tenant-root + - template: templates/cilium-lb-pool.yaml + equal: + path: spec.serviceSelector.matchLabels["app.kubernetes.io/name"] + value: ingress-nginx + + - it: loadBalancer mode with IPv6 address emits /128 CIDR + set: + _cluster: + expose-ingress: tenant-root + expose-external-ips: "2001:db8::1" + expose-mode: loadBalancer + asserts: + - template: templates/cilium-lb-pool.yaml + equal: + path: spec.blocks + value: + - cidr: 2001:db8::1/128 + + - it: loadBalancer mode with mixed IPv4 and IPv6 emits correct CIDR per family + set: + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10,2001:db8::1" + expose-mode: loadBalancer + asserts: + - template: templates/cilium-lb-pool.yaml + equal: + path: spec.blocks + value: + - cidr: 192.0.2.10/32 + - cidr: 2001:db8::1/128 + + - it: loadBalancer mode without externalIPs fails chart render with explicit message + set: + _cluster: + expose-ingress: tenant-root + expose-external-ips: "" + expose-mode: loadBalancer + asserts: + - template: templates/nginx-ingress.yaml + failedTemplate: + errorMessage: "publishing.exposure=loadBalancer requires publishing.externalIPs to contain at least one non-empty address: the CiliumLoadBalancerIPPool has nothing to advertise and the Service would stay in ." + + - it: unknown exposure mode is rejected with a clear error (case-sensitive enum) + set: + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10" + expose-mode: LoadBalancer + asserts: + - template: templates/nginx-ingress.yaml + failedTemplate: + errorMessage: "unknown publishing.exposure mode \"LoadBalancer\": must be \"externalIPs\" or \"loadBalancer\"" + + - it: another typo in exposure mode also fails + set: + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10" + expose-mode: loadbalancer + asserts: + - template: templates/nginx-ingress.yaml + failedTemplate: + errorMessage: "unknown publishing.exposure mode \"loadbalancer\": must be \"externalIPs\" or \"loadBalancer\"" + + - it: loadBalancer mode in a namespace other than publishing.ingressName falls back to LoadBalancer Service without pool + set: + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10" + expose-mode: loadBalancer + release: + namespace: tenant-other + asserts: + - template: templates/nginx-ingress.yaml + equal: + path: spec.values.ingress-nginx.controller.service.type + value: LoadBalancer + - template: templates/nginx-ingress.yaml + equal: + path: spec.values.ingress-nginx.controller.service.externalTrafficPolicy + value: Local + - template: templates/nginx-ingress.yaml + notExists: + path: spec.values.ingress-nginx.controller.service.labels + - template: templates/nginx-ingress.yaml + notExists: + path: spec.values.ingress-nginx.controller.service.externalIPs + - template: templates/cilium-lb-pool.yaml + hasDocuments: + count: 0 + + - it: loadBalancer mode filters out empty entries from externalIPs (trailing comma, leading comma) + set: + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10,,192.0.2.11," + expose-mode: loadBalancer + asserts: + - template: templates/cilium-lb-pool.yaml + hasDocuments: + count: 1 + - template: templates/cilium-lb-pool.yaml + equal: + path: spec.blocks + value: + - cidr: 192.0.2.10/32 + - cidr: 192.0.2.11/32 + + - it: loadBalancer mode with only-empty externalIPs fails chart render (comma-only input) + set: + _cluster: + expose-ingress: tenant-root + expose-external-ips: ",," + expose-mode: loadBalancer + asserts: + - template: templates/nginx-ingress.yaml + failedTemplate: + errorMessage: "publishing.exposure=loadBalancer requires publishing.externalIPs to contain at least one non-empty address: the CiliumLoadBalancerIPPool has nothing to advertise and the Service would stay in ." + + - it: externalIPs mode also filters out empty entries (trailing comma) + set: + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10," + asserts: + - template: templates/nginx-ingress.yaml + equal: + path: spec.values.ingress-nginx.controller.service.externalIPs + value: + - 192.0.2.10 From c34a9db6bd2d052e290cb550713369e82d1f3d0a Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 23 Apr 2026 17:59:49 +0300 Subject: [PATCH 173/250] docs(agents): broaden make generate example to apps packages Address review feedback from coderabbitai on docs/agents/contributing.md:26: Replace the hard-coded packages/extra/ path in the example with packages// so the example matches the preceding text that describes both apps and extra packages. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- docs/agents/contributing.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/agents/contributing.md b/docs/agents/contributing.md index 9626e81e..5b1c6ba5 100644 --- a/docs/agents/contributing.md +++ b/docs/agents/contributing.md @@ -21,8 +21,8 @@ Several files in each package are produced by `make generate` from `values.yaml` **Before committing edits to any of those sources**, run `make generate` inside the package and stage the full diff: ```bash -make -C packages/extra/ generate -git add packages/extra// packages/system/-rd/ +make -C packages// generate +git add packages/// packages/system/-rd/ ``` The repo's pre-commit CI job runs `make generate` in every package and then `git diff --exit-code`. Any unstaged generator output fails the job with exit code 123 and blocks the PR. Also rerun `make generate` after a `git commit --amend` if the amended change touched any of the sources above. From 41fd80711bbe4a3cdb12b3af0bcd8bfdd40993e5 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 23 Apr 2026 18:00:07 +0300 Subject: [PATCH 174/250] docs(agents): fix grammar in regen discovery hint Address review feedback from gemini-code-assist on docs/agents/contributing.md:30: Reword "likely to need regenerated" (regional construction) to "likely needs to be regenerated" for standard technical prose. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- docs/agents/contributing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/agents/contributing.md b/docs/agents/contributing.md index 5b1c6ba5..cef12494 100644 --- a/docs/agents/contributing.md +++ b/docs/agents/contributing.md @@ -27,7 +27,7 @@ git add packages/// packages/system/-rd/ The repo's pre-commit CI job runs `make generate` in every package and then `git diff --exit-code`. Any unstaged generator output fails the job with exit code 123 and blocks the PR. Also rerun `make generate` after a `git commit --amend` if the amended change touched any of the sources above. -To locate packages a WIP branch is likely to need regenerated: +To locate packages a WIP branch likely needs to be regenerated: ```bash git diff --name-only | xargs -n1 dirname | sort -u | grep ^packages/ From 4813566a30fb60eca00ce0ac83254daabf65341f Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 23 Apr 2026 18:02:04 +0300 Subject: [PATCH 175/250] docs(agents): mark scopes list as illustrative examples Address review feedback from gemini-code-assist on docs/agents/contributing.md:11: Scope linters kept flagging valid scopes like 'agents' as unknown because the list read as exhaustive. Annotate it as examples (not exhaustive) and add 'agents' to the Other group so both humans and review bots stop tripping on scopes that are already in regular use across the repo history. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- docs/agents/contributing.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/agents/contributing.md b/docs/agents/contributing.md index cef12494..658dab74 100644 --- a/docs/agents/contributing.md +++ b/docs/agents/contributing.md @@ -43,10 +43,10 @@ git commit --signoff -m "type(scope): brief description" **Types:** `feat`, `fix`, `docs`, `style`, `refactor`, `perf`, `test`, `build`, `ci`, `chore` -**Scopes:** +**Scopes** (e.g., not exhaustive — use any scope that names the component you are touching): - System: `dashboard`, `platform`, `cilium`, `kube-ovn`, `linstor`, `fluxcd`, `cluster-api` - Apps: `postgres`, `mariadb`, `redis`, `kafka`, `clickhouse`, `virtual-machine`, `kubernetes` -- Other: `api`, `hack`, `tests`, `ci`, `docs`, `maintenance` +- Other: `api`, `hack`, `tests`, `ci`, `docs`, `agents`, `maintenance` Breaking changes: append `!` after type/scope (`feat(api)!: ...`) or add a `BREAKING CHANGE:` footer. From 64a3edff01533a0483db31686a6385dffa81ee55 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Thu, 23 Apr 2026 18:13:10 +0300 Subject: [PATCH 176/250] fix(api): prevent IDOR in TenantNamespace Get and Watch handlers Fixed two IDOR vulnerabilities allowing authenticated users to access metadata of any tenant namespace without proper authorization. Changes: - Added hasAccessToNamespace() for efficient single-namespace access checks - Get() now verifies access before returning namespace metadata - Watch() filters events per-namespace with proper authorization - Returns NotFound (not Forbidden) to prevent tenant enumeration Performance optimization: - hasAccessToNamespace() lists RoleBindings only in target namespace instead of listing all cluster RoleBindings (order of magnitude faster) - Watch handler logs authorization errors for security audit Additional fixes: - Handle ServiceAccount subjects with empty namespace correctly - Add klog error logging for failed authorization checks Signed-off-by: IvanHunters --- pkg/registry/core/tenantnamespace/rest.go | 89 ++++++++++++++++++++++- 1 file changed, 87 insertions(+), 2 deletions(-) diff --git a/pkg/registry/core/tenantnamespace/rest.go b/pkg/registry/core/tenantnamespace/rest.go index f1ed3fab..7724c0e1 100644 --- a/pkg/registry/core/tenantnamespace/rest.go +++ b/pkg/registry/core/tenantnamespace/rest.go @@ -24,6 +24,7 @@ import ( "k8s.io/apimachinery/pkg/watch" "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/rest" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" corev1alpha1 "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1" @@ -123,8 +124,18 @@ func (r *REST) Get( return nil, apierrors.NewNotFound(r.gvr.GroupResource(), name) } + // Check if user has access to this namespace + hasAccess, err := r.hasAccessToNamespace(ctx, name) + if err != nil { + return nil, err + } + if !hasAccess { + // Return NotFound instead of Forbidden to prevent enumeration + return nil, apierrors.NewNotFound(r.gvr.GroupResource(), name) + } + ns := &corev1.Namespace{} - err := r.c.Get(ctx, types.NamespacedName{Namespace: "", Name: name}, ns, &client.GetOptions{Raw: opts}) + err = r.c.Get(ctx, types.NamespacedName{Namespace: "", Name: name}, ns, &client.GetOptions{Raw: opts}) if err != nil { return nil, err } @@ -189,6 +200,17 @@ func (r *REST) Watch(ctx context.Context, opts *metainternal.ListOptions) (watch continue } + // Check if user has access to this namespace + hasAccess, err := r.hasAccessToNamespace(ctx, ns.Name) + if err != nil { + klog.Errorf("Failed to check access for namespace %s in watch: %v", ns.Name, err) + continue + } + if !hasAccess { + // User doesn't have access, skip this event + continue + } + out := &corev1alpha1.TenantNamespace{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1alpha1.SchemeGroupVersion.String(), @@ -359,7 +381,11 @@ func (r *REST) filterAccessible( break subjectLoop } case "ServiceAccount": - if u.GetName() == fmt.Sprintf("system:serviceaccount:%s:%s", subj.Namespace, subj.Name) { + saNamespace := subj.Namespace + if saNamespace == "" { + saNamespace = rbs.Items[i].Namespace + } + if u.GetName() == fmt.Sprintf("system:serviceaccount:%s:%s", saNamespace, subj.Name) { allowedNameSet[rbs.Items[i].Namespace] = struct{}{} break subjectLoop } @@ -373,6 +399,65 @@ func (r *REST) filterAccessible( return allowed, nil } +// hasAccessToNamespace checks if the user has access to a single namespace. +// This is optimized for Get/Watch operations where we check one namespace at a time. +// It lists RoleBindings only in the target namespace instead of all cluster RoleBindings. +func (r *REST) hasAccessToNamespace( + ctx context.Context, + namespace string, +) (bool, error) { + u, ok := request.UserFrom(ctx) + if !ok { + return false, fmt.Errorf("user missing in context") + } + + // Check privileged groups + groups := make(map[string]struct{}) + for _, group := range u.GetGroups() { + groups[group] = struct{}{} + } + if _, ok := groups["system:masters"]; ok { + return true, nil + } + if _, ok := groups["cozystack-cluster-admin"]; ok { + return true, nil + } + + // List RoleBindings only in the target namespace + rbs := &rbacv1.RoleBindingList{} + err := r.c.List(ctx, rbs, client.InNamespace(namespace)) + if err != nil { + return false, fmt.Errorf("failed to list rolebindings in %s: %w", namespace, err) + } + + // Check if user is in any RoleBinding subjects + for i := range rbs.Items { + for j := range rbs.Items[i].Subjects { + subj := rbs.Items[i].Subjects[j] + switch subj.Kind { + case "Group": + if _, ok := groups[subj.Name]; ok { + return true, nil + } + case "User": + if subj.Name == u.GetName() { + return true, nil + } + case "ServiceAccount": + saNamespace := subj.Namespace + if saNamespace == "" { + saNamespace = rbs.Items[i].Namespace + } + if u.GetName() == fmt.Sprintf("system:serviceaccount:%s:%s", saNamespace, subj.Name) { + return true, nil + } + } + } + } + + return false, nil +} + // ----------------------------------------------------------------------------- // Boiler-plate // ----------------------------------------------------------------------------- From 5b2501db91c82d8f9ce1d7187325927f7504847c Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Thu, 23 Apr 2026 18:17:47 +0300 Subject: [PATCH 177/250] test(api): add security tests for TenantNamespace IDOR fix Added comprehensive unit tests for authorization logic: - hasAccessToNamespace() tests: - User subject access (positive and negative cases) - Group subject access - ServiceAccount subject access - ServiceAccount with empty namespace (defaults to RoleBinding ns) - Privileged groups (system:masters, cozystack-cluster-admin) - Get() handler tests: - Returns namespace when user has access - Returns NotFound when user lacks access (not Forbidden) - Returns NotFound for non-tenant namespaces All tests verify that authorization correctly enforces RoleBinding-based access control and prevents IDOR vulnerabilities. Signed-off-by: IvanHunters --- .../core/tenantnamespace/rest_test.go | 461 ++++++++++++++++++ 1 file changed, 461 insertions(+) diff --git a/pkg/registry/core/tenantnamespace/rest_test.go b/pkg/registry/core/tenantnamespace/rest_test.go index 7f2979bc..eb678949 100644 --- a/pkg/registry/core/tenantnamespace/rest_test.go +++ b/pkg/registry/core/tenantnamespace/rest_test.go @@ -3,10 +3,18 @@ package tenantnamespace import ( + "context" "testing" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/apiserver/pkg/endpoints/request" + "sigs.k8s.io/controller-runtime/pkg/client/fake" ) func TestMakeListSortsAlphabetically(t *testing.T) { @@ -38,3 +46,456 @@ func TestMakeListSortsAlphabetically(t *testing.T) { } } } + +// Security tests for IDOR fix + +func TestHasAccessToNamespace_WithUserAccess(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = rbacv1.AddToScheme(scheme) + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, + } + + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-binding", + Namespace: "tenant-test", + }, + Subjects: []rbacv1.Subject{ + {Kind: "User", Name: "test-user", APIGroup: "rbac.authorization.k8s.io"}, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: "test-role", + }, + } + + client := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(ns, rb). + Build() + + r := &REST{ + c: client, + gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, + } + + u := &user.DefaultInfo{ + Name: "test-user", + Groups: []string{"system:authenticated"}, + } + ctx := request.WithUser(context.Background(), u) + + hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !hasAccess { + t.Error("expected user to have access, but got false") + } +} + +func TestHasAccessToNamespace_WithoutAccess(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = rbacv1.AddToScheme(scheme) + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, + } + + // RoleBinding for different user + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-binding", + Namespace: "tenant-test", + }, + Subjects: []rbacv1.Subject{ + {Kind: "User", Name: "other-user", APIGroup: "rbac.authorization.k8s.io"}, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: "test-role", + }, + } + + client := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(ns, rb). + Build() + + r := &REST{ + c: client, + gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, + } + + u := &user.DefaultInfo{ + Name: "test-user", + Groups: []string{"system:authenticated"}, + } + ctx := request.WithUser(context.Background(), u) + + hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if hasAccess { + t.Error("expected user to NOT have access, but got true") + } +} + +func TestHasAccessToNamespace_WithGroupAccess(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = rbacv1.AddToScheme(scheme) + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, + } + + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-binding", + Namespace: "tenant-test", + }, + Subjects: []rbacv1.Subject{ + {Kind: "Group", Name: "test-group", APIGroup: "rbac.authorization.k8s.io"}, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: "test-role", + }, + } + + client := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(ns, rb). + Build() + + r := &REST{ + c: client, + gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, + } + + u := &user.DefaultInfo{ + Name: "test-user", + Groups: []string{"system:authenticated", "test-group"}, + } + ctx := request.WithUser(context.Background(), u) + + hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !hasAccess { + t.Error("expected user to have access via group, but got false") + } +} + +func TestHasAccessToNamespace_SystemMasters(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = rbacv1.AddToScheme(scheme) + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, + } + + client := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(ns). + Build() + + r := &REST{ + c: client, + gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, + } + + u := &user.DefaultInfo{ + Name: "admin", + Groups: []string{"system:masters"}, + } + ctx := request.WithUser(context.Background(), u) + + hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !hasAccess { + t.Error("expected system:masters to have access, but got false") + } +} + +func TestHasAccessToNamespace_CozyAdminGroup(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = rbacv1.AddToScheme(scheme) + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, + } + + client := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(ns). + Build() + + r := &REST{ + c: client, + gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, + } + + u := &user.DefaultInfo{ + Name: "cozy-admin", + Groups: []string{"cozystack-cluster-admin"}, + } + ctx := request.WithUser(context.Background(), u) + + hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !hasAccess { + t.Error("expected cozystack-cluster-admin to have access, but got false") + } +} + +func TestHasAccessToNamespace_ServiceAccount(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = rbacv1.AddToScheme(scheme) + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, + } + + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-binding", + Namespace: "tenant-test", + }, + Subjects: []rbacv1.Subject{ + {Kind: "ServiceAccount", Name: "test-sa", Namespace: "tenant-test"}, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: "test-role", + }, + } + + client := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(ns, rb). + Build() + + r := &REST{ + c: client, + gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, + } + + u := &user.DefaultInfo{ + Name: "system:serviceaccount:tenant-test:test-sa", + Groups: []string{"system:authenticated"}, + } + ctx := request.WithUser(context.Background(), u) + + hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !hasAccess { + t.Error("expected service account to have access, but got false") + } +} + +func TestHasAccessToNamespace_ServiceAccountEmptyNamespace(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = rbacv1.AddToScheme(scheme) + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, + } + + // ServiceAccount subject with empty namespace should default to RoleBinding namespace + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-binding", + Namespace: "tenant-test", + }, + Subjects: []rbacv1.Subject{ + {Kind: "ServiceAccount", Name: "test-sa", Namespace: ""}, // Empty namespace + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: "test-role", + }, + } + + client := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(ns, rb). + Build() + + r := &REST{ + c: client, + gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, + } + + u := &user.DefaultInfo{ + Name: "system:serviceaccount:tenant-test:test-sa", + Groups: []string{"system:authenticated"}, + } + ctx := request.WithUser(context.Background(), u) + + hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !hasAccess { + t.Error("expected service account with empty namespace to have access, but got false") + } +} + +func TestGet_WithAccess(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = rbacv1.AddToScheme(scheme) + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, + } + + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-binding", + Namespace: "tenant-test", + }, + Subjects: []rbacv1.Subject{ + {Kind: "User", Name: "test-user", APIGroup: "rbac.authorization.k8s.io"}, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: "test-role", + }, + } + + client := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(ns, rb). + Build() + + r := &REST{ + c: client, + gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, + } + + u := &user.DefaultInfo{ + Name: "test-user", + Groups: []string{"system:authenticated"}, + } + ctx := request.WithUser(context.Background(), u) + + obj, err := r.Get(ctx, "tenant-test", &metav1.GetOptions{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if obj == nil { + t.Fatal("expected object, got nil") + } +} + +func TestGet_WithoutAccess(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = rbacv1.AddToScheme(scheme) + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, + } + + // RoleBinding for different user + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-binding", + Namespace: "tenant-test", + }, + Subjects: []rbacv1.Subject{ + {Kind: "User", Name: "other-user", APIGroup: "rbac.authorization.k8s.io"}, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: "test-role", + }, + } + + client := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(ns, rb). + Build() + + r := &REST{ + c: client, + gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, + } + + u := &user.DefaultInfo{ + Name: "test-user", + Groups: []string{"system:authenticated"}, + } + ctx := request.WithUser(context.Background(), u) + + obj, err := r.Get(ctx, "tenant-test", &metav1.GetOptions{}) + if err == nil { + t.Fatal("expected error, got nil") + } + if obj != nil { + t.Errorf("expected nil object, got %v", obj) + } + + // Verify it returns NotFound (not Forbidden) to prevent enumeration + if !apierrors.IsNotFound(err) { + t.Errorf("expected NotFound error, got %v", err) + } +} + +func TestGet_NonTenantNamespace(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = rbacv1.AddToScheme(scheme) + + client := fake.NewClientBuilder(). + WithScheme(scheme). + Build() + + r := &REST{ + c: client, + gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, + } + + u := &user.DefaultInfo{ + Name: "test-user", + Groups: []string{"system:masters"}, + } + ctx := request.WithUser(context.Background(), u) + + obj, err := r.Get(ctx, "default", &metav1.GetOptions{}) + if err == nil { + t.Fatal("expected error for non-tenant namespace, got nil") + } + if obj != nil { + t.Errorf("expected nil object, got %v", obj) + } + if !apierrors.IsNotFound(err) { + t.Errorf("expected NotFound error, got %v", err) + } +} From 9d552d4086726d2f5c6de5adaa2783082893a8c5 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Thu, 23 Apr 2026 22:19:35 +0500 Subject: [PATCH 178/250] ci(api): broaden codegen drift trigger paths and detect untracked files - Add generated-output dirs (pkg/generated, internal/crdinstall/manifests, packages/system/*/definitions) and Makefile to the workflow paths: filter so PRs that modify only generated artifacts still trigger the drift check. - Mirror the same paths in the root pre-commit hook's files: regex so manual edits to generated files or changes to the root generate target re-run make generate through pre-commit. - Switch drift detection in the workflow from `git diff --exit-code` to `git status --porcelain` so new untracked files produced by make generate (e.g. generated YAML/Go for a new API type) also fail the job; dump `git diff --color=always` on failure for easier debugging. Signed-off-by: Myasnikov Daniil --- .github/workflows/codegen-drift.yml | 10 +++++++++- .pre-commit-config.yaml | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codegen-drift.yml b/.github/workflows/codegen-drift.yml index b08f5d7e..a26caf5e 100644 --- a/.github/workflows/codegen-drift.yml +++ b/.github/workflows/codegen-drift.yml @@ -6,8 +6,15 @@ on: paths: - 'api/**' - 'pkg/apis/**' + - 'pkg/generated/**' + - 'internal/crdinstall/manifests/**' + - 'packages/system/cozystack-controller/definitions/**' + - 'packages/system/application-definition-crd/definition/**' + - 'packages/system/backup-controller/definitions/**' + - 'packages/system/backupstrategy-controller/definitions/**' - 'hack/update-codegen.sh' - 'hack/boilerplate.go.txt' + - 'Makefile' - 'go.mod' - 'go.sum' - '.github/workflows/codegen-drift.yml' @@ -46,8 +53,9 @@ jobs: - name: Fail on drift run: | - if ! git diff --exit-code; then + if [ -n "$(git status --porcelain)" ]; then echo "::error::'make generate' produced changes. Run 'make generate' locally and commit the result." git status --short + git diff --color=always exit 1 fi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 836f79c5..6492b92d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ repos: git diff --color=always | cat ' language: system - files: ^(api/|pkg/apis/|hack/update-codegen\.sh$|hack/boilerplate\.go\.txt$) + files: ^(api/|pkg/apis/|pkg/generated/|internal/crdinstall/manifests/|packages/system/cozystack-controller/definitions/|packages/system/application-definition-crd/definition/|packages/system/backup-controller/definitions/|packages/system/backupstrategy-controller/definitions/|hack/update-codegen\.sh$|hack/boilerplate\.go\.txt$|Makefile$) pass_filenames: false - id: run-make-generate name: Run 'make generate' in all app directories From 3c5521ee992b2dce74ff8956a6d8c581763bf640 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Thu, 23 Apr 2026 22:16:30 +0300 Subject: [PATCH 179/250] fix(hami): remove broken hami-dra subchart The hami-dra subchart renders resources even when dra.enabled=false because Helm dependency conditions don't work for vendored subcharts. The subchart also contains duplicate YAML label keys (app.kubernetes.io/component, app.kubernetes.io/name) and references unpublished container images (v0.0.1-dev). DRA support can be re-added when the upstream project stabilizes it. Signed-off-by: Arsolitt --- .../charts/hami/charts/hami-dra/.helmignore | 24 -- .../charts/hami/charts/hami-dra/Chart.yaml | 18 - .../charts/hami-dra/templates/_commons.tpl | 84 ---- .../charts/hami-dra/templates/_helpers.tpl | 73 ---- .../hami-dra-driver/deviceclass.yaml | 11 - .../nvidia-dra-driver-daemonset.yaml | 145 ------- .../templates/hami-dra-driver/rbac.yaml | 110 ----- .../monitor/monitor-clusterrole.yaml | 11 - .../monitor/monitor-clusterrolebinding.yaml | 15 - .../templates/monitor/monitor-deployment.yaml | 62 --- .../templates/monitor/monitor-service.yaml | 26 -- .../monitor/monitor-serviceaccount.yaml | 8 - .../templates/webhook/cert-manager.yaml | 29 -- .../templates/webhook/cert-secret.yaml | 13 - .../templates/webhook/device-config.yaml | 394 ------------------ .../webhook/mutatingwebhookconfiguration.yaml | 28 -- .../validatingwebhookconfiguration.yaml | 29 -- .../webhook/webhook-clusterrole.yaml | 14 - .../webhook/webhook-clusterrolebinding.yaml | 12 - .../templates/webhook/webhook-deployment.yaml | 59 --- .../templates/webhook/webhook-service.yaml | 15 - .../webhook/webhook-serviceaccount.yaml | 5 - .../charts/hami/charts/hami-dra/values.yaml | 166 -------- 23 files changed, 1351 deletions(-) delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/.helmignore delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/_commons.tpl delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/_helpers.tpl delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/deviceclass.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/nvidia-dra-driver-daemonset.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/rbac.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrole.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrolebinding.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-deployment.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-service.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-serviceaccount.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-manager.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-secret.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/device-config.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/mutatingwebhookconfiguration.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/validatingwebhookconfiguration.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrole.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrolebinding.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-deployment.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-service.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-serviceaccount.yaml delete mode 100644 packages/system/hami/charts/hami/charts/hami-dra/values.yaml diff --git a/packages/system/hami/charts/hami/charts/hami-dra/.helmignore b/packages/system/hami/charts/hami/charts/hami-dra/.helmignore deleted file mode 100644 index 898df488..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/.helmignore +++ /dev/null @@ -1,24 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ - diff --git a/packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml b/packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml deleted file mode 100644 index d01f678f..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/Chart.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: v2 -appVersion: 0.1.0 -description: A Helm chart for HAMi DRA -home: https://github.com/Project-HAMi/HAMi-DRA -keywords: -- webhook -- kubernetes -- admission-controller -- hami -- dra -maintainers: -- name: hami-dra - url: https://github.com/Project-HAMi/HAMi-DRA -name: hami-dra -sources: -- https://github.com/Project-HAMi/HAMi-DRA -type: application -version: 0.1.0 diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/_commons.tpl b/packages/system/hami/charts/hami/charts/hami-dra/templates/_commons.tpl deleted file mode 100644 index 02fbed20..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/_commons.tpl +++ /dev/null @@ -1,84 +0,0 @@ -{{/* vim: set filetype=mustache: */}} - -{{/* -Return the proper image name -{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" $) }} -*/}} -{{- define "common.images.image" -}} -{{- $registryName := .imageRoot.registry -}} -{{- $repositoryName := .imageRoot.repository -}} -{{- $tag := .imageRoot.tag | toString -}} -{{- if .global }} - {{- if .global.imageRegistry }} - {{- $registryName = .global.imageRegistry -}} - {{- end -}} -{{- end -}} -{{- if .tag }} - {{- if .tag.imageTag }} - {{- $tag = .tag.imageTag -}} - {{- end -}} -{{- end -}} -{{- if $registryName }} -{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} -{{- else -}} -{{- printf "%s:%s" $repositoryName $tag -}} -{{- end -}} -{{- end -}} - -{{/* -Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead) -{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }} -*/}} -{{- define "common.images.pullSecrets" -}} - {{- $pullSecrets := list }} - - {{- if .global }} - {{- range .global.imagePullSecrets -}} - {{- $pullSecrets = append $pullSecrets . -}} - {{- end -}} - {{- end -}} - - {{- range .images -}} - {{- range .pullSecrets -}} - {{- $pullSecrets = append $pullSecrets . -}} - {{- end -}} - {{- end -}} - - {{- if (not (empty $pullSecrets)) }} -imagePullSecrets: - {{- range $pullSecrets }} - - name: {{ . }} - {{- end }} - {{- end }} -{{- end -}} - -{{/* -Renders a value that contains template. -Usage: -{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $) }} -*/}} -{{- define "common.tplvalues.render" -}} - {{- if typeIs "string" .value }} - {{- tpl .value .context }} - {{- else }} - {{- tpl (.value | toYaml) .context }} - {{- end }} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "common.names.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/_helpers.tpl b/packages/system/hami/charts/hami/charts/hami-dra/templates/_helpers.tpl deleted file mode 100644 index 99beec77..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/_helpers.tpl +++ /dev/null @@ -1,73 +0,0 @@ -{{- define "hami.dra.webhook.fullname" -}} -{{- printf "%s-%s" (include "common.names.fullname" .) "webhook" | trunc 63 | trimSuffix "-" -}} -{{- end }} - -{{- define "hami.dra.webhook.image" -}} -{{ include "common.images.image" (dict "imageRoot" .Values.webhook.image "global" .Values.global) }} -{{- end -}} - -{{- define "hami.dra.webhook.imagePullSecrets" -}} -{{ include "common.images.pullSecrets" (dict "images" (list .Values.webhook.image) "global" .Values.global) }} -{{- end -}} - -{{- define "hami.dra.driver.nvidia.image" -}} -{{ include "common.images.image" (dict "imageRoot" .Values.drivers.nvidia.image "global" .Values.global) }} -{{- end -}} - -{{- define "hami.dra.driver.nvidia.imagePullSecrets" -}} -{{ include "common.images.pullSecrets" (dict "images" (list .Values.drivers.nvidia.image) "global" .Values.global) }} -{{- end -}} - -{{/* -Common labels -*/}} -{{- define "hami-dra-webhook.labels" -}} -helm.sh/chart: {{ .Chart.Name }} -{{ include "hami-dra-webhook.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "hami-dra-webhook.selectorLabels" -}} -app.kubernetes.io/name: {{ .Release.Name }} -app.kubernetes.io/instance: {{ .Release.Name }} -app.kubernetes.io/component: webhook -{{- end }} - -{{- define "hami.dra.monitor.fullname" -}} -{{- printf "%s-%s" (include "common.names.fullname" .) "monitor" | trunc 63 | trimSuffix "-" -}} -{{- end }} - -{{- define "hami.dra.monitor.image" -}} -{{ include "common.images.image" (dict "imageRoot" .Values.monitor.image "global" .Values.global) }} -{{- end -}} - -{{- define "hami.dra.monitor.imagePullSecrets" -}} -{{ include "common.images.pullSecrets" (dict "images" (list .Values.monitor.image) "global" .Values.global) }} -{{- end -}} - -{{/* -Common labels for monitor -*/}} -{{- define "hami-dra-monitor.labels" -}} -helm.sh/chart: {{ .Chart.Name }} -{{ include "hami-dra-monitor.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels for monitor -*/}} -{{- define "hami-dra-monitor.selectorLabels" -}} -app.kubernetes.io/name: {{ .Release.Name }} -app.kubernetes.io/instance: {{ .Release.Name }} -app.kubernetes.io/component: monitor -{{- end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/deviceclass.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/deviceclass.yaml deleted file mode 100644 index bcb6d2c5..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/deviceclass.yaml +++ /dev/null @@ -1,11 +0,0 @@ -{{- if .Values.drivers.nvidia.enabled }} -apiVersion: resource.k8s.io/v1 -kind: DeviceClass -metadata: - name: hami-core-gpu.project-hami.io -spec: - selectors: - - cel: - expression: |- - device.driver == "hami-core-gpu.project-hami.io" && device.attributes["hami-core-gpu.project-hami.io"].type == "hami-gpu" -{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/nvidia-dra-driver-daemonset.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/nvidia-dra-driver-daemonset.yaml deleted file mode 100644 index a3fb0d01..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/nvidia-dra-driver-daemonset.yaml +++ /dev/null @@ -1,145 +0,0 @@ -{{- if .Values.drivers.nvidia.enabled }} -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: hami-dra-driver-kubelet-plugin - namespace: {{ .Release.Namespace }} -spec: - revisionHistoryLimit: 10 - selector: - matchLabels: - hami-dra-driver-component: kubelet-plugin - template: - metadata: - labels: - app.kubernetes.io/instance: hami-dra-driver - app.kubernetes.io/name: hami-dra-driver - hami-dra-driver-component: kubelet-plugin - spec: - {{- include "hami.dra.driver.nvidia.imagePullSecrets" . | nindent 6 }} - initContainers: - - name: init-container - image: {{ include "hami.dra.driver.nvidia.image" . }} - securityContext: - privileged: true - command: [bash, /usr/bin/kubelet-plugin-prestart.sh] - env: - - name: NVIDIA_DRIVER_ROOT - value: {{ if .Values.drivers.nvidia.containerDriver }}/run/nvidia/driver{{ else }}/{{ end }} - - name: NVIDIA_VISIBLE_DEVICES - value: void - - name: KUBELET_REGISTRAR_DIRECTORY_PATH - value: /var/lib/kubelet/plugins_registry - - name: KUBELET_PLUGINS_DIRECTORY_PATH - value: /var/lib/kubelet/plugins - volumeMounts: - - name: driver-root-parent - mountPath: /driver-root-parent - readOnly: true - containers: - - args: - - |- - # Conditionally mask the params file to prevent this container from - # recreating any missing GPU device nodes. This is necessary, for - # example, when running under nvkind to limit the set GPUs governed - # by the plugin even though it has cgroup access to all of them. - if [ "${MASK_NVIDIA_DRIVER_PARAMS}" = "true" ]; then - cp /proc/driver/nvidia/params root/gpu-params - sed -i 's/^ModifyDeviceFiles: 1$/ModifyDeviceFiles: 0/' root/gpu-params - mount --bind root/gpu-params /proc/driver/nvidia/params - fi - hami-kubelet-plugin -v 6 - command: - - bash - - -c - env: - - name: MASK_NVIDIA_DRIVER_PARAMS - value: "" - - name: NVIDIA_DRIVER_ROOT - value: {{ if .Values.drivers.nvidia.containerDriver }}/run/nvidia/driver{{ else }}/{{ end }} - - name: NVIDIA_VISIBLE_DEVICES - value: void - - name: CDI_ROOT - value: /var/run/cdi - - name: NVIDIA_MIG_CONFIG_DEVICES - value: all - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: KUBELET_REGISTRAR_DIRECTORY_PATH - value: /var/lib/kubelet/plugins_registry - - name: KUBELET_PLUGINS_DIRECTORY_PATH - value: /var/lib/kubelet/plugins - - name: IMAGE_NAME - value: {{ include "hami.dra.driver.nvidia.image" . }} - lifecycle: - postStart: - exec: - command: ["/bin/bash", "-c", "/usr/bin/vgpu-init.sh /usr/local/vgpu/"] - image: {{ include "hami.dra.driver.nvidia.image" . }} - imagePullPolicy: {{ .Values.drivers.nvidia.image.pullPolicy | default "IfNotPresent" }} - name: gpus - securityContext: - privileged: true - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - volumeMounts: - - mountPath: /var/lib/kubelet/plugins_registry - name: plugins-registry - - mountPath: /var/lib/kubelet/plugins - mountPropagation: Bidirectional - name: plugins - - mountPath: /var/run/cdi - name: cdi - - mountPath: /driver-root - mountPropagation: HostToContainer - name: driver-root - readOnly: true - - mountPath: /usr/local/vgpu - name: host-vgpu - - mountPath: /tmp - name: host-tmp - dnsPolicy: ClusterFirst - serviceAccount: hami-dra-driver-service-account - serviceAccountName: hami-dra-driver-service-account - volumes: - - hostPath: - path: /var/lib/kubelet/plugins_registry - type: "" - name: plugins-registry - - hostPath: - path: /var/lib/kubelet/plugins - type: "" - name: plugins - - hostPath: - path: /var/run/cdi - type: "" - name: cdi - - hostPath: - path: {{if .Values.drivers.nvidia.containerDriver }}/run/nvidia{{ else }}/{{ end }} - type: DirectoryOrCreate - name: driver-root-parent - - hostPath: - path: {{if .Values.drivers.nvidia.containerDriver }}/run/nvidia/driver{{ else }}/{{ end }} - type: DirectoryOrCreate - name: driver-root - - hostPath: - path: /dev - type: "" - name: host-dev - - hostPath: - path: /usr/local/vgpu - type: DirectoryOrCreate - name: host-vgpu - - hostPath: - path: /tmp - type: "" - name: host-tmp -{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/rbac.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/rbac.yaml deleted file mode 100644 index 435c37cc..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/hami-dra-driver/rbac.yaml +++ /dev/null @@ -1,110 +0,0 @@ -{{- if .Values.drivers.nvidia.enabled }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: hami-dra-driver-service-account - namespace: {{ .Release.Namespace }} ---- -apiVersion: v1 -items: -- apiVersion: rbac.authorization.k8s.io/v1 - kind: Role - metadata: - name: hami-dra-driver-role - namespace: {{ .Release.Namespace }} - rules: - - apiGroups: - - apps - resources: - - daemonsets - - deployments - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -kind: List -metadata: - resourceVersion: "" ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: hami-dra-driver-role-binding - namespace: {{ .Release.Namespace }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: hami-dra-driver-role -subjects: -- kind: ServiceAccount - name: hami-dra-driver-service-account - namespace: {{ .Release.Namespace }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: hami-dra-driver-clusterrole -rules: -- apiGroups: - - resource.k8s.io - resources: - - resourceclaims - verbs: - - get - - list - - watch -- apiGroups: - - resource.k8s.io - resources: - - resourceclaimtemplates - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - resource.k8s.io - resources: - - resourceslices - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - resource.k8s.io - resources: - - resourceclaims/status - verbs: - - update -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - watch - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: hami-dra-driver-clusterrole-binding-hami-dra-driver -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: hami-dra-driver-clusterrole -subjects: -- kind: ServiceAccount - name: hami-dra-driver-service-account - namespace: {{ .Release.Namespace }} -{{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrole.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrole.yaml deleted file mode 100644 index 45c24863..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrole.yaml +++ /dev/null @@ -1,11 +0,0 @@ -{{- if .Values.monitor.enabled }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "hami.dra.monitor.fullname" . }} -rules: -- apiGroups: ["resource.k8s.io"] - resources: ["resourceslices", "resourceclaims"] - verbs: ["get", "list", "watch"] -{{- end }} - diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrolebinding.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrolebinding.yaml deleted file mode 100644 index 766d3098..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-clusterrolebinding.yaml +++ /dev/null @@ -1,15 +0,0 @@ -{{- if .Values.monitor.enabled }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "hami.dra.monitor.fullname" . }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "hami.dra.monitor.fullname" . }} -subjects: - - kind: ServiceAccount - name: {{ include "hami.dra.monitor.fullname" . }} - namespace: {{ .Release.Namespace }} -{{- end }} - diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-deployment.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-deployment.yaml deleted file mode 100644 index b544b376..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-deployment.yaml +++ /dev/null @@ -1,62 +0,0 @@ -{{- if .Values.monitor.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "hami.dra.monitor.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: - app.kubernetes.io/name: {{ include "hami.dra.monitor.fullname" . }} - {{- include "hami-dra-monitor.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.monitor.replicas | default 1 }} - selector: - matchLabels: - {{- include "hami-dra-monitor.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - {{- include "hami-dra-monitor.selectorLabels" . | nindent 8 }} - spec: - {{- include "hami.dra.monitor.imagePullSecrets" . | nindent 6 }} - serviceAccountName: {{ include "hami.dra.monitor.fullname" . }} - containers: - - name: monitor - image: {{ include "hami.dra.monitor.image" . }} - imagePullPolicy: {{ .Values.monitor.image.pullPolicy | default "IfNotPresent" }} - command: - - /bin/monitor - - --v={{ .Values.monitor.logLevel | default 2 }} - - --metrics-bind-address={{ .Values.monitor.metricsBindAddress | default ":8080" }} - - --health-probe-bind-address={{ .Values.monitor.healthProbeBindAddress | default ":8000" }} - - --kube-api-qps={{ .Values.monitor.kubeAPIQPS | default 40.0 }} - - --kube-api-burst={{ .Values.monitor.kubeAPIBurst | default 60 }} - - --collect-interval={{ .Values.monitor.collectInterval | default "30s" }} - ports: - - containerPort: 8080 - name: metrics - protocol: TCP - - containerPort: 8000 - name: health - protocol: TCP - {{- if .Values.monitor.resources }} - resources: - {{- toYaml .Values.monitor.resources | nindent 12 }} - {{- end }} - livenessProbe: - httpGet: - path: /healthz - port: 8000 - initialDelaySeconds: 10 - periodSeconds: 10 - timeoutSeconds: 5 - failureThreshold: 3 - readinessProbe: - httpGet: - path: /readyz - port: 8000 - initialDelaySeconds: 5 - periodSeconds: 5 - timeoutSeconds: 3 - failureThreshold: 3 -{{- end }} - diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-service.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-service.yaml deleted file mode 100644 index d740cc38..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-service.yaml +++ /dev/null @@ -1,26 +0,0 @@ -{{- if .Values.monitor.enabled }} -apiVersion: v1 -kind: Service -metadata: - name: {{ include "hami.dra.monitor.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: - {{- include "hami-dra-monitor.selectorLabels" . | nindent 4 }} -spec: - type: {{ .Values.monitor.service.type | default "ClusterIP" }} - selector: - {{- include "hami-dra-monitor.selectorLabels" . | nindent 4 }} - ports: - - port: 8080 - targetPort: 8080 - name: metrics - protocol: TCP - {{- if and (eq (.Values.monitor.service.type | default "ClusterIP") "NodePort") .Values.monitor.service.nodePort.metrics }} - nodePort: {{ .Values.monitor.service.nodePort.metrics }} - {{- end }} - - port: 8000 - targetPort: 8000 - name: health - protocol: TCP -{{- end }} - diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-serviceaccount.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-serviceaccount.yaml deleted file mode 100644 index c61ad454..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/monitor/monitor-serviceaccount.yaml +++ /dev/null @@ -1,8 +0,0 @@ -{{- if .Values.monitor.enabled }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "hami.dra.monitor.fullname" . }} - namespace: {{ .Release.Namespace }} -{{- end }} - diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-manager.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-manager.yaml deleted file mode 100644 index dd31c71d..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-manager.yaml +++ /dev/null @@ -1,29 +0,0 @@ -{{- if .Values.certs.certManager.enabled }} -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: {{ .Release.Name }}-dra-webhook-serving-cert - namespace: {{ .Release.Namespace }} - labels: - {{- include "hami-dra-webhook.labels" . | nindent 4 }} -spec: - dnsNames: - - {{ .Release.Name }}-dra-webhook.{{ .Release.Namespace }}.svc - - {{ .Release.Name }}-dra-webhook.{{ .Release.Namespace }}.svc.cluster.local - issuerRef: - kind: Issuer - name: {{ .Release.Name }}-dra-webhook-selfsigned-issuer - secretName: {{ .Release.Name }}-dra-webhook-tls - privateKey: - rotationPolicy: {{ .Values.certs.certManager.privateKeyRotationPolicy | default "Never" }} ---- -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - name: {{ .Release.Name }}-dra-webhook-selfsigned-issuer - namespace: {{ .Release.Namespace }} - labels: - app.kubernetes.io/component: hami-dra-webhook -spec: - selfSigned: {} -{{- end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-secret.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-secret.yaml deleted file mode 100644 index eadbb517..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/cert-secret.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- if eq .Values.certs.mode "custom" }} -apiVersion: v1 -kind: Secret -metadata: - name: {{ include "hami-dra-webhook.tlsSecretName" . }} - namespace: {{ .Release.Namespace }} -type: kubernetes.io/tls -data: - tls.crt: | - {{- .Values.certs.custom.crt | b64enc | indent 4 }} - tls.key: | - {{- .Values.certs.custom.key | b64enc | indent 4 }} -{{- end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/device-config.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/device-config.yaml deleted file mode 100644 index 97283f3e..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/device-config.yaml +++ /dev/null @@ -1,394 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "hami.dra.webhook.fullname" . }}-device-config - namespace: {{ .Release.Namespace }} - labels: - app.kubernetes.io/component: hami-scheduler - {{- include "hami-dra-webhook.selectorLabels" . | nindent 4 }} -data: - device-config.yaml: |- - {{- if .Files.Glob "files/device-config.yaml" }} - {{- .Files.Get "files/device-config.yaml" | nindent 4}} - {{- else }} - nvidia: - resourceCountName: {{ .Values.resourceName }} - resourceMemoryName: {{ .Values.resourceMem }} - resourceMemoryPercentageName: {{ .Values.resourceMemPercentage }} - resourceCoreName: {{ .Values.resourceCores }} - resourcePriorityName: {{ .Values.resourcePriority }} - overwriteEnv: false - defaultMemory: 0 - defaultCores: 0 - defaultGPUNum: 1 - knownMigGeometries: - - models: [ "A30" ] - allowedGeometries: - - - - name: 1g.6gb - core: 25 - memory: 6144 - count: 4 - - - - name: 2g.12gb - core: 50 - memory: 12288 - count: 2 - - - - name: 4g.24gb - core: 100 - memory: 24576 - count: 1 - - models: [ "A100-SXM4-40GB", "A100-40GB-PCIe", "A100-PCIE-40GB"] - allowedGeometries: - - - - name: 1g.5gb - core: 14 - memory: 5120 - count: 7 - - - - name: 1g.5gb - core: 14 - memory: 5120 - count: 1 - - name: 2g.10gb - core: 28 - memory: 10240 - count: 3 - - - - name: 3g.20gb - core: 42 - memory: 20480 - count: 2 - - - - name: 7g.40gb - core: 100 - memory: 40960 - count: 1 - - models: [ "A100-SXM4-80GB", "A100-80GB-PCIe", "A100-PCIE-80GB"] - allowedGeometries: - - - - name: 1g.10gb - core: 14 - memory: 10240 - count: 7 - - - - name: 1g.10gb - core: 14 - memory: 10240 - count: 1 - - name: 2g.20gb - core: 28 - memory: 20480 - count: 3 - - - - name: 3g.40gb - core: 42 - memory: 40960 - count: 2 - - - - name: 7g.79gb - core: 100 - memory: 80896 - count: 1 - - models: [ "H100-PCIE-80GB", "H100-SXM5-80GB"] - allowedGeometries: - - - - name: 1g.10gb - core: 14 - memory: 10240 - count: 7 - - - - name: 1g.10gb - core: 14 - memory: 10240 - count: 1 - - name: 2g.20gb - core: 28 - memory: 20480 - count: 3 - - - - name: 3g.40gb - core: 42 - memory: 40960 - count: 2 - - - - name: 7g.80gb - core: 100 - memory: 81920 - count: 1 - - models: [ "H100-PCIE-94GB", "H100-SXM5-94GB"] - allowedGeometries: - - - - name: 1g.12gb - core: 14 - memory: 12288 - count: 7 - - - - name: 1g.12gb - core: 14 - memory: 12288 - count: 1 - - name: 2g.24gb - core: 28 - memory: 24576 - count: 3 - - - - name: 3g.47gb - core: 42 - memory: 48128 - count: 2 - - - - name: 7g.94gb - core: 100 - memory: 96256 - count: 1 - - models: [ "H20", "H100 on GH200"] - allowedGeometries: - - - - name: 1g.12gb - core: 14 - memory: 12288 - count: 7 - - - - name: 1g.12gb - core: 14 - memory: 12288 - count: 1 - - name: 2g.24gb - core: 28 - memory: 24576 - count: 3 - - - - name: 3g.48gb - core: 42 - memory: 49152 - count: 2 - - - - name: 7g.96gb - core: 100 - memory: 98304 - count: 1 - - models: [ "H200 NVL", "H200-SXM5"] - allowedGeometries: - - - - name: 1g.18gb - core: 14 - memory: 18432 - count: 7 - - - - name: 1g.18gb - core: 14 - memory: 18432 - count: 1 - - name: 2g.35gb - core: 28 - memory: 35840 - count: 3 - - - - name: 3g.71gb - core: 42 - memory: 72704 - count: 2 - - - - name: 7g.141gb - core: 100 - memory: 144384 - count: 1 - - models: [ "B200" ] - allowedGeometries: - - - - name: 1g.23gb - core: 14 - memory: 23552 - count: 7 - - - - name: 1g.23gb - core: 14 - memory: 23552 - count: 1 - - name: 2g.45gb - core: 28 - memory: 46080 - count: 3 - - - - name: 3g.90gb - core: 42 - memory: 92160 - count: 2 - - - - name: 7g.180gb - core: 100 - memory: 184320 - count: 1 - cambricon: - resourceCountName: {{ .Values.mluResourceName }} - resourceMemoryName: {{ .Values.mluResourceMem }} - resourceCoreName: {{ .Values.mluResourceCores }} - hygon: - resourceCountName: {{ .Values.dcuResourceName }} - resourceMemoryName: {{ .Values.dcuResourceMem }} - resourceCoreName: {{ .Values.dcuResourceCores }} - metax: - resourceCountName: "metax-tech.com/gpu" - resourceVCountName: {{ .Values.metaxResourceName }} - resourceVMemoryName: {{ .Values.metaxResourceMem }} - resourceVCoreName: {{ .Values.metaxResourceCore }} - sgpuTopologyAware: {{ .Values.metaxsGPUTopologyAware }} - enflame: - resourceNameGCU: "enflame.com/gcu" - resourceNameVGCU: {{ .Values.enflameResourceNameVGCU }} - resourceNameVGCUPercentage: {{ .Values.enflameResourceNameVGCUPercentage }} - mthreads: - resourceCountName: "mthreads.com/vgpu" - resourceMemoryName: "mthreads.com/sgpu-memory" - resourceCoreName: "mthreads.com/sgpu-core" - iluvatars: - - chipName: MR-V100 - commonWord: MR-V100 - resourceCountName: iluvatar.ai/MR-V100-vgpu - resourceMemoryName: iluvatar.ai/MR-V100.vMem - resourceCoreName: iluvatar.ai/MR-V100.vCore - - chipName: MR-V50 - commonWord: MR-V50 - resourceCountName: iluvatar.ai/MR-V50-vgpu - resourceMemoryName: iluvatar.ai/MR-V50.vMem - resourceCoreName: iluvatar.ai/MR-V50.vCore - - chipName: BI-V150 - commonWord: BI-V150 - resourceCountName: iluvatar.ai/BI-V150-vgpu - resourceMemoryName: iluvatar.ai/BI-V150.vMem - resourceCoreName: iluvatar.ai/BI-V150.vCore - - chipName: BI-V100 - commonWord: BI-V100 - resourceCountName: iluvatar.ai/BI-V100-vgpu - resourceMemoryName: iluvatar.ai/BI-V100.vMem - resourceCoreName: iluvatar.ai/BI-V100.vCore - kunlun: - resourceCountName: {{ .Values.kunlunResourceName }} - resourceVCountName: {{ .Values.kunlunResourceVCountName }} - resourceVMemoryName: {{ .Values.kunlunResourceVMemoryName }} - awsneuron: - resourceCountName: "aws.amazon.com/neuron" - resourceCoreName: "aws.amazon.com/neuroncore" - amd: - resourceCountName: "amd.com/gpu" - vnpus: - - chipName: 910A - commonWord: Ascend910A - resourceName: huawei.com/Ascend910A - resourceMemoryName: huawei.com/Ascend910A-memory - memoryAllocatable: 32768 - memoryCapacity: 32768 - aiCore: 30 - templates: - - name: vir02 - memory: 2184 - aiCore: 2 - - name: vir04 - memory: 4369 - aiCore: 4 - - name: vir08 - memory: 8738 - aiCore: 8 - - name: vir16 - memory: 17476 - aiCore: 16 - - chipName: 910B2 - commonWord: Ascend910B2 - resourceName: huawei.com/Ascend910B2 - resourceMemoryName: huawei.com/Ascend910B2-memory - memoryAllocatable: 65536 - memoryCapacity: 65536 - aiCore: 24 - aiCPU: 6 - templates: - - name: vir03_1c_8g - memory: 8192 - aiCore: 3 - aiCPU: 1 - - name: vir06_1c_16g - memory: 16384 - aiCore: 6 - aiCPU: 1 - - name: vir12_3c_32g - memory: 32768 - aiCore: 12 - aiCPU: 3 - - chipName: 910B3 - commonWord: Ascend910B3 - resourceName: huawei.com/Ascend910B3 - resourceMemoryName: huawei.com/Ascend910B3-memory - memoryAllocatable: 65536 - memoryCapacity: 65536 - aiCore: 20 - aiCPU: 7 - templates: - - name: vir05_1c_16g - memory: 16384 - aiCore: 5 - aiCPU: 1 - - name: vir10_3c_32g - memory: 32768 - aiCore: 10 - aiCPU: 3 - - chipName: 910B4-1 - commonWord: Ascend910B4-1 - resourceName: huawei.com/Ascend910B4-1 - resourceMemoryName: huawei.com/Ascend910B4-1-memory - memoryAllocatable: 65536 - memoryCapacity: 65536 - aiCore: 20 - aiCPU: 7 - templates: - # NOTE: Names of vnpu template for 910B4-1 are fixed by Ascend runtime and must not be changed. - # The memory is used for scheduling so the correct values must be set. - # Template vir05_1c_8g actually provides 16GB memory, - - name: vir05_1c_8g - memory: 16384 - aiCore: 5 - aiCPU: 1 - # Template vir10_3c_16g actually provides 32GB memory - - name: vir10_3c_16g - memory: 32768 - aiCore: 10 - aiCPU: 3 - - chipName: 910B4 - commonWord: Ascend910B4 - resourceName: huawei.com/Ascend910B4 - resourceMemoryName: huawei.com/Ascend910B4-memory - memoryAllocatable: 32768 - memoryCapacity: 32768 - aiCore: 20 - aiCPU: 7 - templates: - - name: vir05_1c_8g - memory: 8192 - aiCore: 5 - aiCPU: 1 - - name: vir10_3c_16g - memory: 16384 - aiCore: 10 - aiCPU: 3 - - chipName: 310P3 - commonWord: Ascend310P - resourceName: huawei.com/Ascend310P - resourceMemoryName: huawei.com/Ascend310P-memory - memoryAllocatable: 21527 - memoryCapacity: 24576 - aiCore: 8 - aiCPU: 7 - templates: - - name: vir01 - memory: 3072 - aiCore: 1 - aiCPU: 1 - - name: vir02 - memory: 6144 - aiCore: 2 - aiCPU: 2 - - name: vir04 - memory: 12288 - aiCore: 4 - aiCPU: 4 - {{ end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/mutatingwebhookconfiguration.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/mutatingwebhookconfiguration.yaml deleted file mode 100644 index c763b6aa..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/mutatingwebhookconfiguration.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{- if .Values.webhook.config.mutating.enabled }} -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - name: {{ .Release.Name }}-mutatingwebhookconfiguration - annotations: - cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ .Release.Name }}-dra-webhook-serving-cert -webhooks: - - name: mutate.hami.io - admissionReviewVersions: ["v1"] - clientConfig: - service: - name: {{ .Release.Name }}-dra-webhook - namespace: {{ .Release.Namespace }} - path: /mutate - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - CREATE - resources: - - pods - scope: '*' - sideEffects: None - timeoutSeconds: 10 -{{- end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/validatingwebhookconfiguration.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/validatingwebhookconfiguration.yaml deleted file mode 100644 index acdcbb96..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/validatingwebhookconfiguration.yaml +++ /dev/null @@ -1,29 +0,0 @@ -{{- if .Values.webhook.config.validating.enabled }} -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: {{ .Release.Name }}-validatingwebhookconfiguration - annotations: - cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ .Release.Name }}-dra-webhook-serving-cert -webhooks: - - name: validate.hami.io - admissionReviewVersions: ["v1"] - clientConfig: - service: - name: {{ .Release.Name }}-dra-webhook - namespace: {{ .Release.Namespace }} - path: /validate - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - DELETE - resources: - - pods - scope: '*' - sideEffects: None - timeoutSeconds: 10 - failurePolicy: Ignore -{{- end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrole.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrole.yaml deleted file mode 100644 index 28916a36..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrole.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "hami.dra.webhook.fullname" . }} -rules: -- apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list", "watch"] -- apiGroups: ["admissionregistration.k8s.io"] - resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"] - verbs: ["get", "list", "watch"] -- apiGroups: ["resource.k8s.io"] - resources: ["resourceclaims"] - verbs: ["*"] diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrolebinding.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrolebinding.yaml deleted file mode 100644 index daad52f2..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-clusterrolebinding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "hami.dra.webhook.fullname" . }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "hami.dra.webhook.fullname" . }} -subjects: - - kind: ServiceAccount - name: {{ include "hami.dra.webhook.fullname" . }} - namespace: {{ .Release.Namespace }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-deployment.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-deployment.yaml deleted file mode 100644 index 8e3cf90c..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-deployment.yaml +++ /dev/null @@ -1,59 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "hami.dra.webhook.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: - app.kubernetes.io/name: {{ include "hami.dra.webhook.fullname" . }} - {{- include "hami-dra-webhook.labels" . | nindent 4 }} -spec: - replicas: 1 - selector: - matchLabels: - {{- include "hami-dra-webhook.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - {{- include "hami-dra-webhook.selectorLabels" . | nindent 8 }} - spec: - {{- include "hami.dra.webhook.imagePullSecrets" . | nindent 6 }} - serviceAccountName: {{ include "hami.dra.webhook.fullname" . }} - containers: - - name: webhook - image: {{ include "hami.dra.webhook.image" . }} - imagePullPolicy: {{ .Values.webhook.image.pullPolicy | default "IfNotPresent" }} - command: - - /bin/webhook - - --v=5 - - --bind-address=0.0.0.0 - - --secure-port=8443 - - --cert-dir=/tls - - --tls-cert-file-name=tls.crt - - --tls-private-key-file-name=tls.key - - --metrics-bind-address=:8080 - - --health-probe-bind-address=:8000 - - --device-config-file=/device-config.yaml - ports: - - containerPort: 8443 - name: webhook - protocol: TCP - - containerPort: 8080 - name: metrics - protocol: TCP - - containerPort: 8000 - name: health - protocol: TCP - volumeMounts: - - name: device-config - mountPath: /device-config.yaml - subPath: device-config.yaml - - name: tls-config - mountPath: /tls - readOnly: true - volumes: - - name: device-config - configMap: - name: {{ include "hami.dra.webhook.fullname" . }}-device-config - - name: tls-config - secret: - secretName: {{ if .Values.certs.certManager.enabled }}{{ .Release.Name }}-dra-webhook-tls{{ else }}{{ .Release.Name }}-tls{{ end }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-service.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-service.yaml deleted file mode 100644 index 290174ec..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ .Release.Name }}-dra-webhook - namespace: {{ .Release.Namespace }} - labels: - {{- include "hami-dra-webhook.selectorLabels" . | nindent 4 }} -spec: - type: "ClusterIP" - selector: - {{- include "hami-dra-webhook.selectorLabels" . | nindent 4 }} - ports: - - port: 443 - targetPort: 8443 - name: webhook diff --git a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-serviceaccount.yaml b/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-serviceaccount.yaml deleted file mode 100644 index b0dc2fb2..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/templates/webhook/webhook-serviceaccount.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "hami.dra.webhook.fullname" . }} - namespace: {{ .Release.Namespace }} diff --git a/packages/system/hami/charts/hami/charts/hami-dra/values.yaml b/packages/system/hami/charts/hami/charts/hami-dra/values.yaml deleted file mode 100644 index ebbbfed7..00000000 --- a/packages/system/hami/charts/hami/charts/hami-dra/values.yaml +++ /dev/null @@ -1,166 +0,0 @@ -## @section Global configuration -## @param global.imageRegistry Global Docker image registry -## @param global.imagePullSecrets Global Docker image pull secrets -global: - ## @param global.imageRegistry Global Docker image registry - imageRegistry: "" - ## E.g. - ## imagePullSecrets: - ## - myRegistryKeySecretName - ## @param global.imagePullSecrets Global Docker image pull secrets - imagePullSecrets: [] - -#Nvidia GPU Parameters -resourceName: "nvidia.com/gpu" -resourceMem: "nvidia.com/gpumem" -resourceMemPercentage: "nvidia.com/gpumem-percentage" -resourceCores: "nvidia.com/gpucores" -resourcePriority: "nvidia.com/priority" - -#MLU Parameters -mluResourceName: "cambricon.com/vmlu" -mluResourceMem: "cambricon.com/mlu.smlu.vmemory" -mluResourceCores: "cambricon.com/mlu.smlu.vcore" - -#Hygon DCU Parameters -dcuResourceName: "hygon.com/dcunum" -dcuResourceMem: "hygon.com/dcumem" -dcuResourceCores: "hygon.com/dcucores" - -#Metax sGPU Parameters -metaxResourceName: "metax-tech.com/sgpu" -metaxResourceCore: "metax-tech.com/vcore" -metaxResourceMem: "metax-tech.com/vmemory" -metaxsGPUTopologyAware: "false" - -#Enflame VGCU Parameters -enflameResourceNameVGCU: "enflame.com/vgcu" -enflameResourceNameVGCUPercentage: "enflame.com/vgcu-percentage" - -#Kunlun XPU Parameters -kunlunResourceName: "kunlunxin.com/xpu" -kunlunResourceVCountName: "kunlunxin.com/vxpu" -kunlunResourceVMemoryName: "kunlunxin.com/vxpu-memory" - -# Webhook deployment configuration -webhook: - image: - registry: ghcr.io - repository: project-hami/hami-dra-webhook - tag: "v0.1.0" - ## Specify a imagePullPolicy - ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' - ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images - ## - pullPolicy: Always - ## Optionally specify an array of imagePullSecrets. - ## Secrets must be manually created in the namespace. - ## Example: - ## pullSecrets: - ## - myRegistryKeySecretName - ## - pullSecrets: [] - ## @param resources controllerManager resource requests and limits - resources: - # If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 128Mi - ## @param nodeSelector controllerManager node labels for pod assignment - args: - webhookName: "" # Default: {{ .Release.Name }}-hami-dra-webhook - secretName: "" # Default: {{ .Release.Name }}-hami-dra-webhook-tls - # Webhook configuration - config: - mutating: - enabled: true - validating: - enabled: true - -# Monitor deployment configuration -monitor: - enabled: true - replicas: 1 - image: - registry: ghcr.io - repository: project-hami/hami-dra-monitor - tag: "v0.1.0" - ## Specify a imagePullPolicy - ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' - ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images - ## - pullPolicy: Always - ## Optionally specify an array of imagePullSecrets. - ## Secrets must be manually created in the namespace. - ## Example: - ## pullSecrets: - ## - myRegistryKeySecretName - ## - pullSecrets: [] - ## @param resources monitor resource requests and limits - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 128Mi - ## Monitor configuration - logLevel: 2 - metricsBindAddress: ":8080" - healthProbeBindAddress: ":8000" - kubeAPIQPS: 40.0 - kubeAPIBurst: 60 - collectInterval: "30s" - ## Monitor service configuration - service: - ## Service type: ClusterIP, NodePort, or LoadBalancer - ## Defaults to ClusterIP - type: ClusterIP - ## NodePort configuration (only used when type is NodePort) - nodePort: - ## Metrics port (NodePort). If not specified, Kubernetes will assign a random port - metrics: "" - -# Certificate configuration -certs: - # Custom certificate (used when mode is "custom") - custom: - crt: "" - key: "" - # Cert-manager configuration (used when mode is "cert-manager") - certManager: - enabled: true - # Private key rotation policy: "Never" or "Always" - # In cert-manager >= v1.18.0, the default changed from "Never" to "Always" - # Setting to "Never" avoids frequent certificate rotation for webhooks - privateKeyRotationPolicy: "Never" - issuer: - clusterIssuerName: "" # Required when type is "clusterIssuer" - -# DRA Drivers configuration -drivers: - nvidia: - enabled: true - # If you are using gpu driver on host, you need to set this to false - containerDriver: true - image: - registry: ghcr.io - repository: projecthami/k8s-dra-driver - tag: "v0.0.1-dev" - ## Specify a imagePullPolicy - ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' - ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images - ## - pullPolicy: IfNotPresent - ## Optionally specify an array of imagePullSecrets. - ## Secrets must be manually created in the namespace. - ## Example: - ## pullSecrets: - ## - myRegistryKeySecretName - ## - pullSecrets: [] From 385ea7a17f8cfc4526c50db036c105c63a5817fc Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Thu, 23 Apr 2026 22:16:34 +0300 Subject: [PATCH 180/250] feat(platform): register HAMi as optional system package Add PackageSource for HAMi with dependency on gpu-operator. Include HAMi in the iaas bundle as an optional package, allowing host-level deployment alongside the existing kubernetes addon path. Signed-off-by: Arsolitt --- packages/core/platform/sources/hami.yaml | 24 +++++++++++++++++++ .../core/platform/templates/bundles/iaas.yaml | 1 + 2 files changed, 25 insertions(+) create mode 100644 packages/core/platform/sources/hami.yaml diff --git a/packages/core/platform/sources/hami.yaml b/packages/core/platform/sources/hami.yaml new file mode 100644 index 00000000..0184e405 --- /dev/null +++ b/packages/core/platform/sources/hami.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: cozystack.io/v1alpha1 +kind: PackageSource +metadata: + name: cozystack.hami +spec: + sourceRef: + kind: OCIRepository + name: cozystack-packages + namespace: cozy-system + path: / + variants: + - name: default + dependsOn: + - cozystack.gpu-operator + components: + - name: hami + path: system/hami + valuesFiles: + - values.yaml + install: + privileged: true + namespace: cozy-hami + releaseName: hami diff --git a/packages/core/platform/templates/bundles/iaas.yaml b/packages/core/platform/templates/bundles/iaas.yaml index ced0a322..66ec98b2 100644 --- a/packages/core/platform/templates/bundles/iaas.yaml +++ b/packages/core/platform/templates/bundles/iaas.yaml @@ -10,6 +10,7 @@ {{include "cozystack.platform.package.default" (list "cozystack.kubevirt-cdi" $) }} {{include "cozystack.platform.package.optional.default" (list "cozystack.vm-default-images" $) }} {{include "cozystack.platform.package.optional.default" (list "cozystack.gpu-operator" $) }} +{{include "cozystack.platform.package.optional.default" (list "cozystack.hami" $) }} {{include "cozystack.platform.package.default" (list "cozystack.kamaji" $) }} {{include "cozystack.platform.package.default" (list "cozystack.capi-operator" $) }} {{include "cozystack.platform.package.default" (list "cozystack.capi-provider-bootstrap-kubeadm" $) }} From 6af74cec0ea436cbd5aefce5e5c8bedf86e5b81a Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 23 Apr 2026 23:24:38 +0300 Subject: [PATCH 181/250] fix(ingress): accept pre-CIDR externalIPs in CiliumLoadBalancerIPPool Address review feedback from gemini-code-assist on packages/extra/ingress/templates/cilium-lb-pool.yaml:19: if the operator passes an externalIP already in CIDR form (192.0.2.10/32 or 2001:db8::1/128), the template appended a second /32 or /128 suffix producing an invalid CiliumLoadBalancerIPPool block. Guard the suffix append on the absence of "/" in the input. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- .../extra/ingress/templates/cilium-lb-pool.yaml | 2 +- packages/extra/ingress/tests/exposure_test.yaml | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/extra/ingress/templates/cilium-lb-pool.yaml b/packages/extra/ingress/templates/cilium-lb-pool.yaml index a383922b..048eb58b 100644 --- a/packages/extra/ingress/templates/cilium-lb-pool.yaml +++ b/packages/extra/ingress/templates/cilium-lb-pool.yaml @@ -16,7 +16,7 @@ metadata: spec: blocks: {{- range $exposeIPsList }} - - cidr: {{ . }}/{{ if contains ":" . }}128{{ else }}32{{ end }} + - cidr: {{ . }}{{ if not (contains "/" .) }}/{{ if contains ":" . }}128{{ else }}32{{ end }}{{ end }} {{- end }} serviceSelector: matchLabels: diff --git a/packages/extra/ingress/tests/exposure_test.yaml b/packages/extra/ingress/tests/exposure_test.yaml index 780e0715..323c8e68 100644 --- a/packages/extra/ingress/tests/exposure_test.yaml +++ b/packages/extra/ingress/tests/exposure_test.yaml @@ -249,3 +249,17 @@ tests: path: spec.values.ingress-nginx.controller.service.externalIPs value: - 192.0.2.10 + + - it: loadBalancer mode accepts pre-CIDR input without double-suffixing + set: + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10/32,2001:db8::1/128" + expose-mode: loadBalancer + asserts: + - template: templates/cilium-lb-pool.yaml + equal: + path: spec.blocks + value: + - cidr: 192.0.2.10/32 + - cidr: 2001:db8::1/128 From 9b87bda06af20d7c1ca8d96b8f338f3bc937d0c5 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Fri, 24 Apr 2026 03:14:41 +0300 Subject: [PATCH 182/250] fix(postgres-operator): block HelmRelease Ready until the cnpg webhook actually serves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cnpg admission webhook's controller pod passes readinessProbe as soon as the local HTTPS server binds to :9443. The HelmRelease marks itself Ready right after that via helm install --wait. But the Service cnpg-webhook-service needs its EndpointSlice populated and the data plane (kube-proxy / cilium) programmed before kube-apiserver can reach the webhook through the Service ClusterIP. That gap is short but not zero, and any HelmRelease that depends on postgres-operator (cozy-keycloak, tenant Postgres apps) can fire its own install inside the window and hit Internal error occurred: failed calling webhook "mcluster.cnpg.io": failed to call webhook: Post "https://cnpg-webhook-service.cozy-postgres-operator.svc:443/...": dial tcp :443: connect: connection refused which fails the install of the downstream release. Seen on cozystack/cozystack#2470 E2E run 24862782568. Add a post-install,post-upgrade Helm hook Job that blocks the release from reporting Ready until the webhook answers /readyz through the apiserver service proxy. Apiserver proxy routes the call over the same Service IP → EndpointSlice → pod path the admission webhook uses, so once it responds, the webhook admission path is also working. RBAC is minimal: a dedicated ServiceAccount with a ClusterRole that only grants get on services/proxy scoped to https:cnpg-webhook-service:webhook-server. The Job times out after 120s with 60 attempts at 2s intervals — longer than any data-plane programming delay seen on E2E, but bounded. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- packages/system/postgres-operator/Makefile | 3 + .../templates/webhook-ready-hook.yaml | 139 ++++++++ .../tests/webhook-ready-hook_test.yaml | 308 ++++++++++++++++++ packages/system/postgres-operator/values.yaml | 25 ++ 4 files changed, 475 insertions(+) create mode 100644 packages/system/postgres-operator/templates/webhook-ready-hook.yaml create mode 100644 packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml diff --git a/packages/system/postgres-operator/Makefile b/packages/system/postgres-operator/Makefile index f6d7ddaa..5279f8a7 100644 --- a/packages/system/postgres-operator/Makefile +++ b/packages/system/postgres-operator/Makefile @@ -3,6 +3,9 @@ export NAMESPACE=cozy-$(NAME) include ../../../hack/package.mk +test: + helm unittest . + update: rm -rf charts helm repo add cnpg https://cloudnative-pg.github.io/charts diff --git a/packages/system/postgres-operator/templates/webhook-ready-hook.yaml b/packages/system/postgres-operator/templates/webhook-ready-hook.yaml new file mode 100644 index 00000000..2ad4e7c9 --- /dev/null +++ b/packages/system/postgres-operator/templates/webhook-ready-hook.yaml @@ -0,0 +1,139 @@ +{{- /* + Post-install gate: block the HelmRelease from reporting Ready until the + cnpg admission webhook actually serves through the cluster Service. Helm + --wait on the controller pod passes once its readinessProbe passes, but + EndpointSlice propagation and kube-proxy/cilium data-plane programming + can lag by a second or two — long enough for any HelmRelease that + depends on postgres-operator (e.g. cozy-keycloak, tenant Postgres apps) + to fire its own install and have kube-apiserver hit the mcluster.cnpg.io + mutating webhook with "dial tcp :443: connect: connection refused". + + The Job uses the apiserver service proxy, which exercises the same + endpoint-resolution and apiserver-initiated pod dial that the admission + webhook path uses. Once /readyz answers through the proxy the data-plane + race is resolved. It does not verify the webhook's TLS CA bundle, so + this gate is scoped to reachability regressions, not cert rotation. + + The service name and port name are hardcoded literals. Upstream cnpg + pins the service name in charts/cloudnative-pg/values.yaml with a + comment "DO NOT CHANGE THE SERVICE NAME as it is currently used to + generate the certificate and can not be configured". The port name is + fixed in charts/cloudnative-pg/templates/service.yaml (ports[0].name: + webhook-server). If a future `make update` ever changes either literal + upstream, the sync-check helm-unittest test + (tests/webhook-ready-hook_test.yaml) renders the subchart Service and + fails if the literal drifts — forcing this template to be updated in + the same change. +*/}} +{{- $_ := required "webhookReady.image.repository must be set to the container image providing kubectl for the post-install readiness Job" .Values.webhookReady.image.repository -}} +{{- $_ := required "webhookReady.image.tag must be set for the post-install readiness Job" .Values.webhookReady.image.tag -}} +{{- /* $svcName and $portName are hardcoded literals; see header comment. */ -}} +{{- $svcName := "cnpg-webhook-service" -}} +{{- /* $portName is the service port NAME, not number — matches ports[0].name in the vendored subchart's Service. */ -}} +{{- $portName := "webhook-server" -}} +{{- $resourceName := printf "https:%s:%s" $svcName $portName -}} +{{- $maxAttempts := .Values.webhookReady.maxAttempts | default 60 -}} +{{- $sleepSeconds := .Values.webhookReady.sleepSeconds | default 2 -}} +{{- /* Derive activeDeadlineSeconds from retries + 60s slack so a values override that raises maxAttempts doesn't get silently cut. */ -}} +{{- $deadline := add (mul (int $maxAttempts) (int $sleepSeconds)) 60 -}} +{{- $image := printf "%s:%s" .Values.webhookReady.image.repository .Values.webhookReady.image.tag -}} +{{- if .Values.webhookReady.image.digest }} +{{- $image = printf "%s@%s" $image .Values.webhookReady.image.digest -}} +{{- end }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }}-webhook-ready + namespace: {{ .Release.Namespace }} + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "0" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Release.Name }}-webhook-ready + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "0" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +rules: + - apiGroups: [""] + resources: ["services/proxy"] + resourceNames: [{{ $resourceName | quote }}] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Release.Name }}-webhook-ready + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "0" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Release.Name }}-webhook-ready +subjects: + - kind: ServiceAccount + name: {{ .Release.Name }}-webhook-ready + namespace: {{ .Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-webhook-ready + namespace: {{ .Release.Namespace }} + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "10" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +spec: + backoffLimit: {{ .Values.webhookReady.backoffLimit | default 2 }} + activeDeadlineSeconds: {{ $deadline }} + template: + spec: + restartPolicy: Never + serviceAccountName: {{ .Release.Name }}-webhook-ready + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: wait + image: {{ $image }} + imagePullPolicy: {{ if .Values.webhookReady.image.digest }}IfNotPresent{{ else }}Always{{ end }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + command: + - sh + - -c + - | + set -e + ns={{ .Release.Namespace }} + proxy="/api/v1/namespaces/${ns}/services/{{ $resourceName }}/proxy/readyz" + max_attempts={{ $maxAttempts }} + sleep_seconds={{ $sleepSeconds }} + i=0 + last_err="" + until last_err=$(kubectl get --raw "$proxy" 2>&1 >/dev/null); do + i=$((i + 1)) + if [ $i -gt $max_attempts ]; then + echo "timeout: cnpg webhook did not respond through the apiserver proxy after ${max_attempts} attempts (${sleep_seconds}s each)" + echo "last error: ${last_err}" + exit 1 + fi + if [ $((i % 10)) -eq 1 ]; then + echo "attempt $i/${max_attempts}: ${last_err}" + fi + sleep "$sleep_seconds" + done + echo "cnpg webhook is ready" diff --git a/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml b/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml new file mode 100644 index 00000000..69f5a6b5 --- /dev/null +++ b/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml @@ -0,0 +1,308 @@ +suite: cnpg webhook post-install readiness gate + +templates: + - templates/webhook-ready-hook.yaml + - charts/cloudnative-pg/templates/service.yaml + +release: + name: postgres-operator + namespace: cozy-postgres-operator + +tests: + - it: renders four hook objects (SA + ClusterRole + ClusterRoleBinding + Job) + template: templates/webhook-ready-hook.yaml + asserts: + - hasDocuments: + count: 4 + + - it: every rendered object carries post-install and post-upgrade hook annotations + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 0 + equal: + path: metadata.annotations["helm.sh/hook"] + value: post-install,post-upgrade + - documentIndex: 1 + equal: + path: metadata.annotations["helm.sh/hook"] + value: post-install,post-upgrade + - documentIndex: 2 + equal: + path: metadata.annotations["helm.sh/hook"] + value: post-install,post-upgrade + - documentIndex: 3 + equal: + path: metadata.annotations["helm.sh/hook"] + value: post-install,post-upgrade + + - it: RBAC is created before the Job (hook-weight ordering) + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 0 + equal: + path: kind + value: ServiceAccount + - documentIndex: 0 + equal: + path: metadata.annotations["helm.sh/hook-weight"] + value: "0" + - documentIndex: 1 + equal: + path: kind + value: ClusterRole + - documentIndex: 1 + equal: + path: metadata.annotations["helm.sh/hook-weight"] + value: "0" + - documentIndex: 2 + equal: + path: kind + value: ClusterRoleBinding + - documentIndex: 2 + equal: + path: metadata.annotations["helm.sh/hook-weight"] + value: "0" + - documentIndex: 3 + equal: + path: kind + value: Job + - documentIndex: 3 + equal: + path: metadata.annotations["helm.sh/hook-weight"] + value: "10" + + - it: RBAC resourceName matches the exact proxy URL segment the Job probes + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 1 + equal: + path: rules[0].resources[0] + value: services/proxy + - documentIndex: 1 + equal: + path: rules[0].resourceNames[0] + value: https:cnpg-webhook-service:webhook-server + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: "/api/v1/namespaces/\\$\\{ns\\}/services/https:cnpg-webhook-service:webhook-server/proxy/readyz" + + - it: hardcoded service name in the hook matches the vendored cnpg subchart Service (drift guard for make update) + template: charts/cloudnative-pg/templates/service.yaml + asserts: + - equal: + path: metadata.name + value: cnpg-webhook-service + - equal: + path: spec.ports[0].name + value: webhook-server + + - it: Job calls kubectl get --raw on the proxy path + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: "kubectl get --raw" + + - it: Job image is digest-pinned when webhookReady.image.digest is set + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: v1.32 + digest: sha256:b9ef7d8dbe65bcc81a46c09b8dc7543103055021c4f43287bf59e92a8f4fe05c + asserts: + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].image + value: docker.io/clastix/kubectl:v1.32@sha256:b9ef7d8dbe65bcc81a46c09b8dc7543103055021c4f43287bf59e92a8f4fe05c + + - it: Job image falls back to tag-only when digest is not configured + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: v1.32 + digest: "" + asserts: + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].image + value: docker.io/clastix/kubectl:v1.32 + + - it: backoffLimit defaults to 2 so transient pod-level failures retry instead of killing the HelmRelease + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 3 + equal: + path: spec.backoffLimit + value: 2 + + - it: backoffLimit is overridable + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + backoffLimit: 5 + asserts: + - documentIndex: 3 + equal: + path: spec.backoffLimit + value: 5 + + - it: chart render fails when webhookReady.image.repository is empty + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: "" + tag: v1.32 + asserts: + - failedTemplate: + errorMessage: "webhookReady.image.repository must be set to the container image providing kubectl for the post-install readiness Job" + + - it: chart render fails when webhookReady.image.tag is empty + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: "" + asserts: + - failedTemplate: + errorMessage: "webhookReady.image.tag must be set for the post-install readiness Job" + + - it: retry loop bounds default when blanked so a wiped override still produces a working Job + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: v1.32 + maxAttempts: null + sleepSeconds: null + asserts: + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: "max_attempts=60" + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: "sleep_seconds=2" + + - it: Job pod runs non-root with seccomp RuntimeDefault for restricted-PSA clusters + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 3 + equal: + path: spec.template.spec.securityContext.runAsNonRoot + value: true + - documentIndex: 3 + equal: + path: spec.template.spec.securityContext.seccompProfile.type + value: RuntimeDefault + + - it: Job container drops all capabilities and runs read-only rootfs + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation + value: false + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem + value: true + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].securityContext.capabilities.drop[0] + value: ALL + + - it: imagePullPolicy is IfNotPresent when digest-pinned, Always when tag-only + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: v1.32 + digest: sha256:b9ef7d8dbe65bcc81a46c09b8dc7543103055021c4f43287bf59e92a8f4fe05c + asserts: + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: IfNotPresent + + - it: imagePullPolicy is Always when no digest is configured + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: v1.32 + digest: "" + asserts: + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always + + - it: retry loop captures and surfaces the last kubectl error message on timeout + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'last_err=\$\(kubectl get --raw' + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'last error: \$\{last_err\}' + + - it: retry loop error message stays in sync when maxAttempts is bumped + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: v1.32 + maxAttempts: 90 + sleepSeconds: 3 + asserts: + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: "max_attempts=90" + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: "sleep_seconds=3" + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'after \$\{max_attempts\} attempts' + + - it: activeDeadlineSeconds scales with retry bounds so an override raise does not silently cut + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: v1.32 + maxAttempts: 180 + sleepSeconds: 3 + asserts: + - documentIndex: 3 + equal: + path: spec.activeDeadlineSeconds + value: 600 + + - it: activeDeadlineSeconds defaults include 60s slack over the default retry window + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 3 + equal: + path: spec.activeDeadlineSeconds + value: 180 diff --git a/packages/system/postgres-operator/values.yaml b/packages/system/postgres-operator/values.yaml index cae3dc53..854a3ac6 100644 --- a/packages/system/postgres-operator/values.yaml +++ b/packages/system/postgres-operator/values.yaml @@ -3,3 +3,28 @@ cloudnative-pg: create: true image: tag: "1.27.3" +# Image used by the post-install webhook-readiness Job (see templates/webhook-ready-hook.yaml). +# Any image with kubectl on PATH works; the Job calls the apiserver service proxy with the +# hook ServiceAccount's token to confirm the mcluster.cnpg.io webhook is reachable end-to-end +# before the HelmRelease reports Ready, closing the "connection refused" bootstrap race. +# +# The tag is digest-pinned so an upstream retag does not change what runs on every install +# and upgrade across the fleet. Refresh by resolving the current manifest-list digest +# (`docker manifest inspect docker.io/clastix/kubectl:v1.32`) and updating `digest` below. +# renovate: datasource=docker depName=docker.io/clastix/kubectl +webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: v1.32 + digest: sha256:b9ef7d8dbe65bcc81a46c09b8dc7543103055021c4f43287bf59e92a8f4fe05c + # Retry loop bounds for the readiness probe. Defaults total ~120s wall clock. + # Both are `default`-coerced in the template so an override that blanks them still + # produces a working Job. + maxAttempts: 60 + sleepSeconds: 2 + # Pod-level retry budget for transient node/registry failures (image pull rate limit, + # OOM, CNI hiccup). The shell loop inside the container only covers reachability retries. + # Pod retries and activeDeadlineSeconds (wall-clock bound on the whole Job across all + # pod retries) are ANDed: a 5-minute ImagePullBackOff on attempt 0 leaves only ~60s for + # subsequent retries before the Job is cut. + backoffLimit: 2 From 3b3540448975391ad7582db58d83ce5b18b15129 Mon Sep 17 00:00:00 2001 From: Denis Chernosov Date: Fri, 24 Apr 2026 11:32:20 +0400 Subject: [PATCH 183/250] fix(cozystack-engine): add managed Kubernetes without visible control-plane nodes support to lineage-controller-webhook (#2417) Signed-off-by: Denis Chernosov --- .../templates/daemonset.yaml | 2 + .../templates/deployment.yaml | 49 +++++++++++++++++++ .../templates/pdb.yaml | 11 +++++ .../lineage-controller-webhook/values.yaml | 5 ++ 4 files changed, 67 insertions(+) create mode 100644 packages/system/lineage-controller-webhook/templates/deployment.yaml create mode 100644 packages/system/lineage-controller-webhook/templates/pdb.yaml diff --git a/packages/system/lineage-controller-webhook/templates/daemonset.yaml b/packages/system/lineage-controller-webhook/templates/daemonset.yaml index 860aee6e..536ea24a 100644 --- a/packages/system/lineage-controller-webhook/templates/daemonset.yaml +++ b/packages/system/lineage-controller-webhook/templates/daemonset.yaml @@ -1,3 +1,4 @@ +{{- if not .Values.lineageControllerWebhook.deployment.enabled }} apiVersion: apps/v1 kind: DaemonSet metadata: @@ -51,3 +52,4 @@ spec: secret: secretName: lineage-controller-webhook-cert defaultMode: 0400 +{{- end }} \ No newline at end of file diff --git a/packages/system/lineage-controller-webhook/templates/deployment.yaml b/packages/system/lineage-controller-webhook/templates/deployment.yaml new file mode 100644 index 00000000..87717fd3 --- /dev/null +++ b/packages/system/lineage-controller-webhook/templates/deployment.yaml @@ -0,0 +1,49 @@ +{{- if .Values.lineageControllerWebhook.deployment.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: lineage-controller-webhook + labels: + app: lineage-controller-webhook +spec: + replicas: {{ .Values.lineageControllerWebhook.deployment.replicas }} + selector: + matchLabels: + app: lineage-controller-webhook + template: + metadata: + labels: + app: lineage-controller-webhook + spec: + serviceAccountName: lineage-controller-webhook + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app: lineage-controller-webhook + containers: + - name: lineage-controller-webhook + image: "{{ .Values.lineageControllerWebhook.image }}" + args: + {{- if .Values.lineageControllerWebhook.debug }} + - --zap-log-level=debug + {{- else }} + - --zap-log-level=info + {{- end }} + ports: + - name: webhook + containerPort: 9443 + volumeMounts: + - name: webhook-certs + mountPath: /tmp/k8s-webhook-server/serving-certs + readOnly: true + volumes: + - name: webhook-certs + secret: + secretName: lineage-controller-webhook-cert + defaultMode: 0400 +{{- end }} \ No newline at end of file diff --git a/packages/system/lineage-controller-webhook/templates/pdb.yaml b/packages/system/lineage-controller-webhook/templates/pdb.yaml new file mode 100644 index 00000000..b0a9d4e4 --- /dev/null +++ b/packages/system/lineage-controller-webhook/templates/pdb.yaml @@ -0,0 +1,11 @@ +{{- if .Values.lineageControllerWebhook.deployment.enabled }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: lineage-controller-webhook +spec: + minAvailable: {{ if gt (int .Values.lineageControllerWebhook.deployment.replicas) 1 }}1{{ else }}0{{ end }} + selector: + matchLabels: + app: lineage-controller-webhook +{{- end }} \ No newline at end of file diff --git a/packages/system/lineage-controller-webhook/values.yaml b/packages/system/lineage-controller-webhook/values.yaml index c5671548..2b84f4e1 100644 --- a/packages/system/lineage-controller-webhook/values.yaml +++ b/packages/system/lineage-controller-webhook/values.yaml @@ -2,9 +2,14 @@ lineageControllerWebhook: image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.3.0@sha256:e8984709686a5eaf19b89da378d7b8c688ea5607e0783a88d9c9e4ccfca96fb0 debug: false localK8sAPIEndpoint: + # incompatable with Deployment mode enabled: true # nodeSelector for the DaemonSet # Talos uses empty value: "node-role.kubernetes.io/control-plane": "" # Generic k8s (k3s, kubeadm) uses: "node-role.kubernetes.io/control-plane": "true" nodeSelector: node-role.kubernetes.io/control-plane: "" + # if enabled, replace DaemonSet by Deployment + deployment: + enabled: false + replicas: 1 From ab7deb2b055c2a677c4c84c1bb22271306ebf372 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Fri, 24 Apr 2026 13:03:49 +0300 Subject: [PATCH 184/250] chore(kubernetes): regenerate after HAMi addon integration Run make generate to update generated types, schema, README, and CRD definitions for the new HAMi addon. Signed-off-by: Arsolitt --- api/apps/v1alpha1/kubernetes/types.go | 12 ++++++++++++ packages/apps/kubernetes/README.md | 3 +++ packages/apps/kubernetes/values.schema.json | 4 ++-- .../system/kubernetes-rd/cozyrds/kubernetes.yaml | 4 ++-- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/api/apps/v1alpha1/kubernetes/types.go b/api/apps/v1alpha1/kubernetes/types.go index 774fcdc5..39a05567 100644 --- a/api/apps/v1alpha1/kubernetes/types.go +++ b/api/apps/v1alpha1/kubernetes/types.go @@ -66,6 +66,9 @@ type Addons struct { // NVIDIA GPU Operator. // +kubebuilder:default:={} GpuOperator GPUOperatorAddon `json:"gpuOperator"` + // HAMi GPU virtualization middleware. + // +kubebuilder:default:={} + Hami HAMiAddon `json:"hami"` // Ingress-NGINX controller. // +kubebuilder:default:={} IngressNginx IngressNginxAddon `json:"ingressNginx"` @@ -157,6 +160,15 @@ type GatewayAPIAddon struct { Enabled bool `json:"enabled"` } +type HAMiAddon struct { + // Enable HAMi (requires GPU Operator). + // +kubebuilder:default:=false + Enabled bool `json:"enabled"` + // Custom Helm values overrides. + // +kubebuilder:default:={} + ValuesOverride k8sRuntime.RawExtension `json:"valuesOverride"` +} + type IngressNginxAddon struct { // Enable the controller (requires nodes labeled `ingress-nginx`). // +kubebuilder:default:=false diff --git a/packages/apps/kubernetes/README.md b/packages/apps/kubernetes/README.md index d62d3a39..d705af64 100644 --- a/packages/apps/kubernetes/README.md +++ b/packages/apps/kubernetes/README.md @@ -128,6 +128,9 @@ See the reference for components utilized in this service: | `addons.gpuOperator` | NVIDIA GPU Operator. | `object` | `{}` | | `addons.gpuOperator.enabled` | Enable GPU Operator. | `bool` | `false` | | `addons.gpuOperator.valuesOverride` | Custom Helm values overrides. | `object` | `{}` | +| `addons.hami` | HAMi GPU virtualization middleware. | `object` | `{}` | +| `addons.hami.enabled` | Enable HAMi (requires GPU Operator). | `bool` | `false` | +| `addons.hami.valuesOverride` | Custom Helm values overrides. | `object` | `{}` | | `addons.fluxcd` | FluxCD GitOps operator. | `object` | `{}` | | `addons.fluxcd.enabled` | Enable FluxCD. | `bool` | `false` | | `addons.fluxcd.valuesOverride` | Custom Helm values overrides. | `object` | `{}` | diff --git a/packages/apps/kubernetes/values.schema.json b/packages/apps/kubernetes/values.schema.json index 3d7a17dd..1e457f1f 100644 --- a/packages/apps/kubernetes/values.schema.json +++ b/packages/apps/kubernetes/values.schema.json @@ -270,7 +270,7 @@ } }, "hami": { - "description": "HAMi GPU virtualization middleware. Enables fractional GPU sharing (memory and compute isolation) for workloads. Requires GPU Operator. Note: workload containers must use glibc < 2.34 (not musl/Alpine) for full compute isolation.", + "description": "HAMi GPU virtualization middleware.", "type": "object", "default": {}, "required": [ @@ -279,7 +279,7 @@ ], "properties": { "enabled": { - "description": "Enable HAMi.", + "description": "Enable HAMi (requires GPU Operator).", "type": "boolean", "default": false }, diff --git a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml index 5e9e8f94..cdf900e1 100644 --- a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml +++ b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml @@ -8,7 +8,7 @@ spec: singular: kubernetes plural: kuberneteses openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"ephemeralStorage":"20Gi","gpus":[],"instanceType":"u1.medium","maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"]}},"additionalProperties":{"type":"object","required":["ephemeralStorage","instanceType","maxReplicas","minReplicas","resources"],"properties":{"ephemeralStorage":{"description":"Ephemeral storage size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string"}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium"},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `.` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","ingressNginx","monitoringAgents","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"large","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}}}} + {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"ephemeralStorage":"20Gi","gpus":[],"instanceType":"u1.medium","maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"]}},"additionalProperties":{"type":"object","required":["ephemeralStorage","instanceType","maxReplicas","minReplicas","resources"],"properties":{"ephemeralStorage":{"description":"Ephemeral storage size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string"}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium"},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `.` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","hami","ingressNginx","monitoringAgents","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"hami":{"description":"HAMi GPU virtualization middleware.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable HAMi (requires GPU Operator).","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"large","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}}}} release: prefix: kubernetes- labels: @@ -26,7 +26,7 @@ spec: tags: - compute icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODFfMjg0NSkiLz4KPHBhdGggZD0iTTcxLjk5NjggMTlDNzAuMzAzOSAxOS4wMDAyIDY4LjkzMTIgMjAuNTMyMiA2OC45MzE0IDIyLjQyMjFDNjguOTMxNCAyMi40NTExIDY4LjkzNzMgMjIuNDc4OCA2OC45Mzc5IDIyLjUwNzZDNjguOTM1NCAyMi43NjQ0IDY4LjkyMzEgMjMuMDczNyA2OC45MzE0IDIzLjI5NzNDNjguOTcxNyAyNC4zODczIDY5LjIwODIgMjUuMjIxNiA2OS4zNTA2IDI2LjIyNThDNjkuNjA4NCAyOC4zNzUyIDY5LjgyNDUgMzAuMTU2OSA2OS42OTEyIDMxLjgxM0M2OS41NjE1IDMyLjQzNzUgNjkuMTAzNyAzMy4wMDg2IDY4LjY5NTYgMzMuNDA1Nkw2OC42MjM1IDM0LjcwODZDNjYuNzgzOSAzNC44NjE3IDY0LjkzMTkgMzUuMTQyMSA2My4wODIxIDM1LjU2NDFDNTUuMTIyNiAzNy4zNzk4IDQ4LjI2OTUgNDEuNDk5MSA0My4wNTIgNDcuMDYwOUM0Mi43MTM0IDQ2LjgyODggNDIuMTIxMSA0Ni40MDE5IDQxLjk0NSA0Ni4yNzEyQzQxLjM5NzcgNDYuMzQ1NCA0MC44NDQ1IDQ2LjUxNTEgNDAuMTI0MSA0Ni4wOTM1QzM4Ljc1MjIgNDUuMTY1NyAzNy41MDI4IDQzLjg4NTEgMzUuOTkxIDQyLjM0MjRDMzUuMjk4MiA0MS42MDQ0IDM0Ljc5NjYgNDAuOTAxOCAzMy45NzM1IDQwLjE5MDRDMzMuNzg2NiA0MC4wMjg5IDMzLjUwMTQgMzkuODEwNCAzMy4yOTIzIDM5LjY0NDJDMzIuNjQ4OSAzOS4xMjg4IDMxLjg5IDM4Ljg2IDMxLjE1NyAzOC44MzQ4QzMwLjIxNDcgMzguODAyNCAyOS4zMDc1IDM5LjE3MjUgMjguNzEzOCAzOS45MjA2QzI3LjY1ODQgNDEuMjUwNiAyNy45OTYzIDQzLjI4MzMgMjkuNDY3MSA0NC40NjE0QzI5LjQ4MiA0NC40NzM0IDI5LjQ5NzkgNDQuNDgyNyAyOS41MTI5IDQ0LjQ5NDNDMjkuNzE1IDQ0LjY1ODkgMjkuOTYyNSA0NC44Njk4IDMwLjE0ODMgNDUuMDA3NkMzMS4wMjE3IDQ1LjY1NTUgMzEuODE5NSA0NS45ODcyIDMyLjY4OTcgNDYuNTAxNUMzNC41MjMxIDQ3LjYzOTEgMzYuMDQzIDQ4LjU4MjMgMzcuMjQ4NiA0OS43MTk2QzM3LjcxOTQgNTAuMjIzNyAzNy44MDE2IDUxLjExMjIgMzcuODY0MyA1MS40OTY0TDM4Ljg0NjggNTIuMzc4MkMzMy41ODcyIDYwLjMzMDggMzEuMTUzIDcwLjE1MzkgMzIuNTkxNSA4MC4xNjI3TDMxLjMwNzcgODAuNTM3OEMzMC45NjkzIDgwLjk3NjggMzAuNDkxMiA4MS42Njc2IDI5Ljk5MTEgODEuODczOEMyOC40MTM4IDgyLjM3MjkgMjYuNjM4NyA4Mi41NTYyIDI0LjQ5NTYgODIuNzgxOUMyMy40ODk0IDgyLjg2NiAyMi42MjEzIDgyLjgxNTggMjEuNTU0NiA4My4wMTg4QzIxLjMxOTggODMuMDYzNSAyMC45OTI3IDgzLjE0OTEgMjAuNzM1OCA4My4yMDk3QzIwLjcyNjkgODMuMjExNiAyMC43MTg2IDgzLjIxNDIgMjAuNzA5NiA4My4yMTYyQzIwLjY5NTYgODMuMjE5NSAyMC42NzcyIDgzLjIyNjMgMjAuNjYzOCA4My4yMjk0QzE4Ljg1NyA4My42NjggMTcuNjk2MyA4NS4zMzY1IDE4LjA2OTkgODYuOTgwNUMxOC40NDM3IDg4LjYyNDggMjAuMjA4NiA4OS42MjQ4IDIyLjAyNjIgODkuMjMxMkMyMi4wMzkzIDg5LjIyODIgMjIuMDU4NCA4OS4yMjc3IDIyLjA3MiA4OS4yMjQ2QzIyLjA5MjYgODkuMjE5OSAyMi4xMTA2IDg5LjIwOTkgMjIuMTMxIDg5LjIwNDlDMjIuMzg0NCA4OS4xNDkgMjIuNzAxOSA4OS4wODY4IDIyLjkyMzYgODkuMDI3MkMyMy45NzIzIDg4Ljc0NTEgMjQuNzMxOCA4OC4zMzA2IDI1LjY3NDYgODcuOTY3N0MyNy43MDI5IDg3LjIzNjggMjkuMzgyOCA4Ni42MjYyIDMxLjAxOTUgODYuMzg4M0MzMS43MDMgODYuMzM0NSAzMi40MjMyIDg2LjgxMiAzMi43ODE0IDg3LjAxMzRMMzQuMTE3NyA4Ni43ODMxQzM3LjE5MjYgOTYuMzYxMyA0My42MzY2IDEwNC4xMDMgNTEuNzk2MyAxMDguOTYxTDUxLjIzOTYgMTEwLjMwM0M1MS40NDAzIDExMC44MjQgNTEuNjYxNiAxMTEuNTMgNTEuNTEyMSAxMTIuMDQ1QzUwLjkxNzEgMTEzLjU5NSA0OS44OTggMTE1LjIzMSA0OC43Mzc0IDExNy4wNTVDNDguMTc1NSAxMTcuODk4IDQ3LjYwMDQgMTE4LjU1MiA0Ny4wOTM0IDExOS41MTZDNDYuOTcyIDExOS43NDcgNDYuODE3NSAxMjAuMTAyIDQ2LjcwMDQgMTIwLjM0NkM0NS45MTI1IDEyMi4wMzkgNDYuNDkwNCAxMjMuOTkgNDguMDAzOCAxMjQuNzIyQzQ5LjUyNjggMTI1LjQ1OSA1MS40MTcxIDEyNC42ODIgNTIuMjM1MiAxMjIuOTg1QzUyLjIzNjQgMTIyLjk4MiA1Mi4yNDA2IDEyMi45OCA1Mi4yNDE3IDEyMi45NzhDNTIuMjQyNiAxMjIuOTc2IDUyLjI0MDkgMTIyLjk3MyA1Mi4yNDE3IDEyMi45NzFDNTIuMzU4MiAxMjIuNzMxIDUyLjUyMzMgMTIyLjQxNSA1Mi42MjE2IDEyMi4xODhDNTMuMDU2IDEyMS4xODkgNTMuMjAwNSAxMjAuMzMyIDUzLjUwNTkgMTE5LjM2NUM1NC4zMTcgMTE3LjMxOCA1NC43NjI2IDExNS4xNyA1NS44NzkxIDExMy44MzJDNTYuMTg0OSAxMTMuNDY2IDU2LjY4MzMgMTEzLjMyNSA1Ny4yMDAxIDExMy4xODZMNTcuODk0NCAxMTEuOTIyQzY1LjAwOCAxMTQuNjY1IDcyLjk3MDUgMTE1LjQwMiA4MC45MjQ1IDExMy41ODdDODIuNzM5MSAxMTMuMTczIDg0LjQ5MDggMTEyLjYzNyA4Ni4xODQzIDExMS45OTRDODYuMzc5NCAxMTIuMzQyIDg2Ljc0MiAxMTMuMDExIDg2LjgzOTMgMTEzLjE3OUM4Ny4zNjQ0IDExMy4zNTEgODcuOTM3NyAxMTMuNDM5IDg4LjQwNDcgMTE0LjEzM0M4OS4yNDAxIDExNS41NjcgODkuODExNCAxMTcuMjYzIDkwLjUwNzMgMTE5LjMxMkM5MC44MTI4IDEyMC4yNzkgOTAuOTYzOCAxMjEuMTM2IDkxLjM5ODEgMTIyLjEzNkM5MS40OTcxIDEyMi4zNjMgOTEuNjYxNCAxMjIuNjg0IDkxLjc3OCAxMjIuOTI1QzkyLjU5NDQgMTI0LjYyOCA5NC40OTA3IDEyNS40MDcgOTYuMDE1OSAxMjQuNjY5Qzk3LjUyOTIgMTIzLjkzNyA5OC4xMDc3IDEyMS45ODYgOTcuMzE5NCAxMjAuMjkzQzk3LjIwMjMgMTIwLjA0OSA5Ny4wNDEyIDExOS42OTUgOTYuOTE5OCAxMTkuNDY0Qzk2LjQxMjcgMTE4LjQ5OSA5NS44Mzc3IDExNy44NTIgOTUuMjc1OCAxMTcuMDA5Qzk0LjExNTIgMTE1LjE4NSA5My4xNTI2IDExMy42NyA5Mi41NTc1IDExMi4xMkM5Mi4zMDg3IDExMS4zMiA5Mi41OTk1IDExMC44MjMgOTIuNzkzMyAxMTAuMzAzQzkyLjY3NzIgMTEwLjE3IDkyLjQyODggMTA5LjQxNCA5Mi4yODI0IDEwOS4wNTlDMTAwLjc2MiAxMDQuMDI5IDEwNy4wMTcgOTUuOTk4NSAxMDkuOTU1IDg2LjcyMzlDMTEwLjM1MSA4Ni43ODY1IDExMS4wNDEgODYuOTA5MSAxMTEuMjY1IDg2Ljk1NDJDMTExLjcyNiA4Ni42NDg3IDExMi4xNDkgODYuMjUwMSAxMTIuOTgxIDg2LjMxNTlDMTE0LjYxNyA4Ni41NTM3IDExNi4yOTcgODcuMTY0NSAxMTguMzI2IDg3Ljg5NTNDMTE5LjI2OCA4OC4yNTgxIDEyMC4wMjggODguNjc5MyAxMjEuMDc3IDg4Ljk2MTRDMTIxLjI5OCA4OS4wMjEgMTIxLjYxNiA4OS4wNzY2IDEyMS44NjkgODkuMTMyNUMxMjEuODg5IDg5LjEzNzUgMTIxLjkwOCA4OS4xNDc1IDEyMS45MjggODkuMTUyMkMxMjEuOTQyIDg5LjE1NTMgMTIxLjk2MSA4OS4xNTU4IDEyMS45NzQgODkuMTU4OEMxMjMuNzkyIDg5LjU1MiAxMjUuNTU3IDg4LjU1MjYgMTI1LjkzIDg2LjkwODFDMTI2LjMwMyA4NS4yNjQxIDEyNS4xNDMgODMuNTk1MiAxMjMuMzM2IDgzLjE1N0MxMjMuMDc0IDgzLjA5NyAxMjIuNzAxIDgyLjk5NSAxMjIuNDQ2IDgyLjk0NjVDMTIxLjM3OSA4Mi43NDM1IDEyMC41MTEgODIuNzkzNSAxMTkuNTA1IDgyLjcwOTVDMTE3LjM2MSA4Mi40ODM5IDExNS41ODYgODIuMzAwNCAxMTQuMDA5IDgxLjgwMTRDMTEzLjM2NiA4MS41NTA3IDExMi45MDggODAuNzgxOSAxMTIuNjg2IDgwLjQ2NTVMMTExLjQ0OCA4MC4xMDM1QzExMi4wOSA3NS40MzggMTExLjkxNyA3MC41ODI1IDExMC44MDYgNjUuNzI0M0MxMDkuNjg1IDYwLjgyMDggMTA3LjcwNCA1Ni4zMzYxIDEwNS4wNjIgNTIuMzg0OEMxMDUuMzc5IDUyLjA5NDggMTA1Ljk3OSA1MS41NjEyIDEwNi4xNDkgNTEuNDA0M0MxMDYuMTk5IDUwLjg1MTcgMTA2LjE1NiA1MC4yNzIyIDEwNi43MjUgNDkuNjYwM0MxMDcuOTMxIDQ4LjUyMyAxMDkuNDUxIDQ3LjU3OTkgMTExLjI4NCA0Ni40NDIzQzExMi4xNTQgNDUuOTI3OSAxMTIuOTU5IDQ1LjU5NjQgMTEzLjgzMiA0NC45NDg0QzExNC4wMyA0NC44MDE5IDExNC4yOTkgNDQuNTY5OSAxMTQuNTA3IDQ0LjQwMjJDMTE1Ljk3NyA0My4yMjM3IDExNi4zMTYgNDEuMTkxMSAxMTUuMjYgMzkuODYxNEMxMTQuMjA0IDM4LjUzMTcgMTEyLjE1OSAzOC40MDY1IDExMC42ODggMzkuNTg1QzExMC40NzkgMzkuNzUxNiAxMTAuMTk1IDM5Ljk2ODggMTEwLjAwNyA0MC4xMzEyQzEwOS4xODQgNDAuODQyNiAxMDguNjc2IDQxLjU0NTIgMTA3Ljk4MyA0Mi4yODMyQzEwNi40NzEgNDMuODI1OSAxMDUuMjIyIDQ1LjExMyAxMDMuODUgNDYuMDQwOUMxMDMuMjU1IDQ2LjM4ODUgMTAyLjM4NSA0Ni4yNjgyIDEwMS45OSA0Ni4yNDQ5TDEwMC44MjQgNDcuMDgwNkM5NC4xNzUzIDQwLjA3NjMgODUuMTIzNSAzNS41OTgyIDc1LjM3NjYgMzQuNzI4M0M3NS4zNDk0IDM0LjMxNzkgNzUuMzEzNyAzMy41NzYxIDc1LjMwNDYgMzMuMzUyOUM3NC45MDU2IDMyLjk2OTMgNzQuNDIzNSAzMi42NDE4IDc0LjMwMjQgMzEuODEzQzc0LjE2OTEgMzAuMTU2OSA3NC4zOTE3IDI4LjM3NTIgNzQuNjQ5NiAyNi4yMjU4Qzc0Ljc5MTkgMjUuMjIxNiA3NS4wMjg0IDI0LjM4NzMgNzUuMDY4OCAyMy4yOTczQzc1LjA3OCAyMy4wNDk1IDc1LjA2MzIgMjIuNjkgNzUuMDYyMiAyMi40MjIxQzc1LjA2MiAyMC41MzIyIDczLjY4OTggMTguOTk5OCA3MS45OTY4IDE5Wk02OC4xNTg1IDQyLjg4ODZMNjcuMjQ4IDU5LjA0NDdMNjcuMTgyNSA1OS4wNzc2QzY3LjEyMTQgNjAuNTIyOSA2NS45Mzc1IDYxLjY3NyA2NC40ODM5IDYxLjY3N0M2My44ODg0IDYxLjY3NyA2My4zMzg4IDYxLjQ4NDkgNjIuODkyMiA2MS4xNTcxTDYyLjg2NiA2MS4xNzAzTDQ5LjY4MDcgNTEuNzc5NEM1My43MzMxIDQ3Ljc3NTkgNTguOTE2NCA0NC44MTcyIDY0Ljg5IDQzLjQ1NDZDNjUuOTgxMiA0My4yMDU2IDY3LjA3MTkgNDMuMDIwOSA2OC4xNTg1IDQyLjg4ODZaTTc1Ljg0MTcgNDIuODg4NkM4Mi44MTU5IDQzLjc1MDQgODkuMjY1NyA0Ni45MjMyIDk0LjIwODEgNTEuNzg2TDgxLjEwOCA2MS4xMTc2TDgxLjA2MjEgNjEuMDk3OUM3OS44OTk0IDYxLjk1MTIgNzguMjYxMSA2MS43Mzk0IDc3LjM1NDggNjAuNTk3OEM3Ni45ODM1IDYwLjEzMDEgNzYuNzg4NyA1OS41ODAxIDc2Ljc2NTMgNTkuMDI0OUw3Ni43NTIyIDU5LjAxODRMNzUuODQxNyA0Mi44ODg2Wk00NC44OTkxIDU3LjgxNEw1Ni45MzgyIDY4LjYzM0w1Ni45MjUxIDY4LjY5ODhDNTguMDExNyA2OS42NDc5IDU4LjE3MiA3MS4yOTQ5IDU3LjI2NTcgNzIuNDM2OEM1Ni44OTQ0IDcyLjkwNDUgNTYuMzk3NSA3My4yMTgyIDU1Ljg2MzkgNzMuMzY0N0w1NS44NTA4IDczLjQxNzNMNDAuNDE4OCA3Ny44OTIzQzM5LjYzMzQgNzAuNjc2NSA0MS4zMjYxIDYzLjY2MjEgNDQuODk5MSA1Ny44MTRaTTk5LjAwOTQgNTcuODIwNkMxMDAuNzk4IDYwLjczMzYgMTAyLjE1MyA2My45ODcxIDEwMi45NTkgNjcuNTE0M0MxMDMuNzU2IDcwLjk5OTEgMTAzLjk1NiA3NC40Nzc4IDEwMy42MjcgNzcuODM5N0w4OC4xMTY2IDczLjM1MTVMODguMTAzNSA3My4yODU3Qzg2LjcxNDUgNzIuOTA0MyA4NS44NjA5IDcxLjQ4NDggODYuMTg0MyA3MC4wNjExQzg2LjMxNjggNjkuNDc3OCA4Ni42MjQ5IDY4Ljk4NDQgODcuMDQyMyA2OC42MTk4TDg3LjAzNTggNjguNTg2OUw5OS4wMDk0IDU3LjgyMDZaTTY5LjUyNzQgNjkuNDY4OEg3NC40NTk2TDc3LjUyNTEgNzMuMzE4Nkw3Ni40MjQ3IDc4LjEyMjZMNzEuOTk2OCA4MC4yNjE0TDY3LjU1NTggNzguMTE2MUw2Ni40NTU0IDczLjMxMkw2OS41Mjc0IDY5LjQ2ODhaTTg1LjMzOTMgODIuNjQzN0M4NS41NDg5IDgyLjYzMzEgODUuNzU3NiA4Mi42NTIgODUuOTYxNiA4Mi42ODk4TDg1Ljk4NzggODIuNjU2OUwxMDEuOTUgODUuMzY4MkM5OS42MTQyIDkxLjk2MjQgOTUuMTQ0IDk3LjY3NSA4OS4xNzExIDEwMS40OThMODIuOTc0NyA4Ni40NjA2TDgyLjk5NDQgODYuNDM0M0M4Mi40MjUyIDg1LjEwNTUgODIuOTk0OCA4My41NDcyIDg0LjMwNDQgODIuOTEzNUM4NC42Mzk3IDgyLjc1MTMgODQuOTkgODIuNjYxNCA4NS4zMzkzIDgyLjY0MzdaTTU4LjUyOTggODIuNzA5NUM1OS43NDggODIuNzI2NyA2MC44NDA2IDgzLjU3NjEgNjEuMTIzNyA4NC44MjJDNjEuMjU2MiA4NS40MDUyIDYxLjE5MTcgODUuOTgzMSA2MC45NzMgODYuNDkzNUw2MS4wMTg5IDg2LjU1MjdMNTQuODg4IDEwMS40MzlDNDkuMTU1OSA5Ny43NDMyIDQ0LjU5MDQgOTIuMjA5OSA0Mi4xNDgxIDg1LjQyMDhMNTcuOTczMSA4Mi43MjI3TDU3Ljk5OTMgODIuNzU1NkM1OC4xNzYzIDgyLjcyMjkgNTguMzU1OCA4Mi43MDcxIDU4LjUyOTggODIuNzA5NVpNNzEuODk4NiA4OS4yMzEyQzcyLjMyMjkgODkuMjE1NSA3Mi43NTM0IDg5LjMwMyA3My4xNjI3IDg5LjUwMUM3My42OTkyIDg5Ljc2MDYgNzQuMTEzNiA5MC4xNjkyIDc0LjM3NDUgOTAuNjU5Mkg3NC40MzM0TDgyLjIzNDYgMTA0LjgyMUM4MS4yMjIxIDEwNS4xNjIgODAuMTgxMyAxMDUuNDU0IDc5LjExNjcgMTA1LjY5N0M3My4xNTA1IDEwNy4wNTggNjcuMjAzMiAxMDYuNjQ1IDYxLjgxOCAxMDQuODAyTDY5LjU5OTUgOTAuNjY1OEg2OS42MTI2QzcwLjA3OTUgODkuNzg4OCA3MC45NjUgODkuMjY1NiA3MS44OTg2IDg5LjIzMTJaIiBmaWxsPSJ3aGl0ZSIgc3Ryb2tlPSJ3aGl0ZSIgc3Ryb2tlLXdpZHRoPSIwLjI1Ii8+CjxkZWZzPgo8bGluZWFyR3JhZGllbnQgaWQ9InBhaW50MF9saW5lYXJfNjgxXzI4NDUiIHgxPSIxMCIgeTE9IjE1LjUiIHgyPSIxNDQiIHkyPSIxMzEuNSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiPgo8c3RvcCBzdG9wLWNvbG9yPSIjNEQ4N0ZGIi8+CjxzdG9wIG9mZnNldD0iMSIgc3RvcC1jb2xvcj0iIzA1NDdEMCIvPgo8L2xpbmVhckdyYWRpZW50Pgo8L2RlZnM+Cjwvc3ZnPgo= - keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "nodeGroups"], ["spec", "nodeGroups", "md0"], ["spec", "nodeGroups", "md0", "minReplicas"], ["spec", "nodeGroups", "md0", "maxReplicas"], ["spec", "nodeGroups", "md0", "instanceType"], ["spec", "nodeGroups", "md0", "ephemeralStorage"], ["spec", "nodeGroups", "md0", "roles"], ["spec", "nodeGroups", "md0", "resources"], ["spec", "nodeGroups", "md0", "gpus"], ["spec", "version"], ["spec", "host"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "fluxcd"], ["spec", "addons", "fluxcd", "enabled"], ["spec", "addons", "fluxcd", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"]] + keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "nodeGroups"], ["spec", "nodeGroups", "md0"], ["spec", "nodeGroups", "md0", "minReplicas"], ["spec", "nodeGroups", "md0", "maxReplicas"], ["spec", "nodeGroups", "md0", "instanceType"], ["spec", "nodeGroups", "md0", "ephemeralStorage"], ["spec", "nodeGroups", "md0", "roles"], ["spec", "nodeGroups", "md0", "resources"], ["spec", "nodeGroups", "md0", "gpus"], ["spec", "version"], ["spec", "host"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "hami"], ["spec", "addons", "hami", "enabled"], ["spec", "addons", "hami", "valuesOverride"], ["spec", "addons", "fluxcd"], ["spec", "addons", "fluxcd", "enabled"], ["spec", "addons", "fluxcd", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"]] secrets: exclude: [] include: From 648ad82dd3283076cf85f6b4b8953ceda8199476 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Fri, 24 Apr 2026 13:31:57 +0300 Subject: [PATCH 185/250] refactor(postgres-operator): scope webhook-ready RBAC to release namespace Address review feedback from gemini-code-assist on packages/system/postgres-operator/templates/webhook-ready-hook.yaml:83. The Job targets the cnpg-webhook-service/services/proxy subresource, which is namespaced and lives in the release namespace. A namespaced Role and RoleBinding grant the exact permission needed without creating global RBAC for a namespaced probe, which is the principle of least privilege. Also update the kind assertions in tests/webhook-ready-hook_test.yaml so the unittest suite tracks the new Role/RoleBinding objects. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- .../postgres-operator/templates/webhook-ready-hook.yaml | 8 +++++--- .../postgres-operator/tests/webhook-ready-hook_test.yaml | 6 +++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/system/postgres-operator/templates/webhook-ready-hook.yaml b/packages/system/postgres-operator/templates/webhook-ready-hook.yaml index 2ad4e7c9..70e51621 100644 --- a/packages/system/postgres-operator/templates/webhook-ready-hook.yaml +++ b/packages/system/postgres-operator/templates/webhook-ready-hook.yaml @@ -52,9 +52,10 @@ metadata: helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded --- apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole +kind: Role metadata: name: {{ .Release.Name }}-webhook-ready + namespace: {{ .Release.Namespace }} annotations: helm.sh/hook: post-install,post-upgrade helm.sh/hook-weight: "0" @@ -66,16 +67,17 @@ rules: verbs: ["get"] --- apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding +kind: RoleBinding metadata: name: {{ .Release.Name }}-webhook-ready + namespace: {{ .Release.Namespace }} annotations: helm.sh/hook: post-install,post-upgrade helm.sh/hook-weight: "0" helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded roleRef: apiGroup: rbac.authorization.k8s.io - kind: ClusterRole + kind: Role name: {{ .Release.Name }}-webhook-ready subjects: - kind: ServiceAccount diff --git a/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml b/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml index 69f5a6b5..eb7863da 100644 --- a/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml +++ b/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml @@ -9,7 +9,7 @@ release: namespace: cozy-postgres-operator tests: - - it: renders four hook objects (SA + ClusterRole + ClusterRoleBinding + Job) + - it: renders four hook objects (SA + Role + RoleBinding + Job) template: templates/webhook-ready-hook.yaml asserts: - hasDocuments: @@ -49,7 +49,7 @@ tests: - documentIndex: 1 equal: path: kind - value: ClusterRole + value: Role - documentIndex: 1 equal: path: metadata.annotations["helm.sh/hook-weight"] @@ -57,7 +57,7 @@ tests: - documentIndex: 2 equal: path: kind - value: ClusterRoleBinding + value: RoleBinding - documentIndex: 2 equal: path: metadata.annotations["helm.sh/hook-weight"] From fc057c039385e7b40cca3f2b6ca77a8f80bab49a Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Fri, 24 Apr 2026 13:32:33 +0300 Subject: [PATCH 186/250] fix(postgres-operator): set resources on webhook-ready wait container Address review feedback from gemini-code-assist on packages/system/postgres-operator/templates/webhook-ready-hook.yaml:115. The wait container had no resource requests or limits, so schedulers treated it as BestEffort and downstream quota enforcement had no signal. Set small requests (10m CPU, 32Mi memory) and conservative limits (100m CPU, 64Mi memory) matching the actual footprint of the kubectl polling loop. Add a matching unittest assertion so the values stay in sync if anyone touches the template. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- .../templates/webhook-ready-hook.yaml | 7 +++++++ .../tests/webhook-ready-hook_test.yaml | 20 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/packages/system/postgres-operator/templates/webhook-ready-hook.yaml b/packages/system/postgres-operator/templates/webhook-ready-hook.yaml index 70e51621..51a8ba81 100644 --- a/packages/system/postgres-operator/templates/webhook-ready-hook.yaml +++ b/packages/system/postgres-operator/templates/webhook-ready-hook.yaml @@ -110,6 +110,13 @@ spec: - name: wait image: {{ $image }} imagePullPolicy: {{ if .Values.webhookReady.image.digest }}IfNotPresent{{ else }}Always{{ end }} + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true diff --git a/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml b/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml index eb7863da..3253c663 100644 --- a/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml +++ b/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml @@ -205,6 +205,26 @@ tests: path: spec.template.spec.securityContext.seccompProfile.type value: RuntimeDefault + - it: wait container declares resource requests and limits for predictable scheduling + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].resources.requests.cpu + value: 10m + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].resources.requests.memory + value: 32Mi + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].resources.limits.cpu + value: 100m + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].resources.limits.memory + value: 64Mi + - it: Job container drops all capabilities and runs read-only rootfs template: templates/webhook-ready-hook.yaml asserts: From 7da29afe6e1f33b69d6040fd1051924d7937a13b Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Fri, 24 Apr 2026 13:33:07 +0300 Subject: [PATCH 187/250] docs(postgres-operator): fix backoffLimit/activeDeadlineSeconds comment math Address review feedback from coderabbitai on packages/system/postgres-operator/values.yaml:30. The old comment's 5-minute ImagePullBackOff scenario conflicted with the 180s activeDeadlineSeconds that the default maxAttempts/sleepSeconds resolve to, so the numbers could not both be taken at face value. Rewrite the comment to state the actual deadline math and frame the two gates as an AND with activeDeadlineSeconds being the shorter one under defaults, so readers understand why backoffLimit has little headroom without an accompanying maxAttempts/sleepSeconds bump. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- packages/system/postgres-operator/values.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/system/postgres-operator/values.yaml b/packages/system/postgres-operator/values.yaml index 854a3ac6..c408220e 100644 --- a/packages/system/postgres-operator/values.yaml +++ b/packages/system/postgres-operator/values.yaml @@ -25,6 +25,8 @@ webhookReady: # Pod-level retry budget for transient node/registry failures (image pull rate limit, # OOM, CNI hiccup). The shell loop inside the container only covers reachability retries. # Pod retries and activeDeadlineSeconds (wall-clock bound on the whole Job across all - # pod retries) are ANDed: a 5-minute ImagePullBackOff on attempt 0 leaves only ~60s for - # subsequent retries before the Job is cut. + # pod retries) are ANDed, so activeDeadlineSeconds is the shorter of the two gates with + # the defaults above: the 60*2+60 = 180s deadline cuts the Job before backoffLimit=2 + # ever matters if pod-level failures eat more than ~60s of the budget. Raise + # maxAttempts/sleepSeconds alongside backoffLimit when tuning for slow image pulls. backoffLimit: 2 From 2734dc0bcb96aaa087cbff6b2ee261e1bde8bffa Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Fri, 24 Apr 2026 14:00:48 +0300 Subject: [PATCH 188/250] fix(hami): clean up leftover hami-dra references and harden defaults Remove broken hami-dra subchart dependency from vendored chart (Chart.yaml, Chart.lock, values.yaml) and strip DRA condition guards from all templates since the subchart was already deleted. Override devicePlugin updateStrategy to OnDelete to prevent destructive rolling updates of GPU workloads. Align gpu-operator template with project pattern (unconditional values emission). Add nodeConfiguration format documentation and test for conflicting valuesOverride scenario. Signed-off-by: Arsolitt --- .../templates/helmreleases/gpu-operator.yaml | 6 ++--- .../tests/gpu_operator_hami_test.yaml | 25 ++++++++++++++++++- packages/system/hami/charts/hami/Chart.lock | 6 ----- packages/system/hami/charts/hami/Chart.yaml | 6 +---- .../templates/device-plugin/configmap.yaml | 2 +- .../device-plugin/daemonsetnvidia.yaml | 2 +- .../templates/device-plugin/monitorrole.yaml | 2 +- .../device-plugin/monitorrolebinding.yaml | 2 +- .../device-plugin/monitorservice.yaml | 2 +- .../device-plugin/monitorserviceaccount.yaml | 2 +- .../device-plugin/runtime-class.yaml | 4 +-- .../hami/templates/scheduler/certmanager.yaml | 2 +- .../hami/templates/scheduler/clusterrole.yaml | 2 -- .../scheduler/clusterrolebinding.yaml | 4 +-- .../hami/templates/scheduler/configmap.yaml | 2 +- .../templates/scheduler/configmapnew.yaml | 2 +- .../hami/templates/scheduler/deployment.yaml | 4 +-- .../templates/scheduler/device-configmap.yaml | 4 +-- .../scheduler/job-patch/clusterrole.yaml | 4 +-- .../job-patch/clusterrolebinding.yaml | 4 +-- .../scheduler/job-patch/job-createSecret.yaml | 4 +-- .../scheduler/job-patch/job-patchWebhook.yaml | 4 +-- .../templates/scheduler/job-patch/psp.yaml | 4 +-- .../templates/scheduler/job-patch/role.yaml | 4 +-- .../scheduler/job-patch/rolebinding.yaml | 4 +-- .../scheduler/job-patch/serviceaccount.yaml | 4 +-- .../charts/hami/templates/scheduler/role.yaml | 4 +-- .../hami/templates/scheduler/rolebinding.yaml | 2 -- .../hami/templates/scheduler/service.yaml | 4 +-- .../templates/scheduler/serviceaccount.yaml | 2 -- .../hami/templates/scheduler/webhook.yaml | 2 +- packages/system/hami/charts/hami/values.yaml | 18 ------------- packages/system/hami/values.yaml | 4 +++ 33 files changed, 55 insertions(+), 92 deletions(-) delete mode 100644 packages/system/hami/charts/hami/Chart.lock diff --git a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml index 995aa430..7a5ecaf1 100644 --- a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml @@ -1,5 +1,5 @@ {{- define "cozystack.defaultGpuOperatorValues" -}} -{{- if and (hasKey .Values.addons "hami") .Values.addons.hami.enabled }} +{{- if .Values.addons.hami.enabled }} gpu-operator: devicePlugin: enabled: false @@ -40,10 +40,8 @@ spec: {{- $defaults := fromYaml (include "cozystack.defaultGpuOperatorValues" .) }} {{- $overrides := deepCopy (default (dict) .Values.addons.gpuOperator.valuesOverride) }} {{- $merged := mergeOverwrite (default (dict) $defaults) $overrides }} - {{- with $merged }} values: - {{- toYaml . | nindent 4 }} - {{- end }} + {{- if $merged }}{{ toYaml $merged | nindent 4 }}{{ end }} dependsOn: {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }} diff --git a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml index 33256ae6..11de743a 100644 --- a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml +++ b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml @@ -26,8 +26,9 @@ tests: enabled: false valuesOverride: {} asserts: - - notExists: + - equal: path: spec.values + value: null - it: should allow user overrides to merge with hami defaults set: @@ -66,6 +67,28 @@ tests: path: spec.values.gpu-operator.devicePlugin.enabled value: true + - it: should let user explicitly override devicePlugin.enabled to true with hami enabled + set: + addons: + gpuOperator: + enabled: true + valuesOverride: + gpu-operator: + devicePlugin: + enabled: true + driver: + enabled: false + hami: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.values.gpu-operator.devicePlugin.enabled + value: true + - equal: + path: spec.values.gpu-operator.driver.enabled + value: false + - it: should not render when gpuOperator is disabled set: addons: diff --git a/packages/system/hami/charts/hami/Chart.lock b/packages/system/hami/charts/hami/Chart.lock deleted file mode 100644 index 25856c07..00000000 --- a/packages/system/hami/charts/hami/Chart.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: hami-dra - repository: https://project-hami.github.io/HAMi-DRA/ - version: 0.1.0 -digest: sha256:374551539570bcee82c1c4dea93eac59e6f410b6d5967d188efd630e203d43bb -generated: "2026-04-17T04:06:47.689117678Z" diff --git a/packages/system/hami/charts/hami/Chart.yaml b/packages/system/hami/charts/hami/Chart.yaml index aba7e95c..55f32ab6 100644 --- a/packages/system/hami/charts/hami/Chart.yaml +++ b/packages/system/hami/charts/hami/Chart.yaml @@ -1,10 +1,6 @@ apiVersion: v2 appVersion: 2.8.1 -dependencies: -- condition: dra.enabled - name: hami-dra - repository: https://project-hami.github.io/HAMi-DRA/ - version: 0.1.0 +dependencies: [] description: Heterogeneous AI Computing Virtualization Middleware keywords: - vgpu diff --git a/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml b/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml index b36347f2..db2ddacb 100644 --- a/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml +++ b/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.devicePlugin.enabled (not .Values.devicePlugin.nodeConfiguration.externalConfigName) (not .Values.dra.enabled) -}} +{{- if and .Values.devicePlugin.enabled (not .Values.devicePlugin.nodeConfiguration.externalConfigName) -}} apiVersion: v1 kind: ConfigMap metadata: diff --git a/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml index 577dc0f9..1f4c24f3 100644 --- a/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml +++ b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) }} +{{- if .Values.devicePlugin.enabled }} apiVersion: apps/v1 kind: DaemonSet metadata: diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml index c51ff6c6..377f6a86 100644 --- a/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}} +{{- if .Values.devicePlugin.enabled -}} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml index 93a118dd..2f0a14ba 100644 --- a/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}} +{{- if .Values.devicePlugin.enabled -}} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml index 24daca9b..0fb156cc 100644 --- a/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}} +{{- if .Values.devicePlugin.enabled -}} apiVersion: v1 kind: Service metadata: diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml index b10d2cb9..6e3c2a44 100644 --- a/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}} +{{- if .Values.devicePlugin.enabled -}} apiVersion: v1 kind: ServiceAccount metadata: diff --git a/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml b/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml index cd3d14cc..ebcc2c9c 100644 --- a/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml +++ b/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml @@ -1,5 +1,4 @@ -{{- if and .Values.devicePlugin.enabled (not .Values.dra.enabled) -}} -{{- if and .Values.devicePlugin.createRuntimeClass .Values.devicePlugin.runtimeClassName }} +{{- if and .Values.devicePlugin.enabled .Values.devicePlugin.createRuntimeClass .Values.devicePlugin.runtimeClassName -}} apiVersion: node.k8s.io/v1 kind: RuntimeClass metadata: @@ -7,5 +6,4 @@ metadata: annotations: helm.sh/hook: pre-install,pre-upgrade handler: nvidia -{{- end }} {{- end -}} diff --git a/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml b/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml index b6edb605..e6d28721 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} +{{- if .Values.scheduler.admissionWebhook.enabled -}} {{- if .Values.scheduler.certManager.enabled }} apiVersion: cert-manager.io/v1 kind: Certificate diff --git a/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml b/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml index 9c510c00..81c4fddf 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml @@ -1,4 +1,3 @@ -{{- if not .Values.dra.enabled -}} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -33,4 +32,3 @@ rules: resources: ["nodes"] verbs: ["get", "update", "list", "patch"] {{- end -}} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml index 69d75986..aa0f3c66 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml @@ -1,4 +1,3 @@ -{{- if not .Values.dra.enabled -}} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -45,5 +44,4 @@ roleRef: subjects: - kind: ServiceAccount name: {{ include "hami-vgpu.scheduler" . }} - namespace: {{ include "hami-vgpu.namespace" . }} -{{- end }} \ No newline at end of file + namespace: {{ include "hami-vgpu.namespace" . }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml b/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml index 75889146..109ecbce 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.scheduler.kubeScheduler.enabled (not .Values.dra.enabled) -}} +{{- if .Values.scheduler.kubeScheduler.enabled -}} apiVersion: v1 kind: ConfigMap metadata: diff --git a/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml b/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml index e2a91d8b..6f6db097 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.scheduler.kubeScheduler.enabled (not .Values.dra.enabled) -}} +{{- if .Values.scheduler.kubeScheduler.enabled -}} apiVersion: v1 kind: ConfigMap metadata: diff --git a/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml b/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml index 6364cae3..12911a42 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml @@ -1,4 +1,3 @@ -{{- if not .Values.dra.enabled -}} apiVersion: apps/v1 kind: Deployment metadata: @@ -223,5 +222,4 @@ spec: {{- end }} {{- if .Values.scheduler.nodeName }} nodeName: {{ .Values.scheduler.nodeName }} - {{- end }} -{{- end }} \ No newline at end of file + {{- end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml b/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml index ccb945ac..8738a06f 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml @@ -1,4 +1,3 @@ -{{- if not .Values.dra.enabled -}} apiVersion: v1 kind: ConfigMap metadata: @@ -406,5 +405,4 @@ data: memory: 12288 aiCore: 4 aiCPU: 4 - {{ end }} -{{- end }} \ No newline at end of file + {{ end }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml index b089ae82..412b1094 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml @@ -1,5 +1,4 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} -{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -27,4 +26,3 @@ rules: - {{ include "hami-vgpu.fullname" . }}-admission {{- end }} {{- end }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml index 64f01ecf..2b82f926 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml @@ -1,5 +1,4 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} -{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -19,4 +18,3 @@ subjects: name: {{ include "hami-vgpu.fullname" . }}-admission namespace: {{ include "hami-vgpu.namespace" . }} {{- end }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml index 400168a7..0e36d95f 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml @@ -1,5 +1,4 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} -{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} apiVersion: batch/v1 kind: Job metadata: @@ -67,4 +66,3 @@ spec: runAsNonRoot: true runAsUser: {{ .Values.scheduler.patch.runAsUser }} {{- end }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml index 71e97f6b..ce52042c 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml @@ -1,5 +1,4 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} -{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} apiVersion: batch/v1 kind: Job metadata: @@ -62,4 +61,3 @@ spec: runAsNonRoot: true runAsUser: {{ .Values.scheduler.patch.runAsUser }} {{- end }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml index a48270e7..1e9cd7da 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml @@ -1,5 +1,4 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} -{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} {{- if .Values.podSecurityPolicy.enabled }} apiVersion: policy/v1beta1 kind: PodSecurityPolicy @@ -37,4 +36,3 @@ spec: - downwardAPI {{- end }} {{- end }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml index 74e45681..56682f8e 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml @@ -1,5 +1,4 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} -{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: @@ -20,4 +19,3 @@ rules: - get - create {{- end }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml index 32da5ba4..7239b128 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml @@ -1,5 +1,4 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} -{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: @@ -20,4 +19,3 @@ subjects: name: {{ include "hami-vgpu.fullname" . }}-admission namespace: {{ include "hami-vgpu.namespace" . }} {{- end }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml index d2eb41de..857c6a8d 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml @@ -1,5 +1,4 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) }} -{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} apiVersion: v1 kind: ServiceAccount metadata: @@ -12,4 +11,3 @@ metadata: {{- include "hami-vgpu.labels" . | nindent 4 }} app.kubernetes.io/component: admission-webhook {{- end }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/role.yaml b/packages/system/hami/charts/hami/templates/scheduler/role.yaml index 6e681967..90987a4f 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/role.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/role.yaml @@ -1,4 +1,3 @@ -{{- if not .Values.dra.enabled -}} apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: @@ -10,5 +9,4 @@ metadata: rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] - verbs: ["create", "list", "watch", "get", "update", "patch"] -{{- end }} \ No newline at end of file + verbs: ["create", "list", "watch", "get", "update", "patch"] \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml index 934f8223..96e175a1 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml @@ -1,4 +1,3 @@ -{{- if not .Values.dra.enabled -}} apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: @@ -30,4 +29,3 @@ subjects: name: {{ include "hami-vgpu.mock-device-plugin" . }} namespace: {{ include "hami-vgpu.namespace" . }} {{- end -}} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/service.yaml b/packages/system/hami/charts/hami/templates/scheduler/service.yaml index 3e18bc98..70378c5c 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/service.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/service.yaml @@ -1,4 +1,3 @@ -{{- if not .Values.dra.enabled }} apiVersion: v1 kind: Service metadata: @@ -32,5 +31,4 @@ spec: protocol: TCP selector: app.kubernetes.io/component: hami-scheduler - {{- include "hami-vgpu.selectorLabels" . | nindent 4 }} -{{- end }} \ No newline at end of file + {{- include "hami-vgpu.selectorLabels" . | nindent 4 }} \ No newline at end of file diff --git a/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml b/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml index 929bf59a..0435c003 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml @@ -1,4 +1,3 @@ -{{- if not .Values.dra.enabled -}} apiVersion: v1 kind: ServiceAccount metadata: @@ -15,4 +14,3 @@ metadata: name: {{ include "hami-vgpu.mock-device-plugin" . }} namespace: {{ include "hami-vgpu.namespace" . }} {{- end -}} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml b/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml index 148feb5c..7296eb4e 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (not .Values.dra.enabled) -}} +{{- if .Values.scheduler.admissionWebhook.enabled -}} apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: diff --git a/packages/system/hami/charts/hami/values.yaml b/packages/system/hami/charts/hami/values.yaml index 9fbeca70..02feeada 100644 --- a/packages/system/hami/charts/hami/values.yaml +++ b/packages/system/hami/charts/hami/values.yaml @@ -392,24 +392,6 @@ mockDevicePlugin: ## - myRegistryKeySecretName ## pullSecrets: [] -# If this option is enabled, the DRA will be installed and the scheduler extender will not be installed. -dra: - enabled: false - -# Configuration for the hami-dra subchart, which includes DRA drivers. -# This is part of config for DRA, please refer to the DRA documentation for more details. -hami-dra: - monitor: - enabled: true - drivers: - nvidia: - enabled: true - # If you are using gpu driver on host, you need to set this to false - containerDriver: true - image: - repository: projecthami/k8s-dra-driver - tag: "v0.0.1-dev" - devices: amd: customresources: diff --git a/packages/system/hami/values.yaml b/packages/system/hami/values.yaml index 1155e2cb..1d1407b7 100644 --- a/packages/system/hami/values.yaml +++ b/packages/system/hami/values.yaml @@ -1,6 +1,10 @@ hami: devicePlugin: runtimeClassName: nvidia + updateStrategy: + type: OnDelete + # See upstream charts/hami/values.yaml for nodeConfiguration format. + # Each entry: {"name": "node-name", "operatingmode": "hami-core", "devicesplitcount": N, ...} nodeConfiguration: config: | { From 37d5ff0c6f0566ca18495e871c5ad626f7976ca1 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Fri, 24 Apr 2026 14:07:17 +0300 Subject: [PATCH 189/250] fix(hami): align templates with project patterns and clean dead code Unconditionally emit values in hami.yaml matching the project pattern. Remove duplicate test case and add coverage for omitted valuesOverride key. Delete dead PSP template and RBAC rules (policy/v1beta1 removed in K8s 1.25). Override kube-scheduler image registry to registry.k8s.io to avoid Chinese registry for international users. Signed-off-by: Arsolitt --- .../templates/helmreleases/hami.yaml | 4 +- .../tests/gpu_operator_hami_test.yaml | 29 ++++++-------- .../scheduler/job-patch/clusterrole.yaml | 7 ---- .../templates/scheduler/job-patch/psp.yaml | 38 ------------------- packages/system/hami/charts/hami/values.yaml | 3 -- packages/system/hami/values.yaml | 5 +++ 6 files changed, 18 insertions(+), 68 deletions(-) delete mode 100644 packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml diff --git a/packages/apps/kubernetes/templates/helmreleases/hami.yaml b/packages/apps/kubernetes/templates/helmreleases/hami.yaml index 1f7a5358..ac4714da 100644 --- a/packages/apps/kubernetes/templates/helmreleases/hami.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/hami.yaml @@ -32,10 +32,8 @@ spec: force: true remediation: retries: -1 - {{- with .Values.addons.hami.valuesOverride }} values: - {{- toYaml . | nindent 4 }} - {{- end }} + {{- if .Values.addons.hami.valuesOverride }}{{ toYaml .Values.addons.hami.valuesOverride | nindent 4 }}{{ end }} dependsOn: {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }} diff --git a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml index 11de743a..c3ec30f2 100644 --- a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml +++ b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml @@ -30,6 +30,18 @@ tests: path: spec.values value: null + - it: should apply hami defaults when valuesOverride key is omitted + set: + addons: + gpuOperator: + enabled: true + hami: + enabled: true + asserts: + - equal: + path: spec.values.gpu-operator.devicePlugin.enabled + value: false + - it: should allow user overrides to merge with hami defaults set: addons: @@ -50,23 +62,6 @@ tests: path: spec.values.gpu-operator.driver.enabled value: false - - it: should let user override devicePlugin back to true if needed - set: - addons: - gpuOperator: - enabled: true - valuesOverride: - gpu-operator: - devicePlugin: - enabled: true - hami: - enabled: true - valuesOverride: {} - asserts: - - equal: - path: spec.values.gpu-operator.devicePlugin.enabled - value: true - - it: should let user explicitly override devicePlugin.enabled to true with hami enabled set: addons: diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml index 412b1094..77e891cc 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml @@ -18,11 +18,4 @@ rules: verbs: - get - update -{{- if .Values.podSecurityPolicy.enabled }} - - apiGroups: ['extensions'] - resources: ['podsecuritypolicies'] - verbs: ['use'] - resourceNames: - - {{ include "hami-vgpu.fullname" . }}-admission -{{- end }} {{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml deleted file mode 100644 index 1e9cd7da..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/psp.yaml +++ /dev/null @@ -1,38 +0,0 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} -{{- if .Values.podSecurityPolicy.enabled }} -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - name: {{ include "hami-vgpu.fullname" . }}-admission - annotations: - "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded - labels: - {{- include "hami-vgpu.labels" . | nindent 4 }} - app.kubernetes.io/component: admission-webhook -spec: - allowPrivilegeEscalation: false - fsGroup: - ranges: - - max: 65535 - min: 1 - rule: MustRunAs - requiredDropCapabilities: - - ALL - runAsUser: - rule: MustRunAsNonRoot - seLinux: - rule: RunAsAny - supplementalGroups: - ranges: - - max: 65535 - min: 1 - rule: MustRunAs - volumes: - - configMap - - emptyDir - - projected - - secret - - downwardAPI -{{- end }} -{{- end }} diff --git a/packages/system/hami/charts/hami/values.yaml b/packages/system/hami/charts/hami/values.yaml index 02feeada..01caca0b 100644 --- a/packages/system/hami/charts/hami/values.yaml +++ b/packages/system/hami/charts/hami/values.yaml @@ -55,9 +55,6 @@ kunlunResourceVMemoryName: "kunlunxin.com/vxpu-memory" schedulerName: "hami-scheduler" -podSecurityPolicy: - enabled: false - scheduler: # @param nodeName defines the node name and the nvidia-vgpu-scheduler-scheduler will schedule to the node. # if we install the nvidia-vgpu-scheduler-scheduler as default scheduler, we need to remove the k8s default diff --git a/packages/system/hami/values.yaml b/packages/system/hami/values.yaml index 1d1407b7..f45d2998 100644 --- a/packages/system/hami/values.yaml +++ b/packages/system/hami/values.yaml @@ -1,4 +1,9 @@ hami: + scheduler: + kubeScheduler: + image: + registry: registry.k8s.io + repository: kube-scheduler devicePlugin: runtimeClassName: nvidia updateStrategy: From 6a9c310a4bfdb8bab3cd4f52724205adfbd2219e Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Fri, 24 Apr 2026 15:31:24 +0300 Subject: [PATCH 190/250] fix(hami): conditional values emission and cosmetic template cleanup Only emit values key in hami.yaml when valuesOverride has content, matching gpu-operator pattern. Add test verifying empty valuesOverride does not produce spec.values. Fix trailing whitespace and missing newlines in vendored chart templates. Signed-off-by: Arsolitt --- .../kubernetes/templates/helmreleases/hami.yaml | 4 +++- packages/apps/kubernetes/tests/hami_test.yaml | 15 +++++++++++++++ .../hami/charts/hami/templates/_commons.tpl | 2 +- .../hami/templates/device-plugin/configmap.yaml | 2 +- .../hami/templates/device-plugin/monitorrole.yaml | 3 +-- .../templates/device-plugin/monitorservice.yaml | 2 +- .../templates/scheduler/clusterrolebinding.yaml | 2 +- .../hami/templates/scheduler/deployment.yaml | 2 +- .../templates/scheduler/device-configmap.yaml | 2 +- .../charts/hami/templates/scheduler/role.yaml | 2 +- .../charts/hami/templates/scheduler/service.yaml | 2 +- .../charts/hami/templates/scheduler/webhook.yaml | 2 +- 12 files changed, 28 insertions(+), 12 deletions(-) diff --git a/packages/apps/kubernetes/templates/helmreleases/hami.yaml b/packages/apps/kubernetes/templates/helmreleases/hami.yaml index ac4714da..8733e675 100644 --- a/packages/apps/kubernetes/templates/helmreleases/hami.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/hami.yaml @@ -32,8 +32,10 @@ spec: force: true remediation: retries: -1 + {{- if .Values.addons.hami.valuesOverride }} values: - {{- if .Values.addons.hami.valuesOverride }}{{ toYaml .Values.addons.hami.valuesOverride | nindent 4 }}{{ end }} + {{- toYaml .Values.addons.hami.valuesOverride | nindent 4 }} + {{- end }} dependsOn: {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }} diff --git a/packages/apps/kubernetes/tests/hami_test.yaml b/packages/apps/kubernetes/tests/hami_test.yaml index e9d32236..bdbe990f 100644 --- a/packages/apps/kubernetes/tests/hami_test.yaml +++ b/packages/apps/kubernetes/tests/hami_test.yaml @@ -118,6 +118,21 @@ tests: name: test-gpu-operator namespace: test-ns + - it: should not render spec.values when valuesOverride is empty + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - hasDocuments: + count: 1 + - notExists: + path: spec.values + - it: should pass through valuesOverride set: addons: diff --git a/packages/system/hami/charts/hami/templates/_commons.tpl b/packages/system/hami/charts/hami/templates/_commons.tpl index 78a262c9..b68018e4 100644 --- a/packages/system/hami/charts/hami/templates/_commons.tpl +++ b/packages/system/hami/charts/hami/templates/_commons.tpl @@ -46,4 +46,4 @@ imagePullSecrets: - name: {{ . }} {{- end }} {{- end }} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml b/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml index db2ddacb..74631e24 100644 --- a/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml +++ b/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml @@ -10,4 +10,4 @@ metadata: data: config.json: | {{- .Values.devicePlugin.nodeConfiguration.config | nindent 4 }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml index 377f6a86..6ac757f2 100644 --- a/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml @@ -25,5 +25,4 @@ rules: - list - patch {{- end -}} - - + diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml index 0fb156cc..104664ea 100644 --- a/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml @@ -26,4 +26,4 @@ spec: selector: app.kubernetes.io/component: hami-device-plugin {{- include "hami-vgpu.selectorLabels" . | nindent 4 }} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml index aa0f3c66..a81d425c 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml @@ -44,4 +44,4 @@ roleRef: subjects: - kind: ServiceAccount name: {{ include "hami-vgpu.scheduler" . }} - namespace: {{ include "hami-vgpu.namespace" . }} \ No newline at end of file + namespace: {{ include "hami-vgpu.namespace" . }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml b/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml index 12911a42..1d89e189 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml @@ -222,4 +222,4 @@ spec: {{- end }} {{- if .Values.scheduler.nodeName }} nodeName: {{ .Values.scheduler.nodeName }} - {{- end }} \ No newline at end of file + {{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml b/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml index 8738a06f..873b813b 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml @@ -405,4 +405,4 @@ data: memory: 12288 aiCore: 4 aiCPU: 4 - {{ end }} \ No newline at end of file + {{ end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/role.yaml b/packages/system/hami/charts/hami/templates/scheduler/role.yaml index 90987a4f..5f7d7e44 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/role.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/role.yaml @@ -9,4 +9,4 @@ metadata: rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] - verbs: ["create", "list", "watch", "get", "update", "patch"] \ No newline at end of file + verbs: ["create", "list", "watch", "get", "update", "patch"] diff --git a/packages/system/hami/charts/hami/templates/scheduler/service.yaml b/packages/system/hami/charts/hami/templates/scheduler/service.yaml index 70378c5c..d7538fed 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/service.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/service.yaml @@ -31,4 +31,4 @@ spec: protocol: TCP selector: app.kubernetes.io/component: hami-scheduler - {{- include "hami-vgpu.selectorLabels" . | nindent 4 }} \ No newline at end of file + {{- include "hami-vgpu.selectorLabels" . | nindent 4 }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml b/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml index 7296eb4e..db9f8029 100644 --- a/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml +++ b/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml @@ -54,4 +54,4 @@ webhooks: scope: '*' sideEffects: None timeoutSeconds: 10 -{{- end }} \ No newline at end of file +{{- end }} From 1131e2f113a8fda8402bf9c92014bff9aaf29f25 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Fri, 24 Apr 2026 15:33:07 +0300 Subject: [PATCH 191/250] docs(hami): reference upstream repo for nodeConfiguration format Signed-off-by: Arsolitt --- packages/system/hami/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/system/hami/values.yaml b/packages/system/hami/values.yaml index f45d2998..64b372a6 100644 --- a/packages/system/hami/values.yaml +++ b/packages/system/hami/values.yaml @@ -8,7 +8,7 @@ hami: runtimeClassName: nvidia updateStrategy: type: OnDelete - # See upstream charts/hami/values.yaml for nodeConfiguration format. + # See https://github.com/Project-HAMi/HAMi/blob/v2.8.1/deployments/charts/hami/values.yaml # Each entry: {"name": "node-name", "operatingmode": "hami-core", "devicesplitcount": N, ...} nodeConfiguration: config: | From 3eeda2ba3596851393fd62d3d352cdd1017be267 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Fri, 24 Apr 2026 16:12:34 +0300 Subject: [PATCH 192/250] chore(kubernetes): regenerate code after hami addon addition Signed-off-by: Arsolitt --- .../kubernetes/zz_generated.deepcopy.go | 17 +++++++++++++++++ api/backups/v1alpha1/zz_generated.deepcopy.go | 5 +++++ 2 files changed, 22 insertions(+) diff --git a/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go b/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go index 2cbe3ba1..3f1cb5ce 100644 --- a/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go +++ b/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go @@ -49,6 +49,7 @@ func (in *Addons) DeepCopyInto(out *Addons) { in.Fluxcd.DeepCopyInto(&out.Fluxcd) out.GatewayAPI = in.GatewayAPI in.GpuOperator.DeepCopyInto(&out.GpuOperator) + in.Hami.DeepCopyInto(&out.Hami) in.IngressNginx.DeepCopyInto(&out.IngressNginx) in.MonitoringAgents.DeepCopyInto(&out.MonitoringAgents) in.Velero.DeepCopyInto(&out.Velero) @@ -260,6 +261,22 @@ func (in *GatewayAPIAddon) DeepCopy() *GatewayAPIAddon { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HAMiAddon) DeepCopyInto(out *HAMiAddon) { + *out = *in + in.ValuesOverride.DeepCopyInto(&out.ValuesOverride) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HAMiAddon. +func (in *HAMiAddon) DeepCopy() *HAMiAddon { + if in == nil { + return nil + } + out := new(HAMiAddon) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IngressNginxAddon) DeepCopyInto(out *IngressNginxAddon) { *out = *in diff --git a/api/backups/v1alpha1/zz_generated.deepcopy.go b/api/backups/v1alpha1/zz_generated.deepcopy.go index 89f6171f..61b8f839 100644 --- a/api/backups/v1alpha1/zz_generated.deepcopy.go +++ b/api/backups/v1alpha1/zz_generated.deepcopy.go @@ -620,6 +620,11 @@ func (in *RestoreJobSpec) DeepCopyInto(out *RestoreJobSpec) { *out = new(v1.TypedLocalObjectReference) (*in).DeepCopyInto(*out) } + if in.Options != nil { + in, out := &in.Options, &out.Options + *out = new(runtime.RawExtension) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestoreJobSpec. From 500816b71b1ad3f916e11ff3ddb1fd6dee9fcc76 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Fri, 24 Apr 2026 16:44:46 +0300 Subject: [PATCH 193/250] refactor(ingress): move CiliumLoadBalancerIPPool to tenant chart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pool is now rendered from packages/apps/tenant/templates/cilium-lb-pool.yaml instead of packages/extra/ingress/templates/cilium-lb-pool.yaml. Cilium LB IPAM forbids overlapping CIDRs across pools regardless of serviceSelector, so both the ingress-loadBalancer path and the upcoming per-tenant Gateway in #2470 cannot each own their own pool on the same publishing.externalIPs range. The tenant chart is the natural per-tenant owner — it already creates the Namespace, the cozystack-values Secret, and the HelmReleases for both ingress and gateway. The new pool uses a namespace-only serviceSelector (io.kubernetes.service.namespace: ), which matches any LoadBalancer Service in the tenant namespace. The metadata.name changed from -ingress to -exposure to reflect that the pool is not ingress-specific. Only the ingress-loadBalancer signal is wired in this commit (_cluster.expose-mode=loadBalancer plus .Values.ingress=true on the publishing tenant). The gateway branch is added in #2470 on top of this commit — it rebases, drops its own packages/extra/gateway/templates/cilium-lb-pool.yaml, and adds an OR branch for .Values.gateway in the tenant template. Pool-rendering unit tests moved from packages/extra/ingress/tests/ to packages/apps/tenant/tests/. The ingress chart tests keep the Service-level asserts. packages/apps/tenant/Makefile gains a test target so hack/helm-unit-tests.sh picks up the new suite. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- packages/apps/tenant/Makefile | 3 + .../tenant}/templates/cilium-lb-pool.yaml | 21 +-- packages/apps/tenant/tests/exposure_test.yaml | 141 ++++++++++++++++++ .../extra/ingress/tests/exposure_test.yaml | 104 +------------ 4 files changed, 158 insertions(+), 111 deletions(-) rename packages/{extra/ingress => apps/tenant}/templates/cilium-lb-pool.yaml (54%) create mode 100644 packages/apps/tenant/tests/exposure_test.yaml diff --git a/packages/apps/tenant/Makefile b/packages/apps/tenant/Makefile index 3fe3810d..2f51d836 100644 --- a/packages/apps/tenant/Makefile +++ b/packages/apps/tenant/Makefile @@ -3,3 +3,6 @@ include ../../../hack/package.mk generate: cozyvalues-gen -m 'tenant' -v values.yaml -s values.schema.json -r README.md -g ../../../api/apps/v1alpha1/tenant/types.go ../../../hack/update-crd.sh + +test: + helm unittest . diff --git a/packages/extra/ingress/templates/cilium-lb-pool.yaml b/packages/apps/tenant/templates/cilium-lb-pool.yaml similarity index 54% rename from packages/extra/ingress/templates/cilium-lb-pool.yaml rename to packages/apps/tenant/templates/cilium-lb-pool.yaml index 048eb58b..63dbcaca 100644 --- a/packages/extra/ingress/templates/cilium-lb-pool.yaml +++ b/packages/apps/tenant/templates/cilium-lb-pool.yaml @@ -1,25 +1,28 @@ -{{- $exposeIngress := (index .Values._cluster "expose-ingress") | default "tenant-root" }} {{- $exposeMode := (index .Values._cluster "expose-mode") | default "externalIPs" }} -{{- $exposeExternalIPs := (index .Values._cluster "expose-external-ips") | default "" | nospace }} -{{- $exposeIPsList := list }} -{{- range splitList "," $exposeExternalIPs }} +{{- $exposeIngress := (index .Values._cluster "expose-ingress") | default "tenant-root" }} +{{- $exposeIPs := (index .Values._cluster "expose-external-ips") | default "" | nospace }} +{{- $ipsList := list }} +{{- range splitList "," $exposeIPs }} {{- $ip := . | trim }} {{- if $ip }} - {{- $exposeIPsList = append $exposeIPsList $ip }} + {{- $ipsList = append $ipsList $ip }} {{- end }} {{- end }} -{{- if and (eq $exposeMode "loadBalancer") (eq $exposeIngress .Release.Namespace) $exposeIPsList }} +{{- $isPublishingIngressLB := and + (eq $exposeMode "loadBalancer") + (eq $exposeIngress .Release.Namespace) + .Values.ingress }} +{{- if and $isPublishingIngressLB $ipsList }} apiVersion: cilium.io/v2 kind: CiliumLoadBalancerIPPool metadata: - name: {{ trimPrefix "tenant-" .Release.Namespace }}-ingress + name: {{ trimPrefix "tenant-" .Release.Namespace }}-exposure spec: blocks: - {{- range $exposeIPsList }} + {{- range $ipsList }} - cidr: {{ . }}{{ if not (contains "/" .) }}/{{ if contains ":" . }}128{{ else }}32{{ end }}{{ end }} {{- end }} serviceSelector: matchLabels: "io.kubernetes.service.namespace": {{ .Release.Namespace | quote }} - "app.kubernetes.io/name": ingress-nginx {{- end }} diff --git a/packages/apps/tenant/tests/exposure_test.yaml b/packages/apps/tenant/tests/exposure_test.yaml new file mode 100644 index 00000000..e86fab5b --- /dev/null +++ b/packages/apps/tenant/tests/exposure_test.yaml @@ -0,0 +1,141 @@ +suite: tenant CiliumLoadBalancerIPPool rendering for publishing.exposure=loadBalancer +templates: + - templates/cilium-lb-pool.yaml + +release: + name: tenant-root + namespace: tenant-root + +tests: + - it: default exposure (externalIPs) renders no pool + set: + ingress: true + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10,192.0.2.11" + asserts: + - hasDocuments: + count: 0 + + - it: loadBalancer mode in publishing tenant with ingress=true renders v2 pool with namespace-only selector + set: + ingress: true + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10,192.0.2.11" + expose-mode: loadBalancer + asserts: + - hasDocuments: + count: 1 + - equal: + path: apiVersion + value: cilium.io/v2 + - equal: + path: kind + value: CiliumLoadBalancerIPPool + - equal: + path: metadata.name + value: root-exposure + - equal: + path: spec.blocks + value: + - cidr: 192.0.2.10/32 + - cidr: 192.0.2.11/32 + - equal: + path: spec.serviceSelector.matchLabels["io.kubernetes.service.namespace"] + value: tenant-root + - notExists: + path: spec.serviceSelector.matchLabels["app.kubernetes.io/name"] + + - it: loadBalancer mode with IPv6 emits /128 CIDR + set: + ingress: true + _cluster: + expose-ingress: tenant-root + expose-external-ips: "2001:db8::1" + expose-mode: loadBalancer + asserts: + - equal: + path: spec.blocks + value: + - cidr: 2001:db8::1/128 + + - it: loadBalancer mode with mixed IPv4 and IPv6 emits correct CIDR per family + set: + ingress: true + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10,2001:db8::1" + expose-mode: loadBalancer + asserts: + - equal: + path: spec.blocks + value: + - cidr: 192.0.2.10/32 + - cidr: 2001:db8::1/128 + + - it: loadBalancer mode accepts pre-CIDR input without double-suffixing + set: + ingress: true + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10/32,2001:db8::1/128" + expose-mode: loadBalancer + asserts: + - equal: + path: spec.blocks + value: + - cidr: 192.0.2.10/32 + - cidr: 2001:db8::1/128 + + - it: loadBalancer mode filters out empty entries from externalIPs (trailing, leading, repeated commas) + set: + ingress: true + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10,,192.0.2.11," + expose-mode: loadBalancer + asserts: + - hasDocuments: + count: 1 + - equal: + path: spec.blocks + value: + - cidr: 192.0.2.10/32 + - cidr: 192.0.2.11/32 + + - it: loadBalancer mode with ingress=false in publishing tenant renders no pool + set: + ingress: false + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10" + expose-mode: loadBalancer + asserts: + - hasDocuments: + count: 0 + + - it: loadBalancer mode in a non-publishing tenant renders no pool + set: + ingress: true + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10" + expose-mode: loadBalancer + release: + name: tenant-u1 + namespace: tenant-u1 + asserts: + - hasDocuments: + count: 0 + + - it: loadBalancer mode in publishing tenant with empty externalIPs renders no pool + set: + ingress: true + _cluster: + expose-ingress: tenant-root + expose-external-ips: "" + expose-mode: loadBalancer + asserts: + - hasDocuments: + count: 0 diff --git a/packages/extra/ingress/tests/exposure_test.yaml b/packages/extra/ingress/tests/exposure_test.yaml index 323c8e68..1c76b3da 100644 --- a/packages/extra/ingress/tests/exposure_test.yaml +++ b/packages/extra/ingress/tests/exposure_test.yaml @@ -1,14 +1,13 @@ suite: ingress exposure modes templates: - templates/nginx-ingress.yaml - - templates/cilium-lb-pool.yaml release: name: ingress namespace: tenant-root tests: - - it: default exposure (externalIPs) renders ClusterIP Service with spec.externalIPs and no CiliumLoadBalancerIPPool + - it: default exposure (externalIPs) renders ClusterIP Service with spec.externalIPs set: _cluster: expose-ingress: tenant-root @@ -31,9 +30,6 @@ tests: - template: templates/nginx-ingress.yaml notExists: path: spec.values.ingress-nginx.controller.service.labels - - template: templates/cilium-lb-pool.yaml - hasDocuments: - count: 0 - it: legacy config without expose-mode falls back to externalIPs behavior set: @@ -45,9 +41,6 @@ tests: equal: path: spec.values.ingress-nginx.controller.service.type value: ClusterIP - - template: templates/cilium-lb-pool.yaml - hasDocuments: - count: 0 - it: externalIPs mode in a namespace other than publishing.ingressName renders LoadBalancer fallback without externalIPs set: @@ -68,11 +61,8 @@ tests: - template: templates/nginx-ingress.yaml notExists: path: spec.values.ingress-nginx.controller.service.externalIPs - - template: templates/cilium-lb-pool.yaml - hasDocuments: - count: 0 - - it: loadBalancer mode renders LoadBalancer Service with lb-pool label and a v2 CiliumLoadBalancerIPPool + - it: loadBalancer mode renders LoadBalancer Service without externalIPs on the Service set: _cluster: expose-ingress: tenant-root @@ -93,62 +83,6 @@ tests: - template: templates/nginx-ingress.yaml notExists: path: spec.values.ingress-nginx.controller.service.externalIPs - - template: templates/cilium-lb-pool.yaml - hasDocuments: - count: 1 - - template: templates/cilium-lb-pool.yaml - equal: - path: apiVersion - value: cilium.io/v2 - - template: templates/cilium-lb-pool.yaml - equal: - path: kind - value: CiliumLoadBalancerIPPool - - template: templates/cilium-lb-pool.yaml - equal: - path: metadata.name - value: root-ingress - - template: templates/cilium-lb-pool.yaml - equal: - path: spec.blocks - value: - - cidr: 192.0.2.10/32 - - cidr: 192.0.2.11/32 - - template: templates/cilium-lb-pool.yaml - equal: - path: spec.serviceSelector.matchLabels["io.kubernetes.service.namespace"] - value: tenant-root - - template: templates/cilium-lb-pool.yaml - equal: - path: spec.serviceSelector.matchLabels["app.kubernetes.io/name"] - value: ingress-nginx - - - it: loadBalancer mode with IPv6 address emits /128 CIDR - set: - _cluster: - expose-ingress: tenant-root - expose-external-ips: "2001:db8::1" - expose-mode: loadBalancer - asserts: - - template: templates/cilium-lb-pool.yaml - equal: - path: spec.blocks - value: - - cidr: 2001:db8::1/128 - - - it: loadBalancer mode with mixed IPv4 and IPv6 emits correct CIDR per family - set: - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10,2001:db8::1" - expose-mode: loadBalancer - asserts: - - template: templates/cilium-lb-pool.yaml - equal: - path: spec.blocks - value: - - cidr: 192.0.2.10/32 - - cidr: 2001:db8::1/128 - it: loadBalancer mode without externalIPs fails chart render with explicit message set: @@ -206,26 +140,6 @@ tests: - template: templates/nginx-ingress.yaml notExists: path: spec.values.ingress-nginx.controller.service.externalIPs - - template: templates/cilium-lb-pool.yaml - hasDocuments: - count: 0 - - - it: loadBalancer mode filters out empty entries from externalIPs (trailing comma, leading comma) - set: - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10,,192.0.2.11," - expose-mode: loadBalancer - asserts: - - template: templates/cilium-lb-pool.yaml - hasDocuments: - count: 1 - - template: templates/cilium-lb-pool.yaml - equal: - path: spec.blocks - value: - - cidr: 192.0.2.10/32 - - cidr: 192.0.2.11/32 - it: loadBalancer mode with only-empty externalIPs fails chart render (comma-only input) set: @@ -249,17 +163,3 @@ tests: path: spec.values.ingress-nginx.controller.service.externalIPs value: - 192.0.2.10 - - - it: loadBalancer mode accepts pre-CIDR input without double-suffixing - set: - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10/32,2001:db8::1/128" - expose-mode: loadBalancer - asserts: - - template: templates/cilium-lb-pool.yaml - equal: - path: spec.blocks - value: - - cidr: 192.0.2.10/32 - - cidr: 2001:db8::1/128 From c8ed1c652cbd3d442036251358c68e7b25ec64a1 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Mon, 27 Apr 2026 02:35:52 +0300 Subject: [PATCH 196/250] chore(ci): adopt CNCF/k8s label conventions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add .github/labels.yml as the canonical label set, synced into the repository by .github/workflows/labels.yaml using EndBug/label-sync. Conventions follow the Kubernetes scheme: https://github.com/kubernetes/test-infra/blob/master/label_sync/labels.md Six namespaced groups: kind/, priority/, triage/, lifecycle/, area/, do-not-merge/. Cozystack-specific labels preserved (epic, community, security/*, size:*). Migration via aliases keeps references on existing issues and PRs: - bug -> kind/bug - enhancement -> kind/feature - documentation -> kind/documentation - question -> kind/support - frozen -> lifecycle/frozen - stale -> lifecycle/stale - do-not-merge -> do-not-merge/work-in-progress delete-other-labels is false on the initial rollout; redundant labels ("do not merge", duplicate, invalid, wontfix) stay until a follow-up PR removes them after stabilisation. The labels workflow has a validate job (python3 schema check) that runs on PR. Sync runs only on push to main, weekly cron, and manual dispatch. Schema invariants: - description <= 100 chars (GitHub REST API limit) - color is 6-char hex without leading # - unique top-level names - aliases do not collide with top-level names PR auto-labeling (.github/workflows/pr-labeler.yaml): - Parses PR title as Conventional Commits header (type, scope, !). - type -> kind/* (feat -> kind/feature, fix -> kind/bug, docs -> kind/documentation, chore/refactor -> kind/cleanup; style, perf, test, build, ci, revert -> no kind label). - scope -> area/* via embedded mapping; composite scopes split on comma. Bracket-style fallback ([scope] description) maps area/* but cannot infer kind/*. - '[Backport release-1.x]' prefix is stripped; area/release and backport labels are added. - '!' after type or 'BREAKING CHANGE:' footer in body adds kind/breaking-change. - Unmapped scope or non-conventional title adds area/uncategorized to flag for human review. - Additive only — never removes existing labels. Hardcoded label references updated: - .github/ISSUE_TEMPLATE/bug_report.md (bug -> kind/bug) - .github/workflows/tags.yaml (documentation -> kind/documentation) AGENTS.md gains an Activation entry pointing agents to labels.yml as the source of truth and to contributing.md for the title auto-labeling table. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- .github/labels.yml | 371 +++++++++++++++++++++++++++ .github/workflows/labels.yaml | 84 ++++++ .github/workflows/pr-labeler.yaml | 199 ++++++++++++++ .github/workflows/tags.yaml | 6 +- AGENTS.md | 6 + docs/agents/contributing.md | 43 ++++ 7 files changed, 707 insertions(+), 4 deletions(-) create mode 100644 .github/labels.yml create mode 100644 .github/workflows/labels.yaml create mode 100644 .github/workflows/pr-labeler.yaml diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 1a119259..fefdab82 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,7 +1,7 @@ --- name: Bug report about: Create a report to help us improve -labels: 'bug' +labels: 'kind/bug' assignees: '' --- diff --git a/.github/labels.yml b/.github/labels.yml new file mode 100644 index 00000000..1369aa48 --- /dev/null +++ b/.github/labels.yml @@ -0,0 +1,371 @@ +# Cozystack repository labels +# +# Label conventions follow the Kubernetes scheme: +# https://github.com/kubernetes/test-infra/blob/master/label_sync/labels.md +# +# Synced into the repository by .github/workflows/labels.yaml +# (EndBug/label-sync@v2). Edit this file via pull request — UI changes +# will be overwritten on the next sync. +# +# Constraints (enforced by the validate job in labels.yaml): +# - description ≤ 100 characters (GitHub REST API limit) +# - color is a 6-character hex string (no leading #) +# - label names are unique +# - aliases do not collide with top-level names +# +# Categories: +# kind/ issue or PR type +# priority/ urgency +# triage/ review state +# lifecycle/ issue or PR lifecycle +# area/ subsystem; extensible — add when 3+ open issues exist +# do-not-merge/ PR merge blockers +# security/ security-finding severity and status (Cozystack-specific) +# size: PR size (auto-applied) +# +# `aliases:` lets EndBug/label-sync rename existing labels without losing +# references on already-tagged issues and PRs. +# +# GitHub-default labels not migrated here (`wontfix`, `invalid`) currently +# carry zero issues/PRs in this repo and will be removed in a follow-up +# cleanup PR rather than aliased to a different namespace. + +# ────────────────────────────────────────────── +# kind/ — issue or PR type +# ────────────────────────────────────────────── + +- name: kind/bug + color: 'd73a4a' + description: Categorizes issue or PR as related to a bug + aliases: ['bug'] + +- name: kind/feature + color: 'a2eeef' + description: Categorizes issue or PR as related to a new feature + aliases: ['enhancement'] + +- name: kind/documentation + color: '0075ca' + description: Categorizes issue or PR as related to documentation + aliases: ['documentation'] + +- name: kind/support + color: 'd876e3' + description: Categorizes issue as a support question + aliases: ['question'] + +- name: kind/cleanup + color: 'c7def8' + description: Categorizes issue or PR as related to cleanup of code, process, or technical debt + +- name: kind/regression + color: 'e11d21' + description: Categorizes issue or PR as related to a regression from a prior release + +- name: kind/flake + color: 'f7c6c7' + description: Categorizes issue or PR as related to a flaky test + +- name: kind/failing-test + color: 'e11d21' + description: Categorizes issue or PR as related to a consistently or frequently failing test + +- name: kind/api-change + color: 'c7def8' + description: Categorizes issue or PR as related to adding, removing, or otherwise changing an API + +- name: kind/breaking-change + color: 'e11d21' + description: Indicates the change introduces a breaking API or behaviour change + +# ────────────────────────────────────────────── +# priority/ — urgency +# ────────────────────────────────────────────── + +- name: priority/critical-urgent + color: 'e11d21' + description: Highest priority. Must be actively worked on as someone's top priority right now + +- name: priority/important-soon + color: 'eb6420' + description: Must be staffed and worked on either currently, or very soon, ideally in time for the next release + +- name: priority/important-longterm + color: 'fbca04' + description: Important over the long term, but may not be staffed and/or may need multiple releases to complete + +- name: priority/backlog + color: 'fef2c0' + description: General backlog priority. Lower than priority/important-longterm + +# ────────────────────────────────────────────── +# triage/ — review state +# ────────────────────────────────────────────── + +- name: triage/needs-triage + color: 'ededed' + description: Indicates an issue needs triage by a maintainer + +- name: triage/accepted + color: '0e8a16' + description: Indicates an issue is ready to be actively worked on + +- name: triage/needs-information + color: 'fbca04' + description: Indicates an issue needs more information in order to work on it + +- name: triage/not-reproducible + color: 'fbca04' + description: Indicates an issue can not be reproduced as described + +- name: triage/duplicate + color: 'cfd3d7' + description: Indicates an issue is a duplicate of another issue + aliases: ['duplicate'] + +- name: triage/unresolved + color: 'cfd3d7' + description: Indicates an issue that can not or will not be resolved + +# ────────────────────────────────────────────── +# lifecycle/ — issue or PR lifecycle +# ────────────────────────────────────────────── + +- name: lifecycle/active + color: '1d76db' + description: Indicates that an issue or PR is actively being worked on by a contributor + +- name: lifecycle/frozen + color: 'db5dd6' + description: Indicates that an issue or PR should not be auto-closed due to staleness + aliases: ['frozen'] + +- name: lifecycle/stale + color: 'dadada' + description: Denotes an issue or PR has remained open with no activity and has become stale + aliases: ['stale'] + +- name: lifecycle/rotten + color: '795548' + description: Denotes an issue or PR that has aged beyond stale and will be auto-closed + +# ────────────────────────────────────────────── +# area/ — subsystem (extensible) +# Add a new area/* when there are 3+ open issues on the topic. +# ────────────────────────────────────────────── + +- name: area/api + color: 'bfd4f2' + description: Issues or PRs related to the cozystack-api aggregated API server + +- name: area/ai + color: 'bfd4f2' + description: Issues or PRs related to AI agent guides, AGENTS.md, docs/agents/ + +- name: area/build + color: 'bfd4f2' + description: Issues or PRs related to image build infrastructure, multi-arch support + +- name: area/ci + color: 'bfd4f2' + description: Issues or PRs related to CI workflows, GitHub Actions, automation + +- name: area/dashboard + color: 'bfd4f2' + description: Issues or PRs related to the dashboard / UI + +- name: area/extra + color: 'bfd4f2' + description: Issues or PRs related to tenant-specific modules (packages/extra/) + +- name: area/database + color: 'bfd4f2' + description: Issues or PRs related to managed databases (postgres, mariadb, redis, etcd, kafka, clickhouse) + +- name: area/kubernetes + color: 'bfd4f2' + description: Issues or PRs related to the tenant Kubernetes app + +- name: area/monitoring + color: 'bfd4f2' + description: Issues or PRs related to the monitoring stack (vlogs, vmstack, grafana, workloadmonitor) + +- name: area/networking + color: 'bfd4f2' + description: Issues or PRs related to networking (ingress, gateway, vpn, metallb, cilium, kube-ovn) + +- name: area/platform + color: 'bfd4f2' + description: Issues or PRs related to platform infrastructure (bundle, flux, talos, installer) + +- name: area/release + color: 'bfd4f2' + description: Issues or PRs related to release tooling (changelog, backport, release pipeline) + +- name: area/storage + color: 'bfd4f2' + description: Issues or PRs related to storage (linstor, seaweedfs, bucket, velero, harbor) + +- name: area/testing + color: 'bfd4f2' + description: Issues or PRs related to testing (e2e, bats, unit tests) + +- name: area/virtualization + color: 'bfd4f2' + description: Issues or PRs related to virtualization (kubevirt, cdi, vmi, vm-import) + +- name: area/uncategorized + color: 'fbca04' + description: PR auto-labeler could not map title scope to a known area/*; please review + +# ────────────────────────────────────────────── +# do-not-merge/ — PR merge blockers (Prow convention) +# ────────────────────────────────────────────── + +- name: do-not-merge/work-in-progress + color: 'e11d21' + description: Indicates that a PR should not merge because it is a work in progress + # Both legacy spellings collapse here. EndBug processes aliases sequentially; + # the second rename hits a name collision and logs a warning — the legacy + # label survives and gets cleaned up in the follow-up dedup PR. + aliases: ['do-not-merge', 'do not merge'] + +- name: do-not-merge/hold + color: 'e11d21' + description: Indicates that a PR should not merge because someone has issued /hold + +# ────────────────────────────────────────────── +# Cozystack-specific (preserved) +# ────────────────────────────────────────────── + +- name: epic + color: 'A335EE' + description: A large development increment that brings definite value to Cozystack users + +- name: community + color: '97458A' + description: Community contributions are welcome in this issue + +- name: help wanted + color: '008672' + description: Extra attention is needed + +- name: good first issue + color: '7057ff' + description: Good for newcomers + +- name: quality-of-life + color: 'aaaaaa' + description: QoL improvements + +- name: upstream-issue + color: 'aaaaaa' + description: Requires resolving an issue in an upstream project + +- name: backport + color: 'FBCA04' + description: Should change be backported on previous release + +- name: backport-previous + color: 'fbd876' + description: Backport target — previous release line + +- name: release + color: 'aaaaaa' + description: Releasing a new Cozystack version + +- name: automated + color: 'ededed' + description: Created by automation + +- name: debug + color: '704479' + description: Debugging in progress + +- name: sponsored + color: '00FF00' + description: Sponsored work + +- name: lgtm + color: '238636' + description: This PR has been approved by a maintainer + +- name: ok-to-test + color: '00FF00' + description: Indicates a non-member PR is safe to run CI on + +# ────────────────────────────────────────────── +# size: — PR size (auto-applied by sizing bot) +# ────────────────────────────────────────────── + +- name: 'size:XS' + color: '00ff00' + description: This PR changes 0-9 lines, ignoring generated files + +- name: 'size:S' + color: '77b800' + description: This PR changes 10-29 lines, ignoring generated files + +- name: 'size:M' + color: 'ebb800' + description: This PR changes 30-99 lines, ignoring generated files + +- name: 'size:L' + color: 'eb9500' + description: This PR changes 100-499 lines, ignoring generated files + +- name: 'size:XL' + color: 'ff823f' + description: This PR changes 500-999 lines, ignoring generated files + +- name: 'size:XXL' + color: 'ffb8b8' + description: This PR changes 1000+ lines, ignoring generated files + +# ────────────────────────────────────────────── +# security/ — security-finding severity and status +# ────────────────────────────────────────────── + +- name: security + color: 'aaaaaa' + description: Security-related issues and features + +- name: security/critical + color: 'd73a4a' + description: Critical security vulnerability + +- name: security/high + color: 'e99695' + description: High severity security finding + +- name: security/medium + color: 'f9c513' + description: Medium severity security finding + +- name: security/low + color: '0e8a16' + description: Low severity security finding + +- name: security/triage-needed + color: 'fbca04' + description: Needs security triage + +- name: security/confirmed + color: '1d76db' + description: Confirmed vulnerability + +- name: security/false-positive + color: 'c5def5' + description: Triaged as false positive + +- name: security/accepted-risk + color: 'bfd4f2' + description: Risk accepted with justification + +- name: security/in-progress + color: '0075ca' + description: Fix in progress + +- name: security/fixed + color: '0e8a16' + description: Fix released diff --git a/.github/workflows/labels.yaml b/.github/workflows/labels.yaml new file mode 100644 index 00000000..69c18329 --- /dev/null +++ b/.github/workflows/labels.yaml @@ -0,0 +1,84 @@ +name: Labels + +on: + pull_request: + paths: + - .github/labels.yml + - .github/workflows/labels.yaml + push: + branches: [main] + paths: + - .github/labels.yml + - .github/workflows/labels.yaml + workflow_dispatch: + schedule: + - cron: '17 4 * * 1' # Mondays at 04:17 UTC + +permissions: + contents: read + +concurrency: + group: labels-sync + cancel-in-progress: false + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Validate labels.yml schema + run: | + python3 - <<'PY' + import re, sys, yaml + + path = '.github/labels.yml' + data = yaml.safe_load(open(path)) + errors = [] + + # 1. description ≤ 100 chars (GitHub REST API limit) + for label in data: + desc = label.get('description', '') or '' + if len(desc) > 100: + errors.append(f"{label['name']}: description {len(desc)} chars (max 100)") + + # 2. color is 6-char hex without leading # + for label in data: + color = label.get('color', '') or '' + if not re.match(r'^[0-9A-Fa-f]{6}$', color): + errors.append(f"{label['name']}: bad color {color!r} (must be 6-char hex without #)") + + # 3. unique top-level names + names = [label['name'] for label in data] + dups = sorted({n for n in names if names.count(n) > 1}) + for n in dups: + errors.append(f"duplicate name: {n}") + + # 4. aliases do not collide with any top-level name + name_set = set(names) + for label in data: + for alias in (label.get('aliases') or []): + if alias in name_set: + errors.append(f"alias {alias!r} (under {label['name']!r}) collides with a top-level name") + + if errors: + for err in errors: + print(f"::error::{err}") + sys.exit(1) + + print(f"labels.yml schema OK ({len(data)} labels)") + PY + + sync: + needs: validate + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - uses: actions/checkout@v4 + - uses: EndBug/label-sync@v2 + with: + config-file: .github/labels.yml + delete-other-labels: false diff --git a/.github/workflows/pr-labeler.yaml b/.github/workflows/pr-labeler.yaml new file mode 100644 index 00000000..d882bc50 --- /dev/null +++ b/.github/workflows/pr-labeler.yaml @@ -0,0 +1,199 @@ +name: PR Auto-Label + +on: + pull_request_target: + types: [opened, edited, reopened, synchronize] + +permissions: + contents: read + pull-requests: write + +jobs: + label: + runs-on: ubuntu-latest + steps: + - name: Apply labels from PR title + uses: actions/github-script@v7 + with: + script: | + // Conventional Commits types accepted by Cozystack (per docs/agents/contributing.md): + // feat, fix, docs, style, refactor, perf, test, build, ci, chore, revert + // Mapping below maps a subset to kind/* — types not listed do not produce a kind/*. + const typeToKind = { + feat: 'kind/feature', + fix: 'kind/bug', + docs: 'kind/documentation', + chore: 'kind/cleanup', + refactor: 'kind/cleanup', + // style, perf, test, build, ci, revert — no kind mapping + }; + + // scope -> area/* mapping. Keys are the scopes observed in cozystack issues + // and PRs. Add new entries when a scope recurs (3+ times). + const scopeToArea = { + // area/api + 'api': 'area/api', + 'cozystack-api': 'area/api', + + // area/ai + 'agents': 'area/ai', + 'ai': 'area/ai', + + // area/build + 'build': 'area/build', + + // area/ci + 'ci': 'area/ci', + + // area/dashboard + 'dashboard': 'area/dashboard', + + // area/database + 'postgres': 'area/database', + 'postgres-operator': 'area/database', + 'mariadb': 'area/database', + 'mariadb-operator': 'area/database', + 'redis': 'area/database', + 'etcd': 'area/database', + 'kafka': 'area/database', + 'clickhouse': 'area/database', + + // area/extra + 'extra': 'area/extra', + + // area/kubernetes + 'kubernetes': 'area/kubernetes', + 'apps/kubernetes': 'area/kubernetes', + + // area/monitoring + 'monitoring': 'area/monitoring', + 'vlogs': 'area/monitoring', + 'vmstack': 'area/monitoring', + 'grafana': 'area/monitoring', + 'workloadmonitor': 'area/monitoring', + + // area/networking + 'ingress': 'area/networking', + 'ingress-nginx': 'area/networking', + 'gateway': 'area/networking', + 'vpn': 'area/networking', + 'metallb': 'area/networking', + 'cilium': 'area/networking', + 'kube-ovn': 'area/networking', + 'tcp-balancer': 'area/networking', + 'securitygroups': 'area/networking', + 'cozy-proxy': 'area/networking', + + // area/platform + 'platform': 'area/platform', + 'bundle': 'area/platform', + 'flux': 'area/platform', + 'fluxcd': 'area/platform', + 'cluster-api': 'area/platform', + 'talos': 'area/platform', + 'installer': 'area/platform', + 'cozyctl': 'area/platform', + 'cozystack-engine': 'area/platform', + 'cozy-lib': 'area/platform', + + // area/release + 'backport': 'area/release', + 'release': 'area/release', + + // area/storage + 'seaweedfs': 'area/storage', + 'seaweedfs-cosi-driver': 'area/storage', + 'bucket': 'area/storage', + 'linstor': 'area/storage', + 'velero': 'area/storage', + 'harbor': 'area/storage', + 'backups': 'area/storage', + + // area/testing + 'tests': 'area/testing', + 'e2e': 'area/testing', + + // area/virtualization + 'kubevirt': 'area/virtualization', + 'cdi': 'area/virtualization', + 'vmi': 'area/virtualization', + 'vm-import': 'area/virtualization', + 'virtual-machine': 'area/virtualization', + 'hami': 'area/virtualization', + 'gpu-operator': 'area/virtualization', + }; + + const pr = context.payload.pull_request; + const title = pr.title || ''; + const body = pr.body || ''; + const existing = new Set(pr.labels.map(l => l.name)); + const toAdd = new Set(); + + // 1. Strip "[Backport release-1.x]" prefix if present. + const backportMatch = title.match(/^\[Backport ([^\]]+)\]\s+(.+)$/); + const cleanTitle = backportMatch ? backportMatch[2] : title; + if (backportMatch) { + toAdd.add('area/release'); + toAdd.add('backport'); + } + + // 2. Try Conventional Commits form: type(scope)?(!)?: description + const conv = cleanTitle.match(/^([a-z]+)(?:\(([^)]+)\))?(!)?:\s*.+$/); + // 3. Fall back to bracket form: [scope] description + const bracket = !conv && cleanTitle.match(/^\[([^\]]+)\]\s+.+$/); + + let type = null, scopeStr = null, breaking = false; + if (conv) { + type = conv[1]; + scopeStr = conv[2] || null; + breaking = !!conv[3]; + } else if (bracket) { + scopeStr = bracket[1]; + } + + // 4. Detect BREAKING CHANGE: footer in body. + if (/^BREAKING CHANGE:/m.test(body)) { + breaking = true; + } + + // 5. Apply kind/* from type. + if (type && typeToKind[type]) { + toAdd.add(typeToKind[type]); + } + + // 6. Apply area/* from scope. Composite scopes split on comma. + const scopes = (scopeStr || '') + .split(/,\s*/) + .map(s => s.trim()) + .filter(Boolean); + for (const s of scopes) { + if (scopeToArea[s]) { + toAdd.add(scopeToArea[s]); + } + } + + // 7. kind/breaking-change. + if (breaking) { + toAdd.add('kind/breaking-change'); + } + + // 8. Fallback: no area/* applied -> area/uncategorized. + const hasArea = [...toAdd].some(l => l.startsWith('area/')); + if (!hasArea) { + toAdd.add('area/uncategorized'); + } + + // 9. Additive only — never remove existing labels. + const newLabels = [...toAdd].filter(l => !existing.has(l)); + if (newLabels.length === 0) { + core.info('No new labels to apply'); + return; + } + + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pr.number, + labels: newLabels, + }); + core.info(`Applied labels: ${newLabels.join(', ')}`); diff --git a/.github/workflows/tags.yaml b/.github/workflows/tags.yaml index 90f04829..1b68ee32 100644 --- a/.github/workflows/tags.yaml +++ b/.github/workflows/tags.yaml @@ -223,7 +223,7 @@ jobs: repo: context.repo.repo, head, base, - title: `Release v${version}`, + title: `chore(release): cut v${version}`, body: `This PR prepares the release \`v${version}\`.`, draft: false }); @@ -411,7 +411,7 @@ jobs: repo: context.repo.repo, head: changelogBranch, base: baseBranch, - title: `docs: add changelog for v${version}`, + title: `docs(release): add changelog for v${version}`, body: `This PR adds the changelog for release \`v${version}\`.\n\n✅ Changelog has been automatically generated in \`docs/changelogs/v${version}.md\`.`, draft: false }); @@ -421,7 +421,7 @@ jobs: owner: context.repo.owner, repo: context.repo.repo, issue_number: pr.data.number, - labels: ['documentation', 'automated'] + labels: ['kind/documentation', 'automated'] }); console.log(`Created PR #${pr.data.number} for changelog`); diff --git a/AGENTS.md b/AGENTS.md index fe9aa8e1..5298f6fd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -27,6 +27,12 @@ working with the **Cozystack** project. - Read: [`contributing.md`](./docs/agents/contributing.md) - Action: Read the file to understand git workflow, commit format, PR process +- **Issue and PR labeling, triage** (e.g., "label this issue", "what label should I use", "triage this", "categorize") + - Read: [`.github/labels.yml`](./.github/labels.yml) + - Action: Use labels defined there. Conventions follow the Kubernetes scheme — `kind/*` (type), `area/*` (subsystem), `priority/*` (urgency), `triage/*` (review state), `lifecycle/*` (auto-close), `do-not-merge/*` (PR blockers), `security/*` (severity) + - For `area/*`: accuracy outweighs reuse. If no existing `area/*` truly fits the change, propose a new one via PR (extend `labels.yml` and the scope mapping in `pr-labeler.yaml`) — do not shoehorn the change into a wrong area. `area/uncategorized` is the auto-labeler fallback; treat it as a signal to pick a fit, create a new area, or correct the PR title + - PR titles: a Conventional Commits header (`type(scope): description`, types from [`contributing.md`](./docs/agents/contributing.md)) auto-applies `kind/*` and `area/*` via `.github/workflows/pr-labeler.yaml`. Append `!` (or add a `BREAKING CHANGE:` footer) to apply `kind/breaking-change` + **Important rules:** - ✅ **ONLY read the file if the task matches the documented process scope** - do not read files for tasks that don't match their purpose - ✅ **ALWAYS read the file FIRST** before starting the task (when applicable) diff --git a/docs/agents/contributing.md b/docs/agents/contributing.md index 658dab74..6ff6eef1 100644 --- a/docs/agents/contributing.md +++ b/docs/agents/contributing.md @@ -57,6 +57,49 @@ git commit --signoff -m "fix(postgres): update operator to version 1.2.3" git commit --signoff -m "docs(contributing): add installation guide" ``` +## PR Title Auto-Labeling + +`.github/workflows/pr-labeler.yaml` parses the PR title on `opened`, `edited`, `reopened`, and `synchronize` events and applies labels additively (never removes). The title is expected to follow Conventional Commits — same format as commit messages above. + +**Type → `kind/*`:** + +| type | label | +| --------- | ------------------ | +| feat | kind/feature | +| fix | kind/bug | +| docs | kind/documentation | +| chore | kind/cleanup | +| refactor | kind/cleanup | +| style, perf, test, build, ci, revert | (no kind label)| + +**Scope → `area/*`** (full mapping in `.github/workflows/pr-labeler.yaml`): + +| scope (examples) | label | +| --- | --- | +| agents, ai | area/ai | +| api, cozystack-api | area/api | +| build | area/build | +| ci | area/ci | +| dashboard | area/dashboard | +| postgres, mariadb, redis, etcd, kafka, clickhouse, postgres-operator, mariadb-operator | area/database | +| extra | area/extra | +| kubernetes | area/kubernetes | +| monitoring, vlogs, vmstack, grafana, workloadmonitor | area/monitoring | +| ingress, gateway, vpn, metallb, cilium, kube-ovn, cozy-proxy, … | area/networking | +| platform, bundle, flux, fluxcd, cluster-api, talos, installer, cozyctl, cozystack-engine, cozy-lib | area/platform | +| backport, release | area/release | +| seaweedfs, bucket, linstor, velero, harbor, backups | area/storage | +| tests, e2e | area/testing | +| kubevirt, cdi, vmi, vm-import, virtual-machine, hami, gpu-operator | area/virtualization | + +**Special handling:** + +- `[Backport release-1.x]` prefix is stripped before parsing; `area/release` and `backport` labels are added. +- Composite scope (`feat(platform, system, apps): …`) — each comma-separated part is mapped independently. +- `!` after type or `BREAKING CHANGE:` footer in the body → `kind/breaking-change`. +- Unmapped scope or non-conventional title → `area/uncategorized` (signals the PR needs manual area selection). +- Bracket-style fallback (`[scope] description`) maps `scope` → `area/*` but cannot infer `kind/*`. + ### AI Agent Attribution When an AI agent authors or materially assists with a commit, add an `Assisted-By:` trailer naming the model: From 91188702a678c445bf42dafcbc3b697b1377b086 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Mon, 27 Apr 2026 03:30:09 +0300 Subject: [PATCH 197/250] chore(ci): normalize hex color case in labels.yml Address review feedback from gemini-code-assist on .github/labels.yml:242: all hex color values use lowercase characters for consistency. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- .github/labels.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/labels.yml b/.github/labels.yml index 1369aa48..2ce04130 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -239,11 +239,11 @@ # ────────────────────────────────────────────── - name: epic - color: 'A335EE' + color: 'a335ee' description: A large development increment that brings definite value to Cozystack users - name: community - color: '97458A' + color: '97458a' description: Community contributions are welcome in this issue - name: help wanted @@ -263,7 +263,7 @@ description: Requires resolving an issue in an upstream project - name: backport - color: 'FBCA04' + color: 'fbca04' description: Should change be backported on previous release - name: backport-previous @@ -283,7 +283,7 @@ description: Debugging in progress - name: sponsored - color: '00FF00' + color: '00ff00' description: Sponsored work - name: lgtm @@ -291,7 +291,7 @@ description: This PR has been approved by a maintainer - name: ok-to-test - color: '00FF00' + color: '00ff00' description: Indicates a non-member PR is safe to run CI on # ────────────────────────────────────────────── From 31f4435eb0d02f905b6d0ba2f06c22034fc437d9 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Mon, 27 Apr 2026 03:30:29 +0300 Subject: [PATCH 198/250] docs(agents): replace Unicode ellipsis with ASCII in contributing.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address review feedback from gemini-code-assist on docs/agents/contributing.md:88: … replaced with ... for compatibility across editors and tools. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- docs/agents/contributing.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/agents/contributing.md b/docs/agents/contributing.md index 6ff6eef1..ac6aaf90 100644 --- a/docs/agents/contributing.md +++ b/docs/agents/contributing.md @@ -85,7 +85,7 @@ git commit --signoff -m "docs(contributing): add installation guide" | extra | area/extra | | kubernetes | area/kubernetes | | monitoring, vlogs, vmstack, grafana, workloadmonitor | area/monitoring | -| ingress, gateway, vpn, metallb, cilium, kube-ovn, cozy-proxy, … | area/networking | +| ingress, gateway, vpn, metallb, cilium, kube-ovn, cozy-proxy, ... | area/networking | | platform, bundle, flux, fluxcd, cluster-api, talos, installer, cozyctl, cozystack-engine, cozy-lib | area/platform | | backport, release | area/release | | seaweedfs, bucket, linstor, velero, harbor, backups | area/storage | @@ -95,7 +95,7 @@ git commit --signoff -m "docs(contributing): add installation guide" **Special handling:** - `[Backport release-1.x]` prefix is stripped before parsing; `area/release` and `backport` labels are added. -- Composite scope (`feat(platform, system, apps): …`) — each comma-separated part is mapped independently. +- Composite scope (`feat(platform, system, apps): ...`) — each comma-separated part is mapped independently. - `!` after type or `BREAKING CHANGE:` footer in the body → `kind/breaking-change`. - Unmapped scope or non-conventional title → `area/uncategorized` (signals the PR needs manual area selection). - Bracket-style fallback (`[scope] description`) maps `scope` → `area/*` but cannot infer `kind/*`. From 10b98ceb62308808aefbaa1ebdd4b1f07d713779 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Mon, 27 Apr 2026 03:30:43 +0300 Subject: [PATCH 199/250] docs(agents): use full path .github/labels.yml in AGENTS.md Address review feedback from gemini-code-assist on AGENTS.md:33: expand bare labels.yml and pr-labeler.yaml to .github/labels.yml and .github/workflows/pr-labeler.yaml for consistency with surrounding refs. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 5298f6fd..1e9c4d15 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,7 +30,7 @@ working with the **Cozystack** project. - **Issue and PR labeling, triage** (e.g., "label this issue", "what label should I use", "triage this", "categorize") - Read: [`.github/labels.yml`](./.github/labels.yml) - Action: Use labels defined there. Conventions follow the Kubernetes scheme — `kind/*` (type), `area/*` (subsystem), `priority/*` (urgency), `triage/*` (review state), `lifecycle/*` (auto-close), `do-not-merge/*` (PR blockers), `security/*` (severity) - - For `area/*`: accuracy outweighs reuse. If no existing `area/*` truly fits the change, propose a new one via PR (extend `labels.yml` and the scope mapping in `pr-labeler.yaml`) — do not shoehorn the change into a wrong area. `area/uncategorized` is the auto-labeler fallback; treat it as a signal to pick a fit, create a new area, or correct the PR title + - For `area/*`: accuracy outweighs reuse. If no existing `area/*` truly fits the change, propose a new one via PR (extend `.github/labels.yml` and the scope mapping in `.github/workflows/pr-labeler.yaml`) — do not shoehorn the change into a wrong area. `area/uncategorized` is the auto-labeler fallback; treat it as a signal to pick a fit, create a new area, or correct the PR title - PR titles: a Conventional Commits header (`type(scope): description`, types from [`contributing.md`](./docs/agents/contributing.md)) auto-applies `kind/*` and `area/*` via `.github/workflows/pr-labeler.yaml`. Append `!` (or add a `BREAKING CHANGE:` footer) to apply `kind/breaking-change` **Important rules:** From 738762994e10f878cab27a685a5dbf116ccc43fd Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Mon, 27 Apr 2026 03:30:52 +0300 Subject: [PATCH 200/250] docs(agents): fix markdown table cell spacing in contributing.md Address review feedback from gemini-code-assist on docs/agents/contributing.md:73: add space before closing pipe in the type to kind mapping table for consistency with other rows. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- docs/agents/contributing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/agents/contributing.md b/docs/agents/contributing.md index ac6aaf90..d2ff94e2 100644 --- a/docs/agents/contributing.md +++ b/docs/agents/contributing.md @@ -70,7 +70,7 @@ git commit --signoff -m "docs(contributing): add installation guide" | docs | kind/documentation | | chore | kind/cleanup | | refactor | kind/cleanup | -| style, perf, test, build, ci, revert | (no kind label)| +| style, perf, test, build, ci, revert | (no kind label) | **Scope → `area/*`** (full mapping in `.github/workflows/pr-labeler.yaml`): From 104b3b3d2b1c25cc06a86669238cad3b744c86d3 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Mon, 20 Apr 2026 18:43:02 +0300 Subject: [PATCH 201/250] feat(operator): add per-package upgradeCRDs policy for HelmRelease Add an opt-in UpgradeCRDs field to ComponentInstall that maps to HelmRelease.Spec.Upgrade.CRDs, allowing a PackageSource component to declare how Flux should handle CRDs from the chart's crds/ directory on upgrade. The helm-controller default on upgrade is Skip, which means new CRDs added between chart versions never reach existing clusters and must be applied manually. Setting upgradeCRDs: CreateReplace makes Flux apply new CRDs declaratively with the chart. Allowed values are restricted to Skip, Create, CreateReplace via a kubebuilder enum marker. Empty / unset preserves the existing Flux default, so all existing PackageSource resources keep working. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- api/v1alpha1/packagesource_types.go | 10 ++ .../cozystack.io_packagesources.yaml | 13 ++ internal/operator/package_reconciler.go | 11 ++ internal/operator/package_reconciler_test.go | 138 ++++++++++++++++++ 4 files changed, 172 insertions(+) create mode 100644 internal/operator/package_reconciler_test.go diff --git a/api/v1alpha1/packagesource_types.go b/api/v1alpha1/packagesource_types.go index c0b943e8..75bf217b 100644 --- a/api/v1alpha1/packagesource_types.go +++ b/api/v1alpha1/packagesource_types.go @@ -132,6 +132,16 @@ type ComponentInstall struct { // DependsOn is a list of component names that must be installed before this component // +optional DependsOn []string `json:"dependsOn,omitempty"` + + // UpgradeCRDs controls how CRDs from the chart's crds/ directory are + // handled on HelmRelease upgrades. Maps to HelmRelease.Spec.Upgrade.CRDs. + // Empty string (default) preserves the helm-controller default (Skip). + // Use "CreateReplace" for operators that evolve their CRD set between + // versions. Warning: CreateReplace overwrites CRDs and may cause data + // loss if upstream drops fields from a CRD with live objects. + // +optional + // +kubebuilder:validation:Enum=Skip;Create;CreateReplace + UpgradeCRDs string `json:"upgradeCRDs,omitempty"` } // Component defines a single Helm release component within a package source diff --git a/internal/crdinstall/manifests/cozystack.io_packagesources.yaml b/internal/crdinstall/manifests/cozystack.io_packagesources.yaml index 0acfcdd2..1d0037df 100644 --- a/internal/crdinstall/manifests/cozystack.io_packagesources.yaml +++ b/internal/crdinstall/manifests/cozystack.io_packagesources.yaml @@ -118,6 +118,19 @@ spec: ReleaseName is the name of the HelmRelease resource that will be created If not specified, defaults to the component Name field type: string + upgradeCRDs: + description: |- + UpgradeCRDs controls how CRDs from the chart's crds/ directory are + handled on HelmRelease upgrades. Maps to HelmRelease.Spec.Upgrade.CRDs. + Empty string (default) preserves the helm-controller default (Skip). + Use "CreateReplace" for operators that evolve their CRD set between + versions. Warning: CreateReplace overwrites CRDs and may cause data + loss if upstream drops fields from a CRD with live objects. + enum: + - Skip + - Create + - CreateReplace + type: string type: object libraries: description: |- diff --git a/internal/operator/package_reconciler.go b/internal/operator/package_reconciler.go index 32e667e4..0e724e49 100644 --- a/internal/operator/package_reconciler.go +++ b/internal/operator/package_reconciler.go @@ -45,6 +45,16 @@ const ( SecretCozystackValues = "cozystack-values" ) +// parseCRDPolicy maps ComponentInstall.UpgradeCRDs to a helmv2.CRDsPolicy. +// Empty / nil preserves the helm-controller default (Skip on upgrade); +// the CRD enum marker restricts the string to Skip/Create/CreateReplace. +func parseCRDPolicy(install *cozyv1alpha1.ComponentInstall) helmv2.CRDsPolicy { + if install == nil || install.UpgradeCRDs == "" { + return "" + } + return helmv2.CRDsPolicy(install.UpgradeCRDs) +} + // PackageReconciler reconciles Package resources type PackageReconciler struct { client.Client @@ -221,6 +231,7 @@ func (r *PackageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct Remediation: &helmv2.UpgradeRemediation{ Retries: -1, }, + CRDs: parseCRDPolicy(component.Install), }, }, } diff --git a/internal/operator/package_reconciler_test.go b/internal/operator/package_reconciler_test.go new file mode 100644 index 00000000..f0ee2c19 --- /dev/null +++ b/internal/operator/package_reconciler_test.go @@ -0,0 +1,138 @@ +/* +Copyright 2025 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package operator + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" + helmv2 "github.com/fluxcd/helm-controller/api/v2" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "sigs.k8s.io/yaml" +) + +func TestParseCRDPolicy(t *testing.T) { + tests := []struct { + name string + install *cozyv1alpha1.ComponentInstall + want helmv2.CRDsPolicy + }{ + { + name: "nil install leaves flux default", + install: nil, + want: "", + }, + { + name: "empty upgradeCRDs leaves flux default", + install: &cozyv1alpha1.ComponentInstall{}, + want: "", + }, + { + name: "Skip is passed through", + install: &cozyv1alpha1.ComponentInstall{UpgradeCRDs: "Skip"}, + want: helmv2.Skip, + }, + { + name: "Create is passed through", + install: &cozyv1alpha1.ComponentInstall{UpgradeCRDs: "Create"}, + want: helmv2.Create, + }, + { + name: "CreateReplace is passed through", + install: &cozyv1alpha1.ComponentInstall{UpgradeCRDs: "CreateReplace"}, + want: helmv2.CreateReplace, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := parseCRDPolicy(tc.install) + if got != tc.want { + t.Errorf("parseCRDPolicy() = %q, want %q", got, tc.want) + } + }) + } +} + +// TestPackageSourceCRDHasUpgradeCRDsEnum guards the generated CRD schema: the +// invalid-value case from the spec is enforced at the API server via a +// kubebuilder enum marker, not in the reconciler. If someone drops the marker +// and forgets to regenerate, this test catches it. +func TestPackageSourceCRDHasUpgradeCRDsEnum(t *testing.T) { + path := filepath.Join("..", "crdinstall", "manifests", "cozystack.io_packagesources.yaml") + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + + var crd apiextensionsv1.CustomResourceDefinition + if err := yaml.Unmarshal(data, &crd); err != nil { + t.Fatalf("unmarshal CRD: %v", err) + } + + var field *apiextensionsv1.JSONSchemaProps + for i := range crd.Spec.Versions { + v := &crd.Spec.Versions[i] + if v.Schema == nil || v.Schema.OpenAPIV3Schema == nil { + continue + } + spec, ok := v.Schema.OpenAPIV3Schema.Properties["spec"] + if !ok { + continue + } + variants, ok := spec.Properties["variants"] + if !ok || variants.Items == nil || variants.Items.Schema == nil { + continue + } + components, ok := variants.Items.Schema.Properties["components"] + if !ok || components.Items == nil || components.Items.Schema == nil { + continue + } + install, ok := components.Items.Schema.Properties["install"] + if !ok { + continue + } + f, ok := install.Properties["upgradeCRDs"] + if !ok { + continue + } + field = &f + break + } + + if field == nil { + t.Fatal("upgradeCRDs field not found in PackageSource CRD schema") + } + + got := map[string]bool{} + for _, e := range field.Enum { + var s string + if err := json.Unmarshal(e.Raw, &s); err != nil { + t.Fatalf("unmarshal enum value %q: %v", e.Raw, err) + } + got[s] = true + } + + for _, want := range []string{"Skip", "Create", "CreateReplace"} { + if !got[want] { + t.Errorf("enum value %q missing from upgradeCRDs; got %v", want, got) + } + } +} From d86bc7760a77c93c6fd837ef5e23c3df3b9b5898 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Mon, 20 Apr 2026 18:43:18 +0300 Subject: [PATCH 202/250] docs(agents): document PackageSource upgradeCRDs field Describe when to set upgradeCRDs: CreateReplace (operators that evolve their CRD set additively between versions) and the data-loss risk of enabling it on operators that drop fields. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- docs/agents/overview.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/agents/overview.md b/docs/agents/overview.md index 35798961..ae0fd1c4 100644 --- a/docs/agents/overview.md +++ b/docs/agents/overview.md @@ -83,6 +83,14 @@ packages/// - Reference PR numbers when available - Keep commits atomic and focused +### PackageSource CRD upgrade policy + +Each component in a `PackageSource` may set `install.upgradeCRDs` to control how CRDs from the chart's `crds/` directory are handled on `HelmRelease` upgrades. Allowed values: `Skip` (default — helm-controller does not touch CRDs on upgrade), `Create` (create new CRDs only), `CreateReplace` (create new and overwrite existing). + +Set `upgradeCRDs: CreateReplace` for operators whose upstream regularly adds new CRDs between versions (etcd-operator, cnpg, kubevirt, kamaji). Without it, new CRDs from a chart bump do not land on existing clusters — only fresh installs get them. + +Do **not** set `CreateReplace` blindly: it overwrites every CRD in `crds/` and can cause silent data loss if upstream drops a field from a CRD that has live objects. Only enable it for operators whose schema evolution is additive-only. When in doubt, leave it unset and apply new CRDs manually. + ### Documentation Documentation is organized as follows: From f527ce683b28401c62cdc420188392b32367401f Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Mon, 20 Apr 2026 19:08:11 +0300 Subject: [PATCH 203/250] docs(agents): clarify that Scopes list is illustrative The Scopes section was read as an exhaustive enumeration, which led to review feedback flagging any scope outside the list as invalid. The intent has always been that contributors pick the most specific scope for the change and extend the list when a genuinely new area appears. Reword the section accordingly and add operator as an example scope. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- docs/agents/contributing.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/agents/contributing.md b/docs/agents/contributing.md index 658dab74..9f3ff779 100644 --- a/docs/agents/contributing.md +++ b/docs/agents/contributing.md @@ -43,10 +43,11 @@ git commit --signoff -m "type(scope): brief description" **Types:** `feat`, `fix`, `docs`, `style`, `refactor`, `perf`, `test`, `build`, `ci`, `chore` -**Scopes** (e.g., not exhaustive — use any scope that names the component you are touching): -- System: `dashboard`, `platform`, `cilium`, `kube-ovn`, `linstor`, `fluxcd`, `cluster-api` -- Apps: `postgres`, `mariadb`, `redis`, `kafka`, `clickhouse`, `virtual-machine`, `kubernetes` -- Other: `api`, `hack`, `tests`, `ci`, `docs`, `agents`, `maintenance` +**Scopes** (examples — not an exhaustive list; pick the most specific scope that describes the change, and introduce a new one if a genuinely new area needs its own): + +- System, e.g.: `dashboard`, `platform`, `operator`, `cilium`, `kube-ovn`, `linstor`, `fluxcd`, `cluster-api` +- Apps, e.g.: `postgres`, `mariadb`, `redis`, `kafka`, `clickhouse`, `virtual-machine`, `kubernetes` +- Other, e.g.: `api`, `hack`, `tests`, `ci`, `docs`, `agents`, `maintenance` Breaking changes: append `!` after type/scope (`feat(api)!: ...`) or add a `BREAKING CHANGE:` footer. From 32ae993d3ebb51a34ece2b6bbe8e6626c72ac300 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Mon, 20 Apr 2026 19:13:06 +0300 Subject: [PATCH 204/250] docs(maintenance): mirror illustrative-scopes wording in PR template Match the wording adopted in docs/agents/contributing.md so that human contributors and AI agents see the same guidance in both places. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- .github/PULL_REQUEST_TEMPLATE.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index b9475846..044dd6a0 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,9 +1,10 @@ + +## Bug Fixes + +* **[vm-instance] Fix `externalMethod: PortList` not filtering ingress ports**: The `vm-instance` chart now sets `networking.cozystack.io/wholeIP: "false"` on the rendered Service when `externalMethod: PortList` is configured, signaling cozy-proxy to install per-port ingress filtering based on `Service.spec.ports`. Previously the annotation was always `"true"`, which kept cozy-proxy in whole-IP passthrough mode and made `PortList` non-functional. **Requires cozy-proxy v0.3.0 or later** (bundled with this chart). ([@mattia-eleuteri](https://github.com/mattia-eleuteri) in #). + +--- + +**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.3.0...v1.3.1 diff --git a/packages/apps/vm-instance/templates/service.yaml b/packages/apps/vm-instance/templates/service.yaml index b7e3a420..4607ffcf 100644 --- a/packages/apps/vm-instance/templates/service.yaml +++ b/packages/apps/vm-instance/templates/service.yaml @@ -9,7 +9,7 @@ metadata: {{- if .Values.external }} service.kubernetes.io/service-proxy-name: "cozy-proxy" annotations: - networking.cozystack.io/wholeIP: "true" + networking.cozystack.io/wholeIP: {{ ternary "true" "false" (eq .Values.externalMethod "WholeIP") | quote }} {{- end }} spec: type: {{ ternary "LoadBalancer" "ClusterIP" .Values.external }} From b0afc9a07c8a842915af1fa98ff1e7d17ccb39c8 Mon Sep 17 00:00:00 2001 From: mattia-eleuteri Date: Tue, 28 Apr 2026 08:37:13 +0200 Subject: [PATCH 221/250] [vm-instance] Add externalAllowICMP knob, drop in-PR changelog - Add `externalAllowICMP` value (default true) propagated as `networking.cozystack.io/allowICMP` annotation on the rendered Service when `externalMethod: PortList`. The cozy-proxy companion (released as part of cozystack/cozy-proxy#11 + #12) drops ICMP by default in port-filter mode, which breaks ping and PMTU discovery; defaulting the chart to "true" preserves user expectations while still allowing operators to opt out by setting `externalAllowICMP: false`. - Remove the v1.3.1.md changelog entry. Project convention is to add changelogs in a dedicated "docs: add changelog for vX.Y.Z" commit at release time, not as part of feature/fix PRs. Signed-off-by: mattia-eleuteri --- api/apps/v1alpha1/vminstance/types.go | 3 ++ docs/changelogs/v1.3.1.md | 11 ---- packages/apps/vm-instance/README.md | 51 ++++++++++--------- .../apps/vm-instance/templates/service.yaml | 3 ++ packages/apps/vm-instance/values.schema.json | 5 ++ packages/apps/vm-instance/values.yaml | 3 ++ .../vm-instance-rd/cozyrds/vm-instance.yaml | 4 +- 7 files changed, 42 insertions(+), 38 deletions(-) delete mode 100644 docs/changelogs/v1.3.1.md diff --git a/api/apps/v1alpha1/vminstance/types.go b/api/apps/v1alpha1/vminstance/types.go index 2bb059c6..9c48724a 100644 --- a/api/apps/v1alpha1/vminstance/types.go +++ b/api/apps/v1alpha1/vminstance/types.go @@ -26,6 +26,9 @@ type ConfigSpec struct { // Ports to forward from outside the cluster. // +kubebuilder:default:={22} ExternalPorts []int `json:"externalPorts,omitempty"` + // Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect. + // +kubebuilder:default:=true + ExternalAllowICMP bool `json:"externalAllowICMP"` // Requested running state of the VirtualMachineInstance // +kubebuilder:default:="Always" RunStrategy RunStrategy `json:"runStrategy"` diff --git a/docs/changelogs/v1.3.1.md b/docs/changelogs/v1.3.1.md deleted file mode 100644 index 7f9ecfab..00000000 --- a/docs/changelogs/v1.3.1.md +++ /dev/null @@ -1,11 +0,0 @@ - - -## Bug Fixes - -* **[vm-instance] Fix `externalMethod: PortList` not filtering ingress ports**: The `vm-instance` chart now sets `networking.cozystack.io/wholeIP: "false"` on the rendered Service when `externalMethod: PortList` is configured, signaling cozy-proxy to install per-port ingress filtering based on `Service.spec.ports`. Previously the annotation was always `"true"`, which kept cozy-proxy in whole-IP passthrough mode and made `PortList` non-functional. **Requires cozy-proxy v0.3.0 or later** (bundled with this chart). ([@mattia-eleuteri](https://github.com/mattia-eleuteri) in #). - ---- - -**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.3.0...v1.3.1 diff --git a/packages/apps/vm-instance/README.md b/packages/apps/vm-instance/README.md index a2b6603e..9d6a52b2 100644 --- a/packages/apps/vm-instance/README.md +++ b/packages/apps/vm-instance/README.md @@ -36,31 +36,32 @@ virtctl ssh @ ### Common parameters -| Name | Description | Type | Value | -| ------------------- | --------------------------------------------------------------------------------------------------------------------------------- | ---------- | ----------- | -| `external` | Enable external access from outside the cluster. | `bool` | `false` | -| `externalMethod` | Method to pass through traffic to the VM. | `string` | `PortList` | -| `externalPorts` | Ports to forward from outside the cluster. | `[]int` | `[22]` | -| `runStrategy` | Requested running state of the VirtualMachineInstance | `string` | `Always` | -| `instanceType` | Virtual Machine instance type. | `string` | `u1.medium` | -| `instanceProfile` | Virtual Machine preferences profile. | `string` | `ubuntu` | -| `disks` | List of disks to attach. | `[]object` | `[]` | -| `disks[i].name` | Disk name. | `string` | `""` | -| `disks[i].bus` | Disk bus type (e.g. "sata"). | `string` | `""` | -| `networks` | Networks to attach the VM to. | `[]object` | `[]` | -| `networks[i].name` | Network attachment name. | `string` | `""` | -| `subnets` | Deprecated: use networks instead. | `[]object` | `[]` | -| `subnets[i].name` | Network attachment name. | `string` | `""` | -| `gpus` | List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). | `[]object` | `[]` | -| `gpus[i].name` | The name of the GPU resource to attach. | `string` | `""` | -| `cpuModel` | Model specifies the CPU model inside the VMI. List of available models https://github.com/libvirt/libvirt/tree/master/src/cpu_map | `string` | `""` | -| `resources` | Resource configuration for the virtual machine. | `object` | `{}` | -| `resources.cpu` | Number of CPU cores allocated. | `quantity` | `""` | -| `resources.memory` | Amount of memory allocated. | `quantity` | `""` | -| `resources.sockets` | Number of CPU sockets (vCPU topology). | `quantity` | `""` | -| `sshKeys` | List of SSH public keys for authentication. | `[]string` | `[]` | -| `cloudInit` | Cloud-init user data. | `string` | `""` | -| `cloudInitSeed` | Seed string to generate SMBIOS UUID for the VM. | `string` | `""` | +| Name | Description | Type | Value | +| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ----------- | +| `external` | Enable external access from outside the cluster. | `bool` | `false` | +| `externalMethod` | Method to pass through traffic to the VM. | `string` | `PortList` | +| `externalPorts` | Ports to forward from outside the cluster. | `[]int` | `[22]` | +| `externalAllowICMP` | Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect. | `bool` | `true` | +| `runStrategy` | Requested running state of the VirtualMachineInstance | `string` | `Always` | +| `instanceType` | Virtual Machine instance type. | `string` | `u1.medium` | +| `instanceProfile` | Virtual Machine preferences profile. | `string` | `ubuntu` | +| `disks` | List of disks to attach. | `[]object` | `[]` | +| `disks[i].name` | Disk name. | `string` | `""` | +| `disks[i].bus` | Disk bus type (e.g. "sata"). | `string` | `""` | +| `networks` | Networks to attach the VM to. | `[]object` | `[]` | +| `networks[i].name` | Network attachment name. | `string` | `""` | +| `subnets` | Deprecated: use networks instead. | `[]object` | `[]` | +| `subnets[i].name` | Network attachment name. | `string` | `""` | +| `gpus` | List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). | `[]object` | `[]` | +| `gpus[i].name` | The name of the GPU resource to attach. | `string` | `""` | +| `cpuModel` | Model specifies the CPU model inside the VMI. List of available models https://github.com/libvirt/libvirt/tree/master/src/cpu_map | `string` | `""` | +| `resources` | Resource configuration for the virtual machine. | `object` | `{}` | +| `resources.cpu` | Number of CPU cores allocated. | `quantity` | `""` | +| `resources.memory` | Amount of memory allocated. | `quantity` | `""` | +| `resources.sockets` | Number of CPU sockets (vCPU topology). | `quantity` | `""` | +| `sshKeys` | List of SSH public keys for authentication. | `[]string` | `[]` | +| `cloudInit` | Cloud-init user data. | `string` | `""` | +| `cloudInitSeed` | Seed string to generate SMBIOS UUID for the VM. | `string` | `""` | ## U Series diff --git a/packages/apps/vm-instance/templates/service.yaml b/packages/apps/vm-instance/templates/service.yaml index 4607ffcf..cfeffa81 100644 --- a/packages/apps/vm-instance/templates/service.yaml +++ b/packages/apps/vm-instance/templates/service.yaml @@ -10,6 +10,9 @@ metadata: service.kubernetes.io/service-proxy-name: "cozy-proxy" annotations: networking.cozystack.io/wholeIP: {{ ternary "true" "false" (eq .Values.externalMethod "WholeIP") | quote }} + {{- if eq .Values.externalMethod "PortList" }} + networking.cozystack.io/allowICMP: {{ ternary "true" "false" (ne .Values.externalAllowICMP false) | quote }} + {{- end }} {{- end }} spec: type: {{ ternary "LoadBalancer" "ClusterIP" .Values.external }} diff --git a/packages/apps/vm-instance/values.schema.json b/packages/apps/vm-instance/values.schema.json index 34f7f634..01bd30a9 100644 --- a/packages/apps/vm-instance/values.schema.json +++ b/packages/apps/vm-instance/values.schema.json @@ -26,6 +26,11 @@ "type": "integer" } }, + "externalAllowICMP": { + "description": "Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect.", + "type": "boolean", + "default": true + }, "runStrategy": { "description": "Requested running state of the VirtualMachineInstance", "type": "string", diff --git a/packages/apps/vm-instance/values.yaml b/packages/apps/vm-instance/values.yaml index f07b75d3..92e399c2 100644 --- a/packages/apps/vm-instance/values.yaml +++ b/packages/apps/vm-instance/values.yaml @@ -31,6 +31,9 @@ externalMethod: PortList externalPorts: - 22 +## @param {bool} externalAllowICMP - Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect. +externalAllowICMP: true + ## @enum {string} RunStrategy - Requested running state of the VirtualMachineInstance ## @value Always - VMI should always be running ## @value Halted - VMI should never be running diff --git a/packages/system/vm-instance-rd/cozyrds/vm-instance.yaml b/packages/system/vm-instance-rd/cozyrds/vm-instance.yaml index e93fa86b..e56bd9dd 100644 --- a/packages/system/vm-instance-rd/cozyrds/vm-instance.yaml +++ b/packages/system/vm-instance-rd/cozyrds/vm-instance.yaml @@ -8,7 +8,7 @@ spec: singular: vminstance plural: vminstances openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"externalMethod":{"description":"Method to pass through traffic to the VM.","type":"string","default":"PortList","enum":["PortList","WholeIP"]},"externalPorts":{"description":"Ports to forward from outside the cluster.","type":"array","default":[22],"items":{"type":"integer"}},"runStrategy":{"description":"Requested running state of the VirtualMachineInstance","type":"string","default":"Always","enum":["Always","Halted","Manual","RerunOnFailure","Once"]},"instanceType":{"description":"Virtual Machine instance type.","type":"string","default":"u1.medium"},"instanceProfile":{"description":"Virtual Machine preferences profile.","type":"string","default":"ubuntu","enum":["alpine","centos.7","centos.7.desktop","centos.stream10","centos.stream10.desktop","centos.stream8","centos.stream8.desktop","centos.stream8.dpdk","centos.stream9","centos.stream9.desktop","centos.stream9.dpdk","cirros","fedora","fedora.arm64","opensuse.leap","opensuse.tumbleweed","rhel.10","rhel.10.arm64","rhel.7","rhel.7.desktop","rhel.8","rhel.8.desktop","rhel.8.dpdk","rhel.9","rhel.9.arm64","rhel.9.desktop","rhel.9.dpdk","rhel.9.realtime","sles","ubuntu","windows.10","windows.10.virtio","windows.11","windows.11.virtio","windows.2k16","windows.2k16.virtio","windows.2k19","windows.2k19.virtio","windows.2k22","windows.2k22.virtio","windows.2k25","windows.2k25.virtio",""]},"disks":{"description":"List of disks to attach.","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"bus":{"description":"Disk bus type (e.g. \"sata\").","type":"string"},"name":{"description":"Disk name.","type":"string"}}}},"networks":{"description":"Networks to attach the VM to.","type":"array","default":[],"items":{"type":"object","properties":{"name":{"description":"Network attachment name.","type":"string"}}}},"subnets":{"description":"Deprecated: use networks instead.","type":"array","default":[],"items":{"type":"object","properties":{"name":{"description":"Network attachment name.","type":"string"}}}},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"name":{"description":"The name of the GPU resource to attach.","type":"string"}}}},"cpuModel":{"description":"Model specifies the CPU model inside the VMI. List of available models https://github.com/libvirt/libvirt/tree/master/src/cpu_map","type":"string","default":""},"resources":{"description":"Resource configuration for the virtual machine.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"sockets":{"description":"Number of CPU sockets (vCPU topology).","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"sshKeys":{"description":"List of SSH public keys for authentication.","type":"array","default":[],"items":{"type":"string"}},"cloudInit":{"description":"Cloud-init user data.","type":"string","default":""},"cloudInitSeed":{"description":"Seed string to generate SMBIOS UUID for the VM.","type":"string","default":""}}} + {"title":"Chart Values","type":"object","properties":{"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"externalMethod":{"description":"Method to pass through traffic to the VM.","type":"string","default":"PortList","enum":["PortList","WholeIP"]},"externalPorts":{"description":"Ports to forward from outside the cluster.","type":"array","default":[22],"items":{"type":"integer"}},"externalAllowICMP":{"description":"Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect.","type":"boolean","default":true},"runStrategy":{"description":"Requested running state of the VirtualMachineInstance","type":"string","default":"Always","enum":["Always","Halted","Manual","RerunOnFailure","Once"]},"instanceType":{"description":"Virtual Machine instance type.","type":"string","default":"u1.medium"},"instanceProfile":{"description":"Virtual Machine preferences profile.","type":"string","default":"ubuntu","enum":["alpine","centos.7","centos.7.desktop","centos.stream10","centos.stream10.desktop","centos.stream8","centos.stream8.desktop","centos.stream8.dpdk","centos.stream9","centos.stream9.desktop","centos.stream9.dpdk","cirros","fedora","fedora.arm64","opensuse.leap","opensuse.tumbleweed","rhel.10","rhel.10.arm64","rhel.7","rhel.7.desktop","rhel.8","rhel.8.desktop","rhel.8.dpdk","rhel.9","rhel.9.arm64","rhel.9.desktop","rhel.9.dpdk","rhel.9.realtime","sles","ubuntu","windows.10","windows.10.virtio","windows.11","windows.11.virtio","windows.2k16","windows.2k16.virtio","windows.2k19","windows.2k19.virtio","windows.2k22","windows.2k22.virtio","windows.2k25","windows.2k25.virtio",""]},"disks":{"description":"List of disks to attach.","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"bus":{"description":"Disk bus type (e.g. \"sata\").","type":"string"},"name":{"description":"Disk name.","type":"string"}}}},"networks":{"description":"Networks to attach the VM to.","type":"array","default":[],"items":{"type":"object","properties":{"name":{"description":"Network attachment name.","type":"string"}}}},"subnets":{"description":"Deprecated: use networks instead.","type":"array","default":[],"items":{"type":"object","properties":{"name":{"description":"Network attachment name.","type":"string"}}}},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"name":{"description":"The name of the GPU resource to attach.","type":"string"}}}},"cpuModel":{"description":"Model specifies the CPU model inside the VMI. List of available models https://github.com/libvirt/libvirt/tree/master/src/cpu_map","type":"string","default":""},"resources":{"description":"Resource configuration for the virtual machine.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"sockets":{"description":"Number of CPU sockets (vCPU topology).","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"sshKeys":{"description":"List of SSH public keys for authentication.","type":"array","default":[],"items":{"type":"string"}},"cloudInit":{"description":"Cloud-init user data.","type":"string","default":""},"cloudInitSeed":{"description":"Seed string to generate SMBIOS UUID for the VM.","type":"string","default":""}}} release: prefix: vm-instance- labels: @@ -26,7 +26,7 @@ spec: tags: - compute icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODdfMzQ1NCkiLz4KPGcgY2xpcC1wYXRoPSJ1cmwoI2NsaXAwXzY4N18zNDU0KSI+CjxwYXRoIGQ9Ik04OS41MDM5IDExMS43MDdINTQuNDk3QzU0LjE3MjcgMTExLjcwNyA1NC4wMTA4IDExMS4yMjEgNTQuMzM0OSAxMTEuMDU5TDU3LjI1MjIgMTA4Ljk1MkM2MC4zMzE0IDEwNi42ODMgNjEuOTUyMiAxMDIuNjMxIDYwLjk3OTcgOTguNzQxMkg4My4wMjFDODIuMDQ4NSAxMDIuNjMxIDgzLjY2OTMgMTA2LjY4MyA4Ni43NDg1IDEwOC45NTJMODkuNjY1OCAxMTEuMDU5Qzg5Ljk5IDExMS4yMjEgODkuODI3OSAxMTEuNzA3IDg5LjUwMzkgMTExLjcwN1oiIGZpbGw9IiNCMEI2QkIiLz4KPHBhdGggZD0iTTExMy4zMjggOTguNzQxSDMwLjY3MjVDMjcuNTkzMSA5OC43NDEgMjUgOTYuMTQ4IDI1IDkzLjA2ODdWMzMuMTAzMkMyNSAzMC4wMjM5IDI3LjU5MzEgMjcuNDMwNyAzMC42NzI1IDI3LjQzMDdIMTEzLjMyOEMxMTYuNDA3IDI3LjQzMDcgMTE5IDMwLjAyMzcgMTE5IDMzLjEwMzJWOTMuMDY4N0MxMTkgOTYuMTQ4IDExNi40MDcgOTguNzQxIDExMy4zMjggOTguNzQxWiIgZmlsbD0iI0U4RURFRSIvPgo8cGF0aCBkPSJNMTE5IDg0LjE1NDlIMjVWMzMuMTAzMkMyNSAzMC4wMjM5IDI3LjU5MzEgMjcuNDMwNyAzMC42NzI1IDI3LjQzMDdIMTEzLjMyOEMxMTYuNDA3IDI3LjQzMDcgMTE5IDMwLjAyMzcgMTE5IDMzLjEwMzJMMTE5IDg0LjE1NDlaIiBmaWxsPSIjMDBCM0ZGIi8+CjxwYXRoIGQ9Ik05MC42Mzc0IDExNi41NjlINTMuMzYxNkM1Mi4wNjUxIDExNi41NjkgNTAuOTMwNyAxMTUuNDM1IDUwLjkzMDcgMTE0LjEzOEM1MC45MzA3IDExMi44NDEgNTIuMDY1MSAxMTEuNzA3IDUzLjM2MTYgMTExLjcwN0g5MC42Mzc0QzkxLjkzMzkgMTExLjcwNyA5My4wNjg0IDExMi44NDEgOTMuMDY4NCAxMTQuMTM4QzkzLjA2ODQgMTE1LjQzNSA5MS45MzM5IDExNi41NjkgOTAuNjM3NCAxMTYuNTY5WiIgZmlsbD0iI0U4RURFRSIvPgo8L2c+CjxwYXRoIGQ9Ik03Mi41Mjc1IDUzLjgzNjdDNzIuNDQzMSA1My44MzUxIDcyLjM2MDUgNTMuODEyMiA3Mi4yODczIDUzLjc3MDFMNTYuNDY5OSA0NC43OTM0QzU2LjM5ODMgNDQuNzUxOSA1Ni4zMzg4IDQ0LjY5MjMgNTYuMjk3MyA0NC42MjA3QzU2LjI1NTkgNDQuNTQ5IDU2LjIzMzggNDQuNDY3OCA1Ni4yMzM0IDQ0LjM4NUM1Ni4yMzM0IDQ0LjIxNjkgNTYuMzI1OCA0NC4wNjE3IDU2LjQ2OTkgNDMuOTc4NUw3Mi4xOTEyIDM1LjA2MDlDNzIuMjYzNyAzNS4wMjEgNzIuMzQ1IDM1IDcyLjQyNzcgMzVDNzIuNTEwNSAzNSA3Mi41OTE4IDM1LjAyMSA3Mi42NjQzIDM1LjA2MDlMODguNDg3MiA0NC4wMzk1Qzg4LjU1OTEgNDQuMDgwMSA4OC42MTg4IDQ0LjEzOTIgODguNjYgNDQuMjEwN0M4OC43MDEzIDQ0LjI4MjIgODguNzIyNyA0NC4zNjM1IDg4LjcyMTkgNDQuNDQ2Qzg4LjcyMjUgNDQuNTI4NSA4OC43MDEgNDQuNjA5NyA4OC42NTk4IDQ0LjY4MTJDODguNjE4NSA0NC43NTI2IDg4LjU1ODkgNDQuODExOCA4OC40ODcyIDQ0Ljg1MjVMNzIuNzcxNCA1My43NjgzQzcyLjY5NzIgNTMuODExNCA3Mi42MTMzIDUzLjgzNDkgNzIuNTI3NSA1My44MzY3IiBmaWxsPSJ3aGl0ZSIvPgo8cGF0aCBvcGFjaXR5PSIwLjciIGQ9Ik03MC4yNTUzIDc1LjY1MTdDNzAuMTcxIDc1LjY1MzUgNzAuMDg3OCA3NS42MzE3IDcwLjAxNTEgNzUuNTg4OEw1NC4yNDU4IDY2LjY0MTdDNTQuMTcxNSA2Ni42MDI0IDU0LjEwOTUgNjYuNTQzNiA1NC4wNjYxIDY2LjQ3MTZDNTQuMDIyOCA2Ni4zOTk3IDU0IDY2LjMxNzMgNTQgNjYuMjMzM1Y0OC4yNzhDNTQgNDguMTA4IDU0LjA5MjQgNDcuOTU0NiA1NC4yNDM5IDQ3Ljg2OTZDNTQuMzE3MiA0Ny44MjcxIDU0LjQwMDQgNDcuODA0NyA1NC40ODUxIDQ3LjgwNDdDNTQuNTY5NyA0Ny44MDQ3IDU0LjY1MjkgNDcuODI3MSA1NC43MjYyIDQ3Ljg2OTZMNzAuNDkzNyA1Ni44MTMxQzcwLjU2NDIgNTYuODU2NSA3MC42MjI1IDU2LjkxNyA3MC42NjMyIDU2Ljk4OTFDNzAuNzAzOSA1Ny4wNjEyIDcwLjcyNTcgNTcuMTQyNCA3MC43MjY1IDU3LjIyNTFWNzUuMTgwNUM3MC43MjU5IDc1LjI2MjggNzAuNzA0MiA3NS4zNDM2IDcwLjY2MzUgNzUuNDE1MUM3MC42MjI3IDc1LjQ4NjYgNzAuNTY0MiA3NS41NDY0IDcwLjQ5MzcgNzUuNTg4OEM3MC40MjA2IDc1LjYyOTEgNzAuMzM4NyA3NS42NTA3IDcwLjI1NTMgNzUuNjUxNyIgZmlsbD0id2hpdGUiLz4KPHBhdGggb3BhY2l0eT0iMC40IiBkPSJNNzQuNzE5OCA3NS42NTExQzc0LjYzMzMgNzUuNjUxMiA3NC41NDgyIDc1LjYyOTYgNzQuNDcyMiA3NS41ODgzQzc0LjQwMTYgNzUuNTQ2MSA3NC4zNDMyIDc1LjQ4NjIgNzQuMzAyNyA3NS40MTQ3Qzc0LjI2MjMgNzUuMzQzMSA3NC4yNDExIDc1LjI2MjIgNzQuMjQxMiA3NS4xOFY1Ny4zMzczQzc0LjI0MTIgNTcuMTcxIDc0LjMzMzYgNTcuMDE1OCA3NC40NzIyIDU2LjkyOUw5MC4yMzk3IDQ3Ljk4NTVDOTAuMzExOSA0Ny45NDM4IDkwLjM5MzggNDcuOTIxOSA5MC40NzcxIDQ3LjkyMTlDOTAuNTYwNSA0Ny45MjE5IDkwLjY0MjQgNDcuOTQzOCA5MC43MTQ2IDQ3Ljk4NTVDOTAuNzg3NiA0OC4wMjU1IDkwLjg0ODUgNDguMDg0MiA5MC44OTExIDQ4LjE1NTdDOTAuOTMzNyA0OC4yMjcyIDkwLjk1NjMgNDguMzA4OCA5MC45NTY2IDQ4LjM5MlY2Ni4yMzI4QzkwLjk1NyA2Ni4zMTY0IDkwLjkzNDcgNjYuMzk4NSA5MC44OTIxIDY2LjQ3MDRDOTAuODQ5NSA2Ni41NDI0IDkwLjc4ODEgNjYuNjAxNCA5MC43MTQ2IDY2LjY0MTFMNzQuOTUyNiA3NS41ODgzQzc0Ljg4MjUgNzUuNjMwNyA3NC44MDE4IDc1LjY1MjUgNzQuNzE5OCA3NS42NTExIiBmaWxsPSJ3aGl0ZSIvPgo8ZGVmcz4KPGxpbmVhckdyYWRpZW50IGlkPSJwYWludDBfbGluZWFyXzY4N18zNDU0IiB4MT0iMTYxIiB5MT0iMTgwIiB4Mj0iMy41OTI4NGUtMDciIHkyPSI0Ljk5OTk4IiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSI+CjxzdG9wLz4KPHN0b3Agb2Zmc2V0PSIxIiBzdG9wLWNvbG9yPSIjNTk1NjU2Ii8+CjwvbGluZWFyR3JhZGllbnQ+CjxjbGlwUGF0aCBpZD0iY2xpcDBfNjg3XzM0NTQiPgo8cmVjdCB3aWR0aD0iOTQiIGhlaWdodD0iOTQiIGZpbGw9IndoaXRlIiB0cmFuc2Zvcm09InRyYW5zbGF0ZSgyNSAyNSkiLz4KPC9jbGlwUGF0aD4KPC9kZWZzPgo8L3N2Zz4K - keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "external"], ["spec", "externalMethod"], ["spec", "externalPorts"], ["spec", "runStrategy"], ["spec", "instanceType"], ["spec", "instanceProfile"], ["spec", "disks"], ["spec", "networks"], ["spec", "subnets"], ["spec", "gpus"], ["spec", "cpuModel"], ["spec", "resources"], ["spec", "sshKeys"], ["spec", "cloudInit"], ["spec", "cloudInitSeed"]] + keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "external"], ["spec", "externalMethod"], ["spec", "externalPorts"], ["spec", "externalAllowICMP"], ["spec", "runStrategy"], ["spec", "instanceType"], ["spec", "instanceProfile"], ["spec", "disks"], ["spec", "networks"], ["spec", "subnets"], ["spec", "gpus"], ["spec", "cpuModel"], ["spec", "resources"], ["spec", "sshKeys"], ["spec", "cloudInit"], ["spec", "cloudInitSeed"]] secrets: exclude: [] include: [] From bb51c88f7817c1d862fdb7947deef7798f717d82 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 11:18:59 +0300 Subject: [PATCH 222/250] fix(monitoring): remove unused namespace variable from GPU efficiency dashboard Address review feedback from coderabbitai on dashboards/gpu/gpu-efficiency.json:839 Signed-off-by: Arsolitt --- dashboards/gpu/gpu-efficiency.json | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/dashboards/gpu/gpu-efficiency.json b/dashboards/gpu/gpu-efficiency.json index f351c964..28a24568 100644 --- a/dashboards/gpu/gpu-efficiency.json +++ b/dashboards/gpu/gpu-efficiency.json @@ -812,30 +812,6 @@ "auto": false, "auto_min": "10s", "auto_count": 30 - }, - { - "type": "query", - "name": "namespace", - "label": "Namespace", - "skipUrlSync": false, - "query": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "multi": true, - "allowCustomValue": true, - "refresh": 2, - "sort": 1, - "includeAll": true, - "auto": false, - "auto_min": "10s", - "auto_count": 30 } ] }, From 452bff45675c55e0c8e01d290dfbb3b1e7f59196 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 11:19:56 +0300 Subject: [PATCH 223/250] fix(monitoring): remove unused namespace variable from GPU performance dashboard Address review feedback from coderabbitai on dashboards/gpu/gpu-performance.json:277 Signed-off-by: Arsolitt --- dashboards/gpu/gpu-performance.json | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/dashboards/gpu/gpu-performance.json b/dashboards/gpu/gpu-performance.json index f9951577..aed9c69a 100644 --- a/dashboards/gpu/gpu-performance.json +++ b/dashboards/gpu/gpu-performance.json @@ -950,30 +950,6 @@ "auto": false, "auto_min": "10s", "auto_count": 30 - }, - { - "type": "query", - "name": "namespace", - "label": "Namespace", - "skipUrlSync": false, - "query": "label_values(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",namespace!~\"cozy-.*|kube-.*\"}, namespace)", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "multi": true, - "allowCustomValue": true, - "refresh": 2, - "sort": 1, - "includeAll": true, - "auto": false, - "auto_min": "10s", - "auto_count": 30 } ] }, From 4c697982b284a9d1f1b47d9cbbdbeab6314d253d Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 11:20:19 +0300 Subject: [PATCH 224/250] fix(monitoring): use Hostname label in GPU tenants dashboard legends Address review feedback from coderabbitai and gemini-code-assist on dashboards/gpu/gpu-tenants.json:532 Signed-off-by: Arsolitt --- dashboards/gpu/gpu-tenants.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dashboards/gpu/gpu-tenants.json b/dashboards/gpu/gpu-tenants.json index e2c400a8..6194a15e 100644 --- a/dashboards/gpu/gpu-tenants.json +++ b/dashboards/gpu/gpu-tenants.json @@ -529,7 +529,7 @@ "targets": [ { "expr": "node:gpu_util:avg", - "legendFormat": "{{node}}", + "legendFormat": "{{Hostname}}", "refId": "A" } ], @@ -585,7 +585,7 @@ "targets": [ { "expr": "node:tensor_active:avg * 100", - "legendFormat": "{{node}}", + "legendFormat": "{{Hostname}}", "refId": "A" } ], @@ -654,7 +654,7 @@ "targets": [ { "expr": "node:power_watts:sum", - "legendFormat": "{{node}}", + "legendFormat": "{{Hostname}}", "refId": "A" } ], @@ -988,7 +988,7 @@ "targets": [ { "expr": "sum_over_time(node:power_watts:sum[24h:1m]) / 60 / 1000", - "legendFormat": "{{node}}", + "legendFormat": "{{Hostname}}", "refId": "A" } ], From bbf338a57d4dca0286bef4dab991ccbce6a2caa2 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 11:20:41 +0300 Subject: [PATCH 225/250] fix(gpu-operator): fail fast on missing artifacts in driver-compat example Address review feedback from coderabbitai on packages/system/gpu-operator/examples/nvidia-driver-compat.yaml:77 Signed-off-by: Arsolitt --- .../examples/nvidia-driver-compat.yaml | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml b/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml index 7706b221..fc9d5981 100644 --- a/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml +++ b/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml @@ -61,16 +61,21 @@ spec: mkdir -p "$DST/bin" - if [ -f "$GLIBC_LIB/libnvidia-ml.so.1" ]; then - cp -f "$GLIBC_LIB/libnvidia-ml.so.1" "$DST/libnvidia-ml.so.1" - echo "Copied libnvidia-ml.so.1" - fi + [ -f "$GLIBC_LIB/libnvidia-ml.so.1" ] || { + echo "missing $GLIBC_LIB/libnvidia-ml.so.1" >&2 + exit 1 + } + [ -f "$SRC_BIN/nvidia-smi" ] || { + echo "missing $SRC_BIN/nvidia-smi" >&2 + exit 1 + } - if [ -f "$SRC_BIN/nvidia-smi" ]; then - cp -f "$SRC_BIN/nvidia-smi" "$DST/bin/nvidia-smi" - chmod +x "$DST/bin/nvidia-smi" - echo "Copied nvidia-smi" - fi + cp -f "$GLIBC_LIB/libnvidia-ml.so.1" "$DST/libnvidia-ml.so.1" + echo "Copied libnvidia-ml.so.1" + + cp -f "$SRC_BIN/nvidia-smi" "$DST/bin/nvidia-smi" + chmod +x "$DST/bin/nvidia-smi" + echo "Copied nvidia-smi" mkdir -p /host/run/nvidia/validations touch /host/run/nvidia/validations/.driver-ctr-ready From 5718740ae3edddd9112e483426ebfebba423ed92 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 11:20:57 +0300 Subject: [PATCH 226/250] docs(gpu-operator): correct gpu-quotas dashboard dependencies in README Address review feedback from coderabbitai on packages/system/gpu-operator/examples/README.md:85 Signed-off-by: Arsolitt --- packages/system/gpu-operator/examples/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/system/gpu-operator/examples/README.md b/packages/system/gpu-operator/examples/README.md index f9094549..b6f170c8 100644 --- a/packages/system/gpu-operator/examples/README.md +++ b/packages/system/gpu-operator/examples/README.md @@ -81,7 +81,7 @@ What each dashboard needs on top of the upstream DCGM Exporter | `gpu-performance` | Per-node, per-GPU deep dive | `DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION` | | `gpu-efficiency` | Per-workload util vs tensor active | `DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION` (via `gpu:*_throttle_fraction:rate5m` recording rules) | | `gpu-fleet` | Cluster-wide admin inventory | `DCGM_FI_DEV_POWER_MGMT_LIMIT` (for the TDP vs draw panel) | -| `gpu-quotas` | Kube-quota vs live usage | `kube_pod_container_resource_requests`, `kube_pod_status_phase`, `DCGM_FI_DEV_GPU_UTIL` (via `namespace:gpu_count:allocated` / `cluster:gpu_count:*` recording rules) | +| `gpu-quotas` | Kube-quota vs live usage | `kube_pod_container_resource_requests`, `kube_pod_status_phase`, `kube_node_status_allocatable` (via `namespace:gpu_count:allocated` / `cluster:gpu_count:*` recording rules) | | `gpu-tenants` | Per-namespace tenant view | nothing (works on default counters) | `DCGM_FI_PROF_PIPE_TENSOR_ACTIVE` and `DCGM_FI_PROF_GR_ENGINE_ACTIVE` From aba5ae3fcd0f4f4fdceae05f24e9371550076731 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 11:21:12 +0300 Subject: [PATCH 227/250] fix(monitoring): prevent many-to-many match in util-per-watt recording rule Address review feedback from coderabbitai on packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml:119 Signed-off-by: Arsolitt --- .../alerts/gpu-recording.rules.yaml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml index b046edf5..5e0f9ea0 100644 --- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml +++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml @@ -109,13 +109,15 @@ spec: # to GPU-identifying labels. - record: gpu:util_per_watt:avg5m expr: | - avg by (Hostname, gpu, UUID) ( + max by (Hostname, gpu, UUID) ( avg_over_time(DCGM_FI_DEV_GPU_UTIL[5m]) - / on (Hostname, gpu, UUID) - clamp_min( - avg_over_time(DCGM_FI_DEV_POWER_USAGE[5m]), - 1 - ) + ) + / on (Hostname, gpu, UUID) + clamp_min( + max by (Hostname, gpu, UUID) ( + avg_over_time(DCGM_FI_DEV_POWER_USAGE[5m]) + ), + 1 ) # Fraction of time power-throttled (TDP cap) — 1.0 = fully throttled. From 8e6266703da78b28109ec34ed6fd030612dc8c68 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 11:43:30 +0300 Subject: [PATCH 228/250] Revert "chore: ignore CLAUDE.local.md" This reverts commit 11f7d3589b3284fca96d0df21a02a5e967dadce7. Signed-off-by: Arsolitt --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 64470222..61003de5 100644 --- a/.gitignore +++ b/.gitignore @@ -84,4 +84,3 @@ tmp/ # build revision marker (generated by make image-packages) packages/core/platform/.build-revision .claude/ -CLAUDE.local.md From 31de9989f62b2f286fc3e6bf58ca85b1b96e54ad Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 11:44:04 +0300 Subject: [PATCH 229/250] fix(gpu-operator): add DCGM_FI_DRIVER_VERSION to custom metrics CSV Signed-off-by: Arsolitt --- packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml b/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml index 8e57788b..5b82fd44 100644 --- a/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml +++ b/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml @@ -16,6 +16,9 @@ data: # If line starts with a '#' it is considered a comment # DCGM FIELD, Prometheus metric type, help message + # Identity + DCGM_FI_DRIVER_VERSION, label, Driver version. + # Clocks DCGM_FI_DEV_SM_CLOCK, gauge, SM clock frequency (in MHz). DCGM_FI_DEV_MEM_CLOCK, gauge, Memory clock frequency (in MHz). From cacd3714bd24d7a3ec7182b8ec7b1057041fcf0a Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 11:44:08 +0300 Subject: [PATCH 230/250] fix(monitoring): include phase label in GPU limits query for consistency Signed-off-by: Arsolitt --- dashboards/gpu/gpu-quotas.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json index ce1e387e..62928b56 100644 --- a/dashboards/gpu/gpu-quotas.json +++ b/dashboards/gpu/gpu-quotas.json @@ -430,7 +430,7 @@ "refId": "requested" }, { - "expr": "kube_pod_container_resource_limits{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left() (kube_pod_status_phase{phase=~\"Running|Pending\"} == 1)", + "expr": "kube_pod_container_resource_limits{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left(phase) (kube_pod_status_phase{phase=~\"Running|Pending\"} == 1)", "instant": true, "range": false, "format": "table", From b0784c0d33d55fefeaaf60cdeabca4bebec1c069 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 11:44:11 +0300 Subject: [PATCH 231/250] fix(monitoring): generalize GPU temperature description in fleet dashboard Signed-off-by: Arsolitt --- dashboards/gpu/gpu-fleet.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboards/gpu/gpu-fleet.json b/dashboards/gpu/gpu-fleet.json index a7d3f085..60c12bfb 100644 --- a/dashboards/gpu/gpu-fleet.json +++ b/dashboards/gpu/gpu-fleet.json @@ -860,7 +860,7 @@ } ], "title": "Max GPU temperature", - "description": "Hottest GPU in the fleet right now. A10 sustained target \u003c83°C.", + "description": "Hottest GPU in the fleet right now. Adjust thresholds to match your GPU model specifications.", "transparent": false, "datasource": { "type": "prometheus", From ed6f9bbd1da5d0daeea5fbe38e5aa42e3a528496 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 12:02:18 +0300 Subject: [PATCH 232/250] fix(monitoring): regenerate gpu-quotas dashboard from SDK Align dashboard JSON with the SDK source of truth. Signed-off-by: Arsolitt --- dashboards/gpu/gpu-quotas.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json index 62928b56..ce1e387e 100644 --- a/dashboards/gpu/gpu-quotas.json +++ b/dashboards/gpu/gpu-quotas.json @@ -430,7 +430,7 @@ "refId": "requested" }, { - "expr": "kube_pod_container_resource_limits{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left(phase) (kube_pod_status_phase{phase=~\"Running|Pending\"} == 1)", + "expr": "kube_pod_container_resource_limits{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left() (kube_pod_status_phase{phase=~\"Running|Pending\"} == 1)", "instant": true, "range": false, "format": "table", From 84f506116f452889011c9d31c0cd48628265d448 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 12:07:31 +0300 Subject: [PATCH 233/250] docs(gpu-operator): clarify violation counter unit ambiguity in DCGM CSV Signed-off-by: Arsolitt --- .../gpu-operator/examples/dcgm-custom-metrics.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml b/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml index 5b82fd44..ef2b0b88 100644 --- a/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml +++ b/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml @@ -66,12 +66,12 @@ data: DCGM_FI_DEV_ROW_REMAP_FAILURE, gauge, Whether remapping of rows has failed. # Throttle / violation counters (crucial for SLA and tenant monitoring) - DCGM_FI_DEV_POWER_VIOLATION, counter, Throttling duration due to power constraints (in us). - DCGM_FI_DEV_THERMAL_VIOLATION, counter, Throttling duration due to thermal constraints (in us). - DCGM_FI_DEV_SYNC_BOOST_VIOLATION, counter, Throttling duration due to sync-boost constraints (in us). - DCGM_FI_DEV_BOARD_LIMIT_VIOLATION, counter, Throttling duration due to board limit constraints (in us). - DCGM_FI_DEV_LOW_UTIL_VIOLATION, counter, Throttling duration due to low utilization (in us). - DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (in us). + DCGM_FI_DEV_POWER_VIOLATION, counter, Throttling duration due to power constraints (us per docs; ns on DCGM 3.x). + DCGM_FI_DEV_THERMAL_VIOLATION, counter, Throttling duration due to thermal constraints (us per docs; ns on DCGM 3.x). + DCGM_FI_DEV_SYNC_BOOST_VIOLATION, counter, Throttling duration due to sync-boost constraints (us per docs; ns on DCGM 3.x). + DCGM_FI_DEV_BOARD_LIMIT_VIOLATION, counter, Throttling duration due to board limit constraints (us per docs; ns on DCGM 3.x). + DCGM_FI_DEV_LOW_UTIL_VIOLATION, counter, Throttling duration due to low utilization (us per docs; ns on DCGM 3.x). + DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (us per docs; ns on DCGM 3.x). # NVLink — DCGM silently drops this metric on GPUs without NVLink, so # enabling it unconditionally is safe and keeps this file reusable From 2a6653e11ae6a97d992f80e4ae0731995deea08a Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 12:08:15 +0300 Subject: [PATCH 234/250] docs(monitoring): add panel descriptions to GPU quotas dashboard Regenerated from SDK source. Signed-off-by: Arsolitt --- dashboards/gpu/gpu-quotas.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json index ce1e387e..865fb95c 100644 --- a/dashboards/gpu/gpu-quotas.json +++ b/dashboards/gpu/gpu-quotas.json @@ -38,6 +38,7 @@ } ], "title": "GPU allocatable", + "description": "Total GPU capacity the cluster can schedule to pods.", "transparent": false, "datasource": { "type": "prometheus", @@ -91,6 +92,7 @@ } ], "title": "GPU requested", + "description": "Sum of GPU requests across all pods cluster-wide, including system namespaces.", "transparent": false, "datasource": { "type": "prometheus", @@ -148,6 +150,7 @@ } ], "title": "Allocation ratio", + "description": "Percentage of allocatable GPUs currently requested by pods.", "transparent": false, "datasource": { "type": "prometheus", @@ -213,6 +216,7 @@ } ], "title": "Pending pods (GPU)", + "description": "Pods requesting GPUs that are stuck in Pending state — indicates capacity shortage.", "transparent": false, "datasource": { "type": "prometheus", @@ -284,6 +288,7 @@ } ], "title": "GPU requested per namespace", + "description": "GPU allocation breakdown by namespace — spot top consumers at a glance.", "transparent": false, "datasource": { "type": "prometheus", @@ -352,6 +357,7 @@ } ], "title": "GPU allocated over time", + "description": "Requested vs allocatable GPUs over time — shows allocation pressure trends.", "transparent": false, "datasource": { "type": "prometheus", @@ -438,6 +444,7 @@ } ], "title": "Pods requesting GPU", + "description": "Per-pod GPU requests and limits with scheduling status — Running or Pending.", "transparent": false, "datasource": { "type": "prometheus", From d20285836cede23c3d7cfe091e5444359bac4320 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Tue, 28 Apr 2026 11:14:27 +0200 Subject: [PATCH 235/250] feat(cozy-proxy): bump to v0.3.0 Pulls in the per-port filtering and allowICMP support that the companion vm-instance chart fix in #2501 relies on. cozy-proxy v0.3.0 also tightens the selector to the standard service.kubernetes.io/service-proxy-name=cozy-proxy label and switches the default ingress mode to port-filter; both are already covered by the vm-instance chart (label landed in #2357, wholeIP/allowICMP wired explicitly in #2501), so VM workloads upgrade transparently. Out-of-tree consumers using cozy-proxy annotations directly (without the label, or relying on the absent-annotation passthrough default) are called out in the upstream v0.3.0 release notes: https://github.com/cozystack/cozy-proxy/releases/tag/v0.3.0 Signed-off-by: Andrei Kvapil --- packages/system/cozy-proxy/charts/cozy-proxy/Chart.yaml | 4 ++-- packages/system/cozy-proxy/charts/cozy-proxy/values.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/system/cozy-proxy/charts/cozy-proxy/Chart.yaml b/packages/system/cozy-proxy/charts/cozy-proxy/Chart.yaml index 13352680..58f166f6 100644 --- a/packages/system/cozy-proxy/charts/cozy-proxy/Chart.yaml +++ b/packages/system/cozy-proxy/charts/cozy-proxy/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: cozy-proxy description: A simple kube-proxy addon for 1:1 NAT services in Kubernetes using an NFT backend type: application -version: 0.2.0 -appVersion: 0.2.0 +version: 0.3.0 +appVersion: 0.3.0 diff --git a/packages/system/cozy-proxy/charts/cozy-proxy/values.yaml b/packages/system/cozy-proxy/charts/cozy-proxy/values.yaml index 8cde5bed..e143e926 100644 --- a/packages/system/cozy-proxy/charts/cozy-proxy/values.yaml +++ b/packages/system/cozy-proxy/charts/cozy-proxy/values.yaml @@ -1,6 +1,6 @@ image: repository: ghcr.io/cozystack/cozystack/cozy-proxy - tag: v0.2.0 + tag: v0.3.0 pullPolicy: IfNotPresent daemonset: From b2a8cca3bb8960ab2f058051deb03f0b7cab0dca Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 12:15:32 +0300 Subject: [PATCH 236/250] fix(monitoring): filter zero-valued series from active tenants count Address review feedback from coderabbitai on dashboards/gpu/gpu-tenants.json:39 Signed-off-by: Arsolitt --- dashboards/gpu/gpu-tenants.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboards/gpu/gpu-tenants.json b/dashboards/gpu/gpu-tenants.json index 6194a15e..7fef321d 100644 --- a/dashboards/gpu/gpu-tenants.json +++ b/dashboards/gpu/gpu-tenants.json @@ -35,7 +35,7 @@ "id": 2, "targets": [ { - "expr": "count(namespace:gpu_count:allocated{namespace=~\"$namespace\"})", + "expr": "count(namespace:gpu_count:allocated{namespace=~\"$namespace\"} \u003e 0)", "refId": "A" } ], From 7257b6aed4bb2b6499168eab467a400196f2e625 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Tue, 28 Apr 2026 12:26:57 +0300 Subject: [PATCH 237/250] fix(api): address review feedback on TenantNamespace RBAC Address review feedback from sircthulhu and CodeRabbit: 1. Return Forbidden instead of NotFound for unauthorized access - Get() now returns 403 Forbidden to follow standard K8s RBAC behavior - Previously returned 404 NotFound for security-by-obscurity - Updated test expectations to match new behavior 2. Propagate field and label selectors in Watch handler - Pass opts.FieldSelector and opts.LabelSelector to upstream Watch - Add defensive filtering before authorization to prevent RBAC bypass - Fixes potential issue with resourceNames restrictions 3. Refactor subject-matching logic to eliminate duplication - Extract matchesSubject() helper for Group/User/ServiceAccount checks - Remove duplicated code from filterAccessible and hasAccessToNamespace - Consolidates ServiceAccount namespace fallback logic All tests pass successfully. Signed-off-by: IvanHunters --- pkg/registry/core/tenantnamespace/rest.go | 94 +++++++++++-------- .../core/tenantnamespace/rest_test.go | 6 +- 2 files changed, 56 insertions(+), 44 deletions(-) diff --git a/pkg/registry/core/tenantnamespace/rest.go b/pkg/registry/core/tenantnamespace/rest.go index 7724c0e1..223a6925 100644 --- a/pkg/registry/core/tenantnamespace/rest.go +++ b/pkg/registry/core/tenantnamespace/rest.go @@ -17,6 +17,8 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metainternal "k8s.io/apimachinery/pkg/apis/meta/internalversion" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" @@ -130,8 +132,8 @@ func (r *REST) Get( return nil, err } if !hasAccess { - // Return NotFound instead of Forbidden to prevent enumeration - return nil, apierrors.NewNotFound(r.gvr.GroupResource(), name) + // Return Forbidden to follow standard K8s RBAC behavior + return nil, apierrors.NewForbidden(r.gvr.GroupResource(), name, fmt.Errorf("access denied")) } ns := &corev1.Namespace{} @@ -155,10 +157,20 @@ func (r *REST) Get( func (r *REST) Watch(ctx context.Context, opts *metainternal.ListOptions) (watch.Interface, error) { nsList := &corev1.NamespaceList{} - nsWatch, err := r.w.Watch(ctx, nsList, &client.ListOptions{Raw: &metav1.ListOptions{ + + // Build upstream watch options with field and label selectors + rawOpts := &metav1.ListOptions{ Watch: true, ResourceVersion: opts.ResourceVersion, - }}) + } + if opts.FieldSelector != nil { + rawOpts.FieldSelector = opts.FieldSelector.String() + } + if opts.LabelSelector != nil { + rawOpts.LabelSelector = opts.LabelSelector.String() + } + + nsWatch, err := r.w.Watch(ctx, nsList, &client.ListOptions{Raw: rawOpts}) if err != nil { return nil, err } @@ -200,6 +212,18 @@ func (r *REST) Watch(ctx context.Context, opts *metainternal.ListOptions) (watch continue } + // Apply defensive filtering for field and label selectors + if opts.FieldSelector != nil { + if !opts.FieldSelector.Matches(fields.Set{"metadata.name": ns.Name}) { + continue + } + } + if opts.LabelSelector != nil { + if !opts.LabelSelector.Matches(labels.Set(ns.Labels)) { + continue + } + } + // Check if user has access to this namespace hasAccess, err := r.hasAccessToNamespace(ctx, ns.Name) if err != nil { @@ -331,6 +355,26 @@ func (r *REST) makeList(src *corev1.NamespaceList, allowed []string) *corev1alph return out } +// matchesSubject checks if a RoleBinding subject matches the user's identity. +// It handles Group, User, and ServiceAccount subjects with proper namespace fallback. +func matchesSubject(subj rbacv1.Subject, bindingNamespace, username string, groups map[string]struct{}) bool { + switch subj.Kind { + case "Group": + _, ok := groups[subj.Name] + return ok + case "User": + return subj.Name == username + case "ServiceAccount": + saNamespace := subj.Namespace + if saNamespace == "" { + saNamespace = bindingNamespace + } + return username == fmt.Sprintf("system:serviceaccount:%s:%s", saNamespace, subj.Name) + default: + return false + } +} + func (r *REST) filterAccessible( ctx context.Context, names []string, @@ -369,26 +413,9 @@ func (r *REST) filterAccessible( subjectLoop: for j := range rbs.Items[i].Subjects { subj := rbs.Items[i].Subjects[j] - switch subj.Kind { - case "Group": - if _, ok = groups[subj.Name]; ok { - allowedNameSet[rbs.Items[i].Namespace] = struct{}{} - break subjectLoop - } - case "User": - if subj.Name == u.GetName() { - allowedNameSet[rbs.Items[i].Namespace] = struct{}{} - break subjectLoop - } - case "ServiceAccount": - saNamespace := subj.Namespace - if saNamespace == "" { - saNamespace = rbs.Items[i].Namespace - } - if u.GetName() == fmt.Sprintf("system:serviceaccount:%s:%s", saNamespace, subj.Name) { - allowedNameSet[rbs.Items[i].Namespace] = struct{}{} - break subjectLoop - } + if matchesSubject(subj, rbs.Items[i].Namespace, u.GetName(), groups) { + allowedNameSet[rbs.Items[i].Namespace] = struct{}{} + break subjectLoop } } } @@ -434,23 +461,8 @@ func (r *REST) hasAccessToNamespace( for i := range rbs.Items { for j := range rbs.Items[i].Subjects { subj := rbs.Items[i].Subjects[j] - switch subj.Kind { - case "Group": - if _, ok := groups[subj.Name]; ok { - return true, nil - } - case "User": - if subj.Name == u.GetName() { - return true, nil - } - case "ServiceAccount": - saNamespace := subj.Namespace - if saNamespace == "" { - saNamespace = rbs.Items[i].Namespace - } - if u.GetName() == fmt.Sprintf("system:serviceaccount:%s:%s", saNamespace, subj.Name) { - return true, nil - } + if matchesSubject(subj, rbs.Items[i].Namespace, u.GetName(), groups) { + return true, nil } } } diff --git a/pkg/registry/core/tenantnamespace/rest_test.go b/pkg/registry/core/tenantnamespace/rest_test.go index eb678949..e1fb1e85 100644 --- a/pkg/registry/core/tenantnamespace/rest_test.go +++ b/pkg/registry/core/tenantnamespace/rest_test.go @@ -462,9 +462,9 @@ func TestGet_WithoutAccess(t *testing.T) { t.Errorf("expected nil object, got %v", obj) } - // Verify it returns NotFound (not Forbidden) to prevent enumeration - if !apierrors.IsNotFound(err) { - t.Errorf("expected NotFound error, got %v", err) + // Verify it returns Forbidden to follow standard K8s RBAC behavior + if !apierrors.IsForbidden(err) { + t.Errorf("expected Forbidden error, got %v", err) } } From 61ed7ad89c0bff66236e6884e6b6cd6038383718 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Tue, 28 Apr 2026 16:33:37 +0500 Subject: [PATCH 238/250] fix(api): address review feedback on TenantNamespace watch path - Hoist user identity extraction out of the Watch goroutine; reuse a cached username and groups map across events instead of re-fetching them per event. Watch now returns Unauthorized up front when no user is present in the context, rather than failing silently per event. - Switch the per-event access-check error log to structured klog.ErrorS to comply with the project Go style guide. - Strengthen TestGet_WithAccess to assert the concrete *TenantNamespace type plus Name, Kind, and APIVersion, so type or metadata regressions fail fast. Assisted-By: Claude Signed-off-by: Myasnikov Daniil --- pkg/registry/core/tenantnamespace/rest.go | 35 +++++++++++++++---- .../core/tenantnamespace/rest_test.go | 16 +++++++-- 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/pkg/registry/core/tenantnamespace/rest.go b/pkg/registry/core/tenantnamespace/rest.go index 223a6925..4ee97eb0 100644 --- a/pkg/registry/core/tenantnamespace/rest.go +++ b/pkg/registry/core/tenantnamespace/rest.go @@ -156,6 +156,18 @@ func (r *REST) Get( // ----------------------------------------------------------------------------- func (r *REST) Watch(ctx context.Context, opts *metainternal.ListOptions) (watch.Interface, error) { + // Extract user identity once for the lifetime of the watch — it does not + // change between events and rebuilding it per event is wasteful. + u, ok := request.UserFrom(ctx) + if !ok { + return nil, apierrors.NewUnauthorized("user missing in context") + } + username := u.GetName() + groups := make(map[string]struct{}) + for _, group := range u.GetGroups() { + groups[group] = struct{}{} + } + nsList := &corev1.NamespaceList{} // Build upstream watch options with field and label selectors @@ -224,10 +236,11 @@ func (r *REST) Watch(ctx context.Context, opts *metainternal.ListOptions) (watch } } - // Check if user has access to this namespace - hasAccess, err := r.hasAccessToNamespace(ctx, ns.Name) + // Check if user has access to this namespace using the cached + // identity — avoids re-extracting user/groups on every event. + hasAccess, err := r.hasAccessToNamespaceForUser(ctx, ns.Name, username, groups) if err != nil { - klog.Errorf("Failed to check access for namespace %s in watch: %v", ns.Name, err) + klog.ErrorS(err, "Failed to check access for namespace in watch", "namespace", ns.Name) continue } if !hasAccess { @@ -437,12 +450,22 @@ func (r *REST) hasAccessToNamespace( if !ok { return false, fmt.Errorf("user missing in context") } - - // Check privileged groups groups := make(map[string]struct{}) for _, group := range u.GetGroups() { groups[group] = struct{}{} } + return r.hasAccessToNamespaceForUser(ctx, namespace, u.GetName(), groups) +} + +// hasAccessToNamespaceForUser is the inner check that does not re-extract user +// identity from context. Use this in hot paths (e.g. the Watch loop) where the +// caller has already cached the user name and groups. +func (r *REST) hasAccessToNamespaceForUser( + ctx context.Context, + namespace, username string, + groups map[string]struct{}, +) (bool, error) { + // Check privileged groups if _, ok := groups["system:masters"]; ok { return true, nil } @@ -461,7 +484,7 @@ func (r *REST) hasAccessToNamespace( for i := range rbs.Items { for j := range rbs.Items[i].Subjects { subj := rbs.Items[i].Subjects[j] - if matchesSubject(subj, rbs.Items[i].Namespace, u.GetName(), groups) { + if matchesSubject(subj, rbs.Items[i].Namespace, username, groups) { return true, nil } } diff --git a/pkg/registry/core/tenantnamespace/rest_test.go b/pkg/registry/core/tenantnamespace/rest_test.go index e1fb1e85..52d2a075 100644 --- a/pkg/registry/core/tenantnamespace/rest_test.go +++ b/pkg/registry/core/tenantnamespace/rest_test.go @@ -15,6 +15,8 @@ import ( "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/endpoints/request" "sigs.k8s.io/controller-runtime/pkg/client/fake" + + corev1alpha1 "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1" ) func TestMakeListSortsAlphabetically(t *testing.T) { @@ -408,8 +410,18 @@ func TestGet_WithAccess(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - if obj == nil { - t.Fatal("expected object, got nil") + tn, ok := obj.(*corev1alpha1.TenantNamespace) + if !ok { + t.Fatalf("expected *TenantNamespace, got %T", obj) + } + if tn.Name != "tenant-test" { + t.Errorf("expected name %q, got %q", "tenant-test", tn.Name) + } + if tn.Kind != "TenantNamespace" { + t.Errorf("expected Kind=TenantNamespace, got %q", tn.Kind) + } + if tn.APIVersion != corev1alpha1.SchemeGroupVersion.String() { + t.Errorf("expected APIVersion=%q, got %q", corev1alpha1.SchemeGroupVersion.String(), tn.APIVersion) } } From 1dc02d44f4ea15253ffd799e85a54ed79ae87fd1 Mon Sep 17 00:00:00 2001 From: Timofei Larkin Date: Mon, 27 Apr 2026 10:27:46 +0300 Subject: [PATCH 239/250] refactor(lineage-controller-webhook): align with cozystack-api shape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Collapse the chart to a single Deployment shape that mirrors cozystack-api: 2 replicas, soft (preferred) nodeAffinity to node-role.kubernetes.io/control-plane via Exists, permissive tolerations, soft podAntiAffinity on hostname, an unconditional PodDisruptionBudget with maxUnavailable: 1, and a Service with spec.trafficDistribution: PreferClose. The same shape works on Talos / kubeadm / k3s and on managed Kubernetes / Cozy-in-Cozy tenant clusters without per-distro overrides — fixing #2417 by making the soft control-plane affinity gracefully fall back to worker scheduling when no control-plane nodes are visible. Drop the DaemonSet path entirely. The previous PR's deployment.enabled toggle, the workload-kind switch in templates/workload.yaml, the fail-guard for localK8sAPIEndpoint+nodeAffinity=[], the conditional PDB, and the values-shape knobs for nodeAffinity and tolerations all go away as a consequence. Override the standard Deployment fields through the usual component-values mechanism if a non-default topology is ever needed. Mark localK8sAPIEndpoint.enabled deprecated and flip the default to false. The flag injects KUBERNETES_SERVICE_HOST=status.hostIP, which is only valid when the pod is actually scheduled on an apiserver- bearing node. With the new soft control-plane affinity, the pod can land off-control-plane and crash-loop. The latency motivation for the flag is real but pending separate webhook performance work; once addressed, the flag can be removed. Revert all changes to packages/core/platform/images/migrations/migrations/20. The earlier ds/...-or-deploy/... fallback was over-engineered: per run-migrations.sh, migration 20 fires only when CURRENT < 20 (i.e. direct upgrades from pre-0.37 to 1.3+), and that path is unsupported anyway. The original ds/... rollout-status line is dead code on every supported install and upgrade path. Add helm-unittest coverage for: the default Deployment shape (replicas, soft nodeAffinity, soft podAntiAffinity, tolerations); no env vars at the default localK8sAPIEndpoint.enabled=false; PDB rendered unconditionally with maxUnavailable: 1, including at replicas=1 (the no-op case); replicas value drives spec.replicas; and that localK8sAPIEndpoint.enabled=true does inject the env vars. The Service test continues to assert trafficDistribution: PreferClose with no internalTrafficPolicy. Add a slim README documenting the topology, parameters, and the deprecation note for localK8sAPIEndpoint. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Timofei Larkin --- .../platform/templates/bundles/system.yaml | 3 - .../lineage-controller-webhook/Makefile | 5 + .../lineage-controller-webhook/README.md | 67 +++++++++++++ .../templates/deployment.yaml | 49 --------- .../templates/pdb.yaml | 4 +- .../templates/service.yaml | 2 +- .../{daemonset.yaml => workload.yaml} | 27 +++-- .../tests/service_test.yaml | 30 ++++++ .../tests/workload_test.yaml | 99 +++++++++++++++++++ .../lineage-controller-webhook/values.yaml | 18 ++-- 10 files changed, 230 insertions(+), 74 deletions(-) create mode 100644 packages/system/lineage-controller-webhook/README.md delete mode 100644 packages/system/lineage-controller-webhook/templates/deployment.yaml rename packages/system/lineage-controller-webhook/templates/{daemonset.yaml => workload.yaml} (67%) create mode 100644 packages/system/lineage-controller-webhook/tests/service_test.yaml create mode 100644 packages/system/lineage-controller-webhook/tests/workload_test.yaml diff --git a/packages/core/platform/templates/bundles/system.yaml b/packages/core/platform/templates/bundles/system.yaml index 43ea7266..6c587c82 100644 --- a/packages/core/platform/templates/bundles/system.yaml +++ b/packages/core/platform/templates/bundles/system.yaml @@ -105,9 +105,6 @@ {{- /* cozystack-api DaemonSet */ -}} {{- $apiValues := dict "cozystackAPI" (dict "nodeSelector" $genericNodeSelector) -}} {{- $_ := set $cozystackEngineComponents "cozystack-api" (dict "values" $apiValues) -}} -{{- /* lineage-controller-webhook DaemonSet */ -}} -{{- $lineageValues := dict "lineageControllerWebhook" (dict "nodeSelector" $genericNodeSelector) -}} -{{- $_ := set $cozystackEngineComponents "lineage-controller-webhook" (dict "values" $lineageValues) -}} {{- end -}} {{- if .Values.authentication.oidc.enabled }} {{include "cozystack.platform.package" (list "cozystack.cozystack-engine" "oidc" $ $cozystackEngineComponents) }} diff --git a/packages/system/lineage-controller-webhook/Makefile b/packages/system/lineage-controller-webhook/Makefile index d5bada31..cdc1cf01 100644 --- a/packages/system/lineage-controller-webhook/Makefile +++ b/packages/system/lineage-controller-webhook/Makefile @@ -4,8 +4,13 @@ NAMESPACE=cozy-system include ../../../hack/common-envs.mk include ../../../hack/package.mk +.PHONY: test + image: image-lineage-controller-webhook +test: + helm unittest . + image-lineage-controller-webhook: docker buildx build -f images/lineage-controller-webhook/Dockerfile ../../.. \ --tag $(REGISTRY)/lineage-controller-webhook:$(call settag,$(TAG)) \ diff --git a/packages/system/lineage-controller-webhook/README.md b/packages/system/lineage-controller-webhook/README.md new file mode 100644 index 00000000..1e43c9c8 --- /dev/null +++ b/packages/system/lineage-controller-webhook/README.md @@ -0,0 +1,67 @@ +# lineage-controller-webhook + +Cozystack system package for the **lineage controller webhook** — a mutating +admission webhook that stamps "lineage" labels onto tenant workloads, linking +each resource back to the Cozystack `Application` that ultimately owns it. + +The webhook intercepts CREATE and UPDATE on `pods`, `secrets`, `services`, +`persistentvolumeclaims`, `ingresses.networking.k8s.io`, and +`workloadmonitors.cozystack.io` outside system namespaces. For each request it +walks the ownership graph upward (Kubernetes `ownerReferences`, then the +`HelmRelease` Flux installed the resource with, then the Cozystack +`Application`-derived CRD that produced the HelmRelease) and writes the +discovered application's group, kind and name as labels +(`apps.cozystack.io/application.{group,kind,name}`) on the incoming object. +Those labels let the aggregated API server, the Cozystack dashboard, and other +lineage-aware consumers reason about which application a resource belongs to. + +The webhook serves TLS on port 9443 with a cert-manager issued certificate, and +runs with `failurePolicy: Fail` — every CREATE/UPDATE on the resources above +is gated on the webhook being reachable. + +## Topology + +The chart ships a single shape, modelled on `cozystack-api`: + +- **Deployment** with two replicas (override via `replicas`). +- **Soft `nodeAffinity`** preferring `node-role.kubernetes.io/control-plane` + (`Exists`, so it matches both Talos's empty value and k3s/kubeadm's `"true"`). + Soft means: the pod lands on a control-plane node when one is reachable, and + on any worker otherwise — no override needed for managed Kubernetes, + Cozy-in-Cozy tenants, or any other cluster where control-plane nodes aren't + visible. +- **Permissive `tolerations`** (`[{operator: Exists}]`) so the pod can land on + tainted control-plane nodes when the soft affinity is satisfiable. +- **Soft `podAntiAffinity`** on `kubernetes.io/hostname` so replicas spread + across nodes when possible (best-effort). +- **`PodDisruptionBudget`** with `maxUnavailable: 1`, unconditional. At + `replicas: 1` it's a useful no-op (allows full drain); at `replicas >= 2` + it caps disruption to one pod. +- **`Service` with `spec.trafficDistribution: PreferClose`** so the apiserver + prefers a webhook endpoint on its own node when one exists, and transparently + falls over to a remote endpoint otherwise. Requires Kubernetes ≥ 1.31; on + older clusters the field is silently ignored and traffic uses default + cluster-wide distribution. + +## Parameters + +All keys live under the top-level `lineageControllerWebhook:` map. + +| Name | Description | Value | +| ----------------------------- | --------------------------------------------------------- | -------------------------------------------------------------------------------- | +| `image` | Container image (digest-pinned) | `ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.3.0@sha256:e898…6fb0` | +| `debug` | Enable `--zap-log-level=debug` instead of `info` | `false` | +| `replicas` | Deployment replica count | `2` | +| `localK8sAPIEndpoint.enabled` | **Deprecated.** See note below. | `false` | + +### `localK8sAPIEndpoint.enabled` (deprecated) + +When enabled, this injects `KUBERNETES_SERVICE_HOST=status.hostIP` and +`KUBERNETES_SERVICE_PORT=6443` so the webhook talks to the kube-apiserver on +its own node. It was originally added to avoid latency on the +webhook-to-apiserver path, but it is only valid when the pod is actually +scheduled on an apiserver-bearing node — which the chart's soft control-plane +affinity no longer guarantees. With this flag enabled and the pod scheduled +off a control-plane node, the controller will crash-loop dialing a non- +apiserver hostIP. Slated for removal once the latency motivation is addressed +in the webhook itself; **leave disabled**. diff --git a/packages/system/lineage-controller-webhook/templates/deployment.yaml b/packages/system/lineage-controller-webhook/templates/deployment.yaml deleted file mode 100644 index 87717fd3..00000000 --- a/packages/system/lineage-controller-webhook/templates/deployment.yaml +++ /dev/null @@ -1,49 +0,0 @@ -{{- if .Values.lineageControllerWebhook.deployment.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: lineage-controller-webhook - labels: - app: lineage-controller-webhook -spec: - replicas: {{ .Values.lineageControllerWebhook.deployment.replicas }} - selector: - matchLabels: - app: lineage-controller-webhook - template: - metadata: - labels: - app: lineage-controller-webhook - spec: - serviceAccountName: lineage-controller-webhook - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - topologyKey: kubernetes.io/hostname - labelSelector: - matchLabels: - app: lineage-controller-webhook - containers: - - name: lineage-controller-webhook - image: "{{ .Values.lineageControllerWebhook.image }}" - args: - {{- if .Values.lineageControllerWebhook.debug }} - - --zap-log-level=debug - {{- else }} - - --zap-log-level=info - {{- end }} - ports: - - name: webhook - containerPort: 9443 - volumeMounts: - - name: webhook-certs - mountPath: /tmp/k8s-webhook-server/serving-certs - readOnly: true - volumes: - - name: webhook-certs - secret: - secretName: lineage-controller-webhook-cert - defaultMode: 0400 -{{- end }} \ No newline at end of file diff --git a/packages/system/lineage-controller-webhook/templates/pdb.yaml b/packages/system/lineage-controller-webhook/templates/pdb.yaml index b0a9d4e4..0786c8ff 100644 --- a/packages/system/lineage-controller-webhook/templates/pdb.yaml +++ b/packages/system/lineage-controller-webhook/templates/pdb.yaml @@ -1,11 +1,9 @@ -{{- if .Values.lineageControllerWebhook.deployment.enabled }} apiVersion: policy/v1 kind: PodDisruptionBudget metadata: name: lineage-controller-webhook spec: - minAvailable: {{ if gt (int .Values.lineageControllerWebhook.deployment.replicas) 1 }}1{{ else }}0{{ end }} + maxUnavailable: 1 selector: matchLabels: app: lineage-controller-webhook -{{- end }} \ No newline at end of file diff --git a/packages/system/lineage-controller-webhook/templates/service.yaml b/packages/system/lineage-controller-webhook/templates/service.yaml index c5df90fb..fbb3ad54 100644 --- a/packages/system/lineage-controller-webhook/templates/service.yaml +++ b/packages/system/lineage-controller-webhook/templates/service.yaml @@ -5,7 +5,7 @@ metadata: labels: app: lineage-controller-webhook spec: - internalTrafficPolicy: Local + trafficDistribution: PreferClose type: ClusterIP ports: - port: 443 diff --git a/packages/system/lineage-controller-webhook/templates/daemonset.yaml b/packages/system/lineage-controller-webhook/templates/workload.yaml similarity index 67% rename from packages/system/lineage-controller-webhook/templates/daemonset.yaml rename to packages/system/lineage-controller-webhook/templates/workload.yaml index 536ea24a..097919c6 100644 --- a/packages/system/lineage-controller-webhook/templates/daemonset.yaml +++ b/packages/system/lineage-controller-webhook/templates/workload.yaml @@ -1,11 +1,11 @@ -{{- if not .Values.lineageControllerWebhook.deployment.enabled }} apiVersion: apps/v1 -kind: DaemonSet +kind: Deployment metadata: name: lineage-controller-webhook labels: app: lineage-controller-webhook spec: + replicas: {{ .Values.lineageControllerWebhook.replicas }} selector: matchLabels: app: lineage-controller-webhook @@ -14,13 +14,25 @@ spec: labels: app: lineage-controller-webhook spec: - {{- with .Values.lineageControllerWebhook.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} + serviceAccountName: lineage-controller-webhook tolerations: - operator: Exists - serviceAccountName: lineage-controller-webhook + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app: lineage-controller-webhook containers: - name: lineage-controller-webhook image: "{{ .Values.lineageControllerWebhook.image }}" @@ -52,4 +64,3 @@ spec: secret: secretName: lineage-controller-webhook-cert defaultMode: 0400 -{{- end }} \ No newline at end of file diff --git a/packages/system/lineage-controller-webhook/tests/service_test.yaml b/packages/system/lineage-controller-webhook/tests/service_test.yaml new file mode 100644 index 00000000..3cdbddd8 --- /dev/null +++ b/packages/system/lineage-controller-webhook/tests/service_test.yaml @@ -0,0 +1,30 @@ +suite: lineage-controller-webhook service +templates: + - templates/service.yaml + +release: + name: lineage-controller-webhook + namespace: cozy-system + +tests: + - it: renders a ClusterIP service on 443 -> 9443 with PreferClose traffic distribution + asserts: + - isKind: + of: Service + - equal: + path: spec.type + value: ClusterIP + - equal: + path: spec.trafficDistribution + value: PreferClose + - notExists: + path: spec.internalTrafficPolicy + - equal: + path: spec.ports[0].port + value: 443 + - equal: + path: spec.ports[0].targetPort + value: 9443 + - equal: + path: spec.selector.app + value: lineage-controller-webhook diff --git a/packages/system/lineage-controller-webhook/tests/workload_test.yaml b/packages/system/lineage-controller-webhook/tests/workload_test.yaml new file mode 100644 index 00000000..bcb24014 --- /dev/null +++ b/packages/system/lineage-controller-webhook/tests/workload_test.yaml @@ -0,0 +1,99 @@ +suite: lineage-controller-webhook workload + PDB +templates: + - templates/workload.yaml + - templates/pdb.yaml + +release: + name: lineage-controller-webhook + namespace: cozy-system + +tests: + # ---- Default Deployment shape ---------------------------------------------- + + - it: defaults render a 2-replica Deployment with soft control-plane nodeAffinity, soft podAntiAffinity, and Exists tolerations + template: templates/workload.yaml + asserts: + - isKind: + of: Deployment + - equal: + path: spec.replicas + value: 2 + - equal: + path: spec.template.spec.tolerations + value: + - operator: Exists + - equal: + path: spec.template.spec.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution + value: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - notExists: + path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution + - equal: + path: spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.topologyKey + value: kubernetes.io/hostname + + - it: defaults inject no localK8sAPIEndpoint env vars (knob is opt-in) + template: templates/workload.yaml + asserts: + - notExists: + path: spec.template.spec.containers[0].env + + # ---- PDB -------------------------------------------------------------------- + + - it: PDB renders unconditionally with maxUnavailable=1 at the default replica count + template: templates/pdb.yaml + asserts: + - isKind: + of: PodDisruptionBudget + - equal: + path: spec.maxUnavailable + value: 1 + - notExists: + path: spec.minAvailable + + - it: PDB still renders at replicas=1 (no-op, but consistent shape) + set: + lineageControllerWebhook.replicas: 1 + template: templates/pdb.yaml + asserts: + - isKind: + of: PodDisruptionBudget + - equal: + path: spec.maxUnavailable + value: 1 + + # ---- Replica override ------------------------------------------------------- + + - it: replicas value drives spec.replicas + set: + lineageControllerWebhook.replicas: 5 + template: templates/workload.yaml + asserts: + - equal: + path: spec.replicas + value: 5 + + # ---- Deprecated localK8sAPIEndpoint knob ----------------------------------- + + - it: localK8sAPIEndpoint.enabled=true injects KUBERNETES_SERVICE_HOST and PORT env vars + set: + lineageControllerWebhook.localK8sAPIEndpoint.enabled: true + template: templates/workload.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: KUBERNETES_SERVICE_HOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + - contains: + path: spec.template.spec.containers[0].env + content: + name: KUBERNETES_SERVICE_PORT + value: "6443" diff --git a/packages/system/lineage-controller-webhook/values.yaml b/packages/system/lineage-controller-webhook/values.yaml index 2b84f4e1..e56d332d 100644 --- a/packages/system/lineage-controller-webhook/values.yaml +++ b/packages/system/lineage-controller-webhook/values.yaml @@ -1,15 +1,13 @@ lineageControllerWebhook: image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.3.0@sha256:e8984709686a5eaf19b89da378d7b8c688ea5607e0783a88d9c9e4ccfca96fb0 debug: false + replicas: 2 + # DEPRECATED. Injects KUBERNETES_SERVICE_HOST=status.hostIP and + # KUBERNETES_SERVICE_PORT=6443 so the webhook talks to the apiserver on its + # own node — only valid when the pod is actually scheduled on an apiserver- + # bearing node. Now that the chart's nodeAffinity to control-plane is soft + # (preferred, not required), the pod can land off-control-plane and crash- + # loop dialing a non-apiserver hostIP. Slated for removal once webhook + # performance work obviates the locality motivation. Leave disabled. localK8sAPIEndpoint: - # incompatable with Deployment mode - enabled: true - # nodeSelector for the DaemonSet - # Talos uses empty value: "node-role.kubernetes.io/control-plane": "" - # Generic k8s (k3s, kubeadm) uses: "node-role.kubernetes.io/control-plane": "true" - nodeSelector: - node-role.kubernetes.io/control-plane: "" - # if enabled, replace DaemonSet by Deployment - deployment: enabled: false - replicas: 1 From 3a8359fa73f5cd96d8363c48d31414d8cadf3447 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 18:59:43 +0300 Subject: [PATCH 240/250] chore(kubernetes): regenerate deepcopy after merge Run `make generate` to pick up the Images type added in main. Signed-off-by: Arsolitt --- .../v1alpha1/kubernetes/zz_generated.deepcopy.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go b/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go index 3f1cb5ce..e021fbaa 100644 --- a/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go +++ b/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go @@ -136,6 +136,7 @@ func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) { } in.Addons.DeepCopyInto(&out.Addons) in.ControlPlane.DeepCopyInto(&out.ControlPlane) + out.Images = in.Images } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigSpec. @@ -277,6 +278,21 @@ func (in *HAMiAddon) DeepCopy() *HAMiAddon { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Images) DeepCopyInto(out *Images) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Images. +func (in *Images) DeepCopy() *Images { + if in == nil { + return nil + } + out := new(Images) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IngressNginxAddon) DeepCopyInto(out *IngressNginxAddon) { *out = *in From 80b86b0c1a39266488b84473724a54b0d3ad7d9c Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 19:02:35 +0300 Subject: [PATCH 241/250] chore(kubernetes): regenerate CRD after merge conflict resolution Per-package `make generate` corrects keysOrder placement and openAPISchema formatting in the kubernetes CRD definition. Signed-off-by: Arsolitt --- api/apps/v1alpha1/kubernetes/types.go | 2 +- packages/system/kubernetes-rd/cozyrds/kubernetes.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/apps/v1alpha1/kubernetes/types.go b/api/apps/v1alpha1/kubernetes/types.go index e0143445..88a3c97e 100644 --- a/api/apps/v1alpha1/kubernetes/types.go +++ b/api/apps/v1alpha1/kubernetes/types.go @@ -6,7 +6,7 @@ package kubernetes import ( resource "k8s.io/apimachinery/pkg/api/resource" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" k8sRuntime "k8s.io/apimachinery/pkg/runtime" ) diff --git a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml index b4ada238..f81eaff0 100644 --- a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml +++ b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml @@ -22,7 +22,7 @@ spec: singular: kubernetes plural: kuberneteses openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"ephemeralStorage":"20Gi","gpus":[],"instanceType":"u1.medium","maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"]}},"additionalProperties":{"type":"object","required":["ephemeralStorage","instanceType","maxReplicas","minReplicas","resources"],"properties":{"ephemeralStorage":{"description":"Ephemeral storage size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string"}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium"},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `.` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","hami","ingressNginx","monitoringAgents","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"hami":{"description":"HAMi GPU virtualization middleware.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable HAMi (requires GPU Operator).","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"large","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"images":{"description":"Optional image overrides for air-gapped or rate-limited registries.","type":"object","default":{},"properties":{"waitForKubeconfig":{"description":"Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.","type":"string","default":""}}}}} + {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"ephemeralStorage":"20Gi","gpus":[],"instanceType":"u1.medium","maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"]}},"additionalProperties":{"type":"object","required":["ephemeralStorage","instanceType","maxReplicas","minReplicas","resources"],"properties":{"ephemeralStorage":{"description":"Ephemeral storage size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string"}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium"},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `.` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","hami","ingressNginx","monitoringAgents","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"hami":{"description":"HAMi GPU virtualization middleware.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable HAMi (requires GPU Operator).","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"large","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"images":{"description":"Optional image overrides for air-gapped or rate-limited registries.","type":"object","default":{},"properties":{"waitForKubeconfig":{"description":"Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.","type":"string","default":""}}}}} release: prefix: kubernetes- labels: @@ -40,7 +40,7 @@ spec: tags: - compute icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODFfMjg0NSkiLz4KPHBhdGggZD0iTTcxLjk5NjggMTlDNzAuMzAzOSAxOS4wMDAyIDY4LjkzMTIgMjAuNTMyMiA2OC45MzE0IDIyLjQyMjFDNjguOTMxNCAyMi40NTExIDY4LjkzNzMgMjIuNDc4OCA2OC45Mzc5IDIyLjUwNzZDNjguOTM1NCAyMi43NjQ0IDY4LjkyMzEgMjMuMDczNyA2OC45MzE0IDIzLjI5NzNDNjguOTcxNyAyNC4zODczIDY5LjIwODIgMjUuMjIxNiA2OS4zNTA2IDI2LjIyNThDNjkuNjA4NCAyOC4zNzUyIDY5LjgyNDUgMzAuMTU2OSA2OS42OTEyIDMxLjgxM0M2OS41NjE1IDMyLjQzNzUgNjkuMTAzNyAzMy4wMDg2IDY4LjY5NTYgMzMuNDA1Nkw2OC42MjM1IDM0LjcwODZDNjYuNzgzOSAzNC44NjE3IDY0LjkzMTkgMzUuMTQyMSA2My4wODIxIDM1LjU2NDFDNTUuMTIyNiAzNy4zNzk4IDQ4LjI2OTUgNDEuNDk5MSA0My4wNTIgNDcuMDYwOUM0Mi43MTM0IDQ2LjgyODggNDIuMTIxMSA0Ni40MDE5IDQxLjk0NSA0Ni4yNzEyQzQxLjM5NzcgNDYuMzQ1NCA0MC44NDQ1IDQ2LjUxNTEgNDAuMTI0MSA0Ni4wOTM1QzM4Ljc1MjIgNDUuMTY1NyAzNy41MDI4IDQzLjg4NTEgMzUuOTkxIDQyLjM0MjRDMzUuMjk4MiA0MS42MDQ0IDM0Ljc5NjYgNDAuOTAxOCAzMy45NzM1IDQwLjE5MDRDMzMuNzg2NiA0MC4wMjg5IDMzLjUwMTQgMzkuODEwNCAzMy4yOTIzIDM5LjY0NDJDMzIuNjQ4OSAzOS4xMjg4IDMxLjg5IDM4Ljg2IDMxLjE1NyAzOC44MzQ4QzMwLjIxNDcgMzguODAyNCAyOS4zMDc1IDM5LjE3MjUgMjguNzEzOCAzOS45MjA2QzI3LjY1ODQgNDEuMjUwNiAyNy45OTYzIDQzLjI4MzMgMjkuNDY3MSA0NC40NjE0QzI5LjQ4MiA0NC40NzM0IDI5LjQ5NzkgNDQuNDgyNyAyOS41MTI5IDQ0LjQ5NDNDMjkuNzE1IDQ0LjY1ODkgMjkuOTYyNSA0NC44Njk4IDMwLjE0ODMgNDUuMDA3NkMzMS4wMjE3IDQ1LjY1NTUgMzEuODE5NSA0NS45ODcyIDMyLjY4OTcgNDYuNTAxNUMzNC41MjMxIDQ3LjYzOTEgMzYuMDQzIDQ4LjU4MjMgMzcuMjQ4NiA0OS43MTk2QzM3LjcxOTQgNTAuMjIzNyAzNy44MDE2IDUxLjExMjIgMzcuODY0MyA1MS40OTY0TDM4Ljg0NjggNTIuMzc4MkMzMy41ODcyIDYwLjMzMDggMzEuMTUzIDcwLjE1MzkgMzIuNTkxNSA4MC4xNjI3TDMxLjMwNzcgODAuNTM3OEMzMC45NjkzIDgwLjk3NjggMzAuNDkxMiA4MS42Njc2IDI5Ljk5MTEgODEuODczOEMyOC40MTM4IDgyLjM3MjkgMjYuNjM4NyA4Mi41NTYyIDI0LjQ5NTYgODIuNzgxOUMyMy40ODk0IDgyLjg2NiAyMi42MjEzIDgyLjgxNTggMjEuNTU0NiA4My4wMTg4QzIxLjMxOTggODMuMDYzNSAyMC45OTI3IDgzLjE0OTEgMjAuNzM1OCA4My4yMDk3QzIwLjcyNjkgODMuMjExNiAyMC43MTg2IDgzLjIxNDIgMjAuNzA5NiA4My4yMTYyQzIwLjY5NTYgODMuMjE5NSAyMC42NzcyIDgzLjIyNjMgMjAuNjYzOCA4My4yMjk0QzE4Ljg1NyA4My42NjggMTcuNjk2MyA4NS4zMzY1IDE4LjA2OTkgODYuOTgwNUMxOC40NDM3IDg4LjYyNDggMjAuMjA4NiA4OS42MjQ4IDIyLjAyNjIgODkuMjMxMkMyMi4wMzkzIDg5LjIyODIgMjIuMDU4NCA4OS4yMjc3IDIyLjA3MiA4OS4yMjQ2QzIyLjA5MjYgODkuMjE5OSAyMi4xMTA2IDg5LjIwOTkgMjIuMTMxIDg5LjIwNDlDMjIuMzg0NCA4OS4xNDkgMjIuNzAxOSA4OS4wODY4IDIyLjkyMzYgODkuMDI3MkMyMy45NzIzIDg4Ljc0NTEgMjQuNzMxOCA4OC4zMzA2IDI1LjY3NDYgODcuOTY3N0MyNy43MDI5IDg3LjIzNjggMjkuMzgyOCA4Ni42MjYyIDMxLjAxOTUgODYuMzg4M0MzMS43MDMgODYuMzM0NSAzMi40MjMyIDg2LjgxMiAzMi43ODE0IDg3LjAxMzRMMzQuMTE3NyA4Ni43ODMxQzM3LjE5MjYgOTYuMzYxMyA0My42MzY2IDEwNC4xMDMgNTEuNzk2MyAxMDguOTYxTDUxLjIzOTYgMTEwLjMwM0M1MS40NDAzIDExMC44MjQgNTEuNjYxNiAxMTEuNTMgNTEuNTEyMSAxMTIuMDQ1QzUwLjkxNzEgMTEzLjU5NSA0OS44OTggMTE1LjIzMSA0OC43Mzc0IDExNy4wNTVDNDguMTc1NSAxMTcuODk4IDQ3LjYwMDQgMTE4LjU1MiA0Ny4wOTM0IDExOS41MTZDNDYuOTcyIDExOS43NDcgNDYuODE3NSAxMjAuMTAyIDQ2LjcwMDQgMTIwLjM0NkM0NS45MTI1IDEyMi4wMzkgNDYuNDkwNCAxMjMuOTkgNDguMDAzOCAxMjQuNzIyQzQ5LjUyNjggMTI1LjQ1OSA1MS40MTcxIDEyNC42ODIgNTIuMjM1MiAxMjIuOTg1QzUyLjIzNjQgMTIyLjk4MiA1Mi4yNDA2IDEyMi45OCA1Mi4yNDE3IDEyMi45NzhDNTIuMjQyNiAxMjIuOTc2IDUyLjI0MDkgMTIyLjk3MyA1Mi4yNDE3IDEyMi45NzFDNTIuMzU4MiAxMjIuNzMxIDUyLjUyMzMgMTIyLjQxNSA1Mi42MjE2IDEyMi4xODhDNTMuMDU2IDEyMS4xODkgNTMuMjAwNSAxMjAuMzMyIDUzLjUwNTkgMTE5LjM2NUM1NC4zMTcgMTE3LjMxOCA1NC43NjI2IDExNS4xNyA1NS44NzkxIDExMy44MzJDNTYuMTg0OSAxMTMuNDY2IDU2LjY4MzMgMTEzLjMyNSA1Ny4yMDAxIDExMy4xODZMNTcuODk0NCAxMTEuOTIyQzY1LjAwOCAxMTQuNjY1IDcyLjk3MDUgMTE1LjQwMiA4MC45MjQ1IDExMy41ODdDODIuNzM5MSAxMTMuMTczIDg0LjQ5MDggMTEyLjYzNyA4Ni4xODQzIDExMS45OTRDODYuMzc5NCAxMTIuMzQyIDg2Ljc0MiAxMTMuMDExIDg2LjgzOTMgMTEzLjE3OUM4Ny4zNjQ0IDExMy4zNTEgODcuOTM3NyAxMTMuNDM5IDg4LjQwNDcgMTE0LjEzM0M4OS4yNDAxIDExNS41NjcgODkuODExNCAxMTcuMjYzIDkwLjUwNzMgMTE5LjMxMkM5MC44MTI4IDEyMC4yNzkgOTAuOTYzOCAxMjEuMTM2IDkxLjM5ODEgMTIyLjEzNkM5MS40OTcxIDEyMi4zNjMgOTEuNjYxNCAxMjIuNjg0IDkxLjc3OCAxMjIuOTI1QzkyLjU5NDQgMTI0LjYyOCA5NC40OTA3IDEyNS40MDcgOTYuMDE1OSAxMjQuNjY5Qzk3LjUyOTIgMTIzLjkzNyA5OC4xMDc3IDEyMS45ODYgOTcuMzE5NCAxMjAuMjkzQzk3LjIwMjMgMTIwLjA0OSA5Ny4wNDEyIDExOS42OTUgOTYuOTE5OCAxMTkuNDY0Qzk2LjQxMjcgMTE4LjQ5OSA5NS44Mzc3IDExNy44NTIgOTUuMjc1OCAxMTcuMDA5Qzk0LjExNTIgMTE1LjE4NSA5My4xNTI2IDExMy42NyA5Mi41NTc1IDExMi4xMkM5Mi4zMDg3IDExMS4zMiA5Mi41OTk1IDExMC44MjMgOTIuNzkzMyAxMTAuMzAzQzkyLjY3NzIgMTEwLjE3IDkyLjQyODggMTA5LjQxNCA5Mi4yODI0IDEwOS4wNTlDMTAwLjc2MiAxMDQuMDI5IDEwNy4wMTcgOTUuOTk4NSAxMDkuOTU1IDg2LjcyMzlDMTEwLjM1MSA4Ni43ODY1IDExMS4wNDEgODYuOTA5MSAxMTEuMjY1IDg2Ljk1NDJDMTExLjcyNiA4Ni42NDg3IDExMi4xNDkgODYuMjUwMSAxMTIuOTgxIDg2LjMxNTlDMTE0LjYxNyA4Ni41NTM3IDExNi4yOTcgODcuMTY0NSAxMTguMzI2IDg3Ljg5NTNDMTE5LjI2OCA4OC4yNTgxIDEyMC4wMjggODguNjc5MyAxMjEuMDc3IDg4Ljk2MTRDMTIxLjI5OCA4OS4wMjEgMTIxLjYxNiA4OS4wNzY2IDEyMS44NjkgODkuMTMyNUMxMjEuODg5IDg5LjEzNzUgMTIxLjkwOCA4OS4xNDc1IDEyMS45MjggODkuMTUyMkMxMjEuOTQyIDg5LjE1NTMgMTIxLjk2MSA4OS4xNTU4IDEyMS45NzQgODkuMTU4OEMxMjMuNzkyIDg5LjU1MiAxMjUuNTU3IDg4LjU1MjYgMTI1LjkzIDg2LjkwODFDMTI2LjMwMyA4NS4yNjQxIDEyNS4xNDMgODMuNTk1MiAxMjMuMzM2IDgzLjE1N0MxMjMuMDc0IDgzLjA5NyAxMjIuNzAxIDgyLjk5NSAxMjIuNDQ2IDgyLjk0NjVDMTIxLjM3OSA4Mi43NDM1IDEyMC41MTEgODIuNzkzNSAxMTkuNTA1IDgyLjcwOTVDMTE3LjM2MSA4Mi40ODM5IDExNS41ODYgODIuMzAwNCAxMTQuMDA5IDgxLjgwMTRDMTEzLjM2NiA4MS41NTA3IDExMi45MDggODAuNzgxOSAxMTIuNjg2IDgwLjQ2NTVMMTExLjQ0OCA4MC4xMDM1QzExMi4wOSA3NS40MzggMTExLjkxNyA3MC41ODI1IDExMC44MDYgNjUuNzI0M0MxMDkuNjg1IDYwLjgyMDggMTA3LjcwNCA1Ni4zMzYxIDEwNS4wNjIgNTIuMzg0OEMxMDUuMzc5IDUyLjA5NDggMTA1Ljk3OSA1MS41NjEyIDEwNi4xNDkgNTEuNDA0M0MxMDYuMTk5IDUwLjg1MTcgMTA2LjE1NiA1MC4yNzIyIDEwNi43MjUgNDkuNjYwM0MxMDcuOTMxIDQ4LjUyMyAxMDkuNDUxIDQ3LjU3OTkgMTExLjI4NCA0Ni40NDIzQzExMi4xNTQgNDUuOTI3OSAxMTIuOTU5IDQ1LjU5NjQgMTEzLjgzMiA0NC45NDg0QzExNC4wMyA0NC44MDE5IDExNC4yOTkgNDQuNTY5OSAxMTQuNTA3IDQ0LjQwMjJDMTE1Ljk3NyA0My4yMjM3IDExNi4zMTYgNDEuMTkxMSAxMTUuMjYgMzkuODYxNEMxMTQuMjA0IDM4LjUzMTcgMTEyLjE1OSAzOC40MDY1IDExMC42ODggMzkuNTg1QzExMC40NzkgMzkuNzUxNiAxMTAuMTk1IDM5Ljk2ODggMTEwLjAwNyA0MC4xMzEyQzEwOS4xODQgNDAuODQyNiAxMDguNjc2IDQxLjU0NTIgMTA3Ljk4MyA0Mi4yODMyQzEwNi40NzEgNDMuODI1OSAxMDUuMjIyIDQ1LjExMyAxMDMuODUgNDYuMDQwOUMxMDMuMjU1IDQ2LjM4ODUgMTAyLjM4NSA0Ni4yNjgyIDEwMS45OSA0Ni4yNDQ5TDEwMC44MjQgNDcuMDgwNkM5NC4xNzUzIDQwLjA3NjMgODUuMTIzNSAzNS41OTgyIDc1LjM3NjYgMzQuNzI4M0M3NS4zNDk0IDM0LjMxNzkgNzUuMzEzNyAzMy41NzYxIDc1LjMwNDYgMzMuMzUyOUM3NC45MDU2IDMyLjk2OTMgNzQuNDIzNSAzMi42NDE4IDc0LjMwMjQgMzEuODEzQzc0LjE2OTEgMzAuMTU2OSA3NC4zOTE3IDI4LjM3NTIgNzQuNjQ5NiAyNi4yMjU4Qzc0Ljc5MTkgMjUuMjIxNiA3NS4wMjg0IDI0LjM4NzMgNzUuMDY4OCAyMy4yOTczQzc1LjA3OCAyMy4wNDk1IDc1LjA2MzIgMjIuNjkgNzUuMDYyMiAyMi40MjIxQzc1LjA2MiAyMC41MzIyIDczLjY4OTggMTguOTk5OCA3MS45OTY4IDE5Wk02OC4xNTg1IDQyLjg4ODZMNjcuMjQ4IDU5LjA0NDdMNjcuMTgyNSA1OS4wNzc2QzY3LjEyMTQgNjAuNTIyOSA2NS45Mzc1IDYxLjY3NyA2NC40ODM5IDYxLjY3N0M2My44ODg0IDYxLjY3NyA2My4zMzg4IDYxLjQ4NDkgNjIuODkyMiA2MS4xNTcxTDYyLjg2NiA2MS4xNzAzTDQ5LjY4MDcgNTEuNzc5NEM1My43MzMxIDQ3Ljc3NTkgNTguOTE2NCA0NC44MTcyIDY0Ljg5IDQzLjQ1NDZDNjUuOTgxMiA0My4yMDU2IDY3LjA3MTkgNDMuMDIwOSA2OC4xNTg1IDQyLjg4ODZaTTc1Ljg0MTcgNDIuODg4NkM4Mi44MTU5IDQzLjc1MDQgODkuMjY1NyA0Ni45MjMyIDk0LjIwODEgNTEuNzg2TDgxLjEwOCA2MS4xMTc2TDgxLjA2MjEgNjEuMDk3OUM3OS44OTk0IDYxLjk1MTIgNzguMjYxMSA2MS43Mzk0IDc3LjM1NDggNjAuNTk3OEM3Ni45ODM1IDYwLjEzMDEgNzYuNzg4NyA1OS41ODAxIDc2Ljc2NTMgNTkuMDI0OUw3Ni43NTIyIDU5LjAxODRMNzUuODQxNyA0Mi44ODg2Wk00NC44OTkxIDU3LjgxNEw1Ni45MzgyIDY4LjYzM0w1Ni45MjUxIDY4LjY5ODhDNTguMDExNyA2OS42NDc5IDU4LjE3MiA3MS4yOTQ5IDU3LjI2NTcgNzIuNDM2OEM1Ni44OTQ0IDcyLjkwNDUgNTYuMzk3NSA3My4yMTgyIDU1Ljg2MzkgNzMuMzY0N0w1NS44NTA4IDczLjQxNzNMNDAuNDE4OCA3Ny44OTIzQzM5LjYzMzQgNzAuNjc2NSA0MS4zMjYxIDYzLjY2MjEgNDQuODk5MSA1Ny44MTRaTTk5LjAwOTQgNTcuODIwNkMxMDAuNzk4IDYwLjczMzYgMTAyLjE1MyA2My45ODcxIDEwMi45NTkgNjcuNTE0M0MxMDMuNzU2IDcwLjk5OTEgMTAzLjk1NiA3NC40Nzc4IDEwMy42MjcgNzcuODM5N0w4OC4xMTY2IDczLjM1MTVMODguMTAzNSA3My4yODU3Qzg2LjcxNDUgNzIuOTA0MyA4NS44NjA5IDcxLjQ4NDggODYuMTg0MyA3MC4wNjExQzg2LjMxNjggNjkuNDc3OCA4Ni42MjQ5IDY4Ljk4NDQgODcuMDQyMyA2OC42MTk4TDg3LjAzNTggNjguNTg2OUw5OS4wMDk0IDU3LjgyMDZaTTY5LjUyNzQgNjkuNDY4OEg3NC40NTk2TDc3LjUyNTEgNzMuMzE4Nkw3Ni40MjQ3IDc4LjEyMjZMNzEuOTk2OCA4MC4yNjE0TDY3LjU1NTggNzguMTE2MUw2Ni40NTU0IDczLjMxMkw2OS41Mjc0IDY5LjQ2ODhaTTg1LjMzOTMgODIuNjQzN0M4NS41NDg5IDgyLjYzMzEgODUuNzU3NiA4Mi42NTIgODUuOTYxNiA4Mi42ODk4TDg1Ljk4NzggODIuNjU2OUwxMDEuOTUgODUuMzY4MkM5OS42MTQyIDkxLjk2MjQgOTUuMTQ0IDk3LjY3NSA4OS4xNzExIDEwMS40OThMODIuOTc0NyA4Ni40NjA2TDgyLjk5NDQgODYuNDM0M0M4Mi40MjUyIDg1LjEwNTUgODIuOTk0OCA4My41NDcyIDg0LjMwNDQgODIuOTEzNUM4NC42Mzk3IDgyLjc1MTMgODQuOTkgODIuNjYxNCA4NS4zMzkzIDgyLjY0MzdaTTU4LjUyOTggODIuNzA5NUM1OS43NDggODIuNzI2NyA2MC44NDA2IDgzLjU3NjEgNjEuMTIzNyA4NC44MjJDNjEuMjU2MiA4NS40MDUyIDYxLjE5MTcgODUuOTgzMSA2MC45NzMgODYuNDkzNUw2MS4wMTg5IDg2LjU1MjdMNTQuODg4IDEwMS40MzlDNDkuMTU1OSA5Ny43NDMyIDQ0LjU5MDQgOTIuMjA5OSA0Mi4xNDgxIDg1LjQyMDhMNTcuOTczMSA4Mi43MjI3TDU3Ljk5OTMgODIuNzU1NkM1OC4xNzYzIDgyLjcyMjkgNTguMzU1OCA4Mi43MDcxIDU4LjUyOTggODIuNzA5NVpNNzEuODk4NiA4OS4yMzEyQzcyLjMyMjkgODkuMjE1NSA3Mi43NTM0IDg5LjMwMyA3My4xNjI3IDg5LjUwMUM3My42OTkyIDg5Ljc2MDYgNzQuMTEzNiA5MC4xNjkyIDc0LjM3NDUgOTAuNjU5Mkg3NC40MzM0TDgyLjIzNDYgMTA0LjgyMUM4MS4yMjIxIDEwNS4xNjIgODAuMTgxMyAxMDUuNDU0IDc5LjExNjcgMTA1LjY5N0M3My4xNTA1IDEwNy4wNTggNjcuMjAzMiAxMDYuNjQ1IDYxLjgxOCAxMDQuODAyTDY5LjU5OTUgOTAuNjY1OEg2OS42MTI2QzcwLjA3OTUgODkuNzg4OCA3MC45NjUgODkuMjY1NiA3MS44OTg2IDg5LjIzMTJaIiBmaWxsPSJ3aGl0ZSIgc3Ryb2tlPSJ3aGl0ZSIgc3Ryb2tlLXdpZHRoPSIwLjI1Ii8+CjxkZWZzPgo8bGluZWFyR3JhZGllbnQgaWQ9InBhaW50MF9saW5lYXJfNjgxXzI4NDUiIHgxPSIxMCIgeTE9IjE1LjUiIHgyPSIxNDQiIHkyPSIxMzEuNSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiPgo8c3RvcCBzdG9wLWNvbG9yPSIjNEQ4N0ZGIi8+CjxzdG9wIG9mZnNldD0iMSIgc3RvcC1jb2xvcj0iIzA1NDdEMCIvPgo8L2xpbmVhckdyYWRpZW50Pgo8L2RlZnM+Cjwvc3ZnPgo= - keysOrder: [["apiVersion"],["appVersion"],["kind"],["metadata"],["metadata","name"],["spec","storageClass"],["spec","nodeGroups"],["spec","nodeGroups","md0"],["spec","nodeGroups","md0","minReplicas"],["spec","nodeGroups","md0","maxReplicas"],["spec","nodeGroups","md0","instanceType"],["spec","nodeGroups","md0","ephemeralStorage"],["spec","nodeGroups","md0","roles"],["spec","nodeGroups","md0","resources"],["spec","nodeGroups","md0","gpus"],["spec","version"],["spec","host"],["spec","addons"],["spec","addons","certManager"],["spec","addons","certManager","enabled"],["spec","addons","certManager","valuesOverride"],["spec","addons","cilium"],["spec","addons","cilium","valuesOverride"],["spec","addons","gatewayAPI"],["spec","addons","gatewayAPI","enabled"],["spec","addons","ingressNginx"],["spec","addons","ingressNginx","enabled"],["spec","addons","ingressNginx","exposeMethod"],["spec","addons","ingressNginx","hosts"],["spec","addons","ingressNginx","valuesOverride"],["spec","addons","gpuOperator"],["spec","addons","gpuOperator","enabled"],["spec","addons","gpuOperator","valuesOverride"],["spec","addons","fluxcd"],["spec","addons","fluxcd","enabled"],["spec","addons","fluxcd","valuesOverride"],["spec","addons","monitoringAgents"],["spec","addons","monitoringAgents","enabled"],["spec","addons","monitoringAgents","valuesOverride"],["spec","addons","verticalPodAutoscaler"],["spec","addons","verticalPodAutoscaler","valuesOverride"],["spec","addons","velero"],["spec","addons","velero","enabled"],["spec","addons","velero","valuesOverride"],["spec","addons","coredns"],["spec","addons","coredns","valuesOverride"],["spec","controlPlane"],["spec","controlPlane","replicas"],["spec","controlPlane","apiServer"],["spec","controlPlane","apiServer","resources"],["spec","controlPlane","apiServer","resourcesPreset"],["spec","controlPlane","controllerManager"],["spec","controlPlane","controllerManager","resources"],["spec","controlPlane","controllerManager","resourcesPreset"],["spec","controlPlane","scheduler"],["spec","controlPlane","scheduler","resources"],["spec","controlPlane","scheduler","resourcesPreset"],["spec","controlPlane","konnectivity"],["spec","controlPlane","konnectivity","server"],["spec","controlPlane","konnectivity","server","resources"],["spec","controlPlane","konnectivity","server","resourcesPreset"],["spec","addons","hami"],["spec","addons","hami","enabled"],["spec","addons","hami","valuesOverride"],["spec","images"],["spec","images","waitForKubeconfig"]] + keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "nodeGroups"], ["spec", "nodeGroups", "md0"], ["spec", "nodeGroups", "md0", "minReplicas"], ["spec", "nodeGroups", "md0", "maxReplicas"], ["spec", "nodeGroups", "md0", "instanceType"], ["spec", "nodeGroups", "md0", "ephemeralStorage"], ["spec", "nodeGroups", "md0", "roles"], ["spec", "nodeGroups", "md0", "resources"], ["spec", "nodeGroups", "md0", "gpus"], ["spec", "version"], ["spec", "host"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "hami"], ["spec", "addons", "hami", "enabled"], ["spec", "addons", "hami", "valuesOverride"], ["spec", "addons", "fluxcd"], ["spec", "addons", "fluxcd", "enabled"], ["spec", "addons", "fluxcd", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"], ["spec", "images"], ["spec", "images", "waitForKubeconfig"]] secrets: exclude: [] include: From 805b3b17cbc4d89206203bc24b45c1412c29a2c8 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Tue, 28 Apr 2026 19:11:26 +0300 Subject: [PATCH 242/250] fix(kubernetes): add CI values to GPU Operator HAMi tests Tests need _namespace.etcd from values-ci.yaml after the merge introduced an etcd-namespace guard on the gpu-operator HelmRelease condition. Signed-off-by: Arsolitt --- packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml index 4fa754fb..44528470 100644 --- a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml +++ b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml @@ -1,6 +1,8 @@ suite: GPU Operator HelmRelease HAMi integration tests templates: - templates/helmreleases/gpu-operator.yaml +values: + - values-ci.yaml tests: - it: should disable devicePlugin when hami is enabled set: From 4541c20e3403a5eda85a15ae3a02b80e6cc2ebbd Mon Sep 17 00:00:00 2001 From: "cozystack-ci[bot]" <274107086+cozystack-ci[bot]@users.noreply.github.com> Date: Wed, 29 Apr 2026 04:38:30 +0000 Subject: [PATCH 243/250] docs: add changelog for v1.3.1 Signed-off-by: cozystack-ci[bot] <274107086+cozystack-ci[bot]@users.noreply.github.com> --- docs/changelogs/v1.3.1.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 docs/changelogs/v1.3.1.md diff --git a/docs/changelogs/v1.3.1.md b/docs/changelogs/v1.3.1.md new file mode 100644 index 00000000..55e6a796 --- /dev/null +++ b/docs/changelogs/v1.3.1.md @@ -0,0 +1,38 @@ +# v1.3.1 (2026-04-29) + +This is a patch release containing fixes and minor improvements. + +## Fixes and Improvements + +* **[linstor-gui] Add package for LINBIT linstor-gui web UI**: [linstor-gui] Add package for LINBIT linstor-gui web UI ([**@myasnikovdaniil**](https://github.com/cozystack/cozystack/pull/2382) in #2382) +* **fix(kamaji): increase memory limits and add startup probe**: fix(kamaji): increase memory limits and add startup probe ([**@IvanHunters**](https://github.com/cozystack/cozystack/pull/2421) in #2421) +* **fix(backups): move velero-configmap Role to velero chart**: fix(backups): move velero-configmap Role to velero chart ([**@myasnikovdaniil**](https://github.com/cozystack/cozystack/pull/2459) in #2459) +* **fix(etcd): remove destructive post-upgrade cert-regeneration hook**: fix(etcd): remove destructive post-upgrade cert-regeneration hook ([**@myasnikovdaniil**](https://github.com/cozystack/cozystack/pull/2462) in #2462) +* **[Backport release-1.3] fix(backups): move velero-configmap Role to velero chart**: [Backport release-1.3] fix(backups): move velero-configmap Role to velero chart ([**@IvanHunters**](https://github.com/cozystack/cozystack/pull/2467) in #2467) +* **fix(api): prevent IDOR in TenantNamespace Get and Watch handlers**: fix(api): prevent IDOR in TenantNamespace Get and Watch handlers ([**@IvanHunters**](https://github.com/cozystack/cozystack/pull/2471) in #2471) +* **[Backport release-1.3] fix(kamaji): increase memory limits and add startup probe**: [Backport release-1.3] fix(kamaji): increase memory limits and add startup probe ([**@app/github-actions**](https://github.com/cozystack/cozystack/pull/2491) in #2491) +* **feat(linstor): bump linstor-csi to v1.10.6 with Protocol-C dual-attach fix**: feat(linstor): bump linstor-csi to v1.10.6 with Protocol-C dual-attach fix ([**@kvaps**](https://github.com/cozystack/cozystack/pull/2496) in #2496) +* **build(linstor): include linstor-gui in root image build target**: build(linstor): include linstor-gui in root image build target ([**@myasnikovdaniil**](https://github.com/cozystack/cozystack/pull/2498) in #2498) +* **[Backport release-1.3] feat(linstor): bump linstor-csi to v1.10.6 with Protocol-C dual-attach fix**: [Backport release-1.3] feat(linstor): bump linstor-csi to v1.10.6 with Protocol-C dual-attach fix ([**@app/github-actions**](https://github.com/cozystack/cozystack/pull/2505) in #2505) +* **[Backport release-1.3] fix(etcd): remove destructive post-upgrade cert-regeneration hook**: [Backport release-1.3] fix(etcd): remove destructive post-upgrade cert-regeneration hook ([**@app/github-actions**](https://github.com/cozystack/cozystack/pull/2511) in #2511) +* **[Backport release-1.3] build(linstor): include linstor-gui in root image build target**: [Backport release-1.3] build(linstor): include linstor-gui in root image build target ([**@app/github-actions**](https://github.com/cozystack/cozystack/pull/2518) in #2518) +* **[Backport release-1.3] fix(api): prevent IDOR in TenantNamespace Get and Watch handlers**: [Backport release-1.3] fix(api): prevent IDOR in TenantNamespace Get and Watch handlers ([**@app/github-actions**](https://github.com/cozystack/cozystack/pull/2524) in #2524) +* **fix namespaces creation**: fix namespaces creation ([**@kvaps**](https://github.com/cozystack/cozystack/pull/435) in #435) + +## Documentation + +* **[website] [docs] Update managed apps reference for v1.3.0**: Documentation updates ([**@myasnikovdaniil**](https://github.com/cozystack/website/pull/507) in cozystack/website#507) +* **[website] feat(blog): add Cozystack 1.3 release announcement**: Documentation updates ([**@tym83**](https://github.com/cozystack/website/pull/512) in cozystack/website#512) +* **[website] fix(telemetry): use April 2026 data as placeholder for quarter and year periods**: Documentation updates ([**@tym83**](https://github.com/cozystack/website/pull/511) in cozystack/website#511) +* **[website] fix(oss-health): correct April 2026 telemetry snapshot and pause fetch cron**: Documentation updates ([**@tym83**](https://github.com/cozystack/website/pull/508) in cozystack/website#508) + +## Contributors + +We thank the following contributors for this release: + +* [**@IvanHunters**](https://github.com/IvanHunters) +* [**@app/github-actions**](https://github.com/app/github-actions) +* [**@kvaps**](https://github.com/kvaps) +* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) + +**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.3.0...v1.3.1 From 875a940033fea97a4800797989b0e004bbb8eca4 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 29 Apr 2026 12:22:08 +0500 Subject: [PATCH 244/250] docs(changelog): rewrite v1.3.1 with the actual release contents The AI-generated v1.3.1 changelog (#2480) was generated from git log v1.3.0..main rather than git log v1.3.0..v1.3.1, because the workflow checked out main while the v1.3.1 tag points to release-1.3. As a result the changelog included: - 8 PRs that were merged to main but never shipped in v1.3.1 - 6 backport PRs that were merged to release-1.3 *after* v1.3.1 was tagged - Both originals and their backports as separate duplicate entries - A 2024 PR (#435) that has nothing to do with this range - Generic "Documentation updates" placeholders for website entries - Title duplicated as both the bold label and the description (`* **fix(...): X**: fix(...): X (...)`) - The cozystack-ci bot listed as a human contributor The actual v1.3.1 release range (v1.3.0..v1.3.1) contains exactly one user-facing change: 41bcb0be [Backport release-1.3] fix(backups): move velero-configmap Role to velero chart (#2467) which is the backport of #2459 (myasnikovdaniil) shipped via #2467 (IvanHunters). This commit replaces the contents of docs/changelogs/v1.3.1.md with that one entry, the matching two-person contributors list, and the standard footer. The workflow + docs fixes that prevent this regression for future patch releases will land in a separate PR against main. Signed-off-by: Myasnikov Daniil --- docs/changelogs/v1.3.1.md | 34 ++++++++-------------------------- 1 file changed, 8 insertions(+), 26 deletions(-) diff --git a/docs/changelogs/v1.3.1.md b/docs/changelogs/v1.3.1.md index 55e6a796..e8318ff4 100644 --- a/docs/changelogs/v1.3.1.md +++ b/docs/changelogs/v1.3.1.md @@ -1,38 +1,20 @@ -# v1.3.1 (2026-04-29) + -This is a patch release containing fixes and minor improvements. +# v1.3.1 (2026-04-23) -## Fixes and Improvements +A patch release with a single bug fix in the backup subsystem. -* **[linstor-gui] Add package for LINBIT linstor-gui web UI**: [linstor-gui] Add package for LINBIT linstor-gui web UI ([**@myasnikovdaniil**](https://github.com/cozystack/cozystack/pull/2382) in #2382) -* **fix(kamaji): increase memory limits and add startup probe**: fix(kamaji): increase memory limits and add startup probe ([**@IvanHunters**](https://github.com/cozystack/cozystack/pull/2421) in #2421) -* **fix(backups): move velero-configmap Role to velero chart**: fix(backups): move velero-configmap Role to velero chart ([**@myasnikovdaniil**](https://github.com/cozystack/cozystack/pull/2459) in #2459) -* **fix(etcd): remove destructive post-upgrade cert-regeneration hook**: fix(etcd): remove destructive post-upgrade cert-regeneration hook ([**@myasnikovdaniil**](https://github.com/cozystack/cozystack/pull/2462) in #2462) -* **[Backport release-1.3] fix(backups): move velero-configmap Role to velero chart**: [Backport release-1.3] fix(backups): move velero-configmap Role to velero chart ([**@IvanHunters**](https://github.com/cozystack/cozystack/pull/2467) in #2467) -* **fix(api): prevent IDOR in TenantNamespace Get and Watch handlers**: fix(api): prevent IDOR in TenantNamespace Get and Watch handlers ([**@IvanHunters**](https://github.com/cozystack/cozystack/pull/2471) in #2471) -* **[Backport release-1.3] fix(kamaji): increase memory limits and add startup probe**: [Backport release-1.3] fix(kamaji): increase memory limits and add startup probe ([**@app/github-actions**](https://github.com/cozystack/cozystack/pull/2491) in #2491) -* **feat(linstor): bump linstor-csi to v1.10.6 with Protocol-C dual-attach fix**: feat(linstor): bump linstor-csi to v1.10.6 with Protocol-C dual-attach fix ([**@kvaps**](https://github.com/cozystack/cozystack/pull/2496) in #2496) -* **build(linstor): include linstor-gui in root image build target**: build(linstor): include linstor-gui in root image build target ([**@myasnikovdaniil**](https://github.com/cozystack/cozystack/pull/2498) in #2498) -* **[Backport release-1.3] feat(linstor): bump linstor-csi to v1.10.6 with Protocol-C dual-attach fix**: [Backport release-1.3] feat(linstor): bump linstor-csi to v1.10.6 with Protocol-C dual-attach fix ([**@app/github-actions**](https://github.com/cozystack/cozystack/pull/2505) in #2505) -* **[Backport release-1.3] fix(etcd): remove destructive post-upgrade cert-regeneration hook**: [Backport release-1.3] fix(etcd): remove destructive post-upgrade cert-regeneration hook ([**@app/github-actions**](https://github.com/cozystack/cozystack/pull/2511) in #2511) -* **[Backport release-1.3] build(linstor): include linstor-gui in root image build target**: [Backport release-1.3] build(linstor): include linstor-gui in root image build target ([**@app/github-actions**](https://github.com/cozystack/cozystack/pull/2518) in #2518) -* **[Backport release-1.3] fix(api): prevent IDOR in TenantNamespace Get and Watch handlers**: [Backport release-1.3] fix(api): prevent IDOR in TenantNamespace Get and Watch handlers ([**@app/github-actions**](https://github.com/cozystack/cozystack/pull/2524) in #2524) -* **fix namespaces creation**: fix namespaces creation ([**@kvaps**](https://github.com/cozystack/cozystack/pull/435) in #435) +## Fixes -## Documentation - -* **[website] [docs] Update managed apps reference for v1.3.0**: Documentation updates ([**@myasnikovdaniil**](https://github.com/cozystack/website/pull/507) in cozystack/website#507) -* **[website] feat(blog): add Cozystack 1.3 release announcement**: Documentation updates ([**@tym83**](https://github.com/cozystack/website/pull/512) in cozystack/website#512) -* **[website] fix(telemetry): use April 2026 data as placeholder for quarter and year periods**: Documentation updates ([**@tym83**](https://github.com/cozystack/website/pull/511) in cozystack/website#511) -* **[website] fix(oss-health): correct April 2026 telemetry snapshot and pause fetch cron**: Documentation updates ([**@tym83**](https://github.com/cozystack/website/pull/508) in cozystack/website#508) +* **fix(backups): move velero-configmap Role to velero chart**: The `backupstrategy-controller` (a default package) declared a Role/RoleBinding scoped to the `cozy-velero` namespace for managing `ResourceModifier` ConfigMaps. On bundles where Velero was not enabled, that namespace did not exist and the HelmRelease failed with `namespaces "cozy-velero" not found`, blocking installation. The Role/RoleBinding now lives in the velero chart, so it is created only when velero is actually deployed. ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2459, backport #2467) ## Contributors -We thank the following contributors for this release: +Thanks to everyone who contributed to this patch release: * [**@IvanHunters**](https://github.com/IvanHunters) -* [**@app/github-actions**](https://github.com/app/github-actions) -* [**@kvaps**](https://github.com/kvaps) * [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) **Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.3.0...v1.3.1 From 1e0d8acb3553e0702a4c96a5775d4aadd6338c6b Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 29 Apr 2026 12:46:18 +0500 Subject: [PATCH 245/250] docs(changelog): add 1.3.1 entries for backports landed after auto-gen The auto-generated changelog only listed #2459/#2467 (velero-configmap Role move). Five additional PRs were backported and merged into release-1.3 between then and the v1.3.1 tag (2026-04-28): - #2471/#2524 - fix(api): IDOR in TenantNamespace Get/Watch - #2496/#2505 - feat(linstor): linstor-csi v1.10.6 (Protocol-C dual-attach) - #2462/#2511 - fix(etcd): remove destructive post-upgrade hook - #2421/#2491 - fix(kamaji): memory limits + startup probe - #2498/#2518 - build(linstor): wire linstor-gui into root build target Update the release date to match the actual tag (2026-04-28), rewrite the intro paragraph, and add @kvaps to contributors. Signed-off-by: Myasnikov Daniil --- docs/changelogs/v1.3.1.md | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/changelogs/v1.3.1.md b/docs/changelogs/v1.3.1.md index e8318ff4..0f149d0d 100644 --- a/docs/changelogs/v1.3.1.md +++ b/docs/changelogs/v1.3.1.md @@ -2,19 +2,36 @@ https://github.com/cozystack/cozystack/releases/tag/v1.3.1 --> -# v1.3.1 (2026-04-23) +# v1.3.1 (2026-04-28) -A patch release with a single bug fix in the backup subsystem. +Patch release covering a TenantNamespace IDOR fix in the API, a destructive `post-upgrade` hook removed from the etcd chart, kamaji controller stability, a `linstor-csi` bump that fixes live migration on Protocol-A/B DRBD resources, the missing `linstor-gui` build wiring, and a velero RBAC fix that unblocked installs on bundles without Velero. + +## Security + +* **fix(api): prevent IDOR in TenantNamespace Get and Watch handlers**: Two IDOR (Insecure Direct Object Reference) vulnerabilities allowed authenticated users to read TenantNamespace metadata they had no RoleBinding for. The `Get` and `Watch` handlers now go through a new `hasAccessToNamespace()` helper that lists RoleBindings scoped only to the target namespace (orders of magnitude faster than the previous all-cluster scan), returns `NotFound` instead of leaking existence on unauthorized access, and applies the same check on the `Watch` filter path. Includes regression tests for the unauthorized paths. ([**@IvanHunters**](https://github.com/IvanHunters) in #2471, backport #2524) + +## Features + +* **feat(linstor): bump linstor-csi to v1.10.6 with Protocol-C dual-attach fix**: Live migration of KubeVirt VMs on Protocol-A/B (async) DRBD volumes no longer fails with `Protocol C required`. `linstor-csi` v1.10.6 now installs a `Protocol=C` override on the resource-definition during dual-attach and reverts it on detach, so `replicated-async` StorageClasses and other Protocol-A/B resource groups support live migration without manual `drbdadm adjust` intervention. ([**@kvaps**](https://github.com/kvaps) in #2496, backport #2505) ## Fixes * **fix(backups): move velero-configmap Role to velero chart**: The `backupstrategy-controller` (a default package) declared a Role/RoleBinding scoped to the `cozy-velero` namespace for managing `ResourceModifier` ConfigMaps. On bundles where Velero was not enabled, that namespace did not exist and the HelmRelease failed with `namespaces "cozy-velero" not found`, blocking installation. The Role/RoleBinding now lives in the velero chart, so it is created only when velero is actually deployed. ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2459, backport #2467) +* **fix(etcd): remove destructive post-upgrade cert-regeneration hook**: The etcd chart ran a `post-upgrade` Helm hook on every upgrade that deleted etcd TLS Secrets (`etcd-ca-tls`, `etcd-peer-ca-tls`, `etcd-client-tls`, `etcd-peer-tls`, `etcd-server-tls`) and then deleted etcd pods, forcing cert-manager to re-issue the entire etcd CA chain. On clusters with Kamaji-managed tenant control planes this put every tenant `kube-apiserver` into CrashLoopBackOff until each DataStore was manually re-reconciled. The hook was a one-shot `2.6.0 → 2.6.1` migration that became a permanent footgun once chart versioning moved to `0.0.0+` (always `< 2.6.1` per semver) and after the underlying `rotationPolicy: Always` issue was fixed in `47d81f70`. The hook is now removed entirely. ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2462, backport #2511) + +* **fix(kamaji): increase memory limits and add startup probe**: The kamaji controller frequently entered CrashLoopBackOff due to OOMKills (exit 137) within ~20–25 seconds of startup, with the readiness probe failing while the controller was still finishing initialization. Memory limit raised from 500Mi to 512Mi, request from 100Mi to 256Mi, and a 60-second startup probe (12 attempts × 5s periods) is added so the controller has room to boot before liveness/readiness probes engage. ([**@IvanHunters**](https://github.com/IvanHunters) in #2421, backport #2491) + +## Build + +* **build(linstor): include linstor-gui in root image build target**: The `linstor-gui` package (added in #2382) was never wired into the root `Makefile`'s `build:` target, so CI never built or published the image. `ghcr.io/cozystack/cozystack/linstor-gui` returned `NAME_UNKNOWN` and `values.yaml` stayed pinned to `tag: 2.3.0` without a digest. The missing build line is added so the next CI run publishes the image and the per-package `Makefile` digest-pins `values.yaml` automatically. ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2498, backport #2518) + ## Contributors Thanks to everyone who contributed to this patch release: * [**@IvanHunters**](https://github.com/IvanHunters) +* [**@kvaps**](https://github.com/kvaps) * [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) **Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.3.0...v1.3.1 From ba8c1a0535d2e0462fb8269df0750ea6f57fdd0b Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Wed, 29 Apr 2026 11:13:12 +0300 Subject: [PATCH 246/250] docs(hami): drop HAMi#173 reference from glibc tracking issues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HAMi#173 was closed as "not planned" and only suggests a typo fix (< 2.3.0 → < 2.30); it does not establish the actual 2.34 boundary. HAMi-core#174 (symbol-level cause) and HAMi#1190 (empirical per-glibc behavior) cover the same ground accurately. Signed-off-by: Arsolitt --- packages/system/hami/README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/system/hami/README.md b/packages/system/hami/README.md index d648364d..aa80d48a 100644 --- a/packages/system/hami/README.md +++ b/packages/system/hami/README.md @@ -38,9 +38,8 @@ Most modern ML/AI base images (CUDA 12.x, PyTorch 2.x, TensorFlow 2.x) use Ubunt **Upstream tracking issues:** -- [HAMi-core#174](https://github.com/Project-HAMi/HAMi-core/issues/174) — `_dl_sym` removal breaks HAMi-core on glibc >= 2.34 -- [HAMi#1190](https://github.com/Project-HAMi/HAMi/issues/1190) — degraded isolation across glibc versions -- [HAMi#173](https://github.com/Project-HAMi/HAMi/issues/173) — documentation incorrectly states glibc < 2.30 (actual boundary is 2.34) +- [HAMi-core#174](https://github.com/Project-HAMi/HAMi-core/issues/174) — `_dl_sym` removal in glibc 2.34 breaks HAMi-core's CUDA symbol resolution at the symbol level +- [HAMi#1190](https://github.com/Project-HAMi/HAMi/issues/1190) — maintainer thread confirming the empirical per-glibc-version isolation behavior shown in the table above ### musl libc (Alpine) incompatibility From 9876acec2fafbadeb09741f70e01250171a42a48 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Wed, 29 Apr 2026 11:13:24 +0300 Subject: [PATCH 247/250] docs(hami): clarify gpu-operator devicePlugin override behavior The HAMi-driven default of disabling gpu-operator's device plugin is applied via valuesOverride and can be re-enabled by users for advanced topologies (mixed HAMi / vanilla NVIDIA device plugin pools). Document this explicitly so the README aligns with the existing template merge order. Signed-off-by: Arsolitt --- packages/system/hami/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/system/hami/README.md b/packages/system/hami/README.md index aa80d48a..68669ac7 100644 --- a/packages/system/hami/README.md +++ b/packages/system/hami/README.md @@ -57,7 +57,7 @@ addons: enabled: true ``` -When HAMi is enabled, GPU Operator's built-in device plugin is automatically disabled to avoid conflicts. +When HAMi is enabled, GPU Operator's built-in device plugin is automatically disabled to avoid conflicts. This default is preserved by setting `addons.gpuOperator.valuesOverride.gpu-operator.devicePlugin.enabled: false`; advanced topologies that partition GPU pools (e.g. some nodes use HAMi while others run the standard NVIDIA device plugin via node selectors) can re-enable it explicitly through `valuesOverride`. ### Requesting fractional GPU resources From d5caffcc8de0177b248899b03391c312d20dde54 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Wed, 29 Apr 2026 11:13:42 +0300 Subject: [PATCH 248/250] fix(kubernetes): gate HAMi HelmRelease on _namespace.etcd Match the pattern used by every other tenant-cluster addon HelmRelease (gpu-operator, cilium, fluxcd, ...): only render when the upstream etcd namespace is ready, not just when the addon is enabled. Add values-ci.yaml to the HAMi test suite so the new gate has the fixture it needs. Signed-off-by: Arsolitt --- packages/apps/kubernetes/templates/helmreleases/hami.yaml | 2 +- packages/apps/kubernetes/tests/hami_test.yaml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/apps/kubernetes/templates/helmreleases/hami.yaml b/packages/apps/kubernetes/templates/helmreleases/hami.yaml index 1f7a5358..f1538c7c 100644 --- a/packages/apps/kubernetes/templates/helmreleases/hami.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/hami.yaml @@ -1,4 +1,4 @@ -{{- if .Values.addons.hami.enabled }} +{{- if and .Values.addons.hami.enabled .Values._namespace.etcd }} {{- if not .Values.addons.gpuOperator.enabled }} {{- fail "addons.hami requires addons.gpuOperator to be enabled" }} {{- end }} diff --git a/packages/apps/kubernetes/tests/hami_test.yaml b/packages/apps/kubernetes/tests/hami_test.yaml index bdbe990f..27f7c75b 100644 --- a/packages/apps/kubernetes/tests/hami_test.yaml +++ b/packages/apps/kubernetes/tests/hami_test.yaml @@ -1,6 +1,8 @@ suite: HAMi HelmRelease tests templates: - templates/helmreleases/hami.yaml +values: + - values-ci.yaml tests: - it: should not render when hami is disabled set: From 9b5848ed268bfe3bc04b42bae38cc724c00a3bf0 Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Wed, 29 Apr 2026 11:13:55 +0300 Subject: [PATCH 249/250] fix(platform): register HAMi as kubernetes-application component MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The HelmRelease at packages/apps/kubernetes/templates/helmreleases/hami.yaml references chartRef.name 'cozystack-kubernetes-application-kubevirt-kubernetes-hami', but that component was missing from the kubernetes-application PackageSource. The HelmRelease would sit in Stalled: ArtifactNotFound at install time. Add kubernetes-hami next to kubernetes-gpu-operator under variant kubevirt, mirroring the existing pattern for every other tenant-cluster addon. The standalone cozystack.hami PackageSource is retained — same shape as gpu-operator, which is registered both standalone (for bundles/iaas) and as a kubernetes-application component (for tenant-cluster HelmReleases). Signed-off-by: Arsolitt --- packages/core/platform/sources/kubernetes-application.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/core/platform/sources/kubernetes-application.yaml b/packages/core/platform/sources/kubernetes-application.yaml index 9787cb19..088383ad 100644 --- a/packages/core/platform/sources/kubernetes-application.yaml +++ b/packages/core/platform/sources/kubernetes-application.yaml @@ -52,6 +52,8 @@ spec: path: system/cilium - name: kubernetes-gpu-operator path: system/gpu-operator + - name: kubernetes-hami + path: system/hami - name: kubernetes-vertical-pod-autoscaler path: system/vertical-pod-autoscaler - name: kubernetes-prometheus-operator-crds From 5d58d0f340f7157370305cc469317f4fa21944af Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Wed, 29 Apr 2026 11:14:41 +0300 Subject: [PATCH 250/250] chore(hami): document and partially automate vendor patches in Makefile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `update:` recipe now reproduces the top-level vendoring overrides (remove broken hami-dra subchart, clear Chart.yaml dependencies, strip dra/hami-dra/podSecurityPolicy from upstream values.yaml) after `helm pull`, so they no longer silently disappear on the next bump. Template-level patches (DRA guards in scheduler/*, indent fix in device-plugin/monitorservice.yaml) are documented with rationale and commit references — they remain a manual step because automating them would be fragile across upstream template restructures. Signed-off-by: Arsolitt --- packages/system/hami/Makefile | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/packages/system/hami/Makefile b/packages/system/hami/Makefile index b7a169c2..83663a66 100644 --- a/packages/system/hami/Makefile +++ b/packages/system/hami/Makefile @@ -4,8 +4,40 @@ export NAMESPACE=cozy-$(NAME) include ../../../hack/common-envs.mk include ../../../hack/package.mk +# When bumping the HAMi version, run `make update` and then review +# the resulting diff in `charts/hami/`. The recipe below reproduces the +# top-level vendoring overrides automatically: +# +# 1. Removes the broken hami-dra subchart. Upstream's NVIDIA DRA driver +# path requires kubelet DRA support that cozystack does not enable +# and has no upstream fix tracked. See commit 3c5521e. +# 2. Empties Chart.yaml dependencies and drops Chart.lock so Helm does +# not try to re-pull hami-dra at build time. See commit 2734dc0. +# 3. Strips dra/hami-dra/podSecurityPolicy blocks from the upstream +# values.yaml since the corresponding code paths are gone. PSP is +# removed from Kubernetes 1.25+ and is unused by cozystack. +# +# Template-level patches are NOT reproduced automatically: +# +# * Scheduler templates have `{{- if .Values.dra.enabled }}` blocks +# that need to be removed because the dra value is gone (commit +# 2734dc0 stripped them). +# * device-plugin/monitorservice.yaml uses `indent` with leading +# whitespace; it must be rewritten to `nindent` for the labels block +# to render correctly when devicePlugin.service.labels is set +# (commit 3685254). +# +# After `make update`, run `git diff -- charts/hami/templates/` and +# replay these template patches against the new upstream version, then +# verify with `helm unittest`. If upstream restructured the affected +# files, the patches may need to be redesigned rather than reapplied. + update: rm -rf charts helm repo add hami-charts https://project-hami.github.io/HAMi/ helm repo update hami-charts helm pull hami-charts/hami --untar --untardir charts + rm -rf charts/hami/charts/hami-dra + yq --inplace '.dependencies = []' charts/hami/Chart.yaml + rm -f charts/hami/Chart.lock + yq --inplace 'del(.dra) | del(.["hami-dra"]) | del(.podSecurityPolicy)' charts/hami/values.yaml