diff --git a/Makefile b/Makefile index 0e94c5e6..10a55314 100644 --- a/Makefile +++ b/Makefile @@ -82,11 +82,19 @@ test: make -C packages/core/testing apply make -C packages/core/testing test -unit-tests: helm-unit-tests bats-unit-tests +unit-tests: helm-unit-tests bats-unit-tests go-unit-tests helm-unit-tests: hack/helm-unit-tests.sh +# Scoped go test over the cozystack-api surface that this repo owns. Kept +# narrow intentionally - running `go test ./...` pulls in generated code +# round-trip suites whose behavior depends on tool versions outside this +# repo's control (kubebuilder, openapi-gen, etc.) and is better exercised +# from their generator workflows. +go-unit-tests: + go test ./pkg/registry/... ./pkg/config/... ./pkg/cmd/server/... + # Discover every hack/*.bats file that is NOT an e2e test and run it # through cozytest.sh. Drop a new *.bats file in hack/ and it is picked # up automatically on the next `make unit-tests` run. diff --git a/api/apps/v1alpha1/kubernetes/types.go b/api/apps/v1alpha1/kubernetes/types.go index 774fcdc5..ea4a2b64 100644 --- a/api/apps/v1alpha1/kubernetes/types.go +++ b/api/apps/v1alpha1/kubernetes/types.go @@ -36,6 +36,9 @@ type ConfigSpec struct { // Kubernetes control-plane configuration. // +kubebuilder:default:={} ControlPlane ControlPlane `json:"controlPlane"` + // Optional image overrides for air-gapped or rate-limited registries. + // +kubebuilder:default:={} + Images Images `json:"images"` } type APIServer struct { @@ -157,6 +160,12 @@ type GatewayAPIAddon struct { Enabled bool `json:"enabled"` } +type Images struct { + // Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag. + // +kubebuilder:default:="" + WaitForKubeconfig string `json:"waitForKubeconfig,omitempty"` +} + type IngressNginxAddon struct { // Enable the controller (requires nodes labeled `ingress-nginx`). // +kubebuilder:default:=false diff --git a/hack/admin-kubeconfig-invariant.bats b/hack/admin-kubeconfig-invariant.bats new file mode 100644 index 00000000..a4b98b00 --- /dev/null +++ b/hack/admin-kubeconfig-invariant.bats @@ -0,0 +1,145 @@ +#!/usr/bin/env bats +# ----------------------------------------------------------------------------- +# Chart-wide invariant for packages/apps/kubernetes: +# +# Every Deployment in this chart that mounts -admin-kubeconfig as a +# Secret volume MUST: +# - declare that volume optional: true (so kubelet does not FailedMount +# while Kamaji is still provisioning the Secret), AND +# - include the wait-for-kubeconfig init container (so the pod becomes +# Ready only after Kamaji publishes the Secret). +# +# The per-template unittests in packages/apps/kubernetes/tests/ lock in +# today's three Deployments (cluster-autoscaler, kccm, csi controller) by +# name. This invariant is stricter: any future Deployment added to this +# chart that mounts the same Secret but forgets the guard will fail here. +# +# Requires: helm, yq (mikefarah v4+), jq. All three are available on the +# project's CI runners and on the maintainer workstation. +# ----------------------------------------------------------------------------- + +@test "every Deployment mounting admin-kubeconfig has optional:true and wait-for-kubeconfig init" { + values_file="packages/apps/kubernetes/tests/values-ci.yaml" + [ -f "$values_file" ] + + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + + helm template invariant packages/apps/kubernetes \ + --namespace tenant-root \ + --values "$values_file" \ + 2>/dev/null > "$tmp/rendered.yaml" + + # yq streams one JSON object per input document. jq -s slurps the stream + # into an array so we can treat all Deployments as a single collection. + yq --output-format=json eval-all '.' "$tmp/rendered.yaml" \ + | jq -s --raw-output ' + map(select(.kind == "Deployment")) | + map({ + name: .metadata.name, + volumes: (.spec.template.spec.volumes // []), + initNames: ((.spec.template.spec.initContainers // []) | map(.name)), + }) | + map( + .name as $n | + .initNames as $ins | + (.volumes[] | select(.secret.secretName | test("-admin-kubeconfig$")?)) + | { + name: $n, + optional: (.secret.optional == true), + hasInit: ($ins | index("wait-for-kubeconfig") != null), + } + ) + ' > "$tmp/summary.json" + + # At least one Deployment must match; if a refactor removes every + # admin-kubeconfig volume from this chart, the test must be updated + # deliberately rather than silently passing. + matched=$(jq 'length' "$tmp/summary.json") + [ "$matched" -ge 1 ] + + offenders=$(jq --raw-output '.[] | select(.optional != true or .hasInit != true) | .name' "$tmp/summary.json") + + if [ -n "$offenders" ]; then + echo "Deployments mounting *-admin-kubeconfig without optional:true + wait-for-kubeconfig init:" >&2 + echo "$offenders" >&2 + echo "Full summary:" >&2 + cat "$tmp/summary.json" >&2 + exit 1 + fi + + echo "Invariant holds for $matched Deployment(s)" +} + +@test "chart emits zero admin-kubeconfig Deployments when tenant has no etcd DataStore" { + # Without a DataStore (parent Tenant has not populated _namespace.etcd yet) + # the control-plane-side Deployments must NOT render at all. If they did, + # the wait-for-kubeconfig init would CrashLoopBackOff indefinitely - there + # would be no KamajiControlPlane to provision the Secret - consuming the + # HelmRelease wait budget and triggering exactly the remediation cycle the + # rest of this chart tries to avoid. This test renders the whole chart + # with etcd empty and asserts no Deployment references the admin-kubeconfig + # Secret. + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + + helm template invariant packages/apps/kubernetes \ + --namespace tenant-root \ + --values packages/apps/kubernetes/tests/values-ci-no-etcd.yaml \ + 2>/dev/null > "$tmp/rendered.yaml" + + matched=$( + yq --output-format=json eval-all '.' "$tmp/rendered.yaml" \ + | jq -s ' + map(select(.kind == "Deployment")) | + map(select( + (.spec.template.spec.volumes // []) + | any(.secret.secretName | test("-admin-kubeconfig$")?) + )) | + length + ' + ) + + if [ "$matched" -ne 0 ]; then + echo "Expected zero Deployments mounting *-admin-kubeconfig when etcd is empty, got $matched:" >&2 + yq --output-format=json eval-all '.' "$tmp/rendered.yaml" \ + | jq -s 'map(select(.kind == "Deployment") | .metadata.name)' >&2 + exit 1 + fi + + echo "No admin-kubeconfig Deployments rendered for empty etcd (as expected)" +} + +@test "chart emits zero admin-kubeconfig HelmReleases when tenant has no etcd DataStore" { + # Same principle as the Deployment variant above, extended to every child + # HelmRelease (cilium, coredns, csi, cert-manager, ...). They reference + # *-admin-kubeconfig via kubeConfig.secretRef and would otherwise sit in + # NotReady forever on an etcd-less tenant, polluting the HelmRelease list + # the operator sees and contradicting the "awaiting-etcd beacon only" + # contract of the soft-skip path. + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + + helm template invariant packages/apps/kubernetes \ + --namespace tenant-root \ + --values packages/apps/kubernetes/tests/values-ci-no-etcd.yaml \ + 2>/dev/null > "$tmp/rendered.yaml" + + matched=$( + yq --output-format=json eval-all '.' "$tmp/rendered.yaml" \ + | jq -s ' + map(select(.kind == "HelmRelease")) | + map(select(.spec.kubeConfig.secretRef.name | test("-admin-kubeconfig$")?)) | + length + ' + ) + + if [ "$matched" -ne 0 ]; then + echo "Expected zero HelmReleases referencing *-admin-kubeconfig when etcd is empty, got $matched:" >&2 + yq --output-format=json eval-all '.' "$tmp/rendered.yaml" \ + | jq -s 'map(select(.kind == "HelmRelease") | .metadata.name)' >&2 + exit 1 + fi + + echo "No admin-kubeconfig HelmReleases rendered for empty etcd (as expected)" +} diff --git a/hack/e2e-apps/remediation-guard.sh b/hack/e2e-apps/remediation-guard.sh new file mode 100644 index 00000000..b45561b9 --- /dev/null +++ b/hack/e2e-apps/remediation-guard.sh @@ -0,0 +1,39 @@ +# Helpers for asserting that a Flux HelmRelease did not fall into an +# install/upgrade remediation cycle during an e2e run. +# +# Background: Flux helm-controller's ClearFailures() zeroes +# .status.installFailures / .status.upgradeFailures on every successful +# reconciliation (see the upstream ClearFailures method on +# HelmReleaseStatus). That makes those counters useless for a guard that +# runs after the HelmRelease has reached Ready - the values are always 0. +# +# What survives a successful reconciliation is .status.history, a bounded +# list of release Snapshots. Each Snapshot carries a status field that +# tracks the Helm release state: deployed, superseded, failed, uninstalled, +# and so on. A remediation cycle leaves the footprint behind: a snapshot +# with status "uninstalled" (from install/upgrade remediation) or "failed" +# (Helm release failure that remediation then uninstalled). Those stay in +# history even after a subsequent successful reinstall. +# +# helmrelease_has_remediation_cycle takes a newline-delimited list of +# snapshot statuses (whatever the caller extracted via kubectl -o jsonpath +# or equivalent) and returns 0 (detected) when any entry is "failed" or +# "uninstalled", 1 otherwise. Empty input is treated as "no history yet, +# no cycle observed". + +helmrelease_has_remediation_cycle() { + statuses="$1" + if [ -z "${statuses}" ]; then + return 1 + fi + # printf + grep over the pipe, rather than a heredoc plus while read. + # printf %s treats the status string as a literal payload, so any stray + # $ in a future caller's input does not trigger shell expansion. grep + # returns 0 iff at least one line matches the allowlist, which is + # exactly the contract the caller wants, so we can return its exit + # status directly. + if printf '%s\n' "${statuses}" | grep --extended-regexp --quiet '^(failed|uninstalled)$'; then + return 0 + fi + return 1 +} diff --git a/hack/e2e-apps/run-kubernetes.sh b/hack/e2e-apps/run-kubernetes.sh index be6dcd6f..761ee57d 100644 --- a/hack/e2e-apps/run-kubernetes.sh +++ b/hack/e2e-apps/run-kubernetes.sh @@ -1,3 +1,5 @@ +. hack/e2e-apps/remediation-guard.sh + run_kubernetes_test() { local version_expr="$1" local test_name="$2" @@ -320,6 +322,35 @@ EOF done kubectl wait hr kubernetes-${test_name}-ingress-nginx -n tenant-test --timeout=5m --for=condition=ready + # Guard: parent HelmRelease must not have entered an install/upgrade remediation cycle. + # A non-zero installFailures/upgradeFailures indicates the helm-wait budget expired while + # admin-kubeconfig was still being provisioned, which would trigger uninstall remediation + # and churn the Cluster CR. + # Flux helm-controller v2 retains per-revision release Snapshots in + # .status.history; each Snapshot's .status reflects the Helm release + # state (deployed/superseded/failed/uninstalled). A remediation cycle + # leaves a "failed" or "uninstalled" entry behind that survives a later + # successful reinstall, unlike the installFailures/upgradeFailures + # counters (which ClearFailures zeroes on every successful reconcile). + # The shape is pinned by hack/remediation-guard.bats; the upstream + # types are github.com/fluxcd/helm-controller/api v2 Snapshot. + history_statuses=$(kubectl get hr -n tenant-test "kubernetes-${test_name}" \ + -ojsonpath='{range .status.history[*]}{.status}{"\n"}{end}') + # Always emit the raw value so a silent future-Flux field rename shows + # up as "empty history on a Ready HR" in CI logs rather than vanishing. + echo "Parent HelmRelease history statuses:" + printf '%s\n' "${history_statuses:-}" + if [ -z "${history_statuses}" ]; then + echo "Unexpected empty .status.history on a Ready HelmRelease - Flux API shape may have changed." >&2 + kubectl -n tenant-test describe hr "kubernetes-${test_name}" >&2 + exit 1 + fi + if helmrelease_has_remediation_cycle "${history_statuses}"; then + echo "Parent HelmRelease entered remediation cycle." >&2 + kubectl -n tenant-test describe hr "kubernetes-${test_name}" >&2 + exit 1 + fi + # Clean up pkill -f "port-forward.*${port}:" 2>/dev/null || true rm -f "tenantkubeconfig-${test_name}" diff --git a/hack/remediation-guard.bats b/hack/remediation-guard.bats new file mode 100644 index 00000000..092fe06d --- /dev/null +++ b/hack/remediation-guard.bats @@ -0,0 +1,127 @@ +#!/usr/bin/env bats +# ----------------------------------------------------------------------------- +# Unit tests for hack/e2e-apps/remediation-guard.sh +# +# helmrelease_has_remediation_cycle takes a newline-delimited list of +# HelmRelease history snapshot status values (deployed/superseded/failed/ +# uninstalled/...) and returns 0 when any entry is "failed" or "uninstalled" +# (meaning flux helm-controller performed install/upgrade remediation). +# +# This is used by the e2e script after the HelmRelease reaches Ready. The +# failure/upgrade counters (.status.installFailures / .status.upgradeFailures) +# are useless there because flux's ClearFailures zeroes them on successful +# reconciliation; .status.history retains the snapshot trail. +# +# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on +# its own line; there is no bats `run` or `$status`. Assertions are +# expressed as direct shell tests that exit non-zero on failure. +# +# Run with: hack/cozytest.sh hack/remediation-guard.bats +# ----------------------------------------------------------------------------- + +@test "empty history returns not-detected" { + . hack/e2e-apps/remediation-guard.sh + if helmrelease_has_remediation_cycle ""; then + echo "expected not-detected for empty history" >&2 + exit 1 + fi +} + +@test "single deployed snapshot returns not-detected" { + . hack/e2e-apps/remediation-guard.sh + if helmrelease_has_remediation_cycle "deployed"; then + echo "expected not-detected for deployed-only history" >&2 + exit 1 + fi +} + +@test "deployed then superseded returns not-detected" { + . hack/e2e-apps/remediation-guard.sh + statuses=$(printf 'deployed\nsuperseded\n') + if helmrelease_has_remediation_cycle "${statuses}"; then + echo "expected not-detected for deployed+superseded history" >&2 + exit 1 + fi +} + +@test "single failed snapshot returns detected" { + . hack/e2e-apps/remediation-guard.sh + if ! helmrelease_has_remediation_cycle "failed"; then + echo "expected detected when history contains failed snapshot" >&2 + exit 1 + fi +} + +@test "single uninstalled snapshot returns detected" { + # The exact signature of the install-remediation race: the first install + # exceeded flux's wait budget, remediation uninstalled, the next retry + # eventually succeeded. History still carries the uninstalled snapshot. + . hack/e2e-apps/remediation-guard.sh + if ! helmrelease_has_remediation_cycle "uninstalled"; then + echo "expected detected when history contains uninstalled snapshot" >&2 + exit 1 + fi +} + +@test "uninstalled then deployed still returns detected" { + . hack/e2e-apps/remediation-guard.sh + statuses=$(printf 'uninstalled\ndeployed\n') + if ! helmrelease_has_remediation_cycle "${statuses}"; then + echo "expected detected despite later successful deploy" >&2 + exit 1 + fi +} + +@test "deployed then failed still returns detected" { + . hack/e2e-apps/remediation-guard.sh + statuses=$(printf 'deployed\nfailed\n') + if ! helmrelease_has_remediation_cycle "${statuses}"; then + echo "expected detected when any entry is failed" >&2 + exit 1 + fi +} + +@test "status.history extraction pins HR v2 status.history shape" { + # Pins the Flux HelmRelease v2 .status.history[].status shape that + # run-kubernetes.sh relies on. If a future flux release renames the + # field, the jsonpath returns nothing, the guard reports no cycle, + # and real remediation loops slip past the e2e assertion. This test + # uses yq to read the exact path used in the e2e script; the upstream + # Snapshot type lives at + # github.com/fluxcd/helm-controller/api/v2.Snapshot (via go.mod). + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + + cat > "$tmp/hr.yaml" <<'YAML' +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: kubernetes-test + namespace: tenant-test +status: + history: + - name: kubernetes-test + namespace: tenant-test + version: 1 + status: uninstalled + - name: kubernetes-test + namespace: tenant-test + version: 2 + status: deployed +YAML + + # Default yq output is yaml scalar format, which for string values emits + # bare unquoted tokens - matching what kubectl -o jsonpath produces in + # e2e. Do not switch to JSON output here; that would quote the values + # and break the loop in helmrelease_has_remediation_cycle. + statuses=$(yq '.status.history[].status' "$tmp/hr.yaml") + + [ -n "$statuses" ] + echo "$statuses" | grep --quiet '^uninstalled$' + + . hack/e2e-apps/remediation-guard.sh + if ! helmrelease_has_remediation_cycle "$statuses"; then + echo "expected detected for pinned HR snippet with uninstalled + deployed history" >&2 + exit 1 + fi +} diff --git a/packages/apps/kubernetes/Makefile b/packages/apps/kubernetes/Makefile index 01cf736d..4fea42ef 100644 --- a/packages/apps/kubernetes/Makefile +++ b/packages/apps/kubernetes/Makefile @@ -4,6 +4,9 @@ KUBERNETES_PKG_TAG = $(shell awk '$$1 == "version:" {print $$2}' Chart.yaml) include ../../../hack/common-envs.mk include ../../../hack/package.mk +test: + helm unittest . + generate: cozyvalues-gen -m 'kubernetes' -v values.yaml -s values.schema.json -r README.md -g ../../../api/apps/v1alpha1/kubernetes/types.go ../../../hack/update-crd.sh @@ -67,3 +70,4 @@ image-cluster-autoscaler: echo "$(REGISTRY)/cluster-autoscaler:$(call settag,$(KUBERNETES_PKG_TAG))@$$(yq e '."containerimage.digest"' images/cluster-autoscaler.json -o json -r)" \ > images/cluster-autoscaler.tag rm -f images/cluster-autoscaler.json + diff --git a/packages/apps/kubernetes/README.md b/packages/apps/kubernetes/README.md index d62d3a39..ecbbc31b 100644 --- a/packages/apps/kubernetes/README.md +++ b/packages/apps/kubernetes/README.md @@ -145,31 +145,33 @@ See the reference for components utilized in this service: ### Kubernetes Control Plane Configuration -| Name | Description | Type | Value | -| --------------------------------------------------- | ------------------------------------------------ | ---------- | ------- | -| `controlPlane` | Kubernetes control-plane configuration. | `object` | `{}` | -| `controlPlane.replicas` | Number of control-plane replicas. | `int` | `2` | -| `controlPlane.apiServer` | API Server configuration. | `object` | `{}` | -| `controlPlane.apiServer.resources` | CPU and memory resources for API Server. | `object` | `{}` | -| `controlPlane.apiServer.resources.cpu` | CPU available. | `quantity` | `""` | -| `controlPlane.apiServer.resources.memory` | Memory (RAM) available. | `quantity` | `""` | -| `controlPlane.apiServer.resourcesPreset` | Preset if `resources` omitted. | `string` | `large` | -| `controlPlane.controllerManager` | Controller Manager configuration. | `object` | `{}` | -| `controlPlane.controllerManager.resources` | CPU and memory resources for Controller Manager. | `object` | `{}` | -| `controlPlane.controllerManager.resources.cpu` | CPU available. | `quantity` | `""` | -| `controlPlane.controllerManager.resources.memory` | Memory (RAM) available. | `quantity` | `""` | -| `controlPlane.controllerManager.resourcesPreset` | Preset if `resources` omitted. | `string` | `micro` | -| `controlPlane.scheduler` | Scheduler configuration. | `object` | `{}` | -| `controlPlane.scheduler.resources` | CPU and memory resources for Scheduler. | `object` | `{}` | -| `controlPlane.scheduler.resources.cpu` | CPU available. | `quantity` | `""` | -| `controlPlane.scheduler.resources.memory` | Memory (RAM) available. | `quantity` | `""` | -| `controlPlane.scheduler.resourcesPreset` | Preset if `resources` omitted. | `string` | `micro` | -| `controlPlane.konnectivity` | Konnectivity configuration. | `object` | `{}` | -| `controlPlane.konnectivity.server` | Konnectivity Server configuration. | `object` | `{}` | -| `controlPlane.konnectivity.server.resources` | CPU and memory resources for Konnectivity. | `object` | `{}` | -| `controlPlane.konnectivity.server.resources.cpu` | CPU available. | `quantity` | `""` | -| `controlPlane.konnectivity.server.resources.memory` | Memory (RAM) available. | `quantity` | `""` | -| `controlPlane.konnectivity.server.resourcesPreset` | Preset if `resources` omitted. | `string` | `micro` | +| Name | Description | Type | Value | +| --------------------------------------------------- | --------------------------------------------------------------------------------------------- | ---------- | ------- | +| `controlPlane` | Kubernetes control-plane configuration. | `object` | `{}` | +| `controlPlane.replicas` | Number of control-plane replicas. | `int` | `2` | +| `controlPlane.apiServer` | API Server configuration. | `object` | `{}` | +| `controlPlane.apiServer.resources` | CPU and memory resources for API Server. | `object` | `{}` | +| `controlPlane.apiServer.resources.cpu` | CPU available. | `quantity` | `""` | +| `controlPlane.apiServer.resources.memory` | Memory (RAM) available. | `quantity` | `""` | +| `controlPlane.apiServer.resourcesPreset` | Preset if `resources` omitted. | `string` | `large` | +| `controlPlane.controllerManager` | Controller Manager configuration. | `object` | `{}` | +| `controlPlane.controllerManager.resources` | CPU and memory resources for Controller Manager. | `object` | `{}` | +| `controlPlane.controllerManager.resources.cpu` | CPU available. | `quantity` | `""` | +| `controlPlane.controllerManager.resources.memory` | Memory (RAM) available. | `quantity` | `""` | +| `controlPlane.controllerManager.resourcesPreset` | Preset if `resources` omitted. | `string` | `micro` | +| `controlPlane.scheduler` | Scheduler configuration. | `object` | `{}` | +| `controlPlane.scheduler.resources` | CPU and memory resources for Scheduler. | `object` | `{}` | +| `controlPlane.scheduler.resources.cpu` | CPU available. | `quantity` | `""` | +| `controlPlane.scheduler.resources.memory` | Memory (RAM) available. | `quantity` | `""` | +| `controlPlane.scheduler.resourcesPreset` | Preset if `resources` omitted. | `string` | `micro` | +| `controlPlane.konnectivity` | Konnectivity configuration. | `object` | `{}` | +| `controlPlane.konnectivity.server` | Konnectivity Server configuration. | `object` | `{}` | +| `controlPlane.konnectivity.server.resources` | CPU and memory resources for Konnectivity. | `object` | `{}` | +| `controlPlane.konnectivity.server.resources.cpu` | CPU available. | `quantity` | `""` | +| `controlPlane.konnectivity.server.resources.memory` | Memory (RAM) available. | `quantity` | `""` | +| `controlPlane.konnectivity.server.resourcesPreset` | Preset if `resources` omitted. | `string` | `micro` | +| `images` | Optional image overrides for air-gapped or rate-limited registries. | `object` | `{}` | +| `images.waitForKubeconfig` | Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag. | `string` | `""` | ## Parameter examples and reference diff --git a/packages/apps/kubernetes/images/busybox.tag b/packages/apps/kubernetes/images/busybox.tag new file mode 100644 index 00000000..39de220a --- /dev/null +++ b/packages/apps/kubernetes/images/busybox.tag @@ -0,0 +1 @@ +docker.io/library/busybox:1.37.0@sha256:1487d0af5f52b4ba31c7e465126ee2123fe3f2305d638e7827681e7cf6c83d5e diff --git a/packages/apps/kubernetes/templates/_helpers.tpl b/packages/apps/kubernetes/templates/_helpers.tpl index 36c06b64..40a8ae7a 100644 --- a/packages/apps/kubernetes/templates/_helpers.tpl +++ b/packages/apps/kubernetes/templates/_helpers.tpl @@ -49,3 +49,52 @@ Selector labels app.kubernetes.io/name: {{ include "kubernetes.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} + +{{/* +wait-for-kubeconfig init container shared by the control-plane-side +Deployments (cluster-autoscaler, kccm, kcsi-controller) that mount the +*-admin-kubeconfig Secret provisioned asynchronously by Kamaji. The +Secret volume is declared optional so kubelet does not FailedMount while +Kamaji is still bootstrapping; this container polls the mounted path and +exits only when super-admin.svc appears, which happens after kubelet's +optional-Secret refresh cycle. + +The 10m deadline stays strictly below the 15m HelmRelease +Install.Timeout set by cozystack-api for the Kubernetes kind (via the +release.cozystack.io/helm-install-timeout annotation) so the +CrashLoopBackOff surfaces before flux remediation fires and uninstalls +the Cluster CR. + +The default image lives in images/busybox.tag and points directly at +docker.io by digest (not mirrored to ghcr.io like the other .tag files +here): the payload is a one-shot sh loop and the digest pin makes the +pull immutable. Operators in air-gapped or rate-limited environments +can override it via .Values.images.waitForKubeconfig (any registry +reference kubelet can pull). When the value is empty the chart falls +back to the bundled digest pin, preserving the prior default. + +Call site owns the surrounding volumes block; the kubeconfig volume +must exist on the pod and mount at /etc/kubernetes/kubeconfig. +*/}} +{{- define "kubernetes.waitForAdminKubeconfig" -}} +- name: wait-for-kubeconfig + image: "{{ default (.Files.Get "images/busybox.tag" | trim) .Values.images.waitForKubeconfig }}" + command: + - sh + - -c + - | + set -eu + deadline=$(( $(date +%s) + 600 )) + until [ -s /etc/kubernetes/kubeconfig/super-admin.svc ]; do + if [ "$(date +%s)" -ge "$deadline" ]; then + echo "admin kubeconfig was not provisioned within 10m; exiting so the pod goes CrashLoopBackOff and surfaces in dashboards" >&2 + exit 1 + fi + echo "waiting for admin kubeconfig (provisioned by Kamaji, visible after kubelet Secret refresh)..." + sleep 5 + done + volumeMounts: + - name: kubeconfig + mountPath: /etc/kubernetes/kubeconfig + readOnly: true +{{- end }} diff --git a/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml b/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml index a00e0155..298d86db 100644 --- a/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml +++ b/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml @@ -1,3 +1,14 @@ +{{- /* + Gate the control-plane-side workloads on the parent tenant having an etcd + DataStore. Without it no KamajiControlPlane is ever created, Kamaji never + provisions -admin-kubeconfig, and rendering these Deployments would cause + the wait-for-kubeconfig init to CrashLoopBackOff indefinitely, consuming + the parent HelmRelease install timeout and triggering the very uninstall + remediation cycle this chart is supposed to avoid. Rendering them only + when $etcd is set keeps the HelmRelease Ready while flux retries on its + interval and picks up the DataStore as soon as the Tenant chart finishes. +*/}} +{{- if .Values._namespace.etcd }} --- apiVersion: apps/v1 kind: Deployment @@ -23,6 +34,8 @@ spec: - key: node-role.kubernetes.io/control-plane operator: Exists effect: "NoSchedule" + initContainers: + {{- include "kubernetes.waitForAdminKubeconfig" $ | nindent 6 }} containers: - image: "{{ $.Files.Get "images/cluster-autoscaler.tag" | trim }}" name: cluster-autoscaler @@ -56,6 +69,7 @@ spec: name: cloud-config - secret: secretName: {{ .Release.Name }}-admin-kubeconfig + optional: true name: kubeconfig serviceAccountName: {{ .Release.Name }}-cluster-autoscaler terminationGracePeriodSeconds: 10 @@ -105,3 +119,4 @@ rules: - list - update - watch +{{- end }} diff --git a/packages/apps/kubernetes/templates/cluster.yaml b/packages/apps/kubernetes/templates/cluster.yaml index 10d6fd80..e1494e64 100644 --- a/packages/apps/kubernetes/templates/cluster.yaml +++ b/packages/apps/kubernetes/templates/cluster.yaml @@ -1,4 +1,15 @@ {{- $etcd := .Values._namespace.etcd }} +{{- /* + When $etcd is empty, the parent Tenant application has not populated + _namespace.etcd in cozystack-values yet - either the operator forgot to + set etcd: true on an ancestor Tenant, or the Tenant HelmRelease is still + reconciling. Either way, rendering a KamajiControlPlane with an empty + dataStoreName would be rejected by Kamaji's admission webhook and the + HelmRelease would fail to install, triggering remediation. Instead, emit + a single ConfigMap as a user-visible status beacon and skip the rest so + flux marks the HelmRelease Ready and retries its 5m reconcile loop until + the Tenant chart catches up. +*/}} {{- $ingress := .Values._namespace.ingress }} {{- $host := .Values._namespace.host }} {{- $kubevirtmachinetemplateNames := list }} @@ -84,6 +95,26 @@ spec: - name: default pod: {} {{- end }} +{{- if not $etcd }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-awaiting-etcd + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/name: kubernetes + app.kubernetes.io/instance: {{ .Release.Name }} +data: + status: "awaiting-etcd" + message: | + No DataStore is available for this tenant Kubernetes cluster. The parent + Tenant application has not populated _namespace.etcd. Set spec.etcd: true + on an ancestor Tenant (usually tenant-root) and wait for its HelmRelease + to reconcile - this HelmRelease will pick up the DataStore on its next + 5m reconcile loop and provision the cluster. +{{- else }} --- apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster @@ -404,3 +435,4 @@ metadata: spec: {{- .spec | toYaml | nindent 2 }} {{- end }} +{{- end }} diff --git a/packages/apps/kubernetes/templates/csi/deploy.yaml b/packages/apps/kubernetes/templates/csi/deploy.yaml index 938b6d67..de62104c 100644 --- a/packages/apps/kubernetes/templates/csi/deploy.yaml +++ b/packages/apps/kubernetes/templates/csi/deploy.yaml @@ -1,3 +1,4 @@ +{{- if .Values._namespace.etcd }} kind: Deployment apiVersion: apps/v1 metadata: @@ -24,6 +25,8 @@ spec: - key: node-role.kubernetes.io/control-plane operator: Exists effect: "NoSchedule" + initContainers: + {{- include "kubernetes.waitForAdminKubeconfig" $ | nindent 6 }} containers: - name: csi-driver imagePullPolicy: Always @@ -234,4 +237,6 @@ spec: emptyDir: {} - secret: secretName: {{ .Release.Name }}-admin-kubeconfig + optional: true name: kubeconfig +{{- end }} diff --git a/packages/apps/kubernetes/templates/helmreleases/cert-manager-crds.yaml b/packages/apps/kubernetes/templates/helmreleases/cert-manager-crds.yaml index be07a8b9..fd9dac7e 100644 --- a/packages/apps/kubernetes/templates/helmreleases/cert-manager-crds.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/cert-manager-crds.yaml @@ -1,4 +1,4 @@ -{{- if .Values.addons.certManager.enabled }} +{{- if and .Values.addons.certManager.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/cert-manager.yaml b/packages/apps/kubernetes/templates/helmreleases/cert-manager.yaml index 6857581a..700b666e 100644 --- a/packages/apps/kubernetes/templates/helmreleases/cert-manager.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/cert-manager.yaml @@ -8,7 +8,7 @@ cert-manager: {{- end }} {{- end }} -{{- if .Values.addons.certManager.enabled }} +{{- if and .Values.addons.certManager.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/cilium.yaml b/packages/apps/kubernetes/templates/helmreleases/cilium.yaml index d032f5b6..d8c90cbf 100644 --- a/packages/apps/kubernetes/templates/helmreleases/cilium.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/cilium.yaml @@ -14,6 +14,7 @@ cilium: {{- end }} {{- end }} +{{- if .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -55,3 +56,4 @@ spec: - name: {{ .Release.Name }}-gateway-api-crds namespace: {{ .Release.Namespace }} {{- end }} +{{- end }} diff --git a/packages/apps/kubernetes/templates/helmreleases/coredns.yaml b/packages/apps/kubernetes/templates/helmreleases/coredns.yaml index bdb6c682..0711a51d 100644 --- a/packages/apps/kubernetes/templates/helmreleases/coredns.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/coredns.yaml @@ -4,6 +4,7 @@ coredns: clusterIP: "10.95.0.10" {{- end }} +{{- if .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -42,3 +43,4 @@ spec: {{- end }} - name: {{ .Release.Name }}-cilium namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/packages/apps/kubernetes/templates/helmreleases/csi.yaml b/packages/apps/kubernetes/templates/helmreleases/csi.yaml index dd2c69a6..109d78e1 100644 --- a/packages/apps/kubernetes/templates/helmreleases/csi.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/csi.yaml @@ -1,3 +1,4 @@ +{{- if .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -39,3 +40,4 @@ spec: - name: {{ .Release.Name }} namespace: {{ .Release.Namespace }} {{- end }} +{{- end }} diff --git a/packages/apps/kubernetes/templates/helmreleases/fluxcd.yaml b/packages/apps/kubernetes/templates/helmreleases/fluxcd.yaml index 76499dfe..25fff01c 100644 --- a/packages/apps/kubernetes/templates/helmreleases/fluxcd.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/fluxcd.yaml @@ -1,4 +1,4 @@ -{{- if .Values.addons.fluxcd.enabled }} +{{- if and .Values.addons.fluxcd.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/gateway-api-crds.yaml b/packages/apps/kubernetes/templates/helmreleases/gateway-api-crds.yaml index 2bcc8d4d..b4172ed1 100644 --- a/packages/apps/kubernetes/templates/helmreleases/gateway-api-crds.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/gateway-api-crds.yaml @@ -1,4 +1,4 @@ -{{- if $.Values.addons.gatewayAPI.enabled }} +{{- if and $.Values.addons.gatewayAPI.enabled $.Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml index 5ef48912..e243bfad 100644 --- a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml @@ -1,4 +1,4 @@ -{{- if .Values.addons.gpuOperator.enabled }} +{{- if and .Values.addons.gpuOperator.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/ingress-nginx.yaml b/packages/apps/kubernetes/templates/helmreleases/ingress-nginx.yaml index 5cafff90..6e2183d3 100644 --- a/packages/apps/kubernetes/templates/helmreleases/ingress-nginx.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/ingress-nginx.yaml @@ -20,7 +20,7 @@ ingress-nginx: node-role.kubernetes.io/ingress-nginx: "" {{- end }} -{{- if .Values.addons.ingressNginx.enabled }} +{{- if and .Values.addons.ingressNginx.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/metrics-server.yaml b/packages/apps/kubernetes/templates/helmreleases/metrics-server.yaml index 3cc81a14..3e6f9660 100644 --- a/packages/apps/kubernetes/templates/helmreleases/metrics-server.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/metrics-server.yaml @@ -1,3 +1,4 @@ +{{- if .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -36,3 +37,4 @@ spec: namespace: {{ .Release.Namespace }} - name: {{ .Release.Name }}-prometheus-operator-crds namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/packages/apps/kubernetes/templates/helmreleases/monitoring-agents.yaml b/packages/apps/kubernetes/templates/helmreleases/monitoring-agents.yaml index ea84dec0..a811f7dd 100644 --- a/packages/apps/kubernetes/templates/helmreleases/monitoring-agents.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/monitoring-agents.yaml @@ -1,6 +1,6 @@ {{- $targetTenant := .Values._namespace.monitoring }} {{- $clusterDomain := (index .Values._cluster "cluster-domain") | default "cozy.local" }} -{{- if .Values.addons.monitoringAgents.enabled }} +{{- if and .Values.addons.monitoringAgents.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/prometheus-operator-crds.yaml b/packages/apps/kubernetes/templates/helmreleases/prometheus-operator-crds.yaml index 600a7994..3038a058 100644 --- a/packages/apps/kubernetes/templates/helmreleases/prometheus-operator-crds.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/prometheus-operator-crds.yaml @@ -1,3 +1,4 @@ +{{- if .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -31,3 +32,4 @@ spec: - name: {{ .Release.Name }} namespace: {{ .Release.Namespace }} {{- end }} +{{- end }} diff --git a/packages/apps/kubernetes/templates/helmreleases/velero.yaml b/packages/apps/kubernetes/templates/helmreleases/velero.yaml index ad236d53..781b9c49 100644 --- a/packages/apps/kubernetes/templates/helmreleases/velero.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/velero.yaml @@ -1,4 +1,4 @@ -{{- if .Values.addons.velero.enabled }} +{{- if and .Values.addons.velero.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler-crds.yaml b/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler-crds.yaml index a3b7a9b4..55a5faac 100644 --- a/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler-crds.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler-crds.yaml @@ -1,4 +1,4 @@ -{{- if .Values.addons.monitoringAgents.enabled }} +{{- if and .Values.addons.monitoringAgents.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler.yaml b/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler.yaml index 178df3e3..74fb5a39 100644 --- a/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler.yaml @@ -24,7 +24,7 @@ vertical-pod-autoscaler: memory: 1600Mi {{- end }} -{{- if .Values.addons.monitoringAgents.enabled }} +{{- if and .Values.addons.monitoringAgents.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/victoria-metrics-operator.yaml b/packages/apps/kubernetes/templates/helmreleases/victoria-metrics-operator.yaml index 99744277..7302f8f5 100644 --- a/packages/apps/kubernetes/templates/helmreleases/victoria-metrics-operator.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/victoria-metrics-operator.yaml @@ -1,4 +1,4 @@ -{{- if .Values.addons.monitoringAgents.enabled }} +{{- if and .Values.addons.monitoringAgents.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/volumesnapshot-crd.yaml b/packages/apps/kubernetes/templates/helmreleases/volumesnapshot-crd.yaml index 025f01b7..d50fd93c 100644 --- a/packages/apps/kubernetes/templates/helmreleases/volumesnapshot-crd.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/volumesnapshot-crd.yaml @@ -1,3 +1,4 @@ +{{- if .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -33,3 +34,4 @@ spec: - name: {{ .Release.Name }} namespace: {{ .Release.Namespace }} {{- end }} +{{- end }} diff --git a/packages/apps/kubernetes/templates/kccm/manager.yaml b/packages/apps/kubernetes/templates/kccm/manager.yaml index 81426d4e..bd9e2798 100644 --- a/packages/apps/kubernetes/templates/kccm/manager.yaml +++ b/packages/apps/kubernetes/templates/kccm/manager.yaml @@ -1,3 +1,4 @@ +{{- if .Values._namespace.etcd }} apiVersion: apps/v1 kind: Deployment metadata: @@ -22,6 +23,8 @@ spec: - key: node-role.kubernetes.io/control-plane operator: Exists effect: "NoSchedule" + initContainers: + {{- include "kubernetes.waitForAdminKubeconfig" $ | nindent 6 }} containers: - name: kubevirt-cloud-controller-manager args: @@ -55,5 +58,7 @@ spec: name: cloud-config - secret: secretName: {{ .Release.Name }}-admin-kubeconfig + optional: true name: kubeconfig serviceAccountName: {{ .Release.Name }}-kccm +{{- end }} diff --git a/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml b/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml new file mode 100644 index 00000000..507e00af --- /dev/null +++ b/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml @@ -0,0 +1,155 @@ +suite: admin-kubeconfig wait guards + +release: + name: test + namespace: tenant-root + +values: + - values-ci.yaml + +tests: + - it: cluster-autoscaler mounts admin-kubeconfig as optional + template: templates/cluster-autoscaler/deployment.yaml + documentSelector: + path: kind + value: Deployment + asserts: + - equal: + path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.secretName + value: test-admin-kubeconfig + - equal: + path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.optional + value: true + + - it: cluster-autoscaler waits for admin-kubeconfig via initContainer + template: templates/cluster-autoscaler/deployment.yaml + documentSelector: + path: kind + value: Deployment + asserts: + - equal: + path: spec.template.spec.initContainers[0].name + value: wait-for-kubeconfig + - contains: + path: spec.template.spec.initContainers[0].volumeMounts + content: + name: kubeconfig + mountPath: /etc/kubernetes/kubeconfig + readOnly: true + + - it: kccm mounts admin-kubeconfig as optional + template: templates/kccm/manager.yaml + documentSelector: + path: kind + value: Deployment + asserts: + - equal: + path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.secretName + value: test-admin-kubeconfig + - equal: + path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.optional + value: true + + - it: kccm waits for admin-kubeconfig via initContainer + template: templates/kccm/manager.yaml + documentSelector: + path: kind + value: Deployment + asserts: + - equal: + path: spec.template.spec.initContainers[0].name + value: wait-for-kubeconfig + - contains: + path: spec.template.spec.initContainers[0].volumeMounts + content: + name: kubeconfig + mountPath: /etc/kubernetes/kubeconfig + readOnly: true + + - it: csi controller mounts admin-kubeconfig as optional + template: templates/csi/deploy.yaml + documentSelector: + path: kind + value: Deployment + asserts: + - equal: + path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.secretName + value: test-admin-kubeconfig + - equal: + path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.optional + value: true + + - it: csi controller waits for admin-kubeconfig via initContainer + template: templates/csi/deploy.yaml + documentSelector: + path: kind + value: Deployment + asserts: + - equal: + path: spec.template.spec.initContainers[0].name + value: wait-for-kubeconfig + - contains: + path: spec.template.spec.initContainers[0].volumeMounts + content: + name: kubeconfig + mountPath: /etc/kubernetes/kubeconfig + readOnly: true + + - it: wait-for-kubeconfig defaults to bundled busybox digest pin + template: templates/cluster-autoscaler/deployment.yaml + documentSelector: + path: kind + value: Deployment + asserts: + - matchRegex: + path: spec.template.spec.initContainers[0].image + pattern: '^docker\.io/library/busybox:[^@]+@sha256:[0-9a-f]{64}$' + + - it: wait-for-kubeconfig honours images.waitForKubeconfig override + template: templates/cluster-autoscaler/deployment.yaml + documentSelector: + path: kind + value: Deployment + set: + images: + waitForKubeconfig: "registry.example.test/library/busybox:1.37.0@sha256:0000000000000000000000000000000000000000000000000000000000000000" + asserts: + - equal: + path: spec.template.spec.initContainers[0].image + value: "registry.example.test/library/busybox:1.37.0@sha256:0000000000000000000000000000000000000000000000000000000000000000" + + - it: cluster.yaml renders and wires dataStoreName when tenant has etcd + template: templates/cluster.yaml + documentSelector: + path: kind + value: KamajiControlPlane + asserts: + - equal: + path: spec.dataStoreName + value: tenant-root + + - it: cluster.yaml skips Cluster resources when tenant has no etcd DataStore + # Must NOT fail rendering - the parent Tenant chart populates + # _namespace.etcd asynchronously, so rendering failures here would cause + # flux install remediation on every cold bootstrap. Instead, emit only a + # ConfigMap status beacon so the HelmRelease reports Ready while flux + # retries on its interval until the DataStore appears. + template: templates/cluster.yaml + set: + _namespace: + etcd: "" + monitoring: "" + ingress: "" + seaweedfs: "" + host: "" + asserts: + - hasDocuments: + count: 1 + - isKind: + of: ConfigMap + - equal: + path: metadata.name + value: test-awaiting-etcd + - equal: + path: data.status + value: awaiting-etcd diff --git a/packages/apps/kubernetes/tests/values-ci-no-etcd.yaml b/packages/apps/kubernetes/tests/values-ci-no-etcd.yaml new file mode 100644 index 00000000..c7c8196f --- /dev/null +++ b/packages/apps/kubernetes/tests/values-ci-no-etcd.yaml @@ -0,0 +1,9 @@ +_namespace: + etcd: "" + monitoring: "" + ingress: "" + seaweedfs: "" + host: "" +_cluster: + cluster-domain: cozy.local +nodeGroups: null diff --git a/packages/apps/kubernetes/tests/values-ci.yaml b/packages/apps/kubernetes/tests/values-ci.yaml new file mode 100644 index 00000000..13365e8c --- /dev/null +++ b/packages/apps/kubernetes/tests/values-ci.yaml @@ -0,0 +1,9 @@ +_namespace: + etcd: tenant-root + monitoring: "" + ingress: "" + seaweedfs: "" + host: "" +_cluster: + cluster-domain: cozy.local +nodeGroups: null diff --git a/packages/apps/kubernetes/values.schema.json b/packages/apps/kubernetes/values.schema.json index 1beeff9c..81e9e2b8 100644 --- a/packages/apps/kubernetes/values.schema.json +++ b/packages/apps/kubernetes/values.schema.json @@ -630,6 +630,18 @@ } } } + }, + "images": { + "description": "Optional image overrides for air-gapped or rate-limited registries.", + "type": "object", + "default": {}, + "properties": { + "waitForKubeconfig": { + "description": "Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.", + "type": "string", + "default": "" + } + } } } } diff --git a/packages/apps/kubernetes/values.yaml b/packages/apps/kubernetes/values.yaml index a67b5d69..68ba07b0 100644 --- a/packages/apps/kubernetes/values.yaml +++ b/packages/apps/kubernetes/values.yaml @@ -197,3 +197,10 @@ controlPlane: server: resources: {} resourcesPreset: "micro" + +## @typedef {struct} Images - Optional image overrides for chart-internal helpers. +## @field {string} [waitForKubeconfig] - Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag. + +## @param {Images} images - Optional image overrides for air-gapped or rate-limited registries. +images: + waitForKubeconfig: "" diff --git a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml index 5e9e8f94..1631c86e 100644 --- a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml +++ b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml @@ -2,13 +2,27 @@ apiVersion: cozystack.io/v1alpha1 kind: ApplicationDefinition metadata: name: kubernetes + annotations: + # Kubernetes tenants boot a Kamaji control plane whose admin-kubeconfig + # Secret is provisioned asynchronously. Cold Kamaji start (image pull + + # etcd + apiserver Ready) plus admin-kubeconfig generation can exceed + # Flux helm-controller's default wait budget, causing remediation loops + # that uninstall the Cluster CR. This override is applied by cozystack-api + # to the HelmRelease Spec.Install.Timeout and Spec.Upgrade.Timeout. + # + # Coupling: the wait-for-kubeconfig init container in + # packages/apps/kubernetes/templates/_helpers.tpl hard-codes a 10m + # deadline chosen to stay strictly below this value so the pod's + # CrashLoopBackOff surfaces before flux remediation fires. If this + # annotation is raised, update that init deadline correspondingly. + release.cozystack.io/helm-install-timeout: "15m" spec: application: kind: Kubernetes singular: kubernetes plural: kuberneteses openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"ephemeralStorage":"20Gi","gpus":[],"instanceType":"u1.medium","maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"]}},"additionalProperties":{"type":"object","required":["ephemeralStorage","instanceType","maxReplicas","minReplicas","resources"],"properties":{"ephemeralStorage":{"description":"Ephemeral storage size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string"}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium"},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `.` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","ingressNginx","monitoringAgents","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"large","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}}}} + {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"ephemeralStorage":"20Gi","gpus":[],"instanceType":"u1.medium","maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"]}},"additionalProperties":{"type":"object","required":["ephemeralStorage","instanceType","maxReplicas","minReplicas","resources"],"properties":{"ephemeralStorage":{"description":"Ephemeral storage size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string"}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium"},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `.` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","ingressNginx","monitoringAgents","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"large","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"images":{"description":"Optional image overrides for air-gapped or rate-limited registries.","type":"object","default":{},"properties":{"waitForKubeconfig":{"description":"Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.","type":"string","default":""}}}}} release: prefix: kubernetes- labels: @@ -26,7 +40,7 @@ spec: tags: - compute icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODFfMjg0NSkiLz4KPHBhdGggZD0iTTcxLjk5NjggMTlDNzAuMzAzOSAxOS4wMDAyIDY4LjkzMTIgMjAuNTMyMiA2OC45MzE0IDIyLjQyMjFDNjguOTMxNCAyMi40NTExIDY4LjkzNzMgMjIuNDc4OCA2OC45Mzc5IDIyLjUwNzZDNjguOTM1NCAyMi43NjQ0IDY4LjkyMzEgMjMuMDczNyA2OC45MzE0IDIzLjI5NzNDNjguOTcxNyAyNC4zODczIDY5LjIwODIgMjUuMjIxNiA2OS4zNTA2IDI2LjIyNThDNjkuNjA4NCAyOC4zNzUyIDY5LjgyNDUgMzAuMTU2OSA2OS42OTEyIDMxLjgxM0M2OS41NjE1IDMyLjQzNzUgNjkuMTAzNyAzMy4wMDg2IDY4LjY5NTYgMzMuNDA1Nkw2OC42MjM1IDM0LjcwODZDNjYuNzgzOSAzNC44NjE3IDY0LjkzMTkgMzUuMTQyMSA2My4wODIxIDM1LjU2NDFDNTUuMTIyNiAzNy4zNzk4IDQ4LjI2OTUgNDEuNDk5MSA0My4wNTIgNDcuMDYwOUM0Mi43MTM0IDQ2LjgyODggNDIuMTIxMSA0Ni40MDE5IDQxLjk0NSA0Ni4yNzEyQzQxLjM5NzcgNDYuMzQ1NCA0MC44NDQ1IDQ2LjUxNTEgNDAuMTI0MSA0Ni4wOTM1QzM4Ljc1MjIgNDUuMTY1NyAzNy41MDI4IDQzLjg4NTEgMzUuOTkxIDQyLjM0MjRDMzUuMjk4MiA0MS42MDQ0IDM0Ljc5NjYgNDAuOTAxOCAzMy45NzM1IDQwLjE5MDRDMzMuNzg2NiA0MC4wMjg5IDMzLjUwMTQgMzkuODEwNCAzMy4yOTIzIDM5LjY0NDJDMzIuNjQ4OSAzOS4xMjg4IDMxLjg5IDM4Ljg2IDMxLjE1NyAzOC44MzQ4QzMwLjIxNDcgMzguODAyNCAyOS4zMDc1IDM5LjE3MjUgMjguNzEzOCAzOS45MjA2QzI3LjY1ODQgNDEuMjUwNiAyNy45OTYzIDQzLjI4MzMgMjkuNDY3MSA0NC40NjE0QzI5LjQ4MiA0NC40NzM0IDI5LjQ5NzkgNDQuNDgyNyAyOS41MTI5IDQ0LjQ5NDNDMjkuNzE1IDQ0LjY1ODkgMjkuOTYyNSA0NC44Njk4IDMwLjE0ODMgNDUuMDA3NkMzMS4wMjE3IDQ1LjY1NTUgMzEuODE5NSA0NS45ODcyIDMyLjY4OTcgNDYuNTAxNUMzNC41MjMxIDQ3LjYzOTEgMzYuMDQzIDQ4LjU4MjMgMzcuMjQ4NiA0OS43MTk2QzM3LjcxOTQgNTAuMjIzNyAzNy44MDE2IDUxLjExMjIgMzcuODY0MyA1MS40OTY0TDM4Ljg0NjggNTIuMzc4MkMzMy41ODcyIDYwLjMzMDggMzEuMTUzIDcwLjE1MzkgMzIuNTkxNSA4MC4xNjI3TDMxLjMwNzcgODAuNTM3OEMzMC45NjkzIDgwLjk3NjggMzAuNDkxMiA4MS42Njc2IDI5Ljk5MTEgODEuODczOEMyOC40MTM4IDgyLjM3MjkgMjYuNjM4NyA4Mi41NTYyIDI0LjQ5NTYgODIuNzgxOUMyMy40ODk0IDgyLjg2NiAyMi42MjEzIDgyLjgxNTggMjEuNTU0NiA4My4wMTg4QzIxLjMxOTggODMuMDYzNSAyMC45OTI3IDgzLjE0OTEgMjAuNzM1OCA4My4yMDk3QzIwLjcyNjkgODMuMjExNiAyMC43MTg2IDgzLjIxNDIgMjAuNzA5NiA4My4yMTYyQzIwLjY5NTYgODMuMjE5NSAyMC42NzcyIDgzLjIyNjMgMjAuNjYzOCA4My4yMjk0QzE4Ljg1NyA4My42NjggMTcuNjk2MyA4NS4zMzY1IDE4LjA2OTkgODYuOTgwNUMxOC40NDM3IDg4LjYyNDggMjAuMjA4NiA4OS42MjQ4IDIyLjAyNjIgODkuMjMxMkMyMi4wMzkzIDg5LjIyODIgMjIuMDU4NCA4OS4yMjc3IDIyLjA3MiA4OS4yMjQ2QzIyLjA5MjYgODkuMjE5OSAyMi4xMTA2IDg5LjIwOTkgMjIuMTMxIDg5LjIwNDlDMjIuMzg0NCA4OS4xNDkgMjIuNzAxOSA4OS4wODY4IDIyLjkyMzYgODkuMDI3MkMyMy45NzIzIDg4Ljc0NTEgMjQuNzMxOCA4OC4zMzA2IDI1LjY3NDYgODcuOTY3N0MyNy43MDI5IDg3LjIzNjggMjkuMzgyOCA4Ni42MjYyIDMxLjAxOTUgODYuMzg4M0MzMS43MDMgODYuMzM0NSAzMi40MjMyIDg2LjgxMiAzMi43ODE0IDg3LjAxMzRMMzQuMTE3NyA4Ni43ODMxQzM3LjE5MjYgOTYuMzYxMyA0My42MzY2IDEwNC4xMDMgNTEuNzk2MyAxMDguOTYxTDUxLjIzOTYgMTEwLjMwM0M1MS40NDAzIDExMC44MjQgNTEuNjYxNiAxMTEuNTMgNTEuNTEyMSAxMTIuMDQ1QzUwLjkxNzEgMTEzLjU5NSA0OS44OTggMTE1LjIzMSA0OC43Mzc0IDExNy4wNTVDNDguMTc1NSAxMTcuODk4IDQ3LjYwMDQgMTE4LjU1MiA0Ny4wOTM0IDExOS41MTZDNDYuOTcyIDExOS43NDcgNDYuODE3NSAxMjAuMTAyIDQ2LjcwMDQgMTIwLjM0NkM0NS45MTI1IDEyMi4wMzkgNDYuNDkwNCAxMjMuOTkgNDguMDAzOCAxMjQuNzIyQzQ5LjUyNjggMTI1LjQ1OSA1MS40MTcxIDEyNC42ODIgNTIuMjM1MiAxMjIuOTg1QzUyLjIzNjQgMTIyLjk4MiA1Mi4yNDA2IDEyMi45OCA1Mi4yNDE3IDEyMi45NzhDNTIuMjQyNiAxMjIuOTc2IDUyLjI0MDkgMTIyLjk3MyA1Mi4yNDE3IDEyMi45NzFDNTIuMzU4MiAxMjIuNzMxIDUyLjUyMzMgMTIyLjQxNSA1Mi42MjE2IDEyMi4xODhDNTMuMDU2IDEyMS4xODkgNTMuMjAwNSAxMjAuMzMyIDUzLjUwNTkgMTE5LjM2NUM1NC4zMTcgMTE3LjMxOCA1NC43NjI2IDExNS4xNyA1NS44NzkxIDExMy44MzJDNTYuMTg0OSAxMTMuNDY2IDU2LjY4MzMgMTEzLjMyNSA1Ny4yMDAxIDExMy4xODZMNTcuODk0NCAxMTEuOTIyQzY1LjAwOCAxMTQuNjY1IDcyLjk3MDUgMTE1LjQwMiA4MC45MjQ1IDExMy41ODdDODIuNzM5MSAxMTMuMTczIDg0LjQ5MDggMTEyLjYzNyA4Ni4xODQzIDExMS45OTRDODYuMzc5NCAxMTIuMzQyIDg2Ljc0MiAxMTMuMDExIDg2LjgzOTMgMTEzLjE3OUM4Ny4zNjQ0IDExMy4zNTEgODcuOTM3NyAxMTMuNDM5IDg4LjQwNDcgMTE0LjEzM0M4OS4yNDAxIDExNS41NjcgODkuODExNCAxMTcuMjYzIDkwLjUwNzMgMTE5LjMxMkM5MC44MTI4IDEyMC4yNzkgOTAuOTYzOCAxMjEuMTM2IDkxLjM5ODEgMTIyLjEzNkM5MS40OTcxIDEyMi4zNjMgOTEuNjYxNCAxMjIuNjg0IDkxLjc3OCAxMjIuOTI1QzkyLjU5NDQgMTI0LjYyOCA5NC40OTA3IDEyNS40MDcgOTYuMDE1OSAxMjQuNjY5Qzk3LjUyOTIgMTIzLjkzNyA5OC4xMDc3IDEyMS45ODYgOTcuMzE5NCAxMjAuMjkzQzk3LjIwMjMgMTIwLjA0OSA5Ny4wNDEyIDExOS42OTUgOTYuOTE5OCAxMTkuNDY0Qzk2LjQxMjcgMTE4LjQ5OSA5NS44Mzc3IDExNy44NTIgOTUuMjc1OCAxMTcuMDA5Qzk0LjExNTIgMTE1LjE4NSA5My4xNTI2IDExMy42NyA5Mi41NTc1IDExMi4xMkM5Mi4zMDg3IDExMS4zMiA5Mi41OTk1IDExMC44MjMgOTIuNzkzMyAxMTAuMzAzQzkyLjY3NzIgMTEwLjE3IDkyLjQyODggMTA5LjQxNCA5Mi4yODI0IDEwOS4wNTlDMTAwLjc2MiAxMDQuMDI5IDEwNy4wMTcgOTUuOTk4NSAxMDkuOTU1IDg2LjcyMzlDMTEwLjM1MSA4Ni43ODY1IDExMS4wNDEgODYuOTA5MSAxMTEuMjY1IDg2Ljk1NDJDMTExLjcyNiA4Ni42NDg3IDExMi4xNDkgODYuMjUwMSAxMTIuOTgxIDg2LjMxNTlDMTE0LjYxNyA4Ni41NTM3IDExNi4yOTcgODcuMTY0NSAxMTguMzI2IDg3Ljg5NTNDMTE5LjI2OCA4OC4yNTgxIDEyMC4wMjggODguNjc5MyAxMjEuMDc3IDg4Ljk2MTRDMTIxLjI5OCA4OS4wMjEgMTIxLjYxNiA4OS4wNzY2IDEyMS44NjkgODkuMTMyNUMxMjEuODg5IDg5LjEzNzUgMTIxLjkwOCA4OS4xNDc1IDEyMS45MjggODkuMTUyMkMxMjEuOTQyIDg5LjE1NTMgMTIxLjk2MSA4OS4xNTU4IDEyMS45NzQgODkuMTU4OEMxMjMuNzkyIDg5LjU1MiAxMjUuNTU3IDg4LjU1MjYgMTI1LjkzIDg2LjkwODFDMTI2LjMwMyA4NS4yNjQxIDEyNS4xNDMgODMuNTk1MiAxMjMuMzM2IDgzLjE1N0MxMjMuMDc0IDgzLjA5NyAxMjIuNzAxIDgyLjk5NSAxMjIuNDQ2IDgyLjk0NjVDMTIxLjM3OSA4Mi43NDM1IDEyMC41MTEgODIuNzkzNSAxMTkuNTA1IDgyLjcwOTVDMTE3LjM2MSA4Mi40ODM5IDExNS41ODYgODIuMzAwNCAxMTQuMDA5IDgxLjgwMTRDMTEzLjM2NiA4MS41NTA3IDExMi45MDggODAuNzgxOSAxMTIuNjg2IDgwLjQ2NTVMMTExLjQ0OCA4MC4xMDM1QzExMi4wOSA3NS40MzggMTExLjkxNyA3MC41ODI1IDExMC44MDYgNjUuNzI0M0MxMDkuNjg1IDYwLjgyMDggMTA3LjcwNCA1Ni4zMzYxIDEwNS4wNjIgNTIuMzg0OEMxMDUuMzc5IDUyLjA5NDggMTA1Ljk3OSA1MS41NjEyIDEwNi4xNDkgNTEuNDA0M0MxMDYuMTk5IDUwLjg1MTcgMTA2LjE1NiA1MC4yNzIyIDEwNi43MjUgNDkuNjYwM0MxMDcuOTMxIDQ4LjUyMyAxMDkuNDUxIDQ3LjU3OTkgMTExLjI4NCA0Ni40NDIzQzExMi4xNTQgNDUuOTI3OSAxMTIuOTU5IDQ1LjU5NjQgMTEzLjgzMiA0NC45NDg0QzExNC4wMyA0NC44MDE5IDExNC4yOTkgNDQuNTY5OSAxMTQuNTA3IDQ0LjQwMjJDMTE1Ljk3NyA0My4yMjM3IDExNi4zMTYgNDEuMTkxMSAxMTUuMjYgMzkuODYxNEMxMTQuMjA0IDM4LjUzMTcgMTEyLjE1OSAzOC40MDY1IDExMC42ODggMzkuNTg1QzExMC40NzkgMzkuNzUxNiAxMTAuMTk1IDM5Ljk2ODggMTEwLjAwNyA0MC4xMzEyQzEwOS4xODQgNDAuODQyNiAxMDguNjc2IDQxLjU0NTIgMTA3Ljk4MyA0Mi4yODMyQzEwNi40NzEgNDMuODI1OSAxMDUuMjIyIDQ1LjExMyAxMDMuODUgNDYuMDQwOUMxMDMuMjU1IDQ2LjM4ODUgMTAyLjM4NSA0Ni4yNjgyIDEwMS45OSA0Ni4yNDQ5TDEwMC44MjQgNDcuMDgwNkM5NC4xNzUzIDQwLjA3NjMgODUuMTIzNSAzNS41OTgyIDc1LjM3NjYgMzQuNzI4M0M3NS4zNDk0IDM0LjMxNzkgNzUuMzEzNyAzMy41NzYxIDc1LjMwNDYgMzMuMzUyOUM3NC45MDU2IDMyLjk2OTMgNzQuNDIzNSAzMi42NDE4IDc0LjMwMjQgMzEuODEzQzc0LjE2OTEgMzAuMTU2OSA3NC4zOTE3IDI4LjM3NTIgNzQuNjQ5NiAyNi4yMjU4Qzc0Ljc5MTkgMjUuMjIxNiA3NS4wMjg0IDI0LjM4NzMgNzUuMDY4OCAyMy4yOTczQzc1LjA3OCAyMy4wNDk1IDc1LjA2MzIgMjIuNjkgNzUuMDYyMiAyMi40MjIxQzc1LjA2MiAyMC41MzIyIDczLjY4OTggMTguOTk5OCA3MS45OTY4IDE5Wk02OC4xNTg1IDQyLjg4ODZMNjcuMjQ4IDU5LjA0NDdMNjcuMTgyNSA1OS4wNzc2QzY3LjEyMTQgNjAuNTIyOSA2NS45Mzc1IDYxLjY3NyA2NC40ODM5IDYxLjY3N0M2My44ODg0IDYxLjY3NyA2My4zMzg4IDYxLjQ4NDkgNjIuODkyMiA2MS4xNTcxTDYyLjg2NiA2MS4xNzAzTDQ5LjY4MDcgNTEuNzc5NEM1My43MzMxIDQ3Ljc3NTkgNTguOTE2NCA0NC44MTcyIDY0Ljg5IDQzLjQ1NDZDNjUuOTgxMiA0My4yMDU2IDY3LjA3MTkgNDMuMDIwOSA2OC4xNTg1IDQyLjg4ODZaTTc1Ljg0MTcgNDIuODg4NkM4Mi44MTU5IDQzLjc1MDQgODkuMjY1NyA0Ni45MjMyIDk0LjIwODEgNTEuNzg2TDgxLjEwOCA2MS4xMTc2TDgxLjA2MjEgNjEuMDk3OUM3OS44OTk0IDYxLjk1MTIgNzguMjYxMSA2MS43Mzk0IDc3LjM1NDggNjAuNTk3OEM3Ni45ODM1IDYwLjEzMDEgNzYuNzg4NyA1OS41ODAxIDc2Ljc2NTMgNTkuMDI0OUw3Ni43NTIyIDU5LjAxODRMNzUuODQxNyA0Mi44ODg2Wk00NC44OTkxIDU3LjgxNEw1Ni45MzgyIDY4LjYzM0w1Ni45MjUxIDY4LjY5ODhDNTguMDExNyA2OS42NDc5IDU4LjE3MiA3MS4yOTQ5IDU3LjI2NTcgNzIuNDM2OEM1Ni44OTQ0IDcyLjkwNDUgNTYuMzk3NSA3My4yMTgyIDU1Ljg2MzkgNzMuMzY0N0w1NS44NTA4IDczLjQxNzNMNDAuNDE4OCA3Ny44OTIzQzM5LjYzMzQgNzAuNjc2NSA0MS4zMjYxIDYzLjY2MjEgNDQuODk5MSA1Ny44MTRaTTk5LjAwOTQgNTcuODIwNkMxMDAuNzk4IDYwLjczMzYgMTAyLjE1MyA2My45ODcxIDEwMi45NTkgNjcuNTE0M0MxMDMuNzU2IDcwLjk5OTEgMTAzLjk1NiA3NC40Nzc4IDEwMy42MjcgNzcuODM5N0w4OC4xMTY2IDczLjM1MTVMODguMTAzNSA3My4yODU3Qzg2LjcxNDUgNzIuOTA0MyA4NS44NjA5IDcxLjQ4NDggODYuMTg0MyA3MC4wNjExQzg2LjMxNjggNjkuNDc3OCA4Ni42MjQ5IDY4Ljk4NDQgODcuMDQyMyA2OC42MTk4TDg3LjAzNTggNjguNTg2OUw5OS4wMDk0IDU3LjgyMDZaTTY5LjUyNzQgNjkuNDY4OEg3NC40NTk2TDc3LjUyNTEgNzMuMzE4Nkw3Ni40MjQ3IDc4LjEyMjZMNzEuOTk2OCA4MC4yNjE0TDY3LjU1NTggNzguMTE2MUw2Ni40NTU0IDczLjMxMkw2OS41Mjc0IDY5LjQ2ODhaTTg1LjMzOTMgODIuNjQzN0M4NS41NDg5IDgyLjYzMzEgODUuNzU3NiA4Mi42NTIgODUuOTYxNiA4Mi42ODk4TDg1Ljk4NzggODIuNjU2OUwxMDEuOTUgODUuMzY4MkM5OS42MTQyIDkxLjk2MjQgOTUuMTQ0IDk3LjY3NSA4OS4xNzExIDEwMS40OThMODIuOTc0NyA4Ni40NjA2TDgyLjk5NDQgODYuNDM0M0M4Mi40MjUyIDg1LjEwNTUgODIuOTk0OCA4My41NDcyIDg0LjMwNDQgODIuOTEzNUM4NC42Mzk3IDgyLjc1MTMgODQuOTkgODIuNjYxNCA4NS4zMzkzIDgyLjY0MzdaTTU4LjUyOTggODIuNzA5NUM1OS43NDggODIuNzI2NyA2MC44NDA2IDgzLjU3NjEgNjEuMTIzNyA4NC44MjJDNjEuMjU2MiA4NS40MDUyIDYxLjE5MTcgODUuOTgzMSA2MC45NzMgODYuNDkzNUw2MS4wMTg5IDg2LjU1MjdMNTQuODg4IDEwMS40MzlDNDkuMTU1OSA5Ny43NDMyIDQ0LjU5MDQgOTIuMjA5OSA0Mi4xNDgxIDg1LjQyMDhMNTcuOTczMSA4Mi43MjI3TDU3Ljk5OTMgODIuNzU1NkM1OC4xNzYzIDgyLjcyMjkgNTguMzU1OCA4Mi43MDcxIDU4LjUyOTggODIuNzA5NVpNNzEuODk4NiA4OS4yMzEyQzcyLjMyMjkgODkuMjE1NSA3Mi43NTM0IDg5LjMwMyA3My4xNjI3IDg5LjUwMUM3My42OTkyIDg5Ljc2MDYgNzQuMTEzNiA5MC4xNjkyIDc0LjM3NDUgOTAuNjU5Mkg3NC40MzM0TDgyLjIzNDYgMTA0LjgyMUM4MS4yMjIxIDEwNS4xNjIgODAuMTgxMyAxMDUuNDU0IDc5LjExNjcgMTA1LjY5N0M3My4xNTA1IDEwNy4wNTggNjcuMjAzMiAxMDYuNjQ1IDYxLjgxOCAxMDQuODAyTDY5LjU5OTUgOTAuNjY1OEg2OS42MTI2QzcwLjA3OTUgODkuNzg4OCA3MC45NjUgODkuMjY1NiA3MS44OTg2IDg5LjIzMTJaIiBmaWxsPSJ3aGl0ZSIgc3Ryb2tlPSJ3aGl0ZSIgc3Ryb2tlLXdpZHRoPSIwLjI1Ii8+CjxkZWZzPgo8bGluZWFyR3JhZGllbnQgaWQ9InBhaW50MF9saW5lYXJfNjgxXzI4NDUiIHgxPSIxMCIgeTE9IjE1LjUiIHgyPSIxNDQiIHkyPSIxMzEuNSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiPgo8c3RvcCBzdG9wLWNvbG9yPSIjNEQ4N0ZGIi8+CjxzdG9wIG9mZnNldD0iMSIgc3RvcC1jb2xvcj0iIzA1NDdEMCIvPgo8L2xpbmVhckdyYWRpZW50Pgo8L2RlZnM+Cjwvc3ZnPgo= - keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "nodeGroups"], ["spec", "nodeGroups", "md0"], ["spec", "nodeGroups", "md0", "minReplicas"], ["spec", "nodeGroups", "md0", "maxReplicas"], ["spec", "nodeGroups", "md0", "instanceType"], ["spec", "nodeGroups", "md0", "ephemeralStorage"], ["spec", "nodeGroups", "md0", "roles"], ["spec", "nodeGroups", "md0", "resources"], ["spec", "nodeGroups", "md0", "gpus"], ["spec", "version"], ["spec", "host"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "fluxcd"], ["spec", "addons", "fluxcd", "enabled"], ["spec", "addons", "fluxcd", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"]] + keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "nodeGroups"], ["spec", "nodeGroups", "md0"], ["spec", "nodeGroups", "md0", "minReplicas"], ["spec", "nodeGroups", "md0", "maxReplicas"], ["spec", "nodeGroups", "md0", "instanceType"], ["spec", "nodeGroups", "md0", "ephemeralStorage"], ["spec", "nodeGroups", "md0", "roles"], ["spec", "nodeGroups", "md0", "resources"], ["spec", "nodeGroups", "md0", "gpus"], ["spec", "version"], ["spec", "host"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "fluxcd"], ["spec", "addons", "fluxcd", "enabled"], ["spec", "addons", "fluxcd", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"], ["spec", "images"], ["spec", "images", "waitForKubeconfig"]] secrets: exclude: [] include: diff --git a/pkg/cmd/server/start.go b/pkg/cmd/server/start.go index f7791332..7ebbbc89 100644 --- a/pkg/cmd/server/start.go +++ b/pkg/cmd/server/start.go @@ -160,6 +160,34 @@ func (o *CozyServerOptions) Complete() error { // Convert to ResourceConfig o.ResourceConfig = &config.ResourceConfig{} for _, crd := range crdList.Items { + release := config.ReleaseConfig{ + Prefix: crd.Spec.Release.Prefix, + Labels: crd.Spec.Release.Labels, + ChartRef: config.ChartRefConfig{ + Kind: crd.Spec.Release.ChartRef.Kind, + Name: crd.Spec.Release.ChartRef.Name, + Namespace: crd.Spec.Release.ChartRef.Namespace, + }, + } + // Per-Application HelmRelease Install/Upgrade timeout. Applications + // whose parent chart contains asynchronously-provisioned resources + // the chart itself depends on (for example, the Kamaji-provisioned + // admin-kubeconfig Secret for Kubernetes tenants) need a longer + // wait budget than the Flux default. Consumed by the REST storage + // layer when building the HelmRelease Spec. The parser rejects + // units Flux would reject at webhook time, so a bad annotation + // surfaces as a loud startup failure instead of a silent drop to + // defaults. + d, err := config.ParseHelmInstallTimeoutAnnotation( + crd.Annotations[config.HelmInstallTimeoutAnnotation], + ) + if err != nil { + return fmt.Errorf( + "ApplicationDefinition %q has invalid %s annotation: %w", + crd.Name, config.HelmInstallTimeoutAnnotation, err, + ) + } + release.HelmInstallTimeout = d resource := config.Resource{ Application: config.ApplicationConfig{ Kind: crd.Spec.Application.Kind, @@ -168,15 +196,7 @@ func (o *CozyServerOptions) Complete() error { ShortNames: []string{}, // TODO: implement shortnames OpenAPISchema: crd.Spec.Application.OpenAPISchema, }, - Release: config.ReleaseConfig{ - Prefix: crd.Spec.Release.Prefix, - Labels: crd.Spec.Release.Labels, - ChartRef: config.ChartRefConfig{ - Kind: crd.Spec.Release.ChartRef.Kind, - Name: crd.Spec.Release.ChartRef.Name, - Namespace: crd.Spec.Release.ChartRef.Namespace, - }, - }, + Release: release, } o.ResourceConfig.Resources = append(o.ResourceConfig.Resources, resource) } diff --git a/pkg/config/config.go b/pkg/config/config.go index 1e123e2c..21de27b2 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -16,6 +16,48 @@ limitations under the License. package config +import ( + "fmt" + "regexp" + "time" +) + +// HelmInstallTimeoutAnnotation is the ApplicationDefinition metadata +// annotation key that overrides the Flux HelmRelease Install.Timeout and +// Upgrade.Timeout for a given Application kind. +const HelmInstallTimeoutAnnotation = "release.cozystack.io/helm-install-timeout" + +// helmTimeoutPattern mirrors the CRD validation pattern used by Flux +// helm-controller on HelmReleaseSpec.Install.Timeout (ms/s/m/h units only). +// time.ParseDuration accepts ns/us/µs, but Flux rejects them - parsing here +// with the same shape avoids feeding the controller a value it will later +// reject at webhook time. See +// github.com/fluxcd/helm-controller/api/v2 HelmReleaseSpec.Install.Timeout +// in the go module cache. +var helmTimeoutPattern = regexp.MustCompile(`^([0-9]+(\.[0-9]+)?(ms|s|m|h))+$`) + +// ParseHelmInstallTimeoutAnnotation parses the value of the +// release.cozystack.io/helm-install-timeout annotation. The empty string is +// treated as "unset" and returns (0, nil) so callers can leave +// HelmInstallTimeout zeroed and let flux defaults apply. Values accepted by +// time.ParseDuration but rejected by Flux (ns/us/µs) return a helpful +// error instead of silently parsing and failing later at HelmRelease +// admission. +func ParseHelmInstallTimeoutAnnotation(raw string) (time.Duration, error) { + if raw == "" { + return 0, nil + } + if !helmTimeoutPattern.MatchString(raw) { + return 0, fmt.Errorf("must match %s (Flux accepts ms/s/m/h units only), got %q", + helmTimeoutPattern, raw) + } + d, err := time.ParseDuration(raw) + if err != nil { + return 0, fmt.Errorf("time.ParseDuration(%q): %w", raw, err) + } + return d, nil +} + // ResourceConfig represents the structure of the configuration file. type ResourceConfig struct { Resources []Resource `yaml:"resources"` @@ -41,6 +83,12 @@ type ReleaseConfig struct { Prefix string `yaml:"prefix"` Labels map[string]string `yaml:"labels"` ChartRef ChartRefConfig `yaml:"chartRef"` + // HelmInstallTimeout overrides the Flux HelmRelease Install.Timeout and + // Upgrade.Timeout for this Application kind. When zero, flux defaults + // apply. Populated from the + // release.cozystack.io/helm-install-timeout annotation on the + // ApplicationDefinition at start-up. + HelmInstallTimeout time.Duration `yaml:"helmInstallTimeout,omitempty"` } // ChartRefConfig references a Flux source artifact for the Helm chart. diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go new file mode 100644 index 00000000..eb6990af --- /dev/null +++ b/pkg/config/config_test.go @@ -0,0 +1,119 @@ +package config + +import ( + "strings" + "testing" + "time" +) + +// Cover the annotation parser used by cozystack-api at startup. The parser +// is consumed by pkg/cmd/server/start.go on every ApplicationDefinition; a +// typo here silently drops back to flux defaults and the Kubernetes tenant +// race described in cozystack#2412 reappears, so the table must exercise: +// - the unset path (empty string treated as "no override"), +// - every unit Flux accepts (ms, s, m, h), +// - compound forms (the CRD pattern accepts repeats), +// - units time.ParseDuration accepts but Flux rejects (ns, us, µs), +// - outright garbage. +func TestParseHelmInstallTimeoutAnnotation(t *testing.T) { + cases := []struct { + name string + input string + want time.Duration + wantErr bool + errMatch string + }{ + { + name: "empty string leaves flux defaults in place", + input: "", + want: 0, + }, + { + name: "minutes", + input: "15m", + want: 15 * time.Minute, + }, + { + name: "hours", + input: "1h", + want: time.Hour, + }, + { + name: "seconds", + input: "45s", + want: 45 * time.Second, + }, + { + name: "milliseconds", + input: "500ms", + want: 500 * time.Millisecond, + }, + { + name: "compound hour and minutes", + input: "2h30m", + want: 2*time.Hour + 30*time.Minute, + }, + { + name: "decimal minutes", + input: "1.5m", + want: 90 * time.Second, + }, + { + name: "nanoseconds rejected - Flux CRD pattern excludes ns", + input: "500ns", + wantErr: true, + errMatch: "Flux accepts ms/s/m/h units only", + }, + { + name: "microseconds rejected - Flux CRD pattern excludes us", + input: "500us", + wantErr: true, + errMatch: "Flux accepts ms/s/m/h units only", + }, + { + name: "microseconds unicode rejected", + input: "500µs", + wantErr: true, + errMatch: "Flux accepts ms/s/m/h units only", + }, + { + name: "bare digits rejected", + input: "15", + wantErr: true, + errMatch: "Flux accepts ms/s/m/h units only", + }, + { + name: "garbage rejected", + input: "abc", + wantErr: true, + errMatch: "Flux accepts ms/s/m/h units only", + }, + { + name: "negative rejected", + input: "-15m", + wantErr: true, + errMatch: "Flux accepts ms/s/m/h units only", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := ParseHelmInstallTimeoutAnnotation(tc.input) + if tc.wantErr { + if err == nil { + t.Fatalf("expected error, got duration=%v", got) + } + if tc.errMatch != "" && !strings.Contains(err.Error(), tc.errMatch) { + t.Errorf("error %q does not contain %q", err.Error(), tc.errMatch) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != tc.want { + t.Errorf("got %v, want %v", got, tc.want) + } + }) + } +} diff --git a/pkg/registry/apps/application/rest.go b/pkg/registry/apps/application/rest.go index 0728ea13..d233ecc3 100644 --- a/pkg/registry/apps/application/rest.go +++ b/pkg/registry/apps/application/rest.go @@ -1528,6 +1528,24 @@ func (r *REST) convertApplicationToHelmRelease(app *appsv1alpha1.Application) (* }, } + // Per-Application HelmRelease wait budget. The mechanism is generic: + // an ApplicationDefinition that sets + // release.cozystack.io/helm-install-timeout gets Install.Timeout and + // Upgrade.Timeout populated from ReleaseConfig.HelmInstallTimeout + // (parsed at startup). Applications that leave it unset keep flux + // defaults so their failed installs remediate on the normal cadence. + // Today only kubernetes-rd carries the annotation because the + // Kubernetes Application's parent chart contains CAPI/Kamaji + // resources whose admin-kubeconfig Secret is provisioned + // asynchronously and Kamaji cold-start routinely exceeds flux's + // default wait budget; any future kind with the same shape can opt + // in by setting the same annotation. + if r.releaseConfig.HelmInstallTimeout > 0 { + timeout := metav1.Duration{Duration: r.releaseConfig.HelmInstallTimeout} + helmRelease.Spec.Install.Timeout = &timeout + helmRelease.Spec.Upgrade.Timeout = &timeout + } + return helmRelease, nil } diff --git a/pkg/registry/apps/application/rest_timeout_test.go b/pkg/registry/apps/application/rest_timeout_test.go new file mode 100644 index 00000000..85ac0b97 --- /dev/null +++ b/pkg/registry/apps/application/rest_timeout_test.go @@ -0,0 +1,116 @@ +package application + +import ( + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" + "github.com/cozystack/cozystack/pkg/config" +) + +func newRESTForTimeout(kind, prefix string, helmInstallTimeout time.Duration) *REST { + return &REST{ + kindName: kind, + releaseConfig: config.ReleaseConfig{ + Prefix: prefix, + ChartRef: config.ChartRefConfig{ + Kind: "HelmChart", + Name: "x", + Namespace: "cozy-system", + }, + HelmInstallTimeout: helmInstallTimeout, + }, + } +} + +// Table-driven: every Application kind carries a per-CRD HelmRelease wait +// budget. The Kubernetes kind's parent chart contains CAPI/Kamaji resources +// whose admin-kubeconfig Secret is provisioned asynchronously, so its +// ApplicationDefinition sets release.cozystack.io/helm-install-timeout=15m +// (or longer). Other kinds leave the annotation unset and keep flux defaults +// so their failed installs remediate on the normal cadence. The test must +// cover both paths: a kind with the timeout set and one without. +func TestConvertApplicationToHelmRelease_AppliesReleaseConfigTimeout(t *testing.T) { + cases := []struct { + name string + kind string + prefix string + configured time.Duration + wantSet bool + }{ + { + name: "Kubernetes kind with 15m configured gets Install and Upgrade Timeout", + kind: "Kubernetes", + prefix: "kubernetes-", + configured: 15 * time.Minute, + wantSet: true, + }, + { + // Fictional kind on purpose: the test is about the unset path + // regardless of which real Application kind ends up needing a + // timeout override. Using a real kind name would create false + // coupling to that Application's ApplicationDefinition. + name: "unrelated kind without configured timeout keeps flux defaults", + kind: "PlaceholderKindForDefaults", + prefix: "placeholder-", + configured: 0, + wantSet: false, + }, + { + name: "arbitrary future kind with 20m configured gets 20m", + kind: "TalosCluster", + prefix: "talos-", + configured: 20 * time.Minute, + wantSet: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := newRESTForTimeout(tc.kind, tc.prefix, tc.configured) + app := &appsv1alpha1.Application{ + ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "tenant-root"}, + } + + hr, err := r.convertApplicationToHelmRelease(app) + if err != nil { + t.Fatalf("convertApplicationToHelmRelease returned error: %v", err) + } + + if hr.Spec.Install == nil || hr.Spec.Upgrade == nil { + t.Fatalf("Spec.Install/Upgrade must be non-nil") + } + + if tc.wantSet { + if hr.Spec.Install.Timeout == nil { + t.Fatalf("Spec.Install.Timeout must be set when HelmInstallTimeout=%v", tc.configured) + } + if hr.Spec.Install.Timeout.Duration != tc.configured { + t.Errorf("Spec.Install.Timeout = %v, want %v", hr.Spec.Install.Timeout.Duration, tc.configured) + } + if hr.Spec.Upgrade.Timeout == nil { + t.Fatalf("Spec.Upgrade.Timeout must be set when HelmInstallTimeout=%v", tc.configured) + } + if hr.Spec.Upgrade.Timeout.Duration != tc.configured { + t.Errorf("Spec.Upgrade.Timeout = %v, want %v", hr.Spec.Upgrade.Timeout.Duration, tc.configured) + } + } else { + if hr.Spec.Install.Timeout != nil { + t.Errorf("Spec.Install.Timeout must be nil when HelmInstallTimeout=0, got %v", hr.Spec.Install.Timeout.Duration) + } + if hr.Spec.Upgrade.Timeout != nil { + t.Errorf("Spec.Upgrade.Timeout must be nil when HelmInstallTimeout=0, got %v", hr.Spec.Upgrade.Timeout.Duration) + } + } + + if hr.Spec.Install.Remediation == nil || hr.Spec.Install.Remediation.Retries != -1 { + t.Errorf("Spec.Install.Remediation.Retries must remain -1, got %+v", hr.Spec.Install.Remediation) + } + if hr.Spec.Upgrade.Remediation == nil || hr.Spec.Upgrade.Remediation.Retries != -1 { + t.Errorf("Spec.Upgrade.Remediation.Retries must remain -1, got %+v", hr.Spec.Upgrade.Remediation) + } + }) + } +}