From b46151a45b4f1d5adabd3457bd3ee75c539e0f59 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Tue, 28 Apr 2026 11:43:02 +0500 Subject: [PATCH 1/2] fix(installer): bootstrap cozy-system via --create-namespace + label hook Removes the chart's `Namespace cozy-system` resource and replaces it with a pre-install/pre-upgrade Job hook (cozy-system-labeler) that patches the required labels onto the namespace after `--create-namespace` creates it. Why: helm v3 has a known chicken-and-egg with charts that ship their own Namespace: - WITH `--create-namespace` on the install command, helm pre-creates the namespace via plain kubectl-create (no helm meta annotations); the chart's own Namespace apply then fails with `already exists`. - WITHOUT `--create-namespace`, helm fails immediately because it cannot write its release-secret to a non-existent namespace. Until now this was hidden by the 3x retry on `Install Cozystack` in `.github/workflows/pull-requests.yaml`: first attempt always fails with the conflict, second attempt sees the existing failed release and takes the upgrade code path which patch-merges instead of strict-create. Reproducible on every cold install. Surfaced cleanly when retries on the install step were dropped. After this change: - install commands use `helm upgrade --install --namespace cozy-system --create-namespace`. Standard pattern, matches kube-prometheus-stack / argo-cd / cert-manager / others. - the pre-install hook (SA + ClusterRole + ClusterRoleBinding + Job) patches `cozystack.io/system=true` and `pod-security.kubernetes.io/enforce=privileged` onto the namespace before main resources apply. - hook-delete-policy=before-hook-creation,hook-succeeded so the RBAC surface only exists during install/upgrade. Verified end-to-end on a kind cluster: cold install in 3.3s, upgrade idempotent, cleanup clean. Image pinned to `alpine/k8s:1.32.0` for the hook (small, public, includes kubectl). Signed-off-by: Myasnikov Daniil --- .../templates/cozy-system-labels.yaml | 82 +++++++++++++++++++ .../templates/cozystack-operator.yaml | 10 --- 2 files changed, 82 insertions(+), 10 deletions(-) create mode 100644 packages/core/installer/templates/cozy-system-labels.yaml diff --git a/packages/core/installer/templates/cozy-system-labels.yaml b/packages/core/installer/templates/cozy-system-labels.yaml new file mode 100644 index 00000000..1ea3e5eb --- /dev/null +++ b/packages/core/installer/templates/cozy-system-labels.yaml @@ -0,0 +1,82 @@ +{{/* + Pre-install/upgrade hook that labels the cozy-system namespace with the + PodSecurity and cozystack identity labels. The chart no longer ships a + Namespace resource (helm v3 cannot adopt a namespace pre-created by + --create-namespace because it lacks helm meta annotations; without + --create-namespace, helm fails to write its release-secret because the + namespace does not exist yet — chicken-and-egg). + Install commands now use --create-namespace to bootstrap the namespace, + and this hook patches the labels the operator depends on (notably + pod-security.kubernetes.io/enforce=privileged for the host-network + operator pod). +*/}} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cozy-system-labeler + namespace: cozy-system + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-200" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cozy-system-labeler + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-180" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +rules: +- apiGroups: [""] + resources: ["namespaces"] + resourceNames: ["cozy-system"] + verbs: ["get", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cozy-system-labeler + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-160" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +subjects: +- kind: ServiceAccount + name: cozy-system-labeler + namespace: cozy-system +roleRef: + kind: ClusterRole + name: cozy-system-labeler + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: cozy-system-labeler + namespace: cozy-system + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-100" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +spec: + backoffLimit: 3 + ttlSecondsAfterFinished: 60 + template: + spec: + serviceAccountName: cozy-system-labeler + restartPolicy: OnFailure + containers: + - name: kubectl + image: alpine/k8s:1.32.0 + imagePullPolicy: IfNotPresent + command: + - kubectl + - label + - --overwrite + - namespace + - cozy-system + - cozystack.io/system=true + - pod-security.kubernetes.io/enforce=privileged diff --git a/packages/core/installer/templates/cozystack-operator.yaml b/packages/core/installer/templates/cozystack-operator.yaml index aded0995..461a71ed 100644 --- a/packages/core/installer/templates/cozystack-operator.yaml +++ b/packages/core/installer/templates/cozystack-operator.yaml @@ -4,16 +4,6 @@ {{- end -}} --- apiVersion: v1 -kind: Namespace -metadata: - name: cozy-system - labels: - cozystack.io/system: "true" - pod-security.kubernetes.io/enforce: privileged - annotations: - helm.sh/resource-policy: keep ---- -apiVersion: v1 kind: ServiceAccount metadata: name: cozystack From c0b76b168ed04154d234c0faf0b0c4a7b3dc8a78 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Tue, 28 Apr 2026 13:21:09 +0500 Subject: [PATCH 2/2] fix(installer): schedule cozy-system labeler hook on NotReady-NoSchedule nodes The pre-install Job runs before any CNI is installed, so all nodes are tainted NotReady-NoSchedule and the pod has no pod network. Without adjustment the Job's pod sits Pending until helm's pre-install timeout fires, blocking the entire install. Surfaced in PR #2500 CI run 25040584552 attempt 2: FailedScheduling 0/3 nodes are available: 3 node(s) had untolerated taint {node.kubernetes.io/not-ready: }. Mirror the cozystack-operator deployment's scheduling pattern: - hostNetwork=true so the pod doesn't depend on CNI - Tolerations matching the operator (not-ready / unreachable / cilium.agent-not-ready / cloudprovider.uninitialized) - Variant-aware KUBERNETES_SERVICE_HOST/PORT env so kubectl reaches the apiserver before kube-proxy + CNI are up: talos: localhost:7445 (KubePrism) generic: cozystack.apiServerHost / Port hosted: default in-cluster Verified on the sandbox kind cluster that the rendered Job manifest schedules and the kubectl container starts (the kind cluster itself doesn't run KubePrism so the env-var path can't be end-to-end-tested there; the Talos E2E will exercise it). Signed-off-by: Myasnikov Daniil --- .../templates/cozy-system-labels.yaml | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/packages/core/installer/templates/cozy-system-labels.yaml b/packages/core/installer/templates/cozy-system-labels.yaml index 1ea3e5eb..995e5e3a 100644 --- a/packages/core/installer/templates/cozy-system-labels.yaml +++ b/packages/core/installer/templates/cozy-system-labels.yaml @@ -68,10 +68,39 @@ spec: spec: serviceAccountName: cozy-system-labeler restartPolicy: OnFailure + # The hook runs BEFORE Cilium / kube-ovn / any CNI is installed, so the + # cluster's nodes are NotReady-NoSchedule and have no pod network. Mirror + # the cozystack-operator deployment's scheduling: hostNetwork=true so the + # pod doesn't need CNI, plus the same tolerations the operator uses. + hostNetwork: true + tolerations: + - key: "node.kubernetes.io/not-ready" + operator: "Exists" + - key: "node.kubernetes.io/unreachable" + operator: "Exists" + - key: "node.cilium.io/agent-not-ready" + operator: "Exists" + - key: "node.cloudprovider.kubernetes.io/uninitialized" + operator: "Exists" containers: - name: kubectl image: alpine/k8s:1.32.0 imagePullPolicy: IfNotPresent + {{- if eq .Values.cozystackOperator.variant "talos" }} + env: + # Talos KubePrism endpoint — same as cozystack-operator uses to reach + # the apiserver before CNI is up. + - name: KUBERNETES_SERVICE_HOST + value: "localhost" + - name: KUBERNETES_SERVICE_PORT + value: "7445" + {{- else if eq .Values.cozystackOperator.variant "generic" }} + env: + - name: KUBERNETES_SERVICE_HOST + value: {{ required "cozystack.apiServerHost is required in generic mode" .Values.cozystack.apiServerHost | quote }} + - name: KUBERNETES_SERVICE_PORT + value: {{ .Values.cozystack.apiServerPort | quote }} + {{- end }} command: - kubectl - label