fix(etcd): remove destructive post-upgrade cert-regeneration hook (#2462)
## What this PR does Removes the `post-upgrade` Helm hook in `packages/extra/etcd` that was `kubectl delete`ing the etcd TLS chain (`etcd-ca-tls`, `etcd-peer-ca-tls`, `etcd-client-tls`, `etcd-peer-tls`, `etcd-server-tls`) and then deleting etcd pods on every chart upgrade. The hook was gated by a semver compare of a stored `etcd-deployed-version` ConfigMap against `2.6.1`. That gate was written when chart versions looked like `2.6.0`, `2.6.1`, etc. Since commit `f871fbdb` ("Remove versions_map logic") all chart versions are stamped as `0.0.0+<git-hash>`, which per semver is always `< 2.6.1`, so the gate always evaluates to "regenerate certs" and the destructive hook fires on every upgrade. On clusters running Kamaji-managed tenant control planes the consequence is severe: wiping the etcd CA triggers cert-manager to re-issue it with a brand-new CA, but Kamaji's `datastore-certificate` Secrets still carry the old CA bundle mounted into tenant `kube-apiserver` pods. Those apiservers hit `x509: certificate signed by unknown authority` against `etcd.<ns>.svc:2379` and go into CrashLoopBackOff until each tenant `DataStore` is individually force-reconciled and the Deployments rolled. Commit `47d81f70` ("Disabled private key rotation in CA certs") already fixed the underlying `rotationPolicy: Always` issue the hook was written to paper over in March 2025, so the migration has no remaining use. Changes: - Delete `templates/hook/{job,role,rolebinding,serviceaccount}.yaml` - Delete `templates/version.yaml` (only the hook read `etcd-deployed-version`) - Add `tests/no-post-upgrade-hook_test.yaml` as a regression guard — if `templates/hook/job.yaml` or `templates/version.yaml` is ever brought back, `make unit-tests` fails ### Release note ```release-note fix(etcd): remove destructive post-upgrade cert-regeneration hook. Previously the etcd chart ran a `post-upgrade` hook on every Helm upgrade that deleted etcd TLS Secrets and pods, causing cert-manager to re-issue the etcd CA. On clusters with Kamaji-managed tenant control planes this put every tenant `kube-apiserver` into CrashLoopBackOff until each DataStore was manually re-reconciled. The hook was a one-shot `2.6.0 -> 2.6.1` migration that became a permanent footgun once chart versioning moved to `0.0.0+<git-hash>` and the underlying `rotationPolicy: Always` issue was fixed in commit `47d81f70`. This is behavior removal, not cosmetic cleanup. ``` <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **Tests** * Added Helm chart test suite verifying absence of post-upgrade hook Job and version ConfigMap. * **Chores** * Removed post-upgrade hook Job and associated RBAC resources (Role, RoleBinding, ServiceAccount). * Removed version ConfigMap. * Added `test` target to Makefile for running Helm chart unit tests. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
commit
36e326d768
7 changed files with 37 additions and 93 deletions
|
|
@ -5,3 +5,6 @@ include ../../../hack/package.mk
|
|||
generate:
|
||||
cozyvalues-gen -v values.yaml -s values.schema.json -r README.md
|
||||
../../../hack/update-crd.sh
|
||||
|
||||
test:
|
||||
helm unittest .
|
||||
|
|
|
|||
|
|
@ -1,39 +0,0 @@
|
|||
{{- $shouldUpdateCerts := true }}
|
||||
{{- $configMap := lookup "v1" "ConfigMap" .Release.Namespace "etcd-deployed-version" }}
|
||||
{{- if $configMap }}
|
||||
{{- $deployedVersion := index $configMap "data" "version" }}
|
||||
{{- if $deployedVersion | semverCompare ">= 2.6.1" }}
|
||||
{{- $shouldUpdateCerts = false }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{- if $shouldUpdateCerts }}
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: etcd-hook
|
||||
annotations:
|
||||
helm.sh/hook: post-upgrade
|
||||
helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
|
||||
spec:
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
policy.cozystack.io/allow-to-apiserver: "true"
|
||||
spec:
|
||||
serviceAccountName: etcd-hook
|
||||
containers:
|
||||
- name: kubectl
|
||||
image: docker.io/alpine/k8s:1.33.4
|
||||
command:
|
||||
- sh
|
||||
args:
|
||||
- -exc
|
||||
- |-
|
||||
kubectl --namespace={{ .Release.Namespace }} delete secrets etcd-ca-tls etcd-peer-ca-tls
|
||||
sleep 10
|
||||
kubectl --namespace={{ .Release.Namespace }} delete secrets etcd-client-tls etcd-peer-tls etcd-server-tls
|
||||
kubectl --namespace={{ .Release.Namespace }} delete pods --selector=app.kubernetes.io/instance=etcd,app.kubernetes.io/managed-by=etcd-operator,app.kubernetes.io/name=etcd,cozystack.io/service=etcd
|
||||
restartPolicy: Never
|
||||
{{- end }}
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
annotations:
|
||||
helm.sh/hook: post-upgrade
|
||||
helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
|
||||
name: etcd-hook
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- secrets
|
||||
- pods
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- delete
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- configmaps
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: etcd-hook
|
||||
annotations:
|
||||
helm.sh/hook: post-upgrade
|
||||
helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: etcd-hook
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: etcd-hook
|
||||
namespace: {{ .Release.Namespace | quote }}
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: etcd-hook
|
||||
annotations:
|
||||
helm.sh/hook: post-upgrade
|
||||
helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: etcd-deployed-version
|
||||
data:
|
||||
version: {{ .Chart.Version }}
|
||||
34
packages/extra/etcd/tests/no-post-upgrade-hook_test.yaml
Normal file
34
packages/extra/etcd/tests/no-post-upgrade-hook_test.yaml
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
suite: etcd chart does not ship a destructive post-upgrade cert-regeneration hook
|
||||
|
||||
release:
|
||||
name: etcd
|
||||
namespace: tenant-root
|
||||
|
||||
templates:
|
||||
- templates/check-release-name.yaml
|
||||
- templates/dashboard-resourcemap.yaml
|
||||
- templates/datastore.yaml
|
||||
- templates/etcd-defrag.yaml
|
||||
- templates/hook/job.yaml
|
||||
- templates/podscrape.yaml
|
||||
- templates/prometheus-rules.yaml
|
||||
- templates/version.yaml
|
||||
|
||||
tests:
|
||||
- it: renders no Job named etcd-hook
|
||||
documentSelector:
|
||||
path: metadata.name
|
||||
value: etcd-hook
|
||||
skipEmptyTemplates: true
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
|
||||
- it: renders no ConfigMap named etcd-deployed-version
|
||||
documentSelector:
|
||||
path: metadata.name
|
||||
value: etcd-deployed-version
|
||||
skipEmptyTemplates: true
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
Loading…
Add table
Add a link
Reference in a new issue