diff --git a/packages/system/postgres-operator/Makefile b/packages/system/postgres-operator/Makefile index f6d7ddaa..5279f8a7 100644 --- a/packages/system/postgres-operator/Makefile +++ b/packages/system/postgres-operator/Makefile @@ -3,6 +3,9 @@ export NAMESPACE=cozy-$(NAME) include ../../../hack/package.mk +test: + helm unittest . + update: rm -rf charts helm repo add cnpg https://cloudnative-pg.github.io/charts diff --git a/packages/system/postgres-operator/templates/webhook-ready-hook.yaml b/packages/system/postgres-operator/templates/webhook-ready-hook.yaml new file mode 100644 index 00000000..51a8ba81 --- /dev/null +++ b/packages/system/postgres-operator/templates/webhook-ready-hook.yaml @@ -0,0 +1,148 @@ +{{- /* + Post-install gate: block the HelmRelease from reporting Ready until the + cnpg admission webhook actually serves through the cluster Service. Helm + --wait on the controller pod passes once its readinessProbe passes, but + EndpointSlice propagation and kube-proxy/cilium data-plane programming + can lag by a second or two — long enough for any HelmRelease that + depends on postgres-operator (e.g. cozy-keycloak, tenant Postgres apps) + to fire its own install and have kube-apiserver hit the mcluster.cnpg.io + mutating webhook with "dial tcp :443: connect: connection refused". + + The Job uses the apiserver service proxy, which exercises the same + endpoint-resolution and apiserver-initiated pod dial that the admission + webhook path uses. Once /readyz answers through the proxy the data-plane + race is resolved. It does not verify the webhook's TLS CA bundle, so + this gate is scoped to reachability regressions, not cert rotation. + + The service name and port name are hardcoded literals. Upstream cnpg + pins the service name in charts/cloudnative-pg/values.yaml with a + comment "DO NOT CHANGE THE SERVICE NAME as it is currently used to + generate the certificate and can not be configured". The port name is + fixed in charts/cloudnative-pg/templates/service.yaml (ports[0].name: + webhook-server). If a future `make update` ever changes either literal + upstream, the sync-check helm-unittest test + (tests/webhook-ready-hook_test.yaml) renders the subchart Service and + fails if the literal drifts — forcing this template to be updated in + the same change. +*/}} +{{- $_ := required "webhookReady.image.repository must be set to the container image providing kubectl for the post-install readiness Job" .Values.webhookReady.image.repository -}} +{{- $_ := required "webhookReady.image.tag must be set for the post-install readiness Job" .Values.webhookReady.image.tag -}} +{{- /* $svcName and $portName are hardcoded literals; see header comment. */ -}} +{{- $svcName := "cnpg-webhook-service" -}} +{{- /* $portName is the service port NAME, not number — matches ports[0].name in the vendored subchart's Service. */ -}} +{{- $portName := "webhook-server" -}} +{{- $resourceName := printf "https:%s:%s" $svcName $portName -}} +{{- $maxAttempts := .Values.webhookReady.maxAttempts | default 60 -}} +{{- $sleepSeconds := .Values.webhookReady.sleepSeconds | default 2 -}} +{{- /* Derive activeDeadlineSeconds from retries + 60s slack so a values override that raises maxAttempts doesn't get silently cut. */ -}} +{{- $deadline := add (mul (int $maxAttempts) (int $sleepSeconds)) 60 -}} +{{- $image := printf "%s:%s" .Values.webhookReady.image.repository .Values.webhookReady.image.tag -}} +{{- if .Values.webhookReady.image.digest }} +{{- $image = printf "%s@%s" $image .Values.webhookReady.image.digest -}} +{{- end }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }}-webhook-ready + namespace: {{ .Release.Namespace }} + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "0" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ .Release.Name }}-webhook-ready + namespace: {{ .Release.Namespace }} + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "0" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +rules: + - apiGroups: [""] + resources: ["services/proxy"] + resourceNames: [{{ $resourceName | quote }}] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Release.Name }}-webhook-ready + namespace: {{ .Release.Namespace }} + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "0" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ .Release.Name }}-webhook-ready +subjects: + - kind: ServiceAccount + name: {{ .Release.Name }}-webhook-ready + namespace: {{ .Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-webhook-ready + namespace: {{ .Release.Namespace }} + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "10" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +spec: + backoffLimit: {{ .Values.webhookReady.backoffLimit | default 2 }} + activeDeadlineSeconds: {{ $deadline }} + template: + spec: + restartPolicy: Never + serviceAccountName: {{ .Release.Name }}-webhook-ready + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: wait + image: {{ $image }} + imagePullPolicy: {{ if .Values.webhookReady.image.digest }}IfNotPresent{{ else }}Always{{ end }} + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + command: + - sh + - -c + - | + set -e + ns={{ .Release.Namespace }} + proxy="/api/v1/namespaces/${ns}/services/{{ $resourceName }}/proxy/readyz" + max_attempts={{ $maxAttempts }} + sleep_seconds={{ $sleepSeconds }} + i=0 + last_err="" + until last_err=$(kubectl get --raw "$proxy" 2>&1 >/dev/null); do + i=$((i + 1)) + if [ $i -gt $max_attempts ]; then + echo "timeout: cnpg webhook did not respond through the apiserver proxy after ${max_attempts} attempts (${sleep_seconds}s each)" + echo "last error: ${last_err}" + exit 1 + fi + if [ $((i % 10)) -eq 1 ]; then + echo "attempt $i/${max_attempts}: ${last_err}" + fi + sleep "$sleep_seconds" + done + echo "cnpg webhook is ready" diff --git a/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml b/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml new file mode 100644 index 00000000..3253c663 --- /dev/null +++ b/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml @@ -0,0 +1,328 @@ +suite: cnpg webhook post-install readiness gate + +templates: + - templates/webhook-ready-hook.yaml + - charts/cloudnative-pg/templates/service.yaml + +release: + name: postgres-operator + namespace: cozy-postgres-operator + +tests: + - it: renders four hook objects (SA + Role + RoleBinding + Job) + template: templates/webhook-ready-hook.yaml + asserts: + - hasDocuments: + count: 4 + + - it: every rendered object carries post-install and post-upgrade hook annotations + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 0 + equal: + path: metadata.annotations["helm.sh/hook"] + value: post-install,post-upgrade + - documentIndex: 1 + equal: + path: metadata.annotations["helm.sh/hook"] + value: post-install,post-upgrade + - documentIndex: 2 + equal: + path: metadata.annotations["helm.sh/hook"] + value: post-install,post-upgrade + - documentIndex: 3 + equal: + path: metadata.annotations["helm.sh/hook"] + value: post-install,post-upgrade + + - it: RBAC is created before the Job (hook-weight ordering) + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 0 + equal: + path: kind + value: ServiceAccount + - documentIndex: 0 + equal: + path: metadata.annotations["helm.sh/hook-weight"] + value: "0" + - documentIndex: 1 + equal: + path: kind + value: Role + - documentIndex: 1 + equal: + path: metadata.annotations["helm.sh/hook-weight"] + value: "0" + - documentIndex: 2 + equal: + path: kind + value: RoleBinding + - documentIndex: 2 + equal: + path: metadata.annotations["helm.sh/hook-weight"] + value: "0" + - documentIndex: 3 + equal: + path: kind + value: Job + - documentIndex: 3 + equal: + path: metadata.annotations["helm.sh/hook-weight"] + value: "10" + + - it: RBAC resourceName matches the exact proxy URL segment the Job probes + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 1 + equal: + path: rules[0].resources[0] + value: services/proxy + - documentIndex: 1 + equal: + path: rules[0].resourceNames[0] + value: https:cnpg-webhook-service:webhook-server + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: "/api/v1/namespaces/\\$\\{ns\\}/services/https:cnpg-webhook-service:webhook-server/proxy/readyz" + + - it: hardcoded service name in the hook matches the vendored cnpg subchart Service (drift guard for make update) + template: charts/cloudnative-pg/templates/service.yaml + asserts: + - equal: + path: metadata.name + value: cnpg-webhook-service + - equal: + path: spec.ports[0].name + value: webhook-server + + - it: Job calls kubectl get --raw on the proxy path + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: "kubectl get --raw" + + - it: Job image is digest-pinned when webhookReady.image.digest is set + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: v1.32 + digest: sha256:b9ef7d8dbe65bcc81a46c09b8dc7543103055021c4f43287bf59e92a8f4fe05c + asserts: + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].image + value: docker.io/clastix/kubectl:v1.32@sha256:b9ef7d8dbe65bcc81a46c09b8dc7543103055021c4f43287bf59e92a8f4fe05c + + - it: Job image falls back to tag-only when digest is not configured + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: v1.32 + digest: "" + asserts: + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].image + value: docker.io/clastix/kubectl:v1.32 + + - it: backoffLimit defaults to 2 so transient pod-level failures retry instead of killing the HelmRelease + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 3 + equal: + path: spec.backoffLimit + value: 2 + + - it: backoffLimit is overridable + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + backoffLimit: 5 + asserts: + - documentIndex: 3 + equal: + path: spec.backoffLimit + value: 5 + + - it: chart render fails when webhookReady.image.repository is empty + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: "" + tag: v1.32 + asserts: + - failedTemplate: + errorMessage: "webhookReady.image.repository must be set to the container image providing kubectl for the post-install readiness Job" + + - it: chart render fails when webhookReady.image.tag is empty + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: "" + asserts: + - failedTemplate: + errorMessage: "webhookReady.image.tag must be set for the post-install readiness Job" + + - it: retry loop bounds default when blanked so a wiped override still produces a working Job + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: v1.32 + maxAttempts: null + sleepSeconds: null + asserts: + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: "max_attempts=60" + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: "sleep_seconds=2" + + - it: Job pod runs non-root with seccomp RuntimeDefault for restricted-PSA clusters + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 3 + equal: + path: spec.template.spec.securityContext.runAsNonRoot + value: true + - documentIndex: 3 + equal: + path: spec.template.spec.securityContext.seccompProfile.type + value: RuntimeDefault + + - it: wait container declares resource requests and limits for predictable scheduling + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].resources.requests.cpu + value: 10m + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].resources.requests.memory + value: 32Mi + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].resources.limits.cpu + value: 100m + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].resources.limits.memory + value: 64Mi + + - it: Job container drops all capabilities and runs read-only rootfs + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation + value: false + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem + value: true + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].securityContext.capabilities.drop[0] + value: ALL + + - it: imagePullPolicy is IfNotPresent when digest-pinned, Always when tag-only + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: v1.32 + digest: sha256:b9ef7d8dbe65bcc81a46c09b8dc7543103055021c4f43287bf59e92a8f4fe05c + asserts: + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: IfNotPresent + + - it: imagePullPolicy is Always when no digest is configured + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: v1.32 + digest: "" + asserts: + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always + + - it: retry loop captures and surfaces the last kubectl error message on timeout + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'last_err=\$\(kubectl get --raw' + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'last error: \$\{last_err\}' + + - it: retry loop error message stays in sync when maxAttempts is bumped + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: v1.32 + maxAttempts: 90 + sleepSeconds: 3 + asserts: + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: "max_attempts=90" + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: "sleep_seconds=3" + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'after \$\{max_attempts\} attempts' + + - it: activeDeadlineSeconds scales with retry bounds so an override raise does not silently cut + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: v1.32 + maxAttempts: 180 + sleepSeconds: 3 + asserts: + - documentIndex: 3 + equal: + path: spec.activeDeadlineSeconds + value: 600 + + - it: activeDeadlineSeconds defaults include 60s slack over the default retry window + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 3 + equal: + path: spec.activeDeadlineSeconds + value: 180 diff --git a/packages/system/postgres-operator/values.yaml b/packages/system/postgres-operator/values.yaml index cae3dc53..c408220e 100644 --- a/packages/system/postgres-operator/values.yaml +++ b/packages/system/postgres-operator/values.yaml @@ -3,3 +3,30 @@ cloudnative-pg: create: true image: tag: "1.27.3" +# Image used by the post-install webhook-readiness Job (see templates/webhook-ready-hook.yaml). +# Any image with kubectl on PATH works; the Job calls the apiserver service proxy with the +# hook ServiceAccount's token to confirm the mcluster.cnpg.io webhook is reachable end-to-end +# before the HelmRelease reports Ready, closing the "connection refused" bootstrap race. +# +# The tag is digest-pinned so an upstream retag does not change what runs on every install +# and upgrade across the fleet. Refresh by resolving the current manifest-list digest +# (`docker manifest inspect docker.io/clastix/kubectl:v1.32`) and updating `digest` below. +# renovate: datasource=docker depName=docker.io/clastix/kubectl +webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: v1.32 + digest: sha256:b9ef7d8dbe65bcc81a46c09b8dc7543103055021c4f43287bf59e92a8f4fe05c + # Retry loop bounds for the readiness probe. Defaults total ~120s wall clock. + # Both are `default`-coerced in the template so an override that blanks them still + # produces a working Job. + maxAttempts: 60 + sleepSeconds: 2 + # Pod-level retry budget for transient node/registry failures (image pull rate limit, + # OOM, CNI hiccup). The shell loop inside the container only covers reachability retries. + # Pod retries and activeDeadlineSeconds (wall-clock bound on the whole Job across all + # pod retries) are ANDed, so activeDeadlineSeconds is the shorter of the two gates with + # the defaults above: the 60*2+60 = 180s deadline cuts the Job before backoffLimit=2 + # ever matters if pod-level failures eat more than ~60s of the budget. Raise + # maxAttempts/sleepSeconds alongside backoffLimit when tuning for slow image pulls. + backoffLimit: 2