diff --git a/packages/apps/tenant/templates/etcd.yaml b/packages/apps/tenant/templates/etcd.yaml index 9a122da2..e5047040 100644 --- a/packages/apps/tenant/templates/etcd.yaml +++ b/packages/apps/tenant/templates/etcd.yaml @@ -20,8 +20,11 @@ spec: interval: 5m timeout: 30m install: + # 10 retries provides reasonable tolerance for transient failures + # (network issues, temporary API unavailability, resource contention) + # while avoiding infinite loops. Total: 10 retries × 30m = 5 hours maximum remediation: - retries: -1 + retries: 10 upgrade: force: true remediation: diff --git a/packages/extra/etcd/README.md b/packages/extra/etcd/README.md index daf9f114..9e40d875 100644 --- a/packages/extra/etcd/README.md +++ b/packages/extra/etcd/README.md @@ -4,12 +4,14 @@ ### Common parameters -| Name | Description | Type | Value | -| ------------------ | ------------------------------------ | ---------- | ------- | -| `size` | Persistent Volume size. | `quantity` | `4Gi` | -| `storageClass` | StorageClass used to store the data. | `string` | `""` | -| `replicas` | Number of etcd replicas. | `int` | `3` | -| `resources` | Resource configuration for etcd. | `object` | `{}` | -| `resources.cpu` | Number of CPU cores allocated. | `quantity` | `1000m` | -| `resources.memory` | Amount of memory allocated. | `quantity` | `512Mi` | +| Name | Description | Type | Value | +| ------------------ | -------------------------------------------------------------------- | ---------- | ----------------------------- | +| `size` | Persistent Volume size. | `quantity` | `4Gi` | +| `storageClass` | StorageClass used to store the data. | `string` | `""` | +| `replicas` | Number of etcd replicas. | `int` | `3` | +| `resources` | Resource configuration for etcd. | `object` | `{}` | +| `resources.cpu` | Number of CPU cores allocated. | `quantity` | `1000m` | +| `resources.memory` | Amount of memory allocated. | `quantity` | `512Mi` | +| `certWaitTimeout` | Timeout in seconds to wait for cert-manager to populate TLS Secrets. | `int` | `300` | +| `kubectlImage` | Container image used for DataStore creation hook Job. | `string` | `docker.io/alpine/k8s:1.33.4` | diff --git a/packages/extra/etcd/templates/datastore.yaml b/packages/extra/etcd/templates/datastore.yaml index 2a4a9e05..2d3d5a67 100644 --- a/packages/extra/etcd/templates/datastore.yaml +++ b/packages/extra/etcd/templates/datastore.yaml @@ -1,35 +1,286 @@ +{{- if not (kindIs "float64" .Values.certWaitTimeout) }} + {{- fail "certWaitTimeout must be a number" }} +{{- end }} +{{- if or (le (.Values.certWaitTimeout | int) 0) (lt (.Values.certWaitTimeout | int) 10) (gt (.Values.certWaitTimeout | int) 3600) }} + {{- fail "certWaitTimeout must be between 10 and 3600 seconds (positive integer)" }} +{{- end }} --- -apiVersion: kamaji.clastix.io/v1alpha1 -kind: DataStore +apiVersion: v1 +kind: ServiceAccount metadata: - name: {{ .Release.Namespace }} + name: etcd-datastore-hook + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "5" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: etcd-datastore-hook + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "5" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +rules: +- apiGroups: [""] + resources: ["secrets"] + resourceNames: ["etcd-ca-tls", "etcd-client-tls"] + verbs: ["get"] +- apiGroups: [""] + resources: ["events"] + verbs: ["list"] +- apiGroups: ["kamaji.clastix.io"] + resources: ["datastores"] + verbs: ["create", "get", "patch"] +- apiGroups: ["cert-manager.io"] + resources: ["certificates", "certificaterequests"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: etcd-datastore-hook + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "5" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: etcd-datastore-hook +subjects: +- kind: ServiceAccount + name: etcd-datastore-hook + namespace: {{ .Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: etcd-datastore-hook-{{ .Release.Revision }} + annotations: + # Weight 10 ensures Job runs after all regular resources including Certificates + # Job waits for cert-manager to populate TLS Secrets before creating DataStore + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "10" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded spec: - driver: etcd - endpoints: - - etcd.{{ $.Release.Namespace }}.svc:2379 - tlsConfig: - certificateAuthority: - certificate: - secretReference: - keyPath: tls.crt - name: etcd-ca-tls - namespace: {{ .Release.Namespace }} - privateKey: - secretReference: - keyPath: tls.key - name: etcd-ca-tls - namespace: {{ .Release.Namespace }} - clientCertificate: - certificate: - secretReference: - keyPath: tls.crt - name: etcd-client-tls - namespace: {{ .Release.Namespace }} - privateKey: - secretReference: - keyPath: tls.key - name: etcd-client-tls - namespace: {{ .Release.Namespace }} + backoffLimit: 10 + template: + metadata: + labels: + policy.cozystack.io/allow-to-apiserver: "true" + spec: + serviceAccountName: etcd-datastore-hook + restartPolicy: OnFailure + containers: + - name: create-datastore + image: {{ .Values.kubectlImage }} + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + env: + - name: CERT_WAIT_TIMEOUT + value: {{ .Values.certWaitTimeout | quote }} + command: + - sh + - -exc + - | + # Enable pipefail to catch errors in pipes + set -o pipefail + + # Install required utilities + # openssl: for certificate validation (checkend, pubkey extraction) + if ! apk add --no-cache openssl; then + echo "ERROR: Failed to install openssl. Check network connectivity and Alpine package repository availability." + exit 1 + fi + + # Verify openssl binary is available + if ! command -v openssl >/dev/null 2>&1; then + echo "ERROR: openssl binary not available after installation" + exit 1 + fi + + # Get namespace from ServiceAccount + NAMESPACE=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace) + + # Validate CERT_WAIT_TIMEOUT is a positive integer + TIMEOUT_SECONDS=${CERT_WAIT_TIMEOUT:-300} + if ! echo "$TIMEOUT_SECONDS" | grep -Eq '^[0-9]+$' || [ "$TIMEOUT_SECONDS" -le 0 ]; then + echo "ERROR: CERT_WAIT_TIMEOUT must be a positive integer, got: $TIMEOUT_SECONDS" + exit 1 + fi + + ITERATIONS=$((TIMEOUT_SECONDS / 2)) + + echo "Waiting for cert-manager to populate TLS Secrets in namespace: $NAMESPACE" + echo "Timeout: $TIMEOUT_SECONDS seconds" + + # Function to check Certificate CR is not in failed state + check_certificate_ready() { + local cert_name=$1 + local namespace=$2 + + # Check Certificate exists + if ! kubectl get certificate "$cert_name" --namespace "$namespace" --request-timeout=10s >/dev/null 2>&1; then + echo "ERROR: Certificate $cert_name not found" + return 1 + fi + + # Check Certificate is not in permanent failed state + local reason=$(kubectl get certificate "$cert_name" --namespace "$namespace" --request-timeout=10s -o jsonpath='{.status.conditions[?(@.type=="Ready")].reason}' 2>/dev/null) + + if [ "$reason" = "Failed" ]; then + echo "ERROR: Certificate $cert_name is in Failed state, will not recover" + kubectl describe certificate "$cert_name" --namespace "$namespace" --request-timeout=10s + return 1 + fi + + return 0 + } + + # Function to validate Secret has both certificate and key + validate_secret() { + local secret_name=$1 + local namespace=$2 + + # Fetch certificate and key data atomically (single kubectl call to avoid TOCTOU) + local secret_data=$(kubectl get secret "$secret_name" --namespace "$namespace" --request-timeout=10s -o jsonpath='{.data}' 2>/dev/null) + local cert_data=$(echo "$secret_data" | grep -o '"tls.crt":"[^"]*"' | cut -d'"' -f4 | base64 -d 2>/dev/null) + local key_data=$(echo "$secret_data" | grep -o '"tls.key":"[^"]*"' | cut -d'"' -f4 | base64 -d 2>/dev/null) + + if [ -z "$cert_data" ] || [ -z "$key_data" ]; then + return 1 + fi + + # Check certificate is valid (not expired) + # -checkend 0 validates notAfter >= now + # Note: notBefore validation is omitted as Kamaji webhook validates + # TLS handshake at admission time, making client-side check redundant + if ! echo "$cert_data" | openssl x509 -noout -checkend 0 2>/dev/null; then + echo "ERROR: Certificate in $secret_name is expired or will expire soon" + return 1 + fi + + # Check private key exists and is valid (supports RSA, ECDSA, Ed25519, etc.) + if ! echo "$key_data" | openssl pkey -noout -check 2>/dev/null; then + return 1 + fi + + # Verify certificate and key match (compare public keys) + local cert_pubkey=$(echo "$cert_data" | openssl x509 -noout -pubkey 2>/dev/null) + local key_pubkey=$(echo "$key_data" | openssl pkey -pubout 2>/dev/null) + + if [ "$cert_pubkey" != "$key_pubkey" ]; then + echo "Certificate and private key mismatch" + return 1 + fi + + return 0 + } + + # Function to print diagnostics on failure + print_diagnostics() { + local secret_name=$1 + local namespace=$2 + + echo "=== Diagnostic Information ===" + echo "Certificate status:" + kubectl describe certificate --namespace "$namespace" --request-timeout=10s 2>&1 || echo "No Certificates found" + echo "" + echo "CertificateRequest status:" + kubectl get certificaterequest --namespace "$namespace" --request-timeout=10s -o wide 2>&1 || echo "No CertificateRequests found" + echo "" + echo "Secret $secret_name metadata (no sensitive data):" + kubectl get secret "$secret_name" --namespace "$namespace" --request-timeout=10s -o jsonpath='{.metadata}' 2>&1 || echo "Secret not found" + } + + # Fail-fast check: ensure Certificate CRs exist and are not in failed state + echo "Verifying Certificate CRs are not in failed state..." + if ! check_certificate_ready "etcd-ca" "$NAMESPACE"; then + exit 1 + fi + if ! check_certificate_ready "etcd-client" "$NAMESPACE"; then + exit 1 + fi + + # Wait for etcd-ca-tls Secret to be populated by cert-manager + echo "Checking etcd-ca-tls Secret..." + for i in $(seq 1 $ITERATIONS); do + if validate_secret "etcd-ca-tls" "$NAMESPACE"; then + echo "etcd-ca-tls Secret populated and valid after $((i * 2)) seconds" + break + fi + if [ $i -eq $ITERATIONS ]; then + echo "ERROR: etcd-ca-tls Secret not populated after $TIMEOUT_SECONDS seconds" + print_diagnostics "etcd-ca-tls" "$NAMESPACE" + exit 1 + fi + sleep 2 + done + + # Wait for etcd-client-tls Secret to be populated by cert-manager + echo "Checking etcd-client-tls Secret..." + for i in $(seq 1 $ITERATIONS); do + if validate_secret "etcd-client-tls" "$NAMESPACE"; then + echo "etcd-client-tls Secret populated and valid after $((i * 2)) seconds" + break + fi + if [ $i -eq $ITERATIONS ]; then + echo "ERROR: etcd-client-tls Secret not populated after $TIMEOUT_SECONDS seconds" + print_diagnostics "etcd-client-tls" "$NAMESPACE" + exit 1 + fi + sleep 2 + done + + echo "All TLS Secrets populated and validated" + + echo "Creating or updating DataStore..." + + # Use kubectl apply for idempotent creation/update + # This eliminates race condition between check and create + # Timeout 90s to accommodate Kamaji ValidatingWebhook etcd connectivity check + sed 's/^ //' <<'EOF' | kubectl apply --namespace "$$NAMESPACE" --request-timeout=90s --filename - + apiVersion: kamaji.clastix.io/v1alpha1 + kind: DataStore + metadata: + name: {{ .Release.Namespace }} + spec: + driver: etcd + endpoints: + - etcd.{{ .Release.Namespace }}.svc:2379 + tlsConfig: + certificateAuthority: + certificate: + secretReference: + keyPath: tls.crt + name: etcd-ca-tls + namespace: {{ .Release.Namespace }} + privateKey: + secretReference: + keyPath: tls.key + name: etcd-ca-tls + namespace: {{ .Release.Namespace }} + clientCertificate: + certificate: + secretReference: + keyPath: tls.crt + name: etcd-client-tls + namespace: {{ .Release.Namespace }} + privateKey: + secretReference: + keyPath: tls.key + name: etcd-client-tls + namespace: {{ .Release.Namespace }} + EOF + + echo "DataStore created or updated successfully" --- apiVersion: v1 kind: Secret diff --git a/packages/extra/etcd/values.schema.json b/packages/extra/etcd/values.schema.json index 3c66664c..23921b29 100644 --- a/packages/extra/etcd/values.schema.json +++ b/packages/extra/etcd/values.schema.json @@ -60,6 +60,16 @@ "x-kubernetes-int-or-string": true } } + }, + "certWaitTimeout": { + "description": "Timeout in seconds to wait for cert-manager to populate TLS Secrets.", + "type": "integer", + "default": 300 + }, + "kubectlImage": { + "description": "Container image used for DataStore creation hook Job.", + "type": "string", + "default": "docker.io/alpine/k8s:1.33.4" } } } diff --git a/packages/extra/etcd/values.yaml b/packages/extra/etcd/values.yaml index 8dac251d..d462b857 100644 --- a/packages/extra/etcd/values.yaml +++ b/packages/extra/etcd/values.yaml @@ -19,3 +19,12 @@ replicas: 3 resources: cpu: 1000m memory: 512Mi + +## @param {int} [certWaitTimeout] - Timeout in seconds to wait for cert-manager to populate TLS Secrets. +## Increase this value for environments with slow cert-manager (high load, external CA, resource limits). +certWaitTimeout: 300 + +## @param {string} [kubectlImage] - Container image used for DataStore creation hook Job. +## Uses alpine/k8s (alpine + kubectl pre-installed) and installs openssl at runtime. +## openssl provides certificate validation utilities (checkend, pubkey extraction). +kubectlImage: docker.io/alpine/k8s:1.33.4 diff --git a/packages/system/etcd-rd/cozyrds/etcd.yaml b/packages/system/etcd-rd/cozyrds/etcd.yaml index 7ff7c39c..77679cee 100644 --- a/packages/system/etcd-rd/cozyrds/etcd.yaml +++ b/packages/system/etcd-rd/cozyrds/etcd.yaml @@ -8,7 +8,7 @@ spec: plural: etcds singular: etcd openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"size":{"description":"Persistent Volume size.","default":"4Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"replicas":{"description":"Number of etcd replicas.","type":"integer","default":3},"resources":{"description":"Resource configuration for etcd.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","default":"1000m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","default":"512Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}}}} + {"title":"Chart Values","type":"object","properties":{"size":{"description":"Persistent Volume size.","default":"4Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"replicas":{"description":"Number of etcd replicas.","type":"integer","default":3},"resources":{"description":"Resource configuration for etcd.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","default":"1000m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","default":"512Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"certWaitTimeout":{"description":"Timeout in seconds to wait for cert-manager to populate TLS Secrets.","type":"integer","default":300},"kubectlImage":{"description":"Container image used for DataStore creation hook Job.","type":"string","default":"docker.io/alpine/k8s:1.33.4"}}} release: prefix: "" labels: @@ -26,7 +26,7 @@ spec: description: Storage for Kubernetes clusters module: true icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHg9Ii0wLjAwMTk1MzEyIiB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgcng9IjI0IiBmaWxsPSJ1cmwoI3BhaW50MF9saW5lYXJfNjgzXzI5NjMpIi8+CjxwYXRoIGQ9Ik0xMjIuNDQyIDczLjQ3MjlDMTIxLjk1OSA3My41MTM0IDEyMS40NzQgNzMuNTMyMiAxMjAuOTU4IDczLjUzMjJDMTE3Ljk2NSA3My41MzIyIDExNS4wNjEgNzIuODMwNCAxMTIuNDQyIDcxLjU0NTFDMTEzLjMxNCA2Ni41NDIxIDExMy42ODUgNjEuNTAxOSAxMTMuNTg4IDU2LjQ4MDJDMTEwLjc0OCA1Mi4zNzIzIDEwNy41MDIgNDguNDk2NSAxMDMuODM4IDQ0LjkyNTdDMTA1LjQyOCA0MS45NDU0IDEwNy43NzggMzkuMzgxMSAxMTAuNzExIDM3LjU2MjhMMTExLjk3MSAzNi43ODQyTDExMC45ODkgMzUuNjc3NEMxMDUuOTMyIDI5Ljk4MzIgOTkuODk3MSAyNS41ODA5IDkzLjA1NDcgMjIuNTkzN0w5MS42OTAyIDIyTDkxLjM0MzcgMjMuNDQyM0M5MC41Mjc3IDI2LjgwMzYgODguODIyMiAyOS44MzYgODYuNDgwNyAzMi4yNjk1QzgxLjk4MDMgMjkuODc3NCA3Ny4yNzg4IDI3Ljk0NCA3Mi40MzA1IDI2LjQ3OTdDNjcuNTkzNyAyNy45NDA4IDYyLjkwMDUgMjkuODY4NiA1OC40MDIgMzIuMjU3MUM1Ni4wNzAxIDI5LjgyNjggNTQuMzY4OCAyNi44MDE4IDUzLjU1NiAyMy40NTAxTDUzLjIwNzIgMjIuMDA4M0w1MS44NDc3IDIyLjU5OTJDNDUuMDkxNCAyNS41NDMxIDM4Ljg5MDEgMzAuMDY0NyAzMy45MTYyIDM1LjY3NDJMMzIuOTMxOCAzNi43ODMzTDM0LjE5IDM3LjU2MTlDMzcuMTE0MiAzOS4zNzMzIDM5LjQ1NzYgNDEuOTIyNCA0MS4wNDQ0IDQ0Ljg4NjZDMzcuMzkxNyA0OC40NDM1IDM0LjE0OTUgNTIuMzA3IDMxLjMxMTkgNTYuMzk1OUMzMS4yMDE0IDYxLjQxNTQgMzEuNTUzNSA2Ni40OTI0IDMyLjQyOTcgNzEuNTY0NEMyOS44MjMxIDcyLjgzNzggMjYuOTM1OCA3My41MzE4IDIzLjk2MjggNzMuNTMxOEMyMy40NDA5IDczLjUzMTggMjIuOTUyNyA3My41MTI5IDIyLjQ3ODIgNzMuNDczM0wyMSA3My4zNjA2TDIxLjEzODUgNzQuODM2NUMyMS44NjI5IDgyLjMwMzMgMjQuMTgxNCA4OS40MDUzIDI4LjAzMzQgOTUuOTQ3MUwyOC43ODUzIDk3LjIyMzdMMjkuOTE0MiA5Ni4yNjU2QzMyLjUzMDUgOTQuMDQ2NSAzNS42OTE3IDkyLjU3NyAzOS4wNTMgOTEuOTg0N0M0MS4yNjg5IDk2LjUxNTUgNDMuODk1MyAxMDAuNzcyIDQ2Ljg3NDcgMTA0LjcyNUM1MS42Mjg3IDEwNi4zODcgNTYuNTgxOSAxMDcuNjI5IDYxLjY5NzEgMTA4LjM2N0M2Mi4xODc3IDExMS43NSA2MS43OTcgMTE1LjI0OSA2MC40NjI0IDExOC40ODRMNTkuODk5NSAxMTkuODU1TDYxLjM0NjkgMTIwLjE3NEM2NS4wNTI5IDEyMC45ODkgNjguNzkxNyAxMjEuNDA0IDcyLjQ1MjYgMTIxLjQwNEw4My41NTUxIDEyMC4xNzRMODUuMDAzOSAxMTkuODU1TDg0LjQzOTcgMTE4LjQ4MkM4My4xMDg3IDExNS4yNDYgODIuNzE4IDExMS43NDMgODMuMjA4NiAxMDguMzZDODguMzAzNiAxMDcuNjIxIDkzLjIzODQgMTA2LjM4MiA5Ny45NzQ4IDEwNC43MjVDMTAwLjk1NyAxMDAuNzY5IDEwMy41ODYgOTYuNTA5NSAxMDUuODA1IDkxLjk3MjhDMTA5LjE3NyA5Mi41NjE0IDExMi4zNTYgOTQuMDMxNyAxMTQuOTg5IDk2LjI1NzNMMTE2LjExOCA5Ny4yMTQxTDExNi44NjYgOTUuOTQwN0MxMjAuNzI1IDg5LjM5MDUgMTIzLjA0MyA4Mi4yODkxIDEyMy43NTYgNzQuODM0MkwxMjMuODk1IDczLjM2MUwxMjIuNDQyIDczLjQ3MjlaTTg4LjMxOTcgOTEuNTE4MUM4My4wNjczIDkyLjk0NjYgNzcuNzMzIDkzLjY2NzcgNzIuNDMwNSA5My42Njc3QzY3LjExMzcgOTMuNjY3NyA2MS43ODU5IDkyLjk0NyA1Ni41MjkgOTEuNTE4MUM1My42NDQ4IDg3LjAzNjYgNTEuMzY0NSA4Mi4yMzU3IDQ5LjcyMzQgNzcuMTgxMkM0OC4wODkyIDcyLjE1MDIgNDcuMTMyOSA2Ni44Nzk1IDQ2Ljg1NTQgNjEuNDUyMkM1MC4yNTA0IDU3LjI1NDcgNTQuMTExIDUzLjU3NzYgNTguMzc2NyA1MC40ODIzQzYyLjcxMTQgNDcuMzI5NCA2Ny40MjcxIDQ0Ljc2NzkgNzIuNDMwNSA0Mi44NDFDNzcuNDI1NiA0NC43NjgzIDgyLjEzMjYgNDcuMzI2MiA4Ni40NTcyIDUwLjQ2NTdDOTAuNzM5NCA1My41Nzc2IDk0LjYxNzEgNTcuMjgzMiA5OC4wMjg3IDYxLjUwN0M5Ny43Mzc4IDY2LjkwMzQgOTYuNzcgNzIuMTQzOCA5NS4xMzMgNzcuMTY2NUM5My40OTYxIDgyLjIyIDkxLjIwODQgODcuMDM2MSA4OC4zMTk3IDkxLjUxODFaTTc2Ljc2ODQgNjYuMTk3NEM3Ni43Njg0IDY5LjkwODEgNzkuNzc1NCA3Mi45MDk2IDgzLjQ4MSA3Mi45MDk2Qzg3LjE4NTcgNzIuOTA5NiA5MC4xODk1IDY5LjkwODYgOTAuMTg5NSA2Ni4xOTc0QzkwLjE4OTUgNjIuNTAxIDg3LjE4NTcgNTkuNDg4MSA4My40ODEgNTkuNDg4MUM3OS43NzU0IDU5LjQ4ODEgNzYuNzY4NCA2Mi41MDEgNzYuNzY4NCA2Ni4xOTc0Wk02OC4wOTU0IDY2LjE5NzRDNjguMDk1NCA2OS45MDgxIDY1LjA4ODggNzIuOTA5NiA2MS4zODMyIDcyLjkwOTZDNTcuNjc0OSA3Mi45MDk2IDU0LjY3NjYgNjkuOTA4NiA1NC42NzY2IDY2LjE5NzRDNTQuNjc2NiA2Mi41MDI0IDU3LjY3NTMgNTkuNDg5NCA2MS4zODMyIDU5LjQ4OTRDNjUuMDg4OCA1OS40ODk0IDY4LjA5NTQgNjIuNTAyNCA2OC4wOTU0IDY2LjE5NzRaIiBmaWxsPSJ3aGl0ZSIvPgo8ZGVmcz4KPGxpbmVhckdyYWRpZW50IGlkPSJwYWludDBfbGluZWFyXzY4M18yOTYzIiB4MT0iNS41IiB5MT0iMTEiIHgyPSIxNDEiIHkyPSIxMjQuNSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiPgo8c3RvcCBzdG9wLWNvbG9yPSIjNTNCMkYwIi8+CjxzdG9wIG9mZnNldD0iMSIgc3RvcC1jb2xvcj0iIzQxOUVEQSIvPgo8L2xpbmVhckdyYWRpZW50Pgo8L2RlZnM+Cjwvc3ZnPgo= - keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "size"], ["spec", "storageClass"], ["spec", "replicas"], ["spec", "resources"], ["spec", "resources", "cpu"], ["spec", "resources", "memory"]] + keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "size"], ["spec", "storageClass"], ["spec", "replicas"], ["spec", "resources"], ["spec", "resources", "cpu"], ["spec", "resources", "memory"], ["spec", "certWaitTimeout"], ["spec", "kubectlImage"]] secrets: exclude: [] include: []