From 41f2c64241a46ee40f8abf5a4bf488dbcb392e13 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Mon, 2 Mar 2026 16:22:20 +0300 Subject: [PATCH 1/3] fix(migration): protect cozy-keycloak namespace during v1.0 migration The migration script only annotated cozy-system namespace with helm.sh/resource-policy=keep, leaving cozy-keycloak unprotected. This could cause Helm to delete the namespace and all its secrets (including keycloak credentials and realm data) during the migration to Package-based configuration. Signed-off-by: IvanHunters --- hack/migrate-to-version-1.0.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hack/migrate-to-version-1.0.sh b/hack/migrate-to-version-1.0.sh index bf648926..ff727276 100755 --- a/hack/migrate-to-version-1.0.sh +++ b/hack/migrate-to-version-1.0.sh @@ -39,6 +39,7 @@ echo "The following resources will be annotated with helm.sh/resource-policy=kee echo "to prevent Helm from deleting them when the installer release is removed:" echo " - Namespace: $NAMESPACE" echo " - ConfigMap: $NAMESPACE/cozystack-version" +echo " - Namespace: cozy-keycloak" echo "" read -p "Do you want to annotate these resources? (y/N) " -n 1 -r echo "" @@ -48,6 +49,10 @@ if [[ $REPLY =~ ^[Yy]$ ]]; then kubectl annotate namespace "$NAMESPACE" helm.sh/resource-policy=keep --overwrite echo "Annotating ConfigMap cozystack-version..." kubectl annotate configmap -n "$NAMESPACE" cozystack-version helm.sh/resource-policy=keep --overwrite 2>/dev/null || echo " ConfigMap cozystack-version not found, skipping." + + echo "Annotating namespace cozy-keycloak..." + kubectl annotate namespace cozy-keycloak helm.sh/resource-policy=keep --overwrite 2>/dev/null || echo " Namespace cozy-keycloak not found, skipping." + echo "" echo "Resources annotated successfully." else From 87fca4dfe280fd35b4f594b4f75fad7b20a90085 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Mon, 2 Mar 2026 20:07:01 +0300 Subject: [PATCH 2/3] fix(migration): protect keycloak HelmReleases during v1.0 upgrade In v0.41.x keycloak HelmReleases were created directly by the platform chart (helmreleases.yaml). In v1.0 this template was removed and keycloak is managed via Package API. Without protection, Helm deletes these HelmReleases during upgrade, triggering FluxCD to run helm uninstall and destroying all keycloak data (credentials, realm config, PostgreSQL DB). Add migration 34 that annotates keycloak, keycloak-operator, and keycloak-configure HelmReleases with helm.sh/resource-policy=keep before the platform chart upgrade. This allows the Package operator to adopt existing HelmReleases and FluxCD to perform helm upgrade instead of a fresh install, preserving all data. Signed-off-by: IvanHunters --- .../platform/images/migrations/migrations/34 | 34 +++++++++++++++++++ packages/core/platform/values.yaml | 2 +- 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100755 packages/core/platform/images/migrations/migrations/34 diff --git a/packages/core/platform/images/migrations/migrations/34 b/packages/core/platform/images/migrations/migrations/34 new file mode 100755 index 00000000..13d3be5f --- /dev/null +++ b/packages/core/platform/images/migrations/migrations/34 @@ -0,0 +1,34 @@ +#!/bin/sh +# Migration 34 --> 35 +# Protect keycloak HelmReleases from deletion during platform chart upgrade. +# +# In v0.41.x keycloak, keycloak-operator, and keycloak-configure HelmReleases +# were created directly by the platform chart (helmreleases.yaml). In v1.0 +# this template was removed and keycloak is managed via Package API instead. +# Without the keep annotation, Helm deletes these HelmReleases during upgrade, +# which triggers FluxCD to run helm uninstall, destroying all keycloak data +# (credentials, realm configuration, PostgreSQL database). +# +# The keep annotation ensures the HelmReleases survive the chart upgrade so +# the Package operator can adopt them and FluxCD performs a helm upgrade +# instead of a fresh install. + +set -euo pipefail + +echo "=== Protecting keycloak HelmReleases from deletion ===" + +for hr_name in keycloak keycloak-operator keycloak-configure; do + if kubectl get helmrelease "$hr_name" -n cozy-keycloak --no-headers 2>/dev/null | grep -q .; then + echo " [ANNOTATE] hr/${hr_name} in cozy-keycloak" + kubectl annotate helmrelease "$hr_name" -n cozy-keycloak \ + helm.sh/resource-policy=keep --overwrite + else + echo " [SKIP] hr/${hr_name} not found in cozy-keycloak" + fi +done + +echo "=== Done ===" + +# Stamp version +kubectl create configmap -n cozy-system cozystack-version \ + --from-literal=version=35 --dry-run=client -o yaml | kubectl apply -f- diff --git a/packages/core/platform/values.yaml b/packages/core/platform/values.yaml index 155cc78a..80ebb936 100644 --- a/packages/core/platform/values.yaml +++ b/packages/core/platform/values.yaml @@ -6,7 +6,7 @@ sourceRef: migrations: enabled: false image: ghcr.io/cozystack/cozystack/platform-migrations:v1.0.0@sha256:68dabdebc38ac439228ae07031cc70e0fa184a24bd4e5b3b22c17466b2a55201 - targetVersion: 34 + targetVersion: 35 # Bundle deployment configuration bundles: system: From d21373e78047907fe3e57e09c54b0137fa6b5b70 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Mon, 2 Mar 2026 21:00:21 +0300 Subject: [PATCH 3/3] fix(migration): delete helm release secrets to protect keycloak data Replace HelmRelease annotation approach with helm secret deletion. Annotating HelmReleases with resource-policy=keep does not prevent FluxCD from running helm uninstall when spec.chart changes to spec.chartRef. Deleting helm release secrets ensures FluxCD has nothing to uninstall and performs helm install instead, adopting existing resources. Update manual migration script with the same protection and fix error handling for cozy-keycloak namespace check. Signed-off-by: IvanHunters --- hack/migrate-to-version-1.0.sh | 46 +++++++++- .../platform/images/migrations/migrations/34 | 90 +++++++++++++++---- 2 files changed, 118 insertions(+), 18 deletions(-) diff --git a/hack/migrate-to-version-1.0.sh b/hack/migrate-to-version-1.0.sh index ff727276..99b79dbd 100755 --- a/hack/migrate-to-version-1.0.sh +++ b/hack/migrate-to-version-1.0.sh @@ -51,7 +51,11 @@ if [[ $REPLY =~ ^[Yy]$ ]]; then kubectl annotate configmap -n "$NAMESPACE" cozystack-version helm.sh/resource-policy=keep --overwrite 2>/dev/null || echo " ConfigMap cozystack-version not found, skipping." echo "Annotating namespace cozy-keycloak..." - kubectl annotate namespace cozy-keycloak helm.sh/resource-policy=keep --overwrite 2>/dev/null || echo " Namespace cozy-keycloak not found, skipping." + if kubectl get namespace cozy-keycloak >/dev/null 2>&1; then + kubectl annotate namespace cozy-keycloak helm.sh/resource-policy=keep --overwrite + else + echo " Namespace cozy-keycloak not found, skipping." + fi echo "" echo "Resources annotated successfully." @@ -61,6 +65,46 @@ else fi echo "" +# Step 0.5: Delete keycloak helm release secrets to prevent FluxCD from running helm uninstall +# When the Package operator recreates keycloak HelmReleases with spec.chartRef +# (replacing spec.chart), FluxCD sees an incompatible change and runs helm uninstall, +# destroying all keycloak data. Without helm secrets, FluxCD has nothing to uninstall +# and performs helm install instead, adopting existing resources. +echo "Step 0.5: Protect keycloak data from destruction" +echo "" +echo "This will delete Helm release secrets for keycloak releases in cozy-keycloak" +echo "namespace. Without these secrets, FluxCD cannot run helm uninstall and will" +echo "adopt existing resources instead of recreating them." +echo "" + +if kubectl get namespace cozy-keycloak >/dev/null 2>&1; then + read -p "Do you want to delete keycloak helm release secrets? (y/N) " -n 1 -r + echo "" + + if [[ $REPLY =~ ^[Yy]$ ]]; then + for release in keycloak keycloak-operator keycloak-configure; do + echo " Deleting helm release secrets for ${release}..." + kubectl delete secrets -n cozy-keycloak -l "name=${release},owner=helm" --ignore-not-found + # Fallback: delete by name pattern + remaining=$(kubectl get secrets -n cozy-keycloak -o name | { grep "^secret/sh\.helm\.release\.v1\.${release}\." || true; }) + if [ -n "$remaining" ]; then + echo "$remaining" | while IFS= read -r secret; do + echo " Deleting $secret" + kubectl delete -n cozy-keycloak "$secret" --ignore-not-found + done + fi + done + echo "" + echo "Keycloak helm release secrets deleted." + else + echo "WARNING: Skipping. Keycloak data may be lost during upgrade if FluxCD" + echo "runs helm uninstall due to chart source change." + fi +else + echo " Namespace cozy-keycloak does not exist, keycloak not deployed — skipping." +fi +echo "" + # Read ConfigMap cozystack echo "Reading ConfigMap cozystack..." COZYSTACK_CM=$(kubectl get configmap -n "$NAMESPACE" cozystack -o json 2>/dev/null || echo "{}") diff --git a/packages/core/platform/images/migrations/migrations/34 b/packages/core/platform/images/migrations/migrations/34 index 13d3be5f..39979f4c 100755 --- a/packages/core/platform/images/migrations/migrations/34 +++ b/packages/core/platform/images/migrations/migrations/34 @@ -1,29 +1,85 @@ #!/bin/sh # Migration 34 --> 35 -# Protect keycloak HelmReleases from deletion during platform chart upgrade. +# Protect keycloak data from destruction during v1.0 upgrade. # -# In v0.41.x keycloak, keycloak-operator, and keycloak-configure HelmReleases -# were created directly by the platform chart (helmreleases.yaml). In v1.0 -# this template was removed and keycloak is managed via Package API instead. -# Without the keep annotation, Helm deletes these HelmReleases during upgrade, -# which triggers FluxCD to run helm uninstall, destroying all keycloak data -# (credentials, realm configuration, PostgreSQL database). +# Problem: +# In v0.41.x keycloak HelmReleases (keycloak, keycloak-operator, keycloak-configure) +# were created via platform chart template (helmreleases.yaml) with spec.chart +# pointing to a HelmRepository. In v1.0 the Package operator recreates these +# HelmReleases with spec.chartRef pointing to an ExternalArtifact. +# FluxCD sees an incompatible chart source change and runs helm uninstall, +# destroying all keycloak resources (PVCs, PostgreSQL databases, credentials, +# realm configuration) before doing a fresh helm install. # -# The keep annotation ensures the HelmReleases survive the chart upgrade so -# the Package operator can adopt them and FluxCD performs a helm upgrade -# instead of a fresh install. +# Solution: +# Delete Helm release secrets (sh.helm.release.v1..*) BEFORE the chart +# source change happens. Without these secrets FluxCD has no release to uninstall, +# so it performs helm install directly. Existing resources with correct +# meta.helm.sh/release-name annotations are adopted by the new release. +# +# Pattern from migration 26 (monitoring migration). set -euo pipefail -echo "=== Protecting keycloak HelmReleases from deletion ===" +NAMESPACE="cozy-keycloak" -for hr_name in keycloak keycloak-operator keycloak-configure; do - if kubectl get helmrelease "$hr_name" -n cozy-keycloak --no-headers 2>/dev/null | grep -q .; then - echo " [ANNOTATE] hr/${hr_name} in cozy-keycloak" - kubectl annotate helmrelease "$hr_name" -n cozy-keycloak \ - helm.sh/resource-policy=keep --overwrite +# Delete all helm release secrets for a given release name in a namespace. +# Uses both label selector and name-pattern matching to ensure complete cleanup. +delete_helm_secrets() { + local ns="$1" + local release="$2" + + # Primary: delete by label selector + kubectl delete secrets -n "$ns" -l "name=${release},owner=helm" --ignore-not-found + + # Fallback: find and delete by name pattern (in case labels were modified) + local remaining + remaining=$(kubectl get secrets -n "$ns" -o name | { grep "^secret/sh\.helm\.release\.v1\.${release}\." || true; }) + if [ -n "$remaining" ]; then + echo " Found secrets not matched by label selector, deleting by name..." + echo "$remaining" | while IFS= read -r secret; do + echo " Deleting $secret" + kubectl delete -n "$ns" "$secret" --ignore-not-found + done + fi + + # Verify all secrets are gone + remaining=$(kubectl get secrets -n "$ns" -o name | { grep "^secret/sh\.helm\.release\.v1\.${release}\." || true; }) + if [ -n "$remaining" ]; then + echo " ERROR: Failed to delete helm release secrets:" + echo "$remaining" + return 1 + fi +} + +echo "=== Protecting keycloak data from destruction during upgrade ===" + +# Check if namespace exists; if not, keycloak was never deployed — skip +if ! kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then + echo " Namespace $NAMESPACE does not exist, keycloak not deployed — skipping" + kubectl create configmap -n cozy-system cozystack-version \ + --from-literal=version=35 --dry-run=client -o yaml | kubectl apply -f- + exit 0 +fi + +for release in keycloak keycloak-operator keycloak-configure; do + # Check if HelmRelease exists (handle missing CRD gracefully) + out=$(kubectl get helmrelease "$release" -n "$NAMESPACE" -o name 2>&1) && found=true || found=false + + if [ "$found" = "true" ]; then + echo " [DELETE SECRETS] Removing helm release secrets for ${release} in ${NAMESPACE}" + delete_helm_secrets "$NAMESPACE" "$release" else - echo " [SKIP] hr/${hr_name} not found in cozy-keycloak" + # Distinguish "not found" from real errors + case "$out" in + *"NotFound"*|*"not found"*|*"the server doesn't have a resource type"*|*"no matches for kind"*) + echo " [SKIP] hr/${release} not found in ${NAMESPACE}" + ;; + *) + echo " [ERROR] Failed to query hr/${release}: $out" >&2 + exit 1 + ;; + esac fi done