diff --git a/deploy/provider-msp/.env.example b/deploy/provider-msp/.env.example index c4c06204d..ea6e70b77 100644 --- a/deploy/provider-msp/.env.example +++ b/deploy/provider-msp/.env.example @@ -60,6 +60,20 @@ PULSE_EMAIL_REPLY_TO=support@example.com # provider MSP backup archive is available: # docker compose run --rm control-plane provider-msp status --require-backup # +# Pre-upgrade / pre-maintenance plan. This is non-mutating and does not pull +# images: +# ./upgrade.sh --dry-run +# +# Apply a provider control-plane upgrade after updating the image pins in this +# file. The runner creates and verifies a fresh backup, dry-runs restore into a +# separate target data dir, pulls provider images, starts Traefik/control-plane, +# and prints the tenant runtime reconcile plan: +# ./upgrade.sh +# +# Also reconcile all client runtimes to the configured CP_PULSE_IMAGE line after +# the provider services are updated: +# ./upgrade.sh --rollout-tenants +# # Fresh-install provider proof. This bootstraps the owner account, checks install # readiness, creates two proof client workspaces, verifies tenant-local agent # ingest and portal handoff, creates and verifies a backup, dry-runs restore and diff --git a/deploy/provider-msp/setup.sh b/deploy/provider-msp/setup.sh index 4675ed992..a72c634eb 100755 --- a/deploy/provider-msp/setup.sh +++ b/deploy/provider-msp/setup.sh @@ -121,6 +121,7 @@ install_deploy_bundle() { "traefik-dynamic.yml" ".env.example" "run-install-proof.sh" + "upgrade.sh" ) if [[ -n "${src_dir}" ]]; then @@ -166,6 +167,7 @@ This script needs these files present on disk: - traefik-dynamic.yml - .env.example - run-install-proof.sh + - upgrade.sh Run it from deploy/provider-msp/, or set PULSE_PROVIDER_MSP_BUNDLE_URL to a tar.gz containing those files. @@ -178,6 +180,7 @@ EOF install -m 0644 "${src_dir}/traefik-dynamic.yml" "${PULSE_PROVIDER_MSP_INSTALL_DIR}/traefik-dynamic.yml" install -m 0644 "${src_dir}/.env.example" "${PULSE_PROVIDER_MSP_INSTALL_DIR}/.env.example" install -m 0755 "${src_dir}/run-install-proof.sh" "${PULSE_PROVIDER_MSP_INSTALL_DIR}/run-install-proof.sh" + install -m 0755 "${src_dir}/upgrade.sh" "${PULSE_PROVIDER_MSP_INSTALL_DIR}/upgrade.sh" } env_value() { diff --git a/deploy/provider-msp/upgrade.sh b/deploy/provider-msp/upgrade.sh new file mode 100755 index 000000000..470222722 --- /dev/null +++ b/deploy/provider-msp/upgrade.sh @@ -0,0 +1,222 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' + +usage() { + cat <<'USAGE' +Usage: + ./upgrade.sh [options] + +Runs the provider-hosted MSP pre-upgrade and upgrade flow: + 1. validates .env and docker-compose.yml + 2. checks provider status and install preflight + 3. creates and verifies a fresh provider MSP backup + 4. dry-runs restore into a separate target data directory + 5. pulls and starts the provider control-plane and Traefik services + 6. prints the tenant runtime reconcile plan + 7. optionally reconciles all tenant runtimes onto the configured runtime line + +Options: + --dry-run Print the non-mutating upgrade plan only + --rollout-tenants Execute tenant-runtime reconcile --all after provider services are updated + --prune-previous Remove preserved pre-rollout tenant containers after successful tenant rollout + --skip-compose-pull Do not run docker compose pull for provider services + --skip-runtime-image-pull Pass --skip-image-pull to provider-msp preflight + --backup-output PATH Write the fresh backup archive to PATH + --restore-target DIR Restore dry-run target data dir (default: /upgrade-restore-drill) + --run-id ID Operator-visible tenant reconcile run id + --health-timeout DURATION Tenant rollout health timeout (default: 90s) + -h, --help Show this help + +Environment equivalents: + PROVIDER_MSP_UPGRADE_DRY_RUN=1 + PROVIDER_MSP_UPGRADE_ROLLOUT_TENANTS=1 + PROVIDER_MSP_UPGRADE_PRUNE_PREVIOUS=1 + PROVIDER_MSP_SKIP_COMPOSE_PULL=1 + PROVIDER_MSP_SKIP_RUNTIME_IMAGE_PULL=1 + PROVIDER_MSP_UPGRADE_BACKUP_OUTPUT=/data/backups/provider-msp/pre-upgrade.tar.gz + PROVIDER_MSP_UPGRADE_RESTORE_TARGET=/data/upgrade-restore-drill + PROVIDER_MSP_UPGRADE_RUN_ID=provider-msp-upgrade-20260602T120000Z + PROVIDER_MSP_UPGRADE_HEALTH_TIMEOUT=90s +USAGE +} + +die() { + echo "error: $*" >&2 + exit 1 +} + +truthy() { + case "$(echo "${1:-}" | tr '[:upper:]' '[:lower:]')" in + true|1|yes|on) return 0 ;; + *) return 1 ;; + esac +} + +env_value() { + local key="$1" + local env_path="${2:-.env}" + local value + if [[ ! -f "${env_path}" ]]; then + return 0 + fi + value="$(grep -E "^${key}=" "${env_path}" | tail -n 1 | cut -d= -f2- || true)" + value="${value%\"}"; value="${value#\"}" + value="${value%\'}"; value="${value#\'}" + echo "${value}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' +} + +extract_field() { + local key="$1" + awk -F= -v key="${key}" '$1 == key { print substr($0, length($1) + 2); exit }' +} + +run_control() { + docker compose run --rm --no-deps control-plane "$@" +} + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "${script_dir}" + +dry_run="${PROVIDER_MSP_UPGRADE_DRY_RUN:-0}" +rollout_tenants="${PROVIDER_MSP_UPGRADE_ROLLOUT_TENANTS:-0}" +prune_previous="${PROVIDER_MSP_UPGRADE_PRUNE_PREVIOUS:-0}" +skip_compose_pull="${PROVIDER_MSP_SKIP_COMPOSE_PULL:-0}" +skip_runtime_image_pull="${PROVIDER_MSP_SKIP_RUNTIME_IMAGE_PULL:-0}" +backup_output="${PROVIDER_MSP_UPGRADE_BACKUP_OUTPUT:-}" +restore_target="${PROVIDER_MSP_UPGRADE_RESTORE_TARGET:-}" +run_id="${PROVIDER_MSP_UPGRADE_RUN_ID:-provider-msp-upgrade-$(date -u +'%Y%m%dT%H%M%SZ')}" +health_timeout="${PROVIDER_MSP_UPGRADE_HEALTH_TIMEOUT:-90s}" + +while (($# > 0)); do + case "$1" in + --dry-run) + dry_run=1 + shift + ;; + --rollout-tenants) + rollout_tenants=1 + shift + ;; + --prune-previous) + prune_previous=1 + shift + ;; + --skip-compose-pull) + skip_compose_pull=1 + shift + ;; + --skip-runtime-image-pull) + skip_runtime_image_pull=1 + shift + ;; + --backup-output) + backup_output="${2:-}" + shift 2 + ;; + --restore-target) + restore_target="${2:-}" + shift 2 + ;; + --run-id) + run_id="${2:-}" + shift 2 + ;; + --health-timeout) + health_timeout="${2:-}" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [[ ! -f .env ]]; then + die "deploy/provider-msp/.env is required; copy .env.example to .env and fill in the provider values" +fi + +provider_data_dir="$(env_value PULSE_PROVIDER_MSP_DATA_DIR .env)" +provider_data_dir="${provider_data_dir:-/data}" +if [[ -z "${restore_target}" ]]; then + restore_target="${provider_data_dir%/}/upgrade-restore-drill" +fi +if [[ "${restore_target%/}" == "${provider_data_dir%/}" ]]; then + die "--restore-target must not point at the live provider data dir (${provider_data_dir})" +fi + +docker compose version >/dev/null +docker version >/dev/null +docker compose config --quiet + +preflight_args=(provider-msp preflight) +if truthy "${skip_runtime_image_pull}" || truthy "${dry_run}"; then + preflight_args+=(--skip-image-pull) +fi + +echo "provider_msp_upgrade_dry_run=$(truthy "${dry_run}" && echo true || echo false)" +echo "provider_msp_upgrade_run_id=${run_id}" +echo "provider_msp_upgrade_restore_target=${restore_target}" + +run_control provider-msp status +run_control "${preflight_args[@]}" + +if truthy "${dry_run}"; then + run_control tenant-runtime reconcile --all --dry-run + echo "tenant_runtime_rollout_applied=false" + echo "provider_msp_upgrade_plan_ok=true" + exit 0 +fi + +backup_args=(provider-msp backup create) +if [[ -n "${backup_output}" ]]; then + backup_args+=(--output "${backup_output}") +fi + +backup_output_text="$(run_control "${backup_args[@]}")" +printf '%s\n' "${backup_output_text}" +archive_path="$(printf '%s\n' "${backup_output_text}" | extract_field archive_path)" +if [[ -z "${archive_path}" ]]; then + die "provider-msp backup create did not print archive_path" +fi + +run_control provider-msp backup verify "${archive_path}" +run_control provider-msp backup restore "${archive_path}" --target-data-dir "${restore_target}" --dry-run +run_control provider-msp status --require-backup + +if ! truthy "${skip_compose_pull}"; then + docker compose pull traefik control-plane +fi +docker compose up -d traefik control-plane +run_control provider-msp status --require-backup +run_control tenant-runtime reconcile --all --dry-run + +if truthy "${rollout_tenants}"; then + reconcile_args=( + tenant-runtime reconcile + --all + --run-id "${run_id}" + --health-timeout "${health_timeout}" + ) + if truthy "${prune_previous}"; then + reconcile_args+=(--prune-previous) + fi + run_control "${reconcile_args[@]}" + run_control provider-msp status --require-backup + echo "tenant_runtime_rollout_applied=true" +else + echo "tenant_runtime_rollout_applied=false" + echo "tenant_runtime_rollout_next_command=docker compose run --rm --no-deps control-plane tenant-runtime reconcile --all --run-id ${run_id} --health-timeout ${health_timeout}" +fi + +docker compose ps + +echo "backup_path=${archive_path}" +echo "restore_target_data_dir=${restore_target}" +echo "provider_msp_upgrade_ok=true" diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index 815ebf482..da7188e96 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -150,6 +150,15 @@ surfaces. provider images, run the one-off `provider-msp install-proof` command through the packaged control-plane service, start the long-running provider stack, and finish with `provider-msp status`. + `deploy/provider-msp/upgrade.sh` is the compose-level pre-upgrade and + pre-maintenance runner for provider-hosted MSP. It must keep dry-run mode + non-mutating, validate the provider `.env` and compose config, check + provider status and preflight, create and verify a fresh backup before + apply, dry-run restore into a separate target data dir, require backup + readiness before and after provider service replacement, update the packaged + Traefik/control-plane services, print the tenant runtime reconcile plan, and + only execute `tenant-runtime reconcile --all` when the operator explicitly + asks for tenant rollout. `deploy/provider-msp/setup.sh` is the first-time provider host setup artifact. It must install the Docker/compose host prerequisites, create the provider data, backup, and Docker-network layout, copy the provider MSP diff --git a/scripts/installtests/provider_msp_deploy_test.go b/scripts/installtests/provider_msp_deploy_test.go index 6ebb7122c..eac5a83e8 100644 --- a/scripts/installtests/provider_msp_deploy_test.go +++ b/scripts/installtests/provider_msp_deploy_test.go @@ -61,6 +61,9 @@ func TestProviderMSPDeployEnvExampleMatchesBootstrapPath(t *testing.T) { "docker compose run --rm control-plane provider-msp preflight", "docker compose run --rm control-plane provider-msp status", "docker compose run --rm control-plane provider-msp status --require-backup", + "./upgrade.sh --dry-run", + "./upgrade.sh", + "./upgrade.sh --rollout-tenants", "./run-install-proof.sh", "docker compose run --rm control-plane provider-msp install-proof", "docker compose run --rm control-plane provider-msp recover --all-degraded --dry-run", @@ -107,6 +110,7 @@ func TestProviderMSPSetupScriptMatchesProviderContract(t *testing.T) { "traefik-dynamic.yml", ".env.example", "run-install-proof.sh", + "upgrade.sh", "CP_PROVIDER_MSP_LICENSE_FILE", "CP_TRIAL_ACTIVATION_PRIVATE_KEY", "PULSE_PROVIDER_MSP_DATA_DIR", @@ -128,6 +132,53 @@ func TestProviderMSPSetupScriptMatchesProviderContract(t *testing.T) { ) } +func TestProviderMSPUpgradeRunnerMatchesComposeContract(t *testing.T) { + scriptPath := repoFile("deploy", "provider-msp", "upgrade.sh") + result := exec.Command("bash", "-n", scriptPath) + if output, err := result.CombinedOutput(); err != nil { + t.Fatalf("provider MSP upgrade runner shell syntax failed: %v\n%s", err, output) + } + scriptBytes, err := os.ReadFile(scriptPath) + if err != nil { + t.Fatalf("read provider MSP upgrade runner: %v", err) + } + text := string(scriptBytes) + assertContainsAll(t, text, + "PROVIDER_MSP_UPGRADE_DRY_RUN", + "PROVIDER_MSP_UPGRADE_ROLLOUT_TENANTS", + "PROVIDER_MSP_UPGRADE_PRUNE_PREVIOUS", + "PROVIDER_MSP_UPGRADE_BACKUP_OUTPUT", + "PROVIDER_MSP_UPGRADE_RESTORE_TARGET", + "PROVIDER_MSP_UPGRADE_RUN_ID", + "PROVIDER_MSP_UPGRADE_HEALTH_TIMEOUT", + "docker compose config --quiet", + "docker version >/dev/null", + "provider-msp preflight", + "provider-msp status", + "provider-msp status --require-backup", + "provider-msp backup create", + "provider-msp backup verify", + "provider-msp backup restore", + "--target-data-dir", + "tenant-runtime reconcile --all --dry-run", + "tenant-runtime reconcile", + "--all", + "--run-id", + "--health-timeout", + "--prune-previous", + "docker compose pull traefik control-plane", + "docker compose up -d traefik control-plane", + "docker compose run --rm --no-deps control-plane", + "provider_msp_upgrade_ok=true", + "tenant_runtime_rollout_applied=true", + "tenant_runtime_rollout_applied=false", + ) + assertNotContainsAny(t, text, + "STRIPE_", + "CP_PUBLIC_CLOUD_SIGNUP_ENABLED", + ) +} + func TestProviderMSPInstallProofRunnerMatchesComposeContract(t *testing.T) { scriptPath := repoFile("deploy", "provider-msp", "run-install-proof.sh") result := exec.Command("bash", "-n", scriptPath)