Add provider MSP upgrade runner

deploy/provider-msp/.env.example

@@ -60,6 +60,20 @@ PULSE_EMAIL_REPLY_TO=support@example.com
 # provider MSP backup archive is available:
 # docker compose run --rm control-plane provider-msp status --require-backup
 #
+# Pre-upgrade / pre-maintenance plan. This is non-mutating and does not pull
+# images:
+# ./upgrade.sh --dry-run
+#
+# Apply a provider control-plane upgrade after updating the image pins in this
+# file. The runner creates and verifies a fresh backup, dry-runs restore into a
+# separate target data dir, pulls provider images, starts Traefik/control-plane,
+# and prints the tenant runtime reconcile plan:
+# ./upgrade.sh
+#
+# Also reconcile all client runtimes to the configured CP_PULSE_IMAGE line after
+# the provider services are updated:
+# ./upgrade.sh --rollout-tenants
+#
 # Fresh-install provider proof. This bootstraps the owner account, checks install
 # readiness, creates two proof client workspaces, verifies tenant-local agent
 # ingest and portal handoff, creates and verifies a backup, dry-runs restore and

deploy/provider-msp/setup.sh

@@ -121,6 +121,7 @@ install_deploy_bundle() {
     "traefik-dynamic.yml"
     ".env.example"
     "run-install-proof.sh"
+    "upgrade.sh"
   )
 
   if [[ -n "${src_dir}" ]]; then
@@ -166,6 +167,7 @@ This script needs these files present on disk:
   - traefik-dynamic.yml
   - .env.example
   - run-install-proof.sh
+  - upgrade.sh
 
 Run it from deploy/provider-msp/, or set PULSE_PROVIDER_MSP_BUNDLE_URL to a
 tar.gz containing those files.
@@ -178,6 +180,7 @@ EOF
   install -m 0644 "${src_dir}/traefik-dynamic.yml" "${PULSE_PROVIDER_MSP_INSTALL_DIR}/traefik-dynamic.yml"
   install -m 0644 "${src_dir}/.env.example" "${PULSE_PROVIDER_MSP_INSTALL_DIR}/.env.example"
   install -m 0755 "${src_dir}/run-install-proof.sh" "${PULSE_PROVIDER_MSP_INSTALL_DIR}/run-install-proof.sh"
+  install -m 0755 "${src_dir}/upgrade.sh" "${PULSE_PROVIDER_MSP_INSTALL_DIR}/upgrade.sh"
 }
 
 env_value() {

deploy/provider-msp/upgrade.sh

@@ -0,0 +1,222 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+
+usage() {
+  cat <<'USAGE'
+Usage:
+  ./upgrade.sh [options]
+
+Runs the provider-hosted MSP pre-upgrade and upgrade flow:
+  1. validates .env and docker-compose.yml
+  2. checks provider status and install preflight
+  3. creates and verifies a fresh provider MSP backup
+  4. dry-runs restore into a separate target data directory
+  5. pulls and starts the provider control-plane and Traefik services
+  6. prints the tenant runtime reconcile plan
+  7. optionally reconciles all tenant runtimes onto the configured runtime line
+
+Options:
+  --dry-run                    Print the non-mutating upgrade plan only
+  --rollout-tenants            Execute tenant-runtime reconcile --all after provider services are updated
+  --prune-previous             Remove preserved pre-rollout tenant containers after successful tenant rollout
+  --skip-compose-pull          Do not run docker compose pull for provider services
+  --skip-runtime-image-pull    Pass --skip-image-pull to provider-msp preflight
+  --backup-output PATH         Write the fresh backup archive to PATH
+  --restore-target DIR         Restore dry-run target data dir (default: <data-dir>/upgrade-restore-drill)
+  --run-id ID                  Operator-visible tenant reconcile run id
+  --health-timeout DURATION    Tenant rollout health timeout (default: 90s)
+  -h, --help                   Show this help
+
+Environment equivalents:
+  PROVIDER_MSP_UPGRADE_DRY_RUN=1
+  PROVIDER_MSP_UPGRADE_ROLLOUT_TENANTS=1
+  PROVIDER_MSP_UPGRADE_PRUNE_PREVIOUS=1
+  PROVIDER_MSP_SKIP_COMPOSE_PULL=1
+  PROVIDER_MSP_SKIP_RUNTIME_IMAGE_PULL=1
+  PROVIDER_MSP_UPGRADE_BACKUP_OUTPUT=/data/backups/provider-msp/pre-upgrade.tar.gz
+  PROVIDER_MSP_UPGRADE_RESTORE_TARGET=/data/upgrade-restore-drill
+  PROVIDER_MSP_UPGRADE_RUN_ID=provider-msp-upgrade-20260602T120000Z
+  PROVIDER_MSP_UPGRADE_HEALTH_TIMEOUT=90s
+USAGE
+}
+
+die() {
+  echo "error: $*" >&2
+  exit 1
+}
+
+truthy() {
+  case "$(echo "${1:-}" | tr '[:upper:]' '[:lower:]')" in
+    true|1|yes|on) return 0 ;;
+    *) return 1 ;;
+  esac
+}
+
+env_value() {
+  local key="$1"
+  local env_path="${2:-.env}"
+  local value
+  if [[ ! -f "${env_path}" ]]; then
+    return 0
+  fi
+  value="$(grep -E "^${key}=" "${env_path}" | tail -n 1 | cut -d= -f2- || true)"
+  value="${value%\"}"; value="${value#\"}"
+  value="${value%\'}"; value="${value#\'}"
+  echo "${value}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//'
+}
+
+extract_field() {
+  local key="$1"
+  awk -F= -v key="${key}" '$1 == key { print substr($0, length($1) + 2); exit }'
+}
+
+run_control() {
+  docker compose run --rm --no-deps control-plane "$@"
+}
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "${script_dir}"
+
+dry_run="${PROVIDER_MSP_UPGRADE_DRY_RUN:-0}"
+rollout_tenants="${PROVIDER_MSP_UPGRADE_ROLLOUT_TENANTS:-0}"
+prune_previous="${PROVIDER_MSP_UPGRADE_PRUNE_PREVIOUS:-0}"
+skip_compose_pull="${PROVIDER_MSP_SKIP_COMPOSE_PULL:-0}"
+skip_runtime_image_pull="${PROVIDER_MSP_SKIP_RUNTIME_IMAGE_PULL:-0}"
+backup_output="${PROVIDER_MSP_UPGRADE_BACKUP_OUTPUT:-}"
+restore_target="${PROVIDER_MSP_UPGRADE_RESTORE_TARGET:-}"
+run_id="${PROVIDER_MSP_UPGRADE_RUN_ID:-provider-msp-upgrade-$(date -u +'%Y%m%dT%H%M%SZ')}"
+health_timeout="${PROVIDER_MSP_UPGRADE_HEALTH_TIMEOUT:-90s}"
+
+while (($# > 0)); do
+  case "$1" in
+    --dry-run)
+      dry_run=1
+      shift
+      ;;
+    --rollout-tenants)
+      rollout_tenants=1
+      shift
+      ;;
+    --prune-previous)
+      prune_previous=1
+      shift
+      ;;
+    --skip-compose-pull)
+      skip_compose_pull=1
+      shift
+      ;;
+    --skip-runtime-image-pull)
+      skip_runtime_image_pull=1
+      shift
+      ;;
+    --backup-output)
+      backup_output="${2:-}"
+      shift 2
+      ;;
+    --restore-target)
+      restore_target="${2:-}"
+      shift 2
+      ;;
+    --run-id)
+      run_id="${2:-}"
+      shift 2
+      ;;
+    --health-timeout)
+      health_timeout="${2:-}"
+      shift 2
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      echo "unknown option: $1" >&2
+      usage >&2
+      exit 2
+      ;;
+  esac
+done
+
+if [[ ! -f .env ]]; then
+  die "deploy/provider-msp/.env is required; copy .env.example to .env and fill in the provider values"
+fi
+
+provider_data_dir="$(env_value PULSE_PROVIDER_MSP_DATA_DIR .env)"
+provider_data_dir="${provider_data_dir:-/data}"
+if [[ -z "${restore_target}" ]]; then
+  restore_target="${provider_data_dir%/}/upgrade-restore-drill"
+fi
+if [[ "${restore_target%/}" == "${provider_data_dir%/}" ]]; then
+  die "--restore-target must not point at the live provider data dir (${provider_data_dir})"
+fi
+
+docker compose version >/dev/null
+docker version >/dev/null
+docker compose config --quiet
+
+preflight_args=(provider-msp preflight)
+if truthy "${skip_runtime_image_pull}" || truthy "${dry_run}"; then
+  preflight_args+=(--skip-image-pull)
+fi
+
+echo "provider_msp_upgrade_dry_run=$(truthy "${dry_run}" && echo true || echo false)"
+echo "provider_msp_upgrade_run_id=${run_id}"
+echo "provider_msp_upgrade_restore_target=${restore_target}"
+
+run_control provider-msp status
+run_control "${preflight_args[@]}"
+
+if truthy "${dry_run}"; then
+  run_control tenant-runtime reconcile --all --dry-run
+  echo "tenant_runtime_rollout_applied=false"
+  echo "provider_msp_upgrade_plan_ok=true"
+  exit 0
+fi
+
+backup_args=(provider-msp backup create)
+if [[ -n "${backup_output}" ]]; then
+  backup_args+=(--output "${backup_output}")
+fi
+
+backup_output_text="$(run_control "${backup_args[@]}")"
+printf '%s\n' "${backup_output_text}"
+archive_path="$(printf '%s\n' "${backup_output_text}" | extract_field archive_path)"
+if [[ -z "${archive_path}" ]]; then
+  die "provider-msp backup create did not print archive_path"
+fi
+
+run_control provider-msp backup verify "${archive_path}"
+run_control provider-msp backup restore "${archive_path}" --target-data-dir "${restore_target}" --dry-run
+run_control provider-msp status --require-backup
+
+if ! truthy "${skip_compose_pull}"; then
+  docker compose pull traefik control-plane
+fi
+docker compose up -d traefik control-plane
+run_control provider-msp status --require-backup
+run_control tenant-runtime reconcile --all --dry-run
+
+if truthy "${rollout_tenants}"; then
+  reconcile_args=(
+    tenant-runtime reconcile
+    --all
+    --run-id "${run_id}"
+    --health-timeout "${health_timeout}"
+  )
+  if truthy "${prune_previous}"; then
+    reconcile_args+=(--prune-previous)
+  fi
+  run_control "${reconcile_args[@]}"
+  run_control provider-msp status --require-backup
+  echo "tenant_runtime_rollout_applied=true"
+else
+  echo "tenant_runtime_rollout_applied=false"
+  echo "tenant_runtime_rollout_next_command=docker compose run --rm --no-deps control-plane tenant-runtime reconcile --all --run-id ${run_id} --health-timeout ${health_timeout}"
+fi
+
+docker compose ps
+
+echo "backup_path=${archive_path}"
+echo "restore_target_data_dir=${restore_target}"
+echo "provider_msp_upgrade_ok=true"

docs/release-control/v6/internal/subsystems/deployment-installability.md

@@ -150,6 +150,15 @@ surfaces.
    provider images, run the one-off `provider-msp install-proof` command through
    the packaged control-plane service, start the long-running provider stack,
    and finish with `provider-msp status`.
+   `deploy/provider-msp/upgrade.sh` is the compose-level pre-upgrade and
+   pre-maintenance runner for provider-hosted MSP. It must keep dry-run mode
+   non-mutating, validate the provider `.env` and compose config, check
+   provider status and preflight, create and verify a fresh backup before
+   apply, dry-run restore into a separate target data dir, require backup
+   readiness before and after provider service replacement, update the packaged
+   Traefik/control-plane services, print the tenant runtime reconcile plan, and
+   only execute `tenant-runtime reconcile --all` when the operator explicitly
+   asks for tenant rollout.
    `deploy/provider-msp/setup.sh` is the first-time provider host setup
    artifact. It must install the Docker/compose host prerequisites, create the
    provider data, backup, and Docker-network layout, copy the provider MSP

scripts/installtests/provider_msp_deploy_test.go

@@ -61,6 +61,9 @@ func TestProviderMSPDeployEnvExampleMatchesBootstrapPath(t *testing.T) {
 		"docker compose run --rm control-plane provider-msp preflight",
 		"docker compose run --rm control-plane provider-msp status",
 		"docker compose run --rm control-plane provider-msp status --require-backup",
+		"./upgrade.sh --dry-run",
+		"./upgrade.sh",
+		"./upgrade.sh --rollout-tenants",
 		"./run-install-proof.sh",
 		"docker compose run --rm control-plane provider-msp install-proof",
 		"docker compose run --rm control-plane provider-msp recover --all-degraded --dry-run",
@@ -107,6 +110,7 @@ func TestProviderMSPSetupScriptMatchesProviderContract(t *testing.T) {
 		"traefik-dynamic.yml",
 		".env.example",
 		"run-install-proof.sh",
+		"upgrade.sh",
 		"CP_PROVIDER_MSP_LICENSE_FILE",
 		"CP_TRIAL_ACTIVATION_PRIVATE_KEY",
 		"PULSE_PROVIDER_MSP_DATA_DIR",
@@ -128,6 +132,53 @@ func TestProviderMSPSetupScriptMatchesProviderContract(t *testing.T) {
 	)
 }
 
+func TestProviderMSPUpgradeRunnerMatchesComposeContract(t *testing.T) {
+	scriptPath := repoFile("deploy", "provider-msp", "upgrade.sh")
+	result := exec.Command("bash", "-n", scriptPath)
+	if output, err := result.CombinedOutput(); err != nil {
+		t.Fatalf("provider MSP upgrade runner shell syntax failed: %v\n%s", err, output)
+	}
+	scriptBytes, err := os.ReadFile(scriptPath)
+	if err != nil {
+		t.Fatalf("read provider MSP upgrade runner: %v", err)
+	}
+	text := string(scriptBytes)
+	assertContainsAll(t, text,
+		"PROVIDER_MSP_UPGRADE_DRY_RUN",
+		"PROVIDER_MSP_UPGRADE_ROLLOUT_TENANTS",
+		"PROVIDER_MSP_UPGRADE_PRUNE_PREVIOUS",
+		"PROVIDER_MSP_UPGRADE_BACKUP_OUTPUT",
+		"PROVIDER_MSP_UPGRADE_RESTORE_TARGET",
+		"PROVIDER_MSP_UPGRADE_RUN_ID",
+		"PROVIDER_MSP_UPGRADE_HEALTH_TIMEOUT",
+		"docker compose config --quiet",
+		"docker version >/dev/null",
+		"provider-msp preflight",
+		"provider-msp status",
+		"provider-msp status --require-backup",
+		"provider-msp backup create",
+		"provider-msp backup verify",
+		"provider-msp backup restore",
+		"--target-data-dir",
+		"tenant-runtime reconcile --all --dry-run",
+		"tenant-runtime reconcile",
+		"--all",
+		"--run-id",
+		"--health-timeout",
+		"--prune-previous",
+		"docker compose pull traefik control-plane",
+		"docker compose up -d traefik control-plane",
+		"docker compose run --rm --no-deps control-plane",
+		"provider_msp_upgrade_ok=true",
+		"tenant_runtime_rollout_applied=true",
+		"tenant_runtime_rollout_applied=false",
+	)
+	assertNotContainsAny(t, text,
+		"STRIPE_",
+		"CP_PUBLIC_CLOUD_SIGNUP_ENABLED",
+	)
+}
+
 func TestProviderMSPInstallProofRunnerMatchesComposeContract(t *testing.T) {
 	scriptPath := repoFile("deploy", "provider-msp", "run-install-proof.sh")
 	result := exec.Command("bash", "-n", scriptPath)