Let provider MSP upgrade.sh move an install to a new release

setup.sh resolves the image pins in .env to digests once, so upgrade.sh
only ever re-pulled the release a provider first installed. Moving to a
new release meant finding the new digests by hand, so none of a release's
fixes, and no new feature such as the portal Plan tab, reached an
existing install.

Run from a newly extracted release bundle, upgrade.sh now works on the
existing install: it resolves the bundle's control-plane and runtime tags
to digests, runs the status gate, preflight and verified backup with the
new release's control plane through a shell override while nothing on
disk changes, then keeps a copy of .env, installs the bundle files and
writes the new pins before starting the new images. --keep-image-pins
keeps hand-set pins.

It also pulls the tenant runtime image before the first status check,
which otherwise failed every upgrade to a new CP_PULSE_IMAGE with "not
present locally", including the documented pin-editing route.
This commit is contained in:
rcourtman 2026-09-24 10:43:01 +01:00
parent 353e88d22b
commit 41abdd0b2c
4 changed files with 331 additions and 6 deletions

View file

@ -113,10 +113,16 @@ PULSE_EMAIL_REPLY_TO=support@example.com
# images:
# ./upgrade.sh --dry-run
#
# Apply a provider control-plane upgrade after updating the image pins in this
# file. The runner creates and verifies a fresh backup, dry-runs restore into a
# separate target data dir, pulls provider images, starts Traefik/control-plane,
# and prints the tenant runtime rollout plan for CP_PULSE_IMAGE:
# Move to a new release: download and verify its provider bundle as for a fresh
# install, then run upgrade.sh from the extracted bundle directory. It installs
# the bundle files here, keeps this file (and a pre-upgrade copy), re-pins
# CONTROL_PLANE_IMAGE and CP_PULSE_IMAGE to that release, creates and verifies a
# fresh backup, dry-runs restore into a separate target data dir, pulls provider
# images, starts Traefik/control-plane, and prints the tenant runtime rollout
# plan for CP_PULSE_IMAGE (--keep-image-pins keeps pins you set by hand):
# sudo -E bash ./upgrade.sh
#
# Run from this directory, upgrade.sh applies the image pins already here:
# ./upgrade.sh
#
# Also roll all client runtimes to the configured CP_PULSE_IMAGE line after the

View file

@ -7,6 +7,14 @@ usage() {
Usage:
./upgrade.sh [options]
To move to a new release, download and verify its provider bundle exactly as
for a fresh install, then run ./upgrade.sh from the extracted bundle directory
(sudo -E bash ./upgrade.sh). It installs the bundle's files into the existing
install (PULSE_PROVIDER_MSP_INSTALL_DIR, default /opt/pulse-provider-msp),
keeps .env and a pre-upgrade copy of it, re-pins CONTROL_PLANE_IMAGE and
CP_PULSE_IMAGE to that release, and then runs the flow below. Run from the
install directory, it applies the image pins already in .env.
Runs the provider-hosted MSP pre-upgrade and upgrade flow:
1. validates .env and docker-compose.yml
2. checks provider status and install preflight
@ -26,6 +34,7 @@ Options:
--restore-target DIR Restore dry-run target data dir (default: <data-dir>/upgrade-restore-drill)
--run-id ID Operator-visible tenant reconcile run id
--health-timeout DURATION Tenant rollout health timeout (default: 90s)
--keep-image-pins From a bundle: install its files but keep the image pins in .env
-h, --help Show this help
Environment equivalents:
@ -75,6 +84,98 @@ run_control() {
docker compose run --rm --no-deps control-plane "$@"
}
set_env_value() {
local key="$1" value="$2" env_path="$3" tmp
tmp="$(mktemp)"
if grep -q -E "^${key}=" "${env_path}"; then
awk -v key="${key}" -v value="${value}" 'BEGIN{done=0} $0 ~ "^" key "=" && done==0 { print key "=" value; done=1; next } { print }' "${env_path}" >"${tmp}"
else
cat "${env_path}" >"${tmp}"
printf '%s=%s\n' "${key}" "${value}" >>"${tmp}"
fi
cat "${tmp}" >"${env_path}"
rm -f "${tmp}"
}
# resolve_image_digest turns a tag into an immutable digest ref, as setup.sh
# does, reading the registry without pulling the image.
resolve_image_digest() {
local ref="$1" manifest_json digest
manifest_json="$(docker buildx imagetools inspect "${ref}" --format '{{json .Manifest}}' 2>/dev/null || true)"
digest="$(printf '%s' "${manifest_json}" | jq -r 'if type == "object" then .digest // empty else empty end' 2>/dev/null || true)"
if [[ "${digest}" != sha256:* ]]; then
digest="$(docker buildx imagetools inspect "${ref}" 2>/dev/null | awk '$1 == "Digest:" {print $2; exit}' || true)"
fi
[[ "${digest}" == sha256:* ]] || return 1
printf '%s@%s\n' "${ref%:*}" "${digest}"
}
# The files setup.sh installs from a bundle. A bundle upgrade installs the same
# set, so an upgraded install matches a fresh one.
bundle_install_files=(docker-compose.yml traefik.yml traefik-dynamic.yml .env.example run-install-proof.sh upgrade.sh)
# A bundle upgrade brings an existing install up to the release bundle this
# script was extracted with. Setup resolves the image pins in .env to digests
# once, so an upgrade used to re-pull only the release the provider first
# installed; moving on meant finding four digests by hand. Nothing on disk
# changes until the platform has been checked and backed up: those steps run
# the new release's control plane as a one-off through a shell override of the
# pins (compose prefers the shell over .env), which also carries fixes to the
# checks themselves. Only then are the bundle files installed and .env
# re-pinned, just before the new images start.
bundle_dir=""
bundle_version=""
bundle_pin_keys=()
bundle_pin_values=()
plan_bundle_upgrade() {
local install_dir="$1" env_path key target current resolved
bundle_dir="$2"
bundle_version="$(tr -d '[:space:]' <"${bundle_dir}/VERSION")"
env_path="${install_dir}/.env"
[[ -f "${env_path}" ]] || die "no provider MSP install at ${install_dir} (missing .env); run ./setup.sh for a fresh install"
[[ -w "${install_dir}" && -w "${env_path}" ]] || die "cannot write ${install_dir}; run: sudo -E bash ./upgrade.sh"
echo "provider_msp_upgrade_bundle_version=${bundle_version}"
echo "provider_msp_upgrade_install_dir=${install_dir}"
if truthy "${keep_image_pins}"; then
echo "provider_msp_upgrade_image_pins=kept"
return 0
fi
for key in CONTROL_PLANE_IMAGE CP_PULSE_IMAGE; do
target="$(env_value "${key}" "${bundle_dir}/.env.example")"
[[ -n "${target}" ]] || die "bundle .env.example has no ${key}; download the release bundle, not the source tree"
current="$(env_value "${key}" "${env_path}")"
if [[ "${target}" == *@sha256:* ]]; then
resolved="${target}"
elif ! resolved="$(resolve_image_digest "${target}")"; then
die "could not resolve a digest for ${target}; check this host can reach the registry"
fi
echo "provider_msp_upgrade_image_pin ${key} current=${current} target=${resolved}"
bundle_pin_keys+=("${key}")
bundle_pin_values+=("${resolved}")
export "${key}=${resolved}"
done
}
apply_bundle_upgrade() {
local install_dir="$1" env_path stamp file i
env_path="${install_dir}/.env"
stamp="$(date -u +'%Y%m%dT%H%M%SZ')"
cp -p "${env_path}" "${env_path}.pre-upgrade-${stamp}"
echo "provider_msp_upgrade_env_backup=${env_path}.pre-upgrade-${stamp}"
for file in "${bundle_install_files[@]}"; do
[[ -f "${bundle_dir}/${file}" ]] || die "bundle is missing ${file}"
case "${file}" in
*.sh) install -m 0755 "${bundle_dir}/${file}" "${install_dir}/${file}" ;;
*) install -m 0644 "${bundle_dir}/${file}" "${install_dir}/${file}" ;;
esac
done
for ((i = 0; i < ${#bundle_pin_keys[@]}; i++)); do
set_env_value "${bundle_pin_keys[$i]}" "${bundle_pin_values[$i]}" "${env_path}"
done
echo "provider_msp_upgrade_bundle_installed=true"
}
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "${script_dir}"
@ -87,7 +188,7 @@ backup_output="${PROVIDER_MSP_UPGRADE_BACKUP_OUTPUT:-}"
restore_target="${PROVIDER_MSP_UPGRADE_RESTORE_TARGET:-}"
run_id="${PROVIDER_MSP_UPGRADE_RUN_ID:-provider-msp-upgrade-$(date -u +'%Y%m%dT%H%M%SZ')}"
health_timeout="${PROVIDER_MSP_UPGRADE_HEALTH_TIMEOUT:-90s}"
keep_image_pins="${PROVIDER_MSP_UPGRADE_KEEP_IMAGE_PINS:-0}"
while (($# > 0)); do
case "$1" in
--dry-run)
@ -126,6 +227,10 @@ while (($# > 0)); do
health_timeout="${2:-}"
shift 2
;;
--keep-image-pins)
keep_image_pins=1
shift
;;
-h|--help)
usage
exit 0
@ -138,13 +243,22 @@ while (($# > 0)); do
esac
done
# Run from an extracted release bundle (it carries VERSION and no .env), work
# on the existing install and move it to that release.
install_dir="${script_dir}"
if [[ -f "${script_dir}/VERSION" && ! -f "${script_dir}/.env" ]]; then
install_dir="${PULSE_PROVIDER_MSP_INSTALL_DIR:-/opt/pulse-provider-msp}"
plan_bundle_upgrade "${install_dir}" "${script_dir}"
cd "${install_dir}"
fi
if [[ ! -f .env ]]; then
die "deploy/provider-msp/.env is required; copy .env.example to .env and fill in the provider values"
fi
provider_data_dir="$(env_value PULSE_PROVIDER_MSP_DATA_DIR .env)"
provider_data_dir="${provider_data_dir:-/data}"
tenant_runtime_image="$(env_value CP_PULSE_IMAGE .env)"
tenant_runtime_image="${CP_PULSE_IMAGE:-$(env_value CP_PULSE_IMAGE .env)}"
if [[ -z "${tenant_runtime_image}" ]]; then
die "CP_PULSE_IMAGE is required"
fi
@ -169,11 +283,21 @@ echo "provider_msp_upgrade_run_id=${run_id}"
echo "provider_msp_upgrade_restore_target=${restore_target}"
echo "provider_msp_upgrade_tenant_runtime_image=${tenant_runtime_image}"
# Status checks that the tenant runtime image is already on this host, and runs
# before anything else would pull it, so a new CP_PULSE_IMAGE failed the gate on
# every upgrade. Pulling changes nothing that is running.
if ! truthy "${skip_runtime_image_pull}"; then
docker pull "${tenant_runtime_image}" >/dev/null
fi
run_control provider-msp status
run_control "${preflight_args[@]}"
if truthy "${dry_run}"; then
run_control tenant-runtime rollout --all --image "${tenant_runtime_image}" --dry-run
if [[ -n "${bundle_dir}" ]]; then
echo "provider_msp_upgrade_bundle_installed=false"
fi
echo "tenant_runtime_rollout_applied=false"
echo "provider_msp_upgrade_plan_ok=true"
exit 0
@ -195,6 +319,11 @@ run_control provider-msp backup verify "${archive_path}"
run_control provider-msp backup restore "${archive_path}" --target-data-dir "${restore_target}" --dry-run
run_control provider-msp status --require-backup
# The platform is checked and backed up; now install the bundle and re-pin.
if [[ -n "${bundle_dir}" ]]; then
apply_bundle_upgrade "${install_dir}"
fi
if ! truthy "${skip_compose_pull}"; then
docker compose pull traefik docker-socket-proxy control-plane
fi

View file

@ -2438,6 +2438,32 @@ artifact-selection behaviour.
## Current State
### Provider MSP upgrades move an install to a new release bundle
`setup.sh` resolves the image pins in `.env` to digests once, so `upgrade.sh`
only ever re-pulled the release a provider first installed; moving to a new
release meant finding the new digests by hand, and none of that release's
fixes reached the install. Run from a newly extracted and verified release
bundle (it carries `VERSION` and no `.env`), `upgrade.sh` now works on the
existing install (`PULSE_PROVIDER_MSP_INSTALL_DIR`, default
`/opt/pulse-provider-msp`): it resolves the bundle's `CONTROL_PLANE_IMAGE` and
`CP_PULSE_IMAGE` tags (stamped by `scripts/build-release.sh`) to digests and
prints current and target pins. It runs the status gate, preflight and the
verified backup with the new release's control plane through a shell override,
which compose prefers over `.env`, so those checks carry the new release's
fixes while nothing on disk changes. Only then does it keep a
`.env.pre-upgrade-<time>` copy, install the same bundle files `setup.sh`
installs, write the new pins and start the new images. `--keep-image-pins`
keeps hand-set pins. The runner also pulls the tenant runtime image before the
first status check, which otherwise failed every upgrade to a new
`CP_PULSE_IMAGE` with "not present locally". Verified on 2026-09-24 against
the walkthrough lab with a v6.4.5-rc.2 bundle: the pins resolved from the real
registry, and when that release's own status check failed the upgrade stopped
with `.env`, the compose file and `upgrade.sh` byte-identical and the control
plane unchanged. Regression coverage:
`TestProviderMSPUpgradeFromBundleRepinsToTheBundleRelease` in
`scripts/installtests/provider_msp_deploy_test.go`.
### Provider MSP setup points buyers at the portal
`deploy/provider-msp/setup.sh` no longer tells an evaluating provider to

View file

@ -643,3 +643,167 @@ func TestProviderMSPSetupLeavesPlatformRunning(t *testing.T) {
}
assertContainsAll(t, first, "DOMAIN", "ACME_EMAIL", "CF_DNS_API_TOKEN")
}
// Setup pins the images in .env to digests once, so an upgrade used to re-pull
// only the release a provider first installed. Run from a newly extracted
// bundle, upgrade.sh must install that bundle into the existing install,
// re-pin the two Pulse images to the bundle's release, keep a copy of the old
// .env, and only then hand over to the backup-gated flow.
func TestProviderMSPUpgradeFromBundleRepinsToTheBundleRelease(t *testing.T) {
newDigest := "sha256:" + strings.Repeat("b", 64)
oldControlPlane := "ghcr.io/rcourtman/pulse-control-plane@sha256:" + strings.Repeat("a", 64)
oldRuntime := "ghcr.io/rcourtman/pulse@sha256:" + strings.Repeat("a", 64)
setup := func(t *testing.T) (bundle, install, dockerLog string) {
t.Helper()
root := t.TempDir()
bundle = filepath.Join(root, "pulse-provider-msp-v6.6.0")
install = filepath.Join(root, "install")
bin := filepath.Join(root, "bin")
for _, dir := range []string{bundle, install, bin} {
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
}
for _, name := range []string{"docker-compose.yml", "traefik.yml", "traefik-dynamic.yml", ".env.example", "run-install-proof.sh", "upgrade.sh", "setup.sh"} {
content, err := os.ReadFile(repoFile("deploy", "provider-msp", name))
if err != nil {
t.Fatal(err)
}
if name == ".env.example" {
// What scripts/build-release.sh stamps into a release bundle.
text := strings.Replace(string(content), "\nCONTROL_PLANE_IMAGE=\n", "\nCONTROL_PLANE_IMAGE=ghcr.io/rcourtman/pulse-control-plane:v6.6.0\n", 1)
text = strings.Replace(text, "\nCP_PULSE_IMAGE=\n", "\nCP_PULSE_IMAGE=ghcr.io/rcourtman/pulse:v6.6.0\n", 1)
content = []byte(text)
}
if err := os.WriteFile(filepath.Join(bundle, name), content, 0o755); err != nil {
t.Fatal(err)
}
}
if err := os.WriteFile(filepath.Join(bundle, "VERSION"), []byte("6.6.0\n"), 0o644); err != nil {
t.Fatal(err)
}
env := "DOMAIN=msp.example.com\nCONTROL_PLANE_IMAGE=" + oldControlPlane + "\nCP_PULSE_IMAGE=" + oldRuntime + "\nPULSE_PROVIDER_MSP_DATA_DIR=/data\n"
if err := os.WriteFile(filepath.Join(install, ".env"), []byte(env), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(install, "upgrade.sh"), []byte("#!/bin/sh\necho old-upgrade-script\n"), 0o755); err != nil {
t.Fatal(err)
}
dockerLog = filepath.Join(root, "docker.log")
// Log each call with the control-plane pin in .env and the image
// compose would use (a shell value wins over .env) at that moment.
fake := "#!/bin/sh\nfile=$(grep '^CONTROL_PLANE_IMAGE=' .env 2>/dev/null | cut -d= -f2)\n" +
"echo \"$* [env=${file}] [uses=${CONTROL_PLANE_IMAGE:-$file}]\" >> " + dockerLog + "\n" +
"case \"$*\" in\n" +
" 'buildx imagetools inspect '*' --format '*) echo '{\"digest\":\"" + newDigest + "\"}' ;;\n" +
" 'buildx imagetools inspect '*) echo 'Digest: " + newDigest + "' ;;\n" +
" *'backup create'*) echo 'archive_path=/data/backups/pre-upgrade.tar.gz' ;;\n" +
"esac\nexit 0\n"
if err := os.WriteFile(filepath.Join(bin, "docker"), []byte(fake), 0o755); err != nil {
t.Fatal(err)
}
return bundle, install, dockerLog
}
run := func(t *testing.T, bundle, install string, args ...string) string {
t.Helper()
cmd := exec.Command("bash", append([]string{filepath.Join(bundle, "upgrade.sh")}, args...)...)
cmd.Env = append(os.Environ(), "PATH="+filepath.Join(filepath.Dir(bundle), "bin")+":"+os.Getenv("PATH"), "PULSE_PROVIDER_MSP_INSTALL_DIR="+install)
output, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("upgrade.sh %v: %v\n%s", args, err, output)
}
return string(output)
}
envOf := func(t *testing.T, install string) string {
t.Helper()
b, err := os.ReadFile(filepath.Join(install, ".env"))
if err != nil {
t.Fatal(err)
}
return string(b)
}
t.Run("upgrade", func(t *testing.T) {
bundle, install, dockerLog := setup(t)
output := run(t, bundle, install)
env := envOf(t, install)
for _, want := range []string{
"CONTROL_PLANE_IMAGE=ghcr.io/rcourtman/pulse-control-plane@" + newDigest,
"CP_PULSE_IMAGE=ghcr.io/rcourtman/pulse@" + newDigest,
"DOMAIN=msp.example.com",
} {
if !strings.Contains(env, want) {
t.Fatalf("upgraded .env lacks %q:\n%s", want, env)
}
}
backups, _ := filepath.Glob(filepath.Join(install, ".env.pre-upgrade-*"))
if len(backups) != 1 {
t.Fatalf("pre-upgrade .env copies = %v, want one", backups)
}
if old, _ := os.ReadFile(backups[0]); !strings.Contains(string(old), oldControlPlane) {
t.Fatal("pre-upgrade copy does not hold the previous pins")
}
installed, _ := os.ReadFile(filepath.Join(install, "upgrade.sh"))
shipped, _ := os.ReadFile(filepath.Join(bundle, "upgrade.sh"))
if string(installed) != string(shipped) {
t.Fatal("bundle upgrade.sh was not installed")
}
logBytes, _ := os.ReadFile(dockerLog)
lines := strings.Split(string(logBytes), "\n")
lineWith := func(needle string) string {
for _, line := range lines {
if strings.Contains(line, needle) {
return line
}
}
t.Fatalf("docker was never called with %q:\n%s", needle, logBytes)
return ""
}
// The check and the backup run the new release's control plane as a
// one-off while .env still holds the old pins; .env changes only
// before the new images start.
newControlPlane := "ghcr.io/rcourtman/pulse-control-plane@" + newDigest
for _, step := range []string{"control-plane provider-msp status", "provider-msp backup create"} {
line := lineWith(step)
if !strings.Contains(line, "[env="+oldControlPlane+"]") || !strings.Contains(line, "[uses="+newControlPlane+"]") {
t.Fatalf("%s should run the new control plane with .env untouched: %s", step, line)
}
}
// The new runtime image is pulled before status checks it is present.
pull := strings.Index(string(logBytes), "pull ghcr.io/rcourtman/pulse@"+newDigest)
status := strings.Index(string(logBytes), "control-plane provider-msp status")
if pull < 0 || status < 0 || pull > status {
t.Fatalf("the new tenant runtime image must be pulled before the status gate:\n%s", logBytes)
}
if line := lineWith("compose up -d traefik docker-socket-proxy control-plane"); !strings.Contains(line, "[env="+newControlPlane+"]") {
t.Fatalf("new images did not start on the new pins: %s", line)
}
if !strings.Contains(output, "provider_msp_upgrade_ok=true") || !strings.Contains(output, "provider_msp_upgrade_bundle_version=6.6.0") {
t.Fatalf("upgrade output:\n%s", output)
}
})
t.Run("keep image pins", func(t *testing.T) {
bundle, install, _ := setup(t)
run(t, bundle, install, "--keep-image-pins")
env := envOf(t, install)
if !strings.Contains(env, "CONTROL_PLANE_IMAGE="+oldControlPlane) || !strings.Contains(env, "CP_PULSE_IMAGE="+oldRuntime) {
t.Fatalf("--keep-image-pins changed the pins:\n%s", env)
}
})
t.Run("dry run changes nothing", func(t *testing.T) {
bundle, install, _ := setup(t)
output := run(t, bundle, install, "--dry-run")
if env := envOf(t, install); !strings.Contains(env, "CONTROL_PLANE_IMAGE="+oldControlPlane) {
t.Fatalf("dry run changed .env:\n%s", env)
}
if installed, _ := os.ReadFile(filepath.Join(install, "upgrade.sh")); string(installed) != "#!/bin/sh\necho old-upgrade-script\n" {
t.Fatal("dry run installed bundle files")
}
if !strings.Contains(output, "current="+oldControlPlane) || !strings.Contains(output, "target=ghcr.io/rcourtman/pulse-control-plane@"+newDigest) {
t.Fatalf("dry run did not print the pin plan:\n%s", output)
}
})
}