mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-03 04:38:48 +00:00
Reconcile published lifecycle rehearsal with reviewed maintenance
Preserve the complete reviewed maintenance tip and published upstream source unchanged. Combine their installability contract and subsystem registry entries without changing runtime behaviour. Change-source: pulse-maintainer
This commit is contained in:
pulse-triage[bot]
commit
f047d7b5db
7 changed files with 1409 additions and 0 deletions
11
.github/workflows/README.md
vendored
11
.github/workflows/README.md
vendored
|
|
@ -140,6 +140,17 @@ and verifies the live service health and exact version. The privileged systemd
|
|||
smoke environment is digest-pinned because a floating container image would
|
||||
otherwise be an unreviewed code path inside the release gate.
|
||||
|
||||
`release-lifecycle-rehearsal.yml` is a daily shadow rehearsal of the systemd
|
||||
lifecycle between two published releases, in the same digest-pinned systemd
|
||||
container. It installs the older release with its signature-verified
|
||||
`install.sh --version`, seeds real settings through the API, upgrades with the
|
||||
installed `/bin/update --version <to>` helper and rolls back with the documented
|
||||
`/bin/update --version <from>`, checking version, health, unit state, settings
|
||||
and data-dir survival after each step. It is read-only, uploads nothing and gates
|
||||
nothing; results are in the job log and step summary. Run
|
||||
`scripts/release_lifecycle_rehearsal.sh --from <tag> --to <tag>` to reproduce
|
||||
it locally (`PULSE_REHEARSAL_ENGINE=podman` on hosts without Docker).
|
||||
|
||||
The shared `install-sh-smoke-body.yml` inherits its caller's token permissions;
|
||||
keep it free of workflow- or job-level permission overrides. Continuity calls
|
||||
that body directly with `contents: read`. The existing `install-sh-smoke.yml`
|
||||
|
|
|
|||
81
.github/workflows/release-lifecycle-rehearsal.yml
vendored
Normal file
81
.github/workflows/release-lifecycle-rehearsal.yml
vendored
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
name: Release Lifecycle Rehearsal
|
||||
|
||||
# Shadow rehearsal of the systemd lifecycle users run between two PUBLISHED
|
||||
# releases: install FROM with its signed install.sh, seed real settings, upgrade
|
||||
# with /bin/update --version TO, then roll back with the documented
|
||||
# /bin/update --version FROM, asserting identity, health, unit state, settings
|
||||
# and data-dir survival after each step. It reports only. No release job
|
||||
# depends on it, and it uploads nothing.
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
from_version:
|
||||
description: 'Published tag to install first (e.g. v6.4.1). Empty = latest stable.'
|
||||
required: false
|
||||
type: string
|
||||
default: ''
|
||||
to_version:
|
||||
description: 'Published tag to upgrade to. Empty = newest published release or prerelease newer than from_version.'
|
||||
required: false
|
||||
type: string
|
||||
default: ''
|
||||
schedule:
|
||||
# Daily, well outside the 02:00-06:00 UTC unattended-update window that an
|
||||
# upgraded install may arm, so the timer cannot race the assertions.
|
||||
- cron: '23 13 * * *'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: release-lifecycle-rehearsal
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
rehearse:
|
||||
name: Install, upgrade and roll back on systemd
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
- name: Checkout rehearsal harness (for README key and scripts)
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Resolve published release pair
|
||||
id: pair
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
REPOSITORY: ${{ github.repository }}
|
||||
FROM_INPUT: ${{ inputs.from_version }}
|
||||
TO_INPUT: ${{ inputs.to_version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
releases_json="$(mktemp)"
|
||||
gh api --paginate "repos/${REPOSITORY}/releases?per_page=100" \
|
||||
| jq -s 'add // []' > "${releases_json}"
|
||||
latest_tag="$(gh api "repos/${REPOSITORY}/releases/latest" --jq '.tag_name')"
|
||||
resolved="$(python3 scripts/release_lifecycle_rehearsal_versions.py \
|
||||
--releases-json "${releases_json}" \
|
||||
--latest-tag "${latest_tag}" \
|
||||
--from "${FROM_INPUT}" \
|
||||
--to "${TO_INPUT}")"
|
||||
rm -f "${releases_json}"
|
||||
from_tag="$(sed -n 's/^from_tag=//p' <<<"${resolved}")"
|
||||
to_tag="$(sed -n 's/^to_tag=//p' <<<"${resolved}")"
|
||||
reason="$(sed -n 's/^reason=//p' <<<"${resolved}")"
|
||||
python3 scripts/write_github_output.py from_tag "${from_tag}"
|
||||
python3 scripts/write_github_output.py to_tag "${to_tag}"
|
||||
echo "Rehearsing ${from_tag} -> ${to_tag} -> ${from_tag} (${reason})"
|
||||
{
|
||||
echo "Release pair: \`${from_tag}\` -> \`${to_tag}\` (${reason})."
|
||||
echo
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Rehearse install, upgrade and rollback
|
||||
env:
|
||||
FROM_TAG: ${{ steps.pair.outputs.from_tag }}
|
||||
TO_TAG: ${{ steps.pair.outputs.to_tag }}
|
||||
PULSE_REHEARSAL_REPO: ${{ github.repository }}
|
||||
PULSE_REHEARSAL_ENGINE: docker
|
||||
run: scripts/release_lifecycle_rehearsal.sh --from "${FROM_TAG}" --to "${TO_TAG}"
|
||||
|
|
@ -842,6 +842,9 @@ release-latency optimization.
|
|||
98. `scripts/verify-github-release-integrity.sh`
|
||||
99. `scripts/verify-release-container-images.sh`
|
||||
100. `scripts/release_control/verify_release_container_images_test.py`
|
||||
101. `.github/workflows/release-lifecycle-rehearsal.yml`
|
||||
102. `scripts/release_lifecycle_rehearsal.sh`
|
||||
103. `scripts/release_lifecycle_rehearsal_versions.py`
|
||||
|
||||
## Shared Boundaries
|
||||
|
||||
|
|
@ -6125,6 +6128,82 @@ the current assignment; a host dependency acquisition on the next launch is
|
|||
required before this contract's Go evidence can be produced. No passing Go
|
||||
suite, installed build or release acceptance is claimed here.
|
||||
|
||||
### Published-release lifecycle rehearsal (shadow)
|
||||
|
||||
`.github/workflows/release-lifecycle-rehearsal.yml` rehearses the systemd
|
||||
install, upgrade and rollback journey between two published releases. It is
|
||||
the shadow first step toward making install, upgrade and rollback a publication
|
||||
gate. It runs daily and on manual dispatch, holds only `contents: read`, runs
|
||||
on the hosted `ubuntu-24.04` image, uses no secrets and uploads no artifacts.
|
||||
No release workflow calls it or waits on it, so it reports without gating.
|
||||
|
||||
`scripts/release_lifecycle_rehearsal_versions.py` chooses the pair from
|
||||
published, non-draft release tags under SemVer precedence. The default FROM is
|
||||
the release GitHub advertises as latest and must be stable. The default TO is
|
||||
the newest published release or prerelease newer than FROM. When FROM was
|
||||
defaulted and nothing newer is published, the pair is the previous stable to
|
||||
the latest stable. An explicit pair must be published and move forward, so a
|
||||
same-version or missing-target run fails instead of passing.
|
||||
|
||||
`scripts/release_lifecycle_rehearsal.sh` runs every phase inside the same
|
||||
digest-pinned `jrei/systemd-debian:12` container as the install smoke.
|
||||
|
||||
1. Install FROM with that release's own `install.sh --version`, after the
|
||||
asset verifies against the README-pinned `pulse-installer` key and passes
|
||||
the server-installer identity checks.
|
||||
2. Assert `/api/version` and `/opt/pulse/bin/pulse --version` both report
|
||||
FROM, `/api/health` reports `healthy`, and `pulse.service` is active and
|
||||
enabled with no newly failed unit.
|
||||
3. Seed state through the real API. First-run security setup uses the
|
||||
bootstrap token from `pulse bootstrap-token`, then a disabled webhook with
|
||||
a secret header and a PVE node with fake token credentials are created. The
|
||||
read-back must show that auth is required, that an unauthenticated request
|
||||
gets 401, and that the API token and password both get 200. The webhook must
|
||||
keep its id, URL, method, service, disabled state and header key. The node
|
||||
must keep its host, token name, stored token secret (`hasToken`), no
|
||||
password and `verifySSL=false`. The `/etc/pulse` file list and checksums
|
||||
become the baseline.
|
||||
4. Upgrade with the installed helper, `/bin/update --version TO`, after
|
||||
checking that the helper is the Pulse server installer's. The helper
|
||||
downloads the latest published `install.sh`, verifies it with its embedded
|
||||
key and runs it.
|
||||
5. After the upgrade, and again after the documented rollback
|
||||
`/bin/update --version FROM`, assert the expected identity, health and unit
|
||||
state. The settings read-back must equal the baseline and meet the fixed
|
||||
expectations. No baseline file may be missing from `/etc/pulse`.
|
||||
`.encryption.key`, `nodes.enc` and `webhooks.enc` must be byte-identical.
|
||||
The API reports only `hasToken` and redacted header values, so byte
|
||||
identity under an unchanged key is what proves the exact seeded secrets
|
||||
survived. A release that legitimately rewrites those stores fails the
|
||||
rehearsal instead of passing unproven. The API token and password checks
|
||||
prove those exact credentials directly.
|
||||
|
||||
Rollback still runs when the upgrade phase fails. A phase also fails if
|
||||
`pulse-update.service` started during the run, because an unattended update
|
||||
would make the version assertions unattributable. Three D-Bus-activated host
|
||||
services that cannot run in the container (`systemd-hostnamed`,
|
||||
`systemd-timedated` and `systemd-localed`) are reported as warnings rather
|
||||
than failures. A unit drop-in sets `PULSE_TELEMETRY=false` so daily CI installs
|
||||
stay out of usage telemetry. It lives outside the installer's files and the
|
||||
data dir.
|
||||
|
||||
Scope: only published assets and the amd64 systemd path are exercised. Docker,
|
||||
Helm, Proxmox LXC creation, the in-app updater and a prerelease's own
|
||||
`install.sh` (the helper always fetches the latest published installer) are
|
||||
not covered. A passing run is not release admission.
|
||||
|
||||
Verification: `scripts/tests/test_release_lifecycle_rehearsal.py` covers pair
|
||||
resolution (SemVer ordering, draft exclusion, forward-only and published-only
|
||||
pairs, the previous-stable fallback), the fixed settings expectations (each lost
|
||||
or altered seeded field fails `--check-snapshot`, and a deleted file or a
|
||||
rewritten secret store fails `--compare-datadir`), the harness contract (the real updater
|
||||
invocation without bypass flags, the signed installer, the shared image digest,
|
||||
and per-phase identity, settings and data-dir checks) and the workflow trust
|
||||
shape. A local podman run (amd64 emulation) of `v6.4.1 -> v6.4.5 -> v6.4.1`
|
||||
passed all four phases. An earlier run correctly failed phase 1 on a new failed
|
||||
unit (`systemd-hostnamed`), which led to the container-only list above. The
|
||||
hosted Docker run is not yet claimed.
|
||||
|
||||
## Release-body updater ownership (30 September 2026)
|
||||
|
||||
Each executable `/bin/update --version` example in a published body must carry
|
||||
|
|
|
|||
|
|
@ -4536,6 +4536,7 @@
|
|||
".github/workflows/recover-release-activation.yml",
|
||||
".github/workflows/release-convergence.yml",
|
||||
".github/workflows/release-dry-run.yml",
|
||||
".github/workflows/release-lifecycle-rehearsal.yml",
|
||||
".github/workflows/retry-release-convergence.yml",
|
||||
".github/workflows/update-demo-server.yml",
|
||||
".github/workflows/validate-release-assets.yml",
|
||||
|
|
@ -4630,6 +4631,8 @@
|
|||
"scripts/release_control/secure_runtime_source_manifest_v7.json",
|
||||
"scripts/release_control/validate_artifact_release_line.py",
|
||||
"scripts/release_ldflags.sh",
|
||||
"scripts/release_lifecycle_rehearsal.sh",
|
||||
"scripts/release_lifecycle_rehearsal_versions.py",
|
||||
"scripts/release_update_key.go",
|
||||
"scripts/run-release-backend-tests.sh",
|
||||
"scripts/run-release-preflight.sh",
|
||||
|
|
@ -4817,6 +4820,21 @@
|
|||
"scripts/release_control/verify_github_release_integrity_test.py"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "release-lifecycle-rehearsal",
|
||||
"label": "published-release install, upgrade and rollback rehearsal proof",
|
||||
"match_prefixes": [],
|
||||
"match_files": [
|
||||
".github/workflows/release-lifecycle-rehearsal.yml",
|
||||
"scripts/release_lifecycle_rehearsal.sh",
|
||||
"scripts/release_lifecycle_rehearsal_versions.py"
|
||||
],
|
||||
"allow_same_subsystem_tests": false,
|
||||
"test_prefixes": [],
|
||||
"exact_files": [
|
||||
"scripts/tests/test_release_lifecycle_rehearsal.py"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "release-candidate-manifest-runtime",
|
||||
"label": "immutable release candidate manifest proof",
|
||||
|
|
|
|||
700
scripts/release_lifecycle_rehearsal.sh
Executable file
700
scripts/release_lifecycle_rehearsal.sh
Executable file
|
|
@ -0,0 +1,700 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# Release lifecycle rehearsal: install, upgrade and roll back Pulse between two
|
||||
# PUBLISHED releases on a real systemd host (a privileged systemd container).
|
||||
#
|
||||
# 1. install FROM with that release's signature-verified install.sh --version
|
||||
# 2. assert FROM identity, health and unit state
|
||||
# 3. seed persistent state through the real API (first-run security setup,
|
||||
# a webhook, a PVE node with fake credentials) and snapshot the data dir
|
||||
# 4. upgrade with the installed updater users run: /bin/update --version TO
|
||||
# 5. assert TO identity, health, unit state, seeded settings and data dir
|
||||
# 6. roll back with the documented command: /bin/update --version FROM
|
||||
# 7. assert FROM identity, health, unit state, seeded settings and data dir
|
||||
#
|
||||
# The result is a phase table on stdout and in $GITHUB_STEP_SUMMARY (when set).
|
||||
# Nothing is uploaded. Exit status is non-zero when any phase fails.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/release_lifecycle_rehearsal.sh --from vX.Y.Z --to vX.Y.Z
|
||||
#
|
||||
# Environment:
|
||||
# PULSE_REHEARSAL_REPO owner/repo for release assets (default rcourtman/Pulse)
|
||||
# PULSE_REHEARSAL_ENGINE docker (default) or podman
|
||||
# PULSE_REHEARSAL_WORKDIR scratch directory (default: mktemp -d)
|
||||
# PULSE_REHEARSAL_README README holding the pinned installer key
|
||||
# (default: README.md next to this script's repo)
|
||||
|
||||
# Programs passed to cexec are single-quoted on purpose: they expand inside
|
||||
# the container from -e variables. Mount options legitimately contain commas.
|
||||
# shellcheck disable=SC2016,SC2054
|
||||
|
||||
set -euo pipefail
|
||||
export LC_ALL=C
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
SYSTEMD_IMAGE="docker.io/jrei/systemd-debian:12@sha256:61d70dc3e574337bd9df794674a60ae73113460fff16ab41a2d234b4a11dcd98"
|
||||
TAG_PATTERN='^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$'
|
||||
API="http://127.0.0.1:7655"
|
||||
DATA_DIR="/etc/pulse"
|
||||
SEED_WEBHOOK_NAME="lifecycle-rehearsal-webhook"
|
||||
SEED_WEBHOOK_URL="https://example.com/pulse-lifecycle-rehearsal"
|
||||
# PVE add-node skips live cluster detection for 192.168.77.x hosts and
|
||||
# "test-" names, so the fake node is persisted without a reachable endpoint.
|
||||
SEED_NODE_NAME="test-lifecycle-rehearsal"
|
||||
SEED_NODE_HOST="https://192.168.77.10:8006"
|
||||
SEED_NODE_TOKEN_NAME="root@pam!rehearsal"
|
||||
SEED_WEBHOOK_HEADER="X-Rehearsal-Secret"
|
||||
SEED_ADMIN_USER="rehearsal-admin"
|
||||
|
||||
FROM_TAG=""
|
||||
TO_TAG=""
|
||||
CHECK_SNAPSHOT=""
|
||||
COMPARE_DATADIR=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--from) FROM_TAG="${2:-}"; shift 2 ;;
|
||||
--to) TO_TAG="${2:-}"; shift 2 ;;
|
||||
# Offline self-tests used by the contract tests: apply the fixed
|
||||
# settings expectations to a snapshot file, or compare two data-dir
|
||||
# manifests (path<TAB>sha256), then exit.
|
||||
--check-snapshot) CHECK_SNAPSHOT="${2:-}"; shift 2 ;;
|
||||
--compare-datadir) COMPARE_DATADIR=("${2:-}" "${3:-}"); shift 3 ;;
|
||||
-h|--help) sed -n '2,/^$/p' "${BASH_SOURCE[0]}"; exit 0 ;;
|
||||
*) echo "unknown argument: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -z "$CHECK_SNAPSHOT" && ${#COMPARE_DATADIR[@]} -eq 0 ]]; then
|
||||
for tag in "$FROM_TAG" "$TO_TAG"; do
|
||||
if [[ ! "$tag" =~ $TAG_PATTERN ]]; then
|
||||
echo "::error::--from and --to must be Pulse release tags (vX.Y.Z[-pre]); got '${tag}'" >&2
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
if [[ "$FROM_TAG" == "$TO_TAG" ]]; then
|
||||
echo "::error::--from and --to must differ; a same-version run proves no upgrade" >&2
|
||||
exit 2
|
||||
fi
|
||||
fi
|
||||
|
||||
REPO="${PULSE_REHEARSAL_REPO:-rcourtman/Pulse}"
|
||||
if [[ ! "$REPO" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]]; then
|
||||
echo "::error::invalid PULSE_REHEARSAL_REPO '${REPO}'" >&2
|
||||
exit 2
|
||||
fi
|
||||
ENGINE="${PULSE_REHEARSAL_ENGINE:-docker}"
|
||||
case "$ENGINE" in
|
||||
docker|podman) ;;
|
||||
*) echo "::error::PULSE_REHEARSAL_ENGINE must be docker or podman" >&2; exit 2 ;;
|
||||
esac
|
||||
README_PATH="${PULSE_REHEARSAL_README:-${ROOT_DIR}/README.md}"
|
||||
WORK_DIR="${PULSE_REHEARSAL_WORKDIR:-$(mktemp -d)}"
|
||||
mkdir -p "${WORK_DIR}/assets" "${WORK_DIR}/state"
|
||||
CONTAINER="pulse-lifecycle-rehearsal-$$"
|
||||
SUMMARY_FILE="${GITHUB_STEP_SUMMARY:-}"
|
||||
|
||||
# Throwaway credentials for an ephemeral container. They never leave it.
|
||||
ADMIN_PASSWORD="$(od -An -N24 -tx1 /dev/urandom | tr -d ' \n')"
|
||||
API_TOKEN="$(od -An -N32 -tx1 /dev/urandom | tr -d ' \n')"
|
||||
NODE_TOKEN_VALUE="$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n')"
|
||||
|
||||
PHASE_ROWS=()
|
||||
FAILURES=()
|
||||
CURRENT_FAILURE=""
|
||||
OVERALL_STATUS=0
|
||||
CONTAINER_STARTED=false
|
||||
BASELINE_FAILED_UNITS=""
|
||||
# D-Bus-activated host services that cannot run inside the test container
|
||||
# (no hostname/clock/locale ownership). Any other new failed unit, and every
|
||||
# pulse* unit, fails the phase.
|
||||
CONTAINER_TOLERATED_UNITS="systemd-hostnamed.service systemd-timedated.service systemd-localed.service"
|
||||
|
||||
log() { printf '[rehearsal] %s\n' "$*"; }
|
||||
note_failure() {
|
||||
CURRENT_FAILURE="${CURRENT_FAILURE:+${CURRENT_FAILURE}; }$*"
|
||||
echo "::error::$*"
|
||||
}
|
||||
|
||||
# Run a bash program inside the container. Data reaches the program only as
|
||||
# environment variables, never by interpolation into the program text.
|
||||
cexec() {
|
||||
local program="$1"
|
||||
shift
|
||||
local -a env_args=(-e PULSE_INSTALL_ALLOW_DOCKER=1)
|
||||
local pair
|
||||
for pair in "$@"; do
|
||||
env_args+=(-e "$pair")
|
||||
done
|
||||
"$ENGINE" exec "${env_args[@]}" "$CONTAINER" bash -c "$program"
|
||||
}
|
||||
|
||||
print_diagnostics() {
|
||||
[[ "$CONTAINER_STARTED" == "true" ]] || return 0
|
||||
echo "::group::Diagnostics: pulse journal (tail)"
|
||||
cexec 'journalctl -u pulse --no-pager | tail -n 150' || true
|
||||
echo "::endgroup::"
|
||||
echo "::group::Diagnostics: unit state"
|
||||
cexec 'systemctl status pulse --no-pager; systemctl list-units --state=failed --no-pager; systemctl list-timers --all --no-pager | grep -i pulse' || true
|
||||
echo "::endgroup::"
|
||||
echo "::group::Diagnostics: pulse-update journal"
|
||||
cexec 'journalctl -u pulse-update --no-pager | tail -n 60' || true
|
||||
echo "::endgroup::"
|
||||
local logfile
|
||||
for logfile in "${WORK_DIR}"/state/*.log; do
|
||||
[[ -f "$logfile" ]] || continue
|
||||
echo "::group::Diagnostics: $(basename "$logfile") (tail)"
|
||||
tail -n 80 "$logfile" || true
|
||||
echo "::endgroup::"
|
||||
done
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
local status=$?
|
||||
if [[ "$OVERALL_STATUS" -ne 0 || "$status" -ne 0 ]]; then
|
||||
print_diagnostics
|
||||
fi
|
||||
if [[ "$CONTAINER_STARTED" == "true" ]]; then
|
||||
"$ENGINE" rm -f "$CONTAINER" >/dev/null 2>&1 || true
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
write_summary() {
|
||||
local table
|
||||
table=$(
|
||||
echo "## Release lifecycle rehearsal"
|
||||
echo
|
||||
echo "\`${FROM_TAG}\` -> \`${TO_TAG}\` -> \`${FROM_TAG}\` on a systemd host (${SYSTEMD_IMAGE%%@*}). Shadow run, not a release gate."
|
||||
echo
|
||||
echo "| Phase | Expected | Reported version | Health | Settings | Data dir | Units | Result |"
|
||||
echo "| --- | --- | --- | --- | --- | --- | --- | --- |"
|
||||
local row
|
||||
for row in "${PHASE_ROWS[@]}"; do
|
||||
echo "$row"
|
||||
done
|
||||
if [[ ${#FAILURES[@]} -gt 0 ]]; then
|
||||
echo
|
||||
echo "### Failures"
|
||||
local failure
|
||||
for failure in "${FAILURES[@]}"; do
|
||||
echo "- ${failure}"
|
||||
done
|
||||
fi
|
||||
)
|
||||
echo
|
||||
echo "$table"
|
||||
if [[ -n "$SUMMARY_FILE" ]]; then
|
||||
printf '%s\n' "$table" >> "$SUMMARY_FILE"
|
||||
fi
|
||||
}
|
||||
|
||||
# record_phase NAME EXPECTED VERSION HEALTH SETTINGS DATADIR UNITS
|
||||
record_phase() {
|
||||
local result="pass"
|
||||
if [[ -n "$CURRENT_FAILURE" ]]; then
|
||||
result="FAIL"
|
||||
FAILURES+=("$1: ${CURRENT_FAILURE}")
|
||||
OVERALL_STATUS=1
|
||||
fi
|
||||
PHASE_ROWS+=("| $1 | \`$2\` | ${3:--} | ${4:--} | ${5:--} | ${6:--} | ${7:--} | ${result} |")
|
||||
CURRENT_FAILURE=""
|
||||
[[ "$result" == "pass" ]]
|
||||
}
|
||||
|
||||
abort_run() {
|
||||
write_summary
|
||||
exit 1
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Published installer, verified against the README-pinned key
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
fetch_and_verify_installer() {
|
||||
local tag="$1"
|
||||
local dest="${WORK_DIR}/assets"
|
||||
local base="https://github.com/${REPO}/releases/download/${tag}"
|
||||
local asset
|
||||
for asset in install.sh install.sh.sshsig; do
|
||||
if ! curl -fsSL --retry 5 --retry-delay 5 --retry-all-errors \
|
||||
-o "${dest}/${asset}" "${base}/${asset}"; then
|
||||
echo "::error::could not download ${base}/${asset}"
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
|
||||
local readme_key allowed_signers
|
||||
readme_key=$(grep -oE 'ssh-ed25519 [A-Za-z0-9+/=]+ pulse-installer' "$README_PATH" | head -1)
|
||||
if [[ -z "$readme_key" ]]; then
|
||||
echo "::error::Could not extract the pulse-installer key from ${README_PATH}"
|
||||
return 1
|
||||
fi
|
||||
allowed_signers="${WORK_DIR}/allowed_signers"
|
||||
printf 'pulse-installer %s\n' "$readme_key" > "$allowed_signers"
|
||||
if ! ssh-keygen -Y verify \
|
||||
-f "$allowed_signers" \
|
||||
-I pulse-installer \
|
||||
-n pulse-install \
|
||||
-s "${dest}/install.sh.sshsig" < "${dest}/install.sh"; then
|
||||
echo "::error::${tag} install.sh.sshsig does not verify against the README's pinned key"
|
||||
return 1
|
||||
fi
|
||||
if ! grep -qE '^# Pulse Installer Script' "${dest}/install.sh" \
|
||||
|| grep -q 'Pulse Unified Agent Installer' "${dest}/install.sh" \
|
||||
|| ! grep -qE '^[[:space:]]*--version\)' "${dest}/install.sh"; then
|
||||
echo "::error::${tag} install.sh is not the Pulse server installer with --version support"
|
||||
return 1
|
||||
fi
|
||||
log "${tag} install.sh signature verifies against the README-pinned key"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Container
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
start_container() {
|
||||
local -a run_args=(run -d --name "$CONTAINER" --privileged
|
||||
-v "${WORK_DIR}/assets:/rehearsal:ro")
|
||||
if [[ "$ENGINE" == "docker" ]]; then
|
||||
# GHA ubuntu-24.04 uses the cgroup v2 unified hierarchy; without
|
||||
# --cgroupns=host systemd PID 1 cannot mount the cgroup tree.
|
||||
run_args+=(--cgroupns=host -v /sys/fs/cgroup:/sys/fs/cgroup:rw
|
||||
--tmpfs /tmp:rw,size=2g --tmpfs /run --tmpfs /run/lock)
|
||||
else
|
||||
# Podman's systemd mode mounts /run, /run/lock and the cgroup tree.
|
||||
# The pinned image is amd64-only.
|
||||
run_args+=(--systemd=always --arch amd64 --tmpfs /tmp:rw,size=2g)
|
||||
fi
|
||||
CONTAINER_STARTED=true
|
||||
if ! "$ENGINE" "${run_args[@]}" "$SYSTEMD_IMAGE" >/dev/null; then
|
||||
echo "::error::${ENGINE} could not start ${SYSTEMD_IMAGE}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local i state
|
||||
for i in $(seq 1 45); do
|
||||
if ! "$ENGINE" inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null | grep -q true; then
|
||||
"$ENGINE" logs "$CONTAINER" || true
|
||||
echo "::error::systemd container exited during boot"
|
||||
return 1
|
||||
fi
|
||||
state=$("$ENGINE" exec "$CONTAINER" systemctl is-system-running 2>/dev/null || true)
|
||||
if [[ "$state" == "running" || "$state" == "degraded" ]]; then
|
||||
break
|
||||
fi
|
||||
if [[ "$i" -eq 45 ]]; then
|
||||
echo "::error::systemd did not become ready inside the container (state: ${state:-unknown})"
|
||||
return 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
BASELINE_FAILED_UNITS=$(cexec 'systemctl list-units --state=failed --no-legend --plain | awk "{print \$1}" | sort')
|
||||
log "systemd is up (pre-existing failed units: ${BASELINE_FAILED_UNITS:-none})"
|
||||
if ! cexec 'export DEBIAN_FRONTEND=noninteractive; apt-get update -qq >/dev/null && apt-get install -y -qq curl ca-certificates jq >/dev/null'; then
|
||||
echo "::error::could not install curl/jq inside the container"
|
||||
return 1
|
||||
fi
|
||||
# Keep a daily CI install out of real usage telemetry. A unit drop-in is
|
||||
# outside the installer's ownership and the data dir, so it does not
|
||||
# change what install, upgrade or rollback do.
|
||||
cexec 'mkdir -p /etc/systemd/system/pulse.service.d && printf "[Service]\nEnvironment=PULSE_TELEMETRY=false\n" > /etc/systemd/system/pulse.service.d/50-rehearsal-no-telemetry.conf'
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Assertions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
PHASE_VERSION=""
|
||||
PHASE_HEALTH=""
|
||||
PHASE_SETTINGS=""
|
||||
PHASE_DATADIR=""
|
||||
PHASE_UNITS=""
|
||||
|
||||
reset_phase_cells() {
|
||||
PHASE_VERSION="-"; PHASE_HEALTH="-"; PHASE_SETTINGS="-"; PHASE_DATADIR="-"; PHASE_UNITS="-"
|
||||
}
|
||||
|
||||
# Identity, health and systemd state for the expected release.
|
||||
assert_runtime() {
|
||||
local expected_tag="$1"
|
||||
local expected="${expected_tag#v}"
|
||||
|
||||
local active="" i
|
||||
for i in $(seq 1 60); do
|
||||
active=$(cexec 'systemctl is-active pulse' 2>/dev/null || true)
|
||||
[[ "$active" == "active" ]] && break
|
||||
sleep 2
|
||||
done
|
||||
|
||||
local health_body health_status
|
||||
health_body=$(cexec 'curl -fsS --retry 30 --retry-delay 2 --retry-connrefused --retry-all-errors "$API/api/health"' "API=${API}" 2>/dev/null || true)
|
||||
health_status=$(jq -r '.status // empty' <<<"$health_body" 2>/dev/null || true)
|
||||
if [[ "$health_status" == "healthy" ]]; then
|
||||
PHASE_HEALTH="healthy"
|
||||
else
|
||||
PHASE_HEALTH="${health_status:-no response}"
|
||||
note_failure "/api/health did not report healthy (got '${PHASE_HEALTH}')"
|
||||
fi
|
||||
|
||||
local version_body reported binary_version
|
||||
version_body=$(cexec 'curl -fsS "$API/api/version"' "API=${API}" 2>/dev/null || true)
|
||||
reported=$(jq -r '.version // empty' <<<"$version_body" 2>/dev/null || true)
|
||||
PHASE_VERSION="${reported:-none}"
|
||||
if [[ "${reported#v}" != "$expected" ]]; then
|
||||
note_failure "/api/version reported '${reported:-none}', expected ${expected_tag}"
|
||||
fi
|
||||
binary_version=$(cexec '/opt/pulse/bin/pulse --version 2>/dev/null | grep -oE "v[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.]+)?" | head -1' 2>/dev/null || true)
|
||||
if [[ "${binary_version#v}" != "$expected" ]]; then
|
||||
note_failure "/opt/pulse/bin/pulse --version reported '${binary_version:-none}', expected ${expected_tag}"
|
||||
fi
|
||||
|
||||
local enabled failed_now new_failed
|
||||
enabled=$(cexec 'systemctl is-enabled pulse' 2>/dev/null || true)
|
||||
failed_now=$(cexec 'systemctl list-units --state=failed --no-legend --plain | awk "{print \$1}" | sort' 2>/dev/null || true)
|
||||
new_failed=$(comm -13 <(printf '%s\n' "$BASELINE_FAILED_UNITS" | sed '/^$/d') \
|
||||
<(printf '%s\n' "$failed_now" | sed '/^$/d') | tr '\n' ' ')
|
||||
PHASE_UNITS="pulse ${active:-unknown}/${enabled:-unknown}"
|
||||
if [[ "$active" != "active" ]]; then
|
||||
note_failure "pulse.service is '${active:-unknown}', expected active"
|
||||
fi
|
||||
if [[ "$enabled" != "enabled" ]]; then
|
||||
note_failure "pulse.service is '${enabled:-unknown}', expected enabled"
|
||||
fi
|
||||
local unit tolerated="" unexpected=""
|
||||
for unit in $new_failed; do
|
||||
if [[ " ${CONTAINER_TOLERATED_UNITS} " == *" ${unit} "* ]]; then
|
||||
tolerated="${tolerated:+${tolerated} }${unit}"
|
||||
else
|
||||
unexpected="${unexpected:+${unexpected} }${unit}"
|
||||
fi
|
||||
done
|
||||
if [[ -n "$unexpected" ]]; then
|
||||
PHASE_UNITS="${PHASE_UNITS}, failed: ${unexpected}"
|
||||
note_failure "systemd units failed during the rehearsal: ${unexpected}"
|
||||
fi
|
||||
if [[ -n "$tolerated" ]]; then
|
||||
PHASE_UNITS="${PHASE_UNITS}, container-only failures ignored: ${tolerated}"
|
||||
echo "::warning::container-environment units failed and were not counted: ${tolerated}"
|
||||
fi
|
||||
|
||||
if ! cexec 'systemctl show -p Environment --value pulse | grep -q "PULSE_TELEMETRY=false"' 2>/dev/null; then
|
||||
echo "::warning::the rehearsal telemetry opt-out drop-in is no longer applied to pulse.service"
|
||||
fi
|
||||
|
||||
# An unattended update firing mid-rehearsal would invalidate the version
|
||||
# assertions, so prove the timer-driven updater never started.
|
||||
local auto_update_started
|
||||
auto_update_started=$(cexec 'systemctl show -p ExecMainStartTimestampMonotonic --value pulse-update.service 2>/dev/null || echo 0' 2>/dev/null || echo 0)
|
||||
if [[ -n "$auto_update_started" && "$auto_update_started" != "0" ]]; then
|
||||
note_failure "pulse-update.service ran during the rehearsal; results are not attributable to the manual updater"
|
||||
fi
|
||||
}
|
||||
|
||||
api_status() {
|
||||
# api_status METHOD PATH [auth: none|token|basic] [body]
|
||||
cexec 'args=(-sS -o /dev/null -w "%{http_code}" -X "$METHOD")
|
||||
case "$AUTH" in
|
||||
token) args+=(-H "X-API-Token: $TOKEN") ;;
|
||||
basic) args+=(-u "$BASIC") ;;
|
||||
esac
|
||||
curl "${args[@]}" "$API$REQ_PATH"' \
|
||||
"API=${API}" "METHOD=$1" "REQ_PATH=$2" "AUTH=${3:-none}" \
|
||||
"TOKEN=${API_TOKEN}" "BASIC=${SEED_ADMIN_USER}:${ADMIN_PASSWORD}"
|
||||
}
|
||||
|
||||
api_json() {
|
||||
# api_json METHOD PATH [body]; authenticates with the seeded API token.
|
||||
cexec 'if [[ -n "$BODY" ]]; then
|
||||
curl -fsS -X "$METHOD" -H "X-API-Token: $TOKEN" -H "Content-Type: application/json" --data "$BODY" "$API$REQ_PATH"
|
||||
else
|
||||
curl -fsS -X "$METHOD" -H "X-API-Token: $TOKEN" "$API$REQ_PATH"
|
||||
fi' "API=${API}" "METHOD=$1" "REQ_PATH=$2" "BODY=${3:-}" "TOKEN=${API_TOKEN}"
|
||||
}
|
||||
|
||||
# Canonical, comparable view of the seeded settings as the API returns them.
|
||||
settings_snapshot() {
|
||||
local webhooks nodes security unauth token basic
|
||||
webhooks=$(api_json GET /api/notifications/webhooks) || return 1
|
||||
nodes=$(api_json GET /api/config/nodes) || return 1
|
||||
security=$(cexec 'curl -fsS "$API/api/security/status"' "API=${API}") || return 1
|
||||
unauth=$(api_status GET /api/config/nodes none)
|
||||
token=$(api_status GET /api/config/nodes token)
|
||||
basic=$(api_status GET /api/config/nodes basic)
|
||||
jq -n -S \
|
||||
--argjson webhooks "$webhooks" \
|
||||
--argjson nodes "$nodes" \
|
||||
--argjson security "$security" \
|
||||
--arg unauth "$unauth" --arg token "$token" --arg basic "$basic" \
|
||||
--arg webhook_name "$SEED_WEBHOOK_NAME" --arg node_name "$SEED_NODE_NAME" '
|
||||
{
|
||||
auth: {
|
||||
requiresAuth: ($security.requiresAuth // null),
|
||||
unauthenticated: $unauth,
|
||||
api_token: $token,
|
||||
password: $basic
|
||||
},
|
||||
webhook: ([$webhooks | .. | objects | select(.name? == $webhook_name)
|
||||
| {id, name, url, method, service, enabled,
|
||||
header_keys: ((.headers // {}) | keys)}] | first),
|
||||
node: ([$nodes | .. | objects | select(.name? == $node_name)
|
||||
| {name, type, host, tokenName, hasToken, hasPassword, verifySSL}] | first)
|
||||
}'
|
||||
}
|
||||
|
||||
# Expected shape of a healthy snapshot (independent of the baseline, so a
|
||||
# baseline that silently lost data cannot make later phases pass).
|
||||
check_snapshot_shape() {
|
||||
local snapshot="$1"
|
||||
local problems
|
||||
problems=$(jq -r \
|
||||
--arg webhook_name "$SEED_WEBHOOK_NAME" --arg webhook_url "$SEED_WEBHOOK_URL" \
|
||||
--arg node_name "$SEED_NODE_NAME" --arg node_host "$SEED_NODE_HOST" \
|
||||
--arg node_token_name "$SEED_NODE_TOKEN_NAME" --arg header "$SEED_WEBHOOK_HEADER" '
|
||||
[ (if .auth.requiresAuth != true then "security status does not require auth" else empty end),
|
||||
(if .auth.unauthenticated != "401" then "unauthenticated request returned \(.auth.unauthenticated), expected 401" else empty end),
|
||||
(if .auth.api_token != "200" then "API token request returned \(.auth.api_token), expected 200" else empty end),
|
||||
(if .auth.password != "200" then "password request returned \(.auth.password), expected 200" else empty end),
|
||||
(if .webhook == null then "seeded webhook \($webhook_name) missing" else empty end),
|
||||
(if .webhook != null and .webhook.url != $webhook_url then "seeded webhook URL changed to \(.webhook.url)" else empty end),
|
||||
(if .webhook != null and .webhook.method != "POST" then "seeded webhook method changed to \(.webhook.method)" else empty end),
|
||||
(if .webhook != null and .webhook.service != "generic" then "seeded webhook service changed to \(.webhook.service)" else empty end),
|
||||
(if .webhook != null and .webhook.enabled != false then "seeded webhook enabled state changed to \(.webhook.enabled)" else empty end),
|
||||
(if .webhook != null and ((.webhook.id // "") == "") then "seeded webhook lost its id" else empty end),
|
||||
(if .webhook != null and .webhook.header_keys != [$header] then "seeded webhook headers changed to \(.webhook.header_keys)" else empty end),
|
||||
(if .node == null then "seeded node \($node_name) missing" else empty end),
|
||||
(if .node != null and .node.type != "pve" then "seeded node type changed to \(.node.type)" else empty end),
|
||||
(if .node != null and .node.host != $node_host then "seeded node host changed to \(.node.host)" else empty end),
|
||||
(if .node != null and .node.tokenName != $node_token_name then "seeded node token name changed to \(.node.tokenName)" else empty end),
|
||||
(if .node != null and .node.hasToken != true then "seeded node lost its API token secret" else empty end),
|
||||
(if .node != null and .node.hasPassword != false then "seeded node gained a password" else empty end),
|
||||
(if .node != null and .node.verifySSL != false then "seeded node verifySSL changed to \(.node.verifySSL)" else empty end)
|
||||
] | .[]' <<<"$snapshot") || problems="snapshot is not valid JSON"
|
||||
if [[ -n "$problems" ]]; then
|
||||
while IFS= read -r problem; do
|
||||
note_failure "settings: ${problem}"
|
||||
done <<<"$problems"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
datadir_manifest() {
|
||||
# path<TAB>sha256 for every regular file, excluding SQLite sidecars and
|
||||
# temp files whose presence depends on shutdown timing.
|
||||
cexec 'cd "$DATA_DIR" && find . -type f ! -name "*-wal" ! -name "*-shm" ! -name "*.tmp" ! -name "*.lock" -printf "%P\n" | LC_ALL=C sort | while IFS= read -r f; do printf "%s\t%s\n" "$f" "$(sha256sum "$f" | cut -d" " -f1)"; done' \
|
||||
"DATA_DIR=${DATA_DIR}"
|
||||
}
|
||||
|
||||
# Encrypted stores holding the seeded secrets (node token value, webhook
|
||||
# header value). The API only exposes hasToken and redacted header values, so
|
||||
# byte identity of these files under an unchanged key is the proof that the
|
||||
# exact secrets survived. A version that legitimately rewrites them fails the
|
||||
# rehearsal loudly rather than passing unproven.
|
||||
SECRET_STORES=".encryption.key nodes.enc webhooks.enc"
|
||||
|
||||
# compare_datadir BASELINE CURRENT LABEL: manifest comparison (pure, testable).
|
||||
compare_datadir() {
|
||||
local baseline="$1" current="$2" label="$3"
|
||||
local missing changed added store before after
|
||||
missing=$(comm -23 <(cut -f1 "$baseline") <(cut -f1 "$current") | sed '/^$/d')
|
||||
added=$(comm -13 <(cut -f1 "$baseline") <(cut -f1 "$current") | sed '/^$/d' | wc -l | tr -d ' ')
|
||||
changed=$(join -t $'\t' "$baseline" "$current" | awk -F'\t' '$2 != $3' | wc -l | tr -d ' ')
|
||||
|
||||
PHASE_DATADIR="$(wc -l < "$current" | tr -d ' ') files, ${changed} changed, ${added} new"
|
||||
log "${DATA_DIR} after ${label}: ${PHASE_DATADIR}"
|
||||
if [[ -n "$missing" ]]; then
|
||||
PHASE_DATADIR="${PHASE_DATADIR}, $(wc -l <<<"$missing" | tr -d ' ') missing"
|
||||
note_failure "files present before the upgrade are gone from ${DATA_DIR}: $(tr '\n' ' ' <<<"$missing")"
|
||||
fi
|
||||
for store in $SECRET_STORES; do
|
||||
before=$(awk -F'\t' -v f="$store" '$1 == f {print $2}' "$baseline")
|
||||
after=$(awk -F'\t' -v f="$store" '$1 == f {print $2}' "$current")
|
||||
if [[ -z "$before" || "$before" != "$after" ]]; then
|
||||
note_failure "${DATA_DIR}/${store} is missing or was rewritten; the seeded secrets are no longer provably intact"
|
||||
fi
|
||||
done
|
||||
if [[ "$changed" -gt 0 ]]; then
|
||||
log "files rewritten in place (informational unless listed above):"
|
||||
join -t $'\t' "$baseline" "$current" | awk -F'\t' '$2 != $3 {print " " $1}'
|
||||
fi
|
||||
}
|
||||
|
||||
check_datadir() {
|
||||
local baseline="${WORK_DIR}/state/datadir.baseline.tsv"
|
||||
local current="${WORK_DIR}/state/datadir.$1.tsv"
|
||||
datadir_manifest > "$current" || { note_failure "could not list ${DATA_DIR}"; PHASE_DATADIR="unreadable"; return 1; }
|
||||
compare_datadir "$baseline" "$current" "$1"
|
||||
}
|
||||
|
||||
check_settings() {
|
||||
local snapshot
|
||||
if ! snapshot=$(settings_snapshot); then
|
||||
note_failure "settings could not be read back through the API"
|
||||
PHASE_SETTINGS="unreadable"
|
||||
return 1
|
||||
fi
|
||||
printf '%s\n' "$snapshot" > "${WORK_DIR}/state/settings.$1.json"
|
||||
local ok=true
|
||||
check_snapshot_shape "$snapshot" || ok=false
|
||||
if ! diff -u "${WORK_DIR}/state/settings.baseline.json" "${WORK_DIR}/state/settings.$1.json"; then
|
||||
note_failure "seeded settings read back differently than before the upgrade (diff above)"
|
||||
ok=false
|
||||
fi
|
||||
if [[ "$ok" == "true" ]]; then
|
||||
PHASE_SETTINGS="auth, webhook, node intact"
|
||||
else
|
||||
PHASE_SETTINGS="drift"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phases
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
phase_install() {
|
||||
reset_phase_cells
|
||||
log "Phase 1: install ${FROM_TAG} with its published install.sh --version"
|
||||
# No TTY: install.sh's prompts take their defaults, as for curl | bash.
|
||||
if ! cexec 'bash /rehearsal/install.sh --version "$FROM_TAG"' "FROM_TAG=${FROM_TAG}" \
|
||||
2>&1 | tee "${WORK_DIR}/state/install-${FROM_TAG}.log"; then
|
||||
note_failure "install.sh --version ${FROM_TAG} exited non-zero"
|
||||
fi
|
||||
assert_runtime "$FROM_TAG"
|
||||
record_phase "1. install" "$FROM_TAG" "$PHASE_VERSION" "$PHASE_HEALTH" "-" "-" "$PHASE_UNITS"
|
||||
}
|
||||
|
||||
phase_seed() {
|
||||
reset_phase_cells
|
||||
log "Phase 2: seed first-run security, a webhook and a PVE node through the API"
|
||||
local token_output setup_token setup_response
|
||||
token_output=$(cexec 'runuser -u pulse -- env PULSE_DATA_DIR="$DATA_DIR" /opt/pulse/bin/pulse bootstrap-token' "DATA_DIR=${DATA_DIR}" 2>&1 || true)
|
||||
setup_token=$(grep -oE 'Token: [^[:space:]]+' <<<"$token_output" | head -1 | cut -d' ' -f2)
|
||||
if [[ -z "$setup_token" ]]; then
|
||||
note_failure "pulse bootstrap-token did not print a setup token"
|
||||
record_phase "2. seed" "$FROM_TAG" "-" "-" "-" "-" "-"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local setup_body
|
||||
setup_body=$(jq -cn --arg u "$SEED_ADMIN_USER" --arg p "$ADMIN_PASSWORD" --arg t "$API_TOKEN" \
|
||||
'{username: $u, password: $p, apiToken: $t}')
|
||||
if ! setup_response=$(cexec 'curl -fsS -X POST -H "Content-Type: application/json" -H "X-Setup-Token: $SETUP_TOKEN" --data "$BODY" "$API/api/security/quick-setup"' \
|
||||
"API=${API}" "SETUP_TOKEN=${setup_token}" "BODY=${setup_body}") \
|
||||
|| [[ "$(jq -r '.success // false' <<<"$setup_response" 2>/dev/null)" != "true" ]]; then
|
||||
note_failure "first-run security setup failed: ${setup_response:-no response}"
|
||||
fi
|
||||
|
||||
local webhook_body node_body
|
||||
webhook_body=$(jq -cn --arg n "$SEED_WEBHOOK_NAME" --arg u "$SEED_WEBHOOK_URL" \
|
||||
--arg hk "$SEED_WEBHOOK_HEADER" --arg hv "$NODE_TOKEN_VALUE" \
|
||||
'{name: $n, url: $u, method: "POST", service: "generic", enabled: false,
|
||||
headers: {($hk): $hv}}')
|
||||
if ! api_json POST /api/notifications/webhooks "$webhook_body" >/dev/null; then
|
||||
note_failure "creating the seed webhook failed"
|
||||
fi
|
||||
node_body=$(jq -cn --arg n "$SEED_NODE_NAME" --arg h "$SEED_NODE_HOST" --arg v "$NODE_TOKEN_VALUE" \
|
||||
--arg tn "$SEED_NODE_TOKEN_NAME" \
|
||||
'{type: "pve", name: $n, host: $h, tokenName: $tn, tokenValue: $v, verifySSL: false}')
|
||||
if ! api_json POST /api/config/nodes "$node_body" >/dev/null; then
|
||||
note_failure "creating the seed PVE node failed"
|
||||
fi
|
||||
|
||||
local snapshot=""
|
||||
if snapshot=$(settings_snapshot) && check_snapshot_shape "$snapshot"; then
|
||||
printf '%s\n' "$snapshot" > "${WORK_DIR}/state/settings.baseline.json"
|
||||
PHASE_SETTINGS="auth, webhook, node seeded"
|
||||
log "Seeded settings as read back through the API:"
|
||||
printf '%s\n' "$snapshot"
|
||||
else
|
||||
if [[ -n "$snapshot" ]]; then
|
||||
printf '%s\n' "$snapshot"
|
||||
fi
|
||||
note_failure "seeded settings did not read back through the API"
|
||||
PHASE_SETTINGS="not readable"
|
||||
fi
|
||||
|
||||
# Persistence is asynchronous for some stores; let writes settle before
|
||||
# the data dir becomes the upgrade baseline.
|
||||
sleep 5
|
||||
if datadir_manifest > "${WORK_DIR}/state/datadir.baseline.tsv" \
|
||||
&& grep -q $'^\\.encryption\\.key\t' "${WORK_DIR}/state/datadir.baseline.tsv"; then
|
||||
PHASE_DATADIR="$(wc -l < "${WORK_DIR}/state/datadir.baseline.tsv" | tr -d ' ') files recorded"
|
||||
log "${DATA_DIR} baseline:"
|
||||
cut -f1 "${WORK_DIR}/state/datadir.baseline.tsv" | sed 's/^/ /'
|
||||
else
|
||||
note_failure "could not record ${DATA_DIR} (or it has no .encryption.key)"
|
||||
PHASE_DATADIR="not recorded"
|
||||
fi
|
||||
record_phase "2. seed" "$FROM_TAG" "-" "-" "$PHASE_SETTINGS" "$PHASE_DATADIR" "-"
|
||||
}
|
||||
|
||||
run_updater() {
|
||||
local target="$1" label="$2"
|
||||
# The helper must be the one the Pulse server installer wrote, not a
|
||||
# community-scripts updater that ignores --version.
|
||||
if ! cexec 'test -x /bin/update && grep -q "^# Pulse update command" /bin/update'; then
|
||||
note_failure "/bin/update is missing or is not the Pulse server installer's update helper"
|
||||
return 1
|
||||
fi
|
||||
# /bin/update downloads the latest published install.sh, verifies it with
|
||||
# its embedded signer key and runs it with the given arguments.
|
||||
if ! cexec '/bin/update --version "$TARGET"' "TARGET=${target}" \
|
||||
2>&1 | tee "${WORK_DIR}/state/${label}-${target}.log"; then
|
||||
note_failure "/bin/update --version ${target} exited non-zero"
|
||||
fi
|
||||
}
|
||||
|
||||
phase_upgrade() {
|
||||
reset_phase_cells
|
||||
log "Phase 3: upgrade with /bin/update --version ${TO_TAG}"
|
||||
run_updater "$TO_TAG" upgrade || true
|
||||
assert_runtime "$TO_TAG"
|
||||
check_settings upgrade || true
|
||||
check_datadir upgrade || true
|
||||
record_phase "3. upgrade (/bin/update)" "$TO_TAG" "$PHASE_VERSION" "$PHASE_HEALTH" "$PHASE_SETTINGS" "$PHASE_DATADIR" "$PHASE_UNITS"
|
||||
}
|
||||
|
||||
phase_rollback() {
|
||||
reset_phase_cells
|
||||
log "Phase 4: roll back with /bin/update --version ${FROM_TAG}"
|
||||
run_updater "$FROM_TAG" rollback || true
|
||||
assert_runtime "$FROM_TAG"
|
||||
check_settings rollback || true
|
||||
check_datadir rollback || true
|
||||
record_phase "4. rollback (/bin/update)" "$FROM_TAG" "$PHASE_VERSION" "$PHASE_HEALTH" "$PHASE_SETTINGS" "$PHASE_DATADIR" "$PHASE_UNITS"
|
||||
}
|
||||
|
||||
main() {
|
||||
log "Rehearsing ${FROM_TAG} -> ${TO_TAG} -> ${FROM_TAG} from ${REPO} with ${ENGINE}"
|
||||
if ! fetch_and_verify_installer "$FROM_TAG"; then
|
||||
CURRENT_FAILURE="published install.sh for ${FROM_TAG} failed verification"
|
||||
record_phase "0. verify installer" "$FROM_TAG" "-" "-" "-" "-" "-" || abort_run
|
||||
fi
|
||||
if ! start_container; then
|
||||
CURRENT_FAILURE="systemd container did not start"
|
||||
record_phase "0. systemd host" "-" "-" "-" "-" "-" "-" || abort_run
|
||||
fi
|
||||
|
||||
phase_install || abort_run
|
||||
phase_seed || abort_run
|
||||
# Rollback runs even when the upgrade fails: it is the recovery path a
|
||||
# user reaches for in exactly that situation.
|
||||
phase_upgrade || true
|
||||
phase_rollback || true
|
||||
|
||||
write_summary
|
||||
return "$OVERALL_STATUS"
|
||||
}
|
||||
|
||||
if [[ -n "$CHECK_SNAPSHOT" ]]; then
|
||||
check_snapshot_shape "$(cat "$CHECK_SNAPSHOT")"
|
||||
exit $?
|
||||
fi
|
||||
if [[ ${#COMPARE_DATADIR[@]} -gt 0 ]]; then
|
||||
compare_datadir "${COMPARE_DATADIR[0]}" "${COMPARE_DATADIR[1]}" self-test
|
||||
[[ -z "$CURRENT_FAILURE" ]]
|
||||
exit $?
|
||||
fi
|
||||
|
||||
main
|
||||
184
scripts/release_lifecycle_rehearsal_versions.py
Executable file
184
scripts/release_lifecycle_rehearsal_versions.py
Executable file
|
|
@ -0,0 +1,184 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Choose the published release pair for the release lifecycle rehearsal.
|
||||
|
||||
The rehearsal installs FROM, upgrades to TO and rolls back to FROM. Both tags
|
||||
must name published (non-draft) Pulse server releases and TO must be strictly
|
||||
newer than FROM under SemVer precedence, so a shadow run can never report a
|
||||
same-version "upgrade" as proof.
|
||||
|
||||
Defaults:
|
||||
FROM = the release GitHub advertises as latest (must be stable).
|
||||
TO = the newest published release or prerelease newer than FROM.
|
||||
When FROM was defaulted and nothing newer exists yet (the common case right
|
||||
after a stable release), the pair becomes previous stable -> latest stable,
|
||||
which is the upgrade users are taking at that moment.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import functools
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
TAG_RE = re.compile(
|
||||
r"^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)"
|
||||
r"(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?$"
|
||||
)
|
||||
|
||||
|
||||
class ResolutionError(ValueError):
|
||||
"""The requested or default release pair cannot be rehearsed."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Selection:
|
||||
from_tag: str
|
||||
to_tag: str
|
||||
reason: str
|
||||
|
||||
|
||||
def parse_tag(tag: str) -> tuple[tuple[int, int, int], tuple[str, ...]]:
|
||||
match = TAG_RE.fullmatch(tag)
|
||||
if not match:
|
||||
raise ResolutionError(f"not a Pulse release tag: {tag!r}")
|
||||
core = (int(match.group(1)), int(match.group(2)), int(match.group(3)))
|
||||
pre = tuple(match.group(4).split(".")) if match.group(4) else ()
|
||||
return core, pre
|
||||
|
||||
|
||||
def is_stable(tag: str) -> bool:
|
||||
return not parse_tag(tag)[1]
|
||||
|
||||
|
||||
def _compare_identifiers(left: tuple[str, ...], right: tuple[str, ...]) -> int:
|
||||
# SemVer 2.0.0 section 11: a release outranks any prerelease of the same
|
||||
# core; numeric identifiers compare numerically and rank below
|
||||
# alphanumeric ones; a longer identifier list wins when all shared
|
||||
# identifiers are equal.
|
||||
if not left and not right:
|
||||
return 0
|
||||
if not left:
|
||||
return 1
|
||||
if not right:
|
||||
return -1
|
||||
for a, b in zip(left, right):
|
||||
if a == b:
|
||||
continue
|
||||
a_num, b_num = a.isdigit(), b.isdigit()
|
||||
if a_num and b_num:
|
||||
return -1 if int(a) < int(b) else 1
|
||||
if a_num != b_num:
|
||||
return -1 if a_num else 1
|
||||
return -1 if a < b else 1
|
||||
if len(left) == len(right):
|
||||
return 0
|
||||
return -1 if len(left) < len(right) else 1
|
||||
|
||||
|
||||
def compare_tags(left: str, right: str) -> int:
|
||||
left_core, left_pre = parse_tag(left)
|
||||
right_core, right_pre = parse_tag(right)
|
||||
if left_core != right_core:
|
||||
return -1 if left_core < right_core else 1
|
||||
return _compare_identifiers(left_pre, right_pre)
|
||||
|
||||
|
||||
def published_tags(releases: list[dict]) -> list[str]:
|
||||
tags: set[str] = set()
|
||||
for release in releases:
|
||||
if not isinstance(release, dict) or release.get("draft") is not False:
|
||||
continue
|
||||
tag = release.get("tag_name")
|
||||
if isinstance(tag, str) and TAG_RE.fullmatch(tag):
|
||||
tags.add(tag)
|
||||
return sorted(tags, key=functools.cmp_to_key(compare_tags))
|
||||
|
||||
|
||||
def resolve(
|
||||
releases: list[dict],
|
||||
latest_tag: str,
|
||||
requested_from: str = "",
|
||||
requested_to: str = "",
|
||||
) -> Selection:
|
||||
tags = published_tags(releases)
|
||||
published = set(tags)
|
||||
|
||||
for label, value in (("from_version", requested_from), ("to_version", requested_to)):
|
||||
if value:
|
||||
parse_tag(value)
|
||||
if value not in published:
|
||||
raise ResolutionError(f"{label} {value} is not a published Pulse release")
|
||||
|
||||
if requested_from:
|
||||
from_tag = requested_from
|
||||
else:
|
||||
if not latest_tag or latest_tag not in published or not is_stable(latest_tag):
|
||||
raise ResolutionError(
|
||||
f"latest advertised release {latest_tag!r} is not a published stable Pulse release"
|
||||
)
|
||||
from_tag = latest_tag
|
||||
|
||||
if requested_to:
|
||||
if compare_tags(requested_to, from_tag) <= 0:
|
||||
raise ResolutionError(
|
||||
f"to_version {requested_to} must be newer than from_version {from_tag}"
|
||||
)
|
||||
return Selection(from_tag, requested_to, "requested pair")
|
||||
|
||||
newer = [tag for tag in tags if compare_tags(tag, from_tag) > 0]
|
||||
if newer:
|
||||
reason = "requested from_version" if requested_from else "latest stable"
|
||||
return Selection(from_tag, newer[-1], f"{reason} -> newest published release")
|
||||
|
||||
if requested_from:
|
||||
raise ResolutionError(f"no published release is newer than from_version {from_tag}")
|
||||
|
||||
older_stable = [
|
||||
tag for tag in tags if is_stable(tag) and compare_tags(tag, from_tag) < 0
|
||||
]
|
||||
if not older_stable:
|
||||
raise ResolutionError(f"no stable release precedes latest stable {from_tag}")
|
||||
return Selection(
|
||||
older_stable[-1],
|
||||
from_tag,
|
||||
"nothing newer than latest stable is published; previous stable -> latest stable",
|
||||
)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument("--releases-json", required=True, type=Path,
|
||||
help="JSON array of GitHub release objects (all pages)")
|
||||
parser.add_argument("--latest-tag", default="",
|
||||
help="tag_name of GET /repos/{repo}/releases/latest")
|
||||
parser.add_argument("--from", dest="from_version", default="")
|
||||
parser.add_argument("--to", dest="to_version", default="")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
try:
|
||||
releases = json.loads(args.releases_json.read_text(encoding="utf-8"))
|
||||
if not isinstance(releases, list):
|
||||
raise ResolutionError("releases JSON must be an array")
|
||||
selection = resolve(
|
||||
releases,
|
||||
args.latest_tag.strip(),
|
||||
args.from_version.strip(),
|
||||
args.to_version.strip(),
|
||||
)
|
||||
except (ResolutionError, json.JSONDecodeError, OSError) as error:
|
||||
print(f"release lifecycle rehearsal: {error}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(f"from_tag={selection.from_tag}")
|
||||
print(f"to_tag={selection.to_tag}")
|
||||
print(f"reason={selection.reason}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
336
scripts/tests/test_release_lifecycle_rehearsal.py
Normal file
336
scripts/tests/test_release_lifecycle_rehearsal.py
Normal file
|
|
@ -0,0 +1,336 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Contract tests for the shadow release lifecycle rehearsal."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
RESOLVER_PATH = REPO_ROOT / "scripts" / "release_lifecycle_rehearsal_versions.py"
|
||||
HARNESS_PATH = REPO_ROOT / "scripts" / "release_lifecycle_rehearsal.sh"
|
||||
WORKFLOW_PATH = REPO_ROOT / ".github" / "workflows" / "release-lifecycle-rehearsal.yml"
|
||||
SMOKE_BODY_PATH = REPO_ROOT / ".github" / "workflows" / "install-sh-smoke-body.yml"
|
||||
|
||||
SPEC = importlib.util.spec_from_file_location("release_lifecycle_rehearsal_versions", RESOLVER_PATH)
|
||||
assert SPEC and SPEC.loader
|
||||
versions = importlib.util.module_from_spec(SPEC)
|
||||
sys.modules[SPEC.name] = versions
|
||||
SPEC.loader.exec_module(versions)
|
||||
|
||||
|
||||
def release(tag: str, *, draft: bool = False, prerelease: bool | None = None) -> dict:
|
||||
return {
|
||||
"tag_name": tag,
|
||||
"draft": draft,
|
||||
"prerelease": ("-" in tag) if prerelease is None else prerelease,
|
||||
}
|
||||
|
||||
|
||||
RELEASES = [
|
||||
release("v6.4.5"),
|
||||
release("helm-chart-6.4.5"),
|
||||
release("v6.4.5-rc.5"),
|
||||
release("v6.4.5-rc.10"),
|
||||
release("v6.4.5-beta.3"),
|
||||
release("v6.4.1"),
|
||||
release("v6.4.0"),
|
||||
release("v6.4.0-rc.9"),
|
||||
release("v6.4.6-rc.1", draft=True),
|
||||
]
|
||||
|
||||
|
||||
class SemverOrderingTest(unittest.TestCase):
|
||||
def test_release_outranks_its_prereleases(self) -> None:
|
||||
self.assertGreater(versions.compare_tags("v6.4.5", "v6.4.5-rc.10"), 0)
|
||||
|
||||
def test_numeric_identifiers_compare_numerically(self) -> None:
|
||||
self.assertGreater(versions.compare_tags("v6.4.5-rc.10", "v6.4.5-rc.5"), 0)
|
||||
|
||||
def test_alphanumeric_identifiers_compare_lexically(self) -> None:
|
||||
self.assertGreater(versions.compare_tags("v6.4.5-rc.1", "v6.4.5-beta.3"), 0)
|
||||
|
||||
def test_rejects_non_release_tags(self) -> None:
|
||||
for tag in ("6.4.5", "helm-chart-6.4.5", "v6.4", "v6.4.5;id", "v06.4.5"):
|
||||
with self.subTest(tag=tag), self.assertRaises(versions.ResolutionError):
|
||||
versions.parse_tag(tag)
|
||||
|
||||
|
||||
class ResolvePairTest(unittest.TestCase):
|
||||
def test_defaults_to_newer_prerelease_after_latest_stable(self) -> None:
|
||||
releases = RELEASES + [release("v6.4.6-rc.2"), release("v6.4.6-beta.1")]
|
||||
selection = versions.resolve(releases, "v6.4.5")
|
||||
self.assertEqual((selection.from_tag, selection.to_tag), ("v6.4.5", "v6.4.6-rc.2"))
|
||||
|
||||
def test_defaults_fall_back_to_previous_stable_when_nothing_is_newer(self) -> None:
|
||||
selection = versions.resolve(RELEASES, "v6.4.5")
|
||||
self.assertEqual((selection.from_tag, selection.to_tag), ("v6.4.1", "v6.4.5"))
|
||||
self.assertIn("previous stable", selection.reason)
|
||||
|
||||
def test_draft_releases_are_never_targets(self) -> None:
|
||||
selection = versions.resolve(RELEASES, "v6.4.5")
|
||||
self.assertNotEqual(selection.to_tag, "v6.4.6-rc.1")
|
||||
with self.assertRaises(versions.ResolutionError):
|
||||
versions.resolve(RELEASES, "v6.4.5", "v6.4.5", "v6.4.6-rc.1")
|
||||
|
||||
def test_explicit_from_selects_newest_newer_release(self) -> None:
|
||||
selection = versions.resolve(RELEASES, "v6.4.5", "v6.4.0")
|
||||
self.assertEqual((selection.from_tag, selection.to_tag), ("v6.4.0", "v6.4.5"))
|
||||
|
||||
def test_explicit_from_without_newer_release_fails_instead_of_passing(self) -> None:
|
||||
with self.assertRaises(versions.ResolutionError):
|
||||
versions.resolve(RELEASES, "v6.4.5", "v6.4.5")
|
||||
|
||||
def test_explicit_pair_must_move_forward(self) -> None:
|
||||
for from_tag, to_tag in (("v6.4.5", "v6.4.1"), ("v6.4.5", "v6.4.5")):
|
||||
with self.subTest(pair=(from_tag, to_tag)), self.assertRaises(versions.ResolutionError):
|
||||
versions.resolve(RELEASES, "v6.4.5", from_tag, to_tag)
|
||||
|
||||
def test_unpublished_or_malformed_requests_fail(self) -> None:
|
||||
for from_tag, to_tag in (("v6.3.0", ""), ("v6.4.1", "v6.9.9"), ("v6.4.1 ; id", "")):
|
||||
with self.subTest(pair=(from_tag, to_tag)), self.assertRaises(versions.ResolutionError):
|
||||
versions.resolve(RELEASES, "v6.4.5", from_tag, to_tag)
|
||||
|
||||
def test_latest_must_be_published_stable(self) -> None:
|
||||
for latest in ("", "v6.4.5-rc.5", "v9.9.9"):
|
||||
with self.subTest(latest=latest), self.assertRaises(versions.ResolutionError):
|
||||
versions.resolve(RELEASES, latest)
|
||||
|
||||
def test_cli_emits_key_value_pair(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = Path(tmp) / "releases.json"
|
||||
path.write_text(json.dumps(RELEASES), encoding="utf-8")
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(RESOLVER_PATH), "--releases-json", str(path),
|
||||
"--latest-tag", "v6.4.5", "--from", "v6.4.0", "--to", "v6.4.1"],
|
||||
capture_output=True, text=True, check=False,
|
||||
)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertIn("from_tag=v6.4.0\n", result.stdout)
|
||||
self.assertIn("to_tag=v6.4.1\n", result.stdout)
|
||||
|
||||
|
||||
class HarnessContractTest(unittest.TestCase):
|
||||
harness = HARNESS_PATH.read_text(encoding="utf-8")
|
||||
|
||||
def test_upgrade_and_rollback_use_the_installed_update_helper(self) -> None:
|
||||
self.assertIn("cexec '/bin/update --version \"$TARGET\"'", self.harness)
|
||||
self.assertIn('run_updater "$TO_TAG" upgrade', self.harness)
|
||||
self.assertIn('run_updater "$FROM_TAG" rollback', self.harness)
|
||||
self.assertIn('grep -q "^# Pulse update command" /bin/update', self.harness)
|
||||
# The updater invocation carries no flags that would bypass the
|
||||
# behavior users get (archives, preflight skips, forced auto-update).
|
||||
updater_lines = [line for line in self.harness.splitlines() if "/bin/update --" in line]
|
||||
for line in updater_lines:
|
||||
for flag in ("--archive", "--skip-upgrade-preflight", "--disable-auto-updates"):
|
||||
self.assertNotIn(flag, line)
|
||||
|
||||
def test_initial_install_uses_signed_published_installer(self) -> None:
|
||||
for needle in (
|
||||
"releases/download/${tag}",
|
||||
"grep -oE 'ssh-ed25519 [A-Za-z0-9+/=]+ pulse-installer'",
|
||||
"-I pulse-installer",
|
||||
"-n pulse-install",
|
||||
"bash /rehearsal/install.sh --version \"$FROM_TAG\"",
|
||||
):
|
||||
self.assertIn(needle, self.harness)
|
||||
|
||||
def test_uses_the_same_pinned_systemd_image_as_install_smoke(self) -> None:
|
||||
digest = re.search(r"jrei/systemd-debian:12@sha256:[0-9a-f]{64}", self.harness)
|
||||
self.assertIsNotNone(digest)
|
||||
self.assertIn(digest.group(0), SMOKE_BODY_PATH.read_text(encoding="utf-8"))
|
||||
|
||||
def test_every_post_change_phase_checks_identity_health_settings_and_data(self) -> None:
|
||||
for phase in ("phase_upgrade", "phase_rollback"):
|
||||
body = self.harness.split(f"{phase}() {{", 1)[1].split("\n}\n", 1)[0]
|
||||
for check in ("assert_runtime", "check_settings", "check_datadir", "record_phase"):
|
||||
with self.subTest(phase=phase, check=check):
|
||||
self.assertIn(check, body)
|
||||
|
||||
def test_settings_are_checked_against_fixed_expectations_not_only_baseline(self) -> None:
|
||||
body = self.harness.split("check_settings() {", 1)[1].split("\n}\n", 1)[0]
|
||||
self.assertIn("check_snapshot_shape", body)
|
||||
self.assertIn("settings.baseline.json", body)
|
||||
|
||||
def test_harness_passes_shellcheck_syntax(self) -> None:
|
||||
result = subprocess.run(["bash", "-n", str(HARNESS_PATH)], capture_output=True, text=True)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
|
||||
def test_rejects_unsafe_tags_before_touching_a_container(self) -> None:
|
||||
for args in (["--from", "v6.4.1", "--to", "v6.4.1"],
|
||||
["--from", "v6.4.1;id", "--to", "v6.4.5"],
|
||||
["--from", "", "--to", "v6.4.5"]):
|
||||
with self.subTest(args=args):
|
||||
result = subprocess.run(
|
||||
["bash", str(HARNESS_PATH), *args],
|
||||
capture_output=True, text=True,
|
||||
env={"PATH": "/usr/bin:/bin", "PULSE_REHEARSAL_ENGINE": "none-such"},
|
||||
)
|
||||
self.assertEqual(result.returncode, 2, result.stdout + result.stderr)
|
||||
|
||||
|
||||
GOOD_SNAPSHOT = {
|
||||
"auth": {"api_token": "200", "password": "200", "requiresAuth": True, "unauthenticated": "401"},
|
||||
"node": {
|
||||
"hasPassword": False,
|
||||
"hasToken": True,
|
||||
"host": "https://192.168.77.10:8006",
|
||||
"name": "test-lifecycle-rehearsal",
|
||||
"tokenName": "root@pam!rehearsal",
|
||||
"type": "pve",
|
||||
"verifySSL": False,
|
||||
},
|
||||
"webhook": {
|
||||
"enabled": False,
|
||||
"header_keys": ["X-Rehearsal-Secret"],
|
||||
"id": "webhook-123",
|
||||
"method": "POST",
|
||||
"name": "lifecycle-rehearsal-webhook",
|
||||
"service": "generic",
|
||||
"url": "https://example.com/pulse-lifecycle-rehearsal",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def mutated(path: tuple[str, ...], value: object) -> dict:
|
||||
snapshot = json.loads(json.dumps(GOOD_SNAPSHOT))
|
||||
target = snapshot
|
||||
for key in path[:-1]:
|
||||
target = target[key]
|
||||
target[path[-1]] = value
|
||||
return snapshot
|
||||
|
||||
|
||||
class SnapshotExpectationTest(unittest.TestCase):
|
||||
"""Lost or altered settings must fail even when the baseline also lost them."""
|
||||
|
||||
def check(self, snapshot: object) -> subprocess.CompletedProcess:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = Path(tmp) / "snapshot.json"
|
||||
path.write_text(json.dumps(snapshot) if not isinstance(snapshot, str) else snapshot,
|
||||
encoding="utf-8")
|
||||
return subprocess.run(
|
||||
["bash", str(HARNESS_PATH), "--check-snapshot", str(path)],
|
||||
capture_output=True, text=True, check=False,
|
||||
env={"PATH": "/usr/bin:/bin:/usr/local/bin:/opt/homebrew/bin"},
|
||||
)
|
||||
|
||||
def test_intact_snapshot_passes(self) -> None:
|
||||
result = self.check(GOOD_SNAPSHOT)
|
||||
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
||||
|
||||
def test_lost_or_changed_settings_fail(self) -> None:
|
||||
cases = {
|
||||
"node token secret lost": (("node", "hasToken"), False),
|
||||
"node token name reset": (("node", "tokenName"), ""),
|
||||
"node TLS verification flipped": (("node", "verifySSL"), True),
|
||||
"node host rewritten": (("node", "host"), "https://192.168.77.11:8006"),
|
||||
"node missing": (("node",), None),
|
||||
"webhook missing": (("webhook",), None),
|
||||
"webhook header lost": (("webhook", "header_keys"), []),
|
||||
"webhook re-enabled": (("webhook", "enabled"), True),
|
||||
"webhook URL changed": (("webhook", "url"), "https://example.org/"),
|
||||
"auth no longer required": (("auth", "requiresAuth"), False),
|
||||
"anonymous access allowed": (("auth", "unauthenticated"), "200"),
|
||||
"API token rejected": (("auth", "api_token"), "401"),
|
||||
"password rejected": (("auth", "password"), "401"),
|
||||
}
|
||||
for label, (path, value) in cases.items():
|
||||
with self.subTest(case=label):
|
||||
result = self.check(mutated(path, value))
|
||||
self.assertEqual(result.returncode, 1, result.stdout + result.stderr)
|
||||
self.assertIn("::error::settings:", result.stdout)
|
||||
|
||||
def test_unparseable_snapshot_fails(self) -> None:
|
||||
self.assertEqual(self.check("not json").returncode, 1)
|
||||
|
||||
|
||||
BASELINE_MANIFEST = {
|
||||
".encryption.key": "a" * 64,
|
||||
".env": "b" * 64,
|
||||
"metrics.db": "c" * 64,
|
||||
"nodes.enc": "d" * 64,
|
||||
"system.json": "e" * 64,
|
||||
"webhooks.enc": "f" * 64,
|
||||
}
|
||||
|
||||
|
||||
class DataDirComparisonTest(unittest.TestCase):
|
||||
def compare(self, current: dict[str, str]) -> subprocess.CompletedProcess:
|
||||
def manifest(entries: dict[str, str]) -> str:
|
||||
return "".join(f"{path}\t{digest}\n" for path, digest in sorted(entries.items()))
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
baseline_path = Path(tmp) / "baseline.tsv"
|
||||
current_path = Path(tmp) / "current.tsv"
|
||||
baseline_path.write_text(manifest(BASELINE_MANIFEST), encoding="utf-8")
|
||||
current_path.write_text(manifest(current), encoding="utf-8")
|
||||
return subprocess.run(
|
||||
["bash", str(HARNESS_PATH), "--compare-datadir", str(baseline_path), str(current_path)],
|
||||
capture_output=True, text=True, check=False,
|
||||
env={"PATH": "/usr/bin:/bin:/usr/local/bin:/opt/homebrew/bin"},
|
||||
)
|
||||
|
||||
def test_rewritten_plain_files_and_new_files_pass(self) -> None:
|
||||
current = dict(BASELINE_MANIFEST, **{"system.json": "1" * 64, "metrics.db": "2" * 64,
|
||||
"new-store.db": "3" * 64})
|
||||
result = self.compare(current)
|
||||
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
||||
|
||||
def test_missing_files_and_rewritten_secret_stores_fail(self) -> None:
|
||||
cases = {
|
||||
"file deleted": {k: v for k, v in BASELINE_MANIFEST.items() if k != "metrics.db"},
|
||||
"encryption key replaced": dict(BASELINE_MANIFEST, **{".encryption.key": "0" * 64}),
|
||||
"node secrets rewritten": dict(BASELINE_MANIFEST, **{"nodes.enc": "0" * 64}),
|
||||
"webhook secrets rewritten": dict(BASELINE_MANIFEST, **{"webhooks.enc": "0" * 64}),
|
||||
"data dir emptied": {},
|
||||
}
|
||||
for label, current in cases.items():
|
||||
with self.subTest(case=label):
|
||||
result = self.compare(current)
|
||||
self.assertEqual(result.returncode, 1, result.stdout + result.stderr)
|
||||
self.assertIn("::error::", result.stdout)
|
||||
|
||||
|
||||
class WorkflowContractTest(unittest.TestCase):
|
||||
workflow = WORKFLOW_PATH.read_text(encoding="utf-8")
|
||||
|
||||
def test_triggers_are_dispatch_and_daily_schedule_only(self) -> None:
|
||||
self.assertIn("workflow_dispatch:", self.workflow)
|
||||
self.assertIn("from_version:", self.workflow)
|
||||
self.assertIn("to_version:", self.workflow)
|
||||
self.assertRegex(self.workflow, r"(?m)^ schedule:\n # [^\n]*\n(?: #[^\n]*\n)* - cron: '23 13 \* \* \*'$")
|
||||
for trigger in ("pull_request", "push:", "workflow_run", "workflow_call", "release:"):
|
||||
self.assertNotIn(trigger, self.workflow)
|
||||
|
||||
def test_read_only_hosted_and_report_only(self) -> None:
|
||||
self.assertRegex(self.workflow, r"(?m)^permissions:\n contents: read$")
|
||||
self.assertNotIn(": write", self.workflow)
|
||||
self.assertNotIn("secrets.", self.workflow)
|
||||
self.assertNotIn("self-hosted", self.workflow)
|
||||
self.assertIn("runs-on: ubuntu-24.04", self.workflow)
|
||||
self.assertIn("persist-credentials: false", self.workflow)
|
||||
self.assertNotIn("upload-artifact", self.workflow)
|
||||
|
||||
def test_dispatch_inputs_reach_shell_only_through_env(self) -> None:
|
||||
for line in self.workflow.splitlines():
|
||||
if "${{" in line and ("inputs." in line or "steps." in line):
|
||||
self.assertRegex(line.strip(), r"^[A-Z_]+: \$\{\{ [a-z_.]+ \}\}$")
|
||||
|
||||
def test_no_release_workflow_depends_on_the_rehearsal(self) -> None:
|
||||
for path in (REPO_ROOT / ".github" / "workflows").glob("*.yml"):
|
||||
if path == WORKFLOW_PATH:
|
||||
continue
|
||||
with self.subTest(workflow=path.name):
|
||||
self.assertNotIn("release-lifecycle-rehearsal", path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue