From cd6b250ae6c4d800fd01c0240760677e08ecc20a Mon Sep 17 00:00:00 2001 From: rcourtman Date: Sat, 4 Jul 2026 22:28:01 +0100 Subject: [PATCH] Fix demo verification and agent update recovery Refs #1515 - restore demo runtime env and verify mock fixtures even when the target version is already installed - require recovered agent update state to include both URL and token before reporting success --- .github/workflows/update-demo-server.yml | 99 ++++++++++-- .../v6/internal/subsystems/agent-lifecycle.md | 16 +- .../subsystems/deployment-installability.md | 16 ++ scripts/install.sh | 12 +- .../installtests/build_release_assets_test.go | 6 + scripts/installtests/install_sh_test.go | 142 ++++++++++++++++++ .../release_promotion_policy_test.py | 7 + 7 files changed, 279 insertions(+), 19 deletions(-) diff --git a/.github/workflows/update-demo-server.yml b/.github/workflows/update-demo-server.yml index a57cc48fd..f68beb92b 100644 --- a/.github/workflows/update-demo-server.yml +++ b/.github/workflows/update-demo-server.yml @@ -436,8 +436,70 @@ jobs: ) ssh -i ~/.ssh/id_ed25519 "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" "bash -s -- $(printf '%q ' "$TAG" "$SERVICE_NAME")" <<<"$REMOTE_SCRIPT" - - name: Verify update - if: steps.current.outputs.skip_current != 'true' + - name: Restore demo runtime configuration + env: + DEMO_SERVER_HOST: ${{ secrets.DEMO_SERVER_HOST }} + DEMO_SERVER_USER: ${{ secrets.DEMO_SERVER_USER }} + run: | + set -euo pipefail + SERVICE_NAME="${{ steps.config.outputs.service_name }}" + REMOTE_SCRIPT=$(cat <<'EOF' + set -euo pipefail + SERVICE_NAME="${1:-pulse}" + DEMO_LOCAL_BASE_URL="$2" + CONFIG_DIR="/etc/${SERVICE_NAME:-pulse}" + ENV_FILE="$CONFIG_DIR/.env" + + set_env_value() { + local key="$1" + local value="$2" + sudo mkdir -p "$CONFIG_DIR" + if sudo test -f "$ENV_FILE"; then + if sudo grep -Eq "^[[:space:]]*${key}=" "$ENV_FILE"; then + sudo sed -i "s|^[[:space:]]*${key}=.*|${key}=${value}|" "$ENV_FILE" + else + printf '\n%s=%s\n' "$key" "$value" | sudo tee -a "$ENV_FILE" >/dev/null + fi + else + printf '# Pulse demo runtime configuration\n%s=%s\n' "$key" "$value" | sudo tee "$ENV_FILE" >/dev/null + fi + } + + set_env_value DEMO_MODE true + set_env_value PULSE_MOCK_MODE true + set_env_value PULSE_MOCK_NODES 5 + set_env_value PULSE_MOCK_VMS_PER_NODE 6 + set_env_value PULSE_MOCK_LXCS_PER_NODE 8 + set_env_value PULSE_MOCK_DOCKER_HOSTS 5 + set_env_value PULSE_MOCK_DOCKER_CONTAINERS 14 + set_env_value PULSE_MOCK_GENERIC_HOSTS 4 + set_env_value PULSE_MOCK_K8S_CLUSTERS 3 + set_env_value PULSE_MOCK_K8S_NODES 5 + set_env_value PULSE_MOCK_K8S_PODS 40 + set_env_value PULSE_MOCK_K8S_DEPLOYMENTS 14 + set_env_value PULSE_MOCK_RANDOM_METRICS true + set_env_value PULSE_MOCK_STOPPED_PERCENT 6 + + sudo chown pulse:pulse "$ENV_FILE" || true + sudo chmod 600 "$ENV_FILE" + sudo systemctl daemon-reload + sudo systemctl restart "$SERVICE_NAME" + + for attempt in $(seq 1 30); do + if curl -fsS "${DEMO_LOCAL_BASE_URL}/api/health" >/dev/null; then + echo "Demo service restarted with governed demo runtime configuration." + exit 0 + fi + sleep 2 + done + + echo "::error::Demo service did not become healthy after restoring demo runtime configuration." + exit 1 + EOF + ) + ssh -i ~/.ssh/id_ed25519 "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" "bash -s -- $(printf '%q ' "$SERVICE_NAME" "$DEMO_LOCAL_BASE_URL")" <<<"$REMOTE_SCRIPT" + + - name: Verify demo runtime env: DEMO_SERVER_HOST: ${{ secrets.DEMO_SERVER_HOST }} DEMO_SERVER_USER: ${{ secrets.DEMO_SERVER_USER }} @@ -459,26 +521,41 @@ jobs: AUTH_USER="${DEMO_AUTH_USER:-demo}" AUTH_PASS="${DEMO_AUTH_PASS:-demo}" - RESOURCES=$(ssh -i ~/.ssh/id_ed25519 "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" \ + ssh -i ~/.ssh/id_ed25519 "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" \ 'AUTH_USER='"$(printf '%q' "$AUTH_USER")"' AUTH_PASS='"$(printf '%q' "$AUTH_PASS")"' DEMO_LOCAL_BASE_URL='"$(printf '%q' "$DEMO_LOCAL_BASE_URL")"' bash -s' <<'EOF' set -euo pipefail - COOKIE_FILE="$HOME/.demo-cookies.txt" + COOKIE_FILE="$(mktemp)" + trap 'rm -f "$COOKIE_FILE"' EXIT curl -fsS -c "$COOKIE_FILE" "${DEMO_LOCAL_BASE_URL}/api/login" -X POST \ -H "Content-Type: application/json" \ -d "{\"username\":\"${AUTH_USER}\",\"password\":\"${AUTH_PASS}\"}" > /dev/null - curl -fsS -b "$COOKIE_FILE" "${DEMO_LOCAL_BASE_URL}/api/state" | jq -r '.resources | if type == "array" then length else 0 end' - rm -f "$COOKIE_FILE" - EOF - ) + curl -fsS -b "$COOKIE_FILE" "${DEMO_LOCAL_BASE_URL}/api/license/runtime-capabilities" >/dev/null + MOCK_ENABLED="false" + RESOURCES="0" + for attempt in $(seq 1 30); do + MOCK_ENABLED=$(curl -fsS -b "$COOKIE_FILE" "${DEMO_LOCAL_BASE_URL}/api/system/mock-mode" | jq -r '.enabled // false') + RESOURCES=$(curl -fsS -b "$COOKIE_FILE" "${DEMO_LOCAL_BASE_URL}/api/state" | jq -r '.resources | if type == "array" then length else 0 end') + if [ "$MOCK_ENABLED" = "true" ] && [ "$RESOURCES" -ge 1 ]; then + break + fi + sleep 2 + done + + echo "Mock mode enabled: $MOCK_ENABLED" echo "Mock resources detected: $RESOURCES" - if [ "$RESOURCES" -ge 1 ]; then - echo "Demo server successfully updated and verified." - else + if [ "$MOCK_ENABLED" != "true" ]; then + echo "::error::Demo server mock mode did not enable after entitlement sync" + exit 1 + fi + if [ "$RESOURCES" -lt 1 ]; then echo "::error::Demo server updated but canonical mock resources are missing" exit 1 fi + EOF + + echo "Demo server successfully updated and verified." - name: Verify frontend parity env: diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index b52191ba4..3c05d660d 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -255,9 +255,13 @@ that Pulse already sees, the copied stale-agent update command must use `scripts/install.sh --update` and recover URL, token, identity, custom CA, and insecure transport from installer-owned saved state instead of asking the operator to mint a fresh install token or exposing agent IDs in the copied -command. Windows stale-agent update commands remain on the existing token-gated -install transport until the Windows installer owns an equivalent saved-state -update mode. +command. If saved state is missing and the Unix shell installer falls back to +legacy running-process or service-unit recovery, recovered connection state is +usable only when both the control-plane URL and token are present. A URL-only +process or unit may seed later recovery attempts, but it must not be logged or +treated as recovered update state. Windows stale-agent update commands remain +on the existing token-gated install transport until the Windows installer owns +an equivalent saved-state update mode. Agent Fleet Doctor diagnostics extend that same read-only lifecycle triage surface: `GET /api/agents/diagnostics` may explain stale versions, missing reports, profile deployment drift, expected Docker/Kubernetes telemetry gaps, @@ -3786,7 +3790,11 @@ When persisted state is absent or partial during update, legacy running-process or service-unit recovery is a fallback into that same lifecycle continuity model, not a separate source of truth: it may only seed the installer-owned state for the upgrade and must not keep raw token arguments in the installed v6 -service command. +service command. It may report recovery success only after URL and token are +both available, even when those values were assembled from different legacy +sources; partial URL-only recovery must fall through to the explicit +missing-state error so operators are not told the update state was recovered +when the installer still cannot reconnect. That same lifecycle ownership must cover service control too: the installer may still choose different platform adapters, but stop/restart semantics for the managed agent must route through shared installer helpers instead of being diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index 7dd8b3d71..e6cbf5fb8 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -1062,6 +1062,15 @@ runtime state contract. `.github/workflows/update-demo-server.yml` must verify mock-mode readiness through the unified `/api/state` `resources[]` collection, not legacy `nodes`, because v6 intentionally strips per-type arrays from the state payload. +That same stable demo-update boundary must also restore the canonical demo +runtime `.env` before verification on every run, including when the service is +already on the requested version and the binary update is skipped. The workflow +must set `DEMO_MODE=true`, converge the governed `PULSE_MOCK_*` fixture +defaults in `/etc//.env`, restart the selected service, force the +release-build demo-fixture entitlement sync through authenticated +`/api/license/runtime-capabilities`, and then prove both +`/api/system/mock-mode.enabled` and `/api/state.resources[]` converge. A +passing `/api/version` or `/api/health` response alone is not demo readiness. That same operator-proof boundary also now owns the canonical hosted staging smoke entrypoint. `scripts/run_hosted_staging_smoke.sh` must stay as the repo-tracked operator command that composes the hosted signup/billing eval pack @@ -1875,6 +1884,13 @@ writer/reader path: `scripts/install.sh` may not keep a heredoc writer plus a second inline field parser for the same `connection.env` contract, because offline uninstall must consume the same persisted install-state artifact the installer wrote instead of reconstructing it ad hoc. +That same shell-agent update recovery path must fail closed on partial +legacy process or service-unit state: a recovered URL without a recovered token +is not usable connection state and must not be logged or treated as recovered. +Fallback recovery may merge URL and token across process args, environment, and +systemd unit data, but it may report success only once both values are present; +otherwise the update command must fall through to the explicit missing-state +error instead of implying recovery succeeded. That same installer ownership now also applies to service lifecycle control: upgrade, reinstall, and platform-specific start/restart flows may not each carry their own stop/start command sequence for the same agent runtime. diff --git a/scripts/install.sh b/scripts/install.sh index dbc765bdd..beaede81c 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -1591,6 +1591,10 @@ apply_recovered_agent_arg_value() { esac } +recovered_connection_state_ready() { + [[ -n "$PULSE_URL" && -n "$PULSE_TOKEN" ]] +} + recover_connection_state_from_arg_stream() { local arg="" local pending_key="" @@ -1688,7 +1692,7 @@ recover_connection_state_from_arg_stream() { esac done - [[ "$RECOVERED_AGENT_ARG_STATE" == "true" && -n "$PULSE_URL" ]] + [[ "$RECOVERED_AGENT_ARG_STATE" == "true" ]] && recovered_connection_state_ready } recover_connection_state_from_env_stream() { @@ -1738,7 +1742,7 @@ recover_connection_state_from_env_stream() { esac done - [[ "$RECOVERED_AGENT_ENV_STATE" == "true" && -n "$PULSE_URL" ]] + [[ "$RECOVERED_AGENT_ENV_STATE" == "true" ]] && recovered_connection_state_ready } collect_running_agent_pids() { @@ -1782,7 +1786,7 @@ recover_connection_state_from_running_agent() { recovered="true" fi fi - if [[ "$recovered" == "true" && -n "$PULSE_URL" ]]; then + if [[ "$recovered" == "true" ]] && recovered_connection_state_ready; then return 0 fi done < <(collect_running_agent_pids) @@ -1820,7 +1824,7 @@ recover_connection_state_from_systemd_unit() { ;; esac done < "$candidate" - if [[ "$recovered" == "true" && -n "$PULSE_URL" ]]; then + if [[ "$recovered" == "true" ]] && recovered_connection_state_ready; then return 0 fi done diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.go index 8e896c518..c451570fb 100644 --- a/scripts/installtests/build_release_assets_test.go +++ b/scripts/installtests/build_release_assets_test.go @@ -807,6 +807,12 @@ func TestUpdateDemoWorkflowUsesGovernedNetworkPath(t *testing.T) { `-name "metrics.db"`, `Removing demo volatile store: %s`, `Demo host does not have enough free space to back up $CONFIG_DIR before install.`, + `Restore demo runtime configuration`, + `set_env_value DEMO_MODE true`, + `set_env_value PULSE_MOCK_MODE true`, + `/api/license/runtime-capabilities`, + `Mock mode enabled`, + `Demo server mock mode did not enable after entitlement sync`, `Verify public browser smoke`, `./scripts/run_demo_public_browser_smoke.sh`, } diff --git a/scripts/installtests/install_sh_test.go b/scripts/installtests/install_sh_test.go index d55ae3ee3..5b614dc9b 100644 --- a/scripts/installtests/install_sh_test.go +++ b/scripts/installtests/install_sh_test.go @@ -645,6 +645,9 @@ func TestInstallSHSupportsSavedStateUpdateMode(t *testing.T) { `recover_connection_state_from_systemd_unit`, `recover_connection_state_from_arg_stream`, `recover_connection_state_from_env_stream`, + `recovered_connection_state_ready() {`, + `[[ "$RECOVERED_AGENT_ARG_STATE" == "true" ]] && recovered_connection_state_ready`, + `[[ "$RECOVERED_AGENT_ENV_STATE" == "true" ]] && recovered_connection_state_ready`, `recover_connection_state_from_existing_agent || true`, `if [[ -n "$PULSE_URL" && -n "$PULSE_TOKEN" ]]; then`, `recover_agent_id_from_state_file() {`, @@ -692,6 +695,7 @@ func TestInstallSHRecoversV5ProcessArgsForSavedStateUpdate(t *testing.T) { RUNTIME_TOKEN_FILE="/var/lib/pulse-agent/token" ` + extractInstallShellFunction(t, "strip_recovered_arg_quotes") + ` ` + extractInstallShellFunction(t, "apply_recovered_agent_arg_value") + ` +` + extractInstallShellFunction(t, "recovered_connection_state_ready") + ` ` + extractInstallShellFunction(t, "recover_connection_state_from_arg_stream") + ` ` + extractInstallShellFunction(t, "build_exec_arg_items") + ` ` + extractInstallShellFunction(t, "join_exec_arg_items") + ` @@ -743,6 +747,144 @@ ARGS } } +func TestInstallSHRejectsPartialRecoveredProcessConnectionState(t *testing.T) { + script := ` + PULSE_URL="" + PULSE_TOKEN="" + INTERVAL="30s" + INTERVAL_EXPLICIT="false" + ENABLE_HOST="true" + HOST_EXPLICIT="false" + ENABLE_DOCKER="" + DOCKER_EXPLICIT="false" + ENABLE_KUBERNETES="" + KUBERNETES_EXPLICIT="false" + KUBECONFIG_PATH="" + ENABLE_PROXMOX="" + PROXMOX_EXPLICIT="false" + PROXMOX_TYPE="" + INSECURE="false" + ENABLE_COMMANDS="false" + ENROLL="false" + HEALTH_ADDR="" + HEALTH_ADDR_SET="false" + AGENT_ID="" + HOSTNAME_OVERRIDE="" + STATE_DIR="/var/lib/pulse-agent" + CURL_CA_BUNDLE="" + KUBE_INCLUDE_ALL_PODS="false" + KUBE_INCLUDE_ALL_DEPLOYMENTS="false" + DISK_EXCLUDES=() +` + extractInstallShellFunction(t, "strip_recovered_arg_quotes") + ` +` + extractInstallShellFunction(t, "apply_recovered_agent_arg_value") + ` +` + extractInstallShellFunction(t, "recovered_connection_state_ready") + ` +` + extractInstallShellFunction(t, "recover_connection_state_from_arg_stream") + ` + if recover_connection_state_from_arg_stream <<'ARGS' +/usr/local/bin/pulse-agent +--url +http://192.168.2.96:7655 +--enable-host +--agent-id +agent-123 +ARGS + then + echo "UNEXPECTED_SUCCESS" + else + echo "EXPECTED_FAILURE" + fi + printf 'URL=%s\nTOKEN=%s\nAGENT_ID=%s\n' "$PULSE_URL" "$PULSE_TOKEN" "$AGENT_ID" + ` + + out, err := exec.Command("bash", "-c", script).CombinedOutput() + if err != nil { + t.Fatalf("bash: %v\n%s", err, out) + } + got := string(out) + if !strings.Contains(got, "EXPECTED_FAILURE") { + t.Fatalf("partial recovered state should fail closed:\n%s", got) + } + if strings.Contains(got, "UNEXPECTED_SUCCESS") { + t.Fatalf("partial recovered state reported success:\n%s", got) + } + if !strings.Contains(got, "URL=http://192.168.2.96:7655") || !strings.Contains(got, "AGENT_ID=agent-123") { + t.Fatalf("partial recovery should still preserve non-secret fields for later sources:\n%s", got) + } + if !strings.Contains(got, "TOKEN=") { + t.Fatalf("expected token to remain empty:\n%s", got) + } +} + +func TestInstallSHCombinesRecoveredProcessArgsAndEnvConnectionState(t *testing.T) { + script := ` + PULSE_URL="" + PULSE_TOKEN="" + INTERVAL="30s" + INTERVAL_EXPLICIT="false" + ENABLE_HOST="true" + HOST_EXPLICIT="false" + ENABLE_DOCKER="" + DOCKER_EXPLICIT="false" + ENABLE_KUBERNETES="" + KUBERNETES_EXPLICIT="false" + KUBECONFIG_PATH="" + ENABLE_PROXMOX="" + PROXMOX_EXPLICIT="false" + PROXMOX_TYPE="" + INSECURE="false" + ENABLE_COMMANDS="false" + ENROLL="false" + HEALTH_ADDR="" + HEALTH_ADDR_SET="false" + AGENT_ID="" + HOSTNAME_OVERRIDE="" + STATE_DIR="/var/lib/pulse-agent" + CURL_CA_BUNDLE="" + KUBE_INCLUDE_ALL_PODS="false" + KUBE_INCLUDE_ALL_DEPLOYMENTS="false" + DISK_EXCLUDES=() +` + extractInstallShellFunction(t, "strip_recovered_arg_quotes") + ` +` + extractInstallShellFunction(t, "apply_recovered_agent_arg_value") + ` +` + extractInstallShellFunction(t, "recovered_connection_state_ready") + ` +` + extractInstallShellFunction(t, "recover_connection_state_from_arg_stream") + ` +` + extractInstallShellFunction(t, "recover_connection_state_from_env_stream") + ` + if recover_connection_state_from_arg_stream <<'ARGS' +/usr/local/bin/pulse-agent +--url +http://192.168.2.96:7655 +ARGS + then + echo "ARGS_READY" + else + echo "ARGS_PARTIAL" + fi + if recover_connection_state_from_env_stream <<'ENV' +PULSE_TOKEN=deadbeef +ENV + then + echo "ENV_READY" + else + echo "ENV_PARTIAL" + fi + printf 'URL=%s\nTOKEN=%s\n' "$PULSE_URL" "$PULSE_TOKEN" + ` + + out, err := exec.Command("bash", "-c", script).CombinedOutput() + if err != nil { + t.Fatalf("bash: %v\n%s", err, out) + } + got := string(out) + for _, needle := range []string{ + "ARGS_PARTIAL", + "ENV_READY", + "URL=http://192.168.2.96:7655", + "TOKEN=deadbeef", + } { + if !strings.Contains(got, needle) { + t.Fatalf("combined recovery missing %q:\n%s", needle, got) + } + } +} + func TestInstallSHUsesCanonicalServiceLifecycleHelpers(t *testing.T) { content, err := os.ReadFile(repoFile("scripts", "install.sh")) if err != nil { diff --git a/scripts/release_control/release_promotion_policy_test.py b/scripts/release_control/release_promotion_policy_test.py index e78b125aa..7206000c1 100644 --- a/scripts/release_control/release_promotion_policy_test.py +++ b/scripts/release_control/release_promotion_policy_test.py @@ -735,6 +735,13 @@ class ReleasePromotionPolicyTest(unittest.TestCase): self.assertIn("DEMO_EXPECTED_HOSTNAME", demo) self.assertIn("Verify target host identity", demo) self.assertIn("Demo environment points at host $REMOTE_HOSTNAME but expected $DEMO_EXPECTED_HOSTNAME.", demo) + self.assertIn("Restore demo runtime configuration", demo) + self.assertIn("set_env_value DEMO_MODE true", demo) + self.assertIn("set_env_value PULSE_MOCK_MODE true", demo) + self.assertIn("Demo service restarted with governed demo runtime configuration.", demo) + self.assertIn("/api/license/runtime-capabilities", demo) + self.assertIn("Mock mode enabled", demo) + self.assertIn("Demo server mock mode did not enable after entitlement sync", demo) self.assertIn(".resources | if type == \"array\" then length else 0 end", demo) self.assertNotIn(".nodes | length", demo) self.assertIn("Mock resources detected", demo)