Fix demo verification and agent update recovery

Refs #1515 - restore demo runtime env and verify mock fixtures even when the target version is already installed - require recovered agent update state to include both URL and token before reporting success
2026-07-10 00:14:38 +00:00 · 2026-07-04 22:28:01 +01:00 · 2026-07-04 22:28:01 +01:00 · cd6b250ae6
commit cd6b250ae6
parent 6becdbf9db
7 changed files with 279 additions and 19 deletions
--- a/.github/workflows/update-demo-server.yml
+++ b/.github/workflows/update-demo-server.yml
@ -436,8 +436,70 @@ jobs:
          )
          ssh -i ~/.ssh/id_ed25519 "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" "bash -s -- $(printf '%q ' "$TAG" "$SERVICE_NAME")" <<<"$REMOTE_SCRIPT"

-      - name: Verify update
-        if: steps.current.outputs.skip_current != 'true'
+      - name: Restore demo runtime configuration
+        env:
+          DEMO_SERVER_HOST: ${{ secrets.DEMO_SERVER_HOST }}
+          DEMO_SERVER_USER: ${{ secrets.DEMO_SERVER_USER }}
+        run: |
+          set -euo pipefail
+          SERVICE_NAME="${{ steps.config.outputs.service_name }}"
+          REMOTE_SCRIPT=$(cat <<'EOF'
+          set -euo pipefail
+          SERVICE_NAME="${1:-pulse}"
+          DEMO_LOCAL_BASE_URL="$2"
+          CONFIG_DIR="/etc/${SERVICE_NAME:-pulse}"
+          ENV_FILE="$CONFIG_DIR/.env"
+
+          set_env_value() {
+            local key="$1"
+            local value="$2"
+            sudo mkdir -p "$CONFIG_DIR"
+            if sudo test -f "$ENV_FILE"; then
+              if sudo grep -Eq "^[[:space:]]*${key}=" "$ENV_FILE"; then
+                sudo sed -i "s|^[[:space:]]*${key}=.*|${key}=${value}|" "$ENV_FILE"
+              else
+                printf '\n%s=%s\n' "$key" "$value" | sudo tee -a "$ENV_FILE" >/dev/null
+              fi
+            else
+              printf '# Pulse demo runtime configuration\n%s=%s\n' "$key" "$value" | sudo tee "$ENV_FILE" >/dev/null
+            fi
+          }
+
+          set_env_value DEMO_MODE true
+          set_env_value PULSE_MOCK_MODE true
+          set_env_value PULSE_MOCK_NODES 5
+          set_env_value PULSE_MOCK_VMS_PER_NODE 6
+          set_env_value PULSE_MOCK_LXCS_PER_NODE 8
+          set_env_value PULSE_MOCK_DOCKER_HOSTS 5
+          set_env_value PULSE_MOCK_DOCKER_CONTAINERS 14
+          set_env_value PULSE_MOCK_GENERIC_HOSTS 4
+          set_env_value PULSE_MOCK_K8S_CLUSTERS 3
+          set_env_value PULSE_MOCK_K8S_NODES 5
+          set_env_value PULSE_MOCK_K8S_PODS 40
+          set_env_value PULSE_MOCK_K8S_DEPLOYMENTS 14
+          set_env_value PULSE_MOCK_RANDOM_METRICS true
+          set_env_value PULSE_MOCK_STOPPED_PERCENT 6
+
+          sudo chown pulse:pulse "$ENV_FILE" || true
+          sudo chmod 600 "$ENV_FILE"
+          sudo systemctl daemon-reload
+          sudo systemctl restart "$SERVICE_NAME"
+
+          for attempt in $(seq 1 30); do
+            if curl -fsS "${DEMO_LOCAL_BASE_URL}/api/health" >/dev/null; then
+              echo "Demo service restarted with governed demo runtime configuration."
+              exit 0
+            fi
+            sleep 2
+          done
+
+          echo "::error::Demo service did not become healthy after restoring demo runtime configuration."
+          exit 1
+          EOF
+          )
+          ssh -i ~/.ssh/id_ed25519 "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" "bash -s -- $(printf '%q ' "$SERVICE_NAME" "$DEMO_LOCAL_BASE_URL")" <<<"$REMOTE_SCRIPT"
+
+      - name: Verify demo runtime
        env:
          DEMO_SERVER_HOST: ${{ secrets.DEMO_SERVER_HOST }}
          DEMO_SERVER_USER: ${{ secrets.DEMO_SERVER_USER }}
@ -459,26 +521,41 @@ jobs:

          AUTH_USER="${DEMO_AUTH_USER:-demo}"
          AUTH_PASS="${DEMO_AUTH_PASS:-demo}"
-          RESOURCES=$(ssh -i ~/.ssh/id_ed25519 "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" \
+          ssh -i ~/.ssh/id_ed25519 "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" \
            'AUTH_USER='"$(printf '%q' "$AUTH_USER")"' AUTH_PASS='"$(printf '%q' "$AUTH_PASS")"' DEMO_LOCAL_BASE_URL='"$(printf '%q' "$DEMO_LOCAL_BASE_URL")"' bash -s' <<'EOF'
          set -euo pipefail
-          COOKIE_FILE="$HOME/.demo-cookies.txt"
+          COOKIE_FILE="$(mktemp)"
+          trap 'rm -f "$COOKIE_FILE"' EXIT
          curl -fsS -c "$COOKIE_FILE" "${DEMO_LOCAL_BASE_URL}/api/login" -X POST \
            -H "Content-Type: application/json" \
            -d "{\"username\":\"${AUTH_USER}\",\"password\":\"${AUTH_PASS}\"}" > /dev/null
-          curl -fsS -b "$COOKIE_FILE" "${DEMO_LOCAL_BASE_URL}/api/state" | jq -r '.resources | if type == "array" then length else 0 end'
-          rm -f "$COOKIE_FILE"
-          EOF
-          )
+          curl -fsS -b "$COOKIE_FILE" "${DEMO_LOCAL_BASE_URL}/api/license/runtime-capabilities" >/dev/null

+          MOCK_ENABLED="false"
+          RESOURCES="0"
+          for attempt in $(seq 1 30); do
+            MOCK_ENABLED=$(curl -fsS -b "$COOKIE_FILE" "${DEMO_LOCAL_BASE_URL}/api/system/mock-mode" | jq -r '.enabled // false')
+            RESOURCES=$(curl -fsS -b "$COOKIE_FILE" "${DEMO_LOCAL_BASE_URL}/api/state" | jq -r '.resources | if type == "array" then length else 0 end')
+            if [ "$MOCK_ENABLED" = "true" ] && [ "$RESOURCES" -ge 1 ]; then
+              break
+            fi
+            sleep 2
+          done
+
+          echo "Mock mode enabled: $MOCK_ENABLED"
          echo "Mock resources detected: $RESOURCES"

-          if [ "$RESOURCES" -ge 1 ]; then
-            echo "Demo server successfully updated and verified."
-          else
+          if [ "$MOCK_ENABLED" != "true" ]; then
+            echo "::error::Demo server mock mode did not enable after entitlement sync"
+            exit 1
+          fi
+          if [ "$RESOURCES" -lt 1 ]; then
            echo "::error::Demo server updated but canonical mock resources are missing"
            exit 1
          fi
+          EOF
+
+          echo "Demo server successfully updated and verified."

      - name: Verify frontend parity
        env:
--- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
+++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
@ -255,9 +255,13 @@ that Pulse already sees, the copied stale-agent update command must use
 `scripts/install.sh --update` and recover URL, token, identity, custom CA, and
 insecure transport from installer-owned saved state instead of asking the
 operator to mint a fresh install token or exposing agent IDs in the copied
-command. Windows stale-agent update commands remain on the existing token-gated
-install transport until the Windows installer owns an equivalent saved-state
-update mode.
+command. If saved state is missing and the Unix shell installer falls back to
+legacy running-process or service-unit recovery, recovered connection state is
+usable only when both the control-plane URL and token are present. A URL-only
+process or unit may seed later recovery attempts, but it must not be logged or
+treated as recovered update state. Windows stale-agent update commands remain
+on the existing token-gated install transport until the Windows installer owns
+an equivalent saved-state update mode.
 Agent Fleet Doctor diagnostics extend that same read-only lifecycle triage
 surface: `GET /api/agents/diagnostics` may explain stale versions, missing
 reports, profile deployment drift, expected Docker/Kubernetes telemetry gaps,
@ -3786,7 +3790,11 @@ When persisted state is absent or partial during update, legacy running-process
 or service-unit recovery is a fallback into that same lifecycle continuity
 model, not a separate source of truth: it may only seed the installer-owned
 state for the upgrade and must not keep raw token arguments in the installed v6
-service command.
+service command. It may report recovery success only after URL and token are
+both available, even when those values were assembled from different legacy
+sources; partial URL-only recovery must fall through to the explicit
+missing-state error so operators are not told the update state was recovered
+when the installer still cannot reconnect.
 That same lifecycle ownership must cover service control too: the installer may
 still choose different platform adapters, but stop/restart semantics for the
 managed agent must route through shared installer helpers instead of being
--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md
+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md
@ -1062,6 +1062,15 @@ runtime state contract. `.github/workflows/update-demo-server.yml` must verify
 mock-mode readiness through the unified `/api/state` `resources[]` collection,
 not legacy `nodes`, because v6 intentionally strips per-type arrays from the
 state payload.
+That same stable demo-update boundary must also restore the canonical demo
+runtime `.env` before verification on every run, including when the service is
+already on the requested version and the binary update is skipped. The workflow
+must set `DEMO_MODE=true`, converge the governed `PULSE_MOCK_*` fixture
+defaults in `/etc/<service>/.env`, restart the selected service, force the
+release-build demo-fixture entitlement sync through authenticated
+`/api/license/runtime-capabilities`, and then prove both
+`/api/system/mock-mode.enabled` and `/api/state.resources[]` converge. A
+passing `/api/version` or `/api/health` response alone is not demo readiness.
 That same operator-proof boundary also now owns the canonical hosted staging
 smoke entrypoint. `scripts/run_hosted_staging_smoke.sh` must stay as the
 repo-tracked operator command that composes the hosted signup/billing eval pack
@ -1875,6 +1884,13 @@ writer/reader path: `scripts/install.sh` may not keep a heredoc writer plus a
 second inline field parser for the same `connection.env` contract, because
 offline uninstall must consume the same persisted install-state artifact the
 installer wrote instead of reconstructing it ad hoc.
+That same shell-agent update recovery path must fail closed on partial
+legacy process or service-unit state: a recovered URL without a recovered token
+is not usable connection state and must not be logged or treated as recovered.
+Fallback recovery may merge URL and token across process args, environment, and
+systemd unit data, but it may report success only once both values are present;
+otherwise the update command must fall through to the explicit missing-state
+error instead of implying recovery succeeded.
 That same installer ownership now also applies to service lifecycle control:
 upgrade, reinstall, and platform-specific start/restart flows may not each
 carry their own stop/start command sequence for the same agent runtime.
--- a/scripts/install.sh
+++ b/scripts/install.sh
@ -1591,6 +1591,10 @@ apply_recovered_agent_arg_value() {
    esac
 }

+recovered_connection_state_ready() {
+    [[ -n "$PULSE_URL" && -n "$PULSE_TOKEN" ]]
+}
+
 recover_connection_state_from_arg_stream() {
    local arg=""
    local pending_key=""
@ -1688,7 +1692,7 @@ recover_connection_state_from_arg_stream() {
        esac
    done

-    [[ "$RECOVERED_AGENT_ARG_STATE" == "true" && -n "$PULSE_URL" ]]
+    [[ "$RECOVERED_AGENT_ARG_STATE" == "true" ]] && recovered_connection_state_ready
 }

 recover_connection_state_from_env_stream() {
@ -1738,7 +1742,7 @@ recover_connection_state_from_env_stream() {
        esac
    done

-    [[ "$RECOVERED_AGENT_ENV_STATE" == "true" && -n "$PULSE_URL" ]]
+    [[ "$RECOVERED_AGENT_ENV_STATE" == "true" ]] && recovered_connection_state_ready
 }

 collect_running_agent_pids() {
@ -1782,7 +1786,7 @@ recover_connection_state_from_running_agent() {
                recovered="true"
            fi
        fi
-        if [[ "$recovered" == "true" && -n "$PULSE_URL" ]]; then
+        if [[ "$recovered" == "true" ]] && recovered_connection_state_ready; then
            return 0
        fi
    done < <(collect_running_agent_pids)
@ -1820,7 +1824,7 @@ recover_connection_state_from_systemd_unit() {
                    ;;
            esac
        done < "$candidate"
-        if [[ "$recovered" == "true" && -n "$PULSE_URL" ]]; then
+        if [[ "$recovered" == "true" ]] && recovered_connection_state_ready; then
            return 0
        fi
    done
--- a/scripts/installtests/build_release_assets_test.go
+++ b/scripts/installtests/build_release_assets_test.go
@ -807,6 +807,12 @@ func TestUpdateDemoWorkflowUsesGovernedNetworkPath(t *testing.T) {
 		`-name "metrics.db"`,
 		`Removing demo volatile store: %s`,
 		`Demo host does not have enough free space to back up $CONFIG_DIR before install.`,
+		`Restore demo runtime configuration`,
+		`set_env_value DEMO_MODE true`,
+		`set_env_value PULSE_MOCK_MODE true`,
+		`/api/license/runtime-capabilities`,
+		`Mock mode enabled`,
+		`Demo server mock mode did not enable after entitlement sync`,
 		`Verify public browser smoke`,
 		`./scripts/run_demo_public_browser_smoke.sh`,
 	}
--- a/scripts/installtests/install_sh_test.go
+++ b/scripts/installtests/install_sh_test.go
@ -645,6 +645,9 @@ func TestInstallSHSupportsSavedStateUpdateMode(t *testing.T) {
 		`recover_connection_state_from_systemd_unit`,
 		`recover_connection_state_from_arg_stream`,
 		`recover_connection_state_from_env_stream`,
+		`recovered_connection_state_ready() {`,
+		`[[ "$RECOVERED_AGENT_ARG_STATE" == "true" ]] && recovered_connection_state_ready`,
+		`[[ "$RECOVERED_AGENT_ENV_STATE" == "true" ]] && recovered_connection_state_ready`,
 		`recover_connection_state_from_existing_agent || true`,
 		`if [[ -n "$PULSE_URL" && -n "$PULSE_TOKEN" ]]; then`,
 		`recover_agent_id_from_state_file() {`,
@ -692,6 +695,7 @@ func TestInstallSHRecoversV5ProcessArgsForSavedStateUpdate(t *testing.T) {
 		RUNTIME_TOKEN_FILE="/var/lib/pulse-agent/token"
 ` + extractInstallShellFunction(t, "strip_recovered_arg_quotes") + `
 ` + extractInstallShellFunction(t, "apply_recovered_agent_arg_value") + `
+` + extractInstallShellFunction(t, "recovered_connection_state_ready") + `
 ` + extractInstallShellFunction(t, "recover_connection_state_from_arg_stream") + `
 ` + extractInstallShellFunction(t, "build_exec_arg_items") + `
 ` + extractInstallShellFunction(t, "join_exec_arg_items") + `
@ -743,6 +747,144 @@ ARGS
 	}
 }

+func TestInstallSHRejectsPartialRecoveredProcessConnectionState(t *testing.T) {
+	script := `
+		PULSE_URL=""
+		PULSE_TOKEN=""
+		INTERVAL="30s"
+		INTERVAL_EXPLICIT="false"
+		ENABLE_HOST="true"
+		HOST_EXPLICIT="false"
+		ENABLE_DOCKER=""
+		DOCKER_EXPLICIT="false"
+		ENABLE_KUBERNETES=""
+		KUBERNETES_EXPLICIT="false"
+		KUBECONFIG_PATH=""
+		ENABLE_PROXMOX=""
+		PROXMOX_EXPLICIT="false"
+		PROXMOX_TYPE=""
+		INSECURE="false"
+		ENABLE_COMMANDS="false"
+		ENROLL="false"
+		HEALTH_ADDR=""
+		HEALTH_ADDR_SET="false"
+		AGENT_ID=""
+		HOSTNAME_OVERRIDE=""
+		STATE_DIR="/var/lib/pulse-agent"
+		CURL_CA_BUNDLE=""
+		KUBE_INCLUDE_ALL_PODS="false"
+		KUBE_INCLUDE_ALL_DEPLOYMENTS="false"
+		DISK_EXCLUDES=()
+` + extractInstallShellFunction(t, "strip_recovered_arg_quotes") + `
+` + extractInstallShellFunction(t, "apply_recovered_agent_arg_value") + `
+` + extractInstallShellFunction(t, "recovered_connection_state_ready") + `
+` + extractInstallShellFunction(t, "recover_connection_state_from_arg_stream") + `
+		if recover_connection_state_from_arg_stream <<'ARGS'
+/usr/local/bin/pulse-agent
+--url
+http://192.168.2.96:7655
+--enable-host
+--agent-id
+agent-123
+ARGS
+		then
+			echo "UNEXPECTED_SUCCESS"
+		else
+			echo "EXPECTED_FAILURE"
+		fi
+		printf 'URL=%s\nTOKEN=%s\nAGENT_ID=%s\n' "$PULSE_URL" "$PULSE_TOKEN" "$AGENT_ID"
+	`
+
+	out, err := exec.Command("bash", "-c", script).CombinedOutput()
+	if err != nil {
+		t.Fatalf("bash: %v\n%s", err, out)
+	}
+	got := string(out)
+	if !strings.Contains(got, "EXPECTED_FAILURE") {
+		t.Fatalf("partial recovered state should fail closed:\n%s", got)
+	}
+	if strings.Contains(got, "UNEXPECTED_SUCCESS") {
+		t.Fatalf("partial recovered state reported success:\n%s", got)
+	}
+	if !strings.Contains(got, "URL=http://192.168.2.96:7655") || !strings.Contains(got, "AGENT_ID=agent-123") {
+		t.Fatalf("partial recovery should still preserve non-secret fields for later sources:\n%s", got)
+	}
+	if !strings.Contains(got, "TOKEN=") {
+		t.Fatalf("expected token to remain empty:\n%s", got)
+	}
+}
+
+func TestInstallSHCombinesRecoveredProcessArgsAndEnvConnectionState(t *testing.T) {
+	script := `
+		PULSE_URL=""
+		PULSE_TOKEN=""
+		INTERVAL="30s"
+		INTERVAL_EXPLICIT="false"
+		ENABLE_HOST="true"
+		HOST_EXPLICIT="false"
+		ENABLE_DOCKER=""
+		DOCKER_EXPLICIT="false"
+		ENABLE_KUBERNETES=""
+		KUBERNETES_EXPLICIT="false"
+		KUBECONFIG_PATH=""
+		ENABLE_PROXMOX=""
+		PROXMOX_EXPLICIT="false"
+		PROXMOX_TYPE=""
+		INSECURE="false"
+		ENABLE_COMMANDS="false"
+		ENROLL="false"
+		HEALTH_ADDR=""
+		HEALTH_ADDR_SET="false"
+		AGENT_ID=""
+		HOSTNAME_OVERRIDE=""
+		STATE_DIR="/var/lib/pulse-agent"
+		CURL_CA_BUNDLE=""
+		KUBE_INCLUDE_ALL_PODS="false"
+		KUBE_INCLUDE_ALL_DEPLOYMENTS="false"
+		DISK_EXCLUDES=()
+` + extractInstallShellFunction(t, "strip_recovered_arg_quotes") + `
+` + extractInstallShellFunction(t, "apply_recovered_agent_arg_value") + `
+` + extractInstallShellFunction(t, "recovered_connection_state_ready") + `
+` + extractInstallShellFunction(t, "recover_connection_state_from_arg_stream") + `
+` + extractInstallShellFunction(t, "recover_connection_state_from_env_stream") + `
+		if recover_connection_state_from_arg_stream <<'ARGS'
+/usr/local/bin/pulse-agent
+--url
+http://192.168.2.96:7655
+ARGS
+		then
+			echo "ARGS_READY"
+		else
+			echo "ARGS_PARTIAL"
+		fi
+		if recover_connection_state_from_env_stream <<'ENV'
+PULSE_TOKEN=deadbeef
+ENV
+		then
+			echo "ENV_READY"
+		else
+			echo "ENV_PARTIAL"
+		fi
+		printf 'URL=%s\nTOKEN=%s\n' "$PULSE_URL" "$PULSE_TOKEN"
+	`
+
+	out, err := exec.Command("bash", "-c", script).CombinedOutput()
+	if err != nil {
+		t.Fatalf("bash: %v\n%s", err, out)
+	}
+	got := string(out)
+	for _, needle := range []string{
+		"ARGS_PARTIAL",
+		"ENV_READY",
+		"URL=http://192.168.2.96:7655",
+		"TOKEN=deadbeef",
+	} {
+		if !strings.Contains(got, needle) {
+			t.Fatalf("combined recovery missing %q:\n%s", needle, got)
+		}
+	}
+}
+
 func TestInstallSHUsesCanonicalServiceLifecycleHelpers(t *testing.T) {
 	content, err := os.ReadFile(repoFile("scripts", "install.sh"))
 	if err != nil {
--- a/scripts/release_control/release_promotion_policy_test.py
+++ b/scripts/release_control/release_promotion_policy_test.py
@ -735,6 +735,13 @@ class ReleasePromotionPolicyTest(unittest.TestCase):
        self.assertIn("DEMO_EXPECTED_HOSTNAME", demo)
        self.assertIn("Verify target host identity", demo)
        self.assertIn("Demo environment points at host $REMOTE_HOSTNAME but expected $DEMO_EXPECTED_HOSTNAME.", demo)
+        self.assertIn("Restore demo runtime configuration", demo)
+        self.assertIn("set_env_value DEMO_MODE true", demo)
+        self.assertIn("set_env_value PULSE_MOCK_MODE true", demo)
+        self.assertIn("Demo service restarted with governed demo runtime configuration.", demo)
+        self.assertIn("/api/license/runtime-capabilities", demo)
+        self.assertIn("Mock mode enabled", demo)
+        self.assertIn("Demo server mock mode did not enable after entitlement sync", demo)
        self.assertIn(".resources | if type == \"array\" then length else 0 end", demo)
        self.assertNotIn(".nodes | length", demo)
        self.assertIn("Mock resources detected", demo)