Bound legacy demo bootstrap workload

This commit is contained in:
rcourtman 2026-08-26 10:27:58 +01:00
parent ad4c82c34e
commit a8e84b4011
8 changed files with 232 additions and 47 deletions

View file

@ -1,8 +1,8 @@
#!/usr/bin/env bash
set -euo pipefail
if [ "$#" -ne 6 ]; then
echo "usage: recover-demo-runtime.sh EXPECTED_HOSTNAME LOCAL_BASE_URL EXPECTED_VERSION MOCK_NODES MOCK_SEED_DURATION MOCK_UPDATE_INTERVAL" >&2
if [ "$#" -ne 15 ]; then
echo "usage: recover-demo-runtime.sh EXPECTED_HOSTNAME LOCAL_BASE_URL EXPECTED_VERSION MOCK_NODES MOCK_VMS_PER_NODE MOCK_LXCS_PER_NODE MOCK_DOCKER_HOSTS MOCK_DOCKER_CONTAINERS MOCK_GENERIC_HOSTS MOCK_K8S_CLUSTERS MOCK_K8S_NODES MOCK_K8S_PODS MOCK_K8S_DEPLOYMENTS MOCK_SEED_DURATION MOCK_UPDATE_INTERVAL" >&2
exit 2
fi
@ -10,8 +10,17 @@ EXPECTED_HOSTNAME="$1"
LOCAL_BASE_URL="${2%/}"
EXPECTED_VERSION="${3#v}"
MOCK_NODES="$4"
MOCK_SEED_DURATION="$5"
MOCK_UPDATE_INTERVAL="$6"
MOCK_VMS_PER_NODE="$5"
MOCK_LXCS_PER_NODE="$6"
MOCK_DOCKER_HOSTS="$7"
MOCK_DOCKER_CONTAINERS="$8"
MOCK_GENERIC_HOSTS="$9"
MOCK_K8S_CLUSTERS="${10}"
MOCK_K8S_NODES="${11}"
MOCK_K8S_PODS="${12}"
MOCK_K8S_DEPLOYMENTS="${13}"
MOCK_SEED_DURATION="${14}"
MOCK_UPDATE_INTERVAL="${15}"
SERVICE_NAME="pulse"
RELAY_SERVICE_NAME="pulse-relay"
EXPECTED_BINARY="/opt/pulse/bin/pulse"
@ -19,7 +28,19 @@ EXPECTED_UNIT="/etc/systemd/system/pulse.service"
MUTATED=false
BACKUP_DIR=""
[[ "$MOCK_NODES" =~ ^[1-9][0-9]*$ ]]
for fixture_count in \
"$MOCK_NODES" \
"$MOCK_VMS_PER_NODE" \
"$MOCK_LXCS_PER_NODE" \
"$MOCK_DOCKER_HOSTS" \
"$MOCK_DOCKER_CONTAINERS" \
"$MOCK_GENERIC_HOSTS" \
"$MOCK_K8S_CLUSTERS" \
"$MOCK_K8S_NODES" \
"$MOCK_K8S_PODS" \
"$MOCK_K8S_DEPLOYMENTS"; do
[[ "$fixture_count" =~ ^[1-9][0-9]*$ ]]
done
[[ "$MOCK_SEED_DURATION" =~ ^[1-9][0-9]*[smhd]$ ]]
[[ "$MOCK_UPDATE_INTERVAL" =~ ^[1-9][0-9]*[smh]$ ]]
@ -71,6 +92,15 @@ set_env_value() {
runtime_profile_matches() {
sudo grep -Fxq "PULSE_MOCK_NODES=${MOCK_NODES}" /etc/pulse/.env &&
sudo grep -Fxq "PULSE_MOCK_VMS_PER_NODE=${MOCK_VMS_PER_NODE}" /etc/pulse/.env &&
sudo grep -Fxq "PULSE_MOCK_LXCS_PER_NODE=${MOCK_LXCS_PER_NODE}" /etc/pulse/.env &&
sudo grep -Fxq "PULSE_MOCK_DOCKER_HOSTS=${MOCK_DOCKER_HOSTS}" /etc/pulse/.env &&
sudo grep -Fxq "PULSE_MOCK_DOCKER_CONTAINERS=${MOCK_DOCKER_CONTAINERS}" /etc/pulse/.env &&
sudo grep -Fxq "PULSE_MOCK_GENERIC_HOSTS=${MOCK_GENERIC_HOSTS}" /etc/pulse/.env &&
sudo grep -Fxq "PULSE_MOCK_K8S_CLUSTERS=${MOCK_K8S_CLUSTERS}" /etc/pulse/.env &&
sudo grep -Fxq "PULSE_MOCK_K8S_NODES=${MOCK_K8S_NODES}" /etc/pulse/.env &&
sudo grep -Fxq "PULSE_MOCK_K8S_PODS=${MOCK_K8S_PODS}" /etc/pulse/.env &&
sudo grep -Fxq "PULSE_MOCK_K8S_DEPLOYMENTS=${MOCK_K8S_DEPLOYMENTS}" /etc/pulse/.env &&
sudo grep -Fxq "PULSE_MOCK_TRENDS_SEED_DURATION=${MOCK_SEED_DURATION}" /etc/pulse/.env &&
sudo grep -Fxq "PULSE_MOCK_UPDATE_INTERVAL=${MOCK_UPDATE_INTERVAL}" /etc/pulse/.env
}
@ -137,6 +167,15 @@ emit_evidence() {
--arg version "$version" \
--arg expected_version "$EXPECTED_VERSION" \
--arg mock_nodes "$MOCK_NODES" \
--arg mock_vms_per_node "$MOCK_VMS_PER_NODE" \
--arg mock_lxcs_per_node "$MOCK_LXCS_PER_NODE" \
--arg mock_docker_hosts "$MOCK_DOCKER_HOSTS" \
--arg mock_docker_containers "$MOCK_DOCKER_CONTAINERS" \
--arg mock_generic_hosts "$MOCK_GENERIC_HOSTS" \
--arg mock_k8s_clusters "$MOCK_K8S_CLUSTERS" \
--arg mock_k8s_nodes "$MOCK_K8S_NODES" \
--arg mock_k8s_pods "$MOCK_K8S_PODS" \
--arg mock_k8s_deployments "$MOCK_K8S_DEPLOYMENTS" \
--arg mock_seed_duration "$MOCK_SEED_DURATION" \
--arg mock_update_interval "$MOCK_UPDATE_INTERVAL" \
'{schema_version: 1, status: $status, mutated: $mutated,
@ -157,6 +196,15 @@ emit_evidence() {
after_runtime_config_sha256: $after_runtime_config_sha256,
version: $version, expected_version: $expected_version,
runtime_profile: {mock_nodes: $mock_nodes,
mock_vms_per_node: $mock_vms_per_node,
mock_lxcs_per_node: $mock_lxcs_per_node,
mock_docker_hosts: $mock_docker_hosts,
mock_docker_containers: $mock_docker_containers,
mock_generic_hosts: $mock_generic_hosts,
mock_k8s_clusters: $mock_k8s_clusters,
mock_k8s_nodes: $mock_k8s_nodes,
mock_k8s_pods: $mock_k8s_pods,
mock_k8s_deployments: $mock_k8s_deployments,
mock_seed_duration: $mock_seed_duration,
mock_update_interval: $mock_update_interval}}'
}
@ -220,6 +268,15 @@ trap rollback_on_error ERR
log "applying the target-compatible demo runtime profile and restarting only pulse.service"
MUTATED=true
set_env_value PULSE_MOCK_NODES "$MOCK_NODES"
set_env_value PULSE_MOCK_VMS_PER_NODE "$MOCK_VMS_PER_NODE"
set_env_value PULSE_MOCK_LXCS_PER_NODE "$MOCK_LXCS_PER_NODE"
set_env_value PULSE_MOCK_DOCKER_HOSTS "$MOCK_DOCKER_HOSTS"
set_env_value PULSE_MOCK_DOCKER_CONTAINERS "$MOCK_DOCKER_CONTAINERS"
set_env_value PULSE_MOCK_GENERIC_HOSTS "$MOCK_GENERIC_HOSTS"
set_env_value PULSE_MOCK_K8S_CLUSTERS "$MOCK_K8S_CLUSTERS"
set_env_value PULSE_MOCK_K8S_NODES "$MOCK_K8S_NODES"
set_env_value PULSE_MOCK_K8S_PODS "$MOCK_K8S_PODS"
set_env_value PULSE_MOCK_K8S_DEPLOYMENTS "$MOCK_K8S_DEPLOYMENTS"
set_env_value PULSE_MOCK_TRENDS_SEED_DURATION "$MOCK_SEED_DURATION"
set_env_value PULSE_MOCK_UPDATE_INTERVAL "$MOCK_UPDATE_INTERVAL"
sudo chown pulse:pulse /etc/pulse/.env

View file

@ -19,21 +19,47 @@ if git grep -q 'mockEagerHistoryPVEGuestLimit' "${TAG_REF}" -- internal/monitori
git grep -q 'UpdateMetricCohort' "${TAG_REF}" -- internal/mock/integration.go; then
PROFILE="large-estate"
MOCK_NODES=50
MOCK_VMS_PER_NODE=10
MOCK_LXCS_PER_NODE=8
MOCK_DOCKER_HOSTS=5
MOCK_DOCKER_CONTAINERS=14
MOCK_GENERIC_HOSTS=4
MOCK_K8S_CLUSTERS=3
MOCK_K8S_NODES=5
MOCK_K8S_PODS=40
MOCK_K8S_DEPLOYMENTS=14
MOCK_SEED_DURATION=48h
MOCK_UPDATE_INTERVAL=2s
else
PROFILE="legacy-bounded"
MOCK_NODES="$({
git show "${TAG_REF}:.github/workflows/update-demo-server.yml" 2>/dev/null || true
} | sed -n -E 's/^[[:space:]]*set_env_value PULSE_MOCK_NODES ([0-9]+)[[:space:]]*$/\1/p' | tail -1)"
if [[ ! "${MOCK_NODES}" =~ ^[1-9][0-9]*$ ]]; then
MOCK_NODES=32
fi
MOCK_SEED_DURATION=6h
MOCK_UPDATE_INTERVAL=10s
# Older runtimes update every fixture on every tick and eagerly retain every
# history series. Keep their public-demo payload deliberately compact; the
# tagged workflow's former 32-node default still produced a 4 MB state
# bootstrap and could leave real browsers parked on the loading shell.
MOCK_NODES=8
MOCK_VMS_PER_NODE=6
MOCK_LXCS_PER_NODE=4
MOCK_DOCKER_HOSTS=2
MOCK_DOCKER_CONTAINERS=8
MOCK_GENERIC_HOSTS=2
MOCK_K8S_CLUSTERS=1
MOCK_K8S_NODES=3
MOCK_K8S_PODS=12
MOCK_K8S_DEPLOYMENTS=4
MOCK_SEED_DURATION=2h
MOCK_UPDATE_INTERVAL=15s
fi
printf 'profile=%s\n' "$PROFILE"
printf 'mock_nodes=%s\n' "$MOCK_NODES"
printf 'mock_vms_per_node=%s\n' "$MOCK_VMS_PER_NODE"
printf 'mock_lxcs_per_node=%s\n' "$MOCK_LXCS_PER_NODE"
printf 'mock_docker_hosts=%s\n' "$MOCK_DOCKER_HOSTS"
printf 'mock_docker_containers=%s\n' "$MOCK_DOCKER_CONTAINERS"
printf 'mock_generic_hosts=%s\n' "$MOCK_GENERIC_HOSTS"
printf 'mock_k8s_clusters=%s\n' "$MOCK_K8S_CLUSTERS"
printf 'mock_k8s_nodes=%s\n' "$MOCK_K8S_NODES"
printf 'mock_k8s_pods=%s\n' "$MOCK_K8S_PODS"
printf 'mock_k8s_deployments=%s\n' "$MOCK_K8S_DEPLOYMENTS"
printf 'mock_seed_duration=%s\n' "$MOCK_SEED_DURATION"
printf 'mock_update_interval=%s\n' "$MOCK_UPDATE_INTERVAL"

View file

@ -18,6 +18,8 @@ jobs:
DEMO_EXPECTED_HOSTNAME: ${{ vars.DEMO_EXPECTED_HOSTNAME }}
DEMO_LOCAL_BASE_URL: ${{ vars.DEMO_LOCAL_BASE_URL }}
DEMO_PUBLIC_HEALTH_URL: ${{ vars.DEMO_PUBLIC_HEALTH_URL }}
DEMO_AUTH_USER: ${{ vars.DEMO_AUTH_USER }}
DEMO_AUTH_PASS: ${{ vars.DEMO_AUTH_PASS }}
steps:
- name: Checkout exact recovery implementation
@ -93,6 +95,15 @@ jobs:
DEMO_SERVER_USER: ${{ secrets.DEMO_SERVER_USER }}
EXPECTED_VERSION: ${{ steps.stable.outputs.tag }}
MOCK_NODES: ${{ steps.runtime_profile.outputs.mock_nodes }}
MOCK_VMS_PER_NODE: ${{ steps.runtime_profile.outputs.mock_vms_per_node }}
MOCK_LXCS_PER_NODE: ${{ steps.runtime_profile.outputs.mock_lxcs_per_node }}
MOCK_DOCKER_HOSTS: ${{ steps.runtime_profile.outputs.mock_docker_hosts }}
MOCK_DOCKER_CONTAINERS: ${{ steps.runtime_profile.outputs.mock_docker_containers }}
MOCK_GENERIC_HOSTS: ${{ steps.runtime_profile.outputs.mock_generic_hosts }}
MOCK_K8S_CLUSTERS: ${{ steps.runtime_profile.outputs.mock_k8s_clusters }}
MOCK_K8S_NODES: ${{ steps.runtime_profile.outputs.mock_k8s_nodes }}
MOCK_K8S_PODS: ${{ steps.runtime_profile.outputs.mock_k8s_pods }}
MOCK_K8S_DEPLOYMENTS: ${{ steps.runtime_profile.outputs.mock_k8s_deployments }}
MOCK_SEED_DURATION: ${{ steps.runtime_profile.outputs.mock_seed_duration }}
MOCK_UPDATE_INTERVAL: ${{ steps.runtime_profile.outputs.mock_update_interval }}
run: |
@ -102,7 +113,7 @@ jobs:
-o StrictHostKeyChecking=yes \
-o UserKnownHostsFile=~/.ssh/known_hosts \
"$DEMO_SERVER_USER@$DEMO_SERVER_HOST" \
"bash -s -- $(printf '%q ' "$DEMO_EXPECTED_HOSTNAME" "$DEMO_LOCAL_BASE_URL" "$EXPECTED_VERSION" "$MOCK_NODES" "$MOCK_SEED_DURATION" "$MOCK_UPDATE_INTERVAL")" \
"bash -s -- $(printf '%q ' "$DEMO_EXPECTED_HOSTNAME" "$DEMO_LOCAL_BASE_URL" "$EXPECTED_VERSION" "$MOCK_NODES" "$MOCK_VMS_PER_NODE" "$MOCK_LXCS_PER_NODE" "$MOCK_DOCKER_HOSTS" "$MOCK_DOCKER_CONTAINERS" "$MOCK_GENERIC_HOSTS" "$MOCK_K8S_CLUSTERS" "$MOCK_K8S_NODES" "$MOCK_K8S_PODS" "$MOCK_K8S_DEPLOYMENTS" "$MOCK_SEED_DURATION" "$MOCK_UPDATE_INTERVAL")" \
< .github/scripts/recover-demo-runtime.sh \
> demo-runtime-recovery.json
jq -e '.schema_version == 1 and (.status == "recovered" or .status == "healthy_noop")' \
@ -139,6 +150,8 @@ jobs:
run: |
set -euo pipefail
export PULSE_PUBLIC_SITE_URL="${DEMO_PUBLIC_HEALTH_URL%/api/health}/"
export PULSE_DEMO_AUTH_USER="${DEMO_AUTH_USER:-demo}"
export PULSE_DEMO_AUTH_PASS="${DEMO_AUTH_PASS:-demo}"
./scripts/run_demo_public_browser_smoke.sh
- name: Compensate failed mutated recovery

View file

@ -50,6 +50,15 @@ jobs:
environment_name: ${{ steps.target.outputs.environment_name }}
skip: ${{ steps.target.outputs.skip || 'false' }}
mock_nodes: ${{ steps.runtime_profile.outputs.mock_nodes }}
mock_vms_per_node: ${{ steps.runtime_profile.outputs.mock_vms_per_node }}
mock_lxcs_per_node: ${{ steps.runtime_profile.outputs.mock_lxcs_per_node }}
mock_docker_hosts: ${{ steps.runtime_profile.outputs.mock_docker_hosts }}
mock_docker_containers: ${{ steps.runtime_profile.outputs.mock_docker_containers }}
mock_generic_hosts: ${{ steps.runtime_profile.outputs.mock_generic_hosts }}
mock_k8s_clusters: ${{ steps.runtime_profile.outputs.mock_k8s_clusters }}
mock_k8s_nodes: ${{ steps.runtime_profile.outputs.mock_k8s_nodes }}
mock_k8s_pods: ${{ steps.runtime_profile.outputs.mock_k8s_pods }}
mock_k8s_deployments: ${{ steps.runtime_profile.outputs.mock_k8s_deployments }}
mock_seed_duration: ${{ steps.runtime_profile.outputs.mock_seed_duration }}
mock_update_interval: ${{ steps.runtime_profile.outputs.mock_update_interval }}
@ -545,6 +554,15 @@ jobs:
set -euo pipefail
SERVICE_NAME="${{ steps.config.outputs.service_name }}"
MOCK_NODES="${{ needs.resolve.outputs.mock_nodes }}"
MOCK_VMS_PER_NODE="${{ needs.resolve.outputs.mock_vms_per_node }}"
MOCK_LXCS_PER_NODE="${{ needs.resolve.outputs.mock_lxcs_per_node }}"
MOCK_DOCKER_HOSTS="${{ needs.resolve.outputs.mock_docker_hosts }}"
MOCK_DOCKER_CONTAINERS="${{ needs.resolve.outputs.mock_docker_containers }}"
MOCK_GENERIC_HOSTS="${{ needs.resolve.outputs.mock_generic_hosts }}"
MOCK_K8S_CLUSTERS="${{ needs.resolve.outputs.mock_k8s_clusters }}"
MOCK_K8S_NODES="${{ needs.resolve.outputs.mock_k8s_nodes }}"
MOCK_K8S_PODS="${{ needs.resolve.outputs.mock_k8s_pods }}"
MOCK_K8S_DEPLOYMENTS="${{ needs.resolve.outputs.mock_k8s_deployments }}"
MOCK_SEED_DURATION="${{ needs.resolve.outputs.mock_seed_duration }}"
MOCK_UPDATE_INTERVAL="${{ needs.resolve.outputs.mock_update_interval }}"
REMOTE_SCRIPT=$(cat <<'EOF'
@ -552,8 +570,17 @@ jobs:
SERVICE_NAME="${1:-pulse}"
DEMO_LOCAL_BASE_URL="$2"
MOCK_NODES="$3"
MOCK_SEED_DURATION="$4"
MOCK_UPDATE_INTERVAL="$5"
MOCK_VMS_PER_NODE="$4"
MOCK_LXCS_PER_NODE="$5"
MOCK_DOCKER_HOSTS="$6"
MOCK_DOCKER_CONTAINERS="$7"
MOCK_GENERIC_HOSTS="$8"
MOCK_K8S_CLUSTERS="$9"
MOCK_K8S_NODES="${10}"
MOCK_K8S_PODS="${11}"
MOCK_K8S_DEPLOYMENTS="${12}"
MOCK_SEED_DURATION="${13}"
MOCK_UPDATE_INTERVAL="${14}"
CONFIG_DIR="/etc/${SERVICE_NAME:-pulse}"
SERVICE_USER="$(systemctl show "$SERVICE_NAME" --property=User --value 2>/dev/null || true)"
SERVICE_GROUP="$(systemctl show "$SERVICE_NAME" --property=Group --value 2>/dev/null || true)"
@ -636,15 +663,15 @@ jobs:
set_env_value DEMO_MODE true
set_env_value PULSE_MOCK_MODE true
set_env_value PULSE_MOCK_NODES "$MOCK_NODES"
set_env_value PULSE_MOCK_VMS_PER_NODE 10
set_env_value PULSE_MOCK_LXCS_PER_NODE 8
set_env_value PULSE_MOCK_DOCKER_HOSTS 5
set_env_value PULSE_MOCK_DOCKER_CONTAINERS 14
set_env_value PULSE_MOCK_GENERIC_HOSTS 4
set_env_value PULSE_MOCK_K8S_CLUSTERS 3
set_env_value PULSE_MOCK_K8S_NODES 5
set_env_value PULSE_MOCK_K8S_PODS 40
set_env_value PULSE_MOCK_K8S_DEPLOYMENTS 14
set_env_value PULSE_MOCK_VMS_PER_NODE "$MOCK_VMS_PER_NODE"
set_env_value PULSE_MOCK_LXCS_PER_NODE "$MOCK_LXCS_PER_NODE"
set_env_value PULSE_MOCK_DOCKER_HOSTS "$MOCK_DOCKER_HOSTS"
set_env_value PULSE_MOCK_DOCKER_CONTAINERS "$MOCK_DOCKER_CONTAINERS"
set_env_value PULSE_MOCK_GENERIC_HOSTS "$MOCK_GENERIC_HOSTS"
set_env_value PULSE_MOCK_K8S_CLUSTERS "$MOCK_K8S_CLUSTERS"
set_env_value PULSE_MOCK_K8S_NODES "$MOCK_K8S_NODES"
set_env_value PULSE_MOCK_K8S_PODS "$MOCK_K8S_PODS"
set_env_value PULSE_MOCK_K8S_DEPLOYMENTS "$MOCK_K8S_DEPLOYMENTS"
set_env_value PULSE_MOCK_RANDOM_METRICS true
set_env_value PULSE_MOCK_STOPPED_PERCENT 6
set_env_value PULSE_MOCK_TRENDS_SEED_DURATION "$MOCK_SEED_DURATION"
@ -670,7 +697,7 @@ jobs:
EOF
)
ssh -i ~/.ssh/id_ed25519 "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" \
"bash -s -- $(printf '%q ' "$SERVICE_NAME" "$DEMO_LOCAL_BASE_URL" "$MOCK_NODES" "$MOCK_SEED_DURATION" "$MOCK_UPDATE_INTERVAL")" <<<"$REMOTE_SCRIPT"
"bash -s -- $(printf '%q ' "$SERVICE_NAME" "$DEMO_LOCAL_BASE_URL" "$MOCK_NODES" "$MOCK_VMS_PER_NODE" "$MOCK_LXCS_PER_NODE" "$MOCK_DOCKER_HOSTS" "$MOCK_DOCKER_CONTAINERS" "$MOCK_GENERIC_HOSTS" "$MOCK_K8S_CLUSTERS" "$MOCK_K8S_NODES" "$MOCK_K8S_PODS" "$MOCK_K8S_DEPLOYMENTS" "$MOCK_SEED_DURATION" "$MOCK_UPDATE_INTERVAL")" <<<"$REMOTE_SCRIPT"
- name: Verify demo runtime
env:
@ -783,6 +810,8 @@ jobs:
exit 1
;;
esac
export PULSE_DEMO_AUTH_USER="${DEMO_AUTH_USER:-demo}"
export PULSE_DEMO_AUTH_PASS="${DEMO_AUTH_PASS:-demo}"
./scripts/run_demo_public_browser_smoke.sh
- name: Cleanup SSH material