From 76ced45c3a4be469aa29746d03b66bedc9fca82b Mon Sep 17 00:00:00 2001 From: rcourtman Date: Thu, 9 Jul 2026 16:51:38 +0100 Subject: [PATCH] Harden demo SSH setup for private deploy hosts --- .github/workflows/deploy-demo-server.yml | 31 +++++++++++++++++-- .github/workflows/update-demo-server.yml | 31 +++++++++++++++++-- .../subsystems/deployment-installability.md | 5 ++- .../installtests/build_release_assets_test.go | 8 +++++ .../release_promotion_policy_test.py | 6 ++++ 5 files changed, 76 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy-demo-server.yml b/.github/workflows/deploy-demo-server.yml index 9321be24f..358c6c31b 100644 --- a/.github/workflows/deploy-demo-server.yml +++ b/.github/workflows/deploy-demo-server.yml @@ -149,8 +149,35 @@ jobs: chmod 700 ~/.ssh echo "$DEMO_SERVER_SSH_KEY" > ~/.ssh/id_ed25519 chmod 600 ~/.ssh/id_ed25519 - ssh-keyscan -H "$DEMO_SERVER_HOST" >> ~/.ssh/known_hosts - chmod 600 ~/.ssh/known_hosts + + : > ~/.ssh/known_hosts + keyscan_output="$(mktemp)" + keyscan_error="$(mktemp)" + MAX_SSH_SETUP_ATTEMPTS=18 + for attempt in $(seq 1 "$MAX_SSH_SETUP_ATTEMPTS"); do + if getent hosts "$DEMO_SERVER_HOST" >/dev/null 2>&1; then + if ssh-keyscan -T 10 -H "$DEMO_SERVER_HOST" > "$keyscan_output" 2>"$keyscan_error" && [ -s "$keyscan_output" ]; then + cat "$keyscan_output" >> ~/.ssh/known_hosts + chmod 600 ~/.ssh/known_hosts + echo "Demo SSH host key captured." + exit 0 + fi + echo "ssh-keyscan did not return demo host keys on attempt ${attempt}/${MAX_SSH_SETUP_ATTEMPTS}." + else + echo "Demo SSH host is not resolvable yet on attempt ${attempt}/${MAX_SSH_SETUP_ATTEMPTS}." + fi + + if [ "$attempt" -lt "$MAX_SSH_SETUP_ATTEMPTS" ]; then + sleep 10 + fi + done + + echo "::error::Timed out waiting for the demo SSH host to resolve and return host keys after Tailscale setup." + tailscale status --peers=false || true + if [ -s "$keyscan_error" ]; then + sed 's/^/ssh-keyscan: /' "$keyscan_error" || true + fi + exit 1 - name: Verify target host identity env: diff --git a/.github/workflows/update-demo-server.yml b/.github/workflows/update-demo-server.yml index 16f095199..7c6cc9b79 100644 --- a/.github/workflows/update-demo-server.yml +++ b/.github/workflows/update-demo-server.yml @@ -256,8 +256,35 @@ jobs: chmod 700 ~/.ssh echo "$DEMO_SERVER_SSH_KEY" > ~/.ssh/id_ed25519 chmod 600 ~/.ssh/id_ed25519 - ssh-keyscan -H "$DEMO_SERVER_HOST" >> ~/.ssh/known_hosts - chmod 600 ~/.ssh/known_hosts + + : > ~/.ssh/known_hosts + keyscan_output="$(mktemp)" + keyscan_error="$(mktemp)" + MAX_SSH_SETUP_ATTEMPTS=18 + for attempt in $(seq 1 "$MAX_SSH_SETUP_ATTEMPTS"); do + if getent hosts "$DEMO_SERVER_HOST" >/dev/null 2>&1; then + if ssh-keyscan -T 10 -H "$DEMO_SERVER_HOST" > "$keyscan_output" 2>"$keyscan_error" && [ -s "$keyscan_output" ]; then + cat "$keyscan_output" >> ~/.ssh/known_hosts + chmod 600 ~/.ssh/known_hosts + echo "Demo SSH host key captured." + exit 0 + fi + echo "ssh-keyscan did not return demo host keys on attempt ${attempt}/${MAX_SSH_SETUP_ATTEMPTS}." + else + echo "Demo SSH host is not resolvable yet on attempt ${attempt}/${MAX_SSH_SETUP_ATTEMPTS}." + fi + + if [ "$attempt" -lt "$MAX_SSH_SETUP_ATTEMPTS" ]; then + sleep 10 + fi + done + + echo "::error::Timed out waiting for the demo SSH host to resolve and return host keys after Tailscale setup." + tailscale status --peers=false || true + if [ -s "$keyscan_error" ]; then + sed 's/^/ssh-keyscan: /' "$keyscan_error" || true + fi + exit 1 - name: Verify target host identity env: diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index 2281e20c7..ba34464a8 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -1183,7 +1183,10 @@ Those same governed demo deploy/update workflows also own the runner-to-host network path. They must establish the canonical Tailscale connectivity step before SSH setup so stable or preview targets may stay on governed private hostnames or Tailscale IPs, rather than silently depending on public SSH -reachability from GitHub-hosted runners. +reachability from GitHub-hosted runners. After Tailscale setup, SSH setup must +wait for the configured demo host to resolve and return host keys with bounded +retries before any installer or binary copy runs; a one-shot `ssh-keyscan` +against a private demo hostname is not sufficient release or deploy proof. Those same workflows also own customer-visible browser truth for the public demo shell. Health checks and entry-asset parity are necessary but not sufficient; after those checks pass, the governed helpers diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.go index 0b8b87586..b6a8bd985 100644 --- a/scripts/installtests/build_release_assets_test.go +++ b/scripts/installtests/build_release_assets_test.go @@ -947,6 +947,10 @@ func TestDeployDemoWorkflowFailsClosedForStableAndVerifiesFrontendParity(t *test ` - stable`, `Capture expected frontend entry asset`, `Verify target host identity`, + `MAX_SSH_SETUP_ATTEMPTS=18`, + `getent hosts "$DEMO_SERVER_HOST"`, + `ssh-keyscan -T 10 -H "$DEMO_SERVER_HOST"`, + `Timed out waiting for the demo SSH host to resolve and return host keys after Tailscale setup.`, `SERVICE_NAME="pulse"`, `Unsupported demo target: ${TARGET}`, `Demo environment points at host $REMOTE_HOSTNAME but expected $DEMO_EXPECTED_HOSTNAME.`, @@ -987,6 +991,10 @@ func TestUpdateDemoWorkflowUsesGovernedNetworkPath(t *testing.T) { `go run ./scripts/release_update_key.go public-key-ssh`, `sed -i "s|^PINNED_RELEASE_SSH_PUBLIC_KEY=.*|PINNED_RELEASE_SSH_PUBLIC_KEY=\"${TRUSTED_SSH_PUBLIC_KEY}\"|" /tmp/pulse-install.sh`, `Verify target host identity`, + `MAX_SSH_SETUP_ATTEMPTS=18`, + `getent hosts "$DEMO_SERVER_HOST"`, + `ssh-keyscan -T 10 -H "$DEMO_SERVER_HOST"`, + `Timed out waiting for the demo SSH host to resolve and return host keys after Tailscale setup.`, `Demo environment points at host $REMOTE_HOSTNAME but expected $DEMO_EXPECTED_HOSTNAME.`, `Prepare demo host storage`, `KEEP_BACKUPS=2`, diff --git a/scripts/release_control/release_promotion_policy_test.py b/scripts/release_control/release_promotion_policy_test.py index bfd5a5a2d..c4bfbf3b6 100644 --- a/scripts/release_control/release_promotion_policy_test.py +++ b/scripts/release_control/release_promotion_policy_test.py @@ -525,6 +525,7 @@ class ReleasePromotionPolicyTest(unittest.TestCase): def test_release_workflow_enforces_rc_lineage_soak_and_v5_notice(self) -> None: content = read(".github/workflows/create-release.yml") update_demo_workflow = read(".github/workflows/update-demo-server.yml") + deploy_demo_workflow = read(".github/workflows/deploy-demo-server.yml") validation_workflow = read(".github/workflows/validate-release-assets.yml") helper = read("scripts/trigger-release.sh") renderer = read("scripts/release_control/render_release_body.py") @@ -610,6 +611,11 @@ class ReleasePromotionPolicyTest(unittest.TestCase): self.assertIn("does not trust the configured release signing key", content) self.assertIn("TRUSTED_SSH_PUBLIC_KEY", update_demo_workflow) self.assertIn('sed -i "s|^PINNED_RELEASE_SSH_PUBLIC_KEY=.*|PINNED_RELEASE_SSH_PUBLIC_KEY=\\"${TRUSTED_SSH_PUBLIC_KEY}\\"|" /tmp/pulse-install.sh', update_demo_workflow) + for demo_workflow in (update_demo_workflow, deploy_demo_workflow): + self.assertIn("MAX_SSH_SETUP_ATTEMPTS=18", demo_workflow) + self.assertIn('getent hosts "$DEMO_SERVER_HOST"', demo_workflow) + self.assertIn('ssh-keyscan -T 10 -H "$DEMO_SERVER_HOST"', demo_workflow) + self.assertIn("Timed out waiting for the demo SSH host to resolve and return host keys after Tailscale setup", demo_workflow) self.assertIn("derive the OpenSSH installer trust key from `PULSE_UPDATE_SIGNING_PUBLIC_KEY`", normalize_ws(contract)) self.assertIn('SYFT_VERSION="1.42.4"', content) self.assertIn('SYFT_ARCHIVE="syft_${SYFT_VERSION}_linux_amd64.tar.gz"', content)