diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index 68a822f264..30e668ebae 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -455,6 +455,14 @@ the managed local stack: `scripts/hot-dev.sh` may only rebuild the backend for runtime Go sources, not `*_test.go` churn, and it must suppress `pulse` binary change events produced by its own successful managed rebuilds, managed backend restarts, or startup build. +That same boundary also owns backend-liveness recovery, not just process- +existence. The managed health monitor in `scripts/hot-dev.sh` must probe +`http://127.0.0.1:${PULSE_DEV_API_PORT}/api/health` in addition to checking +that a `./pulse` process exists, so an alive-but-unresponsive backend (hung +goroutine, panic-recovery loop, port-bind failure with the process still +running) is detected and restarted instead of leaving the dev frontend +talking to a dead listener. Two consecutive missed health probes must trigger +a managed kill and restart of the unresponsive process. That same watcher boundary also owns backend-served demo coherence: `internal/api/frontend-modern/dist` changes must trigger a managed backend rebuild so the `go:embed` frontend on `:7655` cannot drift behind a freshly diff --git a/scripts/hot-dev.sh b/scripts/hot-dev.sh index 9f1ae21d8e..c2d5e976e0 100755 --- a/scripts/hot-dev.sh +++ b/scripts/hot-dev.sh @@ -450,12 +450,22 @@ if ! kill -0 "${BACKEND_PID}" 2>/dev/null; then fi # --- Backend Health Monitor --- -# Restarts Pulse if it dies unexpectedly (not from file watcher rebuild) -# IMPORTANT: Enforces single-instance - kills duplicates if found +# Restarts Pulse if it dies unexpectedly (not from file watcher rebuild), +# enforces single-instance, and detects the alive-but-unresponsive state +# (process exists but /api/health is not serving) which a process-only check +# blind-spots. Two consecutive 5s misses on /api/health -> kill and restart. log_info "Starting backend health monitor..." ( + UNHEALTHY_STREAK=0 + UNHEALTHY_THRESHOLD=2 + + backend_serving() { + curl -sf -o /dev/null --max-time 3 \ + "http://127.0.0.1:${PULSE_DEV_API_PORT:-7655}/api/health" 2>/dev/null + } + while true; do - sleep 10 + sleep 5 PULSE_COUNT=$(pgrep -f "^\./pulse$" 2>/dev/null | wc -l | tr -d ' ') if [[ "$PULSE_COUNT" -eq 0 ]]; then @@ -480,6 +490,7 @@ log_info "Starting backend health monitor..." else log_error "✗ Backend failed to auto-restart!" fi + UNHEALTHY_STREAK=0 elif [[ "$PULSE_COUNT" -gt 1 ]]; then log_error "⚠️ Multiple Pulse processes detected ($PULSE_COUNT), killing all and restarting..." pkill -9 -f "^\./pulse$" 2>/dev/null || true @@ -504,6 +515,38 @@ log_info "Starting backend health monitor..." else log_error "✗ Backend failed to restart after killing duplicates!" fi + UNHEALTHY_STREAK=0 + elif ! backend_serving; then + UNHEALTHY_STREAK=$((UNHEALTHY_STREAK + 1)) + log_warn "⚠️ Pulse alive but /api/health unresponsive (streak ${UNHEALTHY_STREAK}/${UNHEALTHY_THRESHOLD})" + if [[ "$UNHEALTHY_STREAK" -ge "$UNHEALTHY_THRESHOLD" ]]; then + log_error "⚠️ Killing unresponsive Pulse and restarting..." + pkill -9 -f "^\./pulse$" 2>/dev/null || true + sleep 2 + LOG_LEVEL="${LOG_LEVEL:-debug}" \ + FRONTEND_PORT="${PULSE_DEV_API_PORT:-7655}" \ + PORT="${PULSE_DEV_API_PORT:-7655}" \ + PULSE_DATA_DIR="${PULSE_DATA_DIR:-}" \ + PULSE_ENCRYPTION_KEY="${PULSE_ENCRYPTION_KEY:-}" \ + ALLOW_ADMIN_BYPASS="${ALLOW_ADMIN_BYPASS:-1}" \ + PULSE_DEV="${PULSE_DEV:-true}" \ + PULSE_AUTH_USER="${PULSE_AUTH_USER:-}" \ + PULSE_AUTH_PASS="${PULSE_AUTH_PASS:-}" \ + ALLOWED_ORIGINS="${ALLOWED_ORIGINS:-}" \ + LOG_FILE="/tmp/pulse-debug.log" \ + LOG_MAX_SIZE="50" \ + ./pulse /dev/null 2>&1 & + NEW_PID=$! + sleep 2 + if kill -0 "$NEW_PID" 2>/dev/null; then + log_info "✓ Backend restarted after unresponsive state (PID: $NEW_PID)" + else + log_error "✗ Backend failed to restart after unresponsive state!" + fi + UNHEALTHY_STREAK=0 + fi + else + UNHEALTHY_STREAK=0 fi done ) & diff --git a/scripts/tests/test-hot-dev-bg.sh b/scripts/tests/test-hot-dev-bg.sh index 7f337e8b3a..211d1a2827 100755 --- a/scripts/tests/test-hot-dev-bg.sh +++ b/scripts/tests/test-hot-dev-bg.sh @@ -667,6 +667,16 @@ test_hot_dev_script_ignores_test_only_backend_churn() { assert_contains "hot-dev watcher suppresses source churn during managed verification" "${output}" 'verify_lock_active' } +test_hot_dev_health_monitor_probes_api_health() { + local output + output="$(cat "${HOT_DEV}")" + + assert_contains "hot-dev health monitor declares an unhealthy streak threshold" "${output}" 'UNHEALTHY_THRESHOLD=2' + assert_contains "hot-dev health monitor probes /api/health on the dev backend port" "${output}" '"http://127.0.0.1:${PULSE_DEV_API_PORT:-7655}/api/health"' + assert_contains "hot-dev health monitor restarts on alive-but-unresponsive state" "${output}" 'elif ! backend_serving; then' + assert_contains "hot-dev health monitor kills unresponsive Pulse before restart" "${output}" 'Killing unresponsive Pulse and restarting' +} + test_hot_dev_script_marks_managed_rebuild_output_before_build() { local rebuild_block mark_line build_line rebuild_block="$( @@ -1161,6 +1171,7 @@ main() { test_makefile_routes_managed_runtime_through_npm test_hot_dev_script_advertises_foreground_escape_hatch test_hot_dev_script_ignores_test_only_backend_churn + test_hot_dev_health_monitor_probes_api_health test_hot_dev_bg_script_advertises_managed_entrypoint test_hot_dev_bg_usage_prefers_managed_wrappers test_integration_readme_uses_managed_backend_restart_wrapper