mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-13 00:34:43 +00:00
Probe /api/health in hot-dev backend monitor
The supervisor's backend health monitor in scripts/hot-dev.sh only checked whether a `./pulse` process existed via pgrep. If the Go binary hung, panicked into recovery without exiting, or failed to bind :7655 while the process stayed alive, the monitor saw a healthy process count and never restarted, leaving the dev frontend talking to a dead listener. Hit that twice in one session. Add an HTTP probe of /api/health on the dev port. After two consecutive 5s polls where the process exists but /api/health is unreachable, kill the process and restart the same way the dead/duplicate branches do. Cut the poll interval from 10s to 5s so the worst-case unresponsive window is ~10s instead of ~30s. Bring deployment-installability into line by naming alive-but-unresponsive recovery as part of the dev-runtime orchestration boundary, and cover the new branch with static-source assertions in scripts/tests/test-hot-dev-bg.sh.
This commit is contained in:
parent
e6b0d47bd6
commit
35429774b4
3 changed files with 65 additions and 3 deletions
|
|
@ -455,6 +455,14 @@ the managed local stack: `scripts/hot-dev.sh` may only rebuild the backend for
|
|||
runtime Go sources, not `*_test.go` churn, and it must suppress `pulse` binary
|
||||
change events produced by its own successful managed rebuilds, managed backend
|
||||
restarts, or startup build.
|
||||
That same boundary also owns backend-liveness recovery, not just process-
|
||||
existence. The managed health monitor in `scripts/hot-dev.sh` must probe
|
||||
`http://127.0.0.1:${PULSE_DEV_API_PORT}/api/health` in addition to checking
|
||||
that a `./pulse` process exists, so an alive-but-unresponsive backend (hung
|
||||
goroutine, panic-recovery loop, port-bind failure with the process still
|
||||
running) is detected and restarted instead of leaving the dev frontend
|
||||
talking to a dead listener. Two consecutive missed health probes must trigger
|
||||
a managed kill and restart of the unresponsive process.
|
||||
That same watcher boundary also owns backend-served demo coherence:
|
||||
`internal/api/frontend-modern/dist` changes must trigger a managed backend
|
||||
rebuild so the `go:embed` frontend on `:7655` cannot drift behind a freshly
|
||||
|
|
|
|||
|
|
@ -450,12 +450,22 @@ if ! kill -0 "${BACKEND_PID}" 2>/dev/null; then
|
|||
fi
|
||||
|
||||
# --- Backend Health Monitor ---
|
||||
# Restarts Pulse if it dies unexpectedly (not from file watcher rebuild)
|
||||
# IMPORTANT: Enforces single-instance - kills duplicates if found
|
||||
# Restarts Pulse if it dies unexpectedly (not from file watcher rebuild),
|
||||
# enforces single-instance, and detects the alive-but-unresponsive state
|
||||
# (process exists but /api/health is not serving) which a process-only check
|
||||
# blind-spots. Two consecutive 5s misses on /api/health -> kill and restart.
|
||||
log_info "Starting backend health monitor..."
|
||||
(
|
||||
UNHEALTHY_STREAK=0
|
||||
UNHEALTHY_THRESHOLD=2
|
||||
|
||||
backend_serving() {
|
||||
curl -sf -o /dev/null --max-time 3 \
|
||||
"http://127.0.0.1:${PULSE_DEV_API_PORT:-7655}/api/health" 2>/dev/null
|
||||
}
|
||||
|
||||
while true; do
|
||||
sleep 10
|
||||
sleep 5
|
||||
PULSE_COUNT=$(pgrep -f "^\./pulse$" 2>/dev/null | wc -l | tr -d ' ')
|
||||
|
||||
if [[ "$PULSE_COUNT" -eq 0 ]]; then
|
||||
|
|
@ -480,6 +490,7 @@ log_info "Starting backend health monitor..."
|
|||
else
|
||||
log_error "✗ Backend failed to auto-restart!"
|
||||
fi
|
||||
UNHEALTHY_STREAK=0
|
||||
elif [[ "$PULSE_COUNT" -gt 1 ]]; then
|
||||
log_error "⚠️ Multiple Pulse processes detected ($PULSE_COUNT), killing all and restarting..."
|
||||
pkill -9 -f "^\./pulse$" 2>/dev/null || true
|
||||
|
|
@ -504,6 +515,38 @@ log_info "Starting backend health monitor..."
|
|||
else
|
||||
log_error "✗ Backend failed to restart after killing duplicates!"
|
||||
fi
|
||||
UNHEALTHY_STREAK=0
|
||||
elif ! backend_serving; then
|
||||
UNHEALTHY_STREAK=$((UNHEALTHY_STREAK + 1))
|
||||
log_warn "⚠️ Pulse alive but /api/health unresponsive (streak ${UNHEALTHY_STREAK}/${UNHEALTHY_THRESHOLD})"
|
||||
if [[ "$UNHEALTHY_STREAK" -ge "$UNHEALTHY_THRESHOLD" ]]; then
|
||||
log_error "⚠️ Killing unresponsive Pulse and restarting..."
|
||||
pkill -9 -f "^\./pulse$" 2>/dev/null || true
|
||||
sleep 2
|
||||
LOG_LEVEL="${LOG_LEVEL:-debug}" \
|
||||
FRONTEND_PORT="${PULSE_DEV_API_PORT:-7655}" \
|
||||
PORT="${PULSE_DEV_API_PORT:-7655}" \
|
||||
PULSE_DATA_DIR="${PULSE_DATA_DIR:-}" \
|
||||
PULSE_ENCRYPTION_KEY="${PULSE_ENCRYPTION_KEY:-}" \
|
||||
ALLOW_ADMIN_BYPASS="${ALLOW_ADMIN_BYPASS:-1}" \
|
||||
PULSE_DEV="${PULSE_DEV:-true}" \
|
||||
PULSE_AUTH_USER="${PULSE_AUTH_USER:-}" \
|
||||
PULSE_AUTH_PASS="${PULSE_AUTH_PASS:-}" \
|
||||
ALLOWED_ORIGINS="${ALLOWED_ORIGINS:-}" \
|
||||
LOG_FILE="/tmp/pulse-debug.log" \
|
||||
LOG_MAX_SIZE="50" \
|
||||
./pulse </dev/null > /dev/null 2>&1 &
|
||||
NEW_PID=$!
|
||||
sleep 2
|
||||
if kill -0 "$NEW_PID" 2>/dev/null; then
|
||||
log_info "✓ Backend restarted after unresponsive state (PID: $NEW_PID)"
|
||||
else
|
||||
log_error "✗ Backend failed to restart after unresponsive state!"
|
||||
fi
|
||||
UNHEALTHY_STREAK=0
|
||||
fi
|
||||
else
|
||||
UNHEALTHY_STREAK=0
|
||||
fi
|
||||
done
|
||||
) &
|
||||
|
|
|
|||
|
|
@ -667,6 +667,16 @@ test_hot_dev_script_ignores_test_only_backend_churn() {
|
|||
assert_contains "hot-dev watcher suppresses source churn during managed verification" "${output}" 'verify_lock_active'
|
||||
}
|
||||
|
||||
test_hot_dev_health_monitor_probes_api_health() {
|
||||
local output
|
||||
output="$(cat "${HOT_DEV}")"
|
||||
|
||||
assert_contains "hot-dev health monitor declares an unhealthy streak threshold" "${output}" 'UNHEALTHY_THRESHOLD=2'
|
||||
assert_contains "hot-dev health monitor probes /api/health on the dev backend port" "${output}" '"http://127.0.0.1:${PULSE_DEV_API_PORT:-7655}/api/health"'
|
||||
assert_contains "hot-dev health monitor restarts on alive-but-unresponsive state" "${output}" 'elif ! backend_serving; then'
|
||||
assert_contains "hot-dev health monitor kills unresponsive Pulse before restart" "${output}" 'Killing unresponsive Pulse and restarting'
|
||||
}
|
||||
|
||||
test_hot_dev_script_marks_managed_rebuild_output_before_build() {
|
||||
local rebuild_block mark_line build_line
|
||||
rebuild_block="$(
|
||||
|
|
@ -1161,6 +1171,7 @@ main() {
|
|||
test_makefile_routes_managed_runtime_through_npm
|
||||
test_hot_dev_script_advertises_foreground_escape_hatch
|
||||
test_hot_dev_script_ignores_test_only_backend_churn
|
||||
test_hot_dev_health_monitor_probes_api_health
|
||||
test_hot_dev_bg_script_advertises_managed_entrypoint
|
||||
test_hot_dev_bg_usage_prefers_managed_wrappers
|
||||
test_integration_readme_uses_managed_backend_restart_wrapper
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue