vrr/Pulse
2
0
Fork 0
mirror of https://github.com/rcourtman/Pulse.git synced 2026-10-19 02:28:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 13

Keep /api/health off the monitor write lock; tolerate load stalls in the dev watchdog

Browse source 
The liveness handler called SchedulerHealth(), which refreshes provider
instance caches under the monitor's write lock and snapshots queues and
breakers on every probe — it only needed the dead-letter count. Add
Monitor.DeadLetterCount() (queue-local mutex only) and use it.

The hot-dev watchdog killed the backend after 2 missed 3s probes; on a
dev box running race-detector test suites and builds that is routine
CPU starvation, and each kill destroyed any in-flight Assistant chat
stream (observed repeatedly during the chat-feel audit). Default the
miss threshold to 4 (~20s) — genuine wedges still die quickly.
This commit is contained in:
rcourtman 2026-06-11 15:24:00 +01:00
parent 232005a7df
commit b1114f4986
3 changed files with 24 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

7
internal/api/router.go
View file

@ -4468,7 +4468,12 @@ func (r *Router) handleHealth(w http.ResponseWriter, req *http.Request) {
monitorHealthy := r.monitor != nil
schedulerHealthy := false
if monitorHealthy {
schedulerHealthy = r.monitor.SchedulerHealth().DeadLetter.Count == 0
// Liveness probes are watchdog-polled and must stay minimal: the
// full SchedulerHealth refreshes provider caches under the monitor's
// write lock on every call, which this endpoint only needed for the
// dead-letter count. DeadLetterCount touches only the queue's own
// short-lived mutex.
schedulerHealthy = r.monitor.DeadLetterCount() == 0
}
statusCode := http.StatusOK

12
internal/monitoring/monitor.go
View file

@ -2451,6 +2451,18 @@ func emptySchedulerHealthResponse(enabled bool) SchedulerHealthResponse {
}
// SchedulerHealth returns a complete snapshot of scheduler health for API exposure.
// DeadLetterCount reports the number of dead-lettered poll tasks using only
// the queue's own short-lived mutex. Liveness probes (/api/health) use this
// instead of SchedulerHealth, which refreshes provider caches under the
// monitor's write lock on every call — far more work and lock surface than a
// watchdog-polled liveness check should pay for.
func (m *Monitor) DeadLetterCount() int {
if m == nil || m.deadLetterQueue == nil {
return 0
}
return m.deadLetterQueue.Size()
}
func (m *Monitor) SchedulerHealth() SchedulerHealthResponse {
response := emptySchedulerHealthResponse(m.config != nil && m.config.AdaptivePollingEnabled)

9
scripts/hot-dev.sh
View file

@ -23,8 +23,11 @@
# Allow automatic Patrol/discovery/alert AI in dev
# HOT_DEV_BACKEND_HEALTH_STARTUP_GRACE_SECONDS=180
# Backend /api/health grace after starts/restarts
# HOT_DEV_BACKEND_UNHEALTHY_THRESHOLD=2
# HOT_DEV_BACKEND_UNHEALTHY_THRESHOLD=4
# Consecutive failed /api/health probes before restart
# (default tolerates ~20s of load-induced stall; test
# suites and builds on the same machine routinely peg
# the CPU and a kill mid-AI-stream bricks that chat turn)
#
# Pro Features Mode:
# When pulse-enterprise repo exists and HOT_DEV_USE_PRO is not "false",
@ -519,11 +522,11 @@ fi
log_info "Starting backend health monitor..."
(
UNHEALTHY_STREAK=0
UNHEALTHY_THRESHOLD="${HOT_DEV_BACKEND_UNHEALTHY_THRESHOLD:-2}"
UNHEALTHY_THRESHOLD="${HOT_DEV_BACKEND_UNHEALTHY_THRESHOLD:-4}"
BACKEND_HEALTH_STARTUP_GRACE_SECONDS="${HOT_DEV_BACKEND_HEALTH_STARTUP_GRACE_SECONDS:-180}"
BACKEND_PROCESS_MISSING_GRACE_SECONDS="${HOT_DEV_BACKEND_PROCESS_MISSING_GRACE_SECONDS:-10}"
[[ "${UNHEALTHY_THRESHOLD}" =~ ^[1-9][0-9]*$ ]] || UNHEALTHY_THRESHOLD=2
[[ "${UNHEALTHY_THRESHOLD}" =~ ^[1-9][0-9]*$ ]] || UNHEALTHY_THRESHOLD=4
[[ "${BACKEND_HEALTH_STARTUP_GRACE_SECONDS}" =~ ^[0-9]+$ ]] || BACKEND_HEALTH_STARTUP_GRACE_SECONDS=180
[[ "${BACKEND_PROCESS_MISSING_GRACE_SECONDS}" =~ ^[0-9]+$ ]] || BACKEND_PROCESS_MISSING_GRACE_SECONDS=10