mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-19 02:28:59 +00:00
fix(agent): prevent duplicate QNAP watchdogs (#1617)
This commit is contained in:
courtmanr@gmail.com
parent
94c908419b
commit
69475117be
2 changed files with 243 additions and 6 deletions
|
|
@ -1432,6 +1432,7 @@ write_qnap_wrapper_script() {
|
|||
local runtime_binary="$2"
|
||||
local stored_binary="$3"
|
||||
local log_dir="$4"
|
||||
local state_dir="$5"
|
||||
local service_env_lines="$SHELL_EXPORT_LINES"
|
||||
|
||||
cat > "$wrapper_script" <<EOF
|
||||
|
|
@ -1440,6 +1441,10 @@ write_qnap_wrapper_script() {
|
|||
# Auto-generated by Pulse installer.
|
||||
|
||||
WATCHDOG_LOG="${log_dir}/${AGENT_NAME}-watchdog.log"
|
||||
WATCHDOG_PIDFILE="${state_dir}/${AGENT_NAME}.watchdog.pid"
|
||||
AGENT_PIDFILE="${state_dir}/${AGENT_NAME}.pid"
|
||||
LOCK_DIR="${state_dir}/${AGENT_NAME}.watchdog.lock"
|
||||
CURRENT_AGENT_PID=""
|
||||
|
||||
wait_for_file() {
|
||||
path="\$1"
|
||||
|
|
@ -1458,11 +1463,73 @@ trim_watchdog_log() {
|
|||
fi
|
||||
}
|
||||
|
||||
pid_is_live() {
|
||||
_pid="\$1"
|
||||
case "\$_pid" in ''|*[!0-9]*) return 1 ;; esac
|
||||
kill -0 "\$_pid" 2>/dev/null
|
||||
}
|
||||
|
||||
cleanup_watchdog() {
|
||||
trap - EXIT INT TERM HUP
|
||||
|
||||
if pid_is_live "\$CURRENT_AGENT_PID"; then
|
||||
kill "\$CURRENT_AGENT_PID" 2>/dev/null || true
|
||||
wait "\$CURRENT_AGENT_PID" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
if [ -f "\$AGENT_PIDFILE" ] && [ "\$(cat "\$AGENT_PIDFILE" 2>/dev/null)" = "\$CURRENT_AGENT_PID" ]; then
|
||||
rm -f "\$AGENT_PIDFILE"
|
||||
fi
|
||||
|
||||
if [ -f "\$WATCHDOG_PIDFILE" ] && [ "\$(cat "\$WATCHDOG_PIDFILE" 2>/dev/null)" = "\$\$" ]; then
|
||||
rm -f "\$WATCHDOG_PIDFILE"
|
||||
rmdir "\$LOCK_DIR" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
shutdown_watchdog() {
|
||||
cleanup_watchdog
|
||||
exit 0
|
||||
}
|
||||
|
||||
acquire_watchdog_lock() {
|
||||
_waited=0
|
||||
while ! mkdir "\$LOCK_DIR" 2>/dev/null; do
|
||||
_owner=\$(cat "\$WATCHDOG_PIDFILE" 2>/dev/null || true)
|
||||
if pid_is_live "\$_owner"; then
|
||||
echo "\$(date '+%Y-%m-%d %H:%M:%S') [watchdog] Another QNAP watchdog is already running (pid \$_owner); exiting." >> "\$WATCHDOG_LOG"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Give a new owner time to publish its PID before deciding the lock is
|
||||
# stale. This closes the small race between mkdir and the PID write.
|
||||
if [ "\$_waited" -lt 2 ]; then
|
||||
sleep 1
|
||||
_waited=\$((_waited + 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
# The owner no longer exists. Remove only the stale singleton metadata,
|
||||
# then race safely with any other starter for a fresh atomic mkdir.
|
||||
rm -f "\$WATCHDOG_PIDFILE"
|
||||
rmdir "\$LOCK_DIR" 2>/dev/null || true
|
||||
_waited=0
|
||||
done
|
||||
|
||||
echo "\$\$" > "\$WATCHDOG_PIDFILE"
|
||||
return 0
|
||||
}
|
||||
|
||||
mkdir -p "${state_dir}" "${log_dir}" 2>/dev/null || true
|
||||
if ! acquire_watchdog_lock; then
|
||||
exit 0
|
||||
fi
|
||||
trap cleanup_watchdog EXIT
|
||||
trap shutdown_watchdog INT TERM HUP
|
||||
|
||||
wait_for_file "${stored_binary}"
|
||||
|
||||
mkdir -p "${log_dir}" 2>/dev/null || true
|
||||
|
||||
# Kill any existing pulse-agent processes before refreshing the runtime copy.
|
||||
# The singleton owner may replace an orphan left by an older installer.
|
||||
pkill -x "pulse-agent" 2>/dev/null || true
|
||||
sleep 2
|
||||
|
||||
|
|
@ -1479,8 +1546,13 @@ while true; do
|
|||
echo "\$(date '+%Y-%m-%d %H:%M:%S') [watchdog] Starting pulse-agent (agent log: ${log_dir}/${AGENT_NAME}.log)..." >> "\$WATCHDOG_LOG"
|
||||
# The agent writes its own rotating log via --log-file; discard the stdout
|
||||
# mirror so nothing accumulates on the RAM-backed root filesystem.
|
||||
${runtime_binary} ${EXEC_ARGS} > /dev/null 2>> "\$WATCHDOG_LOG"
|
||||
${runtime_binary} ${EXEC_ARGS} > /dev/null 2>> "\$WATCHDOG_LOG" &
|
||||
CURRENT_AGENT_PID=\$!
|
||||
echo "\$CURRENT_AGENT_PID" > "\$AGENT_PIDFILE"
|
||||
wait "\$CURRENT_AGENT_PID"
|
||||
EXIT_CODE=\$?
|
||||
rm -f "\$AGENT_PIDFILE"
|
||||
CURRENT_AGENT_PID=""
|
||||
|
||||
echo "\$(date '+%Y-%m-%d %H:%M:%S') [watchdog] pulse-agent exited with code \$EXIT_CODE, restarting in \${RESTART_DELAY}s..." >> "\$WATCHDOG_LOG"
|
||||
sleep \$RESTART_DELAY
|
||||
|
|
@ -3855,7 +3927,7 @@ if [[ -f /sbin/getcfg ]] || [[ -f /etc/config/qpkg.conf ]]; then
|
|||
pkill -f "start-pulse-agent.sh" 2>/dev/null || true
|
||||
sleep 2
|
||||
|
||||
write_qnap_wrapper_script "$WRAPPER_SCRIPT" "$RUNTIME_BINARY" "$QNAP_STORED_BINARY" "$QNAP_LOG_DIR"
|
||||
write_qnap_wrapper_script "$WRAPPER_SCRIPT" "$RUNTIME_BINARY" "$QNAP_STORED_BINARY" "$QNAP_LOG_DIR" "$STATE_DIR"
|
||||
|
||||
AUTORUN_CONFIGURED=false
|
||||
if [[ -x /etc/init.d/init_disk.sh ]]; then
|
||||
|
|
|
|||
|
|
@ -11,9 +11,12 @@ import (
|
|||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestContainerAgentCompatibilityWrapperDefaultsToUnifiedHostAndDockerMonitoring(t *testing.T) {
|
||||
|
|
@ -1460,7 +1463,7 @@ func TestInstallSHUsesCanonicalQNAPBootstrapRenderer(t *testing.T) {
|
|||
`write_qnap_wrapper_script() {`,
|
||||
`append_qnap_autorun_block() {`,
|
||||
`select_platform_state_dir "${QNAP_VOL}/.pulse-agent"`,
|
||||
`write_qnap_wrapper_script "$WRAPPER_SCRIPT" "$RUNTIME_BINARY" "$QNAP_STORED_BINARY" "$QNAP_LOG_DIR"`,
|
||||
`write_qnap_wrapper_script "$WRAPPER_SCRIPT" "$RUNTIME_BINARY" "$QNAP_STORED_BINARY" "$QNAP_LOG_DIR" "$STATE_DIR"`,
|
||||
`append_qnap_autorun_block "$AUTORUN_PATH" "$WRAPPER_SCRIPT" "$STATE_DIR"`,
|
||||
`complete_installation_flow "$STATE_DIR" "Installation complete! Agent is running." "Upgrade complete! Agent is running." "tail -f ${AGENT_LOG_FILE}"`,
|
||||
}
|
||||
|
|
@ -1661,6 +1664,168 @@ func TestInstallSHWatchdogPathsUseRotatingAgentLog(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// TestInstallSHQNAPWatchdogIsSingleton pins the process-ownership half of issue
|
||||
// #1617. QNAP can launch the persistent wrapper both from autorun.sh and from an
|
||||
// install/upgrade, so the wrapper must elect one supervisor before killing or
|
||||
// starting an agent and must clean up only state that it owns.
|
||||
func TestInstallSHQNAPWatchdogIsSingleton(t *testing.T) {
|
||||
content, err := os.ReadFile(repoFile("scripts", "install.sh"))
|
||||
if err != nil {
|
||||
t.Fatalf("read install.sh: %v", err)
|
||||
}
|
||||
|
||||
script := string(content)
|
||||
required := []string{
|
||||
`WATCHDOG_PIDFILE="${state_dir}/${AGENT_NAME}.watchdog.pid"`,
|
||||
`AGENT_PIDFILE="${state_dir}/${AGENT_NAME}.pid"`,
|
||||
`LOCK_DIR="${state_dir}/${AGENT_NAME}.watchdog.lock"`,
|
||||
`while ! mkdir "\$LOCK_DIR" 2>/dev/null; do`,
|
||||
`if pid_is_live "\$_owner"; then`,
|
||||
`if ! acquire_watchdog_lock; then`,
|
||||
`[ "\$(cat "\$WATCHDOG_PIDFILE" 2>/dev/null)" = "\$\$" ]`,
|
||||
`trap cleanup_watchdog EXIT`,
|
||||
`trap shutdown_watchdog INT TERM HUP`,
|
||||
`CURRENT_AGENT_PID=\$!`,
|
||||
`echo "\$CURRENT_AGENT_PID" > "\$AGENT_PIDFILE"`,
|
||||
`wait "\$CURRENT_AGENT_PID"`,
|
||||
}
|
||||
for _, needle := range required {
|
||||
if !strings.Contains(script, needle) {
|
||||
t.Fatalf("install.sh missing QNAP watchdog singleton contract: %s", needle)
|
||||
}
|
||||
}
|
||||
|
||||
lockPos := strings.Index(script, `if ! acquire_watchdog_lock; then`)
|
||||
killPos := strings.Index(script, `pkill -x "pulse-agent" 2>/dev/null || true`)
|
||||
if lockPos < 0 || killPos < 0 || lockPos > killPos {
|
||||
t.Fatalf("QNAP wrapper must acquire singleton ownership before killing an existing agent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderedQNAPWatchdogRejectsASecondSupervisor(t *testing.T) {
|
||||
content, err := os.ReadFile(repoFile("scripts", "install.sh"))
|
||||
if err != nil {
|
||||
t.Fatalf("read install.sh: %v", err)
|
||||
}
|
||||
|
||||
script := string(content)
|
||||
start := strings.Index(script, "write_qnap_wrapper_script() {")
|
||||
if start < 0 {
|
||||
t.Fatal("install.sh missing QNAP wrapper renderer")
|
||||
}
|
||||
endOffset := strings.Index(script[start:], "\nappend_qnap_autorun_block() {")
|
||||
if endOffset < 0 {
|
||||
t.Fatal("could not isolate QNAP wrapper renderer")
|
||||
}
|
||||
renderer := script[start : start+endOffset]
|
||||
|
||||
tempDir := t.TempDir()
|
||||
stateDir := filepath.Join(tempDir, "state")
|
||||
logDir := filepath.Join(stateDir, "logs")
|
||||
wrapperPath := filepath.Join(stateDir, "start-pulse-agent.sh")
|
||||
storedPath := filepath.Join(stateDir, "stored-agent")
|
||||
runtimePath := filepath.Join(tempDir, "runtime", "pulse-agent")
|
||||
startsPath := filepath.Join(tempDir, "agent-starts")
|
||||
mockBinDir := filepath.Join(tempDir, "bin")
|
||||
|
||||
for _, dir := range []string{stateDir, mockBinDir} {
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatalf("create test directory: %v", err)
|
||||
}
|
||||
}
|
||||
agentScript := "#!/bin/sh\n" +
|
||||
"echo \"$$\" >> " + strconv.Quote(startsPath) + "\n" +
|
||||
"trap 'exit 0' INT TERM HUP\n" +
|
||||
"while :; do sleep 1; done\n"
|
||||
if err := os.WriteFile(storedPath, []byte(agentScript), 0o755); err != nil {
|
||||
t.Fatalf("write fake agent: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(mockBinDir, "pkill"), []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil {
|
||||
t.Fatalf("write fake pkill: %v", err)
|
||||
}
|
||||
|
||||
harness := renderer + `
|
||||
AGENT_NAME=pulse-agent
|
||||
SHELL_EXPORT_LINES=""
|
||||
EXEC_ARGS=""
|
||||
write_qnap_wrapper_script "$1" "$2" "$3" "$4" "$5"
|
||||
`
|
||||
render := exec.Command("bash", "-c", harness, "_", wrapperPath, runtimePath, storedPath, logDir, stateDir)
|
||||
if output, err := render.CombinedOutput(); err != nil {
|
||||
t.Fatalf("render QNAP wrapper: %v\n%s", err, output)
|
||||
}
|
||||
|
||||
env := []string{"PATH=" + mockBinDir + ":/usr/bin:/bin"}
|
||||
first := exec.Command("sh", wrapperPath)
|
||||
first.Env = env
|
||||
if err := first.Start(); err != nil {
|
||||
t.Fatalf("start first watchdog: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if first.Process != nil {
|
||||
_ = first.Process.Signal(syscall.SIGTERM)
|
||||
_, _ = first.Process.Wait()
|
||||
}
|
||||
})
|
||||
|
||||
watchdogPIDFile := filepath.Join(stateDir, "pulse-agent.watchdog.pid")
|
||||
agentPIDFile := filepath.Join(stateDir, "pulse-agent.pid")
|
||||
waitForFile := func(path string) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if _, err := os.Stat(path); err == nil {
|
||||
return
|
||||
}
|
||||
time.Sleep(25 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("timed out waiting for %s", path)
|
||||
}
|
||||
waitForFile(watchdogPIDFile)
|
||||
waitForFile(agentPIDFile)
|
||||
waitForFile(startsPath)
|
||||
|
||||
second := exec.Command("sh", wrapperPath)
|
||||
second.Env = env
|
||||
if output, err := second.CombinedOutput(); err != nil {
|
||||
t.Fatalf("second watchdog should exit cleanly: %v\n%s", err, output)
|
||||
}
|
||||
|
||||
starts, err := os.ReadFile(startsPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read fake-agent starts: %v", err)
|
||||
}
|
||||
if got := len(strings.Fields(string(starts))); got != 1 {
|
||||
t.Fatalf("expected one supervised agent after two watchdog launches, got %d", got)
|
||||
}
|
||||
|
||||
agentPIDBytes, err := os.ReadFile(agentPIDFile)
|
||||
if err != nil {
|
||||
t.Fatalf("read agent pid: %v", err)
|
||||
}
|
||||
agentPID, err := strconv.Atoi(strings.TrimSpace(string(agentPIDBytes)))
|
||||
if err != nil {
|
||||
t.Fatalf("parse agent pid: %v", err)
|
||||
}
|
||||
|
||||
if err := first.Process.Signal(syscall.SIGTERM); err != nil {
|
||||
t.Fatalf("stop first watchdog: %v", err)
|
||||
}
|
||||
if err := first.Wait(); err != nil {
|
||||
t.Fatalf("wait for first watchdog shutdown: %v", err)
|
||||
}
|
||||
first.Process = nil
|
||||
|
||||
if err := syscall.Kill(agentPID, 0); !errors.Is(err, syscall.ESRCH) {
|
||||
t.Fatalf("watchdog shutdown left agent pid %d alive (kill check: %v)", agentPID, err)
|
||||
}
|
||||
for _, path := range []string{watchdogPIDFile, agentPIDFile, filepath.Join(stateDir, "pulse-agent.watchdog.lock")} {
|
||||
if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("watchdog shutdown left singleton state %s (stat: %v)", path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstallSHUsesSharedServiceRenderers(t *testing.T) {
|
||||
content, err := os.ReadFile(repoFile("scripts", "install.sh"))
|
||||
if err != nil {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue