mirror of
https://github.com/openclaw/openclaw.git
synced 2026-10-04 10:10:01 +00:00
fix(e2e): clean interrupted docker harness runs
This commit is contained in:
Vincent Koc
parent
08ae0e6d29
commit
6bfd47af38
3 changed files with 376 additions and 6 deletions
|
|
@ -44,6 +44,68 @@ run_logged_print_heartbeat() {
|
|||
log_file="$(docker_e2e_run_log "$label")"
|
||||
"$@" >"$log_file" 2>&1 &
|
||||
local command_pid=$!
|
||||
local cleanup_done=0
|
||||
local previous_int_trap
|
||||
local previous_term_trap
|
||||
local previous_hup_trap
|
||||
previous_int_trap="$(trap -p INT || true)"
|
||||
previous_term_trap="$(trap -p TERM || true)"
|
||||
previous_hup_trap="$(trap -p HUP || true)"
|
||||
terminate_heartbeat_command() {
|
||||
kill -TERM "$command_pid" 2>/dev/null || true
|
||||
local grace_seconds="${OPENCLAW_DOCKER_E2E_HEARTBEAT_TERM_GRACE_SECONDS:-30}"
|
||||
if ! [[ "$grace_seconds" =~ ^[0-9]+$ ]] || [ "$grace_seconds" -lt 1 ]; then
|
||||
grace_seconds="30"
|
||||
else
|
||||
grace_seconds="$((10#$grace_seconds))"
|
||||
fi
|
||||
local wait_attempt
|
||||
for wait_attempt in $(seq 1 "$((grace_seconds * 10))"); do
|
||||
if ! kill -0 "$command_pid" 2>/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
/bin/sleep 0.1
|
||||
done
|
||||
kill -KILL "$command_pid" 2>/dev/null || true
|
||||
}
|
||||
restore_heartbeat_traps() {
|
||||
if [ -n "$previous_int_trap" ]; then
|
||||
eval "$previous_int_trap"
|
||||
else
|
||||
trap - INT
|
||||
fi
|
||||
if [ -n "$previous_term_trap" ]; then
|
||||
eval "$previous_term_trap"
|
||||
else
|
||||
trap - TERM
|
||||
fi
|
||||
if [ -n "$previous_hup_trap" ]; then
|
||||
eval "$previous_hup_trap"
|
||||
else
|
||||
trap - HUP
|
||||
fi
|
||||
}
|
||||
cleanup_heartbeat_command() {
|
||||
local cleanup_status="${1:-$?}"
|
||||
if [ "$cleanup_done" = "1" ]; then
|
||||
return "$cleanup_status"
|
||||
fi
|
||||
cleanup_done=1
|
||||
trap - INT TERM HUP
|
||||
if kill -0 "$command_pid" 2>/dev/null; then
|
||||
terminate_heartbeat_command
|
||||
wait "$command_pid" 2>/dev/null || true
|
||||
fi
|
||||
rm -f "$log_file"
|
||||
restore_heartbeat_traps
|
||||
if [ "$cleanup_status" -ge 128 ]; then
|
||||
exit "$cleanup_status"
|
||||
fi
|
||||
return "$cleanup_status"
|
||||
}
|
||||
trap 'cleanup_heartbeat_command 130' INT
|
||||
trap 'cleanup_heartbeat_command 143' TERM
|
||||
trap 'cleanup_heartbeat_command 129' HUP
|
||||
local started_at="$SECONDS"
|
||||
local next_heartbeat=$interval_seconds
|
||||
local status=0
|
||||
|
|
@ -65,7 +127,7 @@ run_logged_print_heartbeat() {
|
|||
status=$?
|
||||
set -e
|
||||
docker_e2e_print_log "$log_file"
|
||||
rm -f "$log_file"
|
||||
cleanup_heartbeat_command 0
|
||||
return "$status"
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -224,13 +224,115 @@ docker_e2e_run_with_harness() {
|
|||
local run_status=0
|
||||
local cid_dir
|
||||
local cidfile
|
||||
local docker_run_pid=""
|
||||
local harness_stdin_fd=""
|
||||
local cleanup_done=0
|
||||
local previous_int_trap
|
||||
local previous_term_trap
|
||||
local previous_hup_trap
|
||||
cid_dir="$(mktemp -d "${TMPDIR:-/tmp}/openclaw-docker-e2e-container.XXXXXX")"
|
||||
cidfile="$cid_dir/container.cid"
|
||||
docker_e2e_docker_run_cmd run --rm --cidfile "$cidfile" "${DOCKER_E2E_HARNESS_ARGS[@]}" "$@" ||
|
||||
run_status="$?"
|
||||
docker_e2e_cleanup_container_cidfile "$cidfile"
|
||||
rmdir "$cid_dir" 2>/dev/null || true
|
||||
docker_e2e_cleanup_package_mount_args
|
||||
previous_int_trap="$(trap -p INT || true)"
|
||||
previous_term_trap="$(trap -p TERM || true)"
|
||||
previous_hup_trap="$(trap -p HUP || true)"
|
||||
restore_harness_traps() {
|
||||
if [ -n "$previous_int_trap" ]; then
|
||||
eval "$previous_int_trap"
|
||||
else
|
||||
trap - INT
|
||||
fi
|
||||
if [ -n "$previous_term_trap" ]; then
|
||||
eval "$previous_term_trap"
|
||||
else
|
||||
trap - TERM
|
||||
fi
|
||||
if [ -n "$previous_hup_trap" ]; then
|
||||
eval "$previous_hup_trap"
|
||||
else
|
||||
trap - HUP
|
||||
fi
|
||||
}
|
||||
docker_e2e_harness_descendant_pids() {
|
||||
local parent_pid="$1"
|
||||
local child_pid
|
||||
for child_pid in $(pgrep -P "$parent_pid" 2>/dev/null || true); do
|
||||
docker_e2e_harness_descendant_pids "$child_pid"
|
||||
printf '%s\n' "$child_pid"
|
||||
done
|
||||
}
|
||||
terminate_harness_docker_run() {
|
||||
[ -n "$docker_run_pid" ] || return 0
|
||||
kill -0 "$docker_run_pid" 2>/dev/null || return 0
|
||||
local descendant_pids
|
||||
descendant_pids="$(docker_e2e_harness_descendant_pids "$docker_run_pid")"
|
||||
if [ -n "$descendant_pids" ]; then
|
||||
kill -TERM $descendant_pids 2>/dev/null || true
|
||||
fi
|
||||
kill -TERM "$docker_run_pid" 2>/dev/null || true
|
||||
local grace_seconds="${OPENCLAW_DOCKER_E2E_CONTAINER_TERM_GRACE_SECONDS:-10}"
|
||||
if ! [[ "$grace_seconds" =~ ^[0-9]+$ ]] || [ "$grace_seconds" -lt 1 ]; then
|
||||
grace_seconds="10"
|
||||
else
|
||||
grace_seconds="$((10#$grace_seconds))"
|
||||
fi
|
||||
local wait_attempt
|
||||
for wait_attempt in $(seq 1 "$((grace_seconds * 10))"); do
|
||||
if ! kill -0 "$docker_run_pid" 2>/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
/bin/sleep 0.1
|
||||
done
|
||||
descendant_pids="$(docker_e2e_harness_descendant_pids "$docker_run_pid")"
|
||||
if [ -n "$descendant_pids" ]; then
|
||||
kill -KILL $descendant_pids 2>/dev/null || true
|
||||
fi
|
||||
kill -KILL "$docker_run_pid" 2>/dev/null || true
|
||||
}
|
||||
cleanup_harness_run() {
|
||||
local cleanup_status="${1:-$?}"
|
||||
local exit_after_cleanup="${2:-0}"
|
||||
if [ "$cleanup_done" = "1" ]; then
|
||||
if [ "$exit_after_cleanup" = "1" ]; then
|
||||
exit "$cleanup_status"
|
||||
fi
|
||||
return "$cleanup_status"
|
||||
fi
|
||||
cleanup_done=1
|
||||
trap - INT TERM HUP
|
||||
terminate_harness_docker_run
|
||||
wait "$docker_run_pid" 2>/dev/null || true
|
||||
docker_e2e_cleanup_container_cidfile "$cidfile"
|
||||
rmdir "$cid_dir" 2>/dev/null || true
|
||||
docker_e2e_cleanup_package_mount_args
|
||||
if [ -n "$harness_stdin_fd" ]; then
|
||||
exec {harness_stdin_fd}<&-
|
||||
harness_stdin_fd=""
|
||||
fi
|
||||
restore_harness_traps
|
||||
if [ "$exit_after_cleanup" = "1" ]; then
|
||||
exit "$cleanup_status"
|
||||
fi
|
||||
return "$cleanup_status"
|
||||
}
|
||||
trap 'cleanup_harness_run 130 1' INT
|
||||
trap 'cleanup_harness_run 143 1' TERM
|
||||
trap 'cleanup_harness_run 129 1' HUP
|
||||
exec {harness_stdin_fd}<&0
|
||||
docker_e2e_docker_run_cmd run --rm --cidfile "$cidfile" "${DOCKER_E2E_HARNESS_ARGS[@]}" "$@" <&$harness_stdin_fd &
|
||||
docker_run_pid="$!"
|
||||
local had_errexit=0
|
||||
case "$-" in
|
||||
*e*)
|
||||
had_errexit=1
|
||||
;;
|
||||
esac
|
||||
set +e
|
||||
wait "$docker_run_pid"
|
||||
run_status="$?"
|
||||
if [ "$had_errexit" = "1" ]; then
|
||||
set -e
|
||||
fi
|
||||
cleanup_harness_run 0
|
||||
return "$run_status"
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1812,6 +1812,76 @@ test -f "$TMPDIR/docker-cmd-seen"
|
|||
}
|
||||
});
|
||||
|
||||
it("forwards harness stdin to backgrounded Docker runs", () => {
|
||||
const workDir = mkdtempSync(join(tmpdir(), "openclaw-docker-harness-stdin-"));
|
||||
|
||||
try {
|
||||
const rootDir = process.cwd();
|
||||
const script = `
|
||||
set -euo pipefail
|
||||
ROOT_DIR=${shellQuote(rootDir)}
|
||||
TMPDIR=${shellQuote(workDir)}
|
||||
export ROOT_DIR TMPDIR
|
||||
|
||||
mkdir -p "$TMPDIR/bin"
|
||||
cat >"$TMPDIR/bin/timeout" <<'SH'
|
||||
#!/usr/bin/env bash
|
||||
case "$1" in
|
||||
--kill-after=1s)
|
||||
exit 0
|
||||
;;
|
||||
--kill-after=30s)
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
"$@"
|
||||
SH
|
||||
chmod +x "$TMPDIR/bin/timeout"
|
||||
export PATH="$TMPDIR/bin:$PATH"
|
||||
|
||||
source "$ROOT_DIR/scripts/lib/docker-e2e-package.sh"
|
||||
|
||||
docker() {
|
||||
if [[ "$1" == "rm" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local cidfile=""
|
||||
local expect_cidfile=0
|
||||
local arg
|
||||
for arg in "$@"; do
|
||||
if [[ "$expect_cidfile" == "1" ]]; then
|
||||
cidfile="$arg"
|
||||
expect_cidfile=0
|
||||
continue
|
||||
fi
|
||||
if [[ "$arg" == "--cidfile" ]]; then
|
||||
expect_cidfile=1
|
||||
fi
|
||||
done
|
||||
|
||||
test -n "$cidfile"
|
||||
printf "container-stdin\\n" >"$cidfile"
|
||||
cat >"$TMPDIR/docker-stdin-seen"
|
||||
}
|
||||
export -f docker
|
||||
|
||||
docker_e2e_run_with_harness image-name bash -s <<'SH'
|
||||
printf "heredoc reached docker\\n"
|
||||
SH
|
||||
|
||||
grep -Fxq 'printf "heredoc reached docker\\n"' "$TMPDIR/docker-stdin-seen"
|
||||
`;
|
||||
|
||||
execFileSync("bash", ["-lc", script], { encoding: "utf8" });
|
||||
} finally {
|
||||
rmSync(workDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("cleans Codex npm plugin live package artifacts on every exit path", () => {
|
||||
const runner = readFileSync(CODEX_NPM_PLUGIN_LIVE_DOCKER_E2E_PATH, "utf8");
|
||||
|
||||
|
|
@ -1969,6 +2039,142 @@ output="$(run_logged_print_heartbeat plugins-run 1 bash -c 'printf "captured con
|
|||
}
|
||||
});
|
||||
|
||||
it("cleans the heartbeat command when the wrapper is terminated", () => {
|
||||
const workDir = mkdtempSync(join(tmpdir(), "openclaw-docker-e2e-log-term-cleanup-"));
|
||||
|
||||
try {
|
||||
const rootDir = process.cwd();
|
||||
const script = `
|
||||
set -euo pipefail
|
||||
ROOT_DIR=${shellQuote(rootDir)}
|
||||
TMPDIR=${shellQuote(workDir)}
|
||||
export ROOT_DIR TMPDIR
|
||||
|
||||
source "$ROOT_DIR/scripts/lib/docker-e2e-logs.sh"
|
||||
|
||||
command_pid_file="$TMPDIR/command.pid"
|
||||
(
|
||||
run_logged_print_heartbeat plugins-run 30 bash -c 'printf "%s" "$$" > "$1"; while true; do /bin/sleep 1; done' bash "$command_pid_file"
|
||||
) &
|
||||
wrapper_pid="$!"
|
||||
for _ in $(seq 1 100); do
|
||||
[ -s "$command_pid_file" ] && break
|
||||
/bin/sleep 0.1
|
||||
done
|
||||
if [ ! -s "$command_pid_file" ]; then
|
||||
kill -TERM "$wrapper_pid" 2>/dev/null || true
|
||||
echo "heartbeat command pid was not recorded" >&2
|
||||
exit 1
|
||||
fi
|
||||
command_pid="$(cat "$command_pid_file")"
|
||||
kill -TERM "$wrapper_pid"
|
||||
/bin/sleep 2
|
||||
for _ in $(seq 1 50); do
|
||||
if ! kill -0 "$command_pid" 2>/dev/null; then
|
||||
wait "$wrapper_pid" 2>/dev/null || true
|
||||
exit 0
|
||||
fi
|
||||
/bin/sleep 0.1
|
||||
done
|
||||
kill -TERM "$command_pid" 2>/dev/null || true
|
||||
kill -TERM "$wrapper_pid" 2>/dev/null || true
|
||||
echo "heartbeat command still alive after wrapper termination: $command_pid" >&2
|
||||
exit 1
|
||||
`;
|
||||
|
||||
execFileSync("bash", ["-lc", script], { encoding: "utf8" });
|
||||
} finally {
|
||||
rmSync(workDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("cleans harness containers when heartbeat-wrapped Docker runs are terminated", () => {
|
||||
const workDir = mkdtempSync(join(tmpdir(), "openclaw-docker-e2e-harness-term-cleanup-"));
|
||||
|
||||
try {
|
||||
const rootDir = process.cwd();
|
||||
const script = `
|
||||
set -euo pipefail
|
||||
ROOT_DIR=${shellQuote(rootDir)}
|
||||
TMPDIR=${shellQuote(workDir)}
|
||||
export ROOT_DIR TMPDIR
|
||||
|
||||
mkdir -p "$TMPDIR/bin"
|
||||
cat >"$TMPDIR/bin/timeout" <<'SH'
|
||||
#!/usr/bin/env bash
|
||||
case "$1" in
|
||||
--kill-after=1s)
|
||||
exit 0
|
||||
;;
|
||||
--kill-after=30s)
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
"$@"
|
||||
SH
|
||||
chmod +x "$TMPDIR/bin/timeout"
|
||||
export PATH="$TMPDIR/bin:$PATH"
|
||||
|
||||
source "$ROOT_DIR/scripts/lib/docker-e2e-package.sh"
|
||||
|
||||
docker() {
|
||||
if [[ "$1" == "rm" ]]; then
|
||||
shift
|
||||
test "$1" = "-f"
|
||||
shift
|
||||
printf "%s\\n" "$1" >>"$TMPDIR/docker-rm-seen"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local cidfile=""
|
||||
local expect_cidfile=0
|
||||
local arg
|
||||
for arg in "$@"; do
|
||||
if [[ "$expect_cidfile" == "1" ]]; then
|
||||
cidfile="$arg"
|
||||
expect_cidfile=0
|
||||
continue
|
||||
fi
|
||||
if [[ "$arg" == "--cidfile" ]]; then
|
||||
expect_cidfile=1
|
||||
fi
|
||||
done
|
||||
|
||||
test -n "$cidfile"
|
||||
printf "container-term\\n" >"$cidfile"
|
||||
printf "docker running\\n"
|
||||
while true; do /bin/sleep 10; done
|
||||
}
|
||||
export -f docker
|
||||
|
||||
(
|
||||
docker_e2e_run_logged_print_with_harness plugins-run image-name bash -lc true
|
||||
) &
|
||||
wrapper_pid="$!"
|
||||
for _ in $(seq 1 50); do
|
||||
[ -s "$TMPDIR/docker-rm-seen" ] && break
|
||||
/bin/sleep 0.1
|
||||
kill -0 "$wrapper_pid" 2>/dev/null || true
|
||||
done
|
||||
kill -TERM "$wrapper_pid" 2>/dev/null || true
|
||||
wait "$wrapper_pid" 2>/dev/null || true
|
||||
for _ in $(seq 1 50); do
|
||||
grep -qx "container-term" "$TMPDIR/docker-rm-seen" 2>/dev/null && break
|
||||
/bin/sleep 0.1
|
||||
done
|
||||
grep -qx "container-term" "$TMPDIR/docker-rm-seen"
|
||||
test -z "$(find "$TMPDIR" -maxdepth 1 -name 'openclaw-docker-e2e-container.*' -print)"
|
||||
`;
|
||||
|
||||
execFileSync("bash", ["-lc", script], { encoding: "utf8" });
|
||||
} finally {
|
||||
rmSync(workDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("does not delay fast successful Docker E2E log captures until the next heartbeat", () => {
|
||||
const workDir = mkdtempSync(join(tmpdir(), "openclaw-docker-e2e-log-fast-heartbeat-"));
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue