diff --git a/scripts/lib/docker-e2e-container.sh b/scripts/lib/docker-e2e-container.sh index 2772323b52a7..022ae7b4a7ac 100644 --- a/scripts/lib/docker-e2e-container.sh +++ b/scripts/lib/docker-e2e-container.sh @@ -3,6 +3,15 @@ # Shared helpers for Docker E2E scripts that keep a named container running # while polling readiness from the host. +DOCKER_E2E_CONTAINER_LIB_DIR="${BASH_SOURCE[0]}" +if [[ "$DOCKER_E2E_CONTAINER_LIB_DIR" == */* ]]; then + DOCKER_E2E_CONTAINER_LIB_DIR="${DOCKER_E2E_CONTAINER_LIB_DIR%/*}" +else + DOCKER_E2E_CONTAINER_LIB_DIR=. +fi +DOCKER_E2E_CONTAINER_LIB_DIR="$(cd "$DOCKER_E2E_CONTAINER_LIB_DIR" && pwd)" +source "$DOCKER_E2E_CONTAINER_LIB_DIR/docker-e2e-resource-diagnostics.sh" + docker_e2e_timeout_bin() { if command -v timeout >/dev/null 2>&1; then printf '%s\n' timeout @@ -142,11 +151,7 @@ docker_e2e_docker_cmd() { if [ "${1:-}" = "run" ]; then shift docker_e2e_docker_run_resource_args "$@" || return $? - if [ "${#DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" -gt 0 ]; then - docker_e2e_timeout_cmd "$timeout_value" docker run "${DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" "$@" - else - docker_e2e_timeout_cmd "$timeout_value" docker run "$@" - fi + docker_e2e_docker_run_with_resource_diagnostics "$timeout_value" "$@" return fi docker_e2e_timeout_cmd "$timeout_value" docker "$@" @@ -157,11 +162,7 @@ docker_e2e_docker_run_cmd() { if [ "${1:-}" = "run" ]; then shift docker_e2e_docker_run_resource_args "$@" || return $? - if [ "${#DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" -gt 0 ]; then - docker_e2e_timeout_cmd "$timeout_value" docker run "${DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" "$@" - else - docker_e2e_timeout_cmd "$timeout_value" docker run "$@" - fi + docker_e2e_docker_run_with_resource_diagnostics "$timeout_value" "$@" return fi docker_e2e_timeout_cmd "$timeout_value" docker "$@" diff --git a/scripts/lib/docker-e2e-package.sh b/scripts/lib/docker-e2e-package.sh index a46abc0113fc..c13ec0e6a22a 100644 --- a/scripts/lib/docker-e2e-package.sh +++ b/scripts/lib/docker-e2e-package.sh @@ -7,6 +7,10 @@ DOCKER_E2E_PACKAGE_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT_DIR="${ROOT_DIR:-$(cd "$DOCKER_E2E_PACKAGE_LIB_DIR/../.." && pwd)}" +if ! declare -F docker_e2e_docker_run_with_resource_diagnostics >/dev/null 2>&1; then + source "$DOCKER_E2E_PACKAGE_LIB_DIR/docker-e2e-resource-diagnostics.sh" +fi + if ! declare -F run_logged >/dev/null 2>&1; then source "$DOCKER_E2E_PACKAGE_LIB_DIR/docker-e2e-logs.sh" fi @@ -108,30 +112,10 @@ if ! declare -F docker_e2e_docker_run_resource_args >/dev/null 2>&1; then fi } fi -if ! declare -F docker_e2e_docker_run_cmd >/dev/null 2>&1; then - docker_e2e_docker_run_cmd() { - if [ "${1:-}" = "run" ]; then - shift - docker_e2e_docker_run_resource_args "$@" || return $? - if declare -F docker_e2e_timeout_cmd >/dev/null 2>&1; then - if [ "${#DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" -gt 0 ]; then - docker_e2e_timeout_cmd "${DOCKER_COMMAND_TIMEOUT:-${OPENCLAW_DOCKER_E2E_RUN_TIMEOUT:-3600s}}" docker run "${DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" "$@" - else - docker_e2e_timeout_cmd "${DOCKER_COMMAND_TIMEOUT:-${OPENCLAW_DOCKER_E2E_RUN_TIMEOUT:-3600s}}" docker run "$@" - fi - return - fi - if [ "${#DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" -gt 0 ]; then - set -- run "${DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" "$@" - else - set -- run "$@" - fi - fi - if declare -F docker_e2e_timeout_cmd >/dev/null 2>&1; then - docker_e2e_timeout_cmd "${DOCKER_COMMAND_TIMEOUT:-${OPENCLAW_DOCKER_E2E_RUN_TIMEOUT:-3600s}}" docker "$@" - return - fi - local timeout_value="${DOCKER_COMMAND_TIMEOUT:-${OPENCLAW_DOCKER_E2E_RUN_TIMEOUT:-3600s}}" +if ! declare -F docker_e2e_timeout_cmd >/dev/null 2>&1; then + docker_e2e_timeout_cmd() { + local timeout_value="$1" + shift local timeout_bin="" if command -v timeout >/dev/null 2>&1; then timeout_bin="timeout" @@ -140,9 +124,9 @@ if ! declare -F docker_e2e_docker_run_cmd >/dev/null 2>&1; then fi if [ -n "$timeout_bin" ]; then if "$timeout_bin" --kill-after=1s 1s true >/dev/null 2>&1; then - "$timeout_bin" --kill-after=30s "$timeout_value" docker "$@" + "$timeout_bin" --kill-after=30s "$timeout_value" "$@" else - "$timeout_bin" "$timeout_value" docker "$@" + "$timeout_bin" "$timeout_value" "$@" fi return fi @@ -150,6 +134,19 @@ if ! declare -F docker_e2e_docker_run_cmd >/dev/null 2>&1; then return 127 } fi +if ! declare -F docker_e2e_docker_run_cmd >/dev/null 2>&1; then + docker_e2e_docker_run_cmd() { + if [ "${1:-}" = "run" ]; then + shift + docker_e2e_docker_run_resource_args "$@" || return $? + docker_e2e_docker_run_with_resource_diagnostics \ + "${DOCKER_COMMAND_TIMEOUT:-${OPENCLAW_DOCKER_E2E_RUN_TIMEOUT:-3600s}}" \ + "$@" + return + fi + docker_e2e_timeout_cmd "${DOCKER_COMMAND_TIMEOUT:-${OPENCLAW_DOCKER_E2E_RUN_TIMEOUT:-3600s}}" docker "$@" + } +fi docker_e2e_abs_path() { local file="$1" diff --git a/scripts/lib/docker-e2e-resource-diagnostics.sh b/scripts/lib/docker-e2e-resource-diagnostics.sh new file mode 100644 index 000000000000..4547d802bde2 --- /dev/null +++ b/scripts/lib/docker-e2e-resource-diagnostics.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash + +docker_e2e_resource_limit_error_file() { + local status="$1" + local stderr_file="$2" + local line + [ "$status" = "125" ] || return 1 + + local text="" + while IFS= read -r line || [ -n "$line" ]; do + text="${text}${line} +" + done <"$stderr_file" + + case "$text" in + *"controller pids is not available"* | *"cgroup controller pids is not available"* | \ + *"NanoCPUs can not be set"* | *"CPU CFS scheduler"* | \ + *"cgroup is not mounted"* | *"cgroup not mounted"* | \ + *"resource limit not supported"* | *"resource limits not supported"*) + return 0 + ;; + esac + return 1 +} + +docker_e2e_resource_limit_stderr_file() { + local template="${TMPDIR:-/tmp}/openclaw-docker-resource-limits.XXXXXX" + if command -v mktemp >/dev/null 2>&1; then + mktemp "$template" + return + fi + if [ -x /usr/bin/mktemp ]; then + /usr/bin/mktemp "$template" + return + fi + echo "mktemp command not found; cannot capture Docker resource-limit diagnostics" >&2 + return 127 +} + +docker_e2e_tee_bin() { + if command -v tee >/dev/null 2>&1; then + command -v tee + return + fi + if [ -x /usr/bin/tee ]; then + printf '%s\n' /usr/bin/tee + return + fi + return 1 +} + +docker_e2e_tail_bin() { + if command -v tail >/dev/null 2>&1; then + command -v tail + return + fi + if [ -x /usr/bin/tail ]; then + printf '%s\n' /usr/bin/tail + return + fi + return 1 +} + +docker_e2e_remove_diagnostic_file() { + if command -v rm >/dev/null 2>&1; then + rm -f "$@" + return + fi + /bin/rm -f "$@" +} + +docker_e2e_print_resource_limit_error() { + echo "Docker E2E resource limits are incompatible with this Docker runtime. Fix its cgroup support or explicitly opt out with OPENCLAW_DOCKER_E2E_DISABLE_RESOURCE_LIMITS=1." >&2 +} + +docker_e2e_docker_run_with_resource_diagnostics() { + local timeout_value="$1" + shift + if [ "${#DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" -eq 0 ]; then + docker_e2e_timeout_cmd "$timeout_value" docker run "$@" + return + fi + + local stderr_file="" + if ! stderr_file="$(docker_e2e_resource_limit_stderr_file)"; then + docker_e2e_timeout_cmd \ + "$timeout_value" \ + docker run "${DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" "$@" + return + fi + local tee_bin="" + if ! tee_bin="$(docker_e2e_tee_bin)"; then + docker_e2e_remove_diagnostic_file "$stderr_file" + docker_e2e_timeout_cmd \ + "$timeout_value" \ + docker run "${DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" "$@" + return + fi + local tail_bin="" + if ! tail_bin="$(docker_e2e_tail_bin)"; then + docker_e2e_remove_diagnostic_file "$stderr_file" + docker_e2e_timeout_cmd \ + "$timeout_value" \ + docker run "${DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" "$@" + return + fi + local stderr_fifo="${stderr_file}.stderr.pipe" + local capture_fifo="${stderr_file}.capture.pipe" + local mkfifo_bin="" + if command -v mkfifo >/dev/null 2>&1; then + mkfifo_bin="$(command -v mkfifo)" + elif [ -x /usr/bin/mkfifo ]; then + mkfifo_bin=/usr/bin/mkfifo + fi + local fifo_status=1 + if [ -n "$mkfifo_bin" ]; then + local previous_umask="" + previous_umask="$(umask)" + umask 077 + "$mkfifo_bin" "$stderr_fifo" "$capture_fifo" && fifo_status=0 || fifo_status="$?" + umask "$previous_umask" + fi + if [ "$fifo_status" -ne 0 ]; then + docker_e2e_remove_diagnostic_file "$stderr_file" "$stderr_fifo" "$capture_fifo" + docker_e2e_timeout_cmd \ + "$timeout_value" \ + docker run "${DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" "$@" + return + fi + + "$tail_bin" -c 65536 <"$capture_fifo" >"$stderr_file" & + local tail_pid="$!" + "$tee_bin" "$capture_fifo" <"$stderr_fifo" >&2 & + local tee_pid="$!" + local run_status=0 + if docker_e2e_timeout_cmd \ + "$timeout_value" \ + docker run "${DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" "$@" \ + 2>"$stderr_fifo"; then + run_status=0 + else + run_status="$?" + fi + wait "$tee_pid" || true + wait "$tail_pid" || true + + if docker_e2e_resource_limit_error_file "$run_status" "$stderr_file"; then + docker_e2e_print_resource_limit_error + fi + docker_e2e_remove_diagnostic_file "$stderr_file" "$stderr_fifo" "$capture_fifo" + return "$run_status" +} diff --git a/test/scripts/docker-build-helper.test.ts b/test/scripts/docker-build-helper.test.ts index 61c49c98a2ec..9d118484625c 100644 --- a/test/scripts/docker-build-helper.test.ts +++ b/test/scripts/docker-build-helper.test.ts @@ -635,7 +635,7 @@ docker_build_run e2e-build -t demo-image . chmodSync(join(workDir, "runner.sh"), 0o755); const waitForFile = async (filePath: string) => { - for (let attempt = 0; attempt < 100; attempt += 1) { + for (let attempt = 0; attempt < 500; attempt += 1) { if (existsSync(filePath)) { return; } @@ -648,7 +648,7 @@ docker_build_run e2e-build -t demo-image . child.once("exit", (code, signal) => resolve({ code, signal })); }); const waitForDead = async (pid: number) => { - for (let attempt = 0; attempt < 100; attempt += 1) { + for (let attempt = 0; attempt < 500; attempt += 1) { try { process.kill(pid, 0); } catch { @@ -1316,6 +1316,150 @@ OPENCLAW_DOCKER_E2E_DISABLE_RESOURCE_LIMITS=1 docker_e2e_docker_cmd run demo } }); + it("explains how to opt out when Docker rejects default resource limits", () => { + const workDir = tempDirs.make("openclaw-docker-resource-diagnostic-"); + + try { + const rootDir = process.cwd(); + const script = ` +set -euo pipefail +ROOT_DIR=${shellQuote(rootDir)} +TMPDIR=${shellQuote(workDir)} +export ROOT_DIR TMPDIR +export OPENCLAW_DOCKER_E2E_AVAILABLE_CPUS=8 +unset OPENCLAW_DOCKER_E2E_DISABLE_RESOURCE_LIMITS +unset OPENCLAW_DOCKER_E2E_MEMORY OPENCLAW_DOCKER_E2E_CPUS OPENCLAW_DOCKER_E2E_PIDS_LIMIT + +docker() { + printf "%s\\n" "$*" >>"$TMPDIR/docker-seen" + echo "docker: Error response from daemon: NanoCPUs can not be set, as the cgroup is not mounted" >&2 + return 125 +} + +mkfifo() { + umask >"$TMPDIR/mkfifo-umask" + /usr/bin/mkfifo "$@" +} + +tail() { + printf "%s\\n" "$*" >"$TMPDIR/tail-seen" + /usr/bin/tail "$@" +} + +source "$ROOT_DIR/scripts/lib/docker-e2e-container.sh" +docker_e2e_timeout_cmd() { + shift + "$@" +} + +set +e +printf "before Docker\\n" >"$TMPDIR/stderr" +docker_e2e_docker_cmd run demo 2>>"$TMPDIR/stderr" +status="$?" +set -e + +stderr="$(<"$TMPDIR/stderr")" +[[ "$status" = "125" ]] +[[ "$stderr" = before\\ Docker* ]] +[[ "$stderr" = *"NanoCPUs can not be set"* ]] +[[ "$stderr" = *"Docker E2E resource limits are incompatible with this Docker runtime"* ]] +[[ "$stderr" = *"OPENCLAW_DOCKER_E2E_DISABLE_RESOURCE_LIMITS=1"* ]] +[[ "$(grep -c '^run ' "$TMPDIR/docker-seen")" = "1" ]] +[[ "$(<"$TMPDIR/tail-seen")" = "-c 65536" ]] +[[ "$(<"$TMPDIR/mkfifo-umask")" = "0077" ]] +`; + + execFileSync("bash", ["-lc", script], { encoding: "utf8" }); + } finally { + rmSync(workDir, { recursive: true, force: true }); + } + }); + + it("does not suggest resource opt-out for other Docker failures", () => { + const workDir = tempDirs.make("openclaw-docker-resource-unrelated-"); + + try { + const rootDir = process.cwd(); + const script = ` +set -euo pipefail +ROOT_DIR=${shellQuote(rootDir)} +TMPDIR=${shellQuote(workDir)} +export ROOT_DIR TMPDIR +export OPENCLAW_DOCKER_E2E_AVAILABLE_CPUS=8 +unset OPENCLAW_DOCKER_E2E_DISABLE_RESOURCE_LIMITS +unset OPENCLAW_DOCKER_E2E_MEMORY OPENCLAW_DOCKER_E2E_CPUS OPENCLAW_DOCKER_E2E_PIDS_LIMIT + +docker() { + printf "%s\\n" "$*" >>"$TMPDIR/docker-seen" + echo "docker: Error response from daemon: No such image: cgroup-helper" >&2 + return 125 +} + +source "$ROOT_DIR/scripts/lib/docker-e2e-container.sh" +docker_e2e_timeout_cmd() { + shift + "$@" +} + +set +e +docker_e2e_docker_cmd run demo 2>"$TMPDIR/stderr" +status="$?" +set -e + +stderr="$(<"$TMPDIR/stderr")" +[[ "$status" = "125" ]] +[[ "$stderr" = *"No such image: cgroup-helper"* ]] +[[ "$stderr" != *"OPENCLAW_DOCKER_E2E_DISABLE_RESOURCE_LIMITS"* ]] +[[ "$(grep -c '^run ' "$TMPDIR/docker-seen")" = "1" ]] +`; + + execFileSync("bash", ["-lc", script], { encoding: "utf8" }); + } finally { + rmSync(workDir, { recursive: true, force: true }); + } + }); + + it("runs Docker when resource diagnostic capture is unavailable", () => { + const workDir = tempDirs.make("openclaw-docker-resource-no-temp-"); + + try { + const rootDir = process.cwd(); + const missingTmpDir = join(workDir, "missing"); + const script = ` +set -euo pipefail +ROOT_DIR=${shellQuote(rootDir)} +TMPDIR=${shellQuote(missingTmpDir)} +export ROOT_DIR TMPDIR +export OPENCLAW_DOCKER_E2E_AVAILABLE_CPUS=8 +unset OPENCLAW_DOCKER_E2E_DISABLE_RESOURCE_LIMITS +unset OPENCLAW_DOCKER_E2E_MEMORY OPENCLAW_DOCKER_E2E_CPUS OPENCLAW_DOCKER_E2E_PIDS_LIMIT + +docker() { + printf "%s\\n" "$*" >>${shellQuote(join(workDir, "docker-seen"))} + return 7 +} + +source "$ROOT_DIR/scripts/lib/docker-e2e-container.sh" +docker_e2e_timeout_cmd() { + shift + "$@" +} + +set +e +docker_e2e_docker_cmd run demo 2>/dev/null +status="$?" +set -e + +[[ "$status" = "7" ]] +[[ "$(grep -c '^run ' ${shellQuote(join(workDir, "docker-seen"))})" = "1" ]] +`; + + execFileSync("bash", ["-lc", script], { encoding: "utf8" }); + } finally { + rmSync(workDir, { recursive: true, force: true }); + } + }); + it("rejects invalid Docker run pids limits before invoking docker", () => { const workDir = mkdtempSync(join(tmpdir(), "openclaw-docker-resource-pids-")); @@ -1660,6 +1804,59 @@ docker_e2e_docker_run_cmd run demo } }); + it("diagnoses rejected resource limits in the package-backed fallback", () => { + const workDir = tempDirs.make("openclaw-docker-package-diagnostic-"); + + try { + const rootDir = process.cwd(); + const script = ` +set -euo pipefail +ROOT_DIR=${shellQuote(rootDir)} +TMPDIR=${shellQuote(workDir)} +export ROOT_DIR TMPDIR +export OPENCLAW_DOCKER_E2E_AVAILABLE_CPUS=8 +unset OPENCLAW_DOCKER_E2E_DISABLE_RESOURCE_LIMITS +unset OPENCLAW_DOCKER_E2E_MEMORY OPENCLAW_DOCKER_E2E_CPUS OPENCLAW_DOCKER_E2E_PIDS_LIMIT + +timeout() { + if [[ "$1" = "--kill-after=1s" ]]; then + return 0 + fi + shift 2 + "$@" +} + +docker_e2e_docker_cmd() { + return 0 +} + +docker() { + printf "%s\\n" "$*" >>"$TMPDIR/docker-seen" + echo "OCI runtime create failed: crun: controller pids is not available" >&2 + return 125 +} + +source "$ROOT_DIR/scripts/lib/docker-e2e-package.sh" + +set +e +docker_e2e_docker_run_cmd run demo 2>"$TMPDIR/stderr" +status="$?" +set -e + +stderr="$(<"$TMPDIR/stderr")" +[[ "$status" = "125" ]] +[[ "$stderr" = *"controller pids is not available"* ]] +[[ "$stderr" = *"Docker E2E resource limits are incompatible with this Docker runtime"* ]] +[[ "$stderr" = *"OPENCLAW_DOCKER_E2E_DISABLE_RESOURCE_LIMITS=1"* ]] +[[ "$(grep -c '^run ' "$TMPDIR/docker-seen")" = "1" ]] +`; + + execFileSync("bash", ["-lc", script], { encoding: "utf8" }); + } finally { + rmSync(workDir, { recursive: true, force: true }); + } + }); + it("removes functional Docker build package inputs after the build", () => { const workDir = mkdtempSync(join(tmpdir(), "openclaw-docker-build-cleanup-"));