fix(docker): explain unsupported E2E resource limits (#109587)

* fix(docker): diagnose unsupported E2E resource limits

* test(docker): track diagnostic temp dirs

* fix(docker): bound resource diagnostic capture

* fix(docker): restrict diagnostic FIFO access
This commit is contained in:
Peter Steinberger 2026-07-16 20:56:23 -07:00 • committed by GitHub
parent 6c075724fe
commit ff9b291673
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 385 additions and 38 deletions

View file

@ -3,6 +3,15 @@
# Shared helpers for Docker E2E scripts that keep a named container running
# while polling readiness from the host.
DOCKER_E2E_CONTAINER_LIB_DIR="${BASH_SOURCE[0]}"
if [[ "$DOCKER_E2E_CONTAINER_LIB_DIR" == */* ]]; then
DOCKER_E2E_CONTAINER_LIB_DIR="${DOCKER_E2E_CONTAINER_LIB_DIR%/*}"
else
DOCKER_E2E_CONTAINER_LIB_DIR=.
fi
DOCKER_E2E_CONTAINER_LIB_DIR="$(cd "$DOCKER_E2E_CONTAINER_LIB_DIR" && pwd)"
source "$DOCKER_E2E_CONTAINER_LIB_DIR/docker-e2e-resource-diagnostics.sh"
docker_e2e_timeout_bin() {
if command -v timeout >/dev/null 2>&1; then
printf '%s\n' timeout
@ -142,11 +151,7 @@ docker_e2e_docker_cmd() {
if [ "${1:-}" = "run" ]; then
shift
docker_e2e_docker_run_resource_args "$@" || return $?
if [ "${#DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" -gt 0 ]; then
docker_e2e_timeout_cmd "$timeout_value" docker run "${DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" "$@"
else
docker_e2e_timeout_cmd "$timeout_value" docker run "$@"
fi
docker_e2e_docker_run_with_resource_diagnostics "$timeout_value" "$@"
return
fi
docker_e2e_timeout_cmd "$timeout_value" docker "$@"
@ -157,11 +162,7 @@ docker_e2e_docker_run_cmd() {
if [ "${1:-}" = "run" ]; then
shift
docker_e2e_docker_run_resource_args "$@" || return $?
if [ "${#DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" -gt 0 ]; then
docker_e2e_timeout_cmd "$timeout_value" docker run "${DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" "$@"
else
docker_e2e_timeout_cmd "$timeout_value" docker run "$@"
fi
docker_e2e_docker_run_with_resource_diagnostics "$timeout_value" "$@"
return
fi
docker_e2e_timeout_cmd "$timeout_value" docker "$@"

View file

@ -7,6 +7,10 @@
DOCKER_E2E_PACKAGE_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT_DIR="${ROOT_DIR:-$(cd "$DOCKER_E2E_PACKAGE_LIB_DIR/../.." && pwd)}"
if ! declare -F docker_e2e_docker_run_with_resource_diagnostics >/dev/null 2>&1; then
source "$DOCKER_E2E_PACKAGE_LIB_DIR/docker-e2e-resource-diagnostics.sh"
fi
if ! declare -F run_logged >/dev/null 2>&1; then
source "$DOCKER_E2E_PACKAGE_LIB_DIR/docker-e2e-logs.sh"
fi
@ -108,30 +112,10 @@ if ! declare -F docker_e2e_docker_run_resource_args >/dev/null 2>&1; then
fi
}
fi
if ! declare -F docker_e2e_docker_run_cmd >/dev/null 2>&1; then
docker_e2e_docker_run_cmd() {
if [ "${1:-}" = "run" ]; then
shift
docker_e2e_docker_run_resource_args "$@" || return $?
if declare -F docker_e2e_timeout_cmd >/dev/null 2>&1; then
if [ "${#DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" -gt 0 ]; then
docker_e2e_timeout_cmd "${DOCKER_COMMAND_TIMEOUT:-${OPENCLAW_DOCKER_E2E_RUN_TIMEOUT:-3600s}}" docker run "${DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" "$@"
else
docker_e2e_timeout_cmd "${DOCKER_COMMAND_TIMEOUT:-${OPENCLAW_DOCKER_E2E_RUN_TIMEOUT:-3600s}}" docker run "$@"
fi
return
fi
if [ "${#DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" -gt 0 ]; then
set -- run "${DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" "$@"
else
set -- run "$@"
fi
fi
if declare -F docker_e2e_timeout_cmd >/dev/null 2>&1; then
docker_e2e_timeout_cmd "${DOCKER_COMMAND_TIMEOUT:-${OPENCLAW_DOCKER_E2E_RUN_TIMEOUT:-3600s}}" docker "$@"
return
fi
local timeout_value="${DOCKER_COMMAND_TIMEOUT:-${OPENCLAW_DOCKER_E2E_RUN_TIMEOUT:-3600s}}"
if ! declare -F docker_e2e_timeout_cmd >/dev/null 2>&1; then
docker_e2e_timeout_cmd() {
local timeout_value="$1"
shift
local timeout_bin=""
if command -v timeout >/dev/null 2>&1; then
timeout_bin="timeout"
@ -140,9 +124,9 @@ if ! declare -F docker_e2e_docker_run_cmd >/dev/null 2>&1; then
fi
if [ -n "$timeout_bin" ]; then
if "$timeout_bin" --kill-after=1s 1s true >/dev/null 2>&1; then
"$timeout_bin" --kill-after=30s "$timeout_value" docker "$@"
"$timeout_bin" --kill-after=30s "$timeout_value" "$@"
else
"$timeout_bin" "$timeout_value" docker "$@"
"$timeout_bin" "$timeout_value" "$@"
fi
return
fi
@ -150,6 +134,19 @@ if ! declare -F docker_e2e_docker_run_cmd >/dev/null 2>&1; then
return 127
}
fi
if ! declare -F docker_e2e_docker_run_cmd >/dev/null 2>&1; then
docker_e2e_docker_run_cmd() {
if [ "${1:-}" = "run" ]; then
shift
docker_e2e_docker_run_resource_args "$@" || return $?
docker_e2e_docker_run_with_resource_diagnostics \
"${DOCKER_COMMAND_TIMEOUT:-${OPENCLAW_DOCKER_E2E_RUN_TIMEOUT:-3600s}}" \
"$@"
return
fi
docker_e2e_timeout_cmd "${DOCKER_COMMAND_TIMEOUT:-${OPENCLAW_DOCKER_E2E_RUN_TIMEOUT:-3600s}}" docker "$@"
}
fi
docker_e2e_abs_path() {
local file="$1"

View file

@ -0,0 +1,152 @@
#!/usr/bin/env bash
docker_e2e_resource_limit_error_file() {
local status="$1"
local stderr_file="$2"
local line
[ "$status" = "125" ] || return 1
local text=""
while IFS= read -r line || [ -n "$line" ]; do
text="${text}${line}
"
done <"$stderr_file"
case "$text" in
*"controller pids is not available"* | *"cgroup controller pids is not available"* | \
*"NanoCPUs can not be set"* | *"CPU CFS scheduler"* | \
*"cgroup is not mounted"* | *"cgroup not mounted"* | \
*"resource limit not supported"* | *"resource limits not supported"*)
return 0
;;
esac
return 1
}
docker_e2e_resource_limit_stderr_file() {
local template="${TMPDIR:-/tmp}/openclaw-docker-resource-limits.XXXXXX"
if command -v mktemp >/dev/null 2>&1; then
mktemp "$template"
return
fi
if [ -x /usr/bin/mktemp ]; then
/usr/bin/mktemp "$template"
return
fi
echo "mktemp command not found; cannot capture Docker resource-limit diagnostics" >&2
return 127
}
docker_e2e_tee_bin() {
if command -v tee >/dev/null 2>&1; then
command -v tee
return
fi
if [ -x /usr/bin/tee ]; then
printf '%s\n' /usr/bin/tee
return
fi
return 1
}
docker_e2e_tail_bin() {
if command -v tail >/dev/null 2>&1; then
command -v tail
return
fi
if [ -x /usr/bin/tail ]; then
printf '%s\n' /usr/bin/tail
return
fi
return 1
}
docker_e2e_remove_diagnostic_file() {
if command -v rm >/dev/null 2>&1; then
rm -f "$@"
return
fi
/bin/rm -f "$@"
}
docker_e2e_print_resource_limit_error() {
echo "Docker E2E resource limits are incompatible with this Docker runtime. Fix its cgroup support or explicitly opt out with OPENCLAW_DOCKER_E2E_DISABLE_RESOURCE_LIMITS=1." >&2
}
docker_e2e_docker_run_with_resource_diagnostics() {
local timeout_value="$1"
shift
if [ "${#DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" -eq 0 ]; then
docker_e2e_timeout_cmd "$timeout_value" docker run "$@"
return
fi
local stderr_file=""
if ! stderr_file="$(docker_e2e_resource_limit_stderr_file)"; then
docker_e2e_timeout_cmd \
"$timeout_value" \
docker run "${DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" "$@"
return
fi
local tee_bin=""
if ! tee_bin="$(docker_e2e_tee_bin)"; then
docker_e2e_remove_diagnostic_file "$stderr_file"
docker_e2e_timeout_cmd \
"$timeout_value" \
docker run "${DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" "$@"
return
fi
local tail_bin=""
if ! tail_bin="$(docker_e2e_tail_bin)"; then
docker_e2e_remove_diagnostic_file "$stderr_file"
docker_e2e_timeout_cmd \
"$timeout_value" \
docker run "${DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" "$@"
return
fi
local stderr_fifo="${stderr_file}.stderr.pipe"
local capture_fifo="${stderr_file}.capture.pipe"
local mkfifo_bin=""
if command -v mkfifo >/dev/null 2>&1; then
mkfifo_bin="$(command -v mkfifo)"
elif [ -x /usr/bin/mkfifo ]; then
mkfifo_bin=/usr/bin/mkfifo
fi
local fifo_status=1
if [ -n "$mkfifo_bin" ]; then
local previous_umask=""
previous_umask="$(umask)"
umask 077
"$mkfifo_bin" "$stderr_fifo" "$capture_fifo" && fifo_status=0 || fifo_status="$?"
umask "$previous_umask"
fi
if [ "$fifo_status" -ne 0 ]; then
docker_e2e_remove_diagnostic_file "$stderr_file" "$stderr_fifo" "$capture_fifo"
docker_e2e_timeout_cmd \
"$timeout_value" \
docker run "${DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" "$@"
return
fi
"$tail_bin" -c 65536 <"$capture_fifo" >"$stderr_file" &
local tail_pid="$!"
"$tee_bin" "$capture_fifo" <"$stderr_fifo" >&2 &
local tee_pid="$!"
local run_status=0
if docker_e2e_timeout_cmd \
"$timeout_value" \
docker run "${DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" "$@" \
2>"$stderr_fifo"; then
run_status=0
else
run_status="$?"
fi
wait "$tee_pid" || true
wait "$tail_pid" || true
if docker_e2e_resource_limit_error_file "$run_status" "$stderr_file"; then
docker_e2e_print_resource_limit_error
fi
docker_e2e_remove_diagnostic_file "$stderr_file" "$stderr_fifo" "$capture_fifo"
return "$run_status"
}

View file

@ -635,7 +635,7 @@ docker_build_run e2e-build -t demo-image .
chmodSync(join(workDir, "runner.sh"), 0o755);
const waitForFile = async (filePath: string) => {
for (let attempt = 0; attempt < 100; attempt += 1) {
for (let attempt = 0; attempt < 500; attempt += 1) {
if (existsSync(filePath)) {
return;
}
@ -648,7 +648,7 @@ docker_build_run e2e-build -t demo-image .
child.once("exit", (code, signal) => resolve({ code, signal }));
});
const waitForDead = async (pid: number) => {
for (let attempt = 0; attempt < 100; attempt += 1) {
for (let attempt = 0; attempt < 500; attempt += 1) {
try {
process.kill(pid, 0);
} catch {
@ -1316,6 +1316,150 @@ OPENCLAW_DOCKER_E2E_DISABLE_RESOURCE_LIMITS=1 docker_e2e_docker_cmd run demo
}
});
it("explains how to opt out when Docker rejects default resource limits", () => {
const workDir = tempDirs.make("openclaw-docker-resource-diagnostic-");
try {
const rootDir = process.cwd();
const script = `
set -euo pipefail
ROOT_DIR=${shellQuote(rootDir)}
TMPDIR=${shellQuote(workDir)}
export ROOT_DIR TMPDIR
export OPENCLAW_DOCKER_E2E_AVAILABLE_CPUS=8
unset OPENCLAW_DOCKER_E2E_DISABLE_RESOURCE_LIMITS
unset OPENCLAW_DOCKER_E2E_MEMORY OPENCLAW_DOCKER_E2E_CPUS OPENCLAW_DOCKER_E2E_PIDS_LIMIT
docker() {
printf "%s\\n" "$*" >>"$TMPDIR/docker-seen"
echo "docker: Error response from daemon: NanoCPUs can not be set, as the cgroup is not mounted" >&2
return 125
}
mkfifo() {
umask >"$TMPDIR/mkfifo-umask"
/usr/bin/mkfifo "$@"
}
tail() {
printf "%s\\n" "$*" >"$TMPDIR/tail-seen"
/usr/bin/tail "$@"
}
source "$ROOT_DIR/scripts/lib/docker-e2e-container.sh"
docker_e2e_timeout_cmd() {
shift
"$@"
}
set +e
printf "before Docker\\n" >"$TMPDIR/stderr"
docker_e2e_docker_cmd run demo 2>>"$TMPDIR/stderr"
status="$?"
set -e
stderr="$(<"$TMPDIR/stderr")"
[[ "$status" = "125" ]]
[[ "$stderr" = before\\ Docker* ]]
[[ "$stderr" = *"NanoCPUs can not be set"* ]]
[[ "$stderr" = *"Docker E2E resource limits are incompatible with this Docker runtime"* ]]
[[ "$stderr" = *"OPENCLAW_DOCKER_E2E_DISABLE_RESOURCE_LIMITS=1"* ]]
[[ "$(grep -c '^run ' "$TMPDIR/docker-seen")" = "1" ]]
[[ "$(<"$TMPDIR/tail-seen")" = "-c 65536" ]]
[[ "$(<"$TMPDIR/mkfifo-umask")" = "0077" ]]
`;
execFileSync("bash", ["-lc", script], { encoding: "utf8" });
} finally {
rmSync(workDir, { recursive: true, force: true });
}
});
it("does not suggest resource opt-out for other Docker failures", () => {
const workDir = tempDirs.make("openclaw-docker-resource-unrelated-");
try {
const rootDir = process.cwd();
const script = `
set -euo pipefail
ROOT_DIR=${shellQuote(rootDir)}
TMPDIR=${shellQuote(workDir)}
export ROOT_DIR TMPDIR
export OPENCLAW_DOCKER_E2E_AVAILABLE_CPUS=8
unset OPENCLAW_DOCKER_E2E_DISABLE_RESOURCE_LIMITS
unset OPENCLAW_DOCKER_E2E_MEMORY OPENCLAW_DOCKER_E2E_CPUS OPENCLAW_DOCKER_E2E_PIDS_LIMIT
docker() {
printf "%s\\n" "$*" >>"$TMPDIR/docker-seen"
echo "docker: Error response from daemon: No such image: cgroup-helper" >&2
return 125
}
source "$ROOT_DIR/scripts/lib/docker-e2e-container.sh"
docker_e2e_timeout_cmd() {
shift
"$@"
}
set +e
docker_e2e_docker_cmd run demo 2>"$TMPDIR/stderr"
status="$?"
set -e
stderr="$(<"$TMPDIR/stderr")"
[[ "$status" = "125" ]]
[[ "$stderr" = *"No such image: cgroup-helper"* ]]
[[ "$stderr" != *"OPENCLAW_DOCKER_E2E_DISABLE_RESOURCE_LIMITS"* ]]
[[ "$(grep -c '^run ' "$TMPDIR/docker-seen")" = "1" ]]
`;
execFileSync("bash", ["-lc", script], { encoding: "utf8" });
} finally {
rmSync(workDir, { recursive: true, force: true });
}
});
it("runs Docker when resource diagnostic capture is unavailable", () => {
const workDir = tempDirs.make("openclaw-docker-resource-no-temp-");
try {
const rootDir = process.cwd();
const missingTmpDir = join(workDir, "missing");
const script = `
set -euo pipefail
ROOT_DIR=${shellQuote(rootDir)}
TMPDIR=${shellQuote(missingTmpDir)}
export ROOT_DIR TMPDIR
export OPENCLAW_DOCKER_E2E_AVAILABLE_CPUS=8
unset OPENCLAW_DOCKER_E2E_DISABLE_RESOURCE_LIMITS
unset OPENCLAW_DOCKER_E2E_MEMORY OPENCLAW_DOCKER_E2E_CPUS OPENCLAW_DOCKER_E2E_PIDS_LIMIT
docker() {
printf "%s\\n" "$*" >>${shellQuote(join(workDir, "docker-seen"))}
return 7
}
source "$ROOT_DIR/scripts/lib/docker-e2e-container.sh"
docker_e2e_timeout_cmd() {
shift
"$@"
}
set +e
docker_e2e_docker_cmd run demo 2>/dev/null
status="$?"
set -e
[[ "$status" = "7" ]]
[[ "$(grep -c '^run ' ${shellQuote(join(workDir, "docker-seen"))})" = "1" ]]
`;
execFileSync("bash", ["-lc", script], { encoding: "utf8" });
} finally {
rmSync(workDir, { recursive: true, force: true });
}
});
it("rejects invalid Docker run pids limits before invoking docker", () => {
const workDir = mkdtempSync(join(tmpdir(), "openclaw-docker-resource-pids-"));
@ -1660,6 +1804,59 @@ docker_e2e_docker_run_cmd run demo
}
});
it("diagnoses rejected resource limits in the package-backed fallback", () => {
const workDir = tempDirs.make("openclaw-docker-package-diagnostic-");
try {
const rootDir = process.cwd();
const script = `
set -euo pipefail
ROOT_DIR=${shellQuote(rootDir)}
TMPDIR=${shellQuote(workDir)}
export ROOT_DIR TMPDIR
export OPENCLAW_DOCKER_E2E_AVAILABLE_CPUS=8
unset OPENCLAW_DOCKER_E2E_DISABLE_RESOURCE_LIMITS
unset OPENCLAW_DOCKER_E2E_MEMORY OPENCLAW_DOCKER_E2E_CPUS OPENCLAW_DOCKER_E2E_PIDS_LIMIT
timeout() {
if [[ "$1" = "--kill-after=1s" ]]; then
return 0
fi
shift 2
"$@"
}
docker_e2e_docker_cmd() {
return 0
}
docker() {
printf "%s\\n" "$*" >>"$TMPDIR/docker-seen"
echo "OCI runtime create failed: crun: controller pids is not available" >&2
return 125
}
source "$ROOT_DIR/scripts/lib/docker-e2e-package.sh"
set +e
docker_e2e_docker_run_cmd run demo 2>"$TMPDIR/stderr"
status="$?"
set -e
stderr="$(<"$TMPDIR/stderr")"
[[ "$status" = "125" ]]
[[ "$stderr" = *"controller pids is not available"* ]]
[[ "$stderr" = *"Docker E2E resource limits are incompatible with this Docker runtime"* ]]
[[ "$stderr" = *"OPENCLAW_DOCKER_E2E_DISABLE_RESOURCE_LIMITS=1"* ]]
[[ "$(grep -c '^run ' "$TMPDIR/docker-seen")" = "1" ]]
`;
execFileSync("bash", ["-lc", script], { encoding: "utf8" });
} finally {
rmSync(workDir, { recursive: true, force: true });
}
});
it("removes functional Docker build package inputs after the build", () => {
const workDir = mkdtempSync(join(tmpdir(), "openclaw-docker-build-cleanup-"));