diff --git a/scripts/lib/docker-build.sh b/scripts/lib/docker-build.sh index 38230dd9241a..a2da8a7ddb61 100644 --- a/scripts/lib/docker-build.sh +++ b/scripts/lib/docker-build.sh @@ -5,6 +5,9 @@ DOCKER_BUILD_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" if ! declare -F run_logged >/dev/null 2>&1; then source "$DOCKER_BUILD_LIB_DIR/docker-e2e-logs.sh" fi +if ! declare -F docker_e2e_timeout_cmd >/dev/null 2>&1; then + source "$DOCKER_BUILD_LIB_DIR/docker-e2e-container.sh" +fi docker_build_on_missing_enabled() { case "${OPENCLAW_DOCKER_BUILD_ON_MISSING:-}" in @@ -61,6 +64,27 @@ docker_build_retry_count() { echo 2 } +docker_build_timeout_required() { + case "${OPENCLAW_DOCKER_BUILD_REQUIRE_TIMEOUT:-0}" in + 1 | true | TRUE | yes | YES) + return 0 + ;; + esac + return 1 +} + +docker_build_run_command() { + local timeout_value="$1" + shift + + if docker_e2e_timeout_bin >/dev/null 2>&1 || docker_build_timeout_required; then + docker_e2e_timeout_cmd "$timeout_value" "$@" + return + fi + + "$@" +} + docker_build_with_retries() { local label="$1" shift @@ -74,9 +98,10 @@ docker_build_with_retries() { command+=("$part") done < <(docker_build_command "$@") + local timeout_value="${OPENCLAW_DOCKER_BUILD_TIMEOUT:-3600s}" while true; do log_file="$(docker_e2e_run_log "$label")" - if "${command[@]}" >"$log_file" 2>&1; then + if docker_build_run_command "$timeout_value" "${command[@]}" >"$log_file" 2>&1; then rm -f "$log_file" return 0 fi @@ -103,5 +128,6 @@ docker_build_run() { local label="$1" shift - docker_build_with_retries "$label" "$@" + OPENCLAW_DOCKER_BUILD_REQUIRE_TIMEOUT="${OPENCLAW_DOCKER_BUILD_REQUIRE_TIMEOUT:-1}" \ + docker_build_with_retries "$label" "$@" } diff --git a/test/scripts/docker-build-helper.test.ts b/test/scripts/docker-build-helper.test.ts index ca16b871d81b..2371e4714dea 100644 --- a/test/scripts/docker-build-helper.test.ts +++ b/test/scripts/docker-build-helper.test.ts @@ -110,6 +110,9 @@ describe("docker build helper", () => { expect(helper).toContain("docker buildx build --load"); expect(helper).toContain("docker_build_transient_failure()"); expect(helper).toContain("OPENCLAW_DOCKER_BUILD_RETRIES"); + expect(helper).toContain("OPENCLAW_DOCKER_BUILD_TIMEOUT"); + expect(helper).toContain('docker_build_run_command "$timeout_value" "${command[@]}"'); + expect(helper).toContain("OPENCLAW_DOCKER_BUILD_REQUIRE_TIMEOUT"); expect(helper).toContain("frontend grpc server closed unexpectedly"); }); @@ -182,6 +185,182 @@ describe("docker build helper", () => { ); }); + it("wraps centralized Docker builds with the timeout helper", () => { + const workDir = mkdtempSync(join(tmpdir(), "openclaw-docker-build-timeout-")); + + try { + const binDir = join(workDir, "bin"); + mkdirSync(binDir); + writeFileSync( + join(binDir, "timeout"), + `#!/bin/bash +set -euo pipefail +if [[ "$1" = "--kill-after=1s" ]]; then + exit 0 +fi +printf '%s %s|%s\\n' "$1" "$2" "\${*:3}" >>"$TMPDIR/timeout-seen" +shift 2 +"$@" +`, + ); + chmodSync(join(binDir, "timeout"), 0o755); + writeFileSync( + join(binDir, "docker"), + `#!/bin/sh +printf "%s\\n" "$*" >>"$TMPDIR/docker-seen" +`, + ); + chmodSync(join(binDir, "docker"), 0o755); + const rootDir = process.cwd(); + const script = ` +set -euo pipefail +ROOT_DIR=${shellQuote(rootDir)} +TMPDIR=${shellQuote(workDir)} +export ROOT_DIR TMPDIR +export PATH="$TMPDIR/bin:$PATH" +export OPENCLAW_DOCKER_BUILD_TIMEOUT=17s + +source "$ROOT_DIR/scripts/lib/docker-build.sh" + +docker_build_run e2e-build -t demo-image . + +grep -q '^--kill-after=30s 17s|env DOCKER_BUILDKIT=1 docker build -t demo-image .$' "$TMPDIR/timeout-seen" +grep -q '^build -t demo-image .$' "$TMPDIR/docker-seen" +`; + + execFileSync("bash", ["-lc", script], { encoding: "utf8" }); + } finally { + rmSync(workDir, { recursive: true, force: true }); + } + }); + + it("fails centralized Docker builds fast when timeout is unavailable", () => { + const workDir = mkdtempSync(join(tmpdir(), "openclaw-docker-build-timeout-required-")); + + try { + mkdirSync(join(workDir, "bin")); + const rootDir = process.cwd(); + const script = ` +set -euo pipefail +ROOT_DIR=${shellQuote(rootDir)} +TMPDIR=${shellQuote(workDir)} +export ROOT_DIR TMPDIR +export PATH="$TMPDIR/bin" +export OPENCLAW_DOCKER_BUILD_TIMEOUT=19s + +dirname() { + /usr/bin/dirname "$@" +} + +grep() { + /usr/bin/grep "$@" +} + +cat() { + /bin/cat "$@" +} + +rm() { + /bin/rm "$@" +} + +mktemp() { + /usr/bin/mktemp "$@" +} + +docker() { + printf "%s\\n" "$*" >"$TMPDIR/docker-seen" +} +export -f dirname grep cat rm mktemp docker + +source "$ROOT_DIR/scripts/lib/docker-build.sh" + +set +e +docker_build_run e2e-build -t demo-image . >"$TMPDIR/stdout" 2>"$TMPDIR/stderr" +status="$?" +set -e + +stdout="$(<"$TMPDIR/stdout")" +[[ "$status" = "1" ]] +[[ "$stdout" = *"timeout command not found; cannot bound Docker command after 19s"* ]] +[[ ! -e "$TMPDIR/docker-seen" ]] +`; + + execFileSync("bash", ["-lc", script], { encoding: "utf8" }); + } finally { + rmSync(workDir, { recursive: true, force: true }); + } + }); + + it("keeps setup-style Docker builds compatible when timeout is unavailable", () => { + const workDir = mkdtempSync(join(tmpdir(), "openclaw-docker-build-timeout-optional-")); + + try { + const binDir = join(workDir, "bin"); + mkdirSync(binDir); + writeFileSync( + join(binDir, "env"), + `#!/bin/sh +while [ "$#" -gt 0 ]; do + case "$1" in + *=*) + shift + ;; + *) + break + ;; + esac +done +exec "$@" +`, + ); + chmodSync(join(binDir, "env"), 0o755); + writeFileSync( + join(binDir, "docker"), + `#!/bin/sh +printf "%s\\n" "$*" >"$TMPDIR/docker-seen" +`, + ); + chmodSync(join(binDir, "docker"), 0o755); + const rootDir = process.cwd(); + const script = ` +set -euo pipefail +ROOT_DIR=${shellQuote(rootDir)} +TMPDIR=${shellQuote(workDir)} +export ROOT_DIR TMPDIR +export PATH="$TMPDIR/bin" +export OPENCLAW_DOCKER_BUILD_TIMEOUT=23s + +dirname() { + /usr/bin/dirname "$@" +} + +grep() { + /usr/bin/grep "$@" +} + +rm() { + /bin/rm "$@" +} + +mktemp() { + /usr/bin/mktemp "$@" +} +export -f dirname grep rm mktemp + +source "$ROOT_DIR/scripts/lib/docker-build.sh" + +docker_build_exec -t setup-image . + +[[ "$(<"$TMPDIR/docker-seen")" = "build -t setup-image ." ]] +`; + + execFileSync("bash", ["-lc", script], { encoding: "utf8" }); + } finally { + rmSync(workDir, { recursive: true, force: true }); + } + }); + it("keeps reused Docker image probes behind the timeout-aware helper", () => { const workDir = mkdtempSync(join(tmpdir(), "openclaw-docker-image-reuse-timeout-"));