openclaw/scripts/lib/openclaw-e2e-instance.sh

#!/usr/bin/env bash
# Shared in-container lifecycle helpers for Docker/Bash E2E lanes.
OPENCLAW_E2E_INSTANCE_LIB_DIR="${BASH_SOURCE[0]}"
if [[ "$OPENCLAW_E2E_INSTANCE_LIB_DIR" == */* ]]; then
  OPENCLAW_E2E_INSTANCE_LIB_DIR="${OPENCLAW_E2E_INSTANCE_LIB_DIR%/*}"
else
  OPENCLAW_E2E_INSTANCE_LIB_DIR=.
fi
OPENCLAW_E2E_INSTANCE_LIB_DIR="$(cd "$OPENCLAW_E2E_INSTANCE_LIB_DIR" && pwd)"

openclaw_e2e_eval_test_state_from_b64() {
  local encoded="${1:?missing OpenClaw test-state script}"
  local decoded
  if ! decoded="$(printf '%s' "$encoded" | base64 -d)"; then
    echo "Invalid OpenClaw test-state base64 payload" >&2
    return 1
  fi
  if [ -z "${decoded//[[:space:]]/}" ]; then
    echo "OpenClaw test-state base64 payload decoded to an empty script" >&2
    return 1
  fi
  eval "$decoded"
}
openclaw_e2e_read_positive_int_env() {
  local name="${1:?missing environment variable name}"
  local fallback="${2:?missing fallback value}"
  local value="${!name-}"
  if [ -z "${!name+x}" ]; then
    value="$fallback"
  fi
  if [[ ! "$value" =~ ^[0-9]+$ ]] || (( 10#$value < 1 )); then
    echo "invalid $name: $value" >&2
    return 2
  fi
  printf '%s\n' "$value"
}
openclaw_e2e_read_nonnegative_int_env() {
  local name="${1:?missing environment variable name}"
  local fallback="${2:?missing fallback value}"
  local value="${!name-}"
  if [ -z "${!name+x}" ]; then
    value="$fallback"
  fi
  if [[ ! "$value" =~ ^[0-9]+$ ]]; then
    echo "invalid $name: $value" >&2
    return 2
  fi
  printf '%s\n' "$value"
}
openclaw_e2e_resolve_entrypoint() {
  local entry
  for entry in dist/index.mjs dist/index.js; do
    [ -f "$entry" ] && { printf '%s\n' "$entry"; return 0; }
  done
  echo "OpenClaw entrypoint not found under dist/" >&2
  return 1
}
openclaw_e2e_run_script_entrypoint() {
  local stem="${1:?missing OpenClaw E2E script stem}"
  shift
  if [ -f "$stem.mts" ]; then
    tsx "$stem.mts" "$@"
    return
  fi
  if [ -f "$stem.mjs" ]; then
    node "$stem.mjs" "$@"
    return
  fi
  echo "OpenClaw E2E script entrypoint not found: $stem.{mts,mjs}" >&2
  return 1
}
openclaw_e2e_package_root() {
  local prefix="${1:-}"
  if [ -n "$prefix" ]; then
    printf '%s/lib/node_modules/openclaw\n' "$prefix"
    return 0
  fi
  printf '%s/openclaw\n' "$(npm root -g)"
}
openclaw_e2e_package_entrypoint() {
  local root="${1:?missing package root}"
  local entry
  for entry in "$root/dist/index.mjs" "$root/dist/index.js"; do
    [ -f "$entry" ] && { printf '%s\n' "$entry"; return 0; }
  done
  echo "OpenClaw package entrypoint not found under $root/dist/" >&2
  return 1
}
openclaw_e2e_maybe_timeout() {
  local timeout_value="$1"
  shift
  if [ -z "$timeout_value" ] || [ "$timeout_value" = "0" ]; then
    "$@"
    return
  fi
  local timeout_bin=""
  if command -v timeout >/dev/null 2>&1; then
    timeout_bin="timeout"
  elif command -v gtimeout >/dev/null 2>&1; then
    timeout_bin="gtimeout"
  fi
  if [ -z "$timeout_bin" ]; then
    if command -v node >/dev/null 2>&1; then
      echo "timeout command not found; using Node watchdog for OpenClaw E2E command timeout $timeout_value" >&2
      if [[ "$1" != */* ]]; then
        local resolved_command
        resolved_command="$(command -v "$1" 2>/dev/null || true)"
        if [ -n "$resolved_command" ]; then
          set -- "$resolved_command" "${@:2}"
        fi
      fi
      # Keep stdin attached to the child instead of consuming it as watchdog source.
      node "$OPENCLAW_E2E_INSTANCE_LIB_DIR/docker-e2e-watchdog.mjs" e2e "$timeout_value" "$@"
      return
    fi
    echo "timeout command not found and Node is unavailable; cannot bound OpenClaw E2E command after $timeout_value" >&2
    return 127
  fi
  if "$timeout_bin" --kill-after=1s 1s true >/dev/null 2>&1; then
    "$timeout_bin" --kill-after=30s "$timeout_value" "$@"
  else
    "$timeout_bin" "$timeout_value" "$@"
  fi
}
openclaw_e2e_print_log() {
  local path="$1"
  local max_bytes max_lines redactor_module
  max_bytes="$(openclaw_e2e_read_nonnegative_int_env OPENCLAW_E2E_LOG_TAIL_BYTES 262144)" || return $?
  max_lines="$(openclaw_e2e_read_nonnegative_int_env OPENCLAW_E2E_LOG_TAIL_LINES 120)" || return $?
  [ -f "$path" ] || return 0
  echo "--- $path ---"
  [ -s "$path" ] || return 0
  redactor_module="${OPENCLAW_E2E_REDACTOR_MODULE:-$(openclaw_e2e_package_root)/dist/plugin-sdk/logging-core.js}"
  [ -f "$redactor_module" ] || redactor_module="$PWD/dist/plugin-sdk/logging-core.js"
  if [ ! -f "$redactor_module" ]; then
    echo "[failure log omitted: canonical redactor unavailable]"
    return 0
  fi
  if ! { tail -c "$max_bytes" "$path" 2>/dev/null | tail -n "$max_lines" || tail -n "$max_lines" "$path" || true; } | \
    node --input-type=module -e '
      import { text } from "node:stream/consumers";
      import { pathToFileURL } from "node:url";
      const { redactSensitiveText } = await import(pathToFileURL(process.argv[1]).href);
      process.stdout.write(redactSensitiveText(await text(process.stdin), { mode: "tools" }));
    ' "$redactor_module"; then
    echo "[failure log omitted: canonical redaction failed]"
  fi
}
openclaw_e2e_enable_failure_diagnostics() {
  # Keep diagnostics outside command log redirections, including inherited traps.
  # Scenarios reserve fd 4 and provide dump_debug_logs before enabling this handler.
  exec 4>&2
  set -E
  trap 'status=$?; dump_debug_logs "$status" "$BASH_COMMAND" >&4 2>&4; exit "$status"' ERR
}
openclaw_e2e_install_package() {
  local log_file="$1"
  local label="${2:-mounted OpenClaw package}"
  local prefix="${3:-}"
  local package_tgz="${OPENCLAW_CURRENT_PACKAGE_TGZ:?missing OPENCLAW_CURRENT_PACKAGE_TGZ}"
  local timeout_value="${OPENCLAW_E2E_NPM_INSTALL_TIMEOUT:-600s}"
  local args=(-g)
  if [ -n "$prefix" ]; then
    args+=("--prefix" "$prefix")
  fi
  echo "Installing $label..."
  if openclaw_e2e_maybe_timeout "$timeout_value" npm install "${args[@]}" "$package_tgz" --no-fund --no-audit >"$log_file" 2>&1; then
    echo "Installed $label."
    return 0
  else
    local install_status=$?
    if [ "$install_status" -eq 124 ] || [ "$install_status" -eq 137 ]; then
      echo "npm install timed out after $timeout_value for $label" >&2
    fi
    echo "npm install failed for $label" >&2
    if [ -f "$log_file" ]; then
      openclaw_e2e_print_log "$log_file" >&2
    fi
    return "$install_status"
  fi
}
openclaw_e2e_find_dep_package() {
  local dep_path="$1"
  shift
  find "$@" -path "*/node_modules/$dep_path/package.json" -print -quit 2>/dev/null || true
}
openclaw_e2e_assert_dep_absent() {
  local dep_path="$1"
  shift
  if [ -n "$(openclaw_e2e_find_dep_package "$dep_path" "$@")" ]; then
    echo "$dep_path should not be installed" >&2
    find "$@" -path "*/node_modules/$dep_path/package.json" -print 2>/dev/null >&2 || true
    exit 1
  fi
}
openclaw_e2e_assert_dep_present() {
  local dep_path="$1"
  shift
  if [ -n "$(openclaw_e2e_find_dep_package "$dep_path" "$@")" ]; then
    return 0
  fi
  echo "$dep_path was not installed on demand" >&2
  find "$@" -maxdepth 6 -type d -name node_modules -print 2>/dev/null >&2 || true
  exit 1
}
openclaw_e2e_write_state_env() {
  local target="${1:-/tmp/openclaw-test-state-env}"
  {
    printf 'export HOME=%q\n' "$HOME"
    printf 'export OPENCLAW_HOME=%q\n' "$OPENCLAW_HOME"
    printf 'export OPENCLAW_STATE_DIR=%q\n' "$OPENCLAW_STATE_DIR"
    printf 'export OPENCLAW_CONFIG_PATH=%q\n' "$OPENCLAW_CONFIG_PATH"
    printf 'export OPENCLAW_AGENT_DIR=%q\n' "${OPENCLAW_AGENT_DIR-}"
  } >"$target"
}
openclaw_e2e_install_trash_shim() {
  local shim_dir="${OPENCLAW_E2E_BIN_DIR:-}"
  if [ -z "$shim_dir" ]; then
    if [ -n "${OPENCLAW_STATE_DIR:-}" ]; then
      shim_dir="$OPENCLAW_STATE_DIR/e2e-bin"
    else
      shim_dir="$(mktemp -d "${TMPDIR:-/tmp}/openclaw-bin.XXXXXX")"
    fi
    OPENCLAW_E2E_BIN_DIR="$shim_dir"
    export OPENCLAW_E2E_BIN_DIR
  fi
  case ":$PATH:" in
    *":$shim_dir:"*) ;;
    *) export PATH="$shim_dir:$PATH" ;;
  esac
  mkdir -p "$shim_dir"
  cat >"$shim_dir/trash" <<'TRASH'
#!/usr/bin/env bash
set -euo pipefail
trash_dir="$HOME/.Trash"
mkdir -p "$trash_dir"
for target in "$@"; do
  [ -e "$target" ] || continue
  base="$(basename "$target")"
  dest="$trash_dir/$base"
  [ -e "$dest" ] && dest="$trash_dir/${base}-$(date +%s)-$$"
  mv "$target" "$dest"
done
TRASH
  chmod +x "$shim_dir/trash"
}
openclaw_e2e_run_script_with_pty() {
  local command="$1"
  local log_path="$2"
  local timeout_value="${OPENCLAW_E2E_COMMAND_TIMEOUT:-300s}"
  if script --version >/dev/null 2>&1; then
    openclaw_e2e_maybe_timeout "$timeout_value" script -q -f -c "$command" "$log_path"
  elif node -e 'import("@lydell/node-pty")' >/dev/null 2>&1; then
    openclaw_e2e_maybe_timeout "$timeout_value" node scripts/e2e/lib/run-with-pty.mjs "$log_path" /bin/bash -lc "$command"
  else
    openclaw_e2e_maybe_timeout "$timeout_value" script -q -F "$log_path" /bin/bash -lc "$command"
  fi
}
openclaw_e2e_start_tracked_process() {
  local log_path="${1:?missing OpenClaw E2E process log path}"
  shift
  if command -v setsid >/dev/null 2>&1; then
    setsid "$@" >"$log_path" 2>&1 &
    printf '%s\n' "$!"
    return
  fi
  node --input-type=module - "$log_path" "$@" <<'NODE'
import { closeSync, openSync } from "node:fs";
import { spawn } from "node:child_process";

const [logPath, command, ...args] = process.argv.slice(2);
if (!command) {
  console.error("missing command for OpenClaw E2E tracked process");
  process.exit(1);
}
const logFd = openSync(logPath, "a");
const child = spawn(command, args, {
  detached: process.platform !== "win32",
  env: process.env,
  stdio: ["ignore", logFd, logFd],
});
closeSync(logFd);
child.unref();
console.log(child.pid);
NODE
}
openclaw_e2e_signal_process() {
  local pid="${1:-}" signal="${2:-TERM}"
  [ -n "$pid" ] || return 0
  if kill -0 -- "-$pid" >/dev/null 2>&1; then
    kill "-$signal" -- "-$pid" >/dev/null 2>&1 || true
    return 0
  fi
  kill "-$signal" "$pid" >/dev/null 2>&1 || true
}
openclaw_e2e_process_alive() {
  local pid="${1:-}"
  [ -n "$pid" ] || return 1
  kill -0 "$pid" >/dev/null 2>&1 || kill -0 -- "-$pid" >/dev/null 2>&1
}
openclaw_e2e_stop_process() {
  local pid="${1:-}" _
  [ -n "$pid" ] || return 0
  openclaw_e2e_signal_process "$pid" TERM
  for _ in $(seq 1 40); do
    ! openclaw_e2e_process_alive "$pid" && { wait "$pid" >/dev/null 2>&1 || true; return 0; }
    sleep 0.25
  done
  openclaw_e2e_signal_process "$pid" KILL
  wait "$pid" >/dev/null 2>&1 || true
}
openclaw_e2e_terminate_gateways() {
  openclaw_e2e_stop_process "${1:-}"
}
openclaw_e2e_start_mock_openai() { openclaw_e2e_start_tracked_process "$2" env "MOCK_PORT=$1" node scripts/e2e/mock-openai-server.mjs; }
openclaw_e2e_wait_mock_openai() {
  # Port 0 uses the supplied process/log receipt and prints its allocated port.
  local port="$1" attempts="${2:-80}" timeout_ms="${3:-400}" _
  local base_url="${4:-}" pid="${5:-}" log="${6:-}" ready_port="$port" line
  if [ "$port" = "0" ] && { [ -z "$pid" ] || [ -z "$log" ] || [ -n "$base_url" ]; }; then
    echo "Allocated mock OpenAI readiness requires its process and log, without a base URL override." >&2
    return 2
  fi
  for _ in $(seq 1 "$attempts"); do
    if [ "$port" = "0" ]; then
      # Only the actual listener publishes this complete line. An unrelated
      # server's /health response cannot attest to ownership of the mock port.
      openclaw_e2e_process_alive "$pid" || break
      ready_port=""
      while IFS= read -r line; do
        if [[ "$line" =~ ^mock-openai\ listening\ on\ ([1-9][0-9]{0,4})$ ]] && [ "${BASH_REMATCH[1]}" -le 65535 ]; then
          ready_port="${BASH_REMATCH[1]}"
          break
        fi
      done < <(head -c 65536 "$log" 2>/dev/null)
    fi
    if [ -n "$ready_port" ] && openclaw_e2e_probe_http "${base_url:-http://127.0.0.1:${ready_port}}/health" ok "$timeout_ms"; then
      if [ "$port" = "0" ]; then
        openclaw_e2e_process_alive "$pid" || break
        printf '%s\n' "$ready_port"
      fi
      return 0
    fi
    sleep 0.1
  done
  if [ "$port" = "0" ]; then
    openclaw_e2e_print_log "$log" >&2
    echo "Mock OpenAI did not publish a live listener." >&2
    return 1
  fi
  openclaw_e2e_probe_http "${base_url:-http://127.0.0.1:${port}}/health" ok "$timeout_ms"
}
openclaw_e2e_start_gateway() { openclaw_e2e_start_tracked_process "$3" node "$1" gateway --port "$2" --bind loopback --allow-unconfigured; }
openclaw_e2e_exec_gateway() { exec node "$1" gateway --port "$2" --bind "${3:-loopback}" --allow-unconfigured >"$4" 2>&1; }
openclaw_e2e_gateway_log_port_from_text() {
  sed -nE 's/.*(127\.0\.0\.1|localhost):([0-9]+).*/\2/p' | tail -n 1
}
openclaw_e2e_wait_gateway_ready() {
  local pid="$1" log="$2" attempts="${3:-300}" ready_port="${4:-}" readiness_mode="${5:-strict}" _ saw_ready_log=false
  local ready_scan_offset=0 ready_scan_carry="" ready_scan_carry_chars=256
  local ready_log_pattern='\[gateway\] ready'
  # Published baselines logged their listener before the modern ready marker existed.
  if [ "$readiness_mode" = "legacy-ready-log-ok" ]; then
    ready_log_pattern='\[gateway\] (ready|listening on)'
  fi
  for _ in $(seq 1 "$attempts"); do
    ! kill -0 "$pid" >/dev/null 2>&1 && {
      echo "Gateway exited before becoming ready"
      wait "$pid" || true
      tail -n 120 "$log" 2>/dev/null || true
      return 1
    }
    if [ "$saw_ready_log" != "true" ] && [ -f "$log" ]; then
      local log_bytes="0"
      log_bytes="$(wc -c <"$log" 2>/dev/null || echo 0)"
      log_bytes="${log_bytes//[[:space:]]/}"
      if ! [[ "$log_bytes" =~ ^[0-9]+$ ]]; then
        log_bytes="0"
      fi
      if [ "$log_bytes" -lt "$ready_scan_offset" ]; then
        ready_scan_offset=0
        ready_scan_carry=""
      fi
      if [ "$log_bytes" -gt "$ready_scan_offset" ]; then
        local new_log_text scan_text
        new_log_text="$(tail -c +"$((ready_scan_offset + 1))" "$log" 2>/dev/null || true)"
        ready_scan_offset="$log_bytes"
        scan_text="${ready_scan_carry}${new_log_text}"
        if [ "${#scan_text}" -gt "$ready_scan_carry_chars" ]; then
          ready_scan_carry="${scan_text: -$ready_scan_carry_chars}"
        else
          ready_scan_carry="$scan_text"
        fi
        local ready_log_lines
        ready_log_lines="$(printf "%s" "$scan_text" | grep -E "$ready_log_pattern" || true)"
        if [ -n "$ready_log_lines" ]; then
          saw_ready_log=true
          [ -n "$ready_port" ] || ready_port="$(printf "%s" "$ready_log_lines" | openclaw_e2e_gateway_log_port_from_text)"
        fi
      fi
    fi
    if [ "$saw_ready_log" = "true" ]; then
      [ -n "$ready_port" ] || ready_port="${OPENCLAW_E2E_GATEWAY_READY_PORT:-18789}"
      if [ "$readiness_mode" = "legacy-ready-log-ok" ]; then
        openclaw_e2e_probe_tcp 127.0.0.1 "$ready_port" 400 && return 0
      else
        openclaw_e2e_probe_http "http://127.0.0.1:${ready_port}/readyz" ok 400 && return 0
      fi
    fi
    sleep 0.25
  done
  if [ "$saw_ready_log" = "true" ]; then
    if [ "$readiness_mode" = "legacy-ready-log-ok" ]; then
      echo "Gateway startup log was found, but TCP listener probe never succeeded"
    else
      echo "Gateway log reported ready, but /readyz probe never succeeded"
    fi
  else
    echo "Gateway did not become ready"
  fi
  tail -n 120 "$log" 2>/dev/null || true
  return 1
}
openclaw_e2e_probe_tcp() {
  node --input-type=module -e '
    import net from "node:net";
    const socket = net.createConnection({ host: process.argv[1], port: Number(process.argv[2]) });
    const timeout = setTimeout(() => { socket.destroy(); process.exit(1); }, Number(process.argv[3] ?? 400));
    socket.on("connect", () => { clearTimeout(timeout); socket.end(); process.exit(0); });
    socket.on("error", () => { clearTimeout(timeout); process.exit(1); });
  ' "$1" "$2" "${3:-400}"
}
openclaw_e2e_probe_http() {
  node --input-type=module -e '
    const expected = process.argv[2] ?? "ok";
    const timeoutMs = Number(process.argv[3] ?? 400);
    const controller = new AbortController();
    const timer = setTimeout(() => controller.abort(), timeoutMs);
    let exitCode = 1;
    let response;
    try {
      response = await fetch(process.argv[1], { signal: controller.signal });
      const passed = expected === "ok" ? response.ok : response.status === Number(expected);
      exitCode = passed ? 0 : 1;
    } catch {
      exitCode = 1;
    } finally {
      clearTimeout(timer);
      await response?.body?.cancel?.().catch(() => undefined);
    }
    process.exit(exitCode);
  ' "$1" "${2:-ok}" "${3:-400}"
}
openclaw_e2e_probe_http_status() {
  openclaw_e2e_probe_http "$1" "${2:-200}" "${3:-400}"
}
openclaw_e2e_assert_file() { [ -f "$1" ] || { echo "Missing file: $1"; exit 1; }; }
openclaw_e2e_assert_dir() { [ -d "$1" ] || { echo "Missing dir: $1"; exit 1; }; }
openclaw_e2e_assert_log_not_contains() {
  ! grep -q "$2" "$1" || { echo "Unexpected log output: $2"; exit 1; }
}
openclaw_e2e_run_logged() {
  local label="$1" log_root="${OPENCLAW_E2E_LOG_DIR:-${TMPDIR:-/tmp}}" log_path safe_label
  shift
  safe_label="${label//[^A-Za-z0-9_.-]/-}"
  [ -n "$safe_label" ] || safe_label="command"
  mkdir -p "$log_root"
  log_path="$(mktemp "$log_root/openclaw-${safe_label}.XXXXXX.log")"
  OPENCLAW_E2E_LAST_LOG_PATH="$log_path"
  export OPENCLAW_E2E_LAST_LOG_PATH
  openclaw_e2e_run_command "$@" >"$log_path" 2>&1 || { openclaw_e2e_print_log "$log_path"; exit 1; }
}
openclaw_e2e_run_command() {
  local timeout_value="${OPENCLAW_E2E_COMMAND_TIMEOUT:-300s}"
  openclaw_e2e_maybe_timeout "$timeout_value" "$@"
}
openclaw_e2e_fixture_plugin_command() {
  local runner=()
  while [[ "${1:-}" != "--" && "$#" -gt 0 ]]; do
    runner+=("$1")
    shift
  done
  shift
  local help consent
  # Preserve the successful candidate help result for evidence checks after mutation.
  # Clear it first so a failed replacement probe cannot expose stale capabilities.
  OPENCLAW_E2E_LAST_FIXTURE_PLUGIN_CAPABILITY_CONSENT_SUPPORTED=
  # Use the caller's bounded runner for both calls; never cache across package replacement.
  help="$("${runner[@]}" "$1" "$2" --help)" || {
    local probe_status=$?
    printf '%s\nPlugin fixture help probe failed with status %s: %s %s\n' "$help" "$probe_status" "$1" "$2" >&2
    return "$probe_status"
  }
  consent="$(printf '%s' "$help" | node scripts/e2e/lib/package-compat.mjs fixture-consent)" || return $?
  if [[ -n "$consent" ]]; then
    OPENCLAW_E2E_LAST_FIXTURE_PLUGIN_CAPABILITY_CONSENT_SUPPORTED=1
    set -- "$@" "$consent"
  else
    OPENCLAW_E2E_LAST_FIXTURE_PLUGIN_CAPABILITY_CONSENT_SUPPORTED=0
  fi
  "${runner[@]}" "$@"
}
openclaw_e2e_enable_openclaw_cli_timeout() {
  OPENCLAW_E2E_CLI_BIN="$(type -P openclaw)"
  if [ -z "$OPENCLAW_E2E_CLI_BIN" ]; then
    echo "OpenClaw CLI binary not found on PATH" >&2
    return 1
  fi
  export OPENCLAW_E2E_CLI_BIN
  openclaw() {
    openclaw_e2e_run_command "$OPENCLAW_E2E_CLI_BIN" "$@"
  }
}
openclaw_e2e_dump_logs() {
  local path
  for path in "$@"; do
    openclaw_e2e_print_log "$path"
  done
}