diff --git a/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml b/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml index f58232f75a40..bac91587631c 100644 --- a/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml +++ b/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml @@ -1300,19 +1300,9 @@ jobs: (inputs.live_suite_filter == '' || inputs.live_suite_filter == matrix.suite_id) env: NODE_OPTIONS: --max-old-space-size=8192 + OPENCLAW_BUILD_PRIVATE_QA: "1" run: pnpm build - - name: Configure suite-specific env - if: inputs.live_suite_filter == '' || inputs.live_suite_filter == matrix.suite_id - shell: bash - run: | - set -euo pipefail - case "${{ matrix.suite_id }}" in - openshell-e2e) - echo "OPENCLAW_E2E_OPENSHELL_CONFIG_HOME=$HOME/.config" >> "$GITHUB_ENV" - ;; - esac - - name: Install OpenShell CLI if: | (inputs.live_suite_filter == '' || inputs.live_suite_filter == matrix.suite_id) && @@ -1320,96 +1310,245 @@ jobs: shell: bash run: | set -euo pipefail - export OPENSHELL_VERSION=v0.0.109 - installer_path="$(mktemp "${RUNNER_TEMP}/openshell-install.XXXXXX")" - trap 'rm -f "$installer_path"' EXIT + # The installer also starts a systemd gateway. Install only the verified + # package so the E2E shell below is the sole gateway lifecycle owner. + package_path="$(mktemp "${RUNNER_TEMP}/openshell.XXXXXX.deb")" + trap 'rm -f "$package_path"' EXIT curl -LsSf --connect-timeout 10 --max-time 120 \ - -o "$installer_path" \ - https://raw.githubusercontent.com/NVIDIA/OpenShell/8d67250a5d17348eb96c4fa46226b06d8041f2ba/install.sh - sh "$installer_path" + -o "$package_path" \ + https://github.com/NVIDIA/OpenShell/releases/download/v0.0.109/openshell_0.0.109-1_amd64.deb + printf '%s %s\n' \ + 0364a21f279023a241d967309ebb0177dadb66bd792eeb3daf542283158ca40f \ + "$package_path" | sha256sum --check - + sudo env DEBIAN_FRONTEND=noninteractive apt-get install -y "$package_path" openshell --version - - name: Bootstrap OpenShell gateway + - name: Run ${{ matrix.label }} if: | (inputs.live_suite_filter == '' || inputs.live_suite_filter == matrix.suite_id) && matrix.suite_id == 'openshell-e2e' shell: bash run: | set -euo pipefail - mtls_dir="$HOME/.config/openshell/gateways/openshell/mtls" - gateway_tls_dir="$RUNNER_TEMP/openshell-gateway-certs" - fallback_pid="" - if ! openshell --gateway openshell sandbox list >/dev/null 2>&1; then - rm -rf "$gateway_tls_dir" - openshell-gateway generate-certs \ - --output-dir "$gateway_tls_dir" \ - --server-san 127.0.0.1 \ - --server-san localhost \ - --server-san host.openshell.internal - rm -rf "$mtls_dir" - mkdir -p "$mtls_dir" - cp "$gateway_tls_dir/ca.crt" "$mtls_dir/ca.crt" - cp "$gateway_tls_dir/client/tls.crt" "$mtls_dir/tls.crt" - cp "$gateway_tls_dir/client/tls.key" "$mtls_dir/tls.key" - openshell gateway remove openshell >/dev/null 2>&1 || true - OPENSHELL_LOCAL_TLS_DIR="$gateway_tls_dir" nohup openshell-gateway \ - --bind-address 0.0.0.0 \ - --port 17670 \ - --drivers docker \ - --tls-cert "$gateway_tls_dir/server/tls.crt" \ - --tls-key "$gateway_tls_dir/server/tls.key" \ - --tls-client-ca "$mtls_dir/ca.crt" \ - >"$RUNNER_TEMP/openshell-gateway.log" 2>&1 & - fallback_pid=$! - echo "OPENCLAW_OPENSHELL_FALLBACK_PID=$fallback_pid" >> "$GITHUB_ENV" - for _ in $(seq 1 30); do - if openshell gateway add --local --name openshell https://127.0.0.1:17670; then - break - fi - sleep 1 + umask 077 + fixture_root="$(mktemp -d "${RUNNER_TEMP}/openshell-e2e.XXXXXX")" + gateway_name="openshell-e2e-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + gateway_port=17680 + gateway_pid="" + command_pid="" + run_owned() { + # Async wait handles cancellation immediately; the process group + # includes pnpm's test-runner descendants in the same cleanup. + setsid "$@" & + command_pid=$! + local command_result=0 + wait "$command_pid" || command_result=$? + # Keep surviving descendants owned; a readiness retry must not replace + # their group with the next command's PID. + if kill -0 -- "-$command_pid" 2>/dev/null; then + echo "OpenShell command left a running process group after its leader exited." >&2 + (( command_result != 0 )) || command_result=1 + exit "$command_result" + fi + command_pid="" + return "$command_result" + } + owned_target_running() { + local target="$1" + if [[ "$target" == -* ]]; then + kill -0 -- "$target" 2>/dev/null + # A bare return inside EXIT would inherit the pre-trap failure. + return "$?" + fi + # Bash may reap an early gateway exit while waiting for the suite. + # Only a still-running child job owns its saved numeric PID. + local child + local children=() + mapfile -t children < <(jobs -pr) + for child in "${children[@]}"; do + [[ "$child" == "$target" ]] && return 0 done - openshell gateway select openshell - for _ in $(seq 1 60); do - if openshell --gateway openshell sandbox list >/dev/null 2>&1; then - break + return 1 + } + cleanup() { + local result=$? + trap - EXIT + trap '' INT TERM + local pid + local owned_pids=() + [[ -n "$gateway_pid" ]] && owned_pids+=("$gateway_pid") + [[ -n "$command_pid" ]] && owned_pids+=("-$command_pid") + for pid in "${owned_pids[@]}"; do + if owned_target_running "$pid" && ! kill -TERM -- "$pid" 2>/dev/null; then + if owned_target_running "$pid"; then + (( result != 0 )) || result=1 + fi fi - sleep 1 done - fi - if [[ -z "$fallback_pid" ]]; then - echo "OPENCLAW_OPENSHELL_FALLBACK_PID=" >> "$GITHUB_ENV" - fi - openshell --gateway openshell sandbox list >/dev/null - openshell gateway list - - - name: Validate suite credentials - if: inputs.live_suite_filter == '' || inputs.live_suite_filter == matrix.suite_id - shell: bash - run: | + for _ in $(seq 1 20); do + local remaining=() + for pid in "${owned_pids[@]}"; do + if owned_target_running "$pid"; then + remaining+=("$pid") + else + wait "${pid#-}" 2>/dev/null || true + fi + done + owned_pids=("${remaining[@]}") + (( ${#owned_pids[@]} )) || break + sleep 0.2 + done + local unsettled=0 + for pid in "${owned_pids[@]}"; do + if ! owned_target_running "$pid"; then + wait "${pid#-}" 2>/dev/null || true + continue + fi + kill -KILL -- "$pid" 2>/dev/null || true + (( result != 0 )) || result=1 + wait "${pid#-}" 2>/dev/null || true + if owned_target_running "$pid"; then unsettled=1; fi + done + local docker_cleanup_ok=0 + # Share the former two one-second Docker budgets across owned cleanup. + # timeout owns this subprocess group, including hung Docker children. + if timeout --kill-after=1s 2s bash -s -- "$gateway_name" <<'CLEANUP_DOCKER' set -euo pipefail - case "${{ matrix.suite_id }}" in - openshell-e2e) - ;; - esac + namespace="$1" + listed="$(docker ps -aq \ + --filter "label=openshell.ai/managed-by=openshell" \ + --filter "label=openshell.ai/sandbox-namespace=$namespace")" + helpers="$(docker ps -aq --filter "label=openclaw.ai/openshell-e2e-gateway=$namespace")" + ids=() + while IFS= read -r id; do + [[ -n "$id" ]] || continue + [[ "$id" =~ ^[0-9a-f]+$ ]] || exit 1 + ids+=("$id") + done <<< "$listed"$'\n'"$helpers" + if (( ${#ids[@]} )); then docker rm -f -- "${ids[@]}"; fi + networks="$(docker network ls --format '{{.Name}}')" + if grep -Fxq -- "$namespace" <<< "$networks"; then docker network rm "$namespace"; fi + CLEANUP_DOCKER + then + docker_cleanup_ok=1 + else + (( result != 0 )) || result=1 + fi + if (( unsettled == 0 && docker_cleanup_ok == 1 )); then + if ! rm -rf -- "$fixture_root"; then (( result != 0 )) || result=1; fi + else + echo "Retaining OpenShell fixture state because owned cleanup did not complete." >&2 + fi + exit "$result" + } + trap cleanup EXIT + trap 'exit 130' INT + trap 'exit 143' TERM + export XDG_CONFIG_HOME="$fixture_root/config" + export XDG_STATE_HOME="$fixture_root/state" + export OPENCLAW_E2E_OPENSHELL_CONFIG_HOME="$XDG_CONFIG_HOME" + # Local registration imports this bundle too; a server-only override + # would replace the CLI certificates with the package-managed bundle. + export OPENSHELL_LOCAL_TLS_DIR="$fixture_root/pki" + mkdir -p "$XDG_CONFIG_HOME" "$XDG_STATE_HOME" + run_owned openshell-gateway generate-certs --output-dir "$OPENSHELL_LOCAL_TLS_DIR" \ + --server-san 127.0.0.1 --server-san localhost --server-san host.openshell.internal + cat > "$fixture_root/gateway.toml" < "$RUNNER_TEMP/$gateway_name.raw.log" 2>&1 & + gateway_pid=$! + run_owned openshell gateway add --local --name "$gateway_name" "https://127.0.0.1:$gateway_port" + ready_deadline=$((SECONDS + 60)) + until run_owned timeout --foreground 5s openshell --gateway "$gateway_name" whoami \ + > "$fixture_root/whoami.txt" 2> "$fixture_root/auth-error.txt"; do + owned_target_running "$gateway_pid" || { + echo "OpenShell gateway exited before authenticated readiness." >&2 + exit 1 + } + if (( SECONDS >= ready_deadline )); then + cat "$fixture_root/auth-error.txt" >&2 + exit 1 + fi + sleep 1 + done + owned_target_running "$gateway_pid" + run_owned timeout --foreground 20s openshell --gateway "$gateway_name" sandbox list + echo "OpenShell bootstrap passed: owned gateway and protected whoami/sandbox list." + run_owned ${{ matrix.command }} - - name: Run ${{ matrix.label }} + - name: Redact OpenShell gateway log if: | - ( - (inputs.include_repo_e2e && matrix.requires_repo_e2e) || - (inputs.include_live_suites && matrix.requires_live_suites) - ) && - (inputs.live_suite_filter == '' || inputs.live_suite_filter == matrix.suite_id) - run: ${{ matrix.command }} - - - name: Stop fallback OpenShell gateway - if: always() && matrix.suite_id == 'openshell-e2e' + always() && inputs.include_repo_e2e && + (inputs.live_suite_filter == '' || inputs.live_suite_filter == matrix.suite_id) && + matrix.suite_id == 'openshell-e2e' shell: bash run: | set -euo pipefail - if [[ -n "${OPENCLAW_OPENSHELL_FALLBACK_PID:-}" ]]; then - kill "$OPENCLAW_OPENSHELL_FALLBACK_PID" 2>/dev/null || true + umask 077 + log_base="$RUNNER_TEMP/openshell-e2e-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + trap 'rm -f "$log_base.raw.log" "$log_base.redacted.tmp"' EXIT + trap 'exit 130' INT + trap 'exit 143' TERM + # Logging runs after owned-process/PKI cleanup, never extending its + # cancellation grace. Only this bounded, redacted file may be uploaded. + if timeout --foreground --kill-after=1s 5s node --input-type=module - \ + "$log_base.raw.log" > "$log_base.redacted.tmp" 2>/dev/null <<'NODE' + import fs from "node:fs"; + import { stripVTControlCharacters } from "node:util"; + import { redactSensitiveText } from "./dist/plugin-sdk/logging-core.js"; + const fd = fs.openSync(process.argv[2], "r"); + try { + const size = fs.fstatSync(fd).size; + const start = Math.max(0, size - 1024 * 1024); + const buffer = Buffer.alloc(size - start); + const bytes = fs.readSync(fd, buffer, 0, buffer.length, start); + let text = buffer.subarray(0, bytes).toString("utf8"); + if (start > 0) { + const newline = text.indexOf("\n"); + text = newline < 0 ? "" : text.slice(newline + 1); + } + text = text.slice(0, text.lastIndexOf("\n") + 1); + // v0.0.109 logs SSH bearer UUIDs as session_id/object.id, not token. + // Remove full values before canonical redaction, including in spans. + text = stripVTControlCharacters(text).replace( + /[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}/gi, "[redacted-uuid]", + ); + process.stdout.write(redactSensitiveText(text, { mode: "tools" })); + } finally { + fs.closeSync(fd); + } + NODE + then + mv "$log_base.redacted.tmp" "$log_base.log" + tail -n 120 "$log_base.log" + else + echo "OpenShell gateway log omitted: source or canonical redaction unavailable." fi + - name: Upload redacted OpenShell gateway log + if: | + always() && inputs.include_repo_e2e && + (inputs.live_suite_filter == '' || inputs.live_suite_filter == matrix.suite_id) && + matrix.suite_id == 'openshell-e2e' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: openshell-gateway-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/openshell-e2e-${{ github.run_id }}-${{ github.run_attempt }}.log + if-no-files-found: ignore + retention-days: 7 + validate_docker_e2e: needs: [validate_selected_ref, prepare_docker_e2e_image, plan_release_workflow_matrices] if: (!inputs.prepare_only) && inputs.include_release_path_suites && inputs.docker_lanes == '' && needs.plan_release_workflow_matrices.outputs.docker_e2e_count != '0' diff --git a/extensions/openshell/src/backend.e2e.test.ts b/extensions/openshell/src/backend.e2e.test.ts index bafbdd0a0be3..dfebcbe1098b 100644 --- a/extensions/openshell/src/backend.e2e.test.ts +++ b/extensions/openshell/src/backend.e2e.test.ts @@ -181,7 +181,7 @@ function trimTrailingNewline(value: string): string { return value.replace(/\r?\n$/, ""); } -async function startHostPolicyServer(): Promise { +async function startHostPolicyServer(gatewayName: string): Promise { const port = await allocatePort(); const responseBody = JSON.stringify({ ok: true, message: "hello-from-host" }); const serverScript = `from http.server import BaseHTTPRequestHandler, HTTPServer @@ -218,6 +218,8 @@ HTTPServer(("0.0.0.0", 8000), Handler).serve_forever() "run", "--detach", "--rm", + "--label", + `openclaw.ai/openshell-e2e-gateway=${gatewayName}`, "-e", `RESPONSE_BODY=${responseBody}`, "-p", @@ -452,7 +454,7 @@ describe("openshell sandbox backend e2e", () => { process.env.HOME = env.HOME; process.env.XDG_CONFIG_HOME = env.XDG_CONFIG_HOME; process.env.XDG_CACHE_HOME = env.XDG_CACHE_HOME; - hostPolicyServer = await startHostPolicyServer(); + hostPolicyServer = await startHostPolicyServer(gatewayName); if (!hostPolicyServer) { throw new Error("failed to start host policy server"); } diff --git a/test/scripts/fixtures/openshell-workflow-lifecycle.py b/test/scripts/fixtures/openshell-workflow-lifecycle.py new file mode 100644 index 000000000000..8e7012a09bd1 --- /dev/null +++ b/test/scripts/fixtures/openshell-workflow-lifecycle.py @@ -0,0 +1,283 @@ +"""Synthetic commands and a Linux subreaper for the unchanged workflow shell.""" +import ctypes +import json +import os +import pathlib +import selectors +import select +import shlex +import signal +import socket +import struct +import subprocess +import sys + + +def connect(): + connection = socket.socket(socket.AF_UNIX, socket.SOCK_SEQPACKET) + connection.connect("\0" + os.environ["OPENSHELL_FIXTURE_SOCKET"]) + return connection + + +def request(connection, value): + connection.sendall(json.dumps(value).encode()) + response = connection.recv(65536) + if not response: + raise RuntimeError("fixture controller closed before acknowledging the command") + return json.loads(response) + + +def hold(role, ready_fd=None): + with connect() as connection: + def terminate(_signum, _frame): + connection.sendall(json.dumps({"event": "term", "role": role}).encode()) + os._exit(0) + + signal.signal(signal.SIGTERM, terminate) + request(connection, {"event": "hold", "role": role}) + connection.sendall(json.dumps({"event": "ready", "role": role}).encode()) + if ready_fd is not None: + os.write(ready_fd, b"1") + os.close(ready_fd) + while True: + signal.pause() + + +def actor(command, args): + if command == "openshell-gateway" and args[0] != "generate-certs": + hold("gateway") + with connect() as connection: + answer = request(connection, { + "event": "call", "command": command, "args": args, + "fixtureRoot": str(pathlib.Path(os.environ["XDG_CONFIG_HOME"]).parent), + }) + if answer.get("action") == "descendant": + read_fd, write_fd = os.pipe() + if os.fork() == 0: + os.close(read_fd) + hold("descendant", write_fd) + os.close(write_fd) + if os.read(read_fd, 1) != b"1": + raise RuntimeError("descendant exited before readiness") + os.close(read_fd) + return 42 + if answer.get("action") == "hold": + hold("suite") + sys.stdout.write(answer.get("stdout", "")) + return answer.get("code", 0) + + +def supervise(root, script, scenario): + if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0: + raise OSError(ctypes.get_errno(), "PR_SET_CHILD_SUBREAPER failed") + namespace = "openshell-e2e-123-2" + report = {"calls": [], "terminated": [], "emergency": [], "reapedDescendants": 0} + containers = { + "aa11": {"openshell.ai/managed-by": "openshell", "openshell.ai/sandbox-namespace": namespace}, + "bb22": {"openshell.ai/managed-by": "openshell", "openshell.ai/sandbox-namespace": namespace}, + "cc33": {"openshell.ai/managed-by": "openshell", "openshell.ai/sandbox-namespace": namespace + "-other"}, + "dd44": {"openshell.ai/managed-by": "unrelated", "openshell.ai/sandbox-namespace": namespace}, + "ee55": {"openclaw.ai/openshell-e2e-gateway": namespace}, + "ff66": {"openclaw.ai/openshell-e2e-gateway": namespace + "-other"}, + } + networks = {namespace, namespace + "-other", "another-network"} + bin_dir = root / "bin" + bin_dir.mkdir() + runner_temp = root / "runner" + runner_temp.mkdir() + for command in ("openshell-gateway", "openshell", "fixture-suite", "docker", "sleep"): + executable = bin_dir / command + executable.write_text("#!/bin/sh\nexec " + " ".join(map(shlex.quote, ( + sys.executable, "-I", "-S", str(pathlib.Path(__file__).resolve()), "actor", command, + ))) + ' "$@"\n') + executable.chmod(0o755) + address = "openshell-lifecycle-" + str(os.getpid()) + selector = selectors.DefaultSelector() + listener = socket.socket(socket.AF_UNIX, socket.SOCK_SEQPACKET) + listener.bind("\0" + address) + listener.listen() + selector.register(listener, selectors.EVENT_READ, "accept") + wake_read, wake_write = os.pipe2(os.O_NONBLOCK | os.O_CLOEXEC) + signal.set_wakeup_fd(wake_write) + interrupted = False + + def interrupt(_signum, _frame): + nonlocal interrupted + interrupted = True + + signal.signal(signal.SIGCHLD, lambda *_: None) + signal.signal(signal.SIGTERM, interrupt) + signal.signal(signal.SIGINT, interrupt) + selector.register(wake_read, selectors.EVENT_READ, "signal") + shell = subprocess.Popen(["bash", "--noprofile", "--norc", str(script)], env={ + "PATH": str(bin_dir) + ":" + os.environ["PATH"], "HOME": str(root), + "RUNNER_TEMP": str(runner_temp), "GITHUB_RUN_ID": "123", "GITHUB_RUN_ATTEMPT": "2", + "OPENSHELL_FIXTURE_SOCKET": address, + }) + shell_fd = os.pidfd_open(shell.pid) + owned = {shell.pid: {"fd": shell_fd, "role": "shell", "exited": False, "reaped": False}} + selector.register(shell_fd, selectors.EVENT_READ, ("exit", shell.pid)) + connections = set() + pending = [] + gateway_ready = False + whoami_count = 0 + + def reply(connection, value=None): + connection.sendall(json.dumps(value or {}).encode()) + + def stop_owned(): + for entry in owned.values(): + if entry.get("emergencySent") or select.select([entry["fd"]], [], [], 0)[0]: + continue + try: + signal.pidfd_send_signal(entry["fd"], signal.SIGKILL) + entry["emergencySent"] = True + report["emergency"].append(entry["role"]) + except ProcessLookupError: + pass + + def docker(args): + if args[:2] == ["ps", "-aq"]: + if scenario == "docker-query-failure": + return {"code": 17} + filters = args[2:] + assert len(filters) % 2 == 0 + labels = {} + for flag, value in zip(filters[::2], filters[1::2]): + assert flag == "--filter" and value.startswith("label=") + key, value = value[6:].split("=", 1) + labels[key] = value + selected = [key for key, actual in containers.items() + if all(actual.get(label) == value for label, value in labels.items())] + return {"stdout": "\n".join(selected) + "\n"} + if args[:3] == ["rm", "-f", "--"]: + for key in args[3:]: + del containers[key] + return {} + if args == ["network", "ls", "--format", "{{.Name}}"]: + return {"stdout": "\n".join(sorted(networks)) + "\n"} + if args[:2] == ["network", "rm"] and len(args) == 3: + if scenario == "docker-mutation-failure": + return {"code": 19} + networks.remove(args[2]) + return {} + raise AssertionError("unexpected Docker command: " + repr(args)) + + try: + while True: + for key, _ in selector.select(): + if key.data == "accept": + connection, _ = listener.accept() + connections.add(connection) + selector.register(connection, selectors.EVENT_READ, "message") + elif key.data == "signal": + os.read(wake_read, 65536) + elif isinstance(key.data, tuple): + entry = owned[key.data[1]] + entry["exited"] = True + selector.unregister(key.fd) + else: + connection = key.fileobj + packet = connection.recv(65536) + if not packet: + selector.unregister(connection) + connections.remove(connection) + connection.close() + continue + message = json.loads(packet) + event = message["event"] + if event == "hold": + # The actor waits for this acknowledgement, pinning its lifetime. + pid, _, _ = struct.unpack("3i", connection.getsockopt( + socket.SOL_SOCKET, socket.SO_PEERCRED, struct.calcsize("3i"))) + descriptor = os.pidfd_open(pid) + owned[pid] = {"fd": descriptor, "role": message["role"], + "exited": False, "reaped": False} + selector.register(descriptor, selectors.EVENT_READ, ("exit", pid)) + reply(connection) + elif event == "ready": + if message["role"] == "gateway": + gateway_ready = True + elif message["role"] == "suite" and scenario == "cancel": + signal.pidfd_send_signal(shell_fd, signal.SIGTERM) + elif event == "term": + report["terminated"].append(message["role"]) + else: + assert event == "call" + command, args = message["command"], message["args"] + report["fixtureRoot"] = message["fixtureRoot"] + report["calls"].append({"command": command, "args": args}) + if command == "docker": + reply(connection, docker(args)) + elif command == "sleep": + assert args in (["0.2"], ["1"]) + pending.append((connection, "cleanup" if args == ["0.2"] else "ready")) + elif command == "openshell-gateway": + assert args[0] == "generate-certs" + pathlib.Path(message["fixtureRoot"], "pki").mkdir() + reply(connection) + elif command == "openshell" and args[:2] == ["gateway", "add"]: + pending.append((connection, "gateway")) + elif command == "openshell" and args[-1] == "whoami": + whoami_count += 1 + failed = scenario in ("leader-failure", "docker-mutation-failure") + reply(connection, {"action": "descendant"} if failed and whoami_count == 1 else {}) + elif command == "fixture-suite": + reply(connection, {"action": "hold"} if scenario == "cancel" else {}) + else: + assert command == "openshell" and args[-2:] == ["sandbox", "list"] + reply(connection) + no_children = False + # Drain SIGCHLD notifications here; Popen.wait must not compete for children. + while True: + try: + pid, code = os.waitpid(-1, os.WNOHANG) + except ChildProcessError: + no_children = True + break + if pid == 0: + break + if pid == shell.pid: + shell.returncode = os.waitstatus_to_exitcode(code) + report["shellStatus"] = shell.returncode + if pid in owned: + owned[pid]["reaped"] = True + if owned[pid]["role"] == "descendant": + report["reapedDescendants"] += 1 + if interrupted or (shell.returncode is not None and not no_children): + stop_owned() + for connection, gate in pending[:]: + # Replace elapsed cleanup grace with observed exit and adoption receipts. + settled = all(entry["exited"] and (entry["role"] != "descendant" or entry["reaped"]) + for entry in owned.values() if entry["role"] != "shell") + if gate == "ready" or (gate == "gateway" and gateway_ready) or (gate == "cleanup" and settled): + reply(connection) + pending.remove((connection, gate)) + if shell.returncode is not None and no_children and not connections: + break + assert not interrupted, "fixture controller was interrupted" + finally: + stop_owned() + for connection in connections: + connection.close() + listener.close() + # Closing the protocol releases unregistered short-lived commands too. + while True: + try: + os.waitpid(-1, 0) + except ChildProcessError: + break + for entry in owned.values(): + os.close(entry["fd"]) + signal.set_wakeup_fd(-1) + os.close(wake_read) + os.close(wake_write) + selector.close() + report["containers"] = sorted(containers) + report["networks"] = sorted(networks) + (root / "lifecycle.json").write_text(json.dumps(report)) + + +if sys.argv[1] == "actor": + sys.exit(actor(sys.argv[2], sys.argv[3:])) +supervise(pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]), sys.argv[3]) diff --git a/test/scripts/openshell-workflow.lifecycle.test.ts b/test/scripts/openshell-workflow.lifecycle.test.ts new file mode 100644 index 000000000000..19b817a37c75 --- /dev/null +++ b/test/scripts/openshell-workflow.lifecycle.test.ts @@ -0,0 +1,93 @@ +import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { expectDefined } from "@openclaw/normalization-core"; +import { expect } from "vitest"; +import { parse } from "yaml"; +import { createCommandTest } from "../helpers/command-fixture.js"; + +const test = createCommandTest(); +const workflowPath = ".github/workflows/openclaw-live-and-e2e-checks-reusable.yml"; +const fixturePath = fileURLToPath( + new URL("./fixtures/openshell-workflow-lifecycle.py", import.meta.url), +); +const namespace = "openshell-e2e-123-2"; + +type LifecycleReport = { + shellStatus: number; + calls: { command: string; args: string[] }[]; + terminated: string[]; + emergency: string[]; + reapedDescendants: number; + fixtureRoot: string; + containers: string[]; + networks: string[]; +}; + +test.skipIf(process.platform !== "linux").for([ + { scenario: "success", exit: 0, retained: false, descendant: false, suite: true }, + { scenario: "leader-failure", exit: 42, retained: false, descendant: true, suite: false }, + { scenario: "cancel", exit: 143, retained: false, descendant: false, suite: true }, + { scenario: "docker-query-failure", exit: 1, retained: true, descendant: false, suite: true }, + { scenario: "docker-mutation-failure", exit: 42, retained: true, descendant: true, suite: false }, +])( + "settles the OpenShell workflow's owned resources on $scenario", + async (scenario, { command }) => { + const steps = parse(readFileSync(workflowPath, "utf8")).jobs.validate_special_e2e + .steps as Array<{ + name?: string; + run?: string; + }>; + const run = expectDefined( + steps.find((step) => step.name === "Run ${{ matrix.label }}")?.run, + "OpenShell workflow run step", + ); + const root = command.createTempDir("openclaw-openshell-lifecycle-"); + const script = join(root, "run.sh"); + writeFileSync(script, run.replaceAll("${{ matrix.command }}", "fixture-suite")); + const result = await command.run( + "python3", + ["-I", "-S", fixturePath, root, script, scenario.scenario], + { env: { PATH: process.env.PATH }, maxBuffer: 64 * 1024 }, + ); + + expect(result.error, result.stderr).toBeUndefined(); + expect(result.status, result.stderr).toBe(0); + const report = JSON.parse( + readFileSync(join(root, "lifecycle.json"), "utf8"), + ) as LifecycleReport; + expect(report.shellStatus, result.stderr).toBe(scenario.exit); + expect(report.emergency).toEqual([]); + expect(report.terminated).toContain("gateway"); + expect( + report.calls.filter((call) => call.command === "openshell" && call.args.at(-1) === "whoami"), + ).toHaveLength(1); + expect(report.calls.some((call) => call.command === "fixture-suite")).toBe(scenario.suite); + if (scenario.descendant) { + expect(report.terminated).toContain("descendant"); + expect(report.reapedDescendants).toBe(1); + expect(report.calls.some((call) => call.args.slice(-2).join(" ") === "sandbox list")).toBe( + false, + ); + } + if (scenario.scenario === "cancel") { + expect(report.terminated).toContain("suite"); + } + expect(existsSync(report.fixtureRoot)).toBe(scenario.retained); + expect(report.containers).toEqual( + scenario.scenario === "docker-query-failure" + ? ["aa11", "bb22", "cc33", "dd44", "ee55", "ff66"] + : ["cc33", "dd44", "ff66"], + ); + expect(report.networks).toEqual( + scenario.retained + ? ["another-network", namespace, `${namespace}-other`] + : ["another-network", `${namespace}-other`], + ); + if (scenario.scenario === "docker-query-failure") { + expect( + report.calls.filter((call) => call.command === "docker").map((call) => call.args[0]), + ).toEqual(["ps"]); + } + }, +); diff --git a/test/scripts/openshell-workflow.test.ts b/test/scripts/openshell-workflow.test.ts new file mode 100644 index 000000000000..2d557aad00dd --- /dev/null +++ b/test/scripts/openshell-workflow.test.ts @@ -0,0 +1,157 @@ +import { spawnSync } from "node:child_process"; +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { expectDefined } from "@openclaw/normalization-core"; +import { afterEach, describe, expect, it } from "vitest"; +import { parse } from "yaml"; +import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js"; + +const LIVE_E2E_WORKFLOW_PATH = ".github/workflows/openclaw-live-and-e2e-checks-reusable.yml"; +const tempDirs = useAutoCleanupTempDirTracker(afterEach); + +function readNulSeparatedArgs(filePath: string): string[] { + return readFileSync(filePath, "utf8").split("\0").filter(Boolean); +} + +function workflowSteps() { + return parse(readFileSync(LIVE_E2E_WORKFLOW_PATH, "utf8")).jobs.validate_special_e2e + .steps as Array<{ + name?: string; + run?: string; + env?: Record; + }>; +} + +describe.skipIf(process.platform !== "linux")("OpenShell hosted bootstrap", () => { + it("builds the private QA runtime required by special E2E", () => { + expect( + workflowSteps().find((step) => step.name === "Build dist for special E2E"), + ).toMatchObject({ + run: "pnpm build", + env: { OPENCLAW_BUILD_PRIVATE_QA: "1" }, + }); + }); + + it.each([ + { + name: "success", + curlExit: 0, + checksumExit: 0, + installExit: 0, + calls: "download\nverify\ninstall\nversion\n", + }, + { + name: "partial download", + curlExit: 28, + checksumExit: 0, + installExit: 0, + calls: "download\n", + }, + { + name: "checksum failure", + curlExit: 0, + checksumExit: 1, + installExit: 0, + calls: "download\nverify\n", + }, + { + name: "install failure", + curlExit: 0, + checksumExit: 0, + installExit: 42, + calls: "download\nverify\ninstall\n", + }, + ])("verifies the OpenShell package before installation and cleans it on $name", (scenario) => { + const installStep = expectDefined( + workflowSteps().find((step) => step.name === "Install OpenShell CLI"), + "OpenShell install step", + ); + const run = expectDefined(installStep.run, "OpenShell install command"); + const root = tempDirs.make("openclaw-openshell-package-"); + const result = spawnSync( + "bash", + [ + "--noprofile", + "--norc", + "-c", + ` +curl() { + printf '%s\\0' "$@" > "$RUNNER_TEMP/curl-args" + local output="" + while (( $# )); do + if [[ "$1" == -o ]]; then shift; output="$1"; fi + shift + done + printf '%s' "$output" > "$RUNNER_TEMP/package-path" + printf 'fixture package' > "$output" + printf 'download\\n' >> "$RUNNER_TEMP/calls" + return "$CURL_EXIT" +} +sha256sum() { + printf '%s\\0' "$@" > "$RUNNER_TEMP/checksum-args" + cat > "$RUNNER_TEMP/checksum-input" + printf 'verify\\n' >> "$RUNNER_TEMP/calls" + return "$CHECKSUM_EXIT" +} +sudo() { + printf '%s\\0' "$@" > "$RUNNER_TEMP/install-args" + local package + for package in "$@"; do :; done + cat "$package" > "$RUNNER_TEMP/installed-bytes" + printf 'install\\n' >> "$RUNNER_TEMP/calls" + return "$INSTALL_EXIT" +} +openshell() { + [[ "$*" == --version ]] || return 99 + printf 'version\\n' >> "$RUNNER_TEMP/calls" +} +${run}`, + ], + { + encoding: "utf8", + timeout: 5_000, + env: { + PATH: process.env.PATH, + RUNNER_TEMP: root, + CURL_EXIT: String(scenario.curlExit), + CHECKSUM_EXIT: String(scenario.checksumExit), + INSTALL_EXIT: String(scenario.installExit), + }, + }, + ); + expect(result.error).toBeUndefined(); + expect(result.status, result.stderr).toBe( + scenario.curlExit || scenario.checksumExit || scenario.installExit, + ); + expect(readFileSync(join(root, "calls"), "utf8")).toBe(scenario.calls); + const packagePath = readFileSync(join(root, "package-path"), "utf8"); + expect(readNulSeparatedArgs(join(root, "curl-args"))).toEqual([ + "-LsSf", + "--connect-timeout", + "10", + "--max-time", + "120", + "-o", + packagePath, + "https://github.com/NVIDIA/OpenShell/releases/download/v0.0.109/openshell_0.0.109-1_amd64.deb", + ]); + if (scenario.curlExit === 0) { + expect(readNulSeparatedArgs(join(root, "checksum-args"))).toEqual(["--check", "-"]); + expect(readFileSync(join(root, "checksum-input"), "utf8")).toBe( + `0364a21f279023a241d967309ebb0177dadb66bd792eeb3daf542283158ca40f ${packagePath}\n`, + ); + } + if (scenario.curlExit === 0 && scenario.checksumExit === 0) { + expect(readNulSeparatedArgs(join(root, "install-args"))).toEqual([ + "env", + "DEBIAN_FRONTEND=noninteractive", + "apt-get", + "install", + "-y", + packagePath, + ]); + expect(readFileSync(join(root, "installed-bytes"), "utf8")).toBe("fixture package"); + } + expect(existsSync(packagePath)).toBe(false); + }); +}); diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index 13c31dbf6fa5..9d35f16cc19b 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -12709,7 +12709,7 @@ printf '%s\\n' "$DEEPSEEK_API_KEY" "$DEEPINFRA_API_KEY"`, for (const [jobName, job] of Object.entries(jobs)) { for (const step of job.steps ?? []) { if (step.run === "pnpm build") { - expect(step.env, `${workflowPath}:${jobName}:${step.name}`).toEqual({ + expect(step.env, `${workflowPath}:${jobName}:${step.name}`).toMatchObject({ NODE_OPTIONS: "--max-old-space-size=8192", }); } diff --git a/test/scripts/test-install-sh-docker.test.ts b/test/scripts/test-install-sh-docker.test.ts index 1098855a4f2c..71f0da59b584 100644 --- a/test/scripts/test-install-sh-docker.test.ts +++ b/test/scripts/test-install-sh-docker.test.ts @@ -1234,27 +1234,6 @@ printf 'status=%s\\n' "$status" expect(workflow).toContain("reachable from an OpenClaw branch or release tag"); }); - it("downloads the OpenShell installer completely before execution", () => { - const workflow = parse(readFileSync(LIVE_E2E_WORKFLOW_PATH, "utf8")); - const steps = workflow.jobs.validate_special_e2e.steps as Array<{ - name?: string; - run?: string; - }>; - const installStep = expectDefined( - steps.find((step) => step.name === "Install OpenShell CLI"), - "OpenShell install step", - ); - const run = expectDefined(installStep.run, "OpenShell install command"); - - expect(run).toContain('installer_path="$(mktemp "${RUNNER_TEMP}/openshell-install.XXXXXX")"'); - expect(run).toContain("curl -LsSf --connect-timeout 10 --max-time 120 \\"); - expect(run).toContain('-o "$installer_path"'); - expect(run).toContain('sh "$installer_path"'); - expect(run).toContain("trap 'rm -f \"$installer_path\"' EXIT"); - expect(run.indexOf('-o "$installer_path"')).toBeLessThan(run.indexOf('sh "$installer_path"')); - expect(run).not.toContain("install.sh | sh"); - }); - it("prints package size audits for release smoke tarballs", () => { const script = readFileSync(SCRIPT_PATH, "utf8");