fix(openshell): unblock hosted E2E gateway authentication (#131465)

* fix(openshell): unblock hosted E2E gateway authentication

* test(openshell): retain sanitized hosted gateway diagnostics

* test(openshell): verify hosted package install and QA build

* docs(ci): describe the current bounded typecheck stripes

Correct three comments after the canonical UI shard split. Executable workflow logic and the OpenShell job are unchanged.

.github/workflows/openclaw-live-and-e2e-checks-reusable.yml

@@ -1300,19 +1300,9 @@ jobs:
           (inputs.live_suite_filter == '' || inputs.live_suite_filter == matrix.suite_id)
         env:
           NODE_OPTIONS: --max-old-space-size=8192
+          OPENCLAW_BUILD_PRIVATE_QA: "1"
         run: pnpm build
 
-      - name: Configure suite-specific env
-        if: inputs.live_suite_filter == '' || inputs.live_suite_filter == matrix.suite_id
-        shell: bash
-        run: |
-          set -euo pipefail
-          case "${{ matrix.suite_id }}" in
-            openshell-e2e)
-              echo "OPENCLAW_E2E_OPENSHELL_CONFIG_HOME=$HOME/.config" >> "$GITHUB_ENV"
-              ;;
-          esac
-
       - name: Install OpenShell CLI
         if: |
           (inputs.live_suite_filter == '' || inputs.live_suite_filter == matrix.suite_id) &&
@@ -1320,96 +1310,245 @@ jobs:
         shell: bash
         run: |
           set -euo pipefail
-          export OPENSHELL_VERSION=v0.0.109
-          installer_path="$(mktemp "${RUNNER_TEMP}/openshell-install.XXXXXX")"
-          trap 'rm -f "$installer_path"' EXIT
+          # The installer also starts a systemd gateway. Install only the verified
+          # package so the E2E shell below is the sole gateway lifecycle owner.
+          package_path="$(mktemp "${RUNNER_TEMP}/openshell.XXXXXX.deb")"
+          trap 'rm -f "$package_path"' EXIT
           curl -LsSf --connect-timeout 10 --max-time 120 \
-            -o "$installer_path" \
-            https://raw.githubusercontent.com/NVIDIA/OpenShell/8d67250a5d17348eb96c4fa46226b06d8041f2ba/install.sh
-          sh "$installer_path"
+            -o "$package_path" \
+            https://github.com/NVIDIA/OpenShell/releases/download/v0.0.109/openshell_0.0.109-1_amd64.deb
+          printf '%s  %s\n' \
+            0364a21f279023a241d967309ebb0177dadb66bd792eeb3daf542283158ca40f \
+            "$package_path" | sha256sum --check -
+          sudo env DEBIAN_FRONTEND=noninteractive apt-get install -y "$package_path"
           openshell --version
 
-      - name: Bootstrap OpenShell gateway
+      - name: Run ${{ matrix.label }}
         if: |
           (inputs.live_suite_filter == '' || inputs.live_suite_filter == matrix.suite_id) &&
           matrix.suite_id == 'openshell-e2e'
         shell: bash
         run: |
           set -euo pipefail
-          mtls_dir="$HOME/.config/openshell/gateways/openshell/mtls"
-          gateway_tls_dir="$RUNNER_TEMP/openshell-gateway-certs"
-          fallback_pid=""
-          if ! openshell --gateway openshell sandbox list >/dev/null 2>&1; then
-            rm -rf "$gateway_tls_dir"
-            openshell-gateway generate-certs \
-              --output-dir "$gateway_tls_dir" \
-              --server-san 127.0.0.1 \
-              --server-san localhost \
-              --server-san host.openshell.internal
-            rm -rf "$mtls_dir"
-            mkdir -p "$mtls_dir"
-            cp "$gateway_tls_dir/ca.crt" "$mtls_dir/ca.crt"
-            cp "$gateway_tls_dir/client/tls.crt" "$mtls_dir/tls.crt"
-            cp "$gateway_tls_dir/client/tls.key" "$mtls_dir/tls.key"
-            openshell gateway remove openshell >/dev/null 2>&1 || true
-            OPENSHELL_LOCAL_TLS_DIR="$gateway_tls_dir" nohup openshell-gateway \
-              --bind-address 0.0.0.0 \
-              --port 17670 \
-              --drivers docker \
-              --tls-cert "$gateway_tls_dir/server/tls.crt" \
-              --tls-key "$gateway_tls_dir/server/tls.key" \
-              --tls-client-ca "$mtls_dir/ca.crt" \
-              >"$RUNNER_TEMP/openshell-gateway.log" 2>&1 &
-            fallback_pid=$!
-            echo "OPENCLAW_OPENSHELL_FALLBACK_PID=$fallback_pid" >> "$GITHUB_ENV"
-            for _ in $(seq 1 30); do
-              if openshell gateway add --local --name openshell https://127.0.0.1:17670; then
-                break
-              fi
-              sleep 1
+          umask 077
+          fixture_root="$(mktemp -d "${RUNNER_TEMP}/openshell-e2e.XXXXXX")"
+          gateway_name="openshell-e2e-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          gateway_port=17680
+          gateway_pid=""
+          command_pid=""
+          run_owned() {
+            # Async wait handles cancellation immediately; the process group
+            # includes pnpm's test-runner descendants in the same cleanup.
+            setsid "$@" &
+            command_pid=$!
+            local command_result=0
+            wait "$command_pid" || command_result=$?
+            # Keep surviving descendants owned; a readiness retry must not replace
+            # their group with the next command's PID.
+            if kill -0 -- "-$command_pid" 2>/dev/null; then
+              echo "OpenShell command left a running process group after its leader exited." >&2
+              (( command_result != 0 )) || command_result=1
+              exit "$command_result"
+            fi
+            command_pid=""
+            return "$command_result"
+          }
+          owned_target_running() {
+            local target="$1"
+            if [[ "$target" == -* ]]; then
+              kill -0 -- "$target" 2>/dev/null
+              # A bare return inside EXIT would inherit the pre-trap failure.
+              return "$?"
+            fi
+            # Bash may reap an early gateway exit while waiting for the suite.
+            # Only a still-running child job owns its saved numeric PID.
+            local child
+            local children=()
+            mapfile -t children < <(jobs -pr)
+            for child in "${children[@]}"; do
+              [[ "$child" == "$target" ]] && return 0
             done
-            openshell gateway select openshell
-            for _ in $(seq 1 60); do
-              if openshell --gateway openshell sandbox list >/dev/null 2>&1; then
-                break
+            return 1
+          }
+          cleanup() {
+            local result=$?
+            trap - EXIT
+            trap '' INT TERM
+            local pid
+            local owned_pids=()
+            [[ -n "$gateway_pid" ]] && owned_pids+=("$gateway_pid")
+            [[ -n "$command_pid" ]] && owned_pids+=("-$command_pid")
+            for pid in "${owned_pids[@]}"; do
+              if owned_target_running "$pid" && ! kill -TERM -- "$pid" 2>/dev/null; then
+                if owned_target_running "$pid"; then
+                  (( result != 0 )) || result=1
+                fi
               fi
-              sleep 1
             done
-          fi
-          if [[ -z "$fallback_pid" ]]; then
-            echo "OPENCLAW_OPENSHELL_FALLBACK_PID=" >> "$GITHUB_ENV"
-          fi
-          openshell --gateway openshell sandbox list >/dev/null
-          openshell gateway list
-
-      - name: Validate suite credentials
-        if: inputs.live_suite_filter == '' || inputs.live_suite_filter == matrix.suite_id
-        shell: bash
-        run: |
+            for _ in $(seq 1 20); do
+              local remaining=()
+              for pid in "${owned_pids[@]}"; do
+                if owned_target_running "$pid"; then
+                  remaining+=("$pid")
+                else
+                  wait "${pid#-}" 2>/dev/null || true
+                fi
+              done
+              owned_pids=("${remaining[@]}")
+              (( ${#owned_pids[@]} )) || break
+              sleep 0.2
+            done
+            local unsettled=0
+            for pid in "${owned_pids[@]}"; do
+              if ! owned_target_running "$pid"; then
+                wait "${pid#-}" 2>/dev/null || true
+                continue
+              fi
+              kill -KILL -- "$pid" 2>/dev/null || true
+              (( result != 0 )) || result=1
+              wait "${pid#-}" 2>/dev/null || true
+              if owned_target_running "$pid"; then unsettled=1; fi
+            done
+            local docker_cleanup_ok=0
+            # Share the former two one-second Docker budgets across owned cleanup.
+            # timeout owns this subprocess group, including hung Docker children.
+            if timeout --kill-after=1s 2s bash -s -- "$gateway_name" <<'CLEANUP_DOCKER'
           set -euo pipefail
-          case "${{ matrix.suite_id }}" in
-            openshell-e2e)
-              ;;
-          esac
+          namespace="$1"
+          listed="$(docker ps -aq \
+            --filter "label=openshell.ai/managed-by=openshell" \
+            --filter "label=openshell.ai/sandbox-namespace=$namespace")"
+          helpers="$(docker ps -aq --filter "label=openclaw.ai/openshell-e2e-gateway=$namespace")"
+          ids=()
+          while IFS= read -r id; do
+            [[ -n "$id" ]] || continue
+            [[ "$id" =~ ^[0-9a-f]+$ ]] || exit 1
+            ids+=("$id")
+          done <<< "$listed"$'\n'"$helpers"
+          if (( ${#ids[@]} )); then docker rm -f -- "${ids[@]}"; fi
+          networks="$(docker network ls --format '{{.Name}}')"
+          if grep -Fxq -- "$namespace" <<< "$networks"; then docker network rm "$namespace"; fi
+          CLEANUP_DOCKER
+            then
+              docker_cleanup_ok=1
+            else
+              (( result != 0 )) || result=1
+            fi
+            if (( unsettled == 0 && docker_cleanup_ok == 1 )); then
+              if ! rm -rf -- "$fixture_root"; then (( result != 0 )) || result=1; fi
+            else
+              echo "Retaining OpenShell fixture state because owned cleanup did not complete." >&2
+            fi
+            exit "$result"
+          }
+          trap cleanup EXIT
+          trap 'exit 130' INT
+          trap 'exit 143' TERM
+          export XDG_CONFIG_HOME="$fixture_root/config"
+          export XDG_STATE_HOME="$fixture_root/state"
+          export OPENCLAW_E2E_OPENSHELL_CONFIG_HOME="$XDG_CONFIG_HOME"
+          # Local registration imports this bundle too; a server-only override
+          # would replace the CLI certificates with the package-managed bundle.
+          export OPENSHELL_LOCAL_TLS_DIR="$fixture_root/pki"
+          mkdir -p "$XDG_CONFIG_HOME" "$XDG_STATE_HOME"
+          run_owned openshell-gateway generate-certs --output-dir "$OPENSHELL_LOCAL_TLS_DIR" \
+            --server-san 127.0.0.1 --server-san localhost --server-san host.openshell.internal
+          cat > "$fixture_root/gateway.toml" <<TOML
+          [openshell]
+          version = 1
+          [openshell.gateway]
+          compute_drivers = ["docker"]
+          [openshell.gateway.mtls_auth]
+          enabled = true
+          [openshell.drivers.docker]
+          sandbox_namespace = "$gateway_name"
+          network_name = "$gateway_name"
+          grpc_endpoint = "https://host.openshell.internal:$gateway_port"
+          TOML
+          RUST_LOG=info openshell-gateway --config "$fixture_root/gateway.toml" \
+            --bind-address 0.0.0.0 --port "$gateway_port" \
+            > "$RUNNER_TEMP/$gateway_name.raw.log" 2>&1 &
+          gateway_pid=$!
+          run_owned openshell gateway add --local --name "$gateway_name" "https://127.0.0.1:$gateway_port"
+          ready_deadline=$((SECONDS + 60))
+          until run_owned timeout --foreground 5s openshell --gateway "$gateway_name" whoami \
+            > "$fixture_root/whoami.txt" 2> "$fixture_root/auth-error.txt"; do
+            owned_target_running "$gateway_pid" || {
+              echo "OpenShell gateway exited before authenticated readiness." >&2
+              exit 1
+            }
+            if (( SECONDS >= ready_deadline )); then
+              cat "$fixture_root/auth-error.txt" >&2
+              exit 1
+            fi
+            sleep 1
+          done
+          owned_target_running "$gateway_pid"
+          run_owned timeout --foreground 20s openshell --gateway "$gateway_name" sandbox list
+          echo "OpenShell bootstrap passed: owned gateway and protected whoami/sandbox list."
+          run_owned ${{ matrix.command }}
 
-      - name: Run ${{ matrix.label }}
+      - name: Redact OpenShell gateway log
         if: |
-          (
-            (inputs.include_repo_e2e && matrix.requires_repo_e2e) ||
-            (inputs.include_live_suites && matrix.requires_live_suites)
-          ) &&
-          (inputs.live_suite_filter == '' || inputs.live_suite_filter == matrix.suite_id)
-        run: ${{ matrix.command }}
-
-      - name: Stop fallback OpenShell gateway
-        if: always() && matrix.suite_id == 'openshell-e2e'
+          always() && inputs.include_repo_e2e &&
+          (inputs.live_suite_filter == '' || inputs.live_suite_filter == matrix.suite_id) &&
+          matrix.suite_id == 'openshell-e2e'
         shell: bash
         run: |
           set -euo pipefail
-          if [[ -n "${OPENCLAW_OPENSHELL_FALLBACK_PID:-}" ]]; then
-            kill "$OPENCLAW_OPENSHELL_FALLBACK_PID" 2>/dev/null || true
+          umask 077
+          log_base="$RUNNER_TEMP/openshell-e2e-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          trap 'rm -f "$log_base.raw.log" "$log_base.redacted.tmp"' EXIT
+          trap 'exit 130' INT
+          trap 'exit 143' TERM
+          # Logging runs after owned-process/PKI cleanup, never extending its
+          # cancellation grace. Only this bounded, redacted file may be uploaded.
+          if timeout --foreground --kill-after=1s 5s node --input-type=module - \
+            "$log_base.raw.log" > "$log_base.redacted.tmp" 2>/dev/null <<'NODE'
+          import fs from "node:fs";
+          import { stripVTControlCharacters } from "node:util";
+          import { redactSensitiveText } from "./dist/plugin-sdk/logging-core.js";
+          const fd = fs.openSync(process.argv[2], "r");
+          try {
+            const size = fs.fstatSync(fd).size;
+            const start = Math.max(0, size - 1024 * 1024);
+            const buffer = Buffer.alloc(size - start);
+            const bytes = fs.readSync(fd, buffer, 0, buffer.length, start);
+            let text = buffer.subarray(0, bytes).toString("utf8");
+            if (start > 0) {
+              const newline = text.indexOf("\n");
+              text = newline < 0 ? "" : text.slice(newline + 1);
+            }
+            text = text.slice(0, text.lastIndexOf("\n") + 1);
+            // v0.0.109 logs SSH bearer UUIDs as session_id/object.id, not token.
+            // Remove full values before canonical redaction, including in spans.
+            text = stripVTControlCharacters(text).replace(
+              /[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}/gi, "[redacted-uuid]",
+            );
+            process.stdout.write(redactSensitiveText(text, { mode: "tools" }));
+          } finally {
+            fs.closeSync(fd);
+          }
+          NODE
+          then
+            mv "$log_base.redacted.tmp" "$log_base.log"
+            tail -n 120 "$log_base.log"
+          else
+            echo "OpenShell gateway log omitted: source or canonical redaction unavailable."
           fi
 
+      - name: Upload redacted OpenShell gateway log
+        if: |
+          always() && inputs.include_repo_e2e &&
+          (inputs.live_suite_filter == '' || inputs.live_suite_filter == matrix.suite_id) &&
+          matrix.suite_id == 'openshell-e2e'
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: openshell-gateway-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/openshell-e2e-${{ github.run_id }}-${{ github.run_attempt }}.log
+          if-no-files-found: ignore
+          retention-days: 7
+
   validate_docker_e2e:
     needs: [validate_selected_ref, prepare_docker_e2e_image, plan_release_workflow_matrices]
     if: (!inputs.prepare_only) && inputs.include_release_path_suites && inputs.docker_lanes == '' && needs.plan_release_workflow_matrices.outputs.docker_e2e_count != '0'

extensions/openshell/src/backend.e2e.test.ts

@@ -181,7 +181,7 @@ function trimTrailingNewline(value: string): string {
   return value.replace(/\r?\n$/, "");
 }
 
-async function startHostPolicyServer(): Promise<HostPolicyServer> {
+async function startHostPolicyServer(gatewayName: string): Promise<HostPolicyServer> {
   const port = await allocatePort();
   const responseBody = JSON.stringify({ ok: true, message: "hello-from-host" });
   const serverScript = `from http.server import BaseHTTPRequestHandler, HTTPServer
@@ -218,6 +218,8 @@ HTTPServer(("0.0.0.0", 8000), Handler).serve_forever()
       "run",
       "--detach",
       "--rm",
+      "--label",
+      `openclaw.ai/openshell-e2e-gateway=${gatewayName}`,
       "-e",
       `RESPONSE_BODY=${responseBody}`,
       "-p",
@@ -452,7 +454,7 @@ describe("openshell sandbox backend e2e", () => {
         process.env.HOME = env.HOME;
         process.env.XDG_CONFIG_HOME = env.XDG_CONFIG_HOME;
         process.env.XDG_CACHE_HOME = env.XDG_CACHE_HOME;
-        hostPolicyServer = await startHostPolicyServer();
+        hostPolicyServer = await startHostPolicyServer(gatewayName);
         if (!hostPolicyServer) {
           throw new Error("failed to start host policy server");
         }

test/scripts/fixtures/openshell-workflow-lifecycle.py

@@ -0,0 +1,283 @@
+"""Synthetic commands and a Linux subreaper for the unchanged workflow shell."""
+import ctypes
+import json
+import os
+import pathlib
+import selectors
+import select
+import shlex
+import signal
+import socket
+import struct
+import subprocess
+import sys
+
+
+def connect():
+    connection = socket.socket(socket.AF_UNIX, socket.SOCK_SEQPACKET)
+    connection.connect("\0" + os.environ["OPENSHELL_FIXTURE_SOCKET"])
+    return connection
+
+
+def request(connection, value):
+    connection.sendall(json.dumps(value).encode())
+    response = connection.recv(65536)
+    if not response:
+        raise RuntimeError("fixture controller closed before acknowledging the command")
+    return json.loads(response)
+
+
+def hold(role, ready_fd=None):
+    with connect() as connection:
+        def terminate(_signum, _frame):
+            connection.sendall(json.dumps({"event": "term", "role": role}).encode())
+            os._exit(0)
+
+        signal.signal(signal.SIGTERM, terminate)
+        request(connection, {"event": "hold", "role": role})
+        connection.sendall(json.dumps({"event": "ready", "role": role}).encode())
+        if ready_fd is not None:
+            os.write(ready_fd, b"1")
+            os.close(ready_fd)
+        while True:
+            signal.pause()
+
+
+def actor(command, args):
+    if command == "openshell-gateway" and args[0] != "generate-certs":
+        hold("gateway")
+    with connect() as connection:
+        answer = request(connection, {
+            "event": "call", "command": command, "args": args,
+            "fixtureRoot": str(pathlib.Path(os.environ["XDG_CONFIG_HOME"]).parent),
+        })
+    if answer.get("action") == "descendant":
+        read_fd, write_fd = os.pipe()
+        if os.fork() == 0:
+            os.close(read_fd)
+            hold("descendant", write_fd)
+        os.close(write_fd)
+        if os.read(read_fd, 1) != b"1":
+            raise RuntimeError("descendant exited before readiness")
+        os.close(read_fd)
+        return 42
+    if answer.get("action") == "hold":
+        hold("suite")
+    sys.stdout.write(answer.get("stdout", ""))
+    return answer.get("code", 0)
+
+
+def supervise(root, script, scenario):
+    if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0:
+        raise OSError(ctypes.get_errno(), "PR_SET_CHILD_SUBREAPER failed")
+    namespace = "openshell-e2e-123-2"
+    report = {"calls": [], "terminated": [], "emergency": [], "reapedDescendants": 0}
+    containers = {
+        "aa11": {"openshell.ai/managed-by": "openshell", "openshell.ai/sandbox-namespace": namespace},
+        "bb22": {"openshell.ai/managed-by": "openshell", "openshell.ai/sandbox-namespace": namespace},
+        "cc33": {"openshell.ai/managed-by": "openshell", "openshell.ai/sandbox-namespace": namespace + "-other"},
+        "dd44": {"openshell.ai/managed-by": "unrelated", "openshell.ai/sandbox-namespace": namespace},
+        "ee55": {"openclaw.ai/openshell-e2e-gateway": namespace},
+        "ff66": {"openclaw.ai/openshell-e2e-gateway": namespace + "-other"},
+    }
+    networks = {namespace, namespace + "-other", "another-network"}
+    bin_dir = root / "bin"
+    bin_dir.mkdir()
+    runner_temp = root / "runner"
+    runner_temp.mkdir()
+    for command in ("openshell-gateway", "openshell", "fixture-suite", "docker", "sleep"):
+        executable = bin_dir / command
+        executable.write_text("#!/bin/sh\nexec " + " ".join(map(shlex.quote, (
+            sys.executable, "-I", "-S", str(pathlib.Path(__file__).resolve()), "actor", command,
+        ))) + ' "$@"\n')
+        executable.chmod(0o755)
+    address = "openshell-lifecycle-" + str(os.getpid())
+    selector = selectors.DefaultSelector()
+    listener = socket.socket(socket.AF_UNIX, socket.SOCK_SEQPACKET)
+    listener.bind("\0" + address)
+    listener.listen()
+    selector.register(listener, selectors.EVENT_READ, "accept")
+    wake_read, wake_write = os.pipe2(os.O_NONBLOCK | os.O_CLOEXEC)
+    signal.set_wakeup_fd(wake_write)
+    interrupted = False
+
+    def interrupt(_signum, _frame):
+        nonlocal interrupted
+        interrupted = True
+
+    signal.signal(signal.SIGCHLD, lambda *_: None)
+    signal.signal(signal.SIGTERM, interrupt)
+    signal.signal(signal.SIGINT, interrupt)
+    selector.register(wake_read, selectors.EVENT_READ, "signal")
+    shell = subprocess.Popen(["bash", "--noprofile", "--norc", str(script)], env={
+        "PATH": str(bin_dir) + ":" + os.environ["PATH"], "HOME": str(root),
+        "RUNNER_TEMP": str(runner_temp), "GITHUB_RUN_ID": "123", "GITHUB_RUN_ATTEMPT": "2",
+        "OPENSHELL_FIXTURE_SOCKET": address,
+    })
+    shell_fd = os.pidfd_open(shell.pid)
+    owned = {shell.pid: {"fd": shell_fd, "role": "shell", "exited": False, "reaped": False}}
+    selector.register(shell_fd, selectors.EVENT_READ, ("exit", shell.pid))
+    connections = set()
+    pending = []
+    gateway_ready = False
+    whoami_count = 0
+
+    def reply(connection, value=None):
+        connection.sendall(json.dumps(value or {}).encode())
+
+    def stop_owned():
+        for entry in owned.values():
+            if entry.get("emergencySent") or select.select([entry["fd"]], [], [], 0)[0]:
+                continue
+            try:
+                signal.pidfd_send_signal(entry["fd"], signal.SIGKILL)
+                entry["emergencySent"] = True
+                report["emergency"].append(entry["role"])
+            except ProcessLookupError:
+                pass
+
+    def docker(args):
+        if args[:2] == ["ps", "-aq"]:
+            if scenario == "docker-query-failure":
+                return {"code": 17}
+            filters = args[2:]
+            assert len(filters) % 2 == 0
+            labels = {}
+            for flag, value in zip(filters[::2], filters[1::2]):
+                assert flag == "--filter" and value.startswith("label=")
+                key, value = value[6:].split("=", 1)
+                labels[key] = value
+            selected = [key for key, actual in containers.items()
+                        if all(actual.get(label) == value for label, value in labels.items())]
+            return {"stdout": "\n".join(selected) + "\n"}
+        if args[:3] == ["rm", "-f", "--"]:
+            for key in args[3:]:
+                del containers[key]
+            return {}
+        if args == ["network", "ls", "--format", "{{.Name}}"]:
+            return {"stdout": "\n".join(sorted(networks)) + "\n"}
+        if args[:2] == ["network", "rm"] and len(args) == 3:
+            if scenario == "docker-mutation-failure":
+                return {"code": 19}
+            networks.remove(args[2])
+            return {}
+        raise AssertionError("unexpected Docker command: " + repr(args))
+
+    try:
+        while True:
+            for key, _ in selector.select():
+                if key.data == "accept":
+                    connection, _ = listener.accept()
+                    connections.add(connection)
+                    selector.register(connection, selectors.EVENT_READ, "message")
+                elif key.data == "signal":
+                    os.read(wake_read, 65536)
+                elif isinstance(key.data, tuple):
+                    entry = owned[key.data[1]]
+                    entry["exited"] = True
+                    selector.unregister(key.fd)
+                else:
+                    connection = key.fileobj
+                    packet = connection.recv(65536)
+                    if not packet:
+                        selector.unregister(connection)
+                        connections.remove(connection)
+                        connection.close()
+                        continue
+                    message = json.loads(packet)
+                    event = message["event"]
+                    if event == "hold":
+                        # The actor waits for this acknowledgement, pinning its lifetime.
+                        pid, _, _ = struct.unpack("3i", connection.getsockopt(
+                            socket.SOL_SOCKET, socket.SO_PEERCRED, struct.calcsize("3i")))
+                        descriptor = os.pidfd_open(pid)
+                        owned[pid] = {"fd": descriptor, "role": message["role"],
+                                      "exited": False, "reaped": False}
+                        selector.register(descriptor, selectors.EVENT_READ, ("exit", pid))
+                        reply(connection)
+                    elif event == "ready":
+                        if message["role"] == "gateway":
+                            gateway_ready = True
+                        elif message["role"] == "suite" and scenario == "cancel":
+                            signal.pidfd_send_signal(shell_fd, signal.SIGTERM)
+                    elif event == "term":
+                        report["terminated"].append(message["role"])
+                    else:
+                        assert event == "call"
+                        command, args = message["command"], message["args"]
+                        report["fixtureRoot"] = message["fixtureRoot"]
+                        report["calls"].append({"command": command, "args": args})
+                        if command == "docker":
+                            reply(connection, docker(args))
+                        elif command == "sleep":
+                            assert args in (["0.2"], ["1"])
+                            pending.append((connection, "cleanup" if args == ["0.2"] else "ready"))
+                        elif command == "openshell-gateway":
+                            assert args[0] == "generate-certs"
+                            pathlib.Path(message["fixtureRoot"], "pki").mkdir()
+                            reply(connection)
+                        elif command == "openshell" and args[:2] == ["gateway", "add"]:
+                            pending.append((connection, "gateway"))
+                        elif command == "openshell" and args[-1] == "whoami":
+                            whoami_count += 1
+                            failed = scenario in ("leader-failure", "docker-mutation-failure")
+                            reply(connection, {"action": "descendant"} if failed and whoami_count == 1 else {})
+                        elif command == "fixture-suite":
+                            reply(connection, {"action": "hold"} if scenario == "cancel" else {})
+                        else:
+                            assert command == "openshell" and args[-2:] == ["sandbox", "list"]
+                            reply(connection)
+            no_children = False
+            # Drain SIGCHLD notifications here; Popen.wait must not compete for children.
+            while True:
+                try:
+                    pid, code = os.waitpid(-1, os.WNOHANG)
+                except ChildProcessError:
+                    no_children = True
+                    break
+                if pid == 0:
+                    break
+                if pid == shell.pid:
+                    shell.returncode = os.waitstatus_to_exitcode(code)
+                    report["shellStatus"] = shell.returncode
+                if pid in owned:
+                    owned[pid]["reaped"] = True
+                    if owned[pid]["role"] == "descendant":
+                        report["reapedDescendants"] += 1
+            if interrupted or (shell.returncode is not None and not no_children):
+                stop_owned()
+            for connection, gate in pending[:]:
+                # Replace elapsed cleanup grace with observed exit and adoption receipts.
+                settled = all(entry["exited"] and (entry["role"] != "descendant" or entry["reaped"])
+                              for entry in owned.values() if entry["role"] != "shell")
+                if gate == "ready" or (gate == "gateway" and gateway_ready) or (gate == "cleanup" and settled):
+                    reply(connection)
+                    pending.remove((connection, gate))
+            if shell.returncode is not None and no_children and not connections:
+                break
+        assert not interrupted, "fixture controller was interrupted"
+    finally:
+        stop_owned()
+        for connection in connections:
+            connection.close()
+        listener.close()
+        # Closing the protocol releases unregistered short-lived commands too.
+        while True:
+            try:
+                os.waitpid(-1, 0)
+            except ChildProcessError:
+                break
+        for entry in owned.values():
+            os.close(entry["fd"])
+        signal.set_wakeup_fd(-1)
+        os.close(wake_read)
+        os.close(wake_write)
+        selector.close()
+    report["containers"] = sorted(containers)
+    report["networks"] = sorted(networks)
+    (root / "lifecycle.json").write_text(json.dumps(report))
+
+
+if sys.argv[1] == "actor":
+    sys.exit(actor(sys.argv[2], sys.argv[3:]))
+supervise(pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]), sys.argv[3])

test/scripts/openshell-workflow.lifecycle.test.ts

@@ -0,0 +1,93 @@
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { expectDefined } from "@openclaw/normalization-core";
+import { expect } from "vitest";
+import { parse } from "yaml";
+import { createCommandTest } from "../helpers/command-fixture.js";
+
+const test = createCommandTest();
+const workflowPath = ".github/workflows/openclaw-live-and-e2e-checks-reusable.yml";
+const fixturePath = fileURLToPath(
+  new URL("./fixtures/openshell-workflow-lifecycle.py", import.meta.url),
+);
+const namespace = "openshell-e2e-123-2";
+
+type LifecycleReport = {
+  shellStatus: number;
+  calls: { command: string; args: string[] }[];
+  terminated: string[];
+  emergency: string[];
+  reapedDescendants: number;
+  fixtureRoot: string;
+  containers: string[];
+  networks: string[];
+};
+
+test.skipIf(process.platform !== "linux").for([
+  { scenario: "success", exit: 0, retained: false, descendant: false, suite: true },
+  { scenario: "leader-failure", exit: 42, retained: false, descendant: true, suite: false },
+  { scenario: "cancel", exit: 143, retained: false, descendant: false, suite: true },
+  { scenario: "docker-query-failure", exit: 1, retained: true, descendant: false, suite: true },
+  { scenario: "docker-mutation-failure", exit: 42, retained: true, descendant: true, suite: false },
+])(
+  "settles the OpenShell workflow's owned resources on $scenario",
+  async (scenario, { command }) => {
+    const steps = parse(readFileSync(workflowPath, "utf8")).jobs.validate_special_e2e
+      .steps as Array<{
+      name?: string;
+      run?: string;
+    }>;
+    const run = expectDefined(
+      steps.find((step) => step.name === "Run ${{ matrix.label }}")?.run,
+      "OpenShell workflow run step",
+    );
+    const root = command.createTempDir("openclaw-openshell-lifecycle-");
+    const script = join(root, "run.sh");
+    writeFileSync(script, run.replaceAll("${{ matrix.command }}", "fixture-suite"));
+    const result = await command.run(
+      "python3",
+      ["-I", "-S", fixturePath, root, script, scenario.scenario],
+      { env: { PATH: process.env.PATH }, maxBuffer: 64 * 1024 },
+    );
+
+    expect(result.error, result.stderr).toBeUndefined();
+    expect(result.status, result.stderr).toBe(0);
+    const report = JSON.parse(
+      readFileSync(join(root, "lifecycle.json"), "utf8"),
+    ) as LifecycleReport;
+    expect(report.shellStatus, result.stderr).toBe(scenario.exit);
+    expect(report.emergency).toEqual([]);
+    expect(report.terminated).toContain("gateway");
+    expect(
+      report.calls.filter((call) => call.command === "openshell" && call.args.at(-1) === "whoami"),
+    ).toHaveLength(1);
+    expect(report.calls.some((call) => call.command === "fixture-suite")).toBe(scenario.suite);
+    if (scenario.descendant) {
+      expect(report.terminated).toContain("descendant");
+      expect(report.reapedDescendants).toBe(1);
+      expect(report.calls.some((call) => call.args.slice(-2).join(" ") === "sandbox list")).toBe(
+        false,
+      );
+    }
+    if (scenario.scenario === "cancel") {
+      expect(report.terminated).toContain("suite");
+    }
+    expect(existsSync(report.fixtureRoot)).toBe(scenario.retained);
+    expect(report.containers).toEqual(
+      scenario.scenario === "docker-query-failure"
+        ? ["aa11", "bb22", "cc33", "dd44", "ee55", "ff66"]
+        : ["cc33", "dd44", "ff66"],
+    );
+    expect(report.networks).toEqual(
+      scenario.retained
+        ? ["another-network", namespace, `${namespace}-other`]
+        : ["another-network", `${namespace}-other`],
+    );
+    if (scenario.scenario === "docker-query-failure") {
+      expect(
+        report.calls.filter((call) => call.command === "docker").map((call) => call.args[0]),
+      ).toEqual(["ps"]);
+    }
+  },
+);

test/scripts/openshell-workflow.test.ts

@@ -0,0 +1,157 @@
+import { spawnSync } from "node:child_process";
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { expectDefined } from "@openclaw/normalization-core";
+import { afterEach, describe, expect, it } from "vitest";
+import { parse } from "yaml";
+import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js";
+
+const LIVE_E2E_WORKFLOW_PATH = ".github/workflows/openclaw-live-and-e2e-checks-reusable.yml";
+const tempDirs = useAutoCleanupTempDirTracker(afterEach);
+
+function readNulSeparatedArgs(filePath: string): string[] {
+  return readFileSync(filePath, "utf8").split("\0").filter(Boolean);
+}
+
+function workflowSteps() {
+  return parse(readFileSync(LIVE_E2E_WORKFLOW_PATH, "utf8")).jobs.validate_special_e2e
+    .steps as Array<{
+    name?: string;
+    run?: string;
+    env?: Record<string, string>;
+  }>;
+}
+
+describe.skipIf(process.platform !== "linux")("OpenShell hosted bootstrap", () => {
+  it("builds the private QA runtime required by special E2E", () => {
+    expect(
+      workflowSteps().find((step) => step.name === "Build dist for special E2E"),
+    ).toMatchObject({
+      run: "pnpm build",
+      env: { OPENCLAW_BUILD_PRIVATE_QA: "1" },
+    });
+  });
+
+  it.each([
+    {
+      name: "success",
+      curlExit: 0,
+      checksumExit: 0,
+      installExit: 0,
+      calls: "download\nverify\ninstall\nversion\n",
+    },
+    {
+      name: "partial download",
+      curlExit: 28,
+      checksumExit: 0,
+      installExit: 0,
+      calls: "download\n",
+    },
+    {
+      name: "checksum failure",
+      curlExit: 0,
+      checksumExit: 1,
+      installExit: 0,
+      calls: "download\nverify\n",
+    },
+    {
+      name: "install failure",
+      curlExit: 0,
+      checksumExit: 0,
+      installExit: 42,
+      calls: "download\nverify\ninstall\n",
+    },
+  ])("verifies the OpenShell package before installation and cleans it on $name", (scenario) => {
+    const installStep = expectDefined(
+      workflowSteps().find((step) => step.name === "Install OpenShell CLI"),
+      "OpenShell install step",
+    );
+    const run = expectDefined(installStep.run, "OpenShell install command");
+    const root = tempDirs.make("openclaw-openshell-package-");
+    const result = spawnSync(
+      "bash",
+      [
+        "--noprofile",
+        "--norc",
+        "-c",
+        `
+curl() {
+  printf '%s\\0' "$@" > "$RUNNER_TEMP/curl-args"
+  local output=""
+  while (( $# )); do
+    if [[ "$1" == -o ]]; then shift; output="$1"; fi
+    shift
+  done
+  printf '%s' "$output" > "$RUNNER_TEMP/package-path"
+  printf 'fixture package' > "$output"
+  printf 'download\\n' >> "$RUNNER_TEMP/calls"
+  return "$CURL_EXIT"
+}
+sha256sum() {
+  printf '%s\\0' "$@" > "$RUNNER_TEMP/checksum-args"
+  cat > "$RUNNER_TEMP/checksum-input"
+  printf 'verify\\n' >> "$RUNNER_TEMP/calls"
+  return "$CHECKSUM_EXIT"
+}
+sudo() {
+  printf '%s\\0' "$@" > "$RUNNER_TEMP/install-args"
+  local package
+  for package in "$@"; do :; done
+  cat "$package" > "$RUNNER_TEMP/installed-bytes"
+  printf 'install\\n' >> "$RUNNER_TEMP/calls"
+  return "$INSTALL_EXIT"
+}
+openshell() {
+  [[ "$*" == --version ]] || return 99
+  printf 'version\\n' >> "$RUNNER_TEMP/calls"
+}
+${run}`,
+      ],
+      {
+        encoding: "utf8",
+        timeout: 5_000,
+        env: {
+          PATH: process.env.PATH,
+          RUNNER_TEMP: root,
+          CURL_EXIT: String(scenario.curlExit),
+          CHECKSUM_EXIT: String(scenario.checksumExit),
+          INSTALL_EXIT: String(scenario.installExit),
+        },
+      },
+    );
+    expect(result.error).toBeUndefined();
+    expect(result.status, result.stderr).toBe(
+      scenario.curlExit || scenario.checksumExit || scenario.installExit,
+    );
+    expect(readFileSync(join(root, "calls"), "utf8")).toBe(scenario.calls);
+    const packagePath = readFileSync(join(root, "package-path"), "utf8");
+    expect(readNulSeparatedArgs(join(root, "curl-args"))).toEqual([
+      "-LsSf",
+      "--connect-timeout",
+      "10",
+      "--max-time",
+      "120",
+      "-o",
+      packagePath,
+      "https://github.com/NVIDIA/OpenShell/releases/download/v0.0.109/openshell_0.0.109-1_amd64.deb",
+    ]);
+    if (scenario.curlExit === 0) {
+      expect(readNulSeparatedArgs(join(root, "checksum-args"))).toEqual(["--check", "-"]);
+      expect(readFileSync(join(root, "checksum-input"), "utf8")).toBe(
+        `0364a21f279023a241d967309ebb0177dadb66bd792eeb3daf542283158ca40f  ${packagePath}\n`,
+      );
+    }
+    if (scenario.curlExit === 0 && scenario.checksumExit === 0) {
+      expect(readNulSeparatedArgs(join(root, "install-args"))).toEqual([
+        "env",
+        "DEBIAN_FRONTEND=noninteractive",
+        "apt-get",
+        "install",
+        "-y",
+        packagePath,
+      ]);
+      expect(readFileSync(join(root, "installed-bytes"), "utf8")).toBe("fixture package");
+    }
+    expect(existsSync(packagePath)).toBe(false);
+  });
+});

test/scripts/package-acceptance-workflow.test.ts

@@ -12709,7 +12709,7 @@ printf '%s\\n' "$DEEPSEEK_API_KEY" "$DEEPINFRA_API_KEY"`,
       for (const [jobName, job] of Object.entries(jobs)) {
         for (const step of job.steps ?? []) {
           if (step.run === "pnpm build") {
-            expect(step.env, `${workflowPath}:${jobName}:${step.name}`).toEqual({
+            expect(step.env, `${workflowPath}:${jobName}:${step.name}`).toMatchObject({
               NODE_OPTIONS: "--max-old-space-size=8192",
             });
           }

test/scripts/test-install-sh-docker.test.ts

@@ -1234,27 +1234,6 @@ printf 'status=%s\\n' "$status"
     expect(workflow).toContain("reachable from an OpenClaw branch or release tag");
   });
 
-  it("downloads the OpenShell installer completely before execution", () => {
-    const workflow = parse(readFileSync(LIVE_E2E_WORKFLOW_PATH, "utf8"));
-    const steps = workflow.jobs.validate_special_e2e.steps as Array<{
-      name?: string;
-      run?: string;
-    }>;
-    const installStep = expectDefined(
-      steps.find((step) => step.name === "Install OpenShell CLI"),
-      "OpenShell install step",
-    );
-    const run = expectDefined(installStep.run, "OpenShell install command");
-
-    expect(run).toContain('installer_path="$(mktemp "${RUNNER_TEMP}/openshell-install.XXXXXX")"');
-    expect(run).toContain("curl -LsSf --connect-timeout 10 --max-time 120 \\");
-    expect(run).toContain('-o "$installer_path"');
-    expect(run).toContain('sh "$installer_path"');
-    expect(run).toContain("trap 'rm -f \"$installer_path\"' EXIT");
-    expect(run.indexOf('-o "$installer_path"')).toBeLessThan(run.indexOf('sh "$installer_path"'));
-    expect(run).not.toContain("install.sh | sh");
-  });
-
   it("prints package size audits for release smoke tarballs", () => {
     const script = readFileSync(SCRIPT_PATH, "utf8");