diff --git a/.github/workflows/npm-telegram-beta-e2e.yml b/.github/workflows/npm-telegram-beta-e2e.yml index fb50e914e0d..4d734fb95c2 100644 --- a/.github/workflows/npm-telegram-beta-e2e.yml +++ b/.github/workflows/npm-telegram-beta-e2e.yml @@ -247,14 +247,117 @@ jobs: trap append_telegram_summary EXIT if [[ -n "${PACKAGE_ARTIFACT_NAME// }" ]]; then - mapfile -t package_tgzs < <(find .artifacts/telegram-package-under-test -type f -name "*.tgz" | sort) - if [[ "${#package_tgzs[@]}" -ne 1 ]]; then - echo "package artifact ${PACKAGE_ARTIFACT_NAME} must contain exactly one .tgz; found ${#package_tgzs[@]}" >&2 - exit 1 + package_dir=".artifacts/telegram-package-under-test" + manifest="${package_dir}/preflight-manifest.json" + if [[ -f "${manifest}" ]]; then + package_tgz="$( + node - "${manifest}" "${package_dir}" <<'NODE' + const crypto = require("node:crypto"); + const childProcess = require("node:child_process"); + const fs = require("node:fs"); + const path = require("node:path"); + const [manifestPath, packageDir] = process.argv.slice(2); + const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8")); + const entries = [ + { + packageName: "openclaw", + packageVersion: manifest.packageVersion, + tarballName: manifest.tarballName, + tarballSha256: manifest.tarballSha256, + }, + ...(Array.isArray(manifest.dependencyTarballs) ? manifest.dependencyTarballs : []), + ]; + const packageNames = new Set(); + const tarballNames = new Set(); + for (const entry of entries) { + if ( + !entry.packageName || + !entry.packageVersion || + !entry.tarballName || + !entry.tarballSha256 || + entry.tarballName !== path.basename(entry.tarballName) + ) { + throw new Error("package artifact manifest contains invalid tarball metadata"); + } + if (packageNames.has(entry.packageName) || tarballNames.has(entry.tarballName)) { + throw new Error("package artifact manifest contains duplicate package metadata"); + } + packageNames.add(entry.packageName); + tarballNames.add(entry.tarballName); + const tarballPath = path.join(packageDir, entry.tarballName); + if (!fs.existsSync(tarballPath)) { + throw new Error(`package artifact is missing ${entry.tarballName}`); + } + const digest = crypto.createHash("sha256").update(fs.readFileSync(tarballPath)).digest("hex"); + if (digest !== entry.tarballSha256) { + throw new Error(`package artifact digest mismatch for ${entry.packageName}`); + } + const packageJson = JSON.parse( + childProcess.execFileSync("tar", ["-xOf", tarballPath, "package/package.json"], { + encoding: "utf8", + }), + ); + if (packageJson.name !== entry.packageName || packageJson.version !== entry.packageVersion) { + throw new Error(`package artifact metadata mismatch for ${entry.packageName}`); + } + } + const artifactTarballs = fs + .readdirSync(packageDir, { withFileTypes: true }) + .filter((entry) => entry.isFile() && entry.name.endsWith(".tgz")) + .map((entry) => entry.name); + if ( + artifactTarballs.length !== tarballNames.size || + artifactTarballs.some((name) => !tarballNames.has(name)) + ) { + throw new Error("package artifact tarball set does not match preflight manifest"); + } + process.stdout.write(path.join(packageDir, manifest.tarballName)); + NODE + )" + export OPENCLAW_NPM_TELEGRAM_PACKAGE_DIR="${package_dir}" + else + mapfile -t package_tgzs < <(find "${package_dir}" -type f -name "*.tgz" | sort) + if [[ "${#package_tgzs[@]}" -ne 1 ]]; then + echo "package artifact ${PACKAGE_ARTIFACT_NAME} without a preflight manifest must contain exactly one tgz; found ${#package_tgzs[@]}" >&2 + exit 1 + fi + package_tgz="${package_tgzs[0]}" + candidate_manifest="${package_dir}/package-candidate.json" + node - "${package_tgz}" "${candidate_manifest}" <<'NODE' + const crypto = require("node:crypto"); + const childProcess = require("node:child_process"); + const fs = require("node:fs"); + const path = require("node:path"); + const [tarballPath, manifestPath] = process.argv.slice(2); + const packageJson = JSON.parse( + childProcess.execFileSync("tar", ["-xOf", tarballPath, "package/package.json"], { + encoding: "utf8", + }), + ); + if (packageJson.name !== "openclaw" || typeof packageJson.version !== "string") { + throw new Error("package artifact tgz is not a versioned openclaw package"); + } + if (fs.existsSync(manifestPath)) { + const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8")); + const expectedTarballName = path.basename(manifest.tarball || ""); + if ( + manifest.name !== "openclaw" || + manifest.version !== packageJson.version || + expectedTarballName !== path.basename(tarballPath) || + typeof manifest.sha256 !== "string" + ) { + throw new Error("package candidate manifest does not match the OpenClaw tarball"); + } + const digest = crypto.createHash("sha256").update(fs.readFileSync(tarballPath)).digest("hex"); + if (digest !== manifest.sha256) { + throw new Error("package candidate digest mismatch"); + } + } + NODE fi - export OPENCLAW_NPM_TELEGRAM_PACKAGE_TGZ="${package_tgzs[0]}" + export OPENCLAW_NPM_TELEGRAM_PACKAGE_TGZ="${package_tgz}" if [[ -z "${OPENCLAW_NPM_TELEGRAM_PACKAGE_LABEL// }" ]]; then - export OPENCLAW_NPM_TELEGRAM_PACKAGE_LABEL="$(basename "${package_tgzs[0]}")" + export OPENCLAW_NPM_TELEGRAM_PACKAGE_LABEL="$(basename "${package_tgz}")" fi elif [[ -z "${OPENCLAW_NPM_TELEGRAM_PACKAGE_LABEL// }" ]]; then export OPENCLAW_NPM_TELEGRAM_PACKAGE_LABEL="${OPENCLAW_NPM_TELEGRAM_PACKAGE_SPEC}" diff --git a/.github/workflows/openclaw-npm-release.yml b/.github/workflows/openclaw-npm-release.yml index f1f8f423850..612dad0cb94 100644 --- a/.github/workflows/openclaw-npm-release.yml +++ b/.github/workflows/openclaw-npm-release.yml @@ -396,17 +396,33 @@ jobs: fi TARBALL_NAME="$PACK_NAME" TARBALL_SHA256="$(sha256sum "$PACK_PATH" | awk '{print $1}')" + mapfile -t AI_TARBALLS < <(find "$AI_RUNTIME_TARBALL_DIR" -maxdepth 1 -type f -name 'openclaw-ai-*.tgz' -print | sort) + if [[ "${#AI_TARBALLS[@]}" -ne 1 ]]; then + echo "Expected exactly one packed @openclaw/ai tarball, found ${#AI_TARBALLS[@]}." >&2 + exit 1 + fi + AI_TARBALL_PATH="${AI_TARBALLS[0]}" + AI_TARBALL_NAME="$(basename "$AI_TARBALL_PATH")" + AI_TARBALL_SHA256="$(sha256sum "$AI_TARBALL_PATH" | awk '{print $1}')" + AI_PACKAGE_VERSION="$( + tar -xOf "$AI_TARBALL_PATH" package/package.json | + node -e 'let raw = ""; process.stdin.on("data", (chunk) => (raw += chunk)); process.stdin.on("end", () => process.stdout.write(JSON.parse(raw).version));' + )" + if [[ "$AI_PACKAGE_VERSION" != "$PACKAGE_VERSION" ]]; then + echo "Prepared @openclaw/ai version ${AI_PACKAGE_VERSION} does not match openclaw ${PACKAGE_VERSION}." >&2 + exit 1 + fi ARTIFACT_DIR="$RUNNER_TEMP/openclaw-npm-preflight" rm -rf "$ARTIFACT_DIR" mkdir -p "$ARTIFACT_DIR" cp "$PACK_PATH" "$ARTIFACT_DIR/" - cp "$AI_RUNTIME_TARBALL_DIR"/*.tgz "$ARTIFACT_DIR/" + cp "$AI_TARBALL_PATH" "$ARTIFACT_DIR/" cp "$AI_RUNTIME_TARBALL_DIR/SHA256SUMS" "$ARTIFACT_DIR/ai-runtime-SHA256SUMS" cp -R "$DEPENDENCY_EVIDENCE_DIR" "$ARTIFACT_DIR/dependency-evidence" printf '%s\n' "$RELEASE_TAG" > "$ARTIFACT_DIR/release-tag.txt" printf '%s\n' "$RELEASE_SHA" > "$ARTIFACT_DIR/release-sha.txt" printf '%s\n' "$RELEASE_NPM_DIST_TAG" > "$ARTIFACT_DIR/release-npm-dist-tag.txt" - ARTIFACT_DIR="$ARTIFACT_DIR" RELEASE_TAG="$RELEASE_TAG" RELEASE_SHA="$RELEASE_SHA" RELEASE_NPM_DIST_TAG="$RELEASE_NPM_DIST_TAG" PACKAGE_VERSION="$PACKAGE_VERSION" TARBALL_NAME="$TARBALL_NAME" TARBALL_SHA256="$TARBALL_SHA256" node <<'NODE' + ARTIFACT_DIR="$ARTIFACT_DIR" RELEASE_TAG="$RELEASE_TAG" RELEASE_SHA="$RELEASE_SHA" RELEASE_NPM_DIST_TAG="$RELEASE_NPM_DIST_TAG" PACKAGE_VERSION="$PACKAGE_VERSION" TARBALL_NAME="$TARBALL_NAME" TARBALL_SHA256="$TARBALL_SHA256" AI_PACKAGE_VERSION="$AI_PACKAGE_VERSION" AI_TARBALL_NAME="$AI_TARBALL_NAME" AI_TARBALL_SHA256="$AI_TARBALL_SHA256" node <<'NODE' const fs = require("node:fs"); const path = require("node:path"); const manifest = { @@ -418,6 +434,14 @@ jobs: packageVersion: process.env.PACKAGE_VERSION, tarballName: process.env.TARBALL_NAME, tarballSha256: process.env.TARBALL_SHA256, + dependencyTarballs: [ + { + packageName: "@openclaw/ai", + packageVersion: process.env.AI_PACKAGE_VERSION, + tarballName: process.env.AI_TARBALL_NAME, + tarballSha256: process.env.AI_TARBALL_SHA256, + }, + ], dependencyEvidenceDir: "dependency-evidence", dependencyEvidenceManifest: "dependency-evidence/dependency-evidence-manifest.json", }; diff --git a/scripts/e2e/lib/plugins/npm-registry-server.mjs b/scripts/e2e/lib/plugins/npm-registry-server.mjs index 0c59226f194..acfc52ebabb 100644 --- a/scripts/e2e/lib/plugins/npm-registry-server.mjs +++ b/scripts/e2e/lib/plugins/npm-registry-server.mjs @@ -1,3 +1,4 @@ +import { execFileSync } from "node:child_process"; // Fixture npm registry server for plugin E2E scenarios. import crypto from "node:crypto"; import fs from "node:fs"; @@ -5,6 +6,25 @@ import http from "node:http"; import path from "node:path"; const [portFile, ...packageArgs] = process.argv.slice(2); +function normalizeUpstreamRegistry(raw) { + if (!raw) { + return undefined; + } + const url = new URL(raw); + if ( + (url.protocol !== "http:" && url.protocol !== "https:") || + url.username || + url.password || + url.pathname !== "/" || + url.search || + url.hash + ) { + throw new Error("OPENCLAW_NPM_REGISTRY_UPSTREAM must be an HTTP(S) origin"); + } + return url.origin; +} + +const upstreamRegistry = normalizeUpstreamRegistry(process.env.OPENCLAW_NPM_REGISTRY_UPSTREAM); if (!portFile || packageArgs.length === 0 || packageArgs.length % 3 !== 0) { console.error( @@ -14,6 +34,25 @@ if (!portFile || packageArgs.length === 0 || packageArgs.length % 3 !== 0) { } const packages = new Map(); + +function readPackageManifest(tarballPath, packageName) { + try { + const packageJson = JSON.parse( + execFileSync("tar", ["-xOf", tarballPath, "package/package.json"], { + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }), + ); + return packageJson && typeof packageJson === "object" && !Array.isArray(packageJson) + ? packageJson + : {}; + } catch { + return packageName === "@openclaw/demo-plugin-npm" + ? { dependencies: { "is-number": "7.0.0" } } + : {}; + } +} + for (let index = 0; index < packageArgs.length; index += 3) { const packageName = packageArgs[index]; const version = packageArgs[index + 1]; @@ -28,8 +67,8 @@ for (let index = 0; index < packageArgs.length; index += 3) { existing.latestVersion = version; existing.versions.set(version, { archive, - dependencies: packageName === "@openclaw/demo-plugin-npm" ? { "is-number": "7.0.0" } : {}, integrity: `sha512-${crypto.createHash("sha512").update(archive).digest("base64")}`, + manifest: readPackageManifest(tarballPath, packageName), shasum: crypto.createHash("sha1").update(archive).digest("hex"), tarballName: path.basename(tarballPath), version, @@ -44,7 +83,7 @@ const metadataFor = (entry, baseUrl) => ({ [...entry.versions.entries()].map(([version, versionEntry]) => [ version, { - dependencies: versionEntry.dependencies, + ...versionEntry.manifest, name: entry.packageName, version, dist: { @@ -85,9 +124,46 @@ function findTarballForPath(pathname) { return undefined; } -const server = http.createServer((request, response) => { - const url = new URL(request.url ?? "/", "http://127.0.0.1"); - const baseUrl = `http://127.0.0.1:${server.address().port}`; +function resolveUpstreamRequestUrl(rawRequestUrl) { + const raw = rawRequestUrl || "/"; + if (!raw.startsWith("/") || raw.startsWith("//") || raw.includes("\\")) { + throw new Error(`refusing non-origin registry request URL: ${JSON.stringify(raw)}`); + } + const requestUrl = new URL(raw, "http://127.0.0.1"); + return `${upstreamRegistry}${requestUrl.pathname}${requestUrl.search}`; +} + +async function proxyUpstream(rawRequestUrl, response) { + if (!upstreamRegistry) { + return false; + } + try { + const upstreamUrl = resolveUpstreamRequestUrl(rawRequestUrl); + const upstreamResponse = await fetch(upstreamUrl, { redirect: "manual" }); + const body = Buffer.from(await upstreamResponse.arrayBuffer()); + // Fetch decodes compressed bodies but preserves upstream length metadata. + // Emit the decoded size so npm clients do not truncate proxied responses. + const headers = { "content-length": String(body.length) }; + for (const name of ["content-type", "location"]) { + const value = upstreamResponse.headers.get(name); + if (value) { + headers[name] = value; + } + } + response.writeHead(upstreamResponse.status, headers); + response.end(body); + } catch (error) { + response.writeHead(502, { "content-type": "text/plain" }); + response.end(`upstream registry request failed: ${String(error)}`); + } + return true; +} + +async function handleRequest(request, response) { + const fallbackHost = `127.0.0.1:${server.address().port}`; + const requestHost = request.headers.host || fallbackHost; + const url = new URL(request.url ?? "/", `http://${requestHost}`); + const baseUrl = url.origin; if (request.method !== "GET") { response.writeHead(405, { "content-type": "text/plain" }); response.end("method not allowed"); @@ -111,10 +187,27 @@ const server = http.createServer((request, response) => { return; } + if (await proxyUpstream(request.url, response)) { + return; + } + response.writeHead(404, { "content-type": "text/plain" }); response.end(`not found: ${url.pathname}`); +} + +const server = http.createServer((request, response) => { + void handleRequest(request, response).catch((/** @type {unknown} */ error) => { + if (!response.headersSent) { + response.writeHead(500, { "content-type": "text/plain" }); + response.end(`registry request failed: ${String(error)}`); + return; + } + response.destroy(error instanceof Error ? error : new Error(String(error))); + }); }); -server.listen(0, "127.0.0.1", () => { +const bindHost = process.env.OPENCLAW_NPM_REGISTRY_BIND_HOST || "127.0.0.1"; +const requestedPort = Number(process.env.OPENCLAW_NPM_REGISTRY_PORT || 0); +server.listen(requestedPort, bindHost, () => { fs.writeFileSync(portFile, String(server.address().port)); }); diff --git a/scripts/e2e/npm-telegram-live-docker.sh b/scripts/e2e/npm-telegram-live-docker.sh index 97ccf8ca370..bb529eb5f9a 100755 --- a/scripts/e2e/npm-telegram-live-docker.sh +++ b/scripts/e2e/npm-telegram-live-docker.sh @@ -10,6 +10,7 @@ IMAGE_NAME="$(docker_e2e_resolve_image "openclaw-npm-telegram-live-e2e" OPENCLAW DOCKER_TARGET="${OPENCLAW_NPM_TELEGRAM_DOCKER_TARGET:-build}" PACKAGE_SPEC="${OPENCLAW_NPM_TELEGRAM_PACKAGE_SPEC:-openclaw@beta}" PACKAGE_TGZ="${OPENCLAW_NPM_TELEGRAM_PACKAGE_TGZ:-${OPENCLAW_CURRENT_PACKAGE_TGZ:-}}" +PACKAGE_DIR="${OPENCLAW_NPM_TELEGRAM_PACKAGE_DIR:-}" PACKAGE_LABEL="${OPENCLAW_NPM_TELEGRAM_PACKAGE_LABEL:-}" RUN_ID="${OPENCLAW_NPM_TELEGRAM_RUN_ID:-$(date -u +%Y%m%dT%H%M%SZ)-$$}" OUTPUT_DIR="${OPENCLAW_NPM_TELEGRAM_OUTPUT_DIR:-.artifacts/qa-e2e/npm-telegram-live/$RUN_ID}" @@ -77,11 +78,58 @@ resolve_package_tgz() { printf "%s/%s" "$dir" "$base" } +resolve_package_dir() { + local candidate="$1" + if [ -z "$candidate" ]; then + return 0 + fi + if [ ! -d "$candidate" ]; then + echo "OPENCLAW_NPM_TELEGRAM_PACKAGE_DIR must point to an existing directory; got: $candidate" >&2 + exit 1 + fi + (cd "$candidate" && pwd) +} + +read_package_version() { + tar -xOf "$1" package/package.json | + node -e ' +let raw = ""; +process.stdin.on("data", (chunk) => (raw += chunk)); +process.stdin.on("end", () => { + const version = JSON.parse(raw).version; + if (typeof version !== "string" || !version) { + throw new Error("package tarball is missing a version"); + } + process.stdout.write(version); +}); +' +} + package_mount_args=() +registry_helper_mount_args=() package_install_source="$PACKAGE_SPEC" package_source_kind="npm-package" resolved_package_tgz="$(resolve_package_tgz "$PACKAGE_TGZ")" -if [ -n "$resolved_package_tgz" ]; then +resolved_package_dir="$(resolve_package_dir "$PACKAGE_DIR")" +if [ -n "$resolved_package_dir" ]; then + if [ -z "$resolved_package_tgz" ]; then + echo "OPENCLAW_NPM_TELEGRAM_PACKAGE_DIR requires OPENCLAW_NPM_TELEGRAM_PACKAGE_TGZ" >&2 + exit 1 + fi + case "$resolved_package_tgz" in + "$resolved_package_dir"/*) ;; + *) + echo "OPENCLAW_NPM_TELEGRAM_PACKAGE_TGZ must be inside OPENCLAW_NPM_TELEGRAM_PACKAGE_DIR" >&2 + exit 1 + ;; + esac + package_install_source="openclaw@$(read_package_version "$resolved_package_tgz")" + package_source_kind="prepared-package-set" + package_mount_args=(-v "$resolved_package_dir:/package-under-test:ro") + registry_helper_mount_args=( + -v "$ROOT_DIR/scripts/e2e/lib/plugins/npm-registry-server.mjs:/tmp/openclaw-npm-registry-server.mjs:ro" + ) +elif [ -n "$resolved_package_tgz" ]; then package_install_source="/package-under-test/$(basename "$resolved_package_tgz")" package_source_kind="packed-tarball" package_mount_args=(-v "$resolved_package_tgz:$package_install_source:ro") @@ -248,7 +296,9 @@ run_logged docker_e2e_docker_run_cmd run --rm \ -e OPENCLAW_E2E_NPM_INSTALL_TIMEOUT="${OPENCLAW_E2E_NPM_INSTALL_TIMEOUT:-600s}" \ -e OPENCLAW_NPM_TELEGRAM_INSTALL_SOURCE="$package_install_source" \ -e OPENCLAW_NPM_TELEGRAM_PACKAGE_LABEL="$PACKAGE_LABEL" \ + -e OPENCLAW_NPM_TELEGRAM_PACKAGE_SET="$([ -n "$resolved_package_dir" ] && printf 1 || printf 0)" \ ${package_mount_args[@]+"${package_mount_args[@]}"} \ + ${registry_helper_mount_args[@]+"${registry_helper_mount_args[@]}"} \ -v "$npm_prefix_host:/npm-global" \ -i "$IMAGE_NAME" bash -s <<'EOF' set -euo pipefail @@ -261,6 +311,74 @@ install_source="${OPENCLAW_NPM_TELEGRAM_INSTALL_SOURCE:?missing OPENCLAW_NPM_TEL package_label="${OPENCLAW_NPM_TELEGRAM_PACKAGE_LABEL:-$install_source}" echo "Installing ${package_label} from ${install_source}..." +registry_pid="" +registry_log="" +cleanup_registry() { + if [ -n "$registry_pid" ]; then + kill "$registry_pid" >/dev/null 2>&1 || true + wait "$registry_pid" >/dev/null 2>&1 || true + fi + if [ -n "$registry_log" ]; then + rm -f "$registry_log" + fi +} +trap cleanup_registry EXIT + +if [ "${OPENCLAW_NPM_TELEGRAM_PACKAGE_SET:-0}" = "1" ]; then + shopt -s nullglob + package_tgzs=(/package-under-test/*.tgz) + shopt -u nullglob + if [ "${#package_tgzs[@]}" -eq 0 ]; then + echo "prepared package set contains no tgz files" >&2 + exit 1 + fi + registry_args=() + for package_tgz in "${package_tgzs[@]}"; do + package_metadata="$( + tar -xOf "$package_tgz" package/package.json | + node -e ' +let raw = ""; +process.stdin.on("data", (chunk) => (raw += chunk)); +process.stdin.on("end", () => { + const pkg = JSON.parse(raw); + if (typeof pkg.name !== "string" || !pkg.name || typeof pkg.version !== "string" || !pkg.version) { + throw new Error("package tarball is missing name or version"); + } + process.stdout.write(`${pkg.name}\n${pkg.version}\n`); +}); +' + )" + mapfile -t package_fields <<<"$package_metadata" + registry_args+=("${package_fields[0]}" "${package_fields[1]}" "$package_tgz") + done + registry_port_file="$(mktemp)" + registry_log="$(mktemp)" + OPENCLAW_NPM_REGISTRY_UPSTREAM=https://registry.npmjs.org \ + node /tmp/openclaw-npm-registry-server.mjs \ + "$registry_port_file" \ + "${registry_args[@]}" >"$registry_log" 2>&1 & + registry_pid=$! + for _ in $(seq 1 100); do + if [ -s "$registry_port_file" ]; then + break + fi + if ! kill -0 "$registry_pid" >/dev/null 2>&1; then + cat "$registry_log" >&2 + exit 1 + fi + sleep 0.1 + done + if [ ! -s "$registry_port_file" ]; then + cat "$registry_log" >&2 + echo "prepared package registry did not start" >&2 + exit 1 + fi + registry_url="http://127.0.0.1:$(cat "$registry_port_file")" + rm -f "$registry_port_file" + export NPM_CONFIG_REGISTRY="$registry_url" + export npm_config_registry="$registry_url" +fi + npm_install_timeout="${OPENCLAW_E2E_NPM_INSTALL_TIMEOUT:-600s}" run_npm_install() { if [ -z "$npm_install_timeout" ] || [ "$npm_install_timeout" = "0" ]; then diff --git a/scripts/e2e/parallels/host-server.ts b/scripts/e2e/parallels/host-server.ts index 784fb901b20..b719927e0fe 100644 --- a/scripts/e2e/parallels/host-server.ts +++ b/scripts/e2e/parallels/host-server.ts @@ -1,11 +1,14 @@ // Host Server script supports OpenClaw repository automation. import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { rm } from "node:fs/promises"; import { createServer } from "node:http"; import { createConnection } from "node:net"; +import { tmpdir } from "node:os"; import path from "node:path"; import { sleep as delay } from "../../lib/sleep.mjs"; import { die, run, say, sh, warn } from "./host-command.ts"; -import type { HostServer } from "./types.ts"; +import type { HostServer, NpmRegistryPackage, NpmRegistryServer } from "./types.ts"; const HOST_SERVER_STDERR_LIMIT_BYTES = 64 * 1024; const HOST_SERVER_STDERR_DRAIN_MS = 5_000; @@ -90,6 +93,44 @@ export async function startHostServer(input: { }; } +export async function startNpmRegistryServer(input: { + hostIp: string; + packages: NpmRegistryPackage[]; +}): Promise { + if (input.packages.length === 0) { + die("npm registry server requires at least one package"); + } + const port = allocateHostPort(); + const portFile = path.join(tmpdir(), `openclaw-npm-registry-${randomUUID()}.port`); + const packageArgs = input.packages.flatMap((pkg) => [pkg.name, pkg.version, pkg.tarballPath]); + const child = spawn( + process.execPath, + ["scripts/e2e/lib/plugins/npm-registry-server.mjs", portFile, ...packageArgs], + { + env: { + ...process.env, + OPENCLAW_NPM_REGISTRY_BIND_HOST: "0.0.0.0", + OPENCLAW_NPM_REGISTRY_PORT: String(port), + OPENCLAW_NPM_REGISTRY_UPSTREAM: "https://registry.npmjs.org", + }, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + await waitForHostServer(child, port); + const url = `http://${input.hostIp}:${port}`; + say(`Serve prepared npm package set on ${url}`); + return { + url, + stop: async () => { + try { + await stopHostServerChild(child); + } finally { + await rm(portFile, { force: true }); + } + }, + }; +} + async function stopHostServerChild( child: ChildProcessWithoutNullStreams, terminateTimeoutMs = 2_000, diff --git a/scripts/e2e/parallels/linux-smoke.ts b/scripts/e2e/parallels/linux-smoke.ts index b6ceb1e85f9..b096385a449 100755 --- a/scripts/e2e/parallels/linux-smoke.ts +++ b/scripts/e2e/parallels/linux-smoke.ts @@ -130,6 +130,7 @@ const defaultOptions = (): LinuxOptions => ({ latestVersion: "", mode: "both", modelId: undefined, + npmRegistry: undefined, provider: "openai", snapshotHint: "fresh", targetPackageSpec: "", @@ -157,6 +158,7 @@ Options: --install-version Pin site-installer version/dist-tag for the baseline lane. --target-package-spec Install this npm package tarball instead of packing current main. + --npm-registry Registry used for target package installs. --keep-server Leave temp host HTTP server running. --json Print machine-readable JSON summary. -h, --help Show help. @@ -222,6 +224,10 @@ export function parseArgs(argv: string[]): LinuxOptions { options.targetPackageSpec = ensureValue(args, i, arg); i++; break; + case "--npm-registry": + options.npmRegistry = ensureValue(args, i, arg); + i++; + break; case "--keep-server": options.keepServer = true; break; @@ -524,7 +530,17 @@ fi`); } const tgzUrl = this.server.urlFor(this.artifact.path); this.downloadGuestFile(tgzUrl, `/tmp/${tempName}`); - this.guestExec(["npm", "install", "-g", `/tmp/${tempName}`, "--no-fund", "--no-audit"]); + const npmArgs = ["npm", "install", "-g", `/tmp/${tempName}`, "--no-fund", "--no-audit"]; + this.guestExec( + this.options.npmRegistry + ? [ + "/usr/bin/env", + `NPM_CONFIG_REGISTRY=${this.options.npmRegistry}`, + `npm_config_registry=${this.options.npmRegistry}`, + ...npmArgs, + ] + : npmArgs, + ); this.guestExec(["openclaw", "--version"]); } diff --git a/scripts/e2e/parallels/macos-smoke.ts b/scripts/e2e/parallels/macos-smoke.ts index 12e2a4e9116..836293fd430 100755 --- a/scripts/e2e/parallels/macos-smoke.ts +++ b/scripts/e2e/parallels/macos-smoke.ts @@ -65,6 +65,7 @@ interface MacosOptions { hostIp?: string; latestVersion?: string; installVersion?: string; + npmRegistry?: string; targetPackageSpec?: string; skipLatestRefCheck: boolean; keepServer: boolean; @@ -128,6 +129,7 @@ const defaultOptions = (): MacosOptions => ({ latestVersion: "", mode: "both", modelId: undefined, + npmRegistry: undefined, provider: "openai", skipLatestRefCheck: false, snapshotHint: "macOS 26.5 latest", @@ -155,6 +157,7 @@ Options: --install-version Pin site-installer version/dist-tag for the baseline lane. --target-package-spec Install this npm package tarball instead of packing current main. + --npm-registry Registry used for target package installs. --skip-latest-ref-check Skip the known latest-release ref-mode precheck in upgrade lane. --keep-server Leave temp host HTTP server running. --discord-token-env Host env var name for Discord bot token. @@ -224,6 +227,10 @@ export function parseArgs(argv: string[]): MacosOptions { options.targetPackageSpec = ensureValue(args, i, arg); i++; break; + case "--npm-registry": + options.npmRegistry = ensureValue(args, i, arg); + i++; + break; case "--skip-latest-ref-check": options.skipLatestRefCheck = true; break; @@ -818,11 +825,14 @@ ${guestOpenClaw} --version`, } private installMain(tempName: string): void { + const npmRegistryEnv = this.options.npmRegistry + ? `NPM_CONFIG_REGISTRY=${shellQuote(this.options.npmRegistry)} npm_config_registry=${shellQuote(this.options.npmRegistry)} ` + : ""; if (this.targetInstallsDirectly()) { this .guestSh(`printf 'install-source: registry-spec %s\\n' ${shellQuote(this.options.targetPackageSpec || "")} for attempt in 1 2; do - if ${guestNpm} install -g ${shellQuote(this.options.targetPackageSpec || "")}; then + if ${npmRegistryEnv}${guestNpm} install -g ${shellQuote(this.options.targetPackageSpec || "")}; then break fi if [ "$attempt" -eq 2 ]; then @@ -842,7 +852,7 @@ ${guestOpenClaw} --version`); curl -fsSL --connect-timeout 10 --max-time 120 --retry 2 --retry-delay 2 ${shellQuote( tgzUrl, )} -o /tmp/${tempName} -${guestNpm} install -g /tmp/${tempName} +${npmRegistryEnv}${guestNpm} install -g /tmp/${tempName} ${guestOpenClaw} --version`); } diff --git a/scripts/e2e/parallels/npm-update-scripts.ts b/scripts/e2e/parallels/npm-update-scripts.ts index b8a5cf1a326..d36fd142fd8 100644 --- a/scripts/e2e/parallels/npm-update-scripts.ts +++ b/scripts/e2e/parallels/npm-update-scripts.ts @@ -21,6 +21,7 @@ import type { Platform, ProviderAuth } from "./types.ts"; interface NpmUpdateScriptInput { auth: ProviderAuth; expectedNeedle: string; + npmRegistry?: string; updateTarget: string; } @@ -29,6 +30,14 @@ const macosGuestPath = "/opt/homebrew/bin:/opt/homebrew/opt/node/bin:/usr/local/bin:/usr/local/sbin:/opt/homebrew/sbin:/usr/bin:/bin:/usr/sbin:/sbin"; const macosOpenClawCommand = '"$OPENCLAW_BIN"'; +function posixNpmRegistryEnv(registry: string | undefined): string { + if (!registry) { + return ""; + } + const quoted = shellQuote(registry); + return `NPM_CONFIG_REGISTRY=${quoted} npm_config_registry=${quoted} `; +} + function posixModelProviderConfigCommands( command: string, modelId: string, @@ -120,8 +129,11 @@ fi`; } function windowsUpdateWithBundledPluginsDisabled(input: NpmUpdateScriptInput): string { + const registryEntry = input.npmRegistry + ? `; NPM_CONFIG_REGISTRY = ${psSingleQuote(input.npmRegistry)}` + : ""; return `$script:OpenClawUpdateExit = 0 -$updateOutput = Invoke-WithScopedEnv @{ OPENCLAW_DISABLE_BUNDLED_PLUGINS = '1'; OPENCLAW_ALLOW_OLDER_BINARY_DESTRUCTIVE_ACTIONS = '1' } { +$updateOutput = Invoke-WithScopedEnv @{ OPENCLAW_DISABLE_BUNDLED_PLUGINS = '1'; OPENCLAW_ALLOW_OLDER_BINARY_DESTRUCTIVE_ACTIONS = '1'${registryEntry} } { Invoke-OpenClaw update --tag ${psSingleQuote(input.updateTarget)} --yes --json --no-restart 2>&1 $script:OpenClawUpdateExit = $LASTEXITCODE } @@ -255,7 +267,7 @@ wait_for_gateway() { } scrub_future_plugin_entries stop_openclaw_gateway_processes -OPENCLAW_ALLOW_OLDER_BINARY_DESTRUCTIVE_ACTIONS=1 OPENCLAW_DISABLE_BUNDLED_PLUGINS=1 "$OPENCLAW_BIN" update --tag ${shellQuote(input.updateTarget)} --yes --json --no-restart +${posixNpmRegistryEnv(input.npmRegistry)}OPENCLAW_ALLOW_OLDER_BINARY_DESTRUCTIVE_ACTIONS=1 OPENCLAW_DISABLE_BUNDLED_PLUGINS=1 "$OPENCLAW_BIN" update --tag ${shellQuote(input.updateTarget)} --yes --json --no-restart ${posixVersionCheck(macosOpenClawCommand, input.expectedNeedle)} start_openclaw_gateway wait_for_gateway @@ -395,7 +407,7 @@ wait_for_gateway() { } scrub_future_plugin_entries stop_openclaw_gateway_processes -OPENCLAW_ALLOW_OLDER_BINARY_DESTRUCTIVE_ACTIONS=1 OPENCLAW_DISABLE_BUNDLED_PLUGINS=1 openclaw update --tag ${shellQuote(input.updateTarget)} --yes --json --no-restart +${posixNpmRegistryEnv(input.npmRegistry)}OPENCLAW_ALLOW_OLDER_BINARY_DESTRUCTIVE_ACTIONS=1 OPENCLAW_DISABLE_BUNDLED_PLUGINS=1 openclaw update --tag ${shellQuote(input.updateTarget)} --yes --json --no-restart ${posixVersionCheck("openclaw", input.expectedNeedle)} start_openclaw_gateway wait_for_gateway diff --git a/scripts/e2e/parallels/npm-update-smoke.ts b/scripts/e2e/parallels/npm-update-smoke.ts index 1b443e914b7..26b0af6b20a 100755 --- a/scripts/e2e/parallels/npm-update-smoke.ts +++ b/scripts/e2e/parallels/npm-update-smoke.ts @@ -14,12 +14,12 @@ import { import { die, ensureValue, + extractPackageJsonFromTgz, extractLastOpenClawVersionFromLog, isLikelyMacosDesktopHome, makeTempDir, packOpenClaw, packageBuildCommitFromTgz, - packageVersionFromTgz, parseMacosDsclUserHomeLine, parsePlatformList, parseProvider, @@ -34,10 +34,13 @@ import { say, shellQuote, startHostServer, + startNpmRegistryServer, withProgressOnStderr, writeSummaryMarkdown, writeJson, type HostServer, + type NpmRegistryPackage, + type NpmRegistryServer, type PackageArtifact, type Platform, type Provider, @@ -53,6 +56,7 @@ const LOGGED_POST_FORCE_KILL_WAIT_MS = 1_000; interface NpmUpdateOptions { betaValidation?: string; + dependencyTarballs: string[]; freshTargetSpec?: string; hostIp?: string; macosVm?: string; @@ -373,6 +377,7 @@ Options: --update-target Target passed to guest 'openclaw update --tag'. Default: host-served tgz packed from current checkout. --target-tarball Host-serve this prepared tgz for update and fresh install. + --dependency-tarball Companion package tgz required by the target. Repeatable. --fresh-target Also run fresh install smoke for this package after update lanes. --beta-validation [target] Resolve a beta tag/alias/version, then run latest->target update plus fresh target install. Default target when flag is bare: beta. @@ -395,6 +400,7 @@ export function parseArgs(argv: string[]): NpmUpdateOptions { const options: NpmUpdateOptions = { apiKeyEnv: undefined, betaValidation: undefined, + dependencyTarballs: [], freshTargetSpec: undefined, json: false, macosVm: undefined, @@ -422,6 +428,10 @@ export function parseArgs(argv: string[]): NpmUpdateOptions { options.targetTarball = ensureValue(args, i, arg); i++; break; + case "--dependency-tarball": + options.dependencyTarballs.push(ensureValue(args, i, arg)); + i++; + break; case "--fresh-target": options.freshTargetSpec = ensureValue(args, i, arg); i++; @@ -481,6 +491,9 @@ export function parseArgs(argv: string[]): NpmUpdateOptions { "--target-tarball cannot be combined with --beta-validation, --update-target, or --fresh-target", ); } + if (options.dependencyTarballs.length > 0 && !options.targetTarball) { + throw new Error("--dependency-tarball requires --target-tarball"); + } return options; } @@ -564,6 +577,7 @@ export class NpmUpdateSmoke { private harnessTargetFamily = ""; private hostIp = ""; protected server: HostServer | null = null; + private registryServer: NpmRegistryServer | null = null; private artifact: PackageArtifact | null = null; private freshTargetSpec = ""; private startedAt = Date.now(); @@ -574,7 +588,9 @@ export class NpmUpdateSmoke { private updateTargetTarball = ""; private targetTarballPath = ""; private targetTarballBuildCommit = ""; + private targetDependencyPackages: NpmRegistryPackage[] = []; private targetTarballVersion = ""; + private targetRegistryUrl = ""; private macosVm = macosVmDefault; private linuxVm = linuxVmDefault; @@ -605,6 +621,7 @@ export class NpmUpdateSmoke { await this.runSteps(); } finally { await this.server?.stop().catch(() => undefined); + await this.registryServer?.stop().catch(() => undefined); await rm(this.tgzDir, { force: true, recursive: true }).catch(() => undefined); } } @@ -756,6 +773,9 @@ export class NpmUpdateSmoke { auth.apiKeyEnv, "--target-package-spec", packageSpec, + ...(phase === "fresh-target" && this.targetRegistryUrl + ? ["--npm-registry", this.targetRegistryUrl] + : []), "--json", ...extraArgs, ]; @@ -799,6 +819,31 @@ export class NpmUpdateSmoke { path: hostedTarballPath, version: this.targetTarballVersion, }; + if (this.targetDependencyPackages.length > 0) { + // Prepared sibling packages publish before core, so pre-publish VM installs need + // a local registry that serves the exact package set without touching public npm. + this.registryServer = await startNpmRegistryServer({ + hostIp: this.hostIp, + packages: [ + { + name: "openclaw", + version: this.targetTarballVersion, + tarballPath: hostedTarballPath, + }, + ...this.targetDependencyPackages, + ], + }); + this.targetRegistryUrl = this.registryServer.url; + this.updateTargetTarball = `${this.registryServer.url}/openclaw/-/${path.basename( + hostedTarballPath, + )}`; + this.updateTargetEffective = this.targetTarballVersion; + this.freshTargetSpec = this.updateTargetTarball; + this.updateExpectedNeedle = this.targetTarballVersion; + this.updateTargetPackageVersion = this.targetTarballVersion; + this.updateTargetBuildCommit = this.artifact.buildCommitShort ?? ""; + return; + } this.server = await startHostServer({ artifactPath: this.artifact.path, dir: this.tgzDir, @@ -979,6 +1024,7 @@ export class NpmUpdateSmoke { const input = { auth: this.authForPlatform(platform), expectedNeedle: this.updateExpectedNeedle, + npmRegistry: this.targetRegistryUrl, updateTarget: this.updateTargetEffective, }; switch (platform) { @@ -1385,10 +1431,42 @@ export class NpmUpdateSmoke { throw new Error(`target tarball does not exist: ${targetTarballPath}`); } this.targetTarballPath = targetTarballPath; - [this.targetTarballVersion, this.targetTarballBuildCommit] = await Promise.all([ - packageVersionFromTgz(targetTarballPath), + const [targetPackageJson, targetBuildCommit] = await Promise.all([ + extractPackageJsonFromTgz<{ + dependencies?: Record; + version?: string; + }>(targetTarballPath, "package/package.json"), packageBuildCommitFromTgz(targetTarballPath), ]); + this.targetTarballVersion = targetPackageJson.version ?? ""; + this.targetTarballBuildCommit = targetBuildCommit; + this.targetDependencyPackages = await Promise.all( + this.options.dependencyTarballs.map(async (dependencyTarball) => { + const tarballPath = path.resolve(dependencyTarball); + if (!existsSync(tarballPath)) { + throw new Error(`dependency tarball does not exist: ${tarballPath}`); + } + const dependencyPackage = await extractPackageJsonFromTgz<{ + name?: string; + version?: string; + }>(tarballPath, "package/package.json"); + const name = dependencyPackage.name ?? ""; + const version = dependencyPackage.version ?? ""; + if (!name || !version || name === "openclaw") { + throw new Error(`dependency tarball has invalid package metadata: ${tarballPath}`); + } + if (targetPackageJson.dependencies?.[name] !== version) { + throw new Error( + `target tarball requires ${name}@${targetPackageJson.dependencies?.[name] ?? ""}, but companion tarball provides ${version}`, + ); + } + return { name, version, tarballPath }; + }), + ); + const dependencyNames = new Set(this.targetDependencyPackages.map((pkg) => pkg.name)); + if (dependencyNames.size !== this.targetDependencyPackages.length) { + throw new Error("dependency tarballs must have unique package names"); + } if (!this.targetTarballVersion || !this.targetTarballBuildCommit) { throw new Error( `target tarball is missing package or build metadata: ${targetTarballPath}`, diff --git a/scripts/e2e/parallels/smoke-common.ts b/scripts/e2e/parallels/smoke-common.ts index 27f6675aa23..8f2258afb2b 100644 --- a/scripts/e2e/parallels/smoke-common.ts +++ b/scripts/e2e/parallels/smoke-common.ts @@ -24,6 +24,7 @@ export interface SmokeRunOptions { json: boolean; keepServer: boolean; mode: Mode; + npmRegistry?: string; provider: Provider; snapshotHint: string; targetPackageSpec?: string; diff --git a/scripts/e2e/parallels/types.ts b/scripts/e2e/parallels/types.ts index 19f54bb4a9b..f4b357356d1 100644 --- a/scripts/e2e/parallels/types.ts +++ b/scripts/e2e/parallels/types.ts @@ -45,3 +45,14 @@ export interface HostServer { urlFor(filePath: string): string; stop(): Promise; } + +export interface NpmRegistryPackage { + name: string; + version: string; + tarballPath: string; +} + +export interface NpmRegistryServer { + url: string; + stop(): Promise; +} diff --git a/scripts/e2e/parallels/windows-smoke.ts b/scripts/e2e/parallels/windows-smoke.ts index 6994f82fddc..17751d9019e 100755 --- a/scripts/e2e/parallels/windows-smoke.ts +++ b/scripts/e2e/parallels/windows-smoke.ts @@ -108,6 +108,7 @@ const defaultOptions = (): WindowsOptions => ({ latestVersion: "", mode: "both", modelId: undefined, + npmRegistry: undefined, provider: "openai", skipLatestRefCheck: false, snapshotHint: "pre-openclaw-native-e2e-2026-03-12", @@ -142,6 +143,7 @@ Options: then run openclaw update --channel dev. --target-package-spec Install this npm package tarball instead of packing current main. + --npm-registry Registry used for target package installs. --skip-latest-ref-check Skip latest-release ref-mode precheck. --keep-server Leave temp host HTTP server running. --json Print machine-readable JSON summary. @@ -175,6 +177,9 @@ export function parseArgs(argv: string[]): WindowsOptions { "--model": (value) => { options.modelId = value; }, + "--npm-registry": (value) => { + options.npmRegistry = value; + }, "--openai-api-key-env": (value) => { options.apiKeyEnv = value; }, @@ -567,11 +572,15 @@ Invoke-OpenClaw --version die("package artifact/server missing"); } const tgzUrl = this.server.urlFor(this.artifact.path); + const registryScript = this.options.npmRegistry + ? `$env:NPM_CONFIG_REGISTRY = ${psSingleQuote(this.options.npmRegistry)}` + : ""; return this.guestPowerShellBackground( `install-main-${tempName.replaceAll(/[^A-Za-z0-9_-]/g, "-")}`, `$ErrorActionPreference = 'Stop' $tgz = Join-Path $env:TEMP ${psSingleQuote(tempName)} curl.exe -fsSL --connect-timeout 10 --max-time 120 --retry 2 --retry-delay 2 ${psSingleQuote(tgzUrl)} -o $tgz +${registryScript} npm.cmd install -g $tgz --no-fund --no-audit --loglevel=error if ($LASTEXITCODE -ne 0) { throw "npm install failed with exit code $LASTEXITCODE" } Invoke-OpenClaw --version diff --git a/scripts/release-candidate-checklist.mjs b/scripts/release-candidate-checklist.mjs index 60e4d5536da..4160557571c 100644 --- a/scripts/release-candidate-checklist.mjs +++ b/scripts/release-candidate-checklist.mjs @@ -625,7 +625,7 @@ export function buildPublishCommand(options) { .join(" "); } -function validatePreflightManifest(manifest, params) { +export function validatePreflightManifest(manifest, params) { if (manifest.releaseTag !== params.tag) { throw new Error( `npm preflight tag mismatch: expected ${params.tag}, got ${manifest.releaseTag}`, @@ -644,6 +644,20 @@ function validatePreflightManifest(manifest, params) { if (!manifest.tarballName || !manifest.tarballSha256) { throw new Error("npm preflight manifest missing tarball metadata"); } + if (!Array.isArray(manifest.dependencyTarballs)) { + throw new Error("npm preflight manifest missing dependency tarball metadata"); + } + for (const dependency of manifest.dependencyTarballs) { + if ( + !dependency?.packageName || + !dependency.packageVersion || + !dependency.tarballName || + !dependency.tarballSha256 || + dependency.tarballName !== basename(dependency.tarballName) + ) { + throw new Error("npm preflight manifest contains invalid dependency tarball metadata"); + } + } } export function validateFullManifest(manifest, params) { @@ -676,11 +690,22 @@ export function validateFullManifest(manifest, params) { } } -export function candidateParallelsArgs(tarballPath) { - return ["test:parallels:npm-update", "--", "--target-tarball", tarballPath, "--json"]; +export function candidateParallelsArgs(tarballPath, dependencyTarballPaths = []) { + return [ + "test:parallels:npm-update", + "--", + "--target-tarball", + tarballPath, + ...dependencyTarballPaths.flatMap((dependency) => ["--dependency-tarball", dependency]), + "--json", + ]; } -export function candidateParallelsShellCommand(tarballPath, timeoutBin) { +export function candidateParallelsShellCommand( + tarballPath, + timeoutBin, + dependencyTarballPaths = [], +) { return [ 'set -a; source "$HOME/.profile" >/dev/null 2>&1 || true; set +a;', "exec", @@ -688,18 +713,18 @@ export function candidateParallelsShellCommand(tarballPath, timeoutBin) { "--foreground", "150m", "pnpm", - ...candidateParallelsArgs(tarballPath).map(shellQuote), + ...candidateParallelsArgs(tarballPath, dependencyTarballPaths).map(shellQuote), ].join(" "); } -async function runParallelsIfNeeded(options, tarballPath) { +async function runParallelsIfNeeded(options, tarballPath, dependencyTarballPaths) { if (options.skipParallels) { return { status: "skipped", reason: "operator skipped --skip-parallels" }; } const timeoutBin = run("bash", ["-lc", "command -v gtimeout || command -v timeout"], { capture: true, }).trim(); - const command = candidateParallelsShellCommand(tarballPath, timeoutBin); + const command = candidateParallelsShellCommand(tarballPath, timeoutBin, dependencyTarballPaths); run("bash", ["-lc", command]); return { status: "passed", @@ -827,8 +852,21 @@ async function main() { `prepared tarball digest mismatch: expected ${npmManifest.tarballSha256}, got ${actualTarballSha}`, ); } + const dependencyTarballPaths = npmManifest.dependencyTarballs.map((dependency) => { + const dependencyPath = join(npmDir, dependency.tarballName); + if (!existsSync(dependencyPath)) { + throw new Error(`prepared dependency tarball missing: ${dependencyPath}`); + } + const actualDependencySha = sha256(dependencyPath); + if (actualDependencySha !== dependency.tarballSha256) { + throw new Error( + `prepared dependency tarball digest mismatch for ${dependency.packageName}: expected ${dependency.tarballSha256}, got ${actualDependencySha}`, + ); + } + return dependencyPath; + }); - const parallels = await runParallelsIfNeeded(options, tarballPath); + const parallels = await runParallelsIfNeeded(options, tarballPath, dependencyTarballPaths); const npmTelegram = await runTelegramIfNeeded(options, npmArtifactName); options.npmTelegramRunId = npmTelegram.runId ?? ""; const pluginNpmPlan = await collectPluginPlanWithRetry( diff --git a/test/scripts/npm-telegram-live.test.ts b/test/scripts/npm-telegram-live.test.ts index 1ed5e786315..2058779a1f7 100644 --- a/test/scripts/npm-telegram-live.test.ts +++ b/test/scripts/npm-telegram-live.test.ts @@ -130,6 +130,22 @@ describe("package Telegram live Docker E2E", () => { ); }); + it("installs prepared root and companion tarballs through an exact local registry", () => { + const script = readFileSync(DOCKER_SCRIPT_PATH, "utf8"); + + expect(script).toContain("OPENCLAW_NPM_TELEGRAM_PACKAGE_DIR"); + expect(script).toContain('package_source_kind="prepared-package-set"'); + expect(script).toContain('package_install_source="openclaw@$(read_package_version'); + expect(script).toContain('-v "$resolved_package_dir:/package-under-test:ro"'); + expect(script).toContain( + '-v "$ROOT_DIR/scripts/e2e/lib/plugins/npm-registry-server.mjs:/tmp/openclaw-npm-registry-server.mjs:ro"', + ); + expect(script).toContain("OPENCLAW_NPM_TELEGRAM_PACKAGE_SET"); + expect(script).toContain("node /tmp/openclaw-npm-registry-server.mjs"); + expect(script).toContain("OPENCLAW_NPM_REGISTRY_UPSTREAM=https://registry.npmjs.org"); + expect(script).toContain('export NPM_CONFIG_REGISTRY="$registry_url"'); + }); + it("keeps live Docker artifacts isolated by default", () => { const script = readFileSync(DOCKER_SCRIPT_PATH, "utf8"); diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index 3fecc1b7c80..f0d074169eb 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -1782,7 +1782,15 @@ describe("package artifact reuse", () => { "package_spec must be openclaw@alpha", ]); expectTextToIncludeAll(runStep.run, [ - 'export OPENCLAW_NPM_TELEGRAM_PACKAGE_TGZ="${package_tgzs[0]}"', + 'manifest="${package_dir}/preflight-manifest.json"', + 'candidate_manifest="${package_dir}/package-candidate.json"', + 'find "${package_dir}" -type f -name "*.tgz"', + "package artifact manifest contains duplicate package metadata", + "package artifact tarball set does not match preflight manifest", + "package candidate manifest does not match the OpenClaw tarball", + "package candidate digest mismatch", + 'export OPENCLAW_NPM_TELEGRAM_PACKAGE_DIR="${package_dir}"', + 'export OPENCLAW_NPM_TELEGRAM_PACKAGE_TGZ="${package_tgz}"', ]); }); @@ -1962,6 +1970,14 @@ describe("package artifact reuse", () => { "Workflow-dispatched real publish requires release_publish_run_id", ); expect(npmWorkflow).toContain("tarballSha256"); + expect(npmWorkflow).toContain("dependencyTarballs"); + expect(npmWorkflow).toContain('packageName: "@openclaw/ai"'); + expect(npmWorkflow).toContain("AI_TARBALL_SHA256"); + expect(npmWorkflow).toContain("does not match openclaw"); + const npmTelegramWorkflow = readFileSync(NPM_TELEGRAM_WORKFLOW, "utf8"); + expect(npmTelegramWorkflow).toContain("preflight-manifest.json"); + expect(npmTelegramWorkflow).toContain("OPENCLAW_NPM_TELEGRAM_PACKAGE_DIR"); + expect(npmTelegramWorkflow).toContain("package artifact digest mismatch"); expect(workflow).toContain("Checkout release SHA"); expect(workflow).toContain('git show "${TARGET_SHA}:CHANGELOG.md" > "${changelog_file}"'); expect(workflow).toContain('$0 == "## Unreleased" { in_section = 1; next }'); diff --git a/test/scripts/parallels-npm-update-smoke.test.ts b/test/scripts/parallels-npm-update-smoke.test.ts index 3cfc89b343b..3811b2b60ca 100644 --- a/test/scripts/parallels-npm-update-smoke.test.ts +++ b/test/scripts/parallels-npm-update-smoke.test.ts @@ -106,7 +106,15 @@ afterEach(() => { describe("parallels npm update smoke", () => { it("accepts one prepared tarball target for update and fresh install", () => { - expect(parseArgs(["--target-tarball", "/tmp/openclaw-candidate.tgz"])).toMatchObject({ + expect( + parseArgs([ + "--target-tarball", + "/tmp/openclaw-candidate.tgz", + "--dependency-tarball", + "/tmp/openclaw-ai-candidate.tgz", + ]), + ).toMatchObject({ + dependencyTarballs: ["/tmp/openclaw-ai-candidate.tgz"], targetTarball: "/tmp/openclaw-candidate.tgz", updateTarget: "", freshTargetSpec: undefined, @@ -114,6 +122,9 @@ describe("parallels npm update smoke", () => { expect(() => parseArgs(["--target-tarball", "/tmp/openclaw-candidate.tgz", "--update-target", "beta"]), ).toThrow("--target-tarball cannot be combined"); + expect(() => parseArgs(["--dependency-tarball", "/tmp/openclaw-ai-candidate.tgz"])).toThrow( + "--dependency-tarball requires --target-tarball", + ); }); it("stops the host artifact server when the wrapper fails mid-run", async () => { @@ -231,18 +242,33 @@ exit 1 expect(script).toContain("freshTargetStatus"); }); - it("host-serves a prepared candidate tarball for both proof phases", () => { + it("serves a prepared package set for both proof phases", () => { const script = readFileSync(SCRIPT_PATH, "utf8"); expect(script).toContain("--target-tarball "); + expect(script).toContain("--dependency-tarball "); expect(script).toContain('label: "prepared candidate tgz"'); expect(script).toContain("await copyFile(this.targetTarballPath, hostedTarballPath)"); - expect(script).toContain("dir: this.tgzDir"); - expect(script).toContain("this.updateTargetEffective = targetUrl"); - expect(script).toContain("this.freshTargetSpec = targetUrl"); + expect(script).toContain("startNpmRegistryServer"); + expect(script).toContain("this.updateTargetEffective = this.targetTarballVersion"); + expect(script).toContain("this.freshTargetSpec = this.updateTargetTarball"); expect(script).toContain("this.updateExpectedNeedle = this.targetTarballVersion"); }); + it("routes update installs through the prepared package registry", () => { + const registry = "http://192.0.2.2:48123"; + const input = { + auth: TEST_AUTH, + expectedNeedle: "2026.7.1-beta.3", + npmRegistry: registry, + updateTarget: "2026.7.1-beta.3", + }; + + expect(macosUpdateScript(input)).toContain(`NPM_CONFIG_REGISTRY='${registry}'`); + expect(linuxUpdateScript(input)).toContain(`NPM_CONFIG_REGISTRY='${registry}'`); + expect(windowsUpdateScript(input)).toContain(`NPM_CONFIG_REGISTRY = '${registry}'`); + }); + it("accepts keyed and nested npm metadata for published update targets", () => { const script = readFileSync(SCRIPT_PATH, "utf8"); diff --git a/test/scripts/parallels-smoke-model.test.ts b/test/scripts/parallels-smoke-model.test.ts index 93d15c51c0d..16e6c08a8e6 100644 --- a/test/scripts/parallels-smoke-model.test.ts +++ b/test/scripts/parallels-smoke-model.test.ts @@ -282,6 +282,11 @@ describe("Parallels smoke model selection", () => { expect(parseMacosSmokeArgs(["--host-port", "65535"]).hostPort).toBe(65535); expect(parseLinuxSmokeArgs(["--host-port", "65535"]).hostPort).toBe(65535); expect(parseWindowsSmokeArgs(["--host-port", "65535"]).hostPort).toBe(65535); + for (const parseArgs of [parseMacosSmokeArgs, parseLinuxSmokeArgs, parseWindowsSmokeArgs]) { + expect(parseArgs(["--npm-registry", "http://192.0.2.2:48123"]).npmRegistry).toBe( + "http://192.0.2.2:48123", + ); + } expect(parseNpmUpdateSmokeArgs(["--", "--package-spec", "openclaw@2026.5.1"]).packageSpec).toBe( "openclaw@2026.5.1", ); @@ -406,6 +411,8 @@ describe("Parallels smoke model selection", () => { expect(parallelsVm).toContain("export function resolveMacosVmName"); expect(parallelsVm).toContain("export function waitForVmStatus"); expect(hostServer).toContain("export async function startHostServer"); + expect(hostServer).toContain("export async function startNpmRegistryServer"); + expect(hostServer).toContain('OPENCLAW_NPM_REGISTRY_UPSTREAM: "https://registry.npmjs.org"'); expect(hostServer).toContain("http.server"); expect(snapshots).toContain("export function resolveSnapshot"); expect(smokeCommon).toContain("runSmokeLane"); diff --git a/test/scripts/plugins-assertions.test.ts b/test/scripts/plugins-assertions.test.ts index 7e4a8581734..f5456840493 100644 --- a/test/scripts/plugins-assertions.test.ts +++ b/test/scripts/plugins-assertions.test.ts @@ -12,6 +12,7 @@ import { import { createServer, request as httpRequest } from "node:http"; import { tmpdir } from "node:os"; import path from "node:path"; +import { gzipSync } from "node:zlib"; import { describe, expect, it } from "vitest"; import { createBoundedChildOutput } from "../helpers/bounded-child-output.js"; import { cleanupTempDirs, makeTempDir } from "../helpers/temp-dir.js"; @@ -159,10 +160,11 @@ async function waitForPortFile(portFile: string): Promise { function requestFixtureRegistry( port: number, requestPath: string, -): Promise<{ body: string; statusCode: number | undefined }> { + headers: Record = {}, +): Promise<{ body: string; contentLength: string | undefined; statusCode: number | undefined }> { return new Promise((resolve, reject) => { const request = httpRequest( - { host: "127.0.0.1", method: "GET", path: requestPath, port }, + { headers, host: "127.0.0.1", method: "GET", path: requestPath, port }, (response) => { let body = ""; response.setEncoding("utf8"); @@ -170,7 +172,11 @@ function requestFixtureRegistry( body += chunk; }); response.on("end", () => { - resolve({ body, statusCode: response.statusCode }); + resolve({ + body, + contentLength: response.headers["content-length"], + statusCode: response.statusCode, + }); }); }, ); @@ -602,6 +608,235 @@ test -d "$OPENCLAW_PLUGINS_TMP_DIR" } }); + it("serves tarball dependencies using the request-visible registry origin", async () => { + const tempDirs: string[] = []; + const root = makeTempDir(tempDirs, "openclaw-plugin-npm-fixture-package-"); + const packageDir = path.join(root, "package"); + const portFile = path.join(root, "port"); + const tarballPath = path.join(root, "openclaw.tgz"); + mkdirSync(packageDir); + writeJson(path.join(packageDir, "package.json"), { + name: "openclaw", + version: "2026.7.1-beta.3", + dependencies: { + "@openclaw/ai": "2026.7.1-beta.3", + zod: "4.3.6", + }, + optionalDependencies: { + "sqlite-vec": "0.1.7-alpha.2", + }, + }); + const packed = spawnSync("tar", ["-czf", tarballPath, "-C", root, "package"], { + encoding: "utf8", + }); + expect(packed.status, packed.stderr).toBe(0); + + const child = spawn( + process.execPath, + [ + "scripts/e2e/lib/plugins/npm-registry-server.mjs", + portFile, + "openclaw", + "2026.7.1-beta.3", + tarballPath, + ], + { + cwd: process.cwd(), + stdio: ["ignore", "pipe", "pipe"], + }, + ); + + try { + const port = await waitForPortFile(portFile); + const response = await requestFixtureRegistry(port, "/openclaw", { + host: `192.0.2.2:${port}`, + }); + const metadata = JSON.parse(response.body); + + expect(response.statusCode).toBe(200); + expect(metadata.versions["2026.7.1-beta.3"].dependencies).toEqual({ + "@openclaw/ai": "2026.7.1-beta.3", + zod: "4.3.6", + }); + expect(metadata.versions["2026.7.1-beta.3"].optionalDependencies).toEqual({ + "sqlite-vec": "0.1.7-alpha.2", + }); + expect(metadata.versions["2026.7.1-beta.3"].dist.tarball).toBe( + `http://192.0.2.2:${port}/openclaw/-/openclaw.tgz`, + ); + } finally { + if (child.exitCode === null) { + child.kill(); + await new Promise((resolve) => { + child.once("close", resolve); + }); + } + cleanupTempDirs(tempDirs); + } + }); + + it("recomputes proxied content length after fetch decodes the response", async () => { + const tempDirs: string[] = []; + const root = makeTempDir(tempDirs, "openclaw-plugin-npm-fixture-proxy-"); + const portFile = path.join(root, "port"); + const tarballPath = path.join(root, "demo-plugin.tgz"); + const upstreamBody = JSON.stringify({ payload: "x".repeat(1_000) }); + const compressedBody = gzipSync(upstreamBody); + writeFileSync(tarballPath, "fixture package archive", "utf8"); + + const upstream = createServer((_request, response) => { + response.writeHead(200, { + "content-encoding": "gzip", + "content-length": String(compressedBody.length), + "content-type": "application/json", + }); + response.end(compressedBody); + }); + await new Promise((resolve) => { + upstream.listen(0, "127.0.0.1", resolve); + }); + const upstreamAddress = upstream.address(); + if (!upstreamAddress || typeof upstreamAddress === "string") { + throw new Error("expected upstream registry address"); + } + + const child = spawn( + process.execPath, + [ + "scripts/e2e/lib/plugins/npm-registry-server.mjs", + portFile, + "@openclaw/demo-plugin-npm", + "1.0.0", + tarballPath, + ], + { + cwd: process.cwd(), + env: { + ...process.env, + OPENCLAW_NPM_REGISTRY_UPSTREAM: `http://127.0.0.1:${upstreamAddress.port}`, + }, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + + try { + const port = await waitForPortFile(portFile); + const response = await requestFixtureRegistry(port, "/upstream-package"); + + expect(response.statusCode).toBe(200); + expect(response.body).toBe(upstreamBody); + expect(response.contentLength).toBe(String(Buffer.byteLength(upstreamBody))); + } finally { + if (child.exitCode === null) { + child.kill(); + await new Promise((resolve) => { + child.once("close", resolve); + }); + } + await new Promise((resolve) => { + upstream.close(() => resolve()); + }); + cleanupTempDirs(tempDirs); + } + }); + + it("does not let absolute-form request targets escape the configured upstream", async () => { + const tempDirs: string[] = []; + const root = makeTempDir(tempDirs, "openclaw-plugin-npm-fixture-proxy-origin-"); + const portFile = path.join(root, "port"); + const tarballPath = path.join(root, "demo-plugin.tgz"); + let configuredUpstreamHits = 0; + let escapeServerHits = 0; + let configuredUpstreamTarget: string | undefined; + writeFileSync(tarballPath, "fixture package archive", "utf8"); + + const configuredUpstream = createServer((request, response) => { + configuredUpstreamHits += 1; + configuredUpstreamTarget = request.url; + response.writeHead(200, { "content-type": "text/plain" }); + response.end("configured upstream"); + }); + const escapeServer = createServer((_request, response) => { + escapeServerHits += 1; + response.writeHead(200, { "content-type": "text/plain" }); + response.end("escaped upstream"); + }); + await Promise.all([ + new Promise((resolve) => { + configuredUpstream.listen(0, "127.0.0.1", resolve); + }), + new Promise((resolve) => { + escapeServer.listen(0, "127.0.0.1", resolve); + }), + ]); + const configuredAddress = configuredUpstream.address(); + const escapeAddress = escapeServer.address(); + if ( + !configuredAddress || + typeof configuredAddress === "string" || + !escapeAddress || + typeof escapeAddress === "string" + ) { + throw new Error("expected upstream registry addresses"); + } + + const child = spawn( + process.execPath, + [ + "scripts/e2e/lib/plugins/npm-registry-server.mjs", + portFile, + "@openclaw/demo-plugin-npm", + "1.0.0", + tarballPath, + ], + { + cwd: process.cwd(), + env: { + ...process.env, + OPENCLAW_NPM_REGISTRY_UPSTREAM: `http://127.0.0.1:${configuredAddress.port}`, + }, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + + try { + const port = await waitForPortFile(portFile); + const escaped = await requestFixtureRegistry( + port, + `http://registry.invalid//127.0.0.1:${escapeAddress.port}/probe`, + ); + + expect(escaped.statusCode).toBe(502); + expect(escaped.body).toContain("refusing non-origin registry request URL"); + expect(configuredUpstreamHits).toBe(0); + expect(escapeServerHits).toBe(0); + + const valid = await requestFixtureRegistry(port, "/pkg?x=1"); + + expect(valid.statusCode).toBe(200); + expect(valid.body).toBe("configured upstream"); + expect(configuredUpstreamHits).toBe(1); + expect(configuredUpstreamTarget).toBe("/pkg?x=1"); + expect(escapeServerHits).toBe(0); + } finally { + if (child.exitCode === null) { + child.kill(); + await new Promise((resolve) => { + child.once("close", resolve); + }); + } + await Promise.all([ + new Promise((resolve) => { + configuredUpstream.close(() => resolve()); + }), + new Promise((resolve) => { + escapeServer.close(() => resolve()); + }), + ]); + cleanupTempDirs(tempDirs); + } + }); + it("rejects invalid plugin fixture log byte limits before npm fixture setup", () => { const root = mkdtempSync(path.join(tmpdir(), "openclaw-plugin-npm-fixture-log-invalid-")); try { diff --git a/test/scripts/release-candidate-checklist.test.ts b/test/scripts/release-candidate-checklist.test.ts index 365389688fe..c14a951167e 100644 --- a/test/scripts/release-candidate-checklist.test.ts +++ b/test/scripts/release-candidate-checklist.test.ts @@ -10,6 +10,7 @@ import { resolveArtifactName, requireRunIdFromDispatchOutput, validateFullManifest, + validatePreflightManifest, validateWindowsSourceRelease, } from "../../scripts/release-candidate-checklist.mjs"; @@ -69,8 +70,64 @@ describe("release candidate checklist", () => { candidateParallelsShellCommand( ".artifacts/preflight/openclaw candidate.tgz", "/opt/homebrew/bin/gtimeout", + [".artifacts/preflight/openclaw-ai candidate.tgz"], ), ).toContain("'--target-tarball' '.artifacts/preflight/openclaw candidate.tgz'"); + expect( + candidateParallelsArgs(".artifacts/preflight/openclaw.tgz", [ + ".artifacts/preflight/openclaw-ai.tgz", + ]), + ).toEqual([ + "test:parallels:npm-update", + "--", + "--target-tarball", + ".artifacts/preflight/openclaw.tgz", + "--dependency-tarball", + ".artifacts/preflight/openclaw-ai.tgz", + "--json", + ]); + }); + + it("requires exact dependency tarball metadata in npm preflight manifests", () => { + const manifest = { + releaseTag: "v2026.7.1-beta.3", + releaseSha: "candidate-sha", + npmDistTag: "beta", + tarballName: "openclaw-2026.7.1-beta.3.tgz", + tarballSha256: "root-sha", + dependencyTarballs: [ + { + packageName: "@openclaw/ai", + packageVersion: "2026.7.1-beta.3", + tarballName: "openclaw-ai-2026.7.1-beta.3.tgz", + tarballSha256: "ai-sha", + }, + ], + }; + const params = { + tag: "v2026.7.1-beta.3", + targetSha: "candidate-sha", + npmDistTag: "beta", + }; + + expect(() => validatePreflightManifest(manifest, params)).not.toThrow(); + expect(() => + validatePreflightManifest({ ...manifest, dependencyTarballs: undefined }, params), + ).toThrow("missing dependency tarball metadata"); + expect(() => + validatePreflightManifest( + { + ...manifest, + dependencyTarballs: [ + { + ...manifest.dependencyTarballs[0], + tarballName: "../openclaw-ai.tgz", + }, + ], + }, + params, + ), + ).toThrow("invalid dependency tarball metadata"); }); it("requires run ids when dispatch is disabled", () => { @@ -87,7 +144,10 @@ describe("release candidate checklist", () => { secondValue: string, prefix = requiredArgs, ): [string, string[]] => [flag, [...prefix, flag, firstValue, flag, secondValue]]; - const duplicateFlag = (flag: string): [string, string[]] => [flag, [...requiredArgs, flag, flag]]; + const duplicateFlag = (flag: string): [string, string[]] => [ + flag, + [...requiredArgs, flag, flag], + ]; const duplicateCases = [ duplicateOption("--tag", "v2026.5.14-beta.3", "v2026.5.14-beta.4", []), duplicateOption("--workflow-ref", "release/a", "release/b"),