From dc2b976b5b707d137ff92fc82dc5e0835be8674a Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sat, 19 Sep 2026 06:02:03 +0800 Subject: [PATCH] fix(crabbox): classify source fetch failures (#152138) --- scripts/crabbox-source-receiver.mts | 38 +++++++++++-- test/scripts/crabbox-wrapper.test.ts | 85 ++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 4 deletions(-) diff --git a/scripts/crabbox-source-receiver.mts b/scripts/crabbox-source-receiver.mts index da52c5e3ec5e..7731b9edeb90 100644 --- a/scripts/crabbox-source-receiver.mts +++ b/scripts/crabbox-source-receiver.mts @@ -8,6 +8,7 @@ const path = require("node:path"); const { createHash } = require("node:crypto"); const { isUtf8 } = require("node:buffer"); const { spawnSync } = require("node:child_process"); +const { getSystemErrorMap } = require("node:util"); const expected = JSON.parse(process.argv[1]); const syncRoot = process.cwd(); const cwd = process.argv[2] ?? syncRoot; @@ -37,6 +38,33 @@ function hashFile(file, algorithm, blob = false) { return hash.digest("hex"); } finally { fs.closeSync(fd); } } +const gitFailureCause = Object.freeze({ + noSpace: "no-space", permissionDenied: "permission-denied", + commandUnavailable: "command-unavailable", outputLimit: "output-limit", + terminated: "terminated", remoteRefMissing: "remote-ref-missing", + invalidObjectData: "invalid-object-data", dns: "dns", connection: "connection", + auth: "auth", unknown: "unknown", +}); +function gitFailure(phase, result) { + const errors = getSystemErrorMap(); + const rawCode = result.error?.code; + const code = [...errors.values()].some(([name]) => name === rawCode) ? rawCode : null; + const text = Buffer.isBuffer(result.stderr) ? result.stderr.toString("utf8").trim() : ""; + const cause = code === "ENOSPC" ? gitFailureCause.noSpace + : code === "EACCES" || code === "EPERM" ? gitFailureCause.permissionDenied + : code === "ENOENT" ? gitFailureCause.commandUnavailable + : code === "ENOBUFS" ? gitFailureCause.outputLimit + : result.signal !== null ? gitFailureCause.terminated + : /^fatal: couldn't find remote ref [^\r\n]+$/u.test(text) ? gitFailureCause.remoteRefMissing + : /^fatal: (?:pack has bad object(?: at offset \d+)?|bad object [a-f0-9]+|index-pack failed|fetch-pack: invalid index-pack output)$/u.test(text) ? gitFailureCause.invalidObjectData + : /^fatal: unable to access '[^'\r\n]+': Could not resolve (?:host|proxy): [^\r\n]+$/u.test(text) ? gitFailureCause.dns + : /^fatal: unable to access '[^'\r\n]+': (?:Failed to connect to [^\r\n]+|Recv failure: Connection reset by peer)$/u.test(text) ? gitFailureCause.connection + : /^fatal: (?:Authentication failed for '[^'\r\n]+'|could not read Username for '[^'\r\n]+': [^\r\n]+)$/u.test(text) ? gitFailureCause.auth + : gitFailureCause.unknown; + return { phase, baseSha: /^[a-f0-9]{40}$/u.test(expected.baseSha) ? expected.baseSha : null, + status: result.status, signal: result.signal, spawnError: Boolean(result.error), + code, errno: errors.has(result.error?.errno) ? result.error.errno : null, cause }; +} try { if (process.argv[2] && (cwd === syncRoot || cwd.startsWith(syncRoot + path.sep) || syncRoot.startsWith(cwd + path.sep))) fail("Testbox execution and sync workspaces overlap; stop this lease and warm a fresh one"); @@ -58,10 +86,10 @@ try { delete env.GIT_ALTERNATE_OBJECT_DIRECTORIES; delete env.GIT_SHALLOW_FILE; function git(args, options = {}) { - const { encoding, ...spawnOptions } = options; + const { encoding, phase = args[0], ...spawnOptions } = options; const result = spawnSync("git", ["-c", "core.hooksPath=/dev/null", "-c", "core.fsmonitor=false", ...args], { cwd, env, maxBuffer: 64 * 1024 * 1024, ...spawnOptions }); - if (result.status !== 0) fail("source Git operation failed: " + args[0]); + if (result.status !== 0) fail("source Git operation failed: " + JSON.stringify(gitFailure(phase, result))); if (encoding === "buffer") return result.stdout; if (result.stdout === null) return ""; if (!isUtf8(result.stdout)) fail("unsupported non-UTF-8 Git metadata"); @@ -69,10 +97,12 @@ try { } git(["init", "-q"]); git(["remote", "add", "origin", "https://github.com/openclaw/openclaw.git"]); - git(["fetch", "-q", "--depth=2", "origin", expected.baseSha + ":refs/remotes/origin/main"]); + git(["fetch", "-q", "--depth=2", "origin", expected.baseSha + ":refs/remotes/origin/main"], + { phase: "base-fetch" }); if (git(["rev-parse", "refs/remotes/origin/main"]).trim() !== expected.baseSha) fail("source base mismatch"); - git(["fetch", "-q", bundle, "refs/openclaw/source-capsule:refs/heads/openclaw-source"]); + git(["fetch", "-q", bundle, "refs/openclaw/source-capsule:refs/heads/openclaw-source"], + { phase: "capsule-fetch" }); for (const [ref, value] of [ ["refs/heads/openclaw-source", expected.carrier], ["refs/heads/openclaw-source^{tree}", expected.tree], diff --git a/test/scripts/crabbox-wrapper.test.ts b/test/scripts/crabbox-wrapper.test.ts index 359145261002..cd8145854990 100644 --- a/test/scripts/crabbox-wrapper.test.ts +++ b/test/scripts/crabbox-wrapper.test.ts @@ -25,6 +25,7 @@ import { homedir, tmpdir } from "node:os"; import path from "node:path"; import { setTimeout as delay } from "node:timers/promises"; import { pathToFileURL } from "node:url"; +import { getSystemErrorMap } from "node:util"; import { build, type BuildOptions } from "esbuild"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; import { parse } from "yaml"; @@ -5141,6 +5142,90 @@ process.on("uncaughtExceptionMonitor", (error) => { // Shared receiver failure paths need one full real-Git fixture; provider/history // variants above retain independent successful source identity checks. if (provider === "blacksmith-testbox" && !shallow) { + const errnoFor = (code: string) => + [...getSystemErrorMap()].find(([, [name]]) => name === code)?.[0]; + const gitText = { + ref: "fatal: couldn't find remote ref fixture\n", + object: "fatal: pack has bad object at offset 12\n", + dns: "fatal: unable to access 'https://private.invalid/repo': Could not resolve host: private.invalid\n", + connection: + "fatal: unable to access 'https://private.invalid/repo': Failed to connect to private.invalid port 443\n", + auth: "fatal: Authentication failed for 'https://private.invalid/repo'\n", + nearAuth: "warning: authentication failed later PRIVATE_SENTINEL\n", + nearRef: "fatal: couldn't find remote reference PRIVATE_SENTINEL\n", + }; + const fetchFailures = [ + ["base-fetch", "no-space", "ENOSPC"], + ["capsule-fetch", "permission-denied", "EACCES"], + ["base-fetch", "permission-denied", "EPERM"], + ["capsule-fetch", "command-unavailable", "ENOENT"], + ["base-fetch", "output-limit", "ENOBUFS"], + ["capsule-fetch", "terminated", undefined, "SIGTERM"], + ["base-fetch", "remote-ref-missing", undefined, undefined, gitText.ref], + ["capsule-fetch", "invalid-object-data", undefined, undefined, gitText.object], + ["base-fetch", "dns", undefined, undefined, gitText.dns], + ["capsule-fetch", "connection", undefined, undefined, gitText.connection], + ["base-fetch", "auth", undefined, undefined, gitText.auth], + ["capsule-fetch", "unknown", undefined, undefined, gitText.nearAuth], + ["base-fetch", "unknown", undefined, undefined, gitText.nearRef], + ] as const; + for (const [index, [phase, cause, code, signal, stderr]] of fetchFailures.entries()) { + const preload = path.join(root, `fetch-failure-${index}.cjs`); + const errno = code ? errnoFor(code) : undefined; + writeFileSync( + preload, + `const cp = require("node:child_process"); +const original = cp.spawnSync; +const fault = ${JSON.stringify({ phase, code, errno, signal, stderr })}; +cp.spawnSync = (command, args, options) => { + const fetchIndex = args.indexOf("fetch"); + if (command !== "git" || fetchIndex < 0 || args.slice(fetchIndex + 1).includes("origin") !== (fault.phase === "base-fetch")) + return original(command, args, options); + return { status: fault.code || fault.signal ? null : 128, signal: fault.signal ?? null, + stdout: Buffer.alloc(0), stderr: Buffer.from(fault.stderr ?? "PRIVATE_SENTINEL\\n"), + error: fault.code ? Object.assign(new Error("PRIVATE_ERROR_MESSAGE"), { code: fault.code, errno: fault.errno }) : undefined }; +};\n`, + ); + const argvPath = path.join(root, `fetch-failure-${index}.json`); + let priorIndex: Buffer | undefined; + const rejected = receive( + `fetch-failure-${index}`, + candidate.remoteCommand, + candidate.bundle, + origin, + { NODE_OPTIONS: `--require=${preload}`, TRANSPORT_FIXTURE_ARGV: argvPath }, + true, + [], + (receiver) => { + priorIndex = readFileSync(path.join(receiver, ".git", "index")); + }, + ); + const prefix = "[crabbox] source verification failed: source Git operation failed: "; + const line = rejected.result.stderr.split("\n").find((entry) => entry.startsWith(prefix)); + expect(line, failureDetail(rejected.result)).toBeDefined(); + expect(JSON.parse(line!.slice(prefix.length))).toEqual({ + phase, + baseSha: base, + status: code || signal ? null : 128, + signal: signal ?? null, + spawnError: Boolean(code), + code: code ?? null, + errno: errno ?? null, + cause, + }); + expect(rejected.result.status, failureDetail(rejected.result)).toBe(2); + expect(rejected.result.stdout).toBe(""); + expect(rejected.result.stderr).not.toMatch(/PRIVATE_|private\.invalid/u); + expect(existsSync(argvPath)).toBe(false); + expect(git(rejected.receiver, ["rev-parse", "HEAD"])).toBe(base); + expect(readFileSync(path.join(rejected.receiver, ".git", "index"))).toEqual(priorIndex); + expect(readFileSync(path.join(rejected.receiver, "owner.txt"), "utf8")).toBe( + "native stale bytes\n", + ); + expect( + readdirSync(rejected.receiver).filter((file) => file.startsWith(".openclaw-source-")), + ).toEqual([]); + } for (const [fault, file, message] of [ ["bytes", "newer-source.txt", "source bytes mismatch"], ["mode", "newer-source.txt", "source mode mismatch"],