fix(crabbox): classify source fetch failures (#152138)

scripts/crabbox-source-receiver.mts

@@ -8,6 +8,7 @@ const path = require("node:path");
 const { createHash } = require("node:crypto");
 const { isUtf8 } = require("node:buffer");
 const { spawnSync } = require("node:child_process");
+const { getSystemErrorMap } = require("node:util");
 const expected = JSON.parse(process.argv[1]);
 const syncRoot = process.cwd();
 const cwd = process.argv[2] ?? syncRoot;
@@ -37,6 +38,33 @@ function hashFile(file, algorithm, blob = false) {
     return hash.digest("hex");
   } finally { fs.closeSync(fd); }
 }
+const gitFailureCause = Object.freeze({
+  noSpace: "no-space", permissionDenied: "permission-denied",
+  commandUnavailable: "command-unavailable", outputLimit: "output-limit",
+  terminated: "terminated", remoteRefMissing: "remote-ref-missing",
+  invalidObjectData: "invalid-object-data", dns: "dns", connection: "connection",
+  auth: "auth", unknown: "unknown",
+});
+function gitFailure(phase, result) {
+  const errors = getSystemErrorMap();
+  const rawCode = result.error?.code;
+  const code = [...errors.values()].some(([name]) => name === rawCode) ? rawCode : null;
+  const text = Buffer.isBuffer(result.stderr) ? result.stderr.toString("utf8").trim() : "";
+  const cause = code === "ENOSPC" ? gitFailureCause.noSpace
+    : code === "EACCES" || code === "EPERM" ? gitFailureCause.permissionDenied
+    : code === "ENOENT" ? gitFailureCause.commandUnavailable
+    : code === "ENOBUFS" ? gitFailureCause.outputLimit
+    : result.signal !== null ? gitFailureCause.terminated
+    : /^fatal: couldn't find remote ref [^\r\n]+$/u.test(text) ? gitFailureCause.remoteRefMissing
+    : /^fatal: (?:pack has bad object(?: at offset \d+)?|bad object [a-f0-9]+|index-pack failed|fetch-pack: invalid index-pack output)$/u.test(text) ? gitFailureCause.invalidObjectData
+    : /^fatal: unable to access '[^'\r\n]+': Could not resolve (?:host|proxy): [^\r\n]+$/u.test(text) ? gitFailureCause.dns
+    : /^fatal: unable to access '[^'\r\n]+': (?:Failed to connect to [^\r\n]+|Recv failure: Connection reset by peer)$/u.test(text) ? gitFailureCause.connection
+    : /^fatal: (?:Authentication failed for '[^'\r\n]+'|could not read Username for '[^'\r\n]+': [^\r\n]+)$/u.test(text) ? gitFailureCause.auth
+    : gitFailureCause.unknown;
+  return { phase, baseSha: /^[a-f0-9]{40}$/u.test(expected.baseSha) ? expected.baseSha : null,
+    status: result.status, signal: result.signal, spawnError: Boolean(result.error),
+    code, errno: errors.has(result.error?.errno) ? result.error.errno : null, cause };
+}
 try {
   if (process.argv[2] && (cwd === syncRoot || cwd.startsWith(syncRoot + path.sep) || syncRoot.startsWith(cwd + path.sep)))
     fail("Testbox execution and sync workspaces overlap; stop this lease and warm a fresh one");
@@ -58,10 +86,10 @@ try {
   delete env.GIT_ALTERNATE_OBJECT_DIRECTORIES;
   delete env.GIT_SHALLOW_FILE;
   function git(args, options = {}) {
-    const { encoding, ...spawnOptions } = options;
+    const { encoding, phase = args[0], ...spawnOptions } = options;
     const result = spawnSync("git", ["-c", "core.hooksPath=/dev/null", "-c", "core.fsmonitor=false", ...args],
       { cwd, env, maxBuffer: 64 * 1024 * 1024, ...spawnOptions });
-    if (result.status !== 0) fail("source Git operation failed: " + args[0]);
+    if (result.status !== 0) fail("source Git operation failed: " + JSON.stringify(gitFailure(phase, result)));
     if (encoding === "buffer") return result.stdout;
     if (result.stdout === null) return "";
     if (!isUtf8(result.stdout)) fail("unsupported non-UTF-8 Git metadata");
@@ -69,10 +97,12 @@ try {
   }
   git(["init", "-q"]);
   git(["remote", "add", "origin", "https://github.com/openclaw/openclaw.git"]);
-  git(["fetch", "-q", "--depth=2", "origin", expected.baseSha + ":refs/remotes/origin/main"]);
+  git(["fetch", "-q", "--depth=2", "origin", expected.baseSha + ":refs/remotes/origin/main"],
+    { phase: "base-fetch" });
   if (git(["rev-parse", "refs/remotes/origin/main"]).trim() !== expected.baseSha)
     fail("source base mismatch");
-  git(["fetch", "-q", bundle, "refs/openclaw/source-capsule:refs/heads/openclaw-source"]);
+  git(["fetch", "-q", bundle, "refs/openclaw/source-capsule:refs/heads/openclaw-source"],
+    { phase: "capsule-fetch" });
   for (const [ref, value] of [
     ["refs/heads/openclaw-source", expected.carrier],
     ["refs/heads/openclaw-source^{tree}", expected.tree],

test/scripts/crabbox-wrapper.test.ts

@@ -25,6 +25,7 @@ import { homedir, tmpdir } from "node:os";
 import path from "node:path";
 import { setTimeout as delay } from "node:timers/promises";
 import { pathToFileURL } from "node:url";
+import { getSystemErrorMap } from "node:util";
 import { build, type BuildOptions } from "esbuild";
 import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
 import { parse } from "yaml";
@@ -5141,6 +5142,90 @@ process.on("uncaughtExceptionMonitor", (error) => {
       // Shared receiver failure paths need one full real-Git fixture; provider/history
       // variants above retain independent successful source identity checks.
       if (provider === "blacksmith-testbox" && !shallow) {
+        const errnoFor = (code: string) =>
+          [...getSystemErrorMap()].find(([, [name]]) => name === code)?.[0];
+        const gitText = {
+          ref: "fatal: couldn't find remote ref fixture\n",
+          object: "fatal: pack has bad object at offset 12\n",
+          dns: "fatal: unable to access 'https://private.invalid/repo': Could not resolve host: private.invalid\n",
+          connection:
+            "fatal: unable to access 'https://private.invalid/repo': Failed to connect to private.invalid port 443\n",
+          auth: "fatal: Authentication failed for 'https://private.invalid/repo'\n",
+          nearAuth: "warning: authentication failed later PRIVATE_SENTINEL\n",
+          nearRef: "fatal: couldn't find remote reference PRIVATE_SENTINEL\n",
+        };
+        const fetchFailures = [
+          ["base-fetch", "no-space", "ENOSPC"],
+          ["capsule-fetch", "permission-denied", "EACCES"],
+          ["base-fetch", "permission-denied", "EPERM"],
+          ["capsule-fetch", "command-unavailable", "ENOENT"],
+          ["base-fetch", "output-limit", "ENOBUFS"],
+          ["capsule-fetch", "terminated", undefined, "SIGTERM"],
+          ["base-fetch", "remote-ref-missing", undefined, undefined, gitText.ref],
+          ["capsule-fetch", "invalid-object-data", undefined, undefined, gitText.object],
+          ["base-fetch", "dns", undefined, undefined, gitText.dns],
+          ["capsule-fetch", "connection", undefined, undefined, gitText.connection],
+          ["base-fetch", "auth", undefined, undefined, gitText.auth],
+          ["capsule-fetch", "unknown", undefined, undefined, gitText.nearAuth],
+          ["base-fetch", "unknown", undefined, undefined, gitText.nearRef],
+        ] as const;
+        for (const [index, [phase, cause, code, signal, stderr]] of fetchFailures.entries()) {
+          const preload = path.join(root, `fetch-failure-${index}.cjs`);
+          const errno = code ? errnoFor(code) : undefined;
+          writeFileSync(
+            preload,
+            `const cp = require("node:child_process");
+const original = cp.spawnSync;
+const fault = ${JSON.stringify({ phase, code, errno, signal, stderr })};
+cp.spawnSync = (command, args, options) => {
+  const fetchIndex = args.indexOf("fetch");
+  if (command !== "git" || fetchIndex < 0 || args.slice(fetchIndex + 1).includes("origin") !== (fault.phase === "base-fetch"))
+    return original(command, args, options);
+  return { status: fault.code || fault.signal ? null : 128, signal: fault.signal ?? null,
+    stdout: Buffer.alloc(0), stderr: Buffer.from(fault.stderr ?? "PRIVATE_SENTINEL\\n"),
+    error: fault.code ? Object.assign(new Error("PRIVATE_ERROR_MESSAGE"), { code: fault.code, errno: fault.errno }) : undefined };
+};\n`,
+          );
+          const argvPath = path.join(root, `fetch-failure-${index}.json`);
+          let priorIndex: Buffer | undefined;
+          const rejected = receive(
+            `fetch-failure-${index}`,
+            candidate.remoteCommand,
+            candidate.bundle,
+            origin,
+            { NODE_OPTIONS: `--require=${preload}`, TRANSPORT_FIXTURE_ARGV: argvPath },
+            true,
+            [],
+            (receiver) => {
+              priorIndex = readFileSync(path.join(receiver, ".git", "index"));
+            },
+          );
+          const prefix = "[crabbox] source verification failed: source Git operation failed: ";
+          const line = rejected.result.stderr.split("\n").find((entry) => entry.startsWith(prefix));
+          expect(line, failureDetail(rejected.result)).toBeDefined();
+          expect(JSON.parse(line!.slice(prefix.length))).toEqual({
+            phase,
+            baseSha: base,
+            status: code || signal ? null : 128,
+            signal: signal ?? null,
+            spawnError: Boolean(code),
+            code: code ?? null,
+            errno: errno ?? null,
+            cause,
+          });
+          expect(rejected.result.status, failureDetail(rejected.result)).toBe(2);
+          expect(rejected.result.stdout).toBe("");
+          expect(rejected.result.stderr).not.toMatch(/PRIVATE_|private\.invalid/u);
+          expect(existsSync(argvPath)).toBe(false);
+          expect(git(rejected.receiver, ["rev-parse", "HEAD"])).toBe(base);
+          expect(readFileSync(path.join(rejected.receiver, ".git", "index"))).toEqual(priorIndex);
+          expect(readFileSync(path.join(rejected.receiver, "owner.txt"), "utf8")).toBe(
+            "native stale bytes\n",
+          );
+          expect(
+            readdirSync(rejected.receiver).filter((file) => file.startsWith(".openclaw-source-")),
+          ).toEqual([]);
+        }
         for (const [fault, file, message] of [
           ["bytes", "newer-source.txt", "source bytes mismatch"],
           ["mode", "newer-source.txt", "source mode mismatch"],