fix(crabbox): retry cold metadata probes so a slow run --help does not block validation (#102159)

The wrapper probes `crabbox --version` and `crabbox run --help` once each with a
snappy 5s timeout to enumerate providers before delegating. A cold Crabbox — the
first call right after a version bump, or one on a loaded machine — can exceed
that timeout while rendering `run --help` (52KB, emitted on stderr in 0.36) or
doing first-run init, and can emit nothing on that first call. When that happens
both guards fire and hard-exit the wrapper ("selected binary failed basic
--version/--help sanity checks" or "could not parse provider list from --help;
refusing to run"), which blocks ALL remote validation (`pnpm check:changed`,
remote `pnpm test` lanes) even though the binary is fine.

Retry each metadata probe once with a generous 20s timeout when the first attempt
is killed or returns empty. The warm path is unchanged (one ~instant probe); only
a slow/empty first probe pays for the retry, after which the sanity/provider-list
guards judge the recovered result. Add a regression test: a fake Crabbox whose
`run --help` is slower than the default probe timeout (and, like 0.36, writes help
to stderr) is now recovered by the retry instead of hard-failing.
This commit is contained in:
Peter Steinberger 2026-07-08 12:53:48 +01:00 committed by GitHub
parent bd4d4c0d31
commit 83ebbcb3ac
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 66 additions and 4 deletions

View file

@ -28,6 +28,11 @@ import { resolvePathEnvKey, resolveWindowsCmdExePath } from "./windows-cmd-helpe
const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
const CRABBOX_METADATA_PROBE_TIMEOUT_MS = 5_000; const CRABBOX_METADATA_PROBE_TIMEOUT_MS = 5_000;
// A cold Crabbox (first call after an upgrade, or one on a loaded machine) can
// exceed the snappy default probe timeout while it renders `run --help` or does
// first-run init. Retry the metadata probes once with this generous timeout so a
// single slow probe does not hard-fail the wrapper and block all remote validation.
const CRABBOX_METADATA_PROBE_RETRY_TIMEOUT_MS = 20_000;
const ignoreRepoBinary = process.env.OPENCLAW_CRABBOX_WRAPPER_IGNORE_REPO_BINARY === "1"; const ignoreRepoBinary = process.env.OPENCLAW_CRABBOX_WRAPPER_IGNORE_REPO_BINARY === "1";
const repoLocal = ignoreRepoBinary ? null : resolveCrabboxBinary(process.env, process.platform); const repoLocal = ignoreRepoBinary ? null : resolveCrabboxBinary(process.env, process.platform);
const pathLocal = resolvePathBinary("crabbox", process.env, process.platform); const pathLocal = resolvePathBinary("crabbox", process.env, process.platform);
@ -360,14 +365,14 @@ function buildBatchCommandLine(command, commandArgs) {
return `"${[escapedCommand, ...escapedArgs].join(" ")}"`; return `"${[escapedCommand, ...escapedArgs].join(" ")}"`;
} }
function checkedOutput(command, commandArgs) { function checkedOutput(command, commandArgs, timeoutMs = resolveMetadataProbeTimeoutMs(process.env)) {
const invocation = spawnInvocation(command, commandArgs, process.env, process.platform); const invocation = spawnInvocation(command, commandArgs, process.env, process.platform);
const result = spawnSync(invocation.command, invocation.args, { const result = spawnSync(invocation.command, invocation.args, {
cwd: repoRoot, cwd: repoRoot,
encoding: "utf8", encoding: "utf8",
stdio: ["ignore", "pipe", "pipe"], stdio: ["ignore", "pipe", "pipe"],
windowsVerbatimArguments: invocation.windowsVerbatimArguments, windowsVerbatimArguments: invocation.windowsVerbatimArguments,
timeout: resolveMetadataProbeTimeoutMs(process.env), timeout: timeoutMs,
killSignal: "SIGKILL", killSignal: "SIGKILL",
}); });
const timedOut = result.error?.name === "Error" && result.signal === "SIGKILL"; const timedOut = result.error?.name === "Error" && result.signal === "SIGKILL";
@ -378,6 +383,19 @@ function checkedOutput(command, commandArgs) {
}; };
} }
// Probe Crabbox metadata (`--version` / `run --help`) with one generous retry.
// A cold Crabbox can be SIGKILLed by the snappy default timeout or emit nothing
// on the first call, then be instant and clean on the next. Retrying keeps the
// warm path fast (one ~instant probe) while stopping a single slow probe from
// tripping the sanity/provider-list guards and blocking all remote validation.
function probeCrabboxMetadata(command, commandArgs) {
const first = checkedOutput(command, commandArgs);
if (first.status === 0 && first.text.length > 0) {
return first;
}
return checkedOutput(command, commandArgs, CRABBOX_METADATA_PROBE_RETRY_TIMEOUT_MS);
}
function parseCrabboxVersion(value) { function parseCrabboxVersion(value) {
const match = `${value}`.match(/\bv?(\d+)\.(\d+)\.(\d+)(?:-([^\s+]+))?(?:\+[^\s]+)?\b/u); const match = `${value}`.match(/\bv?(\d+)\.(\d+)\.(\d+)(?:-([^\s+]+))?(?:\+[^\s]+)?\b/u);
if (!match) { if (!match) {
@ -3155,8 +3173,8 @@ function injectFullCheckoutLeaseReclaim(commandArgs) {
return normalizedArgs; return normalizedArgs;
} }
const version = checkedOutput(binary, ["--version"]); const version = probeCrabboxMetadata(binary, ["--version"]);
const help = checkedOutput(binary, ["run", "--help"]); const help = probeCrabboxMetadata(binary, ["run", "--help"]);
const providers = parseProvidersFromHelp(help.text); const providers = parseProvidersFromHelp(help.text);
const displayBinary = binary === "crabbox" ? "crabbox" : relative(repoRoot, binary); const displayBinary = binary === "crabbox" ? "crabbox" : relative(repoRoot, binary);
const provider = selectedProvider(args, providers); const provider = selectedProvider(args, providers);

View file

@ -227,6 +227,32 @@ function makeSlowVersionCrabbox(helpText: string): string {
return binDir; return binDir;
} }
// Fake Crabbox whose `run --help` is slow on every call and, like real Crabbox
// 0.36, renders the provider help to stderr. Used to prove the wrapper retries a
// cold/slow metadata probe instead of hard-failing.
function makeSlowHelpCrabbox(helpText: string, delayMs: number): string {
const binDir = mkdtempSync(path.join(tmpdir(), "openclaw-slow-help-crabbox-"));
tempDirs.push(binDir);
const crabboxPath = path.join(binDir, "crabbox");
const script = [
"#!/usr/bin/env node",
"const args = process.argv.slice(2);",
"if (args[0] === '--version') {",
" console.log(process.env.OPENCLAW_FAKE_CRABBOX_VERSION || 'crabbox 0.22.1');",
" process.exit(0);",
"} else if (args[0] === 'run' && args[1] === '--help') {",
` setTimeout(() => { process.stderr.write(${JSON.stringify(helpText)}); process.exit(0); }, ${delayMs});`,
"} else {",
" process.exit(0);",
"}",
].join("\n");
writeFileSync(crabboxPath, `${script}\n`, "utf8");
writeFileSync(`${crabboxPath}.cmd`, windowsNodeCmdShim("crabbox"), "utf8");
chmodSync(crabboxPath, 0o755);
return binDir;
}
function testTimingPreload(options: { clockScale?: number; spawnTimeoutMs?: number }): string { function testTimingPreload(options: { clockScale?: number; spawnTimeoutMs?: number }): string {
const key = JSON.stringify(options); const key = JSON.stringify(options);
let preloadPath = timingPreloads.get(key); let preloadPath = timingPreloads.get(key);
@ -3128,6 +3154,24 @@ describe("scripts/crabbox-wrapper", () => {
expect(result.stderr).toContain("selected binary failed basic --version/--help sanity checks"); expect(result.stderr).toContain("selected binary failed basic --version/--help sanity checks");
}); });
it("retries a cold Crabbox whose run --help is slower than the default probe timeout", () => {
const helpText = "provider: hetzner, aws, local-container, blacksmith-testbox, or cloudflare\n";
// First probe is SIGKILLed at 150ms; the retry gets the full generous timeout
// and reads the (600ms) stderr help, so the wrapper must not hard-fail.
const result = runWrapper(helpText, ["--version"], {
env: { OPENCLAW_TEST_CRABBOX_METADATA_PROBE_TIMEOUT_MS: "150" },
extraPathEntries: [makeSlowHelpCrabbox(helpText, 600)],
});
expect(result.error).toBeUndefined();
expect(result.status).toBe(0);
expect(result.stderr).not.toContain("could not parse provider list");
expect(result.stderr).not.toContain("selected binary failed basic --version/--help sanity checks");
expect(result.stderr).toContain(
"providers=hetzner,aws,local-container,blacksmith-testbox,cloudflare",
);
});
it("parses provider choices from the --provider flag help format", () => { it("parses provider choices from the --provider flag help format", () => {
const helpText = const helpText =
"Usage: crabbox run [options]\n --provider hetzner|aws|local-container|blacksmith-testbox|cloudflare\n"; "Usage: crabbox run [options]\n --provider hetzner|aws|local-container|blacksmith-testbox|cloudflare\n";