mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-09 15:59:30 +00:00
fix(crabbox): retry cold metadata probes so a slow run --help does not block validation (#102159)
The wrapper probes `crabbox --version` and `crabbox run --help` once each with a
snappy 5s timeout to enumerate providers before delegating. A cold Crabbox — the
first call right after a version bump, or one on a loaded machine — can exceed
that timeout while rendering `run --help` (52KB, emitted on stderr in 0.36) or
doing first-run init, and can emit nothing on that first call. When that happens
both guards fire and hard-exit the wrapper ("selected binary failed basic
--version/--help sanity checks" or "could not parse provider list from --help;
refusing to run"), which blocks ALL remote validation (`pnpm check:changed`,
remote `pnpm test` lanes) even though the binary is fine.
Retry each metadata probe once with a generous 20s timeout when the first attempt
is killed or returns empty. The warm path is unchanged (one ~instant probe); only
a slow/empty first probe pays for the retry, after which the sanity/provider-list
guards judge the recovered result. Add a regression test: a fake Crabbox whose
`run --help` is slower than the default probe timeout (and, like 0.36, writes help
to stderr) is now recovered by the retry instead of hard-failing.
This commit is contained in:
parent
bd4d4c0d31
commit
83ebbcb3ac
2 changed files with 66 additions and 4 deletions
|
|
@ -28,6 +28,11 @@ import { resolvePathEnvKey, resolveWindowsCmdExePath } from "./windows-cmd-helpe
|
|||
|
||||
const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const CRABBOX_METADATA_PROBE_TIMEOUT_MS = 5_000;
|
||||
// A cold Crabbox (first call after an upgrade, or one on a loaded machine) can
|
||||
// exceed the snappy default probe timeout while it renders `run --help` or does
|
||||
// first-run init. Retry the metadata probes once with this generous timeout so a
|
||||
// single slow probe does not hard-fail the wrapper and block all remote validation.
|
||||
const CRABBOX_METADATA_PROBE_RETRY_TIMEOUT_MS = 20_000;
|
||||
const ignoreRepoBinary = process.env.OPENCLAW_CRABBOX_WRAPPER_IGNORE_REPO_BINARY === "1";
|
||||
const repoLocal = ignoreRepoBinary ? null : resolveCrabboxBinary(process.env, process.platform);
|
||||
const pathLocal = resolvePathBinary("crabbox", process.env, process.platform);
|
||||
|
|
@ -360,14 +365,14 @@ function buildBatchCommandLine(command, commandArgs) {
|
|||
return `"${[escapedCommand, ...escapedArgs].join(" ")}"`;
|
||||
}
|
||||
|
||||
function checkedOutput(command, commandArgs) {
|
||||
function checkedOutput(command, commandArgs, timeoutMs = resolveMetadataProbeTimeoutMs(process.env)) {
|
||||
const invocation = spawnInvocation(command, commandArgs, process.env, process.platform);
|
||||
const result = spawnSync(invocation.command, invocation.args, {
|
||||
cwd: repoRoot,
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
windowsVerbatimArguments: invocation.windowsVerbatimArguments,
|
||||
timeout: resolveMetadataProbeTimeoutMs(process.env),
|
||||
timeout: timeoutMs,
|
||||
killSignal: "SIGKILL",
|
||||
});
|
||||
const timedOut = result.error?.name === "Error" && result.signal === "SIGKILL";
|
||||
|
|
@ -378,6 +383,19 @@ function checkedOutput(command, commandArgs) {
|
|||
};
|
||||
}
|
||||
|
||||
// Probe Crabbox metadata (`--version` / `run --help`) with one generous retry.
|
||||
// A cold Crabbox can be SIGKILLed by the snappy default timeout or emit nothing
|
||||
// on the first call, then be instant and clean on the next. Retrying keeps the
|
||||
// warm path fast (one ~instant probe) while stopping a single slow probe from
|
||||
// tripping the sanity/provider-list guards and blocking all remote validation.
|
||||
function probeCrabboxMetadata(command, commandArgs) {
|
||||
const first = checkedOutput(command, commandArgs);
|
||||
if (first.status === 0 && first.text.length > 0) {
|
||||
return first;
|
||||
}
|
||||
return checkedOutput(command, commandArgs, CRABBOX_METADATA_PROBE_RETRY_TIMEOUT_MS);
|
||||
}
|
||||
|
||||
function parseCrabboxVersion(value) {
|
||||
const match = `${value}`.match(/\bv?(\d+)\.(\d+)\.(\d+)(?:-([^\s+]+))?(?:\+[^\s]+)?\b/u);
|
||||
if (!match) {
|
||||
|
|
@ -3155,8 +3173,8 @@ function injectFullCheckoutLeaseReclaim(commandArgs) {
|
|||
return normalizedArgs;
|
||||
}
|
||||
|
||||
const version = checkedOutput(binary, ["--version"]);
|
||||
const help = checkedOutput(binary, ["run", "--help"]);
|
||||
const version = probeCrabboxMetadata(binary, ["--version"]);
|
||||
const help = probeCrabboxMetadata(binary, ["run", "--help"]);
|
||||
const providers = parseProvidersFromHelp(help.text);
|
||||
const displayBinary = binary === "crabbox" ? "crabbox" : relative(repoRoot, binary);
|
||||
const provider = selectedProvider(args, providers);
|
||||
|
|
|
|||
|
|
@ -227,6 +227,32 @@ function makeSlowVersionCrabbox(helpText: string): string {
|
|||
return binDir;
|
||||
}
|
||||
|
||||
// Fake Crabbox whose `run --help` is slow on every call and, like real Crabbox
|
||||
// 0.36, renders the provider help to stderr. Used to prove the wrapper retries a
|
||||
// cold/slow metadata probe instead of hard-failing.
|
||||
function makeSlowHelpCrabbox(helpText: string, delayMs: number): string {
|
||||
const binDir = mkdtempSync(path.join(tmpdir(), "openclaw-slow-help-crabbox-"));
|
||||
tempDirs.push(binDir);
|
||||
const crabboxPath = path.join(binDir, "crabbox");
|
||||
|
||||
const script = [
|
||||
"#!/usr/bin/env node",
|
||||
"const args = process.argv.slice(2);",
|
||||
"if (args[0] === '--version') {",
|
||||
" console.log(process.env.OPENCLAW_FAKE_CRABBOX_VERSION || 'crabbox 0.22.1');",
|
||||
" process.exit(0);",
|
||||
"} else if (args[0] === 'run' && args[1] === '--help') {",
|
||||
` setTimeout(() => { process.stderr.write(${JSON.stringify(helpText)}); process.exit(0); }, ${delayMs});`,
|
||||
"} else {",
|
||||
" process.exit(0);",
|
||||
"}",
|
||||
].join("\n");
|
||||
writeFileSync(crabboxPath, `${script}\n`, "utf8");
|
||||
writeFileSync(`${crabboxPath}.cmd`, windowsNodeCmdShim("crabbox"), "utf8");
|
||||
chmodSync(crabboxPath, 0o755);
|
||||
return binDir;
|
||||
}
|
||||
|
||||
function testTimingPreload(options: { clockScale?: number; spawnTimeoutMs?: number }): string {
|
||||
const key = JSON.stringify(options);
|
||||
let preloadPath = timingPreloads.get(key);
|
||||
|
|
@ -3128,6 +3154,24 @@ describe("scripts/crabbox-wrapper", () => {
|
|||
expect(result.stderr).toContain("selected binary failed basic --version/--help sanity checks");
|
||||
});
|
||||
|
||||
it("retries a cold Crabbox whose run --help is slower than the default probe timeout", () => {
|
||||
const helpText = "provider: hetzner, aws, local-container, blacksmith-testbox, or cloudflare\n";
|
||||
// First probe is SIGKILLed at 150ms; the retry gets the full generous timeout
|
||||
// and reads the (600ms) stderr help, so the wrapper must not hard-fail.
|
||||
const result = runWrapper(helpText, ["--version"], {
|
||||
env: { OPENCLAW_TEST_CRABBOX_METADATA_PROBE_TIMEOUT_MS: "150" },
|
||||
extraPathEntries: [makeSlowHelpCrabbox(helpText, 600)],
|
||||
});
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stderr).not.toContain("could not parse provider list");
|
||||
expect(result.stderr).not.toContain("selected binary failed basic --version/--help sanity checks");
|
||||
expect(result.stderr).toContain(
|
||||
"providers=hetzner,aws,local-container,blacksmith-testbox,cloudflare",
|
||||
);
|
||||
});
|
||||
|
||||
it("parses provider choices from the --provider flag help format", () => {
|
||||
const helpText =
|
||||
"Usage: crabbox run [options]\n --provider hetzner|aws|local-container|blacksmith-testbox|cloudflare\n";
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue