mirror of
https://github.com/openclaw/openclaw.git
synced 2026-10-03 01:29:56 +00:00
d457b2c611
* fix(test): checkout fixture budgets cut healthy runs on loaded hosts
On a loaded macOS host (load 55-83), ci-platform-checkout.test.ts and the
other checkout fixture consumers failed through three fixture-internal
wall-clock budgets, not through the workflow under test:
- the supervisor's mock-resolution preflight ran bash under its own 2 s
timeout ("mock command resolution failed: spawnSync bash ETIMEDOUT");
- the POSIX process census shadowed its caller's operation deadline with a
fresh 1 s budget for all singleton ps queries ("Fixture process census
failed (ETIMEDOUT)"); the owner-ancestry walk had the same 1 s shape;
- the supervisor cut every run at 45 s ("fixture deadline exceeded"), nested
under the helper's 50 s close race only because the helper could not see
the Vitest test timeout.
Measured breakdown: a multi-attempt scenario serially starts 25-36 Node
actors plus the Python owner; at load ~72 Linux git-failure spent 19.4 s,
of which the fixture's own 82 ps calls were 1.6 s. The work scales with
scheduler latency and is the contract under test, so the fix is ownership
of the time bound, not less work.
The owning test's AbortSignal now bounds a supervised run.
withCiCheckoutFixture takes it, rejects its close join on abort, and asks
the live supervisor to cancel over IPC so its own stop() retires the
detached shell group and actors, keeping the SIGKILL fallback for a wedged
supervisor. The supervisor drops its 45 s watchdog; its operation deadline
becomes the existing 60 s actor lifetime ceiling, which only bounds orphans.
Census, preflight, and ancestry queries borrow that operation deadline, as
the Windows census witness already did.
Every runCiGitStep and withCiCheckoutFixture caller passes its test signal;
.each registrations that need the context move to .for, which is the only
form Vitest 5 hands the context to.
Regressions: the shared slow-witness preload now gives POSIX supervisors a
first native query that spends 1.1 s on their clock (fails main's fixture
7/7 with "census failed (unverified)"), and the outer-runner retention
proof gains a cancel fault proving the aborted supervisor retires its shell.
* fix(test): keep checkout signal bindings lint-clean
414 lines
15 KiB
TypeScript
414 lines
15 KiB
TypeScript
import { beforeAll, expect, it, vi } from "vitest";
|
|
import { runCiGitStep } from "./ci-git-owner.test-support.js";
|
|
import type { PerformanceFixtureOptions } from "./openclaw-performance-workflow.test-support.js";
|
|
|
|
// Each case owns its checkout and process trees. Overlap their real timeout and
|
|
// drain waits while keeping subprocess pressure bounded.
|
|
beforeAll(() => {
|
|
vi.setConfig({ maxConcurrency: 2 });
|
|
return () => vi.resetConfig();
|
|
});
|
|
|
|
// Performance jobs run on Ubuntu. Exercise their POSIX bodies here; the shared
|
|
// ci-platform-checkout suite owns native Windows Job Object proof.
|
|
const posixIt = it.skipIf(process.platform === "win32").concurrent;
|
|
|
|
const steps = {
|
|
target: ["resolve_target", "Resolve OpenClaw target ref"],
|
|
record: ["source_performance", "Record source performance revision"],
|
|
tested: ["kova", "Record tested revision"],
|
|
kova: ["kova", "Install OCM and Kova"],
|
|
baseline: ["source_performance", "Fetch previous source performance baseline"],
|
|
prepare: ["publish", "Prepare clawgrit report commit"],
|
|
publish: ["publish", "Publish to clawgrit reports"],
|
|
} as const;
|
|
|
|
function performanceRun(
|
|
signal: AbortSignal,
|
|
mode: PerformanceFixtureOptions["mode"],
|
|
options: Partial<Parameters<typeof runCiGitStep>[0]> = {},
|
|
) {
|
|
const [job, step] = steps[mode];
|
|
return runCiGitStep({
|
|
signal,
|
|
workflow: { file: ".github/workflows/openclaw-performance.yml", job, step },
|
|
fetchResults: [],
|
|
performance: { mode },
|
|
...options,
|
|
});
|
|
}
|
|
|
|
// The previous semantic tests used short-lived stubs or replayed Git by hand.
|
|
// These actual workflow bodies must drain real parent/child/grandchild writers
|
|
// before every command, output, consumer and exit, while a sentinel stays alive.
|
|
posixIt.for(Object.keys(steps) as PerformanceFixtureOptions["mode"][])(
|
|
"Performance %s drains Git trees before every continuation",
|
|
{ timeout: 55_000 },
|
|
async (mode, { signal }) => {
|
|
const report = await performanceRun(signal, mode);
|
|
expect(report.code, report.output).toBe(0);
|
|
expect(report.readyAttempts.length).toBeGreaterThan(0);
|
|
if (mode === "prepare") {
|
|
expect(report.githubOutput).toContain("ready=true\n");
|
|
}
|
|
if (mode === "publish") {
|
|
expect(report.pushes).toHaveLength(1);
|
|
expect(report.fetches).toHaveLength(0);
|
|
expect(report.githubSummary).toContain("### Clawgrit report published");
|
|
}
|
|
},
|
|
);
|
|
|
|
posixIt.for([23, 125])(
|
|
"baseline ordinary fetch %s is advisory after extinction",
|
|
{ timeout: 55_000 },
|
|
async (code, { signal }) => {
|
|
const report = await performanceRun(signal, "baseline", { fetchResults: [code, code, code] });
|
|
expect(report.code, report.output).toBe(0);
|
|
expect(report.githubSummary).toBe(
|
|
"No previous source performance baseline could be fetched.\n",
|
|
);
|
|
expect(report.githubEnv).toBe("");
|
|
expect(report.commands.at(-1)?.args[0]).toBe("fetch");
|
|
},
|
|
);
|
|
|
|
posixIt.for(["absent", "invalid", "trailing-newline"] as const)(
|
|
"baseline %s pointer preserves advisory result",
|
|
{ timeout: 55_000 },
|
|
async (baseline, { signal }) => {
|
|
const report = await performanceRun(signal, "baseline", {
|
|
performance: { mode: "baseline", baseline },
|
|
});
|
|
expect(report.code, report.output).toBe(0);
|
|
expect(report.githubSummary).toContain(
|
|
baseline === "absent"
|
|
? "No previous source performance baseline exists"
|
|
: "Previous source performance baseline pointer is invalid.",
|
|
);
|
|
expect(report.githubEnv).toBe("");
|
|
expect(report.checkouts).toHaveLength(0);
|
|
},
|
|
);
|
|
|
|
posixIt.for(["ls-tree", "show"])(
|
|
"baseline %s failure never becomes absence or invalid JSON",
|
|
{ timeout: 55_000 },
|
|
async (operation, { signal }) => {
|
|
const report = await performanceRun(signal, "baseline", {
|
|
gitFault: { match: `^${operation} `, code: 128 },
|
|
});
|
|
expect(report.code, report.output).toBe(128);
|
|
expect(report.githubSummary).toBe("");
|
|
expect(report.githubEnv).toBe("");
|
|
expect(report.commands.at(-1)?.args[0]).toBe(operation);
|
|
},
|
|
);
|
|
|
|
const terminalCases = [
|
|
...["rev-parse"].map((operation) => ({ mode: "target" as const, operation })),
|
|
{ mode: "record" as const, operation: "rev-parse" },
|
|
{ mode: "tested" as const, operation: "rev-parse" },
|
|
...["fetch", "checkout"].map((operation) => ({ mode: "kova" as const, operation })),
|
|
...["fetch", "ls-tree", "show", "sparse-checkout init", "sparse-checkout set", "checkout"].map(
|
|
(operation) => ({ mode: "baseline" as const, operation }),
|
|
),
|
|
...["fetch", "ls-tree", "diff"].map((operation) => ({ mode: "prepare" as const, operation })),
|
|
...[
|
|
"config",
|
|
"push",
|
|
"fetch",
|
|
"ls-tree",
|
|
"checkout",
|
|
"cherry-pick -X",
|
|
"cherry-pick --abort",
|
|
"rev-parse",
|
|
].map((operation) => ({ mode: "publish" as const, operation })),
|
|
];
|
|
// Every injected lifecycle failure must stop at its command, before later policy actions.
|
|
posixIt.for(
|
|
terminalCases.flatMap((entry) =>
|
|
(["cleanup-failure", "cancel"] as const).map((code) => ({
|
|
mode: entry.mode,
|
|
operation: entry.operation,
|
|
code,
|
|
})),
|
|
),
|
|
)(
|
|
"$mode $operation $code fences every later action",
|
|
{ timeout: 55_000 },
|
|
async ({ mode, operation, code }, { signal }) => {
|
|
const report = await performanceRun(signal, mode, {
|
|
gitFault: { match: `^${operation}(?: |$)`, code },
|
|
...(mode === "publish" && operation !== "config" && operation !== "push"
|
|
? { pushResults: [23] }
|
|
: {}),
|
|
...(operation === "cherry-pick --abort"
|
|
? { gitFaults: [{ match: "^cherry-pick -X ", code: 23 }] }
|
|
: {}),
|
|
});
|
|
expect(report.code, report.output).toBe(code === "cancel" ? 143 : 125);
|
|
expect(report.commands.at(-1)?.args.join(" ")).toMatch(new RegExp(`^${operation}(?: |$)`));
|
|
expect(report.githubSummary).toBe("");
|
|
expect(report.githubOutput).not.toMatch(/ready=true|already_published=|report_url=/u);
|
|
expect(report.githubEnv).not.toContain("SOURCE_PERF_BASELINE_DIR=");
|
|
expect(report.output).not.toMatch(
|
|
/Unable to replay|Unable to refresh|No previous|Published report:/u,
|
|
);
|
|
const failedPush = mode === "publish" && operation === "push";
|
|
if (failedPush) {
|
|
expect(report.fetches).toHaveLength(0);
|
|
expect(report.output).not.toContain("fixture backoff:");
|
|
}
|
|
},
|
|
);
|
|
|
|
posixIt.for(["hang", 23, 125] as const)(
|
|
"prepare initial fetch %s cannot reach checkout, commit or token readiness",
|
|
{ timeout: 55_000 },
|
|
async (failure, { signal }) => {
|
|
const report = await performanceRun(signal, "prepare", {
|
|
fetchResults: [failure, failure, failure],
|
|
});
|
|
expect(report.code, report.output).toBe(1);
|
|
expect(report.fetches).toHaveLength(3);
|
|
expect(report.fetches.every(({ args }) => args.includes("--depth=1"))).toBe(true);
|
|
expect(report.checkouts).toHaveLength(0);
|
|
expect(report.commands.at(-1)?.args[0]).toBe("fetch");
|
|
expect(report.githubOutput).toBe("ready=false\n");
|
|
expect(report.githubSummary).toBe("");
|
|
},
|
|
);
|
|
|
|
posixIt(
|
|
"initial duplicate is verified before token or push",
|
|
async ({ signal }) => {
|
|
const report = await performanceRun(signal, "prepare", {
|
|
performance: { mode: "prepare", duplicate: true },
|
|
});
|
|
expect(report.code, report.output).toBe(0);
|
|
expect(report.githubOutput).toContain("already_published=true\n");
|
|
expect(report.githubOutput).toContain("ready=true\n");
|
|
expect(report.githubOutput).not.toContain("report_commit=");
|
|
expect(report.githubSummary).toContain("### Clawgrit report already published");
|
|
expect(report.pushes).toHaveLength(0);
|
|
expect(report.commands.some(({ args }) => args[0] === "commit")).toBe(false);
|
|
expect(
|
|
report.commands.every(
|
|
({ envProbe, args }) =>
|
|
!JSON.parse(envProbe!).token && JSON.parse(envProbe!).auth === (args[0] === "fetch"),
|
|
),
|
|
).toBe(true);
|
|
},
|
|
55_000,
|
|
);
|
|
|
|
posixIt.for([128, 125])(
|
|
"prepare duplicate inspection %s is terminal",
|
|
{ timeout: 55_000 },
|
|
async (code, { signal }) => {
|
|
const report = await performanceRun(signal, "prepare", {
|
|
gitFault: { match: "^ls-tree ", code },
|
|
});
|
|
expect(report.code, report.output).toBe(code);
|
|
expect(report.githubOutput).toBe("ready=false\n");
|
|
expect(report.commands.at(-1)?.args[0]).toBe("ls-tree");
|
|
},
|
|
);
|
|
|
|
posixIt.for([0, 1, 125])(
|
|
"cached diff status %s commits only for ordinary 1",
|
|
{ timeout: 55_000 },
|
|
async (code, { signal }) => {
|
|
const report = await performanceRun(signal, "prepare", {
|
|
gitFault: { match: "^diff --cached --quiet$", code, output: "" },
|
|
});
|
|
expect(report.code, report.output).toBe(code > 1 ? code : 0);
|
|
expect(report.commands.filter(({ args }) => args[0] === "commit")).toHaveLength(
|
|
code === 1 ? 1 : 0,
|
|
);
|
|
expect(report.githubOutput.includes("ready=true\n")).toBe(code <= 1);
|
|
},
|
|
);
|
|
|
|
posixIt.for([125, "hang"] as const)(
|
|
"ambiguous push %s reconciles only after extinction",
|
|
{ timeout: 55_000 },
|
|
async (code, { signal }) => {
|
|
const report = await performanceRun(signal, "publish", { pushResults: [code] });
|
|
expect(report.code, report.output).toBe(0);
|
|
expect(report.pushes).toHaveLength(2);
|
|
expect(report.fetches).toHaveLength(1);
|
|
expect(report.checkouts).toHaveLength(1);
|
|
expect(report.output.match(/fixture backoff: \d+/gu)).toEqual(["fixture backoff: 2"]);
|
|
expect(report.githubSummary).toContain("### Clawgrit report published");
|
|
expect(report.performance?.config).not.toContain("AUTHORIZATION");
|
|
for (const command of report.commands) {
|
|
const scope = JSON.parse(command.envProbe!);
|
|
expect(scope.token).toBe(false);
|
|
const authenticated = [
|
|
"push",
|
|
"fetch",
|
|
"ls-tree",
|
|
"checkout",
|
|
"cherry-pick",
|
|
"rev-parse",
|
|
].includes(command.args[0]!);
|
|
expect(scope.auth).toBe(authenticated);
|
|
if (scope.auth) {
|
|
expect(scope).toMatchObject({ count: "3", prompt: "0" });
|
|
}
|
|
if (command.args[0] === "push") {
|
|
expect(command.configuration).toContain("core.hooksPath=/dev/null");
|
|
}
|
|
}
|
|
expect(JSON.stringify(report)).not.toContain("fixture-performance-token");
|
|
expect(JSON.stringify(report)).not.toContain(
|
|
Buffer.from("x-access-token:fixture-performance-token").toString("base64"),
|
|
);
|
|
},
|
|
);
|
|
|
|
posixIt.for([false, true])(
|
|
"five failed pushes always get five fetches (fetch fails=%s)",
|
|
{ timeout: 55_000 },
|
|
async (fetchFails, { signal }) => {
|
|
const report = await performanceRun(signal, "publish", {
|
|
pushResults: [23, 23, 23, 23, 23],
|
|
fetchResults: fetchFails ? [23, 23, 23, 23, 23] : [],
|
|
});
|
|
expect(report.code, report.output).toBe(1);
|
|
expect(report.pushes).toHaveLength(5);
|
|
expect(report.boundaries.filter(({ name }) => name === "backoff")).toHaveLength(5);
|
|
expect(report.fetches.map(({ args }) => args)).toEqual(
|
|
Array.from({ length: 5 }, () => ["fetch", "--depth=1", "origin", "main"]),
|
|
);
|
|
expect(report.output.match(/fixture backoff: \d+/gu)).toEqual(
|
|
[2, 4, 6, 8, 10].map((seconds) => `fixture backoff: ${seconds}`),
|
|
);
|
|
expect(report.checkouts).toHaveLength(fetchFails ? 0 : 4);
|
|
expect(report.commands.filter(({ args }) => args[0] === "cherry-pick")).toHaveLength(
|
|
fetchFails ? 0 : 4,
|
|
);
|
|
expect(report.output.match(/::warning::Unable to refresh/gu) ?? []).toHaveLength(
|
|
fetchFails ? 4 : 0,
|
|
);
|
|
expect(report.githubSummary).toContain("failed after 5 attempts.");
|
|
expect(report.githubSummary).not.toContain("Published report:");
|
|
},
|
|
);
|
|
|
|
posixIt.for([1, 5])(
|
|
"remote duplicate after ambiguous attempt %s succeeds without replay",
|
|
{ timeout: 55_000 },
|
|
async (attempt, { signal }) => {
|
|
const report = await performanceRun(signal, "publish", {
|
|
performance: { mode: "publish", remoteDuplicateAttempt: attempt },
|
|
pushResults: Array.from({ length: attempt }, () => 124),
|
|
});
|
|
expect(report.code, report.output).toBe(0);
|
|
expect(report.pushes).toHaveLength(attempt);
|
|
expect(report.fetches).toHaveLength(attempt);
|
|
expect(report.checkouts).toHaveLength(attempt - 1);
|
|
expect(report.githubSummary).toContain("### Clawgrit report published");
|
|
},
|
|
);
|
|
|
|
posixIt.for([23, 125])(
|
|
"ordinary cherry-pick and abort %s failures remain visible publish failure",
|
|
{ timeout: 55_000 },
|
|
async (code, { signal }) => {
|
|
const report = await performanceRun(signal, "publish", {
|
|
pushResults: [23],
|
|
gitFaults: [
|
|
{ match: "^cherry-pick -X ", code: 23 },
|
|
{ match: "^cherry-pick --abort$", code },
|
|
],
|
|
});
|
|
expect(report.code, report.output).toBe(1);
|
|
expect(report.commands.at(-1)?.args).toEqual(["cherry-pick", "--abort"]);
|
|
expect(report.pushes).toHaveLength(1);
|
|
expect(report.output).toContain(
|
|
"::error::Unable to replay the clawgrit report after a concurrent publish.",
|
|
);
|
|
expect(report.githubSummary).toBe("");
|
|
},
|
|
);
|
|
|
|
posixIt.for(["owner", "python", "git"] as const)(
|
|
"prepare setup failure %s cannot publish readiness",
|
|
{ timeout: 55_000 },
|
|
async (setupFailure, { signal }) => {
|
|
const report = await performanceRun(signal, "prepare", { setupFailure });
|
|
expect(report.code, report.output).not.toBe(0);
|
|
expect(report.pushes).toHaveLength(0);
|
|
expect(report.githubOutput).not.toContain("ready=true");
|
|
},
|
|
);
|
|
|
|
posixIt.for([125, "hang"] as const)(
|
|
"reconciliation fetch %s warns once and retries without replay",
|
|
{ timeout: 55_000 },
|
|
async (code, { signal }) => {
|
|
const report = await performanceRun(signal, "publish", {
|
|
pushResults: [23],
|
|
fetchResults: [code],
|
|
});
|
|
expect(report.code, report.output).toBe(0);
|
|
expect(report.pushes).toHaveLength(2);
|
|
expect(report.fetches).toHaveLength(1);
|
|
expect(report.checkouts).toHaveLength(0);
|
|
expect(report.output.match(/::warning::Unable to refresh/gu)).toHaveLength(1);
|
|
},
|
|
);
|
|
|
|
posixIt.for([125, 128])(
|
|
"remote duplicate read %s is terminal, never absence",
|
|
{ timeout: 55_000 },
|
|
async (code, { signal }) => {
|
|
const report = await performanceRun(signal, "publish", {
|
|
pushResults: [23],
|
|
gitFault: { match: "^ls-tree ", code },
|
|
});
|
|
expect(report.code, report.output).toBe(code);
|
|
expect(report.commands.at(-1)?.args[0]).toBe("ls-tree");
|
|
expect(report.githubSummary).toBe("");
|
|
expect(report.checkouts).toHaveLength(0);
|
|
},
|
|
);
|
|
|
|
posixIt.for(["prepare", "publish"] as const)(
|
|
"%s cancellation during real TERM-resistant cleanup prevents continuation",
|
|
{ timeout: 55_000 },
|
|
async (mode, { signal }) => {
|
|
const report = await performanceRun(signal, mode, {
|
|
cancelDuringCleanup: true,
|
|
cleanupCancelMatch: mode === "prepare" ? "^fetch " : "^push ",
|
|
...(mode === "prepare" ? { fetchResults: ["hang"] } : { pushResults: ["hang"] }),
|
|
});
|
|
expect(report.cancelledDuringCleanup).toBe(true);
|
|
expect(report.code, report.output).toBe(143);
|
|
expect(report.commands.at(-1)?.args[0]).toBe(mode === "prepare" ? "fetch" : "push");
|
|
expect(report.githubOutput).not.toContain("ready=true");
|
|
expect(report.githubSummary).toBe("");
|
|
expect(report.output).not.toContain("fixture backoff:");
|
|
},
|
|
);
|
|
|
|
posixIt(
|
|
"cancellation during owned backoff prevents reconciliation fetch",
|
|
async ({ signal }) => {
|
|
const report = await performanceRun(signal, "publish", {
|
|
pushResults: [23],
|
|
realClock: true,
|
|
cooperativeTrees: true,
|
|
cancelDuringBackoff: true,
|
|
});
|
|
expect(report.code, report.output).toBe(143);
|
|
expect(report.pushes).toHaveLength(1);
|
|
expect(report.fetches).toHaveLength(0);
|
|
expect(report.githubSummary).toBe("");
|
|
expect(report.boundaries.some(({ name }) => name === "backoff-cancel")).toBe(true);
|
|
},
|
|
55_000,
|
|
);
|