From d457b2c611311366d0c5358f1b05192fa44fad57 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 1 Oct 2026 13:42:36 -0700 Subject: [PATCH] fix(test): checkout fixture tests fail healthy runs on loaded macOS hosts (#162888) * fix(test): checkout fixture budgets cut healthy runs on loaded hosts On a loaded macOS host (load 55-83), ci-platform-checkout.test.ts and the other checkout fixture consumers failed through three fixture-internal wall-clock budgets, not through the workflow under test: - the supervisor's mock-resolution preflight ran bash under its own 2 s timeout ("mock command resolution failed: spawnSync bash ETIMEDOUT"); - the POSIX process census shadowed its caller's operation deadline with a fresh 1 s budget for all singleton ps queries ("Fixture process census failed (ETIMEDOUT)"); the owner-ancestry walk had the same 1 s shape; - the supervisor cut every run at 45 s ("fixture deadline exceeded"), nested under the helper's 50 s close race only because the helper could not see the Vitest test timeout. Measured breakdown: a multi-attempt scenario serially starts 25-36 Node actors plus the Python owner; at load ~72 Linux git-failure spent 19.4 s, of which the fixture's own 82 ps calls were 1.6 s. The work scales with scheduler latency and is the contract under test, so the fix is ownership of the time bound, not less work. The owning test's AbortSignal now bounds a supervised run. withCiCheckoutFixture takes it, rejects its close join on abort, and asks the live supervisor to cancel over IPC so its own stop() retires the detached shell group and actors, keeping the SIGKILL fallback for a wedged supervisor. The supervisor drops its 45 s watchdog; its operation deadline becomes the existing 60 s actor lifetime ceiling, which only bounds orphans. Census, preflight, and ancestry queries borrow that operation deadline, as the Windows census witness already did. Every runCiGitStep and withCiCheckoutFixture caller passes its test signal; .each registrations that need the context move to .for, which is the only form Vitest 5 hands the context to. Regressions: the shared slow-witness preload now gives POSIX supervisors a first native query that spends 1.1 s on their clock (fails main's fixture 7/7 with "census failed (unverified)"), and the outer-runner retention proof gains a cancel fault proving the aborted supervisor retires its shell. * fix(test): keep checkout signal bindings lint-clean --- test/scripts/ci-checkout.test-support.ts | 56 ++- test/scripts/ci-git-owner-settlement.test.ts | 3 +- test/scripts/ci-git-owner.test-support.ts | 2 + test/scripts/ci-git-owner.test.ts | 321 ++++++++++-------- test/scripts/ci-git-prerequisites.test.ts | 10 +- test/scripts/ci-linux-git.test.ts | 286 +++++++++------- test/scripts/ci-platform-checkout.test.ts | 85 +++-- .../ci-windows-process-census.test-support.ts | 30 +- test/scripts/ci-workflow-guards.test.ts | 7 +- .../scripts/fixtures/ci-platform-checkout.mjs | 24 +- ...openclaw-performance-git-lifecycle.test.ts | 145 ++++---- .../openclaw-performance-workflow.test.ts | 10 +- .../plugin-release-git-lifecycle.test.ts | 137 ++++---- .../release-workflow-git-lifecycle.test.ts | 107 +++--- 14 files changed, 691 insertions(+), 532 deletions(-) diff --git a/test/scripts/ci-checkout.test-support.ts b/test/scripts/ci-checkout.test-support.ts index 875209055096..554fc8fe786b 100644 --- a/test/scripts/ci-checkout.test-support.ts +++ b/test/scripts/ci-checkout.test-support.ts @@ -192,6 +192,7 @@ export function expectCiCheckoutCleanup(report: Report) { export async function withCiCheckoutFixture( scenario: string, + signal: AbortSignal, prepare: (root: string) => NodeJS.ProcessEnv | void, inspect: (report: Report, result: CloseResult, stderr: string, root: string) => T | Promise, ): Promise { @@ -215,39 +216,64 @@ export async function withCiCheckoutFixture( throw error; } let stderr = ""; + let closeResult: CloseResult | undefined; // An error can precede close, including failed spawn. Never reject this join. const closed = new Promise((resolve) => { - supervisor.once("close", (code, signal) => { - resolve({ code, signal }); + supervisor.once("close", (code, exitSignal) => { + closeResult = { code, signal: exitSignal }; + resolve(closeResult); }); }); supervisor.stderr?.on("data", (data) => (stderr += String(data))); supervisor.on("error", (error) => (stderr += `${error}\n`)); - let timer: NodeJS.Timeout | undefined; + const joinClose = async (deadline: number) => { + let timer: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + closed.then(() => true), + new Promise((resolve) => { + timer = setTimeout(() => resolve(false), Math.max(0, deadline - Date.now())); + }), + ]); + } finally { + clearTimeout(timer); + } + }; let report: Report | undefined; + let onAbort = () => {}; try { - const completed = await Promise.race([ - closed, - new Promise((_, reject) => { - timer = setTimeout( - () => reject(new Error("Checkout supervisor did not close within 50000ms")), - 50_000, - ); - }), - ]); - clearTimeout(timer); + // The owning test bounds the run; a slow host never races a fixture deadline. + // Vitest does not unwind a suspended body on timeout, so its abort must reject + // this join to reach cleanup. Same contract as withinTest, kept inline because + // the outer-runner proof loads this module in plain Node. + const completed = await new Promise((resolve, reject) => { + onAbort = () => { + const reason: unknown = signal.reason; + reject(reason instanceof Error ? reason : new Error("test aborted", { cause: reason })); + }; + signal.addEventListener("abort", onAbort, { once: true }); + if (signal.aborted) { + onAbort(); + } + void closed.then(resolve); + }); report = reportSchema.parse(JSON.parse(readFileSync(path.join(root, "report.json"), "utf8"))); return await inspect(report, completed, stderr, root); } finally { - clearTimeout(timer); + signal.removeEventListener("abort", onAbort); if (report) { // A consumer assertion failure does not revoke the producer's release receipt. rmSync(root, { recursive: true, force: true }); } else { - const deadline = Date.now() + 4_000; // Keep IPC attached through termination: explicit disconnect can suppress Node's close. // Let lease-bound Git descendants stop even if the supervisor cannot run cleanup. rmSync(path.join(root, "lease"), { force: true }); + if (!closeResult && supervisor.connected) { + // A cancelled run still has a live owner: let it retire its shell group and actors. + supervisor.send({ type: "ci-checkout:cancel" }, () => {}); + await joinClose(Date.now() + 4_000); + } + const deadline = Date.now() + 4_000; const termination = terminateManagedChild(supervisor, "SIGKILL", { taskkillTimeoutMs: 2_000, processGroupFallback: "never", diff --git a/test/scripts/ci-git-owner-settlement.test.ts b/test/scripts/ci-git-owner-settlement.test.ts index 7432d8e96023..ff21171e8748 100644 --- a/test/scripts/ci-git-owner-settlement.test.ts +++ b/test/scripts/ci-git-owner-settlement.test.ts @@ -11,9 +11,10 @@ import { it.skipIf(process.platform !== "win32")( "settles original member handles before the Git owner returns", - async () => { + async ({ signal }) => { await withCiCheckoutFixture( "harness-timeout", + signal, (root) => { const source = readFileSync(".github/actions/git-owner/owner.py", "utf8"); const probe = path.join(root, "settlement-probe.py"); diff --git a/test/scripts/ci-git-owner.test-support.ts b/test/scripts/ci-git-owner.test-support.ts index 576f3045f6f4..8f461759e3c0 100644 --- a/test/scripts/ci-git-owner.test-support.ts +++ b/test/scripts/ci-git-owner.test-support.ts @@ -114,6 +114,7 @@ function readWorkflowStep({ file, job, step: name }: WorkflowTarget): Step & { r } export async function runCiGitStep(options: { + signal: AbortSignal; workflow?: "workflow-sanity" | WorkflowTarget; job?: string; action?: @@ -241,6 +242,7 @@ export async function runCiGitStep(options: { let publisherFixture: ReturnType | undefined; return withCiCheckoutFixture( `linux:${options.scenario ?? "configured"}`, + options.signal, (root) => { const actions = path.join(root, "trusted-actions"); if (options.performance) { diff --git a/test/scripts/ci-git-owner.test.ts b/test/scripts/ci-git-owner.test.ts index 8815fbee7d36..e97c24958d41 100644 --- a/test/scripts/ci-git-owner.test.ts +++ b/test/scripts/ci-git-owner.test.ts @@ -512,13 +512,15 @@ exec "$REAL_GIT" "$@"`, } }); -releasePolicyIt.each([ +releasePolicyIt.for([ { label: "timeout", failure: "hang" }, { label: "Git failure", failure: 23 }, ] as const)( "retries a drained release ancestry fetch after $label", - async ({ failure }) => { + { timeout: 55_000 }, + async ({ failure }, { signal }) => { const report = await runCiGitStep({ + signal, policy: failure === "hang" ? fastReleaseAncestryPolicy : releaseAncestryPolicy, env: { RELEASE_ANCESTRY_MODE: "merge-base", @@ -539,13 +541,13 @@ releasePolicyIt.each([ expect(report.fetches).toHaveLength(2); expect(report.output).toContain("fetch failed on attempt 1; retrying"); }, - 55_000, ); releasePolicyIt( "preserves the final release ancestry Git failure after bounded retries", - async () => { + async ({ signal }) => { const report = await runCiGitStep({ + signal, policy: releaseAncestryPolicy, env: { RELEASE_ANCESTRY_MODE: "merge-base", @@ -561,18 +563,22 @@ releasePolicyIt( }, ); -releasePolicyIt("returns 124 when the release ancestry total budget is exhausted", async () => { - const report = await runCiGitStep({ - policy: expiredReleaseAncestryPolicy, - env: { - RELEASE_ANCESTRY_MODE: "merge-base", - RELEASE_ANCESTRY_TARGET_REF: "refs/heads/main", - }, - fetchResults: [], - }); - expect(report.code, report.output).toBe(124); - expect(report.commands).toEqual([]); -}); +releasePolicyIt( + "returns 124 when the release ancestry total budget is exhausted", + async ({ signal }) => { + const report = await runCiGitStep({ + signal, + policy: expiredReleaseAncestryPolicy, + env: { + RELEASE_ANCESTRY_MODE: "merge-base", + RELEASE_ANCESTRY_TARGET_REF: "refs/heads/main", + }, + fetchResults: [], + }); + expect(report.code, report.output).toBe(124); + expect(report.commands).toEqual([]); + }, +); it("materializes an executable preflight manifest from the workflow revision", async ({ command, @@ -729,15 +735,18 @@ it("binds read-only checkout authentication only to the workflow repository", () expect(ownedCheckouts).toBeGreaterThan(0); }); -it.each([false, true])("preserves linked Git metadata (reclaim locks=%s)", async (reclaimLocks) => { - const invocation = reclaimLocks - ? 'run_git(os.getcwd(), "fetch", "origin", "fixture", reclaim_locks=True)' - : 'print(git_output(os.getcwd(), "rev-parse", "HEAD"), end="")'; - const report = await runCiGitStep({ - fetchResults: [], - policy: - policyImport + - `from pathlib import Path +it.for([false, true])( + "preserves linked Git metadata (reclaim locks=%s)", + async (reclaimLocks, { signal }) => { + const invocation = reclaimLocks + ? 'run_git(os.getcwd(), "fetch", "origin", "fixture", reclaim_locks=True)' + : 'print(git_output(os.getcwd(), "rev-parse", "HEAD"), end="")'; + const report = await runCiGitStep({ + signal, + fetchResults: [], + policy: + policyImport + + `from pathlib import Path shared = Path.cwd().parent / "shared-git" shared.mkdir() lock = shared / "shallow.lock" @@ -750,18 +759,20 @@ finally: assert metadata.read_text() == "gitdir: ../shared-git\\n" assert lock.read_text() == "not invocation-owned\\n" `, - }); - expect(report.code, report.output).toBe(reclaimLocks ? 125 : 0); - expect(report.commands.map(({ args }) => args)).toEqual( - reclaimLocks ? [] : [["rev-parse", "HEAD"]], - ); - if (!reclaimLocks) { - expect(report.output).toBe(`${head}${EOL}`); - } -}); + }); + expect(report.code, report.output).toBe(reclaimLocks ? 125 : 0); + expect(report.commands.map(({ args }) => args)).toEqual( + reclaimLocks ? [] : [["rev-parse", "HEAD"]], + ); + if (!reclaimLocks) { + expect(report.output).toBe(`${head}${EOL}`); + } + }, +); -it("reclaims failed supplemental-fetch locks before the next attempt", async () => { +it("reclaims failed supplemental-fetch locks before the next attempt", async ({ signal }) => { const report = await runCiGitStep({ + signal, job: "checks-fast-core", step: "Prepare release-gate ratchet merge tree", fetchResults: ["hang", 0], @@ -774,8 +785,9 @@ it("reclaims failed supplemental-fetch locks before the next attempt", async () linuxIt( "bootstraps only action-owned bytes outside the candidate with isolated Python", - async () => { + async ({ signal }) => { const report = await runCiGitStep({ + signal, action: "git-owner", fetchResults: [], poisonPython: true, @@ -958,8 +970,9 @@ runpy.run_path(os.environ["BASE_REAL_POLICY_PATH"], run_name="__main__") linuxIt( "drains a timed-out exact fetch before deepening for the base", - async () => { + async ({ signal }) => { const report = await runCiGitStep({ + signal, action: "ensure-base-commit", baseAvailableAfter: 2, fetchResults: ["hang", 0], @@ -973,7 +986,7 @@ linuxIt( 55_000, ); -linuxIt.each([ +linuxIt.for([ { label: "empty", sha: "", code: 0, commands: 0 }, { label: "all-zero", sha: "00000", code: 0, commands: 0 }, { label: "invalid SHA", sha: "--help", code: 2, commands: 0 }, @@ -984,8 +997,9 @@ linuxIt.each([ { label: "already available", sha: base, code: 0, commands: 2 }, ])( "base policy preserves $label validation and skip behavior", - async ({ sha, code, commands, invalidRef }) => { + async ({ sha, code, commands, invalidRef }, { signal }) => { const report = await runCiGitStep({ + signal, action: "ensure-base-commit", env: { BASE_SHA: sha }, invalidRef, @@ -998,10 +1012,12 @@ linuxIt.each([ }, ); -linuxIt.each([1, 2, 3, 4, 5, 6, undefined])( +linuxIt.for([1, 2, 3, 4, 5, 6, undefined])( "base policy preserves exact/deepen/plain-ref order (available after %s)", - async (baseAvailableAfter) => { + { timeout: 55_000 }, + async (baseAvailableAfter, { signal }) => { const report = await runCiGitStep({ + signal, action: "ensure-base-commit", baseAvailableAfter, fetchResults: [0, 23, 0, 23, 0, 0], @@ -1037,13 +1053,14 @@ linuxIt.each([1, 2, 3, 4, 5, 6, undefined])( expect(report.output).toContain("::error title=ensure-base-commit missing base::"); } }, - 55_000, ); -linuxIt.each([125, 143, "hang"] as const)( +linuxIt.for([125, 143, "hang"] as const)( "base remains available after safely drained ordinary outcome %s", - async (failure) => { + { timeout: 55_000 }, + async (failure, { signal }) => { const report = await runCiGitStep({ + signal, action: "ensure-base-commit", baseAvailableAfter: 1, fetchResults: [failure], @@ -1053,10 +1070,9 @@ linuxIt.each([125, 143, "hang"] as const)( expect(report.output).toContain("exact fetch failed"); expect(report.output).toContain("Resolved base commit after exact fetch"); }, - 55_000, ); -linuxIt.each([ +linuxIt.for([ { label: "inspection failure", result: "cleanup-failure", code: 125 }, { label: "cancellation", result: "hang", scenario: "cancel-SIGTERM", code: 143 }, { @@ -1067,8 +1083,10 @@ linuxIt.each([ }, ] as const)( "base policy stops before availability/retry on $label", - async ({ result, code, ...entry }) => { + { timeout: 55_000 }, + async ({ result, code, ...entry }, { signal }) => { const report = await runCiGitStep({ + signal, action: "ensure-base-commit", baseAvailableAfter: 1, fetchResults: [result], @@ -1082,13 +1100,13 @@ linuxIt.each([ expect(report.output).not.toContain("Resolved base commit"); expect(report.cancelledDuringCleanup).toBe("cancelDuringCleanup" in entry); }, - 55_000, ); linuxIt( "keeps the base action's 30-second fetch deadline and drains before recovery", - async () => { + async ({ signal }) => { const report = await runCiGitStep({ + signal, action: "ensure-base-commit", baseAvailableAfter: 1, fetchResults: ["hang"], @@ -1108,8 +1126,9 @@ linuxIt( linuxIt( "fences later calls even if a trusted policy accidentally catches an ownership failure", - async () => { + async ({ signal }) => { const report = await runCiGitStep({ + signal, fetchResults: ["cleanup-failure"], policy: policyImport + @@ -1130,15 +1149,17 @@ except Exception: }, ); -linuxIt.each( +linuxIt.for( [false, true].flatMap((inlinePolicy) => ([125, "cleanup-failure"] as const).map((failure) => ({ inlinePolicy, failure })), ), )( "preserves generic output and typed recovery (stdin=$inlinePolicy, outcome=$failure)", - async ({ inlinePolicy, failure }) => { + { timeout: 55_000 }, + async ({ inlinePolicy, failure }, { signal }) => { const output = " \tpath\0another path\r\n\n\n"; const report = await runCiGitStep({ + signal, fetchResults: [failure], inlinePolicy, revisions: { HEAD: output.slice(0, -1) }, @@ -1171,14 +1192,15 @@ sys.stdout.write(git_output(os.getcwd(), "rev-parse", "HEAD", env={"CI_OWNER_PRO ]); } }, - 55_000, ); -linuxIt.each([0, 23, "cleanup-failure"] as const)( +linuxIt.for([0, 23, "cleanup-failure"] as const)( "generic Git output drains its writers before consumption (%s)", - async (code) => { + { timeout: 55_000 }, + async (code, { signal }) => { const output = `${head}\trefs/heads/main\n`; const report = await runCiGitStep({ + signal, policy: policyImport + 'import sys\nsys.stdout.write(git_output(os.getcwd(), "ls-remote", "origin", "refs/heads/main"))\n', @@ -1196,7 +1218,6 @@ linuxIt.each([0, 23, "cleanup-failure"] as const)( expect(report.output).not.toContain(output); } }, - 55_000, ); const posixIt = it.skipIf(process.platform === "win32").concurrent; @@ -1219,8 +1240,12 @@ function requireAuditObject(ref: string, file: string) { } return object; } -const sanity = (options: Omit[0], "workflow">) => +const sanity = ( + signal: AbortSignal, + options: Omit[0], "workflow" | "signal">, +) => runCiGitStep({ + signal, ...options, workflow: "workflow-sanity", objects: { ...auditObjects, ...options.objects }, @@ -1283,10 +1308,11 @@ const sanityFetchCases: SanityFetchCase[] = [ }, ]; -posixIt.each(sanityFetchCases)( +posixIt.for(sanityFetchCases)( "workflow sanity preserves fetch policy: $label", - async ({ fetchResults, baseAvailableAfter, refs, warnings, code }) => { - const report = await sanity({ fetchResults, baseAvailableAfter }); + { timeout: 55_000 }, + async ({ fetchResults, baseAvailableAfter, refs, warnings, code }, { signal }) => { + const report = await sanity(signal, { fetchResults, baseAvailableAfter }); expect(report.code, report.output).toBe(code); expect(report.fetches.map(({ args }) => args)).toEqual( refs.map((ref) => [ @@ -1324,17 +1350,17 @@ posixIt.each(sanityFetchCases)( expect(report.trustedZizmor).toBe(""); } }, - 55_000, ); -posixIt.each([ +posixIt.for([ { label: "30-second fetch deadline", fetchResults: ["hang", 0], warnings: 1 }, { label: "five-second backoff", fetchResults: [137, 0], warnings: 1 }, ] as const)( "workflow sanity retains $label", - async ({ fetchResults, warnings }) => { + { timeout: 55_000 }, + async ({ fetchResults, warnings }, { signal }) => { const readyFetchClockAdvanceSeconds = fetchResults[0] === "hang" ? 30 : undefined; - const report = await sanity({ + const report = await sanity(signal, { fetchResults: [...fetchResults], realClock: true, virtualBackoff: true, @@ -1356,10 +1382,9 @@ posixIt.each([ (report.backoffClockAdvancedSeconds + (report.fetchClockAdvancedSeconds ?? 0)) * 1000; expect(elapsed).toBeGreaterThanOrEqual(fetchResults[0] === "hang" ? 35_000 : 5_000); }, - 55_000, ); -posixIt.each([ +posixIt.for([ { label: "owner inspection failure", fetchResults: ["cleanup-failure"], code: 125 }, { label: "fetch cancellation", fetchResults: ["hang"], scenario: "cancel-SIGTERM", code: 143 }, { @@ -1390,8 +1415,9 @@ posixIt.each([ fetchResults: FetchResult[]; })[])( "workflow sanity never recovers or publishes after $label", - async ({ label: _label, code, ...options }) => { - const report = await sanity(options); + { timeout: 55_000 }, + async ({ label: _label, code, ...options }, { signal }) => { + const report = await sanity(signal, options); if (code === "launcher") { // Bash versions differ for a found executable whose interpreter is missing. expect([126, 127], report.output).toContain(report.code); @@ -1408,13 +1434,13 @@ posixIt.each([ Boolean(options.cancelDuringBackoff), ); }, - 55_000, ); -posixIt.each([[0], [1]].map((missing) => ({ missing })))( +posixIt.for([[0], [1]].map((missing) => ({ missing })))( "workflow sanity selects missing exact configs independently ($missing)", - async ({ missing }) => { - const report = await sanity({ + { timeout: 55_000 }, + async ({ missing }, { signal }) => { + const report = await sanity(signal, { fetchResults: [], baseAvailableAfter: 0, objects: Object.fromEntries( @@ -1449,18 +1475,18 @@ posixIt.each([[0], [1]].map((missing) => ({ missing })))( `PRE_COMMIT_CONFIG_PATH=${report.runnerTemp}/pre-commit-base.yaml\n`, ); }, - 55_000, ); -posixIt.each( +posixIt.for( auditFiles.flatMap((file) => [ { file, fallback: false }, { file, fallback: true }, ]), )( "workflow sanity rejects partial $file show (fallback=$fallback)", - async ({ file, fallback }) => { - const report = await sanity({ + { timeout: 55_000 }, + async ({ file, fallback }, { signal }) => { + const report = await sanity(signal, { fetchResults: [], baseAvailableAfter: 0, objects: { @@ -1481,11 +1507,10 @@ posixIt.each( expect(report.output).toContain(`Could not read ${file} from ${base} or origin/main.`); } }, - 55_000, ); -posixIt("workflow sanity rejects a config without the Zizmor reference", async () => { - const report = await sanity({ +posixIt("workflow sanity rejects a config without the Zizmor reference", async ({ signal }) => { + const report = await sanity(signal, { fetchResults: [], baseAvailableAfter: 0, objects: { [`${base}:${auditFiles[0]}`]: { text: "repos: []\n" } }, @@ -1512,8 +1537,9 @@ const maturityEnvironment = { posixIt( "generated publisher drains real Git descendants before every continuation", - async () => { + async ({ signal }) => { const report = await runCiGitStep({ + signal, action: "publish-generated-pr", step: "Publish generated pull request", fetchResults: [], @@ -1532,8 +1558,12 @@ posixIt( 55_000, ); -function publisherRun(options: Partial[0]> = {}) { +function publisherRun( + signal: AbortSignal, + options: Partial[0]> = {}, +) { return runCiGitStep({ + signal, action: "publish-generated-pr", step: "Publish generated pull request", fetchResults: [], @@ -1541,8 +1571,12 @@ function publisherRun(options: Partial[0]> = {}) ...options, }); } -function maturityRun(options: Partial[0]> = {}) { +function maturityRun( + signal: AbortSignal, + options: Partial[0]> = {}, +) { return runCiGitStep({ + signal, workflow: maturityValidation, env: maturityEnvironment, fetchResults: [], @@ -1553,14 +1587,15 @@ function maturityRun(options: Partial[0]> = {}) // Actual-body fault injection covers the former conditional-errexit hole and // lifecycle/status collisions; the existing real-repository cases own tree semantics. -posixIt.each( +posixIt.for( ["fetch", "ls-remote", "push", "ls-tree"].flatMap((operation) => (["cleanup-failure", "cancel"] as const).map((code) => ({ operation, code })), ), )( "generated publisher $code at $operation is terminal before any continuation", - async ({ operation, code }) => { - const report = await publisherRun({ gitFault: { match: `^${operation} `, code } }); + { timeout: 55_000 }, + async ({ operation, code }, { signal }) => { + const report = await publisherRun(signal, { gitFault: { match: `^${operation} `, code } }); expect(report.code, report.output).toBe(code === "cancel" ? 143 : 125); expect(report.commands.at(-1)?.args[0]).toBe(operation); expect(report.githubSummary).toBe(""); @@ -1570,13 +1605,13 @@ posixIt.each( /refusing a doomed retry|moved concurrently|merged|Deferred|Generated pull request:/u, ); }, - 55_000, ); -posixIt.each([124, 125, 143])( +posixIt.for([124, 125, 143])( "generated publisher ordinary push %s drains before semantic failure reporting", - async (code) => { - const report = await publisherRun({ + { timeout: 55_000 }, + async (code, { signal }) => { + const report = await publisherRun(signal, { gitFault: { match: "^push ", code, output: "GH013 repository rule violations\n" }, }); expect(report.code, report.output).toBe(code === 124 ? 0 : code); @@ -1596,13 +1631,15 @@ posixIt.each([124, 125, 143])( expect(report.githubSummary).toBe(""); } }, - 55_000, ); -posixIt.each(["fetch", "ls-remote", "push"])( +posixIt.for(["fetch", "ls-remote", "push"])( "generated publisher %s timeout has bounded recovery", - async (operation) => { - const report = await publisherRun({ gitFault: { match: `^${operation} `, code: "hang" } }); + { timeout: 55_000 }, + async (operation, { signal }) => { + const report = await publisherRun(signal, { + gitFault: { match: `^${operation} `, code: "hang" }, + }); expect(report.code, report.output).toBe(operation === "push" ? 0 : 124); expect(report.fetches).toHaveLength(1); expect(report.pushes).toHaveLength(operation === "push" ? 2 : 0); @@ -1613,10 +1650,9 @@ posixIt.each(["fetch", "ls-remote", "push"])( ); expect(report.authHeaderPresent).toBe(false); }, - 55_000, ); -posixIt.each([ +posixIt.for([ { label: "overlap candidate diff", match: "^diff --name-only", occurrence: 2 }, { label: "overlap tree read", match: "^ls-tree ", occurrence: 1 }, { label: "invalidation diff", match: "^diff --quiet ", occurrence: 1 }, @@ -1632,8 +1668,9 @@ posixIt.each([ }, ])( "generated publisher ordinary failure inside $label never becomes success", - async ({ match, occurrence, merged, noChange, overlap }) => { - const report = await publisherRun({ + { timeout: 55_000 }, + async ({ match, occurrence, merged, noChange, overlap }, { signal }) => { + const report = await publisherRun(signal, { publisher: { mergeGeneratedPush: merged, noGeneratedChange: noChange, @@ -1648,13 +1685,13 @@ posixIt.each([ /Generated output was merged|Deferred stale|Neutralized stale/u, ); }, - 55_000, ); -posixIt.each([0, 2, 23, 125, 143, "hang", "cleanup-failure", "cancel"] as const)( +posixIt.for([0, 2, 23, 125, 143, "hang", "cleanup-failure", "cancel"] as const)( "maturity branch lookup %s preserves 0/2/ordinary/fatal policy after drain", - async (code) => { - const report = await maturityRun({ + { timeout: 55_000 }, + async (code, { signal }) => { + const report = await maturityRun(signal, { env: { ...maturityEnvironment, INPUT_REF: "release/2026.8.1" }, gitFault: { match: "^ls-remote ", code }, }); @@ -1686,13 +1723,12 @@ posixIt.each([0, 2, 23, 125, 143, "hang", "cleanup-failure", "cancel"] as const) } } }, - 55_000, ); posixIt( "generated publisher retries one timed-out push under the unchanged lease", - async () => { - const report = await publisherRun({ + async ({ signal }) => { + const report = await publisherRun(signal, { gitFault: { match: "^push ", occurrence: 1, code: "hang" }, }); expect(report.code, report.output).toBe(0); @@ -1705,7 +1741,7 @@ posixIt( 55_000, ); -posixIt.each( +posixIt.for( [ { match: "^fetch ", occurrence: 1 }, { match: "^fetch ", occurrence: 2 }, @@ -1717,8 +1753,9 @@ posixIt.each( ), )( "maturity $code at $match/$occurrence stops before fallback/output", - async ({ match, occurrence, code }) => { - const report = await maturityRun({ + { timeout: 55_000 }, + async ({ match, occurrence, code }, { signal }) => { + const report = await maturityRun(signal, { env: { ...maturityEnvironment, EXPECTED_SHA: "" }, gitFault: { match, occurrence, code }, }); @@ -1727,18 +1764,18 @@ posixIt.each( expect(report.githubOutput).toBe(""); expect(report.githubSummary).toBe(""); }, - 55_000, ); -posixIt.each([ +posixIt.for([ { race: "delete", secondFailure: false, code: 0, pushes: 2, fetches: 2 }, { race: "advance", secondFailure: false, code: 1, pushes: 1, fetches: 1 }, { race: "recreate", secondFailure: false, code: 1, pushes: 2, fetches: 2 }, { race: "delete", secondFailure: true, code: 1, pushes: 2, fetches: 2 }, ] as const)( "generated publisher exact deletion-race lease policy ($race, second failure=$secondFailure)", - async ({ race, secondFailure, code, pushes, fetches }) => { - const report = await publisherRun({ + { timeout: 55_000 }, + async ({ race, secondFailure, code, pushes, fetches }, { signal }) => { + const report = await publisherRun(signal, { publisher: { existingPr: true, race, failGeneratedPush: secondFailure }, }); expect(report.code, report.output).toBe(code); @@ -1776,10 +1813,9 @@ posixIt.each([ ).toEqual([]); } }, - 55_000, ); -posixIt.each( +posixIt.for( [ { match: "^fetch ", occurrence: 2 }, { match: "^ls-tree ", occurrence: 5 }, @@ -1788,8 +1824,9 @@ posixIt.each( ), )( "generated publisher verify_publication $code at $match is terminal", - async ({ match, occurrence, code }) => { - const report = await publisherRun({ + { timeout: 55_000 }, + async ({ match, occurrence, code }, { signal }) => { + const report = await publisherRun(signal, { publisher: { reconciliation: "missing" }, gitFault: { match, occurrence, code }, }); @@ -1801,13 +1838,13 @@ posixIt.each( expect(report.commands.at(code === 23 ? -2 : -1)?.args.join(" ")).toMatch(new RegExp(match)); expect(report.output).not.toContain("Generated output was merged"); }, - 55_000, ); -posixIt.each([0, 5, 125, "cleanup-failure", "cancel"] as const)( +posixIt.for([0, 5, 125, "cleanup-failure", "cancel"] as const)( "generated publisher auth cleanup keeps ordinary tolerance but fences fatal %s", - async (code) => { - const report = await publisherRun({ + { timeout: 55_000 }, + async (code, { signal }) => { + const report = await publisherRun(signal, { gitFault: { match: "^config --local --unset-all ", code }, }); expect(report.code, report.output).toBe( @@ -1825,13 +1862,12 @@ posixIt.each([0, 5, 125, "cleanup-failure", "cancel"] as const)( expect(text).not.toContain("test-token"); } }, - 55_000, ); posixIt( "generated publisher removes Git auth after an unexpected policy exception", - async () => { - const report = await publisherRun({ + async ({ signal }) => { + const report = await publisherRun(signal, { publisher: { autoMerge: true, malformedAutoMergeRecord: true }, }); expect(report.code, report.output).toBe(125); @@ -1847,16 +1883,17 @@ posixIt( 55_000, ); -posixIt.each(["main-ancestor", "release-tag", "release-branch-head", "floating-main"])( +posixIt.for(["main-ancestor", "release-tag", "release-branch-head", "floating-main"])( "maturity preserves exact trust order, output hash bytes and fetches: %s", - async (reason) => { + { timeout: 55_000 }, + async (reason, { signal }) => { const release = "release/2026.8.1"; const floating = reason === "floating-main"; const tag = reason === "release-tag"; const releaseBranch = reason === "release-branch-head"; const revision = floating ? "d".repeat(40) : head; const publicationBase = releaseBranch ? release : "main"; - const report = await maturityRun({ + const report = await maturityRun(signal, { realClock: true, realDrain: false, env: { @@ -1922,29 +1959,30 @@ posixIt.each(["main-ancestor", "release-tag", "release-branch-head", "floating-m ], ); }, - 55_000, ); -posixIt.each( +posixIt.for( ["publisher", "maturity"].flatMap((surface) => (["owner", "python", "git"] as const).map((setupFailure) => ({ surface, setupFailure })), ), )( "$surface setup failure ($setupFailure) never reaches Git, GH, or outputs", - async ({ surface, setupFailure }) => { - const report = await (surface === "publisher" ? publisherRun : maturityRun)({ setupFailure }); + { timeout: 55_000 }, + async ({ surface, setupFailure }, { signal }) => { + const report = await (surface === "publisher" ? publisherRun : maturityRun)(signal, { + setupFailure, + }); expect(report.code).not.toBe(0); expect(report.commands).toEqual([]); expect(report.githubOutput).toBe(""); expect(report.githubSummary).toBe(""); }, - 55_000, ); posixIt( "generated publisher reconciliation accepts a tree merged after PR mutation", - async () => { - const report = await publisherRun({ publisher: { reconciliation: "merged" } }); + async ({ signal }) => { + const report = await publisherRun(signal, { publisher: { reconciliation: "merged" } }); expect(report.code, report.output).toBe(0); expect(report.fetches).toHaveLength(2); expect(report.pushes).toHaveLength(1); @@ -1956,10 +1994,11 @@ posixIt( 55_000, ); -posixIt.each([125, 143])( +posixIt.for([125, 143])( "generated publisher ordinary stale-lease %s permits the exact deletion rebuild", - async (code) => { - const report = await publisherRun({ + { timeout: 55_000 }, + async (code, { signal }) => { + const report = await publisherRun(signal, { publisher: { existingPr: true, race: "delete" }, gitFault: { match: "^push ", code, output: "stale info\n" }, }); @@ -1972,10 +2011,9 @@ posixIt.each([125, 143])( expect(report.publication?.generatedA).toBe("desired-a"); expect(report.authHeaderPresent).toBe(false); }, - 55_000, ); -posixIt.each([ +posixIt.for([ { label: "invalid expected SHA", env: { EXPECTED_SHA: "bad" }, @@ -2015,8 +2053,12 @@ posixIt.each([ }, ])( "maturity rejects $label without outputs", - async ({ env, fault, fetches, diagnostic, code }) => { - const report = await maturityRun({ env: { ...maturityEnvironment, ...env }, gitFault: fault }); + { timeout: 55_000 }, + async ({ env, fault, fetches, diagnostic, code }, { signal }) => { + const report = await maturityRun(signal, { + env: { ...maturityEnvironment, ...env }, + gitFault: fault, + }); expect(report.code, report.output).toBe(code ?? 1); expect(report.fetches).toHaveLength(fetches); expect(report.githubOutput).toBe(""); @@ -2025,5 +2067,4 @@ posixIt.each([ expect(report.output).toContain(diagnostic); } }, - 55_000, ); diff --git a/test/scripts/ci-git-prerequisites.test.ts b/test/scripts/ci-git-prerequisites.test.ts index 51a023947d42..6a65cc166a08 100644 --- a/test/scripts/ci-git-prerequisites.test.ts +++ b/test/scripts/ci-git-prerequisites.test.ts @@ -45,8 +45,11 @@ it.each([false, true])( }, ); -it("fetches selected history with the initial checkout before the test worker runs", async () => { +it("fetches selected history with the initial checkout before the test worker runs", async ({ + signal, +}) => { const report = await runCiGitStep({ + signal, job: "checks-node-core-test-nondist-shard", env: { CHECKOUT_GIT_COMMITS_JSON: JSON.stringify([reader.commit]) }, fetchResults: [0, 0], @@ -56,10 +59,11 @@ it("fetches selected history with the initial checkout before the test worker ru expect(report.fetches).toHaveLength(2); }); -it.each(["{}", '"main"', '["--upload-pack=bad"]', '["abc"]', "[null]"])( +it.for(["{}", '"main"', '["--upload-pack=bad"]', '["abc"]', "[null]"])( "rejects malformed immutable history before checkout mutation: %s", - async (input) => { + async (input, { signal }) => { const report = await runCiGitStep({ + signal, job: "checks-node-core-test-nondist-shard", env: { CHECKOUT_GIT_COMMITS_JSON: input }, fetchResults: [], diff --git a/test/scripts/ci-linux-git.test.ts b/test/scripts/ci-linux-git.test.ts index 0fd9ca549db2..41481be2597c 100644 --- a/test/scripts/ci-linux-git.test.ts +++ b/test/scripts/ci-linux-git.test.ts @@ -38,10 +38,14 @@ const resetCases: { label: string; fetchResults: FetchResult[]; code: number; at { label: "timeouts exhausted", fetchResults: Array(5).fill("hang"), code: 1, attempts: 5 }, { label: "unverified cleanup", fetchResults: ["cleanup-failure"], code: 125, attempts: 1 }, ]; -linuxIt.each(resetProfiles.flatMap((profile) => resetCases.map((entry) => ({ profile, entry }))))( +linuxIt.for(resetProfiles.flatMap((profile) => resetCases.map((entry) => ({ profile, entry }))))( "$profile.job drains descendants before reset/reuse ($entry.label)", - async ({ profile: { job, step, target, remote }, entry: { fetchResults, code, attempts } }) => { - const report = await runCiGitStep({ job, step, fetchResults }); + { timeout: 55_000 }, + async ( + { profile: { job, step, target, remote }, entry: { fetchResults, code, attempts } }, + { signal }, + ) => { + const report = await runCiGitStep({ signal, job, step, fetchResults }); expect(report.code).toBe(code); expect(report.readyAttempts).toHaveLength(attempts); expect(report.fetches).toHaveLength(attempts); @@ -61,17 +65,17 @@ linuxIt.each(resetProfiles.flatMap((profile) => resetCases.map((entry) => ({ pro .every(({ args }) => args.at(-1) === `https://github.com/${remote}.git`), ).toBe(true); }, - 55_000, ); -linuxIt.each([ +linuxIt.for([ { label: "timeout recovery", fetchResults: ["hang", 0], code: 0, attempts: 2 }, { label: "timeouts exhausted", fetchResults: ["hang", "hang", "hang"], code: 124, attempts: 3 }, { label: "ordinary Git failure", fetchResults: [23], code: 23, attempts: 1 }, ] satisfies { label: string; fetchResults: FetchResult[]; code: number; attempts: number }[])( "skills preserves exact-SHA retries without a fallback ($label)", - async ({ fetchResults, code, attempts }) => { - const report = await runCiGitStep({ job: "skills-python", fetchResults }); + { timeout: 55_000 }, + async ({ fetchResults, code, attempts }, { signal }) => { + const report = await runCiGitStep({ signal, job: "skills-python", fetchResults }); expect(report.code).toBe(code); expect(report.fetches).toHaveLength(attempts); expect( @@ -83,16 +87,16 @@ linuxIt.each([ expect(report.checkouts).toHaveLength(code === 0 ? 1 : 0); expect(report.boundaries.some(({ name }) => name === "delete")).toBe(false); }, - 55_000, ); -linuxIt.each([ +linuxIt.for([ { phase: "fetch", fetchResults: [23, 0], checkoutResults: [], firstCheckout: false }, { phase: "checkout", fetchResults: [0, 0], checkoutResults: [23, 0], firstCheckout: true }, ])( "Android resets only after safely joined $phase failure", - async ({ fetchResults, checkoutResults, firstCheckout }) => { - const report = await runCiGitStep({ job: "android", fetchResults, checkoutResults }); + { timeout: 55_000 }, + async ({ fetchResults, checkoutResults, firstCheckout }, { signal }) => { + const report = await runCiGitStep({ signal, job: "android", fetchResults, checkoutResults }); expect(report.code).toBe(0); expect(report.readyAttempts).toEqual([1, 2]); expect(report.fetches.map(({ args }) => args.at(-1))).toEqual([ @@ -112,14 +116,13 @@ linuxIt.each([ "checkout", ]); }, - 55_000, ); const manualProfiles = [ { job: "preflight", step: "Checkout", depth: 1 }, { job: "security-fast", step: "Checkout manual target", depth: 2 }, ]; -linuxIt.each( +linuxIt.for( manualProfiles.flatMap((profile) => [ { ...profile, label: "missing branch", fetchResults: [128, 0] as FetchResult[], code: 0 }, { @@ -137,8 +140,10 @@ linuxIt.each( ]), )( "$job only falls back after a safely joined unavailable target ($label)", - async ({ job, step, depth, fetchResults, code }) => { + { timeout: 55_000 }, + async ({ job, step, depth, fetchResults, code }, { signal }) => { const report = await runCiGitStep({ + signal, job, step, fetchResults, @@ -162,13 +167,13 @@ linuxIt.each( ); expect(report.checkouts).toHaveLength(code === 0 ? 1 : 0); }, - 55_000, ); linuxIt( "preflight pins a moved exact SHA and retries only its parent metadata", - async () => { + async ({ signal }) => { const report = await runCiGitStep({ + signal, job: "preflight", fetchResults: [0, 0, 23, 0], env: { GITHUB_EVENT_NAME: "workflow_dispatch" }, @@ -193,8 +198,9 @@ linuxIt( linuxIt( "manual security never refetches an unavailable equal fallback", - async () => { + async ({ signal }) => { const report = await runCiGitStep({ + signal, job: "security-fast", step: "Checkout manual target", env: { GITHUB_EVENT_NAME: "workflow_dispatch" }, @@ -211,8 +217,9 @@ linuxIt( linuxIt( "preflight rejects a fallback that cannot satisfy the requested exact SHA", - async () => { + async ({ signal }) => { const report = await runCiGitStep({ + signal, job: "preflight", env: { GITHUB_EVENT_NAME: "workflow_dispatch", CHECKOUT_REF: moved }, fetchResults: [128, 0], @@ -252,15 +259,15 @@ const preflightCases: { code: 1, }, ]; -linuxIt.each(preflightCases)( +linuxIt.for(preflightCases)( "preflight fails closed: $label", - async ({ env, fetchResults, code }) => { - const report = await runCiGitStep({ job: "preflight", env, fetchResults }); + { timeout: 55_000 }, + async ({ env, fetchResults, code }, { signal }) => { + const report = await runCiGitStep({ signal, job: "preflight", env, fetchResults }); expect(report.code).toBe(code); expect(report.fetches).toHaveLength(fetchResults.length); expect(report.checkouts).toEqual([]); }, - 55_000, ); const historyProfiles: { @@ -283,7 +290,7 @@ const historyProfiles: { }, ]; -linuxIt.each( +linuxIt.for( historyProfiles.flatMap((profile) => [ { ...profile, label: "successful leader exit", fetchResults: [0] as FetchResult[], code: 0 }, { @@ -295,8 +302,10 @@ linuxIt.each( ]), )( "$job/$step joins supplemental history before consumption ($label, $target)", - async ({ job, step, env, target, fetchResults, code }) => { + { timeout: 55_000 }, + async ({ job, step, env, target, fetchResults, code }, { signal }) => { const report = await runCiGitStep({ + signal, job, step, env, @@ -315,13 +324,13 @@ linuxIt.each( expect(report.checkouts.map(({ args }) => args.at(-1))).toEqual(code === 0 ? [merge] : []); } }, - 55_000, ); linuxIt( "ratchet retries a stale merge parent before checkout and base publication", - async () => { + async ({ signal }) => { const report = await runCiGitStep({ + signal, job: "checks-fast-core", step: "Prepare release-gate ratchet merge tree", fetchResults: [0, 0], @@ -351,8 +360,8 @@ linuxIt( posixIt( "fetches the CI harness without a second full-repository snapshot", - async () => { - const report = await runCiGitStep({ job: "checks-fast-core", fetchResults: [0, 0] }); + async ({ signal }) => { + const report = await runCiGitStep({ signal, job: "checks-fast-core", fetchResults: [0, 0] }); expect(report.code).toBe(0); const harnessDirectory = path.join(report.workspace, ".ci-harness"); const harnessCommands = report.commands.filter( @@ -474,8 +483,9 @@ const qaGitCases: QaGitCase[] = [ }, ]; -function runQaGitCase(profile: QaGitCase, fetchResults: FetchResult[]) { +function runQaGitCase(signal: AbortSignal, profile: QaGitCase, fetchResults: FetchResult[]) { return runCiGitStep({ + signal, workflow: { file: ".github/workflows/qa-profile-evidence.yml", job: profile.job, @@ -506,10 +516,12 @@ function runQaGitCase(profile: QaGitCase, fetchResults: FetchResult[]) { }); } -posixIt.each(qaGitCases)( +posixIt.for(qaGitCases)( "QA Git owner drains descendants before the next boundary: $label", - async (profile) => { + { timeout: 55_000 }, + async (profile, { signal }) => { const report = await runQaGitCase( + signal, profile, profile.fetches.map(() => 0), ); @@ -553,13 +565,13 @@ posixIt.each(qaGitCases)( expect(report.githubEnv).toBe(""); expect(report.githubPath).toBe(""); }, - 55_000, ); -posixIt.each(qaGitCases.filter(({ label, reason }) => !reason || label === "main validation"))( +posixIt.for(qaGitCases.filter(({ label, reason }) => !reason || label === "main validation"))( "QA Git owner stops without downstream work after cleanup failure: $label", - async (profile) => { - const report = await runQaGitCase(profile, ["cleanup-failure"]); + { timeout: 55_000 }, + async (profile, { signal }) => { + const report = await runQaGitCase(signal, profile, ["cleanup-failure"]); expect(report.code, report.output).toBe(125); expect(report.readyAttempts).toEqual([1]); expect(report.fetches.map(({ args }) => args)).toEqual([profile.fetches[0]]); @@ -574,7 +586,6 @@ posixIt.each(qaGitCases.filter(({ label, reason }) => !reason || label === "main expect(report.githubPath).toBe(""); expect(report.output).toContain("Git ownership/setup failed"); }, - 55_000, ); const mantisReleaseRef = "release/2026.8.1"; @@ -603,7 +614,7 @@ const mantisCases = [ mismatch?: boolean; }[]; -posixIt.each([ +posixIt.for([ ...mantisCases.map((entry) => Object.assign({}, entry, { failure: 0 as FetchResult })), ...[true, false].flatMap((shared) => (["cleanup-failure", 23] satisfies FetchResult[]).map((failure) => ({ @@ -614,13 +625,15 @@ posixIt.each([ ), ])( "Mantis ref Git owner drains before trust probes and publication: $label", - async (profile) => { + { timeout: 55_000 }, + async (profile, { signal }) => { const { shared, failure } = profile; const baseline = "baseline" in profile && profile.baseline; const release = "release" in profile && profile.release; const mismatch = "mismatch" in profile && profile.mismatch; const fetches = release ? [qaMainFetch, mantisReleaseFetch] : [qaMainFetch]; const report = await runCiGitStep({ + signal, ...(shared ? ({ action: "mantis-validate-trusted-ref", step: "Validate refs are trusted" } as const) : { @@ -704,7 +717,6 @@ posixIt.each([ expect(report.output).toContain("not trusted for this secret-bearing Mantis run"); } }, - 55_000, ); const mantisWorktrees = [ @@ -731,13 +743,15 @@ const mantisWorktrees = [ }, ]; -posixIt.each([ +posixIt.for([ ...mantisWorktrees.map((profile) => ({ ...profile, failure: false })), { ...mantisWorktrees[0]!, failure: true }, ])( "Mantis worktree Git owner drains before next worktree/install/build: $workflow (cleanup failure=$failure)", - async ({ workflow, job, lanes, offline, build, failure }) => { + { timeout: 55_000 }, + async ({ workflow, job, lanes, offline, build, failure }, { signal }) => { const report = await runCiGitStep({ + signal, workflow: { file: `.github/workflows/mantis-${workflow}.yml`, job, @@ -817,7 +831,6 @@ posixIt.each([ expect(report.output).toContain("Git ownership/setup failed"); } }, - 55_000, ); const newer = "d".repeat(40); @@ -847,8 +860,13 @@ const commit = [ ["commit", "-m", `chore(sync): mirror docs from fixture/checkout@${candidate}`], ]; -function runDocs(step: string, options: Partial[0]> = {}) { +function runDocs( + signal: AbortSignal, + step: string, + options: Partial[0]> = {}, +) { return runCiGitStep({ + signal, workflow: { file: ".github/workflows/docs-sync-publish.yml", job: "sync-publish-repo", step }, fetchResults: [], objects: { [sourceObject]: { text: JSON.stringify({ sha: candidate }) } }, @@ -865,10 +883,14 @@ function backoffs(report: Awaited>) { return [...report.output.matchAll(/fixture backoff: (\d+)/gu)].map((match) => Number(match[1])); } -posixIt.each(["directory", "file", "symlink"] as const)( +posixIt.for(["directory", "file", "symlink"] as const)( "docs clone drains an ordinary failure before deleting/retrying (%s)", - async (publishPath) => { - const report = await runDocs("Clone publish repo", { cloneResults: [23, 0], publishPath }); + { timeout: 55_000 }, + async (publishPath, { signal }) => { + const report = await runDocs(signal, "Clone publish repo", { + cloneResults: [23, 0], + publishPath, + }); expect(report.code, report.output).toBe(0); expect(report.readyAttempts).toEqual([1, 2]); expect(gitArgs(report)).toEqual( @@ -889,13 +911,14 @@ posixIt.each(["directory", "file", "symlink"] as const)( expect(report.output).toContain("Clone attempt 1 failed; retrying."); expect(report.output).not.toContain("fixture-docs-token"); }, - 55_000, ); posixIt( "docs clone cleanup uncertainty is terminal before another deletion or clone", - async () => { - const report = await runDocs("Clone publish repo", { cloneResults: ["cleanup-failure"] }); + async ({ signal }) => { + const report = await runDocs(signal, "Clone publish repo", { + cloneResults: ["cleanup-failure"], + }); expect(report.code, report.output).toBe(125); expect(report.clones).toHaveLength(1); expect(report.boundaries.map(({ name }) => name)).toEqual(["delete", "clone:1", "exit"]); @@ -906,10 +929,13 @@ posixIt( 55_000, ); -posixIt.each([125, "hang"] satisfies FetchResult[])( +posixIt.for([125, "hang"] satisfies FetchResult[])( "docs advisory fetch drains before config/add/commit and still continues (%s)", - async (failure) => { - const report = await runDocs("Commit publish repo sync", { fetchResults: [failure, 0] }); + { timeout: 55_000 }, + async (failure, { signal }) => { + const report = await runDocs(signal, "Commit publish repo sync", { + fetchResults: [failure, 0], + }); expect(report.code, report.output).toBe(0); expect(gitArgs(report)).toEqual([ diff, @@ -950,17 +976,17 @@ posixIt.each([125, "hang"] satisfies FetchResult[])( "exit", ]); }, - 55_000, ); -posixIt.each([ +posixIt.for([ { operation: "push", failure: 23, lockChange: true }, { operation: "rebase", failure: 125, lockChange: false }, { operation: "push", failure: 143, lockChange: false }, ])( "docs publication drains failed $operation ($failure) before abort/next fetch and then succeeds", - async ({ operation, failure, lockChange }) => { - const report = await runDocs("Commit publish repo sync", { + { timeout: 55_000 }, + async ({ operation, failure, lockChange }, { signal }) => { + const report = await runDocs(signal, "Commit publish repo sync", { env: lockChange ? { FIXTURE_DOCS_LOCK_AFTER_REBASE: "1" } : {}, rebaseResults: operation === "rebase" ? [failure, 0] : [], pushResults: operation === "push" ? [failure, 0] : [], @@ -990,13 +1016,12 @@ posixIt.each([ expect(report.output).toContain("Reused 1 unchanged successful page check(s)."); } }, - 55_000, ); posixIt( "docs publication rejects invalid content introduced by the final rebase", - async () => { - const report = await runDocs("Commit publish repo sync", { + async ({ signal }) => { + const report = await runDocs(signal, "Commit publish repo sync", { env: { FIXTURE_DOCS_MDX_AFTER_REBASE: "# Rebased page\n\n{unfinished\n" }, }); expect(report.code, report.output).toBe(125); @@ -1007,10 +1032,11 @@ posixIt( 55_000, ); -posixIt.each(["advisory fetch", "fetch", "rebase", "manifest", "lock", "push"] as const)( +posixIt.for(["advisory fetch", "fetch", "rebase", "manifest", "lock", "push"] as const)( "docs publication cleanup uncertainty at %s prevents abort/retry/next Git", - async (operation) => { - const report = await runDocs("Commit publish repo sync", { + { timeout: 55_000 }, + async (operation, { signal }) => { + const report = await runDocs(signal, "Commit publish repo sync", { fetchResults: operation === "advisory fetch" ? ["cleanup-failure"] @@ -1056,14 +1082,14 @@ posixIt.each(["advisory fetch", "fetch", "rebase", "manifest", "lock", "push"] a expect(report.output).toContain("Git ownership/setup failed"); expect(report.output).not.toContain("retrying"); }, - 55_000, ); -posixIt.each(["Clone publish repo", "Commit publish repo sync"])( +posixIt.for(["Clone publish repo", "Commit publish repo sync"])( "docs %s preserves five attempts and every backoff including the terminal one", - async (step) => { + { timeout: 55_000 }, + async (step, { signal }) => { const cloning = step === "Clone publish repo"; - const report = await runDocs(step, { + const report = await runDocs(signal, step, { cloneResults: cloning ? Array(5).fill(23) : [], fetchResults: cloning ? [] : [0, ...Array(5).fill(23)], }); @@ -1085,16 +1111,16 @@ posixIt.each(["Clone publish repo", "Commit publish repo sync"])( ), ).toBe(true); }, - 55_000, ); -posixIt.each([ +posixIt.for([ { label: "no changes", diffResult: 0, stale: false }, { label: "stale source", diffResult: 1, stale: true }, ])( "docs publication exits successfully without committing for $label", - async ({ diffResult, stale }) => { - const report = await runDocs("Commit publish repo sync", { + { timeout: 55_000 }, + async ({ diffResult, stale }, { signal }) => { + const report = await runDocs(signal, "Commit publish repo sync", { diffResult, objects: { [sourceObject]: { text: JSON.stringify({ sha: newer }) } }, mergeBase: { ancestor: true, revision: candidate }, @@ -1112,10 +1138,9 @@ posixIt.each([ expect(report.commands.at(-1)?.cwd).toBe(report.workspace); } }, - 55_000, ); -posixIt.each([ +posixIt.for([ { label: "missing metadata", text: "", code: 128, ancestor: false }, { label: "malformed JSON", text: "{", code: 0, ancestor: false }, { label: "non-JSON constant", text: `{"sha":"${newer}","value":NaN}`, code: 0, ancestor: true }, @@ -1124,8 +1149,9 @@ posixIt.each([ { label: "unrelated source", text: JSON.stringify({ sha: newer }), code: 0, ancestor: false }, ])( "docs publication retains the changed path for $label", - async ({ text, code, ancestor, label }) => { - const report = await runDocs("Commit publish repo sync", { + { timeout: 55_000 }, + async ({ text, code, ancestor, label }, { signal }) => { + const report = await runDocs(signal, "Commit publish repo sync", { objects: { [sourceObject]: { text, code } }, mergeBase: { ancestor, revision: candidate }, }); @@ -1146,10 +1172,9 @@ posixIt.each([ push, ]); }, - 55_000, ); -posixIt.each([ +posixIt.for([ { label: "malformed remote manifest", text: "{", @@ -1168,8 +1193,9 @@ posixIt.each([ }, ])( "docs publication rejects $label before push without Git retries", - async ({ text, validates, error }) => { - const report = await runDocs("Commit publish repo sync", { + { timeout: 55_000 }, + async ({ text, validates, error }, { signal }) => { + const report = await runDocs(signal, "Commit publish repo sync", { objects: { [sourceObject]: { text: JSON.stringify({ sha: candidate }) }, [dependencyReads[0]![1]!]: { text }, @@ -1192,13 +1218,13 @@ posixIt.each([ expect(report.rebases.map(({ args }) => args)).toEqual([rebase]); expect(backoffs(report)).toEqual([]); }, - 55_000, ); -posixIt.each([0, 23, "cleanup-failure"] satisfies FetchResult[])( +posixIt.for([0, 23, "cleanup-failure"] satisfies FetchResult[])( "docs ClawHub HEAD is owned before Node consumption (%s)", - async (revParseResult) => { - const report = await runDocs("Sync docs into publish repo", { revParseResult }); + { timeout: 55_000 }, + async (revParseResult, { signal }) => { + const report = await runDocs(signal, "Sync docs into publish repo", { revParseResult }); expect(report.code, report.output).toBe( revParseResult === "cleanup-failure" ? 125 : revParseResult, ); @@ -1226,13 +1252,13 @@ posixIt.each([0, 23, "cleanup-failure"] satisfies FetchResult[])( : [], ); }, - 55_000, ); -posixIt.each(["Clone publish repo", "Commit publish repo sync"])( +posixIt.for(["Clone publish repo", "Commit publish repo sync"])( "docs %s cancellation never reaches retry, abort, or the next Git call", - async (step) => { - const report = await runDocs(step, { + { timeout: 55_000 }, + async (step, { signal }) => { + const report = await runDocs(signal, step, { scenario: "cancel-SIGTERM", cloneResults: ["hang"], fetchResults: ["hang"], @@ -1251,7 +1277,6 @@ posixIt.each(["Clone publish repo", "Commit publish repo sync"])( step === "Clone publish repo" ? 1 : 0, ); }, - 55_000, ); const agentGate = "Gate trusted main activity and hourly cadence"; @@ -1272,8 +1297,13 @@ const agentCommitCommands = [ const agentOutput = (reviewBase = base) => `run_agent=true\nbase_sha=${candidate}\nreview_base_sha=${reviewBase}\nreview_head_sha=${candidate}\n`; -function runDocsAgent(step: string, options: Partial[0]> = {}) { +function runDocsAgent( + signal: AbortSignal, + step: string, + options: Partial[0]> = {}, +) { return runCiGitStep({ + signal, ...options, workflow: { file: ".github/workflows/docs-agent.yml", job: "update-docs", step }, fetchResults: options.fetchResults ?? [], @@ -1290,10 +1320,11 @@ function runDocsAgent(step: string, options: Partial { - const report = await runDocsAgent(agentGate, { + { timeout: 55_000 }, + async (code, { signal }) => { + const report = await runDocsAgent(signal, agentGate, { env: { EVENT_NAME: "workflow_dispatch" }, commandResults: { "rev-parse HEAD": { code: 0 }, [`rev-parse ${candidate}^`]: { code } }, }); @@ -1305,13 +1336,13 @@ posixIt.each([0, 128, 125])( expect(report.commands.filter(({ tool }) => tool === "gh")).toEqual([]); expect(report.githubOutput).toBe(agentOutput(code === 0 ? base : candidate)); }, - 55_000, ); -posixIt.each([125, "hang"] satisfies FetchResult[])( +posixIt.for([125, "hang"] satisfies FetchResult[])( "Docs Agent gate drains failed fetch before retry, remote read, gh and output (%s)", - async (failure) => { - const report = await runDocsAgent(agentGate, { fetchResults: [failure, 0] }); + { timeout: 55_000 }, + async (failure, { signal }) => { + const report = await runDocsAgent(signal, agentGate, { fetchResults: [failure, 0] }); expect(report.code, report.output).toBe(0); expect(report.fetches.map(({ args }) => args)).toEqual([agentFetch, agentFetch]); expect(backoffs(report)).toEqual([2]); @@ -1332,13 +1363,13 @@ posixIt.each([125, "hang"] satisfies FetchResult[])( ], ]); }, - 55_000, ); -posixIt.each([false, true])( +posixIt.for([false, true])( "Docs Agent gate stops before gh/output/retry on fatal cleanup (cancel=%s)", - async (cancel) => { - const report = await runDocsAgent(agentGate, { + { timeout: 55_000 }, + async (cancel, { signal }) => { + const report = await runDocsAgent(signal, agentGate, { fetchResults: cancel ? ["hang"] : ["cleanup-failure"], ...(cancel ? { scenario: "cancel-SIGTERM", cooperativeTrees: true, realClock: true } : {}), }); @@ -1349,13 +1380,12 @@ posixIt.each([false, true])( expect(backoffs(report)).toEqual([]); expect(report.output).not.toContain("retrying"); }, - 55_000, ); posixIt( "Docs Agent superseded gate drains before false output without gh", - async () => { - const report = await runDocsAgent(agentGate, { revisions: { "origin/main": moved } }); + async ({ signal }) => { + const report = await runDocsAgent(signal, agentGate, { revisions: { "origin/main": moved } }); expect(report.code, report.output).toBe(0); expect(gitArgs(report)).toEqual([agentFetch, ["rev-parse", "origin/main"]]); expect(report.githubOutput).toBe("run_agent=false\n"); @@ -1367,7 +1397,7 @@ posixIt( 55_000, ); -posixIt.each([ +posixIt.for([ { probe: 128, parent: 0, code: 0, reviewBase: base }, { probe: 128, parent: 128, code: 0, reviewBase: candidate }, { probe: 0, parent: 0, code: 0, reviewBase: moved }, @@ -1375,8 +1405,9 @@ posixIt.each([ { probe: 128, parent: "cleanup-failure", code: 125, reviewBase: "" }, ] satisfies { probe: FetchResult; parent: FetchResult; code: number; reviewBase: string }[])( "Docs Agent review base only falls back after ordinary Git failure ($probe/$parent)", - async ({ probe, parent, code, reviewBase }) => { - const report = await runDocsAgent(agentGate, { + { timeout: 55_000 }, + async ({ probe, parent, code, reviewBase }, { signal }) => { + const report = await runDocsAgent(signal, agentGate, { workflowRuns: [ { id: 122, @@ -1436,13 +1467,12 @@ posixIt.each([ ]); expect(backoffs(report)).toEqual([]); }, - 55_000, ); posixIt( "Docs Agent no-change commit owns diff before successful exit", - async () => { - const report = await runDocsAgent(agentCommit, { + async ({ signal }) => { + const report = await runDocsAgent(signal, agentCommit, { commandResults: { "diff HEAD --quiet": { code: 0 } }, }); expect(report.code, report.output).toBe(0); @@ -1452,10 +1482,11 @@ posixIt( 55_000, ); -posixIt.each([125, "hang"] satisfies FetchResult[])( +posixIt.for([125, "hang"] satisfies FetchResult[])( "Docs Agent commit drains diff before config/commit and failed fetch before retry (%s)", - async (failure) => { - const report = await runDocsAgent(agentCommit, { + { timeout: 55_000 }, + async (failure, { signal }) => { + const report = await runDocsAgent(signal, agentCommit, { commandResults: { "diff HEAD --quiet": { code: failure === 125 ? 125 : 1 } }, fetchResults: [failure, 0], }); @@ -1465,13 +1496,13 @@ posixIt.each([125, "hang"] satisfies FetchResult[])( expect(report.output).toContain("Fetch attempt 1 failed; retrying."); expect(report.output).not.toContain("fixture-docs-agent-token"); }, - 55_000, ); -posixIt.each([false, true])( +posixIt.for([false, true])( "Docs Agent push failure drains before owned read and retry/stale success (advanced=%s)", - async (advanced) => { - const report = await runDocsAgent(agentCommit, { + { timeout: 55_000 }, + async (advanced, { signal }) => { + const report = await runDocsAgent(signal, agentCommit, { pushResults: [143, 0], revParseResult: 0, revisions: { "origin/main": advanced ? moved : candidate }, @@ -1491,14 +1522,14 @@ posixIt.each([false, true])( : "Docs update attempt 1 failed; retrying.", ); }, - 55_000, ); -posixIt.each(["diff", "config", "commit", "fetch", "push", "read"])( +posixIt.for(["diff", "config", "commit", "fetch", "push", "read"])( "Docs Agent commit cleanup failure at %s is terminal before retry/stale success", - async (operation) => { + { timeout: 55_000 }, + async (operation, { signal }) => { const index = operation === "diff" ? 0 : operation === "config" ? 1 : 4; - const report = await runDocsAgent(agentCommit, { + const report = await runDocsAgent(signal, agentCommit, { commandResults: ["diff", "config", "commit"].includes(operation) ? { [agentCommitCommands[index]!.join(" ")]: { code: "cleanup-failure" } } : {}, @@ -1521,14 +1552,14 @@ posixIt.each(["diff", "config", "commit", "fetch", "push", "read"])( expect(backoffs(report)).toEqual([]); expect(report.output).not.toMatch(/retrying|skipping stale|No docs changes/u); }, - 55_000, ); -posixIt.each(["gate", "commit fetch", "commit push"])( +posixIt.for(["gate", "commit fetch", "commit push"])( "Docs Agent %s preserves five attempts and terminal backoff contract", - async (phase) => { + { timeout: 55_000 }, + async (phase, { signal }) => { const gate = phase === "gate"; - const report = await runDocsAgent(gate ? agentGate : agentCommit, { + const report = await runDocsAgent(signal, gate ? agentGate : agentCommit, { fetchResults: phase === "commit push" ? [] : Array(5).fill(23), pushResults: phase === "commit push" ? Array(5).fill(23) : [], }); @@ -1552,7 +1583,6 @@ posixIt.each(["gate", "commit fetch", "commit push"])( ), ).toBe(true); }, - 55_000, ); const agentProducers = [ @@ -1562,15 +1592,15 @@ const agentProducers = [ ["diff", "HEAD", "--name-only"], ["diff", "--cached", "HEAD", "--name-only"], ]; -posixIt.each(agentProducers.map((args, index) => ({ args, index })))( +posixIt.for(agentProducers.map((args, index) => ({ args, index })))( "Docs Agent enforcement stops on failed producer $args before consuming partial output", - async ({ args, index }) => { - const report = await runDocsAgent("Enforce existing-docs-only patch", { + { timeout: 55_000 }, + async ({ args, index }, { signal }) => { + const report = await runDocsAgent(signal, "Enforce existing-docs-only patch", { commandResults: { [args.join(" ")]: { code: 23, output: "src/forbidden.ts\n" } }, }); expect(report.code, report.output).toBe(23); expect(gitArgs(report)).toEqual(agentProducers.slice(0, index + 1)); expect(report.output).not.toContain("forbidden"); }, - 55_000, ); diff --git a/test/scripts/ci-platform-checkout.test.ts b/test/scripts/ci-platform-checkout.test.ts index 07dc945778c1..61a8be79f520 100644 --- a/test/scripts/ci-platform-checkout.test.ts +++ b/test/scripts/ci-platform-checkout.test.ts @@ -99,18 +99,20 @@ const linuxCases = { scenario: "non-executable-find", attempts: 0, code: null, checkout: false, deletions: 0 }, ]; -it.concurrent.each([ +it.concurrent.for([ ...platformCases.map((entry) => Object.assign(entry, { linux: false, deletions: 0 })), ...linuxCases.map((entry) => Object.assign(entry, { linux: true })), ])( "preserves checkout ownership and fixture isolation (Linux=$linux, $scenario)", - async ({ scenario, attempts, code, checkout, linux, deletions }) => { + { timeout: 55_000 }, + async ({ scenario, attempts, code, checkout, linux, deletions }, { signal: testSignal }) => { const setupFailure = scenario.startsWith("non-executable-"); const run = readCiCheckoutStep(linux ? "checks-fast-core" : "checks-windows").run; const policyScenario = `${linux ? "linux:" : ""}${scenario}`; await withCiCheckoutFixture( policyScenario, + testSignal, (root) => { const workspace = path.join(root, "workspace"); if (scenario === "cancel-SIGTERM") { @@ -138,7 +140,14 @@ it.concurrent.each([ path.join(root, "checkout.sh"), setupFailure ? "printf 'unexpected workflow invocation\\n' >&2\nexit 99\n" : accelerated, ); - if (process.platform === "win32" || scenario === "git-exit-124") { + // A slow census witness must not replace the workflow's real outcome. + const slowWitness = [ + "timeouts-exhausted", + "recovery", + "early-leader-exit", + "harness-timeout", + ].includes(scenario); + if (process.platform === "win32" || slowWitness || scenario === "git-exit-124") { return censusPreload( root, scenario === "git-exit-124" @@ -176,9 +185,7 @@ if (process.argv[2] === "sentinel") { syncFixtureBuiltinExports(); ` : "", - ["timeouts-exhausted", "recovery", "early-leader-exit", "harness-timeout"].includes( - scenario, - ), + slowWitness, ); } return undefined; @@ -312,10 +319,9 @@ syncFixtureBuiltinExports(); }, ); }, - 55_000, ); -it.concurrent.each([ +it.concurrent.for([ ...[ ...(process.platform === "win32" ? [] : [{ kind: "linux-node", retained: false }]), ...(process.platform === "win32" @@ -376,7 +382,8 @@ it.concurrent.each([ ].map((entry) => Object.assign(entry, { kind: "preflight", retained: false }))), ])( "materializes $kind trusted harness ($event, workflow=$workflow, target=$target, retained=$retained) without mutating the candidate", - async ({ kind, retained, event, workflow, target, code, fetches }) => { + { timeout: 55_000 }, + async ({ kind, retained, event, workflow, target, code, fetches }, { signal }) => { const linux = kind !== "platform"; const preflight = kind === "preflight"; const posix = process.platform !== "win32"; @@ -441,6 +448,7 @@ it.concurrent.each([ let readSourceStatus: (() => string[]) | undefined; await withCiCheckoutFixture( `${linux ? "linux:" : ""}configured`, + signal, (root) => { const source = path.join(root, "source"); mkdirSync(source); @@ -743,18 +751,19 @@ it.concurrent.each([ }, ); }, - 55_000, ); registerWindowsCensusTests(); -it.each(["prepare", "inspect"])( +it.for(["prepare", "inspect"])( "removes checkout artifacts after %s assertion failure", - async (phase) => { + { timeout: 55_000 }, + async (phase, { signal }) => { let root: string | undefined; await expect( withCiCheckoutFixture( "early-leader-exit", + signal, (directory) => { root = directory; expect(phase, "injected prepare assertion").not.toBe("prepare"); @@ -769,10 +778,9 @@ it.each(["prepare", "inspect"])( ).rejects.toThrow(`injected ${phase} assertion`); expect(existsSync(expectDefined(root, "created checkout root"))).toBe(false); }, - 55_000, ); -it.skipIf(process.platform === "win32").each(["census", "corrupt-report", "timeout"])( +it.skipIf(process.platform === "win32").each(["census", "corrupt-report", "timeout", "cancel"])( "retains checkout artifacts across failed outer-runner cleanup (%s)", async (fault) => { const preload = String.raw` @@ -802,6 +810,10 @@ if (process.argv[2] === "supervise") { }); }); } + if (fault === "cancel" && args[1]?.[1] !== "sentinel") { + // The detached workflow shell is running; only the supervisor can retire its group. + queueMicrotask(() => process.send({ type: "ci-checkout:shell-started", pids: [process.pid, ...children] })); + } return child; }; cp.spawnSync = (...args) => { @@ -834,15 +846,18 @@ import fs from "node:fs"; import { fixturePreloadEnv, syncFixtureBuiltinExports } from ${JSON.stringify(new URL("./fixtures/ci-fixture-runtime.cjs", import.meta.url).href)}; import { tmpdir } from "node:os"; import path from "node:path"; -import { mock } from "node:test"; const timeoutFault = process.argv[2] === "timeout"; +const cancelledFault = timeoutFault || process.argv[2] === "cancel"; +const cancellation = new AbortController(); let root, failure; let supervisor, ready, onReady; const fork = cp.fork; -if (timeoutFault) { +if (cancelledFault) { ready = new Promise(resolve => { onReady = message => { - if (message?.type === "ci-checkout:sentinel-created") resolve(message.pids); + if (message?.type === (timeoutFault ? "ci-checkout:sentinel-created" : "ci-checkout:shell-started")) { + resolve(message.pids); + } }; }); cp.fork = (...args) => { @@ -854,10 +869,9 @@ if (timeoutFault) { } try { const { withCiCheckoutFixture } = await import(process.argv[1]); - if (timeoutFault) mock.timers.enable({ apis: ["setTimeout"] }); - const completed = withCiCheckoutFixture("early-leader-exit", directory => { + const completed = withCiCheckoutFixture("early-leader-exit", cancellation.signal, directory => { root = directory; - fs.writeFileSync(path.join(root, "checkout.sh"), "exit 0\n"); + fs.writeFileSync(path.join(root, "checkout.sh"), process.argv[2] === "cancel" ? "exec sleep 30\n" : "exit 0\n"); const preload = path.join(root, "fault.mjs"); fs.writeFileSync(preload, "const fault = " + JSON.stringify(process.argv[2]) + ";\n" + process.argv[3]); return fixturePreloadEnv(preload); @@ -868,11 +882,11 @@ try { failure = String(error); }); try { - if (timeoutFault) { + if (cancelledFault) { const pids = await Promise.race([ready, completed.then(() => { - throw new Error("supervisor completed before the timeout probe was ready"); + throw new Error("supervisor completed before the cancellation probe was ready"); })]); - assert.equal(pids.length, 2); + assert.equal(pids.length, timeoutFault ? 2 : 3); assert.equal(pids[0], supervisor.pid); assert.notEqual(pids[1], supervisor.pid); for (const pid of pids) { @@ -881,11 +895,10 @@ try { } } } finally { - if (timeoutFault) { + if (cancelledFault) { // Creation belongs to the supervisor, not a child's delayed self-registration. - // Restore timers before the expired controller deadline starts real cleanup. - mock.timers.tick(50_000); - mock.timers.reset(); + // Cancel the run the way a Vitest timeout aborts its owning test. + cancellation.abort(new Error("owning test cancelled the supervised run")); } await completed; } @@ -893,8 +906,7 @@ try { console.error(error); failure = String(error); } finally { - if (timeoutFault) { - mock.timers.reset(); + if (cancelledFault) { supervisor?.off("message", onReady); cp.fork = fork; syncFixtureBuiltinExports(); @@ -950,8 +962,15 @@ process.exitCode = 1; expect(existsSync(path.join(evidence.root, "report.json"))).toBe(false); } else if (fault === "timeout") { expect(evidence.pids).toHaveLength(2); - expect(evidence.failure).toContain("did not close within 50000ms"); + expect(evidence.failure).toContain("owning test cancelled the supervised run"); expect(existsSync(path.join(evidence.root, "report.json"))).toBe(false); + } else if (fault === "cancel") { + // The detached shell group died, so the live supervisor ran its own cleanup. + expect(evidence.pids).toHaveLength(3); + expect(evidence.failure).toContain("owning test cancelled the supervised run"); + expect( + JSON.parse(readFileSync(path.join(evidence.root, "report.json"), "utf8")), + ).toMatchObject({ error: "test cancelled", cleanupRemaining: [] }); } else { expect(evidence.failure).not.toContain("unexpected completed report"); expect(readFileSync(path.join(evidence.root, "report.json"), "utf8")).toBe("null"); @@ -967,8 +986,9 @@ process.exitCode = 1; it.skipIf(process.platform === "win32")( "waits for legal slow tree startup before cancellation", - async () => { + async ({ signal }) => { const report = await runCiGitStep({ + signal, job: "checks-windows", env: { CHECKOUT_KIND: "platform" }, fetchResults: ["hang"], @@ -984,8 +1004,9 @@ it.skipIf(process.platform === "win32")( it.skipIf(process.platform === "win32")( "reports owner exit and output instead of a cleanup readiness timeout", - async () => { + async ({ signal }) => { const report = await runCiGitStep({ + signal, policy: 'print("owner exited before cleanup readiness", flush=True)\nraise SystemExit(23)\n', fetchResults: [], cancelDuringCleanup: true, diff --git a/test/scripts/ci-windows-process-census.test-support.ts b/test/scripts/ci-windows-process-census.test-support.ts index 13f95f2dae42..c7f9fce140a3 100644 --- a/test/scripts/ci-windows-process-census.test-support.ts +++ b/test/scripts/ci-windows-process-census.test-support.ts @@ -23,8 +23,19 @@ const censusArgs = args => delayed && args?.[2]?.endsWith("ci-windows-process-ce ? ["-I", "-S", "-c", "import runpy,sys,time; time.sleep(1.1); sys.argv=sys.argv[1:]; runpy.run_path(sys.argv[0],run_name='__main__')", ...args.slice(2)] : args; const spawn = cp.spawn, spawnSync = cp.spawnSync; +// POSIX has no census interpreter to delay; its first native query instead +// spends as long as a loaded host can, on the supervisor's clock. +let slowNativeQuery = delayed && process.argv[2] === "supervise"; // Delay the actual interpreter entry point, including the pre-fix synchronous path. -cp.spawnSync = (command, args, options) => spawnSync(command, censusArgs(args), options); +cp.spawnSync = (command, args, options) => { + const result = spawnSync(command, censusArgs(args), options); + if (slowNativeQuery && command === "/bin/ps") { + slowNativeQuery = false; + const now = Date.now; + Date.now = () => now() + 1_100; + } + return result; +}; cp.spawn = (command, args, options) => { const child = spawn(command, censusArgs(args), options); if (command === "python" && process.argv[2] === "supervise") { @@ -274,11 +285,13 @@ fs.rmSync(root, { recursive: true }); censusTestTimeoutMs, ); - it.each(["sentinel", ...(process.platform === "win32" ? ["startup", "query", "lease"] : [])])( + it.for(["sentinel", ...(process.platform === "win32" ? ["startup", "query", "lease"] : [])])( "retains startup errors and joins census before registration (%s)", - async (fault) => { + { timeout: 55_000 }, + async (fault, { signal }) => { await withCiCheckoutFixture( "early-leader-exit", + signal, (root) => { writeFileSync(path.join(root, "checkout.sh"), "exit 99\n"); const sampler = String.raw` @@ -351,12 +364,12 @@ for line in sys.stdin: }, ); }, - 55_000, ); - it("joins an unregistered sentinel before supervisor close on disconnect", async () => { + it("joins an unregistered sentinel before supervisor close on disconnect", async ({ signal }) => { await withCiCheckoutFixture( "early-leader-exit", + signal, (root) => { writeFileSync(path.join(root, "checkout.sh"), "exit 99\n"); // Fault only the asynchronous startup boundary; keep the real safe preflight. @@ -414,9 +427,10 @@ if (mode === "supervise") { ); }, 55_000); - it.each(["direct-child-close", "truthful-final-census"])( + it.for(["direct-child-close", "truthful-final-census"])( "rejects expired supervisor cleanup and joins census (%s)", - async (fault) => { + { timeout: 55_000 }, + async (fault, { signal }) => { type BoundaryEvent = { event: string; role?: string; @@ -436,6 +450,7 @@ if (mode === "supervise") { let failure: unknown; await withCiCheckoutFixture( "early-leader-exit", + signal, (directory) => { root = directory; writeFileSync(path.join(root, "checkout.sh"), "exit 0\n"); @@ -593,6 +608,5 @@ if (mode === "supervise") { // Every spawned writer above also has a creator-held close; failures retain evidence. rmSync(directory, { recursive: true }); }, - 55_000, ); } diff --git a/test/scripts/ci-workflow-guards.test.ts b/test/scripts/ci-workflow-guards.test.ts index 9054fd22c6af..412ed2bc1bb7 100644 --- a/test/scripts/ci-workflow-guards.test.ts +++ b/test/scripts/ci-workflow-guards.test.ts @@ -4999,7 +4999,7 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}" }); }); - it.skipIf(process.platform === "win32").each([ + it.skipIf(process.platform === "win32").for([ { task: "bundled-protocol", eventName: "pull_request" }, { task: "bundled-protocol", eventName: "workflow_dispatch" }, { task: "guards", eventName: "pull_request" }, @@ -5008,7 +5008,8 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}" { task: "npm-lock", eventName: "workflow_dispatch" }, ] as const)( "uses prefetched CI base without later network access ($task, $eventName)", - async ({ task, eventName }) => { + { timeout: 55_000 }, + async ({ task, eventName }, { signal }) => { const base = "c".repeat(40); const baseRef = "refs/remotes/origin/ci-ratchet-base"; const jobName = task === "bundled-protocol" ? "checks-fast-core" : "check-shard"; @@ -5024,6 +5025,7 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}" preflightOutputs: { diff_base_revision: base }, }); const report = await runCiGitStep({ + signal, job: jobName, step: task === "bundled-protocol" @@ -5086,7 +5088,6 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}" ]); } }, - 55_000, ); it.each([ diff --git a/test/scripts/fixtures/ci-platform-checkout.mjs b/test/scripts/fixtures/ci-platform-checkout.mjs index 5c4e363dc113..53943e5f9a8f 100644 --- a/test/scripts/fixtures/ci-platform-checkout.mjs +++ b/test/scripts/fixtures/ci-platform-checkout.mjs @@ -17,6 +17,9 @@ const instance = randomUUID(); let ownWindowsCreationTime; let census; let actorLease; +// Orphan ceiling for every fixture process. The owning test's abort signal ends a +// supervised run; this only bounds processes whose owner died or never cancels. +const lifetimeCeilingMs = 60_000; let operationDeadline; const workspace = path.join(root, "workspace"); const runnerTemp = path.join(root, "temp"); @@ -210,9 +213,9 @@ async function liveRecords(deadline = operationDeadline) { } } else { // Linux can census the owned PID set in one ps call. Apple ps scans the - // whole host for multiple PIDs; keep its singleton queries under one budget. + // whole host for multiple PIDs; its singleton queries share the caller's + // operation deadline, like the Windows witness. const pidLists = process.platform === "linux" ? [[...pids]] : [...pids].map((pid) => [pid]); - const deadline = Date.now() + 1_000; for (const selectedPids of pidLists) { const remaining = deadline - Date.now(); if (remaining <= 0) { @@ -262,8 +265,7 @@ async function liveRecords(deadline = operationDeadline) { }); } -function isWorkflowDescendant(pid, shellPid) { - const deadline = Date.now() + 1_000; +function isWorkflowDescendant(pid, shellPid, deadline = operationDeadline) { const visited = new Set(); while (pid > 1 && !visited.has(pid)) { if (pid === shellPid) return true; @@ -386,7 +388,7 @@ function holdLease() { }; // Orphans stop themselves when the supervisor releases the lease; no PID discovery/kills. // The independent ceiling also covers a supervisor killed before it can unlink the lease. - const deadline = Date.now() + 60_000; + const deadline = Date.now() + lifetimeCeilingMs; const checkLease = () => { if (!isLive() || Date.now() >= deadline) { process.exit(0); @@ -1285,8 +1287,12 @@ async function supervise() { for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"]) { process.once(signal, () => void stop(`supervisor received ${signal}`)); } - operationDeadline = Date.now() + 45_000; - setTimeout(() => void stop("fixture deadline exceeded"), 45_000); + // The owning test's signal bounds the run: a slow host must not lose a race + // against a fixture deadline. Cancellation still runs this owned cleanup. + process.on("message", (message) => { + if (message?.type === "ci-checkout:cancel") void stop("test cancelled"); + }); + operationDeadline = Date.now() + lifetimeCeilingMs; try { if (process.platform === "win32") { census = createWindowsProcessCensus({ @@ -1297,7 +1303,7 @@ async function supervise() { void stop(error); }, }); - // Interpreter startup belongs to the existing supervisor watchdog, not a query deadline. + // Interpreter startup belongs to the owning test's cancellation, not a query deadline. await census.ready; if (stopping) { await stopping; @@ -1322,7 +1328,7 @@ async function supervise() { cwd: workspace, env: { PATH: commandPath }, encoding: "utf8", - timeout: 2_000, + timeout: Math.max(1, operationDeadline - Date.now()), killSignal: "SIGKILL", }, ); diff --git a/test/scripts/openclaw-performance-git-lifecycle.test.ts b/test/scripts/openclaw-performance-git-lifecycle.test.ts index e2b23d53fadb..23b03f5a171e 100644 --- a/test/scripts/openclaw-performance-git-lifecycle.test.ts +++ b/test/scripts/openclaw-performance-git-lifecycle.test.ts @@ -24,11 +24,13 @@ const steps = { } as const; function performanceRun( + signal: AbortSignal, mode: PerformanceFixtureOptions["mode"], options: Partial[0]> = {}, ) { const [job, step] = steps[mode]; return runCiGitStep({ + signal, workflow: { file: ".github/workflows/openclaw-performance.yml", job, step }, fetchResults: [], performance: { mode }, @@ -39,10 +41,11 @@ function performanceRun( // The previous semantic tests used short-lived stubs or replayed Git by hand. // These actual workflow bodies must drain real parent/child/grandchild writers // before every command, output, consumer and exit, while a sentinel stays alive. -posixIt.each(Object.keys(steps) as PerformanceFixtureOptions["mode"][])( +posixIt.for(Object.keys(steps) as PerformanceFixtureOptions["mode"][])( "Performance %s drains Git trees before every continuation", - async (mode) => { - const report = await performanceRun(mode); + { timeout: 55_000 }, + async (mode, { signal }) => { + const report = await performanceRun(signal, mode); expect(report.code, report.output).toBe(0); expect(report.readyAttempts.length).toBeGreaterThan(0); if (mode === "prepare") { @@ -54,13 +57,13 @@ posixIt.each(Object.keys(steps) as PerformanceFixtureOptions["mode"][])( expect(report.githubSummary).toContain("### Clawgrit report published"); } }, - 55_000, ); -posixIt.each([23, 125])( +posixIt.for([23, 125])( "baseline ordinary fetch %s is advisory after extinction", - async (code) => { - const report = await performanceRun("baseline", { fetchResults: [code, code, code] }); + { timeout: 55_000 }, + async (code, { signal }) => { + const report = await performanceRun(signal, "baseline", { fetchResults: [code, code, code] }); expect(report.code, report.output).toBe(0); expect(report.githubSummary).toBe( "No previous source performance baseline could be fetched.\n", @@ -68,13 +71,13 @@ posixIt.each([23, 125])( expect(report.githubEnv).toBe(""); expect(report.commands.at(-1)?.args[0]).toBe("fetch"); }, - 55_000, ); -posixIt.each(["absent", "invalid", "trailing-newline"] as const)( +posixIt.for(["absent", "invalid", "trailing-newline"] as const)( "baseline %s pointer preserves advisory result", - async (baseline) => { - const report = await performanceRun("baseline", { + { timeout: 55_000 }, + async (baseline, { signal }) => { + const report = await performanceRun(signal, "baseline", { performance: { mode: "baseline", baseline }, }); expect(report.code, report.output).toBe(0); @@ -86,13 +89,13 @@ posixIt.each(["absent", "invalid", "trailing-newline"] as const)( expect(report.githubEnv).toBe(""); expect(report.checkouts).toHaveLength(0); }, - 55_000, ); -posixIt.each(["ls-tree", "show"])( +posixIt.for(["ls-tree", "show"])( "baseline %s failure never becomes absence or invalid JSON", - async (operation) => { - const report = await performanceRun("baseline", { + { timeout: 55_000 }, + async (operation, { signal }) => { + const report = await performanceRun(signal, "baseline", { gitFault: { match: `^${operation} `, code: 128 }, }); expect(report.code, report.output).toBe(128); @@ -100,7 +103,6 @@ posixIt.each(["ls-tree", "show"])( expect(report.githubEnv).toBe(""); expect(report.commands.at(-1)?.args[0]).toBe(operation); }, - 55_000, ); const terminalCases = [ @@ -124,7 +126,7 @@ const terminalCases = [ ].map((operation) => ({ mode: "publish" as const, operation })), ]; // Every injected lifecycle failure must stop at its command, before later policy actions. -posixIt.each( +posixIt.for( terminalCases.flatMap((entry) => (["cleanup-failure", "cancel"] as const).map((code) => ({ mode: entry.mode, @@ -134,8 +136,9 @@ posixIt.each( ), )( "$mode $operation $code fences every later action", - async ({ mode, operation, code }) => { - const report = await performanceRun(mode, { + { timeout: 55_000 }, + async ({ mode, operation, code }, { signal }) => { + const report = await performanceRun(signal, mode, { gitFault: { match: `^${operation}(?: |$)`, code }, ...(mode === "publish" && operation !== "config" && operation !== "push" ? { pushResults: [23] } @@ -158,13 +161,15 @@ posixIt.each( expect(report.output).not.toContain("fixture backoff:"); } }, - 55_000, ); -posixIt.each(["hang", 23, 125] as const)( +posixIt.for(["hang", 23, 125] as const)( "prepare initial fetch %s cannot reach checkout, commit or token readiness", - async (failure) => { - const report = await performanceRun("prepare", { fetchResults: [failure, failure, failure] }); + { timeout: 55_000 }, + async (failure, { signal }) => { + const report = await performanceRun(signal, "prepare", { + fetchResults: [failure, failure, failure], + }); expect(report.code, report.output).toBe(1); expect(report.fetches).toHaveLength(3); expect(report.fetches.every(({ args }) => args.includes("--depth=1"))).toBe(true); @@ -173,13 +178,12 @@ posixIt.each(["hang", 23, 125] as const)( expect(report.githubOutput).toBe("ready=false\n"); expect(report.githubSummary).toBe(""); }, - 55_000, ); posixIt( "initial duplicate is verified before token or push", - async () => { - const report = await performanceRun("prepare", { + async ({ signal }) => { + const report = await performanceRun(signal, "prepare", { performance: { mode: "prepare", duplicate: true }, }); expect(report.code, report.output).toBe(0); @@ -199,21 +203,24 @@ posixIt( 55_000, ); -posixIt.each([128, 125])( +posixIt.for([128, 125])( "prepare duplicate inspection %s is terminal", - async (code) => { - const report = await performanceRun("prepare", { gitFault: { match: "^ls-tree ", code } }); + { timeout: 55_000 }, + async (code, { signal }) => { + const report = await performanceRun(signal, "prepare", { + gitFault: { match: "^ls-tree ", code }, + }); expect(report.code, report.output).toBe(code); expect(report.githubOutput).toBe("ready=false\n"); expect(report.commands.at(-1)?.args[0]).toBe("ls-tree"); }, - 55_000, ); -posixIt.each([0, 1, 125])( +posixIt.for([0, 1, 125])( "cached diff status %s commits only for ordinary 1", - async (code) => { - const report = await performanceRun("prepare", { + { timeout: 55_000 }, + async (code, { signal }) => { + const report = await performanceRun(signal, "prepare", { gitFault: { match: "^diff --cached --quiet$", code, output: "" }, }); expect(report.code, report.output).toBe(code > 1 ? code : 0); @@ -222,13 +229,13 @@ posixIt.each([0, 1, 125])( ); expect(report.githubOutput.includes("ready=true\n")).toBe(code <= 1); }, - 55_000, ); -posixIt.each([125, "hang"] as const)( +posixIt.for([125, "hang"] as const)( "ambiguous push %s reconciles only after extinction", - async (code) => { - const report = await performanceRun("publish", { pushResults: [code] }); + { timeout: 55_000 }, + async (code, { signal }) => { + const report = await performanceRun(signal, "publish", { pushResults: [code] }); expect(report.code, report.output).toBe(0); expect(report.pushes).toHaveLength(2); expect(report.fetches).toHaveLength(1); @@ -260,13 +267,13 @@ posixIt.each([125, "hang"] as const)( Buffer.from("x-access-token:fixture-performance-token").toString("base64"), ); }, - 55_000, ); -posixIt.each([false, true])( +posixIt.for([false, true])( "five failed pushes always get five fetches (fetch fails=%s)", - async (fetchFails) => { - const report = await performanceRun("publish", { + { timeout: 55_000 }, + async (fetchFails, { signal }) => { + const report = await performanceRun(signal, "publish", { pushResults: [23, 23, 23, 23, 23], fetchResults: fetchFails ? [23, 23, 23, 23, 23] : [], }); @@ -289,13 +296,13 @@ posixIt.each([false, true])( expect(report.githubSummary).toContain("failed after 5 attempts."); expect(report.githubSummary).not.toContain("Published report:"); }, - 55_000, ); -posixIt.each([1, 5])( +posixIt.for([1, 5])( "remote duplicate after ambiguous attempt %s succeeds without replay", - async (attempt) => { - const report = await performanceRun("publish", { + { timeout: 55_000 }, + async (attempt, { signal }) => { + const report = await performanceRun(signal, "publish", { performance: { mode: "publish", remoteDuplicateAttempt: attempt }, pushResults: Array.from({ length: attempt }, () => 124), }); @@ -305,13 +312,13 @@ posixIt.each([1, 5])( expect(report.checkouts).toHaveLength(attempt - 1); expect(report.githubSummary).toContain("### Clawgrit report published"); }, - 55_000, ); -posixIt.each([23, 125])( +posixIt.for([23, 125])( "ordinary cherry-pick and abort %s failures remain visible publish failure", - async (code) => { - const report = await performanceRun("publish", { + { timeout: 55_000 }, + async (code, { signal }) => { + const report = await performanceRun(signal, "publish", { pushResults: [23], gitFaults: [ { match: "^cherry-pick -X ", code: 23 }, @@ -326,37 +333,40 @@ posixIt.each([23, 125])( ); expect(report.githubSummary).toBe(""); }, - 55_000, ); -posixIt.each(["owner", "python", "git"] as const)( +posixIt.for(["owner", "python", "git"] as const)( "prepare setup failure %s cannot publish readiness", - async (setupFailure) => { - const report = await performanceRun("prepare", { setupFailure }); + { timeout: 55_000 }, + async (setupFailure, { signal }) => { + const report = await performanceRun(signal, "prepare", { setupFailure }); expect(report.code, report.output).not.toBe(0); expect(report.pushes).toHaveLength(0); expect(report.githubOutput).not.toContain("ready=true"); }, - 55_000, ); -posixIt.each([125, "hang"] as const)( +posixIt.for([125, "hang"] as const)( "reconciliation fetch %s warns once and retries without replay", - async (code) => { - const report = await performanceRun("publish", { pushResults: [23], fetchResults: [code] }); + { timeout: 55_000 }, + async (code, { signal }) => { + const report = await performanceRun(signal, "publish", { + pushResults: [23], + fetchResults: [code], + }); expect(report.code, report.output).toBe(0); expect(report.pushes).toHaveLength(2); expect(report.fetches).toHaveLength(1); expect(report.checkouts).toHaveLength(0); expect(report.output.match(/::warning::Unable to refresh/gu)).toHaveLength(1); }, - 55_000, ); -posixIt.each([125, 128])( +posixIt.for([125, 128])( "remote duplicate read %s is terminal, never absence", - async (code) => { - const report = await performanceRun("publish", { + { timeout: 55_000 }, + async (code, { signal }) => { + const report = await performanceRun(signal, "publish", { pushResults: [23], gitFault: { match: "^ls-tree ", code }, }); @@ -365,13 +375,13 @@ posixIt.each([125, 128])( expect(report.githubSummary).toBe(""); expect(report.checkouts).toHaveLength(0); }, - 55_000, ); -posixIt.each(["prepare", "publish"] as const)( +posixIt.for(["prepare", "publish"] as const)( "%s cancellation during real TERM-resistant cleanup prevents continuation", - async (mode) => { - const report = await performanceRun(mode, { + { timeout: 55_000 }, + async (mode, { signal }) => { + const report = await performanceRun(signal, mode, { cancelDuringCleanup: true, cleanupCancelMatch: mode === "prepare" ? "^fetch " : "^push ", ...(mode === "prepare" ? { fetchResults: ["hang"] } : { pushResults: ["hang"] }), @@ -383,13 +393,12 @@ posixIt.each(["prepare", "publish"] as const)( expect(report.githubSummary).toBe(""); expect(report.output).not.toContain("fixture backoff:"); }, - 55_000, ); posixIt( "cancellation during owned backoff prevents reconciliation fetch", - async () => { - const report = await performanceRun("publish", { + async ({ signal }) => { + const report = await performanceRun(signal, "publish", { pushResults: [23], realClock: true, cooperativeTrees: true, diff --git a/test/scripts/openclaw-performance-workflow.test.ts b/test/scripts/openclaw-performance-workflow.test.ts index f98fc095153b..7c1827d86923 100644 --- a/test/scripts/openclaw-performance-workflow.test.ts +++ b/test/scripts/openclaw-performance-workflow.test.ts @@ -1214,7 +1214,7 @@ printf '%s\\n' \ } }); - posixIt.each([ + posixIt.for([ { name: "direct", pushResults: [], fetchResults: [], success: true }, { name: "remote duplicate", pushResults: [124], fetchResults: [], success: true, duplicate: 1 }, { @@ -1226,8 +1226,10 @@ printf '%s\\n' \ { name: "missing token", pushResults: [], fetchResults: [], success: false, token: "" }, ])( "advertises a clawgrit URL only after verified success ($name)", - async ({ name, pushResults, fetchResults, success, duplicate, token }) => { + { timeout: 55_000 }, + async ({ name, pushResults, fetchResults, success, duplicate, token }, { signal }) => { const report = await runCiGitStep({ + signal, workflow: { file: WORKFLOW, job: "publish", step: "Publish to clawgrit reports" }, performance: { mode: "publish", remoteDuplicateAttempt: duplicate }, fetchResults, @@ -1245,13 +1247,13 @@ printf '%s\\n' \ expect(report.githubSummary).toContain("ClawSweeper GitHub App installation"); } }, - 55_000, ); posixIt( "preserves both reports when concurrent writers update one latest pointer", - async () => { + async ({ signal }) => { const report = await runCiGitStep({ + signal, workflow: { file: WORKFLOW, job: "publish", step: "Publish to clawgrit reports" }, performance: { mode: "publish", race: true }, fetchResults: [], diff --git a/test/scripts/plugin-release-git-lifecycle.test.ts b/test/scripts/plugin-release-git-lifecycle.test.ts index 32515336af6b..03cebed27564 100644 --- a/test/scripts/plugin-release-git-lifecycle.test.ts +++ b/test/scripts/plugin-release-git-lifecycle.test.ts @@ -106,9 +106,10 @@ const modes: Record< }, }; -function pluginRun(mode: PluginMode, options: RunOptions = {}) { +function pluginRun(signal: AbortSignal, mode: PluginMode, options: RunOptions = {}) { const selected = modes[mode]; return runCiGitStep({ + signal, workflow: selected.workflow, fetchResults: [], ...options, @@ -121,7 +122,7 @@ function gitCommands(report: Awaited>) { return report.commands.filter(({ tool }) => tool === "git").map(({ args }) => args); } -posixIt.each([ +posixIt.for([ { mode: "clawhub-resolve" as const, commands: [ @@ -162,8 +163,9 @@ posixIt.each([ }, ])( "$mode drains every Git tree before success or output", - async ({ commands, mode, output }) => { - const report = await pluginRun(mode); + { timeout: 55_000 }, + async ({ commands, mode, output }, { signal }) => { + const report = await pluginRun(signal, mode); expect(report.code, report.output).toBe(0); expect(gitCommands(report)).toEqual(commands); expect(report.githubOutput).toBe(output); @@ -172,14 +174,14 @@ posixIt.each([ expect(report.boundaries.some(({ name }) => name === "output")).toBe(true); } }, - 55_000, ); -posixIt.each(["npm-preflight-read", "npm-publish-read"] as const)( +posixIt.for(["npm-preflight-read", "npm-publish-read"] as const)( "%s preserves exact source package bytes before the next consumer", - async (mode) => { + { timeout: 55_000 }, + async (mode, { signal }) => { const sourceRef = mode === "npm-preflight-read" ? "SOURCE_SHA" : "TARGET_SHA"; - const report = await pluginRun(mode, { + const report = await pluginRun(signal, mode, { commandResults: { [`show ${sha}:${packageDir}/package.json`]: { code: 0, output: packageJson }, }, @@ -192,13 +194,13 @@ posixIt.each(["npm-preflight-read", "npm-publish-read"] as const)( expect(report.pluginSourcePackage).toBe(packageJson); expect(modes[mode].env[sourceRef]).toBe(sha); }, - 55_000, ); -posixIt.each(["npm-preflight-read", "npm-publish-read"] as const)( +posixIt.for(["npm-preflight-read", "npm-publish-read"] as const)( "%s rejects partial source package output after ordinary show failure", - async (mode) => { - const report = await pluginRun(mode, { + { timeout: 55_000 }, + async (mode, { signal }) => { + const report = await pluginRun(signal, mode, { commandResults: { [`show ${sha}:${packageDir}/package.json`]: { code: 23, output: "{partial" }, }, @@ -207,28 +209,28 @@ posixIt.each(["npm-preflight-read", "npm-publish-read"] as const)( expect(gitCommands(report).at(-1)?.[0]).toBe("show"); expect(report.pluginSourcePackage).toBe(""); }, - 55_000, ); -posixIt.each( +posixIt.for( (["npm-preflight-read", "npm-publish-read"] as const).flatMap((mode) => ([23, 125, "hang"] as const).map((failure) => ({ failure, mode })), ), )( "$mode fetch failure $failure stops before source package readback", - async ({ failure, mode }) => { - const report = await pluginRun(mode, { fetchResults: [failure] }); + { timeout: 55_000 }, + async ({ failure, mode }, { signal }) => { + const report = await pluginRun(signal, mode, { fetchResults: [failure] }); expect(report.code, report.output).toBe(failure === "hang" ? 124 : failure); expect(gitCommands(report).at(-1)?.[0]).toBe("fetch"); expect(report.pluginSourcePackage).toBe(""); }, - 55_000, ); -posixIt.each([1, 125, 143])( +posixIt.for([1, 125, 143])( "ClawHub resolves origin fallback after safely drained ordinary local probe failure %s", - async (code) => { - const report = await pluginRun("clawhub-resolve", { + { timeout: 55_000 }, + async (code, { signal }) => { + const report = await pluginRun(signal, "clawhub-resolve", { env: { TARGET_REF: "release/fixture" }, commandResults: { "rev-parse --verify --quiet release/fixture^{commit}": { code, output: "" }, @@ -252,13 +254,12 @@ posixIt.each([1, 125, 143])( ]); expect(report.githubOutput).toBe(`sha=${sha}\n`); }, - 55_000, ); posixIt( "ClawHub release tags retain their second bounded fetch", - async () => { - const report = await pluginRun("clawhub-resolve", { + async ({ signal }) => { + const report = await pluginRun(signal, "clawhub-resolve", { env: { RELEASE_TAG: releaseTag }, }); expect(report.code, report.output).toBe(0); @@ -278,8 +279,8 @@ posixIt( posixIt( "ClawHub protected tooling validates the exact peeled release target", - async () => { - const report = await pluginRun("clawhub-oidc", { + async ({ signal }) => { + const report = await pluginRun(signal, "clawhub-oidc", { env: { RELEASE_PUBLISH_RUN_ATTEMPT: "2", RELEASE_PUBLISH_RUN_ID: "123", @@ -299,10 +300,11 @@ posixIt( 55_000, ); -posixIt.each([1, 125])( +posixIt.for([1, 125])( "ClawHub protected tag ordinary lookup failure %s retains OIDC rejection", - async (code) => { - const report = await pluginRun("clawhub-oidc", { + { timeout: 55_000 }, + async (code, { signal }) => { + const report = await pluginRun(signal, "clawhub-oidc", { env: { RELEASE_PUBLISH_RUN_ATTEMPT: "2", RELEASE_PUBLISH_RUN_ID: "123", @@ -318,13 +320,13 @@ posixIt.each([1, 125])( "Plugin ClawHub OIDC publish target is not bound to protected tooling and the exact release tag.", ); }, - 55_000, ); -posixIt.each(["clawhub-trust", "npm-trust"] as const)( +posixIt.for(["clawhub-trust", "npm-trust"] as const)( "%s accepts a release branch only after successful enumeration", - async (mode) => { - const report = await pluginRun(mode, { + { timeout: 55_000 }, + async (mode, { signal }) => { + const report = await pluginRun(signal, mode, { commandResults: { "merge-base --is-ancestor HEAD origin/main": { code: 1 }, "for-each-ref --format=%(refname) refs/remotes/origin/release": { @@ -342,13 +344,12 @@ posixIt.each(["clawhub-trust", "npm-trust"] as const)( "refs/remotes/origin/release/2026.8.1", ]); }, - 55_000, ); posixIt( "npm-trust rejects a Tideclaw alpha publish after main and release misses", - async () => { - const report = await pluginRun("npm-trust", { + async ({ signal }) => { + const report = await pluginRun(signal, "npm-trust", { env: { WORKFLOW_REF: `refs/heads/${alphaBranch}` }, commandResults: { "merge-base --is-ancestor HEAD origin/main": { code: 1 }, @@ -366,8 +367,8 @@ posixIt( posixIt( "clawhub-trust rejects a retired Tideclaw alpha branch before ancestry admission", - async () => { - const report = await pluginRun("clawhub-trust", { + async ({ signal }) => { + const report = await pluginRun(signal, "clawhub-trust", { env: { TRUSTED_PUBLISH_BRANCH: alphaBranch }, commandResults: { "merge-base --is-ancestor HEAD origin/main": { code: 1 }, @@ -382,11 +383,12 @@ posixIt( 55_000, ); -posixIt.each(["refs/heads/extended-stable/2026.8.33", "refs/heads/main"])( +posixIt.for(["refs/heads/extended-stable/2026.8.33", "refs/heads/main"])( "npm extended-stable preflight retains exact-tip admission from %s", - async (workflowRef) => { + { timeout: 55_000 }, + async (workflowRef, { signal }) => { const branch = "extended-stable/2026.8.33"; - const report = await pluginRun("npm-trust", { + const report = await pluginRun(signal, "npm-trust", { env: { PREFLIGHT_ONLY: "true", NPM_DIST_TAG: "extended-stable", @@ -407,7 +409,6 @@ posixIt.each(["refs/heads/extended-stable/2026.8.33", "refs/heads/main"])( // Preflight adds its exact-source check before the extended-stable tip check. expect(gitCommands(report).filter(([operation]) => operation === "rev-parse")).toHaveLength(6); }, - 55_000, ); const candidateAdmissionCases: Array<{ @@ -471,10 +472,11 @@ const candidateAdmissionCases: Array<{ }, ]; -posixIt.each(candidateAdmissionCases)( +posixIt.for(candidateAdmissionCases)( "npm canonical candidate admission: $name", - async ({ env, commands, code, message }) => { - const report = await pluginRun("npm-trust", { + { timeout: 55_000 }, + async ({ env, commands, code, message }, { signal }) => { + const report = await pluginRun(signal, "npm-trust", { env: { NPM_DIST_TAG: "extended-stable", PUBLISH_SCOPE: "all-publishable", @@ -497,18 +499,18 @@ posixIt.each(candidateAdmissionCases)( ]); } }, - 55_000, ); -posixIt.each([ +posixIt.for([ ["moved canonical tip", "refs/heads/main", "c".repeat(40)], ["untrusted workflow branch", "refs/heads/topic", sha], ["same-name main tag", "refs/tags/main", sha], -])( +] as const)( "npm extended-stable preflight rejects %s", - async (_name, workflowRef, branchSha) => { + { timeout: 55_000 }, + async ([_name, workflowRef, branchSha], { signal }) => { const branch = "extended-stable/2026.8.33"; - const report = await pluginRun("npm-trust", { + const report = await pluginRun(signal, "npm-trust", { env: { PREFLIGHT_ONLY: "true", NPM_DIST_TAG: "extended-stable", @@ -524,13 +526,12 @@ posixIt.each([ expect(report.code, report.output).toBe(1); expect(report.output).toContain("Extended-stable plugin"); }, - 55_000, ); posixIt( "npm preflight rejects before Tideclaw fallback after main and release misses", - async () => { - const report = await pluginRun("npm-trust", { + async ({ signal }) => { + const report = await pluginRun(signal, "npm-trust", { env: { PREFLIGHT_ONLY: "true", SOURCE_REF: sha, WORKFLOW_REF: `refs/heads/${alphaBranch}` }, revisions: { [`${sha}^{commit}`]: sha }, commandResults: { @@ -548,23 +549,24 @@ posixIt( 55_000, ); -posixIt.each(["clawhub-trust", "npm-trust"] as const)( +posixIt.for(["clawhub-trust", "npm-trust"] as const)( "%s treats merge-base errors other than ordinary 1 as terminal", - async (mode) => { - const report = await pluginRun(mode, { + { timeout: 55_000 }, + async (mode, { signal }) => { + const report = await pluginRun(signal, mode, { commandResults: { "merge-base --is-ancestor HEAD origin/main": { code: 23 } }, }); expect(report.code, report.output).toBe(23); expect(gitCommands(report)).toHaveLength(mode === "npm-trust" ? 2 : 1); expect(report.fetches).toHaveLength(mode === "npm-trust" ? 1 : 0); }, - 55_000, ); -posixIt.each(["clawhub-trust", "npm-trust"] as const)( +posixIt.for(["clawhub-trust", "npm-trust"] as const)( "%s treats release-ref enumeration failure as terminal", - async (mode) => { - const report = await pluginRun(mode, { + { timeout: 55_000 }, + async (mode, { signal }) => { + const report = await pluginRun(signal, mode, { commandResults: { "merge-base --is-ancestor HEAD origin/main": { code: 1 }, "for-each-ref --format=%(refname) refs/remotes/origin/release": { code: 23 }, @@ -574,7 +576,6 @@ posixIt.each(["clawhub-trust", "npm-trust"] as const)( expect(gitCommands(report).at(-1)?.[0]).toBe("for-each-ref"); expect(report.fetches).toHaveLength(mode === "npm-trust" ? 1 : 0); }, - 55_000, ); const terminalCases: Array<{ @@ -657,7 +658,7 @@ const terminalCases: Array<{ }, ]; -posixIt.each( +posixIt.for( terminalCases.flatMap((entry) => (["cleanup-failure", "cancel"] as const).map((failure) => Object.assign({}, entry, { failure }), @@ -665,8 +666,9 @@ posixIt.each( ), )( "$mode $operation $failure fences every later Git/output/consumer boundary", - async ({ commandResults, env, failure, match, mode, operation, revisions }) => { - const report = await pluginRun(mode, { + { timeout: 55_000 }, + async ({ commandResults, env, failure, match, mode, operation, revisions }, { signal }) => { + const report = await pluginRun(signal, mode, { commandResults, env, revisions, @@ -680,20 +682,19 @@ posixIt.each( expect(report.githubOutput).toBe(""); expect(report.commands.some(({ tool }) => ["node", "pnpm"].includes(tool))).toBe(false); }, - 55_000, ); -posixIt.each( +posixIt.for( (["clawhub-resolve", "npm-resolve", "npm-preflight-read", "npm-publish-read"] as const).flatMap( (mode) => (["owner", "python", "git"] as const).map((setupFailure) => ({ mode, setupFailure })), ), )( "$mode setup failure $setupFailure cannot publish or consume Git output", - async ({ mode, setupFailure }) => { - const report = await pluginRun(mode, { setupFailure }); + { timeout: 55_000 }, + async ({ mode, setupFailure }, { signal }) => { + const report = await pluginRun(signal, mode, { setupFailure }); expect(report.code, report.output).not.toBe(0); expect(report.githubOutput).toBe(""); expect(report.commands.some(({ tool }) => ["node", "pnpm"].includes(tool))).toBe(false); }, - 55_000, ); diff --git a/test/scripts/release-workflow-git-lifecycle.test.ts b/test/scripts/release-workflow-git-lifecycle.test.ts index c0ac8352e725..1d13c9e8bb3d 100644 --- a/test/scripts/release-workflow-git-lifecycle.test.ts +++ b/test/scripts/release-workflow-git-lifecycle.test.ts @@ -51,9 +51,10 @@ const releases: Record< }, }; -function releaseRun(mode: ReleaseMode, options: RunOptions = {}) { +function releaseRun(signal: AbortSignal, mode: ReleaseMode, options: RunOptions = {}) { const release = releases[mode]; return runCiGitStep({ + signal, workflow: release.workflow, fetchResults: [], ...options, @@ -66,10 +67,11 @@ function gitCommands(report: Awaited>) { return report.commands.filter(({ tool }) => tool === "git").map(({ args }) => args); } -posixIt.each([releaseTag, `${releaseTag}-2`])( +posixIt.for([releaseTag, `${releaseTag}-2`])( "Linux admits a stable tag from its matching release branch: %s", - async (tag) => { - const report = await releaseRun("linux", { + { timeout: 55_000 }, + async (tag, { signal }) => { + const report = await releaseRun(signal, "linux", { env: { RELEASE_TAG: tag }, revisions: { [`refs/tags/${tag}^{commit}`]: sha }, commandResults: { @@ -86,10 +88,9 @@ posixIt.each([releaseTag, `${releaseTag}-2`])( "+refs/heads/release/2026.8.1:refs/remotes/origin/release/2026.8.1", ]); }, - 55_000, ); -posixIt.each([ +posixIt.for([ { mode: "linux" as const, commands: [ @@ -120,8 +121,9 @@ posixIt.each([ }, ])( "$mode admission drains every Git tree before output or consumer", - async ({ mode, commands, output }) => { - const report = await releaseRun(mode); + { timeout: 55_000 }, + async ({ mode, commands, output }, { signal }) => { + const report = await releaseRun(signal, mode); expect(report.code, report.output).toBe(0); expect(gitCommands(report)).toEqual(commands); expect(report.githubOutput).toBe(output); @@ -135,33 +137,33 @@ posixIt.each([ expect(report.boundaries.some(({ name }) => name === "output")).toBe(true); } }, - 55_000, ); -posixIt.each( +posixIt.for( (["linux", "macos", "placeholder"] as const).flatMap((mode) => ([23, 125, "hang"] as const).map((failure) => ({ failure, mode })), ), )( "$mode fetch failure $failure stops before output or consumer", - async ({ failure, mode }) => { - const report = await releaseRun(mode, { fetchResults: [failure] }); + { timeout: 55_000 }, + async ({ failure, mode }, { signal }) => { + const report = await releaseRun(signal, mode, { fetchResults: [failure] }); expect(report.code, report.output).toBe(failure === "hang" ? 124 : failure); expect(gitCommands(report).at(-1)?.[0]).toBe("fetch"); expect(report.githubOutput).toBe(""); expect(report.commands.some(({ tool }) => tool === "pnpm")).toBe(false); }, - 55_000, ); -posixIt.each( +posixIt.for( (["linux", "macos"] as const).flatMap((mode) => ([23, 125] as const).map((code) => ({ code, mode })), ), )( "$mode ordinary rev-parse status $code remains terminal", - async ({ code, mode }) => { - const report = await releaseRun(mode, { + { timeout: 55_000 }, + async ({ code, mode }, { signal }) => { + const report = await releaseRun(signal, mode, { gitFault: { match: "^rev-parse ", code }, }); expect(report.code, report.output).toBe(code); @@ -169,13 +171,12 @@ posixIt.each( expect(report.githubOutput).toBe(""); expect(report.commands.some(({ tool }) => tool === "pnpm")).toBe(false); }, - 55_000, ); posixIt( "macOS rejects an invalid public branch before any Git command", - async () => { - const report = await releaseRun("macos", { + async ({ signal }) => { + const report = await releaseRun(signal, "macos", { env: { PUBLIC_RELEASE_BRANCH: "feature/not-a-release" }, }); expect(report.code, report.output).toBe(1); @@ -210,7 +211,7 @@ const terminalOperations = [ }, ]; -posixIt.each( +posixIt.for( terminalOperations.flatMap((entry) => (["cleanup-failure", "cancel"] as const).map((failure) => Object.assign({}, entry, { failure }), @@ -218,8 +219,9 @@ posixIt.each( ), )( "$mode $operation $failure is terminal before every later boundary", - async ({ failure, match, mode, occurrence, operation }) => { - const report = await releaseRun(mode, { + { timeout: 55_000 }, + async ({ failure, match, mode, occurrence, operation }, { signal }) => { + const report = await releaseRun(signal, mode, { gitFault: { match, occurrence, code: failure }, }); expect(report.code, report.output).toBe(failure === "cancel" ? 143 : 125); @@ -228,26 +230,25 @@ posixIt.each( expect(report.commands.some(({ tool }) => tool === "pnpm")).toBe(false); expect(report.output).not.toMatch(/not reachable|requires ref to equal/u); }, - 55_000, ); -posixIt.each([23, 125])( +posixIt.for([23, 125])( "Linux ordinary merge-base status %s is terminal without trying another branch", - async (code) => { - const report = await releaseRun("linux", { + { timeout: 55_000 }, + async (code, { signal }) => { + const report = await releaseRun(signal, "linux", { gitFault: { match: "^merge-base ", code }, }); expect(report.code, report.output).toBe(code); expect(gitCommands(report).at(-1)?.[0]).toBe("merge-base"); expect(report.githubOutput).toBe(""); }, - 55_000, ); posixIt( "Linux rejects a tag outside main and its matching release branch", - async () => { - const report = await releaseRun("linux", { + async ({ signal }) => { + const report = await releaseRun(signal, "linux", { commandResults: { [`merge-base --is-ancestor ${sha} origin/main`]: { code: 1 }, [`merge-base --is-ancestor ${sha} refs/remotes/origin/release/2026.8.1`]: { code: 1 }, @@ -264,8 +265,8 @@ posixIt( posixIt( "Linux rejects tooling outside main before inspecting the candidate", - async () => { - const report = await releaseRun("linux", { + async ({ signal }) => { + const report = await releaseRun(signal, "linux", { commandResults: { [`merge-base --is-ancestor ${otherSha} origin/main`]: { code: 1 } }, }); expect(report.code, report.output).toBe(1); @@ -276,10 +277,11 @@ posixIt( 55_000, ); -posixIt.each([128, "cleanup-failure", "cancel"] as const)( +posixIt.for([128, "cleanup-failure", "cancel"] as const)( "Linux matching release branch fetch failure %s cannot admit a stale ref", - async (failure) => { - const report = await releaseRun("linux", { + { timeout: 55_000 }, + async (failure, { signal }) => { + const report = await releaseRun(signal, "linux", { commandResults: { [`merge-base --is-ancestor ${sha} origin/main`]: { code: 1 } }, gitFault: { match: "^fetch ", occurrence: 2, code: failure }, }); @@ -289,16 +291,16 @@ posixIt.each([128, "cleanup-failure", "cancel"] as const)( expect(gitCommands(report).at(-1)?.[0]).toBe("fetch"); expect(report.githubOutput).toBe(""); }, - 55_000, ); -posixIt.each([ +posixIt.for([ { occurrence: 1, message: "workflow revision is not reachable" }, { occurrence: 2, message: "target must be reachable" }, ])( "placeholder ordinary merge-base failure $occurrence keeps its custom rejection", - async ({ message, occurrence }) => { - const report = await releaseRun("placeholder", { + { timeout: 55_000 }, + async ({ message, occurrence }, { signal }) => { + const report = await releaseRun(signal, "placeholder", { gitFault: { match: "^merge-base ", occurrence, code: 23 }, }); expect(report.code, report.output).toBe(1); @@ -308,13 +310,13 @@ posixIt.each([ ); expect(report.githubOutput).toBe(""); }, - 55_000, ); -posixIt.each([23, 125])( +posixIt.for([23, 125])( "placeholder rev-parse status %s retains exact-SHA rejection", - async (code) => { - const report = await releaseRun("placeholder", { + { timeout: 55_000 }, + async (code, { signal }) => { + const report = await releaseRun(signal, "placeholder", { gitFault: { match: "^rev-parse ", code }, }); expect(report.code, report.output).toBe(1); @@ -324,7 +326,6 @@ posixIt.each([23, 125])( expect(gitCommands(report)).toHaveLength(1); expect(report.githubOutput).toBe(""); }, - 55_000, ); const placeholderIdentityMismatches: Array<{ @@ -341,22 +342,22 @@ const placeholderIdentityMismatches: Array<{ }, ]; -posixIt.each(placeholderIdentityMismatches)( +posixIt.for(placeholderIdentityMismatches)( "placeholder rejects non-Git identity mismatch before checkout inspection", - async ({ env, message }) => { - const report = await releaseRun("placeholder", { env }); + { timeout: 55_000 }, + async ({ env, message }, { signal }) => { + const report = await releaseRun(signal, "placeholder", { env }); expect(report.code, report.output).toBe(1); expect(report.output).toContain(message); expect(gitCommands(report)).toEqual([]); expect(report.githubOutput).toBe(""); }, - 55_000, ); posixIt( "placeholder rejects a checked-out SHA mismatch before fetch", - async () => { - const report = await releaseRun("placeholder", { + async ({ signal }) => { + const report = await releaseRun(signal, "placeholder", { commandResults: { "rev-parse HEAD": { code: 0, output: `${otherSha}\n` } }, }); expect(report.code, report.output).toBe(1); @@ -369,17 +370,17 @@ posixIt( 55_000, ); -posixIt.each( +posixIt.for( (["linux", "macos", "placeholder"] as const).flatMap((mode) => (["owner", "python", "git"] as const).map((setupFailure) => ({ mode, setupFailure })), ), )( "$mode setup failure $setupFailure cannot publish or consume admission", - async ({ mode, setupFailure }) => { - const report = await releaseRun(mode, { setupFailure }); + { timeout: 55_000 }, + async ({ mode, setupFailure }, { signal }) => { + const report = await releaseRun(signal, mode, { setupFailure }); expect(report.code, report.output).not.toBe(0); expect(report.githubOutput).toBe(""); expect(report.commands.some(({ tool }) => tool === "pnpm")).toBe(false); }, - 55_000, );