diff --git a/docs/help/testing/suites.md b/docs/help/testing/suites.md index 04fb513cefde..885f674b83fb 100644 --- a/docs/help/testing/suites.md +++ b/docs/help/testing/suites.md @@ -141,6 +141,9 @@ Native dependency policy: the remaining value must use plain decimal digits without leading zeros, so values such as `1e5` or `007` are rejected. Unset the variable to disable the watchdog. + Canceling a compiler shard batch joins each compiler, including forced + termination when needed, before releasing checkout artifact ownership so + the next build or check can proceed. diff --git a/scripts/run-tsgo-core-test-shards.mts b/scripts/run-tsgo-core-test-shards.mts index 408613fca72c..8659bb008865 100644 --- a/scripts/run-tsgo-core-test-shards.mts +++ b/scripts/run-tsgo-core-test-shards.mts @@ -6,12 +6,8 @@ import { existsSync, realpathSync } from "node:fs"; import path from "node:path"; import type { CoreTsgoGraph } from "./check-tsgo-core-boundary.mts"; import { isDirectRunUrl } from "./lib/direct-run.mjs"; -import { - distArtifactEntryArgs, - withDistArtifactOwnership, -} from "./lib/dist-artifact-ownership.mts"; +import { withDistArtifactOwnership } from "./lib/dist-artifact-ownership.mts"; import { resolveLocalCheckEnv } from "./lib/local-check-runtime.mts"; -import { runManagedCommand } from "./lib/managed-child-process.mts"; import { resolveRepoRoot } from "./lib/repo-root.mjs"; import { selectTsgoCoreTestShards, @@ -22,31 +18,28 @@ import { TSGO_CORE_TEST_SHARDS, selectTsgoCoreTestStripe, } from "./lib/tsgo-core-test-shards.mts"; +import { prepareTsgoCommand, runPreparedTsgoCommand } from "./run-tsgo.mts"; const repoRoot = resolveRepoRoot(import.meta.url); -function runShard(config: string, env: NodeJS.ProcessEnv): Promise { - return runManagedCommand({ - bin: process.execPath, - shell: false, - args: distArtifactEntryArgs( - path.join(repoRoot, "scripts/run-tsgo.mts"), - // These graphs have no project references. Project mode rechecks root - // membership even when a restored build-info file is newer than a new root. - [ - "-p", - config, - "--incremental", - // The package command pins this config's cache at the repository root. - ...(config === "test/tsconfig/tsconfig.test.root.json" - ? ["--tsBuildInfoFile", ".artifacts/tsgo-cache/test-root.tsbuildinfo"] - : []), - ], - { native: true }, - ), - cwd: repoRoot, +async function runShard(config: string, env: NodeJS.ProcessEnv): Promise { + const command = prepareTsgoCommand( + // These graphs have no project references. Project mode rechecks root + // membership even when a restored build-info file is newer than a new root. + [ + "-p", + config, + "--incremental", + // The package command pins this config's cache at the repository root. + ...(config === "test/tsconfig/tsconfig.test.root.json" + ? ["--tsBuildInfoFile", ".artifacts/tsgo-cache/test-root.tsbuildinfo"] + : []), + ], env, - requireProcessTreeExit: process.platform !== "win32", - }); + repoRoot, + ); + // One owner joins each native compiler. A second wrapper's escalation deadline + // can kill the cleanup owner before it releases its artifact claim. + return command ? await runPreparedTsgoCommand(command) : 0; } /** Runs selected canonical graphs under the same output and child-process owner. */ diff --git a/scripts/run-tsgo.mts b/scripts/run-tsgo.mts index 23168107a783..7492c5131151 100644 --- a/scripts/run-tsgo.mts +++ b/scripts/run-tsgo.mts @@ -80,6 +80,32 @@ export function prepareTsgoCommand( }; } +/** The caller holds artifact ownership until this compiler and its output are joined. */ +export async function runPreparedTsgoCommand( + command: NonNullable>, +): Promise { + try { + const tsBuildInfoFile = readFlagValue(command.args, "--tsBuildInfoFile"); + if (tsBuildInfoFile) { + fs.mkdirSync(path.dirname(path.resolve(command.cwd, tsBuildInfoFile)), { recursive: true }); + } + // Managed cleanup forwards SIGTERM before bounded SIGKILL escalation, then + // joins the compiler group and output before reporting a timeout. + return await runManagedCommand({ + ...command, + requireProcessTreeExit: process.platform !== "win32", + }); + } catch (error) { + if ((error as { code?: string } | undefined)?.code !== "ETIMEDOUT") { + throw error; + } + console.error( + `[tsgo] no completion after ${command.timeoutMs}ms; killed the tsgo process tree. Raise OPENCLAW_TSGO_TIMEOUT_MS for intentionally longer builds, or unset it to disable the watchdog.`, + ); + return 1; + } +} + async function main(): Promise { let command: ReturnType; try { @@ -92,31 +118,11 @@ async function main(): Promise { if (!command) { return; } - try { - // Preflight must refuse or skip before installed bootstrap dependencies load. - // Output mutation and the compiler still remain inside their checkout owner. - const { withDistArtifactOwnership } = await import("./lib/dist-artifact-ownership.mts"); - await withDistArtifactOwnership(command.cwd, async () => { - const tsBuildInfoFile = readFlagValue(command.args, "--tsBuildInfoFile"); - if (tsBuildInfoFile) { - fs.mkdirSync(path.dirname(path.resolve(command.cwd, tsBuildInfoFile)), { recursive: true }); - } - // Managed cleanup forwards SIGTERM before bounded SIGKILL escalation, then - // joins the compiler group and output before reporting a timeout. - process.exitCode = await runManagedCommand({ - ...command, - requireProcessTreeExit: process.platform !== "win32", - }); - }); - } catch (error) { - if ((error as { code?: string } | undefined)?.code !== "ETIMEDOUT") { - throw error; - } - console.error( - `[tsgo] no completion after ${command.timeoutMs}ms; killed the tsgo process tree. Raise OPENCLAW_TSGO_TIMEOUT_MS for intentionally longer builds, or unset it to disable the watchdog.`, - ); - process.exitCode = 1; - } + // Preflight must refuse or skip before installed bootstrap dependencies load. + const { withDistArtifactOwnership } = await import("./lib/dist-artifact-ownership.mts"); + process.exitCode = await withDistArtifactOwnership(command.cwd, () => + runPreparedTsgoCommand(command), + ); } if (isDirectRunUrl(process.argv[1], import.meta.url)) { diff --git a/test/scripts/dist-artifact-fixture.ts b/test/scripts/dist-artifact-fixture.ts new file mode 100644 index 000000000000..c1baa831a04e --- /dev/null +++ b/test/scripts/dist-artifact-fixture.ts @@ -0,0 +1,48 @@ +import fs from "node:fs"; +import path from "node:path"; +import { materializeNativeCompiler } from "./native-boundary-fixture.js"; + +function write(root: string, relative: string, content: string) { + const target = path.join(root, relative); + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.writeFileSync(target, content); + return target; +} + +export function installDistArtifactScripts( + root: string, + scripts: string[], + { compiler = true, dependencies = ["tsx", "@openclaw/fs-safe"] } = {}, +) { + const sourceRoot = process.cwd(); + // Keep the checkpoint launcher when installCompiler already owns this toolchain. + if (compiler && !fs.existsSync(path.join(root, "node_modules/typescript/package.json"))) { + materializeNativeCompiler(root); + } + for (const script of ["tsx.mjs", ...scripts]) { + write( + root, + `scripts/${script}`, + fs.readFileSync(path.join(sourceRoot, "scripts", script), "utf8"), + ); + } + for (const file of [ + "scripts/lib", + "scripts/windows-cmd-helpers.mjs", + "packages/normalization-core/src", + "packages/normalization-core/package.json", + ]) { + fs.mkdirSync(path.dirname(path.join(root, file)), { recursive: true }); + fs.cpSync(path.join(sourceRoot, file), path.join(root, file), { recursive: true }); + } + write(root, "scripts/lib/plugin-sdk-entrypoints.json", '["qa-channel-protocol"]'); + fs.mkdirSync(path.join(root, "node_modules"), { recursive: true }); + for (const name of dependencies) { + fs.mkdirSync(path.dirname(path.join(root, "node_modules", name)), { recursive: true }); + fs.symlinkSync( + path.join(sourceRoot, "node_modules", name), + path.join(root, "node_modules", name), + process.platform === "win32" ? "junction" : "dir", + ); + } +} diff --git a/test/scripts/dist-artifact-ownership.test.ts b/test/scripts/dist-artifact-ownership.test.ts index c7c1a1c77493..8fdb0f496694 100644 --- a/test/scripts/dist-artifact-ownership.test.ts +++ b/test/scripts/dist-artifact-ownership.test.ts @@ -19,6 +19,7 @@ import { createVitestResourceOwner } from "../../scripts/lib/vitest-resource-own import { resolveTestNodeExecPath } from "../../src/test-utils/node-process.js"; import { createFixtureLifetime } from "../helpers/fixture-lifetime.js"; import { waitForDead } from "../helpers/process-wait.js"; +import { installDistArtifactScripts as installScripts } from "./dist-artifact-fixture.js"; import { materializeNativeCompiler, overrideNativeFixtureExecutable, @@ -100,43 +101,6 @@ function installBuildCheckpoint(root: string, checkpoint: string) { write(root, "pnpm.cjs", 'import("./node_modules/tsdown/dist/run.mjs");\n'); } -function installScripts( - root: string, - scripts: string[], - { compiler = true, dependencies = ["tsx", "@openclaw/fs-safe"] } = {}, -) { - // Keep the checkpoint launcher when installCompiler already owns this toolchain. - if (compiler && !fs.existsSync(path.join(root, "node_modules/typescript/package.json"))) { - materializeNativeCompiler(root); - } - for (const script of ["tsx.mjs", ...scripts]) { - write( - root, - `scripts/${script}`, - fs.readFileSync(path.join(sourceRoot, "scripts", script), "utf8"), - ); - } - for (const file of [ - "scripts/lib", - "scripts/windows-cmd-helpers.mjs", - "packages/normalization-core/src", - "packages/normalization-core/package.json", - ]) { - fs.mkdirSync(path.dirname(path.join(root, file)), { recursive: true }); - fs.cpSync(path.join(sourceRoot, file), path.join(root, file), { recursive: true }); - } - write(root, "scripts/lib/plugin-sdk-entrypoints.json", '["qa-channel-protocol"]'); - fs.mkdirSync(path.join(root, "node_modules"), { recursive: true }); - for (const name of dependencies) { - fs.mkdirSync(path.dirname(path.join(root, "node_modules", name)), { recursive: true }); - fs.symlinkSync( - path.join(sourceRoot, "node_modules", name), - path.join(root, "node_modules", name), - process.platform === "win32" ? "junction" : "dir", - ); - } -} - function withProcesses(...args: Parameters) { return fixture.run(() => runWithProcesses(...args)); } @@ -315,7 +279,12 @@ describe("native check launchers in paths with spaces", () => { const root = createCheckout("openclaw check launchers "); installScripts( root, - ["run-tsgo-core-test-shards.mts", "run-oxlint.mts", "run-oxlint-shards.mts"], + [ + "run-tsgo-core-test-shards.mts", + "run-tsgo.mts", + "run-oxlint.mts", + "run-oxlint-shards.mts", + ], { compiler: false, dependencies: ["tsx", "@openclaw/fs-safe", "json5", "p-map", "koffi"], @@ -325,17 +294,17 @@ describe("native check launchers in paths with spaces", () => { write(root, nativeJob, fs.readFileSync(path.join(sourceRoot, nativeJob), "utf8")); const compiler = script === "run-tsgo-core-test-shards.mts"; const workload = compiler - ? "run-tsgo.mts" - : "prepare-extension-package-boundary-artifacts.mts"; + ? "node_modules/typescript/compiler.mjs" + : "scripts/prepare-extension-package-boundary-artifacts.mts"; const observed = path.join(root, "child.json"); const settled = path.join(root, "child-settled"); const consumed = path.join(root, "lint-consumed"); // Keep the real CLI, artifact handoff and managed process owner. Only the // terminal compiler/preparation workload waits at this completion barrier. - write( + const workloadPath = write( root, - `scripts/${workload}`, - ` + workload, + `#!/usr/bin/env node import fs from 'node:fs'; import { createRequire } from 'node:module'; const require = createRequire(import.meta.url); @@ -348,6 +317,13 @@ describe("native check launchers in paths with spaces", () => { process.exitCode = ${exitCode}; `, ); + if (compiler) { + fs.chmodSync(workloadPath, 0o755); + overrideNativeFixtureExecutable(root, workloadPath); + if (process.platform === "win32") { + write(root, `${workload}.cmd`, `@"${testNodeExecPath}" "%~dp0compiler.mjs" %*\r\n`); + } + } const lint = write( root, "node_modules/.bin/oxlint", @@ -373,14 +349,16 @@ describe("native check launchers in paths with spaces", () => { const child: { argv: string[]; pid: number } = JSON.parse( fs.readFileSync(observed, "utf8"), ); - expect(child.argv).toEqual( + expect(compiler ? child.argv.slice(0, 3) : child.argv).toEqual( compiler ? ["-p", TSGO_CORE_TEST_SHARDS[0].config, "--incremental"] : ["--mode=package-boundary"], ); const lock = resolveDistArtifactLockPath(root); expect(fs.existsSync(path.join(lock, "owner.json"))).toBe(true); - expect(fs.readdirSync(lock)).toContain(`child-${child.pid}`); + if (!compiler) { + expect(fs.readdirSync(lock)).toContain(`child-${child.pid}`); + } expect(fs.existsSync(settled)).toBe(false); expect(fs.existsSync(consumed)).toBe(false); gate.write("continue"); diff --git a/test/scripts/tsgo-shard-cancellation.test.ts b/test/scripts/tsgo-shard-cancellation.test.ts new file mode 100644 index 000000000000..4db8cecac348 --- /dev/null +++ b/test/scripts/tsgo-shard-cancellation.test.ts @@ -0,0 +1,117 @@ +import type { ChildProcess } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { createInterface } from "node:readline"; +import { pathToFileURL } from "node:url"; +import { afterEach, expect, it } from "vitest"; +import { withDistArtifactOwnership } from "../../scripts/lib/dist-artifact-ownership.mts"; +import { runManagedCommand } from "../../scripts/lib/managed-child-process.mts"; +import { TSGO_CORE_TEST_SHARDS } from "../../scripts/lib/tsgo-core-test-shards.mts"; +import { createDeferredCore } from "../../src/shared/deferred.js"; +import { resolveTestNodeExecPath } from "../../src/test-utils/node-process.js"; +import { createFixtureLifetime } from "../helpers/fixture-lifetime.js"; +import { isProcessAlive, waitForDead } from "../helpers/process-wait.js"; +import { installDistArtifactScripts } from "./dist-artifact-fixture.js"; +import { overrideNativeFixtureExecutable } from "./native-boundary-fixture.js"; + +const lifetime = createFixtureLifetime(); +afterEach(() => lifetime.cleanup()); + +it.runIf(process.platform !== "win32")( + "releases shard artifacts after canceling a compiler that requires forced termination", + ({ signal }) => + lifetime.run(async () => { + const root = fs.realpathSync(lifetime.createTempDir("openclaw-cancel-shards-")); + fs.writeFileSync(path.join(root, "package.json"), '{"type":"module"}'); + fs.writeFileSync(path.join(root, "pnpm-workspace.yaml"), "packages: []\n"); + installDistArtifactScripts(root, ["run-tsgo-core-test-shards.mts", "run-tsgo.mts"], { + compiler: false, + dependencies: ["@openclaw/fs-safe"], + }); + const compiler = path.join(root, "compiler.cjs"); + fs.writeFileSync( + compiler, + `#!${resolveTestNodeExecPath()} +process.on('SIGTERM', () => {}); +process.stdin.resume(); +console.log(JSON.stringify({ pid: process.pid })); +`, + ); + fs.chmodSync(compiler, 0o755); + overrideNativeFixtureExecutable(root, compiler); + const clock = path.join(root, "supervisor-clock.mjs"); + // Accelerate every supervisor's existing grace period equally. Readiness and + // process completion still use real pipes and OS signals, without sleeps. + fs.writeFileSync( + clock, + `const now = Date.now.bind(Date), start = now(); +Date.now = () => start + (now() - start) * 10; +`, + ); + const ready = createDeferredCore<{ pid: number }>(); + let child: ChildProcess | undefined; + let output = ""; + let compilerPid: number | undefined; + const completion = lifetime.track( + runManagedCommand({ + bin: resolveTestNodeExecPath(), + args: [ + path.join(root, "scripts/run-tsgo-core-test-shards.mts"), + "--stripe", + `1/${TSGO_CORE_TEST_SHARDS.length}`, + ], + cwd: root, + env: { + ...process.env, + NODE_OPTIONS: `--import=${pathToFileURL(clock).href}`, + }, + stdio: ["pipe", "pipe", "pipe"], + signal, + requireProcessTreeExit: true, + onReady: (started) => { + child = started; + createInterface({ input: started.stdout! }).once("line", (line) => { + ready.resolve(JSON.parse(line) as { pid: number }); + }); + started.stderr!.on("data", (chunk) => { + output += String(chunk); + }); + }, + }), + ); + try { + const compilerReady = await Promise.race([ + ready.promise, + completion.then((code) => { + throw new Error(`Shards exited before compiler readiness: ${code}\n${output}`); + }), + ]); + compilerPid = compilerReady.pid; + if (!child) { + throw new Error("Missing shard process"); + } + child.kill("SIGTERM"); + expect(await completion, output).toBe(143); + expect(() => process.kill(compilerReady.pid, 0)).toThrow(); + expect(fs.readdirSync(path.join(root, ".artifacts/dist-artifacts.lock"))).toEqual([]); + await withDistArtifactOwnership(root, async () => { + fs.writeFileSync(path.join(root, "next-check"), "acquired"); + }); + expect(fs.readFileSync(path.join(root, "next-check"), "utf8")).toBe("acquired"); + } finally { + await lifetime.verifyCleanup(async () => { + child?.stdin?.end(); + try { + await completion; + } finally { + if (compilerPid !== undefined) { + if (isProcessAlive(compilerPid)) { + process.kill(compilerPid, "SIGKILL"); + } + await waitForDead(compilerPid, 2_000); + } + } + }); + } + }), +);