From cfe8791ca920d6a2d3e16a35196e7edd4d0130bb Mon Sep 17 00:00:00 2001 From: Shakker <165377636+shakkernerd@users.noreply.github.com> Date: Thu, 1 Oct 2026 16:08:26 +0100 Subject: [PATCH] fix: reuse prepared test runtimes after source refreshes (#162744) Reuse compatible test runtimes across source refreshes and test-only edits while preserving input invalidation and strict UI/deployment checks. Related: #57204. --- docs/reference/test/runner-internals.md | 9 + scripts/build-all.mts | 84 +++++- scripts/lib/local-build-metadata.mts | 38 ++- scripts/lib/run-node-input-state.mts | 247 +++++++++++++++++- scripts/lib/tsdown-output-roots.mts | 21 ++ scripts/lib/vitest-build-prerequisites.mts | 7 +- scripts/prepare-vitest-runtime.mts | 39 ++- scripts/run-node-watch-paths.mts | 8 +- scripts/run-node.mts | 140 +++++++++- scripts/tsdown-build.mts | 22 +- src/infra/build-stamp.test.ts | 226 +++++++++++++++- test/scripts/build-all.test.ts | 69 +++++ test/scripts/prepare-vitest-runtime.test.ts | 14 + test/scripts/runtime-postbuild-stamp.test.ts | 109 ++++++-- .../test-projects-build-admission.test.ts | 21 +- test/scripts/tsdown-declaration-fixture.ts | 1 + 16 files changed, 975 insertions(+), 80 deletions(-) diff --git a/docs/reference/test/runner-internals.md b/docs/reference/test/runner-internals.md index be94b282233b..ea3717953975 100644 --- a/docs/reference/test/runner-internals.md +++ b/docs/reference/test/runner-internals.md @@ -41,6 +41,15 @@ Runtime-consuming tests prepare checkout artifacts through the explicit build ow not by launching the CLI with `--version`. Preparation reuses source-runner freshness checks and checkout artifact ownership, without updater service or database-maintenance custody. Current artifacts need no writable checkout or service inspection. +`build-all` fingerprints production inputs in the existing stamps. Test preparation +can reuse a coherent runtime after a source refresh or test-only correction, +including a new private transport commit. Ordinary CLI and immutable deployment +HEAD checks remain strict; UI E2E preparation also keeps its current-head checks. +Changed production inputs, build configuration, dependencies, compiler identity, +or required missing outputs still require preparation. Partial postbuilds that skip +static assets cannot satisfy readers requiring those assets. A full build before E2E +checks should use `OPENCLAW_BUILD_PRIVATE_QA=1 pnpm build`; scope that flag to the +build command so its artifacts satisfy the strongest test prerequisite. Before writing, automatic preparation requires verified artifact separation or an observed offline managed Gateway. On Linux it reads the loaded command location without reading service diff --git a/scripts/build-all.mts b/scripts/build-all.mts index dc8c1b037ff0..f77430fbe822 100644 --- a/scripts/build-all.mts +++ b/scripts/build-all.mts @@ -1,7 +1,9 @@ #!/usr/bin/env node // Builds OpenClaw packages and plugin SDK artifacts with cache-aware orchestration. -import type { SpawnSyncOptions } from "node:child_process"; +import { spawnSync, type SpawnSyncOptions } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; import { performance } from "node:perf_hooks"; import { resolveNodeRuntimeExecutable } from "../src/infra/node-runtime-executable.ts"; import { @@ -18,8 +20,15 @@ import { } from "./lib/dist-artifact-ownership.mts"; import { formatDurationElapsed } from "./lib/format-duration.mts"; import { resolveLiveManagedGatewayDistFence } from "./lib/live-gateway-dist-fence.mts"; +import { + BUILD_STAMP_FILE, + RUNTIME_POSTBUILD_STAMP_FILE, + writeBuildStamp, + writeRuntimePostBuildStamp, +} from "./lib/local-build-metadata.mts"; import { runManagedCommand } from "./lib/managed-child-process.mts"; import type { MemoryLimitParams } from "./lib/process-memory.mts"; +import { captureRunNodeInputState } from "./lib/run-node-input-state.mts"; import { preflightInstalledSourceArtifacts } from "./lib/source-update-artifact-preflight.mts"; import { TSDOWN_PACKAGE_CONFIG_GROUP, @@ -571,10 +580,60 @@ export async function runBuildAllSteps( const resolveCacheState = params.resolveCacheState ?? resolveBuildStepCacheState; const restoreCache = params.restoreCache ?? restoreBuildStepCacheOutputs; const finalizeCache = params.finalizeCache ?? finalizeBuildStepCache; + const cwd = params.cwd ?? process.cwd(); + const inputDeps = { cwd, distRoot: path.join(cwd, "dist"), fs, env: buildEnv, spawnSync }; + const capturesNativeInputs = + !params.runStep && steps.some((step) => step.label.endsWith("build-stamp")); + const hasAssetBuild = + capturesNativeInputs && steps.some((step) => step.label === "plugins:assets:build"); + const assetInputState = hasAssetBuild + ? captureRunNodeInputState(inputDeps, "build", { assetPhase: true }) + : null; + let buildInputState = + capturesNativeInputs && !hasAssetBuild && steps.some((step) => step.label === "build-stamp") + ? captureRunNodeInputState(inputDeps, "build") + : null; + const runtimeEnv = { + ...buildEnv, + ...steps.find((step) => step.label === "runtime-postbuild")?.env, + }; + let runtimeInputState = + capturesNativeInputs && + !hasAssetBuild && + steps.some((step) => step.label === "runtime-postbuild-stamp") + ? captureRunNodeInputState({ ...inputDeps, env: runtimeEnv }, "runtime") + : null; + let stampsInvalidated = false; + const invalidateInputStamps = () => { + // Injected steps own their fixture writes; native writers share this lifecycle. + if ( + stampsInvalidated || + params.runStep || + !steps.some((step) => step.label.endsWith("build-stamp")) + ) { + return; + } + for (const name of [BUILD_STAMP_FILE, RUNTIME_POSTBUILD_STAMP_FILE]) { + fs.rmSync(path.join(cwd, "dist", name), { force: true }); + } + stampsInvalidated = true; + }; const runStep = params.runStep ?? (async (invocation: ReturnType) => { const script = invocation.args[2]; + if ( + script === "scripts/build-stamp.mts" || + script === "scripts/runtime-postbuild-stamp.mts" + ) { + const buildStamp = script === "scripts/build-stamp.mts"; + (buildStamp ? writeBuildStamp : writeRuntimePostBuildStamp)({ + cwd, + env: buildStamp ? buildEnv : runtimeEnv, + inputState: buildStamp ? buildInputState : runtimeInputState, + }); + return { status: 0 }; + } return { status: await runManagedCommand({ bin: invocation.command, @@ -609,8 +668,11 @@ export async function runBuildAllSteps( let stepToRun = step; let reusedCache = false; if (cacheEnabled && cacheState.fresh) { - if (cacheState.restorable && !restoreCache(cacheState)) { - throw new Error(`Build cache changed before restoration: ${step.label}; rerun the build`); + if (cacheState.restorable) { + invalidateInputStamps(); + if (!restoreCache(cacheState)) { + throw new Error(`Build cache changed before restoration: ${step.label}; rerun the build`); + } } const cacheHitStep = resolveBuildAllStepOnCacheHit(step); if (!cacheHitStep) { @@ -624,6 +686,7 @@ export async function runBuildAllSteps( } logger.error(`[build-all] ${step.label}${reusedCache ? " (cache restored)" : ""}`); const invocation = resolveBuildAllStep(stepToRun, { env: buildEnv }); + invalidateInputStamps(); const result = await runStep(invocation); params.signal?.throwIfAborted(); const durationMs = cacheDurationMs + now() - startedAt; @@ -633,6 +696,21 @@ export async function runBuildAllSteps( exitCode = typeof result.status === "number" ? result.status : 1; break; } + if (step.label === "plugins:assets:build" && !params.runStep) { + const current = captureRunNodeInputState(inputDeps, "build", { assetPhase: true }); + if ( + assetInputState && + (!current || + current.signature !== assetInputState.signature || + current.generation !== assetInputState.generation) + ) { + throw new Error("Build inputs changed during asset preparation; rerun the build"); + } + buildInputState = assetInputState ? captureRunNodeInputState(inputDeps, "build") : null; + runtimeInputState = assetInputState + ? captureRunNodeInputState({ ...inputDeps, env: runtimeEnv }, "runtime") + : null; + } // Runtime-only tsdown cleans its output roots. Cache hits restore // declarations again after that pass so the full build stays complete. if (!finalizeCache(step, cacheState, { env: buildEnv, reusedCache })) { diff --git a/scripts/lib/local-build-metadata.mts b/scripts/lib/local-build-metadata.mts index d72bc9f9cda5..a516ce1cf52c 100644 --- a/scripts/lib/local-build-metadata.mts +++ b/scripts/lib/local-build-metadata.mts @@ -4,7 +4,13 @@ import fs from "node:fs"; import path from "node:path"; import process from "node:process"; import { BUILD_STAMP_FILE, RUNTIME_POSTBUILD_STAMP_FILE } from "./local-build-metadata-paths.mts"; -import { hasDirtyRuntimePostBuildInputs, hasDirtySourceTree } from "./run-node-input-state.mts"; +import { + hasDirtyRuntimePostBuildInputs, + hasDirtySourceTree, + captureRunNodeInputState, + type RunNodeInputState, +} from "./run-node-input-state.mts"; +import { shouldCopyStaticExtensionAssets } from "./static-extension-assets.mts"; export { BUILD_STAMP_FILE, RUNTIME_POSTBUILD_STAMP_FILE }; @@ -19,6 +25,7 @@ type BuildMetadataParams = { fs?: typeof fs; env?: NodeJS.ProcessEnv; now?: () => number; + inputState?: RunNodeInputState | null; spawnSync?: BuildMetadataSpawnSync; }; @@ -36,6 +43,31 @@ function resolveInputsClean(params: BuildMetadataParams, scope: "build" | "runti return dirty === null ? null : !dirty; } +function verifiedInputSignature(params: BuildMetadataParams, scope: "build" | "runtime") { + if (!params.inputState) { + return {}; + } + const cwd = params.cwd ?? process.cwd(); + const current = captureRunNodeInputState( + { + cwd, + distRoot: path.join(cwd, "dist"), + fs: params.fs ?? fs, + env: params.env ?? process.env, + spawnSync: params.spawnSync ?? spawnSync, + }, + scope, + ); + if ( + !current || + current.signature !== params.inputState.signature || + current.generation !== params.inputState.generation + ) { + throw new Error("Build inputs changed during preparation; rerun the build"); + } + return { inputSignature: current.signature }; +} + /** Resolve the current git HEAD for build stamp metadata. */ export function resolveGitHead(params: BuildMetadataParams = {}) { const cwd = params.cwd ?? process.cwd(); @@ -71,7 +103,7 @@ export function writeBuildStamp(params: BuildMetadataParams = {}) { fsImpl.mkdirSync(distRoot, { recursive: true }); fsImpl.writeFileSync( buildStampPath, - `${JSON.stringify({ builtAt: now(), head, inputsClean: resolveInputsClean(params, "build") })}\n`, + `${JSON.stringify({ builtAt: now(), head, inputsClean: resolveInputsClean(params, "build"), ...verifiedInputSignature(params, "build") })}\n`, "utf8", ); return buildStampPath; @@ -97,6 +129,8 @@ export function writeRuntimePostBuildStamp(params: BuildMetadataParams = {}) { syncedAt: now(), ...(head ? { head } : {}), inputsClean: resolveInputsClean(params, "runtime"), + staticAssets: shouldCopyStaticExtensionAssets({ env: params.env ?? process.env }), + ...verifiedInputSignature(params, "runtime"), }, null, 2, diff --git a/scripts/lib/run-node-input-state.mts b/scripts/lib/run-node-input-state.mts index 1fb801ce00f7..b6e3555d30ba 100644 --- a/scripts/lib/run-node-input-state.mts +++ b/scripts/lib/run-node-input-state.mts @@ -1,16 +1,27 @@ import type { SpawnSyncOptionsWithStringEncoding } from "node:child_process"; // Canonical watched-input state shared by artifact producers and freshness readers. +import { createHash } from "node:crypto"; import type fs from "node:fs"; +import { createRequire } from "node:module"; import path from "node:path"; import { extensionRestartMetadataFiles, isBuildRelevantRunNodePath, + isIgnoredRunNodeSourcePath, normalizeRunNodePath as normalizePath, runNodeWatchedPaths, } from "../run-node-watch-paths.mts"; -import { collectSourceCheckoutPluginBuildEntries } from "./bundled-plugin-build-entries.mjs"; +import { + BUNDLED_PLUGIN_BUILD_ENV_NAMES, + collectSourceCheckoutPluginBuildEntries, +} from "./bundled-plugin-build-entries.mjs"; import { BUNDLED_PLUGIN_PATH_PREFIX, BUNDLED_PLUGIN_ROOT_DIR } from "./bundled-plugin-paths.mjs"; -import { listStaticExtensionAssetSources } from "./static-extension-assets.mts"; +import { isRecord } from "./record-shared.mjs"; +import { + listGeneratedExtensionAssetSources, + listStaticExtensionAssetSources, +} from "./static-extension-assets.mts"; +import { TSDOWN_PACKAGES_CACHE_INPUT } from "./tsdown-output-roots.mts"; export type RunNodeInputDeps = { cwd: string; @@ -55,7 +66,11 @@ const runtimePostBuildScriptPaths = new Set( ); const runtimePostBuildStaticAssetPaths = new Set(listStaticExtensionAssetSources()); -const readGitStatus = (deps: RunNodeInputDeps, paths: string[] = runNodeWatchedPaths) => { +const readGitStatus = ( + deps: RunNodeInputDeps, + paths: string[] = runNodeWatchedPaths, + untracked: "normal" | "all" = "normal", +) => { try { const result = deps.spawnSync( "git", @@ -65,7 +80,7 @@ const readGitStatus = (deps: RunNodeInputDeps, paths: string[] = runNodeWatchedP "--porcelain=v1", "-z", "--no-renames", - "--untracked-files=normal", + `--untracked-files=${untracked}`, "--", ...paths, ], @@ -73,6 +88,7 @@ const readGitStatus = (deps: RunNodeInputDeps, paths: string[] = runNodeWatchedP cwd: deps.cwd, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"], + maxBuffer: 16 * 1024 * 1024, }, ); if (result.status !== 0) { @@ -176,3 +192,226 @@ const isDirtyBundledPluginPackageEntryChangeWithoutBuiltOutputs = ( (filePath) => !deps.fs.existsSync(filePath), ); }; + +const buildToolPaths = [ + "scripts", + TSDOWN_PACKAGES_CACHE_INPUT.path, + "package.json", + "config", + ":(glob)packages/**/package.json", + ":(glob)extensions/*/package.json", + ":(glob)extensions/*/openclaw.plugin.json", + ":(glob)packages/**/tsconfig*.json", + "pnpm-lock.yaml", + "pnpm-workspace.yaml", + "node-version.mjs", + ".npmrc", + "patches", + ":(glob)tsconfig*.json", + ":(glob)tsdown*.ts", +]; +const isBuildToolPath = (file: string) => + file.startsWith("scripts/") || + file.startsWith("config/") || + file.startsWith("patches/") || + /^packages\/.+\/(?:package|tsconfig[^/]*)\.json$/u.test(file) || + /^extensions\/[^/]+\/(?:package|openclaw\.plugin)\.json$/u.test(file) || + buildToolPaths.includes(file) || + /^(?:tsconfig.*\.json|tsdown.*\.ts)$/u.test(file); + +function isRunNodeInputPath( + file: string, + scope: "build" | "runtime", + generatedAssets: ReadonlySet, +) { + const packageSource = + scope === "build" && + file.startsWith(`${TSDOWN_PACKAGES_CACHE_INPUT.path}/`) && + !file + .split("/") + .some((part) => TSDOWN_PACKAGES_CACHE_INPUT.excludeDirectories.includes(part)) && + TSDOWN_PACKAGES_CACHE_INPUT.extensions.some((extension) => file.endsWith(extension)) && + !isIgnoredRunNodeSourcePath(file); + return ( + packageSource || + (scope === "build" && + file.startsWith(BUNDLED_PLUGIN_PATH_PREFIX) && + !isIgnoredRunNodeSourcePath(file) && + ![...generatedAssets].some((output) => file === output || file.startsWith(`${output}/`))) || + isBuildToolPath(file) || + (scope === "build" ? isBuildRelevantRunNodePath(file) : isRuntimePostBuildRelevantPath(file)) + ); +} + +function listRunNodeInputFiles(deps: RunNodeInputDeps, scope: "build" | "runtime") { + const watched = scope === "build" ? runNodeWatchedPaths : runtimePostBuildWatchedPaths; + const result = deps.spawnSync( + "git", + [ + "ls-files", + "--cached", + "--others", + "--exclude-standard", + "-z", + "--", + ...watched, + ...buildToolPaths, + ], + { + cwd: deps.cwd, + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + maxBuffer: 16 * 1024 * 1024, + }, + ); + if (result.status !== 0) { + return null; + } + const generatedAssets = new Set( + listGeneratedExtensionAssetSources({ rootDir: deps.cwd, fs: deps.fs }), + ); + return [...new Set((result.stdout ?? "").split("\0"))] + .filter((file) => file && isRunNodeInputPath(file, scope, generatedAssets)) + .toSorted(); +} + +// A Control UI manifest contains both authored metadata and generated entry paths. +function generatedControlUiManifest(deps: RunNodeInputDeps, file: string) { + if (!/^extensions\/[^/]+\/openclaw\.plugin\.json$/u.test(file)) { + return false; + } + const packagePath = path.join(deps.cwd, path.dirname(file), "package.json"); + if (!deps.fs.existsSync(packagePath)) { + return false; + } + const pkg: unknown = JSON.parse(deps.fs.readFileSync(packagePath, "utf8")); + const openclaw = isRecord(pkg) && isRecord(pkg.openclaw) ? pkg.openclaw : {}; + const assets = isRecord(openclaw.assetScripts) ? openclaw.assetScripts : {}; + return ( + typeof openclaw.controlUi === "string" && + openclaw.controlUi.trim().length > 0 && + Array.isArray(assets.buildOutputs) && + assets.buildOutputs.includes("openclaw.plugin.json") + ); +} + +/** Identifies effective production inputs, independently of transport-carrier commits. */ +export function resolveRunNodeInputSignature( + deps: RunNodeInputDeps, + scope: "build" | "runtime", +): string | null { + try { + const files = listRunNodeInputFiles(deps, scope); + if (!files) { + return null; + } + const hash = createHash("sha256"); + hash.update( + JSON.stringify([ + 2, + scope, + process.version, + process.platform, + process.arch, + BUNDLED_PLUGIN_BUILD_ENV_NAMES.map((key) => [key, deps.env[key] ?? ""]), + ]), + ); + const capture = (file: string, optional = false, identity = file) => { + hash.update(`\0${identity}\0`); + const absolute = path.resolve(deps.cwd, file); + try { + const before = deps.fs.statSync(absolute); + const link = deps.fs.lstatSync(absolute); + let contents = deps.fs.readFileSync(absolute); + if (generatedControlUiManifest(deps, file)) { + const manifest: unknown = JSON.parse(contents.toString()); + if (!isRecord(manifest)) { + throw new Error(`Invalid plugin manifest: ${file}`); + } + const { controlUi: _generated, ...source } = manifest; + contents = Buffer.from(JSON.stringify(source)); + } + hash.update( + JSON.stringify([ + link.mode, + contents.length, + link.isSymbolicLink() ? deps.fs.readlinkSync(absolute) : null, + ]), + ); + hash.update(contents); + const after = deps.fs.statSync(absolute); + if ( + before.ctimeMs !== after.ctimeMs || + before.size !== after.size || + before.ino !== after.ino + ) { + throw new Error(`Build input changed while reading: ${file}`); + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT" || !optional) { + throw error; + } + hash.update("missing"); + } + }; + for (const file of files) { + capture(file, true); + } + // Keep the declared compiler installation in the identity without loading it. + capture("node_modules/.modules.yaml", true); + const require = createRequire(import.meta.url); + for (const name of ["tsdown", "typescript"]) { + capture(require.resolve(`${name}/package.json`), false, `${name}/package.json`); + capture(require.resolve(name), false, `${name}/entry`); + } + return hash.digest("hex"); + } catch { + return null; + } +} + +export type RunNodeInputState = { signature: string; generation: string }; + +/** Transient mutation evidence includes clean inputs that can change and revert during a build. */ +export function captureRunNodeInputState( + deps: RunNodeInputDeps, + scope: "build" | "runtime", + options: { assetPhase?: boolean } = {}, +): RunNodeInputState | null { + const signature = resolveRunNodeInputSignature(deps, scope); + if (!signature) { + return null; + } + try { + const files = listRunNodeInputFiles(deps, scope); + if (!files) { + return null; + } + const generation = createHash("sha256"); + for (const file of files) { + generation.update(`${file}\0`); + // The asset writer atomically replaces these files. Their authored fields + // remain guarded by the signature; later compiler phases also guard stat identity. + if (options.assetPhase && generatedControlUiManifest(deps, file)) { + continue; + } + try { + const absolute = path.resolve(deps.cwd, file); + const link = deps.fs.lstatSync(absolute, { bigint: true }); + const stat = deps.fs.statSync(absolute, { bigint: true }); + generation.update( + [link.dev, link.ino, link.ctimeNs, stat.dev, stat.ino, stat.ctimeNs, stat.size].join(":"), + ); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + throw error; + } + generation.update("missing"); + } + generation.update("\0"); + } + return { signature, generation: generation.digest("hex") }; + } catch { + return null; + } +} diff --git a/scripts/lib/tsdown-output-roots.mts b/scripts/lib/tsdown-output-roots.mts index 06a26231f1ba..c7b9e91498fe 100644 --- a/scripts/lib/tsdown-output-roots.mts +++ b/scripts/lib/tsdown-output-roots.mts @@ -36,3 +36,24 @@ export function tsdownPackageOutputRoot(packageName: string): string { function packageOutputRoot(packageName: string): string { return `packages/${packageName}/dist`; } + +const TSDOWN_SOURCE_EXTENSIONS = [ + ".cjs", + ".cts", + ".js", + ".json", + ".json5", + ".mjs", + ".mts", + ".sql", + ".ts", + ".tsx", + ".yaml", + ".yml", +]; + +export const TSDOWN_PACKAGES_CACHE_INPUT = { + path: "packages", + extensions: TSDOWN_SOURCE_EXTENSIONS, + excludeDirectories: ["dist", "node_modules"], +}; diff --git a/scripts/lib/vitest-build-prerequisites.mts b/scripts/lib/vitest-build-prerequisites.mts index e614c668e7f2..9fb0a323752a 100644 --- a/scripts/lib/vitest-build-prerequisites.mts +++ b/scripts/lib/vitest-build-prerequisites.mts @@ -524,11 +524,14 @@ export async function prepareVitestRuntime( } options.signal?.throwIfAborted(); const cwd = path.resolve(import.meta.dirname, "../.."); - if (!options.runtimePrepared) { + if (!options.runtimePrepared || controlUi) { console.error(`[test] preparing ${mode} runtime before Vitest workers`); const code = await runManagedCommand({ bin: process.execPath, - args: ["scripts/prepare-vitest-runtime.mjs"], + args: [ + "scripts/prepare-vitest-runtime.mjs", + ...(controlUi ? ["--require-current-head"] : []), + ], cwd, env: { ...env, ...(mode === "private-qa" ? { OPENCLAW_BUILD_PRIVATE_QA: "1" } : {}) }, signal: options.signal, diff --git a/scripts/prepare-vitest-runtime.mts b/scripts/prepare-vitest-runtime.mts index 0b62469f741b..1af7204f5306 100644 --- a/scripts/prepare-vitest-runtime.mts +++ b/scripts/prepare-vitest-runtime.mts @@ -1,10 +1,17 @@ +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; // Explicit test builds use the same checkout ownership and live-dist fence as pnpm build. import { runBuildAllSteps } from "./build-all.mts"; import { runCancelableCommand } from "./lib/cancelable-command.mts"; import { isDirectRunUrl } from "./lib/direct-run.mjs"; import { withDistArtifactOwnership } from "./lib/dist-artifact-ownership.mts"; import { resolveLiveManagedGatewayDistFence } from "./lib/live-gateway-dist-fence.mts"; -import { writeRuntimePostBuildStamp } from "./lib/local-build-metadata.mts"; +import { + RUNTIME_POSTBUILD_STAMP_FILE, + writeRuntimePostBuildStamp, +} from "./lib/local-build-metadata.mts"; +import { captureRunNodeInputState } from "./lib/run-node-input-state.mts"; import { resolveRunNodePreparation } from "./run-node.mts"; import { runRuntimePostBuild } from "./runtime-postbuild.mts"; import { listTsdownOutputRoots } from "./tsdown-build.mts"; @@ -13,9 +20,12 @@ export async function prepareTestRuntime( cwd = process.cwd(), env: NodeJS.ProcessEnv = process.env, signal?: AbortSignal, + options: { requireCurrentHead?: boolean } = {}, ): Promise { signal?.throwIfAborted(); - const initial = resolveRunNodePreparation(cwd, env); + const initial = resolveRunNodePreparation(cwd, env, { + allowEquivalentInputs: !options.requireCurrentHead, + }); if (!initial.build && !initial.runtime) { return 0; } @@ -23,7 +33,9 @@ export async function prepareTestRuntime( cwd, async () => { signal?.throwIfAborted(); - const preparation = resolveRunNodePreparation(cwd, env); + const preparation = resolveRunNodePreparation(cwd, env, { + allowEquivalentInputs: !options.requireCurrentHead, + }); if (preparation.immutable) { throw new Error( "Cannot prepare tests in an immutable deployment; use a separate source checkout.", @@ -50,9 +62,20 @@ export async function prepareTestRuntime( console.error(fence.message); return 1; } + const inputState = captureRunNodeInputState( + { + cwd, + distRoot: path.join(cwd, "dist"), + fs, + env, + spawnSync, + }, + "runtime", + ); + fs.rmSync(path.join(cwd, "dist", RUNTIME_POSTBUILD_STAMP_FILE), { force: true }); runRuntimePostBuild({ cwd, env }); signal?.throwIfAborted(); - writeRuntimePostBuildStamp({ cwd, env }); + writeRuntimePostBuildStamp({ cwd, env, inputState }); } return 0; }, @@ -61,7 +84,13 @@ export async function prepareTestRuntime( } if (isDirectRunUrl(process.argv[1], import.meta.url)) { + const args = process.argv.slice(2); + if (args.some((arg) => arg !== "--require-current-head")) { + throw new Error("Usage: prepare-vitest-runtime.mjs [--require-current-head]"); + } process.exitCode = await runCancelableCommand((signal) => - prepareTestRuntime(process.cwd(), process.env, signal), + prepareTestRuntime(process.cwd(), process.env, signal, { + requireCurrentHead: args.includes("--require-current-head"), + }), ); } diff --git a/scripts/run-node-watch-paths.mts b/scripts/run-node-watch-paths.mts index 2e05df01dad4..55f75caefde9 100644 --- a/scripts/run-node-watch-paths.mts +++ b/scripts/run-node-watch-paths.mts @@ -48,13 +48,13 @@ const extensionSourceFilePattern = /\.(?:[cm]?[jt]sx?)$/; export const normalizeRunNodePath = (filePath: unknown): string => (typeof filePath === "string" ? filePath : "").replaceAll(path.sep, "/").replace(/^\.\/+/, ""); -const isIgnoredSourcePath = (relativePath: string): boolean => +export const isIgnoredRunNodeSourcePath = (relativePath: string): boolean => relativePath.endsWith(".test.ts") || relativePath.endsWith(".test.tsx") || relativePath.endsWith("test-helpers.ts"); const isBuildRelevantSourcePath = (relativePath: string): boolean => - extensionSourceFilePattern.test(relativePath) && !isIgnoredSourcePath(relativePath); + extensionSourceFilePattern.test(relativePath) && !isIgnoredRunNodeSourcePath(relativePath); const isRestartRelevantExtensionPath = (relativePath: string): boolean => { if (extensionRestartMetadataFiles.has(path.posix.basename(relativePath))) { @@ -79,11 +79,11 @@ const isRelevantRunNodePath = ( return true; } if (normalizedPath.startsWith("src/")) { - return !isIgnoredSourcePath(normalizedPath.slice("src/".length)); + return !isIgnoredRunNodeSourcePath(normalizedPath.slice("src/".length)); } for (const sourceRoot of RUN_NODE_PACKAGE_SOURCE_ROOTS) { if (normalizedPath.startsWith(`${sourceRoot}/`)) { - return !isIgnoredSourcePath(normalizedPath.slice(sourceRoot.length + 1)); + return !isIgnoredRunNodeSourcePath(normalizedPath.slice(sourceRoot.length + 1)); } } if (normalizedPath.startsWith(BUNDLED_PLUGIN_PATH_PREFIX)) { diff --git a/scripts/run-node.mts b/scripts/run-node.mts index 7f1520cd495d..d120f42c12e6 100644 --- a/scripts/run-node.mts +++ b/scripts/run-node.mts @@ -29,8 +29,11 @@ import { writeRuntimePostBuildStamp as writeDistRuntimePostBuildStamp, } from "./lib/local-build-metadata.mts"; import { + captureRunNodeInputState, + type RunNodeInputState, collectRunNodeBundledPluginBuildEntries, hasDirtySourceTree, + resolveRunNodeInputSignature, hasDirtyRuntimePostBuildInputs, isRuntimePostBuildRelevantPath, listBundledPluginRuntimeEntryPaths, @@ -231,12 +234,12 @@ const findLatestMtime = ( const readJsonStamp = (filePath: string, deps: RunNodeRequirementDeps) => { const mtime = statMtime(filePath, deps.fs); if (mtime == null) { - return { mtime: null, head: null, inputsClean: null }; + return { mtime: null, head: null, inputsClean: null, inputSignature: null, staticAssets: null }; } try { const raw = deps.fs.readFileSync(filePath, "utf8").trim(); if (!raw.startsWith("{")) { - return { mtime, head: null, inputsClean: null }; + return { mtime, head: null, inputsClean: null, inputSignature: null, staticAssets: null }; } const parsed = JSON.parse(raw); const head = typeof parsed?.head === "string" && parsed.head.trim() ? parsed.head.trim() : null; @@ -244,9 +247,15 @@ const readJsonStamp = (filePath: string, deps: RunNodeRequirementDeps) => { mtime, head, inputsClean: typeof parsed?.inputsClean === "boolean" ? parsed.inputsClean : null, + staticAssets: Object.hasOwn(parsed, "staticAssets") ? parsed.staticAssets === true : null, + inputSignature: Object.hasOwn(parsed, "inputSignature") + ? typeof parsed.inputSignature === "string" && /^[a-f0-9]{64}$/u.test(parsed.inputSignature) + ? parsed.inputSignature + : "" + : null, }; } catch { - return { mtime, head: null, inputsClean: null }; + return { mtime, head: null, inputsClean: null, inputSignature: null, staticAssets: null }; } }; @@ -500,7 +509,10 @@ const hasMissingRequiredRuntimePostBuildOutput = (deps: RunNodeRequirementDeps) }; /** Decides whether source changes require a new dev build. */ -export const resolveBuildRequirement = (deps: RunNodeRequirementDeps): BuildRequirement => { +export const resolveBuildRequirement = ( + deps: RunNodeRequirementDeps, + options: { allowEquivalentInputs?: boolean } = {}, +): BuildRequirement => { if (deps.env.OPENCLAW_FORCE_BUILD === "1") { return { shouldBuild: true, reason: "force_build" }; } @@ -521,14 +533,47 @@ export const resolveBuildRequirement = (deps: RunNodeRequirementDeps): BuildRequ } const currentHead = resolveGitHead(deps); + if ( + stamp.inputSignature === "" || + (!currentHead && stamp.inputSignature !== null && stamp.inputsClean !== true) + ) { + return { shouldBuild: true, reason: "build_inputs_unverified" }; + } if (currentHead && !stamp.head) { return { shouldBuild: true, reason: "build_stamp_missing_head" }; } - if (currentHead && stamp.head && currentHead !== stamp.head) { + const headChanged = Boolean(currentHead && stamp.head && currentHead !== stamp.head); + const immutable = isImmutableGitDeployment(deps); + if (headChanged && (!options.allowEquivalentInputs || immutable)) { return { shouldBuild: true, reason: "git_head_changed" }; } if (currentHead) { const dirty = hasDirtySourceTree(deps); + // Preserve the portable clean-artifact contract; fingerprints are needed + // for dirty inputs and test capsules whose private carrier HEAD changed. + if ( + !headChanged && + dirty === false && + stamp.inputsClean === true && + (!options.allowEquivalentInputs || + isImmutableGitDeployment(deps) || + stamp.inputSignature === null) + ) { + return hasMissingBuiltBundledPluginRuntimeEntryOutput(deps) + ? { shouldBuild: true, reason: "missing_bundled_plugin_dist_entry" } + : { shouldBuild: false, reason: "clean" }; + } + if (options.allowEquivalentInputs && stamp.inputSignature !== null && !immutable) { + if (stamp.inputSignature !== resolveRunNodeInputSignature(deps, "build")) { + return { shouldBuild: true, reason: "build_inputs_changed" }; + } + return hasMissingBuiltBundledPluginRuntimeEntryOutput(deps) + ? { shouldBuild: true, reason: "missing_bundled_plugin_dist_entry" } + : { shouldBuild: false, reason: "clean" }; + } + if (headChanged) { + return { shouldBuild: true, reason: "git_head_changed" }; + } if (dirty === true) { return { shouldBuild: true, reason: "dirty_watched_tree" }; } @@ -563,7 +608,7 @@ export const resolveBuildRequirement = (deps: RunNodeRequirementDeps): BuildRequ /** Decides whether runtime postbuild artifacts need to be regenerated. */ export const resolveRuntimePostBuildRequirement = ( deps: RunNodeRuntimeRequirementDeps, - options: { requireCleanInputs?: boolean } = {}, + options: { requireCleanInputs?: boolean; allowEquivalentInputs?: boolean } = {}, ): RuntimePostBuildRequirement => { if (deps.env.OPENCLAW_FORCE_RUNTIME_POSTBUILD === "1") { return { shouldSync: true, reason: "force_runtime_postbuild" }; @@ -574,6 +619,13 @@ export const resolveRuntimePostBuildRequirement = ( return { shouldSync: true, reason: "missing_runtime_postbuild_stamp" }; } + if ( + shouldCopyStaticExtensionAssets({ env: deps.env }) && + (stamp.staticAssets === false || (stamp.inputSignature !== null && stamp.staticAssets !== true)) + ) { + return { shouldSync: true, reason: "static_assets_not_prepared" }; + } + const buildStamp = readBuildStamp(deps); if (buildStamp.mtime == null) { return { shouldSync: true, reason: "missing_build_stamp" }; @@ -583,14 +635,53 @@ export const resolveRuntimePostBuildRequirement = ( } const currentHead = resolveGitHead(deps); + if ( + stamp.inputSignature === "" || + (!currentHead && stamp.inputSignature !== null && stamp.inputsClean !== true) + ) { + return { shouldSync: true, reason: "runtime_inputs_unverified" }; + } if (currentHead && !stamp.head) { return { shouldSync: true, reason: "runtime_postbuild_stamp_missing_head" }; } - if (currentHead && stamp.head && currentHead !== stamp.head) { + const headChanged = Boolean(currentHead && stamp.head && currentHead !== stamp.head); + if ( + headChanged && + (!options.allowEquivalentInputs || + isImmutableGitDeployment(deps) || + buildStamp.head !== stamp.head) + ) { return { shouldSync: true, reason: "git_head_changed" }; } if (currentHead) { const dirty = hasDirtyRuntimePostBuildInputs(deps); + if ( + !headChanged && + dirty === false && + stamp.inputsClean === true && + (!options.allowEquivalentInputs || + isImmutableGitDeployment(deps) || + stamp.inputSignature === null) + ) { + return hasMissingRequiredRuntimePostBuildOutput(deps) + ? { shouldSync: true, reason: "missing_runtime_postbuild_output" } + : { shouldSync: false, reason: "clean" }; + } + if ( + options.allowEquivalentInputs && + stamp.inputSignature !== null && + !isImmutableGitDeployment(deps) + ) { + if (stamp.inputSignature !== resolveRunNodeInputSignature(deps, "runtime")) { + return { shouldSync: true, reason: "runtime_inputs_changed" }; + } + return hasMissingRequiredRuntimePostBuildOutput(deps) + ? { shouldSync: true, reason: "missing_runtime_postbuild_output" } + : { shouldSync: false, reason: "clean" }; + } + if (headChanged) { + return { shouldSync: true, reason: "git_head_changed" }; + } if (dirty === true) { return { shouldSync: true, reason: "dirty_runtime_postbuild_inputs" }; } @@ -622,6 +713,7 @@ const BUILD_REASON_LABELS = { missing_dist_entry: "dist entry missing", config_newer: "config newer than build stamp", build_stamp_missing_head: "build stamp missing git head", + build_inputs_changed: "build input bytes or toolchain changed", build_inputs_unverified: "build inputs were not verified clean", git_head_changed: "git head changed", dirty_watched_tree: "dirty watched source tree", @@ -638,6 +730,8 @@ const RUNTIME_POSTBUILD_REASON_LABELS = { missing_build_stamp: "build stamp missing", build_stamp_newer: "build stamp newer than runtime postbuild stamp", runtime_postbuild_stamp_missing_head: "runtime postbuild stamp missing git head", + static_assets_not_prepared: "runtime static assets were not verified", + runtime_inputs_changed: "runtime input bytes or toolchain changed", runtime_inputs_unverified: "runtime postbuild inputs were not verified clean", git_head_changed: "git head changed", dirty_runtime_postbuild_inputs: "dirty runtime postbuild inputs", @@ -1387,16 +1481,19 @@ const syncRuntimeArtifacts = async (deps: RunNodeDeps) => { return true; }; -const writeRuntimePostBuildStamp = (deps: RunNodeDeps) => { +const writeRuntimePostBuildStamp = (deps: RunNodeDeps, inputState: RunNodeInputState | null) => { try { writeDistRuntimePostBuildStamp({ cwd: deps.cwd, fs: deps.fs, env: deps.env, spawnSync: deps.spawnSync, + inputState, }); + return true; } catch (error) { logRunner(`Failed to write runtime postbuild stamp: ${getErrorMessage(error)}`, deps); + return false; } }; @@ -1424,12 +1521,14 @@ const syncRuntimeArtifactsAndStamp = async (deps: RunNodeDeps) => return false; } deps.cancellation.signal.throwIfAborted(); + const inputState = captureRunNodeInputState(deps, "runtime"); + deps.fs.rmSync(deps.runtimePostBuildStampPath, { force: true }); const synced = await syncRuntimeArtifacts(deps); deps.cancellation.signal.throwIfAborted(); if (synced) { - writeRuntimePostBuildStamp(deps); + return writeRuntimePostBuildStamp(deps, inputState); } - return synced; + return false; }); }, deps.cancellation.signal, @@ -1578,10 +1677,25 @@ function createRunNodeDeps(params: RunNodeMainParams) { } /** Read-only build admission shared by explicit test preparation and the source runner. */ -export function resolveRunNodePreparation(cwd: string, env: NodeJS.ProcessEnv) { +export function resolveRunNodePreparation( + cwd: string, + env: NodeJS.ProcessEnv, + options: { allowEquivalentInputs?: boolean } = {}, +) { const deps = createRunNodeDeps({ cwd, env, args: [] }); - const build = resolveBuildRequirement(deps).shouldBuild; - const runtime = !build && resolveRuntimePostBuildRequirement(deps).shouldSync; + let build = resolveBuildRequirement(deps, options).shouldBuild; + let runtime = !build && resolveRuntimePostBuildRequirement(deps, options).shouldSync; + // A partial refresh must not relabel an older compiled generation while CLI + // and UI metadata still retain its identity. Reuse only a coherent generation. + if ( + !build && + runtime && + options.allowEquivalentInputs && + resolveGitHead(deps) !== readBuildStamp(deps).head + ) { + build = true; + runtime = false; + } return { build, runtime, immutable: (build || runtime) && isImmutableGitDeployment(deps) }; } diff --git a/scripts/tsdown-build.mts b/scripts/tsdown-build.mts index 9a2c590a2e3c..8a7a8ccd9745 100644 --- a/scripts/tsdown-build.mts +++ b/scripts/tsdown-build.mts @@ -40,6 +40,7 @@ import { } from "./lib/tsdown-config-groups.mts"; import { TSDOWN_PACKAGE_OUTPUT_ROOTS, + TSDOWN_PACKAGES_CACHE_INPUT, tsdownPackageOutputRoot, } from "./lib/tsdown-output-roots.mts"; @@ -72,21 +73,6 @@ export const TSDOWN_DECLARATION_EXTENSIONS = [".d.ts", ".d.mts", ".d.cts"]; const SOURCE_DECLARATION_SOURCE_EXTENSIONS = [".ts", ".tsx", ".mts", ".cts", ".js", ".mjs", ".cjs"]; const RUN_NODE_SKIP_DTS_BUILD_ENV = "OPENCLAW_RUN_NODE_SKIP_DTS_BUILD"; -const TSDOWN_SOURCE_EXTENSIONS = [ - ".cjs", - ".cts", - ".js", - ".json", - ".json5", - ".mjs", - ".mts", - ".sql", - ".ts", - ".tsx", - ".yaml", - ".yml", -]; - export const TSDOWN_DECLARATION_TOOL_INPUTS = [ "package.json", "pnpm-lock.yaml", @@ -116,11 +102,7 @@ export const TSDOWN_DECLARATION_TOOL_INPUTS = [ "scripts/lib/tsdown-declaration-boundary.mts", "scripts/lib/tsdown-output-roots.mts", ]; -export const TSDOWN_PACKAGES_CACHE_INPUT = { - path: "packages", - extensions: TSDOWN_SOURCE_EXTENSIONS, - excludeDirectories: ["dist", "node_modules"], -}; +export { TSDOWN_PACKAGES_CACHE_INPUT }; export const TSDOWN_UNIFIED_CACHE_ENV = [ "OPENCLAW_BUILD_PRIVATE_QA", ...BUNDLED_PLUGIN_BUILD_ENV_NAMES, diff --git a/src/infra/build-stamp.test.ts b/src/infra/build-stamp.test.ts index 85708a9f5876..3707f5e0497e 100644 --- a/src/infra/build-stamp.test.ts +++ b/src/infra/build-stamp.test.ts @@ -1,7 +1,19 @@ +import { execFileSync, spawnSync } from "node:child_process"; +import fsSync from "node:fs"; import fs from "node:fs/promises"; +import path from "node:path"; import { describe, expect, it } from "vitest"; import { BUILD_STAMP_FILE } from "../../scripts/lib/local-build-metadata-paths.mts"; -import { writeBuildStamp } from "../../scripts/lib/local-build-metadata.mts"; +import { + writeBuildStamp, + writeRuntimePostBuildStamp, +} from "../../scripts/lib/local-build-metadata.mts"; +import { captureRunNodeInputState } from "../../scripts/lib/run-node-input-state.mts"; +import { resolveBuildRequirement, resolveRunNodePreparation } from "../../scripts/run-node.mts"; +import { + setupStampedProject, + trackProjectWithGit, +} from "../../test/scripts/run-node.test-support.js"; import { withTestDir } from "../test-helpers/temp-dir.js"; describe("build-stamp script", () => { @@ -34,3 +46,215 @@ describe("build-stamp script", () => { }); }); }); + +it("reuses built dirty inputs but rejects changed production, dependencies and missing output", async () => { + await withTestDir({ prefix: "openclaw-dirty-build-" }, async (cwd) => { + const write = async (file: string, value: string) => { + await fs.mkdir(path.dirname(path.join(cwd, file)), { recursive: true }); + await fs.writeFile(path.join(cwd, file), value); + }; + await write(".gitignore", "dist/\n"); + await write("package.json", '{"name":"openclaw"}'); + await write("src/index.ts", "export const value = 1;\n"); + await write("src/index.test.ts", "original fixture\n"); + await write("src/stable.ts", "export const stable = 1;\n"); + await write("pnpm-lock.yaml", "original lockfile\n"); + for (const args of [ + ["init", "-q"], + ["add", "."], + [ + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.test", + "commit", + "-qm", + "fixture", + ], + ]) { + execFileSync( + "git", + ["-c", "commit.gpgsign=false", "-c", "core.hooksPath=/dev/null", ...args], + { cwd }, + ); + } + const deps = { + cwd, + env: {}, + fs: fsSync, + spawnSync, + distRoot: path.join(cwd, "dist"), + distEntry: path.join(cwd, "dist/entry.js"), + buildStampPath: path.join(cwd, "dist/.buildstamp"), + sourceRoots: [], + configFiles: [], + }; + await write("src/index.ts", "export const value = 2;\n"); + const inputState = captureRunNodeInputState(deps, "build"); + expect(inputState?.signature).toMatch(/^[a-f0-9]{64}$/u); + await write("dist/entry.js", "export const value = 2;\n"); + writeBuildStamp({ cwd }); + expect(resolveBuildRequirement(deps, { allowEquivalentInputs: true }).reason).toBe( + "dirty_watched_tree", + ); + writeBuildStamp({ cwd, inputState }); + expect(resolveBuildRequirement(deps, { allowEquivalentInputs: true }).shouldBuild).toBe(false); + await write("src/stable.ts", "export const stable = 2;\n"); + await write("src/stable.ts", "export const stable = 1;\n"); + expect(() => writeBuildStamp({ cwd, inputState })).toThrow("Build inputs changed"); + expect( + resolveBuildRequirement({ + ...deps, + spawnSync: () => ({ status: 1, stdout: "" }), + }).shouldBuild, + ).toBe(true); + const validStamp = await fs.readFile(deps.buildStampPath, "utf8"); + for (const malformed of ["", null, false]) { + await fs.writeFile( + deps.buildStampPath, + JSON.stringify({ + ...JSON.parse(validStamp), + inputSignature: malformed, + }), + ); + expect(resolveBuildRequirement(deps, { allowEquivalentInputs: true }).shouldBuild).toBe(true); + } + await fs.writeFile(deps.buildStampPath, validStamp); + await write( + "deployment.json", + JSON.stringify({ kind: "git", sourceHead: JSON.parse(validStamp).head }), + ); + expect(resolveBuildRequirement(deps, { allowEquivalentInputs: true }).shouldBuild).toBe(true); + await fs.unlink(path.join(cwd, "deployment.json")); + await write("src/index.test.ts", "corrected fixture\n"); + expect(resolveBuildRequirement(deps, { allowEquivalentInputs: true }).shouldBuild).toBe(false); + await fs.rename(path.join(cwd, "src/index.ts"), path.join(cwd, "src/renamed.ts")); + expect(resolveBuildRequirement(deps, { allowEquivalentInputs: true }).shouldBuild).toBe(true); + await fs.rename(path.join(cwd, "src/renamed.ts"), path.join(cwd, "src/index.ts")); + await fs.unlink(path.join(cwd, "src/index.ts")); + expect(resolveBuildRequirement(deps, { allowEquivalentInputs: true }).shouldBuild).toBe(true); + await write("src/index.ts", "export const value = 3;\n"); + await fs.utimes(path.join(cwd, "src/index.ts"), new Date(0), new Date(0)); + expect(resolveBuildRequirement(deps, { allowEquivalentInputs: true }).reason).toBe( + "build_inputs_changed", + ); + expect(() => writeBuildStamp({ cwd, inputState })).toThrow("Build inputs changed"); + await write("src/index.ts", "export const value = 2;\n"); + expect(() => writeBuildStamp({ cwd, inputState })).toThrow("Build inputs changed"); + await write("src/index.ts", "export const value = 1;\n"); + expect(resolveBuildRequirement(deps, { allowEquivalentInputs: true }).shouldBuild).toBe(true); + await write("src/index.ts", "export const value = 2;\n"); + await write("pnpm-lock.yaml", "changed dependency\n"); + expect(resolveBuildRequirement(deps, { allowEquivalentInputs: true }).shouldBuild).toBe(true); + await write("pnpm-lock.yaml", "original lockfile\n"); + expect(resolveBuildRequirement(deps, { allowEquivalentInputs: true }).shouldBuild).toBe(false); + await write("src/index.ts", "export const value = 1;\n"); + await write("config/tsconfig/base.json", "{}\n"); + await write("dist/entry.js", "export const value = 1;\n"); + writeBuildStamp({ cwd, inputState: captureRunNodeInputState(deps, "build") }); + expect(resolveBuildRequirement(deps, { allowEquivalentInputs: true }).shouldBuild).toBe(false); + await fs.rm(path.join(cwd, "config"), { recursive: true }); + expect(resolveBuildRequirement(deps, { allowEquivalentInputs: true }).shouldBuild).toBe(true); + await write( + "extensions/example/package.json", + JSON.stringify({ + openclaw: { + controlUi: "./browser/index.ts", + assetScripts: { build: "build-ui", buildOutputs: ["openclaw.plugin.json"] }, + }, + }), + ); + const manifest = (entry: string, enabled = true) => + JSON.stringify({ id: "example", configSchema: { enabled }, controlUi: { entry } }); + await write("extensions/example/openclaw.plugin.json", manifest("old.js")); + await write("extensions/example/browser/index.css", "body { color: red; }"); + const assetsBefore = captureRunNodeInputState(deps, "build", { assetPhase: true }); + const compilerBefore = captureRunNodeInputState(deps, "build"); + await write("extensions/example/openclaw.plugin.json", manifest("new.js")); + expect(captureRunNodeInputState(deps, "build", { assetPhase: true })).toEqual(assetsBefore); + expect(captureRunNodeInputState(deps, "build")?.generation).not.toBe( + compilerBefore?.generation, + ); + await write("extensions/example/browser/index.test.ts", "fixture correction"); + expect(captureRunNodeInputState(deps, "build", { assetPhase: true })).toEqual(assetsBefore); + await write("extensions/example/openclaw.plugin.json", manifest("new.js", false)); + expect(captureRunNodeInputState(deps, "build", { assetPhase: true })?.signature).not.toBe( + assetsBefore?.signature, + ); + await write("extensions/example/openclaw.plugin.json", manifest("new.js")); + await write("extensions/example/browser/index.css", "body { color: blue; }"); + expect(captureRunNodeInputState(deps, "build", { assetPhase: true })?.signature).not.toBe( + assetsBefore?.signature, + ); + await fs.unlink(deps.distEntry); + expect(resolveBuildRequirement(deps, { allowEquivalentInputs: true }).reason).toBe( + "missing_dist_entry", + ); + }); +}); + +it("reuses clean test capsules across carrier commits but preserves strict CLI and deployment heads", async () => { + await withTestDir({ prefix: "openclaw-carrier-build-" }, async (cwd) => { + await setupStampedProject(cwd, { + files: { + "src/entry.test.ts": "original fixture\n", + "packages/session-url-contract/src/index.ts": "export const value = 1;\n", + }, + }); + const { git, deps } = await trackProjectWithGit(cwd); + const commit = () => + git( + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.test", + "-c", + "commit.gpgsign=false", + "commit", + "--allow-empty", + "-qm", + "carrier", + ); + writeBuildStamp({ cwd, inputState: captureRunNodeInputState(deps, "build") }); + writeRuntimePostBuildStamp({ cwd, inputState: captureRunNodeInputState(deps, "runtime") }); + commit(); + expect(resolveBuildRequirement(deps).reason).toBe("git_head_changed"); + expect(resolveRunNodePreparation(cwd, {}, { allowEquivalentInputs: true })).toEqual({ + build: false, + runtime: false, + immutable: false, + }); + const stampFile = deps.buildStampPath; + const validStamp = await fs.readFile(stampFile, "utf8"); + const validStampTime = await fs.stat(stampFile); + const legacy = JSON.parse(validStamp); + delete legacy.inputSignature; + await fs.writeFile(stampFile, JSON.stringify(legacy)); + expect(resolveBuildRequirement(deps, { allowEquivalentInputs: true }).reason).toBe( + "git_head_changed", + ); + await fs.writeFile(stampFile, validStamp); + await fs.utimes(stampFile, validStampTime.atime, validStampTime.mtime); + await fs.writeFile(path.join(cwd, "src/entry.test.ts"), "corrected fixture\n"); + git("add", "src/entry.test.ts"); + commit(); + expect(resolveRunNodePreparation(cwd, {}, { allowEquivalentInputs: true }).build).toBe(false); + await fs.writeFile( + path.join(cwd, "deployment.json"), + JSON.stringify({ kind: "git", sourceHead: git("rev-parse", "HEAD") }), + ); + expect(resolveRunNodePreparation(cwd, {}, { allowEquivalentInputs: true }).immutable).toBe( + true, + ); + await fs.unlink(path.join(cwd, "deployment.json")); + await fs.writeFile( + path.join(cwd, "packages/session-url-contract/src/index.ts"), + "export const value = 2;\n", + ); + git("add", "packages/session-url-contract/src/index.ts"); + commit(); + expect(resolveBuildRequirement(deps, { allowEquivalentInputs: true }).reason).toBe( + "build_inputs_changed", + ); + }); +}); diff --git a/test/scripts/build-all.test.ts b/test/scripts/build-all.test.ts index ac24566a513b..eb56ea7c8fa7 100644 --- a/test/scripts/build-all.test.ts +++ b/test/scripts/build-all.test.ts @@ -287,6 +287,75 @@ describe("resolveBuildAllSteps", () => { expect(runner.logger.error).toHaveBeenCalledWith(message); }); + it.each(["gatewayWatch", "cliStartup"])( + "records %s runtime phase completeness", + async (profile) => { + const cwd = tempDirs.make("openclaw-phase-stamp-"); + const steps = resolveBuildAllSteps(profile, {}) + .filter((step) => ["runtime-postbuild", "runtime-postbuild-stamp"].includes(step.label)) + .map((step) => { + if (step.kind === "pnpm") { + throw new Error("Runtime metadata steps must use the native Node owner"); + } + return step.label === "runtime-postbuild" + ? Object.assign({}, step, { args: ["-e", "process.exit(0)"] }) + : step; + }); + const result = await runBuildAllSteps(profile, { + cwd, + env: {}, + steps, + logger: { error() {}, warn() {} }, + memoryLimit: buildMemoryLimit(16), + }); + expect(result.exitCode).toBe(0); + expect( + JSON.parse(fs.readFileSync(path.join(cwd, "dist/.runtime-postbuildstamp"), "utf8")) + .staticAssets, + ).toBe(false); + }, + ); + + it("invalidates old runtime stamps before a failed declaration-cache restoration", async () => { + const cwd = tempDirs.make("openclaw-restore-stamps-"); + fs.mkdirSync(path.join(cwd, "dist")); + const stamps = [".buildstamp", ".runtime-postbuildstamp"].map((name) => + path.join(cwd, "dist", name), + ); + for (const stamp of stamps) { + fs.writeFileSync(stamp, "previous valid generation"); + } + await expect( + runBuildAllSteps("pluginSdkStrictSmoke", { + cwd, + env: {}, + memoryLimit: buildMemoryLimit(16), + logger: { error() {}, warn() {} }, + steps: [getBuildAllStep("tsdown-ai"), getBuildAllStep("build-stamp")], + resolveCacheState: () => ({ + cacheable: true, + fresh: true, + restorable: true, + reason: "fresh-cache", + signature: "fixture", + outputRoot: cwd, + stampPath: path.join(cwd, "cache.json"), + inputFiles: 1, + outputFiles: 1, + relativeOutputFiles: ["dist/entry.js"], + stampedOutputs: ["dist/entry.js"], + record: undefined, + }), + restoreCache() { + expect(stamps.some((file) => fs.existsSync(file))).toBe(false); + fs.writeFileSync(path.join(cwd, "dist/entry.js"), "partial restoration"); + return false; + }, + }), + ).rejects.toThrow("Build cache changed before restoration"); + expect(stamps.some((file) => fs.existsSync(file))).toBe(false); + }); + it("admits package once and freezes its heap for every child", async () => { const profile = "package"; const tsdownSteps = resolveBuildAllSteps(profile).filter( diff --git a/test/scripts/prepare-vitest-runtime.test.ts b/test/scripts/prepare-vitest-runtime.test.ts index e716cbb1f367..f683de4a583f 100644 --- a/test/scripts/prepare-vitest-runtime.test.ts +++ b/test/scripts/prepare-vitest-runtime.test.ts @@ -289,3 +289,17 @@ it("joins a canceled compiler before releasing checkout ownership", async () => const next = await acquireDistArtifactOwnership(root); await next.release(); }); + +it("keeps UI preparation current-head even after another runtime group was prepared", async () => { + commands.prepare.mockResolvedValueOnce(7); + expect( + await prepareVitestRuntime([{ configs: ["test/vitest/vitest.ui-e2e.config.ts"] }], env, { + runtimePrepared: true, + }), + ).toBe(7); + expect(commands.prepare).toHaveBeenCalledWith( + expect.objectContaining({ + args: ["scripts/prepare-vitest-runtime.mjs", "--require-current-head"], + }), + ); +}); diff --git a/test/scripts/runtime-postbuild-stamp.test.ts b/test/scripts/runtime-postbuild-stamp.test.ts index 12912263e1ae..ebc7279b0e4b 100644 --- a/test/scripts/runtime-postbuild-stamp.test.ts +++ b/test/scripts/runtime-postbuild-stamp.test.ts @@ -3,13 +3,21 @@ import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { RUNTIME_POSTBUILD_STAMP_FILE } from "../../scripts/lib/local-build-metadata-paths.mts"; import { writeRuntimePostBuildStamp } from "../../scripts/lib/local-build-metadata.mts"; +import { captureRunNodeInputState } from "../../scripts/lib/run-node-input-state.mts"; +import { + copyStaticExtensionAssets, + copyStaticExtensionAssetsToRuntimeOverlay, +} from "../../scripts/lib/static-extension-assets.mts"; +import { resolveRuntimePostBuildRequirement } from "../../scripts/run-node.mts"; import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js"; +import { setupStampedProject, trackProjectWithGit } from "./run-node.test-support.js"; describe("runtime-postbuild-stamp script", () => { const tempDirs = useAutoCleanupTempDirTracker(afterEach); it.each([ { name: "clean", gitStatus: "", inputsClean: true }, + { name: "skipped assets", gitStatus: "", inputsClean: true }, { name: "dirty metadata", gitStatus: " M extensions/demo/openclaw.plugin.json\0", @@ -17,24 +25,91 @@ describe("runtime-postbuild-stamp script", () => { }, { name: "source-only change", gitStatus: " M src/index.ts\0", inputsClean: true }, { name: "unknown", gitStatus: null, inputsClean: null }, - ])("records $name runtime inputs with the current git head", ({ gitStatus, inputsClean }) => { - const rootDir = tempDirs.make("openclaw-runtime-postbuild-stamp-"); - const stampPath = writeRuntimePostBuildStamp({ - cwd: rootDir, - now: () => 123, - spawnSync: (_command, args) => - args[0] === "rev-parse" - ? { status: 0, stdout: "abc123\n" } - : gitStatus === null - ? { status: 1, stdout: "" } - : { status: 0, stdout: gitStatus }, - }); + ])( + "records $name runtime inputs with the current git head", + ({ name, gitStatus, inputsClean }) => { + const rootDir = tempDirs.make("openclaw-runtime-postbuild-stamp-"); + const stampPath = writeRuntimePostBuildStamp({ + cwd: rootDir, + now: () => 123, + env: { OPENCLAW_RUNTIME_POSTBUILD_STATIC_ASSETS: name === "skipped assets" ? "0" : "1" }, + spawnSync: (_command, args) => + args[0] === "rev-parse" + ? { status: 0, stdout: "abc123\n" } + : gitStatus === null + ? { status: 1, stdout: "" } + : { status: 0, stdout: gitStatus }, + }); - expect(path.relative(rootDir, stampPath)).toBe(path.join("dist", RUNTIME_POSTBUILD_STAMP_FILE)); - expect(JSON.parse(fs.readFileSync(stampPath, "utf8"))).toEqual({ - syncedAt: 123, - head: "abc123", - inputsClean, + expect(path.relative(rootDir, stampPath)).toBe( + path.join("dist", RUNTIME_POSTBUILD_STAMP_FILE), + ); + expect(JSON.parse(fs.readFileSync(stampPath, "utf8"))).toEqual({ + syncedAt: 123, + head: "abc123", + inputsClean, + staticAssets: name !== "skipped assets", + }); + }, + ); + it("does not reuse skipped static assets for a full runtime reader", async () => { + const cwd = tempDirs.make("runtime-stamp-assets-"); + const source = "extensions/apple-fm/assets/AppleFoundationModels.swift"; + const output = `dist/${source}`; + const overlay = `dist-runtime/${source}`; + await setupStampedProject(cwd, { + files: { + "extensions/apple-fm/package.json": JSON.stringify({ + name: "@openclaw/apple-fm", + openclaw: { + build: { + staticAssets: [ + { + source: "assets/AppleFoundationModels.swift", + output: "assets/AppleFoundationModels.swift", + }, + ], + }, + }, + }), + [source]: "A", + [output]: "A", + [overlay]: "A", + }, }); + const { deps } = await trackProjectWithGit(cwd); + fs.writeFileSync(path.join(cwd, source), "B"); + const env = { OPENCLAW_RUNTIME_POSTBUILD_STATIC_ASSETS: "0" }; + writeRuntimePostBuildStamp({ + cwd, + env, + inputState: captureRunNodeInputState({ ...deps, env }, "runtime"), + }); + expect(fs.readFileSync(path.join(cwd, output), "utf8")).toBe("A"); + expect(resolveRuntimePostBuildRequirement(deps, { allowEquivalentInputs: true }).reason).toBe( + "static_assets_not_prepared", + ); + expect( + resolveRuntimePostBuildRequirement({ ...deps, env }, { allowEquivalentInputs: true }) + .shouldSync, + ).toBe(false); + const inputState = captureRunNodeInputState(deps, "runtime"); + copyStaticExtensionAssets({ rootDir: cwd, env: {} }); + copyStaticExtensionAssetsToRuntimeOverlay({ rootDir: cwd, env: {} }); + writeRuntimePostBuildStamp({ cwd, env: {}, inputState }); + expect(fs.readFileSync(path.join(cwd, output), "utf8")).toBe("B"); + expect( + resolveRuntimePostBuildRequirement(deps, { allowEquivalentInputs: true }).shouldSync, + ).toBe(false); + expect( + resolveRuntimePostBuildRequirement({ ...deps, env }, { allowEquivalentInputs: true }) + .shouldSync, + ).toBe(false); + const stamp = path.join(cwd, "dist", RUNTIME_POSTBUILD_STAMP_FILE); + const recorded = JSON.parse(fs.readFileSync(stamp, "utf8")); + fs.writeFileSync(stamp, JSON.stringify({ ...recorded, staticAssets: "unknown" })); + expect(resolveRuntimePostBuildRequirement(deps, { allowEquivalentInputs: true }).reason).toBe( + "static_assets_not_prepared", + ); }); }); diff --git a/test/scripts/test-projects-build-admission.test.ts b/test/scripts/test-projects-build-admission.test.ts index 8931f6345679..d15e3466744c 100644 --- a/test/scripts/test-projects-build-admission.test.ts +++ b/test/scripts/test-projects-build-admission.test.ts @@ -1283,6 +1283,10 @@ describe("test-projects build admission", () => { await start(mixed ? [e2eTarget, avatarTarget] : [avatarTarget]); await Promise.race([runtime.started, terminal.promise]); const ui = createPreparationGate(commands.prepare); + if (mixed) { + // E2E reuse still needs the UI's stricter current-head preparation. + commands.prepare.mockResolvedValueOnce(0); + } try { expect(commands.reader).not.toHaveBeenCalled(); expect(commands.uiAssets).not.toHaveBeenCalled(); @@ -1309,11 +1313,10 @@ describe("test-projects build admission", () => { } expect(outcome).toMatch(/^\[test\] passed /u); expect(commands.prepareE2e).toHaveBeenCalledTimes(mixed ? 1 : 0); - expect(commands.prepare.mock.calls.map(([command]) => command.args)).toEqual( - mixed - ? [["scripts/ui.js", "build"]] - : [["scripts/prepare-vitest-runtime.mjs"], ["scripts/ui.js", "build"]], - ); + expect(commands.prepare.mock.calls.map(([command]) => command.args)).toEqual([ + ["scripts/prepare-vitest-runtime.mjs", "--require-current-head"], + ["scripts/ui.js", "build"], + ]); expect(commands.uiAssets).toHaveBeenCalledTimes(2); expect(process.exitCode).toBe(0); }, @@ -1334,7 +1337,7 @@ describe("test-projects build admission", () => { await terminal.promise; expect(commands.reader).not.toHaveBeenCalled(); expect(commands.prepare.mock.calls.map(([command]) => command.args)).toEqual([ - ["scripts/prepare-vitest-runtime.mjs"], + ["scripts/prepare-vitest-runtime.mjs", "--require-current-head"], ["scripts/ui.js", "build"], ]); expect(process.exitCode).toBe(outcome === "nonzero" ? 7 : 1); @@ -1373,9 +1376,9 @@ describe("test-projects build admission", () => { source.resolve(); await rejected; } - expect(commands.prepare.mock.calls.map(([command]) => command.args)).toEqual( - mixed ? [] : [["scripts/prepare-vitest-runtime.mjs"]], - ); + expect(commands.prepare.mock.calls.map(([command]) => command.args)).toEqual([ + ["scripts/prepare-vitest-runtime.mjs", "--require-current-head"], + ]); expect(commands.uiAssets).not.toHaveBeenCalled(); expect(commands.reader).not.toHaveBeenCalled(); expect(exitBySignal).toHaveBeenCalledExactlyOnceWith("SIGTERM"); diff --git a/test/scripts/tsdown-declaration-fixture.ts b/test/scripts/tsdown-declaration-fixture.ts index 6ade0aeb7d70..490df2ee68f5 100644 --- a/test/scripts/tsdown-declaration-fixture.ts +++ b/test/scripts/tsdown-declaration-fixture.ts @@ -183,6 +183,7 @@ export function createFixture(groups: readonly string[], root: string) { "generate-kysely-types.mts", "tsdown-build.mts", "pnpm-runner.mts", + "run-node-watch-paths.mts", "windows-cmd-helpers.mjs", "write-plugin-sdk-entry-dts.ts", "write-unified-entry-dts.ts",