diff --git a/docs/cli/update/how-updates-run.md b/docs/cli/update/how-updates-run.md index 5b1a1b7b6d96..e4ce45ce3ce8 100644 --- a/docs/cli/update/how-updates-run.md +++ b/docs/cli/update/how-updates-run.md @@ -697,6 +697,10 @@ the sentinel. Stable, beta, and dev updates install dependencies and build in a temporary worktree while the old Gateway serves. Dev rebases the staged checkout first so local commits are preserved and the build validates the exact source that will be activated. On POSIX, staging uses a private directory in the checkout's existing ignored `.artifacts` area. By default, the full workspace stays on the checkout filesystem, not a potentially small system temporary filesystem. An existing `.artifacts` redirect is honored as an operator storage choice, just like the build cache. Existing checkout, parent, and artifact directory permissions are not changed. Windows keeps its short system-drive staging path. Only dev updates walk back through earlier commits; stable and beta updates validate their selected target. + Git object transfer reads its prepared pack directly from disk. Packs above 256 MiB record a size warning and continue when the installed Git object volume has room for the measured pack and index. A known shortfall reports `snapshot-capacity-insufficient` before stopping the Gateway; unknown free space remains a warning. The pack import duration is recorded with the update steps. This check is separate from state-snapshot placement and runtime build-cache exclusions. + + This repair runs in the installed updater. An older updater that refuses a pack above its fixed limit cannot acquire the repair through that same failing update; update the source installation manually using the [source-checkout reference script](/install/updating/update-methods#source-checkout-servers-reference-script). + The updater prepares the built runtime (`dist`, `dist-runtime`, and dependencies, including nested workspace outputs) on the destination filesystem and removes the temporary Git worktree registration before changing the live checkout. Cleanup failures remain visible in the update result. If an interruption leaves staging behind, artifact-area staging does not dirty the checkout or block the next update's clean check. Dev can walk back up to 10 commits to find the newest buildable version. Confirmed ENOSPC storage failures stop immediately with `preflight-insufficient-space`; free space on the preflight staging and package-manager store filesystems before retrying. Shared package-manager stores are not deleted. Update builds skip TypeScript declaration generation by default. Set `OPENCLAW_RUN_NODE_SKIP_DTS_BUILD=0` to explicitly request declarations. Set `OPENCLAW_UPDATE_PREFLIGHT_LINT=1` to also run source lint during this preflight; lint runs in constrained serial mode because user update hosts are often smaller than CI runners. diff --git a/src/infra/update-runner-git-admission.test.ts b/src/infra/update-runner-git-admission.test.ts index 38747dbd8db5..965ed2fad341 100644 --- a/src/infra/update-runner-git-admission.test.ts +++ b/src/infra/update-runner-git-admission.test.ts @@ -7,6 +7,7 @@ import { pathToFileURL } from "node:url"; import { afterEach, describe, expect, it, vi } from "vitest"; import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; import { withEnvAsync } from "../test-utils/env.js"; +import * as diskSpace from "./disk-space.js"; import { renderUpdateRunReport, updateRunReportInputFromResult } from "./update-run-report.js"; import { buildUpdateCommandRunner } from "./update-runner-command.js"; import { updateGitCheckout } from "./update-runner-git.js"; @@ -235,6 +236,51 @@ describe("Git database admission", () => { }, ); + it("refuses insufficient object-volume capacity before stopping the Gateway", async () => { + const state = fixture(); + const before = state.git(state.install, "rev-parse", "HEAD"); + const prepareMutation = vi.fn(); + const capacity = vi.spyOn(diskSpace, "tryReadDiskSpace").mockReturnValue({ + targetPath: state.install, + checkedPath: state.install, + availableBytes: 0, + totalBytes: 1024, + }); + try { + const result = await state.run({ beforeGitMutation: prepareMutation }); + expect(result).toMatchObject({ status: "error", reason: "snapshot-capacity-insufficient" }); + expect(result.steps).toContainEqual( + expect.objectContaining({ + name: "git update pack capacity", + stderrTail: expect.stringContaining("0 bytes available"), + }), + ); + expect(prepareMutation).not.toHaveBeenCalled(); + expect(state.git(state.install, "rev-parse", "HEAD")).toBe(before); + } finally { + capacity.mockRestore(); + } + }); + + it("continues with a warning when object-volume capacity is unknown", async () => { + const state = fixture(); + const capacity = vi.spyOn(diskSpace, "tryReadDiskSpace").mockReturnValue(null); + try { + const result = await state.run({ beforeGitMutation: async () => undefined }); + expect(result.status, JSON.stringify(result)).toBe("ok"); + expect(result.steps).toContainEqual( + expect.objectContaining({ + name: "git update pack capacity", + exitCode: 0, + warnings: [expect.stringContaining("free space could not be measured")], + }), + ); + expect(state.git(state.install, "rev-parse", "HEAD")).toBe(state.target); + } finally { + capacity.mockRestore(); + } + }); + it("does not release another owner's keep file after import", async () => { const state = fixture(); let keepPath = ""; diff --git a/src/infra/update-runner-git-transfer.test.ts b/src/infra/update-runner-git-transfer.test.ts index 5708c270c60c..c0e2a948adb3 100644 --- a/src/infra/update-runner-git-transfer.test.ts +++ b/src/infra/update-runner-git-transfer.test.ts @@ -120,13 +120,15 @@ it "none", "inventory", "missing-pack", + "large-pack", "retry", "missing-before", "legacy-git", "configured-limit", - ] as const)("stages complete Git transfers (failure=%s)", async (failure) => { + ] as const)("transfers Git objects without buffering the pack (scenario=%s)", async (failure) => { const overflow = failure === "inventory"; const missingPack = failure === "missing-pack"; + const largePack = failure === "large-pack"; const root = temporary.make("git-transfer-bounds-"); const source = path.join(root, "source"); const install = path.join(root, "install"); @@ -179,6 +181,13 @@ it ); fs.writeFileSync(path.join(source, `object-${index}`), bytes); } + if (largePack) { + // Uncompressed Git objects keep this sparse fixture above the old pack cap. + await git(source, "config", "core.compression", "0"); + const payload = path.join(source, "large-payload"); + fs.writeFileSync(payload, ""); + fs.truncateSync(payload, 257 * 1024 * 1024); + } await git(source, "add", "."); await git(source, "commit", "-m", "candidate"); const candidateSha = await git(source, "rev-parse", "HEAD"); @@ -239,7 +248,7 @@ it totalSteps: 1, results, }); - await using initialTransfer = await prepareGitCandidateTransfer({ + const initialTransfer = await prepareGitCandidateTransfer({ candidateSha, beforeSha, installedRoot: install, @@ -266,15 +275,28 @@ it expect(await git(install, "rev-parse", "HEAD")).toBe(beforeSha); return; } - expect(transfer).toBeDefined(); + expect(transfer, JSON.stringify(results.filter((entry) => entry.exitCode !== 0))).toBeDefined(); + if (transfer?.status !== "ok") { + throw new Error("Git transfer preparation failed"); + } + await using admittedTransfer = transfer; expect(inventoryBytes).toBeGreaterThan(8000); if (failure === "none") { // The pinned descriptor survives removal of the staging pathname. const packName = fs.readdirSync(source).find((name) => name.endsWith(".pack"))!; fs.unlinkSync(path.join(source, packName)); } - expect(await transfer!.importInto(step(install))).toBe(true); + expect(await admittedTransfer.importInto(step(install))).toBe(true); expect(packBytes).toBeGreaterThan(8000); + if (largePack) { + expect(packBytes).toBeGreaterThan(256 * 1024 * 1024); + expect(results).toContainEqual( + expect.objectContaining({ + exitCode: 0, + warnings: [expect.stringContaining("Large Git update pack")], + }), + ); + } if (failure === "none") { expect(packBytes).toBeLessThan(baseBytes.length); } @@ -286,7 +308,7 @@ it const inspection = path.join(root, "inspection.git"); await git(root, "clone", "--mirror", "--shared", install, inspection); await git(inspection, "update-ref", "refs/heads/candidate", candidateSha); - await using retryTransfer = await prepareGitCandidateTransfer({ + const retryTransfer = await prepareGitCandidateTransfer({ candidateSha, beforeSha, installedRoot: install, @@ -295,8 +317,12 @@ it step: step(inspection), }); transfer = retryTransfer; - expect(transfer).toBeDefined(); - expect(await transfer!.importInto(step(install))).toBe(true); + expect(transfer, JSON.stringify(results.filter((entry) => entry.exitCode !== 0))).toBeDefined(); + if (transfer?.status !== "ok") { + throw new Error("Git transfer retry preparation failed"); + } + await using admittedRetryTransfer = transfer; + expect(await admittedRetryTransfer.importInto(step(install))).toBe(true); await git(install, "repack", "-a", "-d"); } await git(install, "checkout", "--detach", candidateSha); @@ -305,6 +331,12 @@ it expect(packBytes).toBeGreaterThan(1024 * 1024); expect(await git(source, "config", "pack.packSizeLimit")).toBe("1m"); } + if (largePack) { + expect(fs.statSync(path.join(install, "large-payload")).size).toBe(257 * 1024 * 1024); + expect(await git(install, "rev-parse", "HEAD:large-payload")).toBe( + await git(source, "rev-parse", "HEAD:large-payload"), + ); + } expect(fs.readFileSync(path.join(install, "base"))).toEqual(baseBytes); for (let index = 0; index < 250; index++) { expect(fs.readFileSync(path.join(install, `object-${index}`))).toEqual( diff --git a/src/infra/update-runner-git-transfer.ts b/src/infra/update-runner-git-transfer.ts index ae4a1c033bd3..177c7b5dfe20 100644 --- a/src/infra/update-runner-git-transfer.ts +++ b/src/infra/update-runner-git-transfer.ts @@ -1,12 +1,15 @@ import { randomUUID } from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; +import { tryReadDiskSpace } from "./disk-space.js"; import { hasErrnoCode } from "./errno.js"; import { openLocalFileSafely, type OpenResult } from "./fs-safe.js"; import { runStep } from "./update-runner-command.js"; import { classifyPartialCloneGitFailure } from "./update-runner-git-target.js"; import type { RunStepOptions, UpdateStepResult } from "./update-runner-types.js"; +const LARGE_CANDIDATE_PACK_WARNING_BYTES = 256 * 1024 * 1024; + function recordStagingFailure( step: RunStepOptions, name: string, @@ -191,13 +194,16 @@ export async function prepareGitCandidateTransfer(params: { if (!hash) { return undefined; } + await using stagedPack = new AsyncDisposableStack(); let pack: OpenResult; const packPath = `${prefix}-${hash}.pack`; const readStarted = Date.now(); + let requiredBytes: number; try { // Pin the staged file before admission; Git reads this descriptor directly // instead of retaining and copying the entire pack through JavaScript. - pack = await openLocalFileSafely({ filePath: packPath }); + pack = stagedPack.use(await openLocalFileSafely({ filePath: packPath })); + requiredBytes = pack.stat.size + (await fs.stat(`${prefix}-${hash}.idx`)).size; } catch (error) { return recordStagingFailure( step, @@ -207,9 +213,54 @@ export async function prepareGitCandidateTransfer(params: { Date.now() - readStarted, ); } + const objectDirectory = await runGit( + "git update object directory", + ["rev-parse", "--path-format=absolute", "--git-path", "objects"], + undefined, + installedRoot, + ); + if (!objectDirectory) { + return undefined; + } + // Objects must live on this volume; the state snapshot allocator still owns + // choosing among temporary volumes for the separate rollback snapshot. + const capacity = tryReadDiskSpace(objectDirectory); + if (capacity && capacity.availableBytes < requiredBytes) { + const reason = "snapshot-capacity-insufficient" as const; + recordStagingFailure( + step, + "git update pack capacity", + "measure Git update pack capacity", + `${reason}: Git update pack and index need ${requiredBytes} bytes in ${objectDirectory}; ${capacity.availableBytes} bytes available. Free space on this volume and retry; the installed checkout is unchanged.`, + Date.now() - readStarted, + ); + return { status: "error" as const, reason }; + } + const warnings: string[] = []; + if (pack.stat.size > LARGE_CANDIDATE_PACK_WARNING_BYTES) { + warnings.push( + `Large Git update pack: ${pack.stat.size} bytes; importing from disk without buffering it in memory.`, + ); + } + if (!capacity) { + warnings.push("Git object-volume free space could not be measured; continuing the update."); + } + const measured: UpdateStepResult = { + name: "git update pack capacity", + command: "measure Git update pack capacity", + cwd: installedRoot, + durationMs: Date.now() - readStarted, + exitCode: 0, + stdoutTail: `Git update pack and index: ${requiredBytes} bytes; ${capacity ? `${capacity.availableBytes} bytes available` : "free space unknown"} in ${objectDirectory}.`, + ...(warnings.length ? { warnings } : {}), + }; + step.results?.push(measured); + step.progress?.onStepComplete?.({ ...measured, index: step.stepIndex, total: step.totalSteps }); const keepMessage = `openclaw-update-${randomUUID()}`; + const retainedPack = stagedPack.move(); return { - [Symbol.asyncDispose]: () => pack[Symbol.asyncDispose](), + status: "ok" as const, + [Symbol.asyncDispose]: () => retainedPack[Symbol.asyncDispose](), async importInto(target: RunStepOptions): Promise { const imported = await runStep({ ...target, diff --git a/src/infra/update-runner-git.ts b/src/infra/update-runner-git.ts index af3ddbbc0e67..53a4cbb311d8 100644 --- a/src/infra/update-runner-git.ts +++ b/src/infra/update-runner-git.ts @@ -94,7 +94,9 @@ export async function updateGitCheckout(params: { let mutationPrepared = false; let sourceMutationStarted = false; let runtimePromotion: Awaited> | undefined; - let candidateTransfer: Awaited>; + let candidateTransfer: + | Extract>, { status: "ok" }> + | undefined; let stateMigrationStarted = false; let recovery = await verifyGitUpdateRecovery({ root: gitRoot, sha: beforeSha }); let rollbackOutcome: NonNullable = { @@ -352,7 +354,7 @@ export async function updateGitCheckout(params: { const importCandidate = async (candidateSha: string, upstreamRef?: string) => { // Close the pinned pack on every exit, including admission refusal, // before the surrounding inspection checkout is removed. - await using transfer = await prepareGitCandidateTransfer({ + const transfer = await prepareGitCandidateTransfer({ candidateSha, beforeSha, installedRoot: gitRoot, @@ -361,16 +363,17 @@ export async function updateGitCheckout(params: { step: inspectionWorkStep("git-pack-update", [], inspectionRoot), probeTimeoutMs: timeoutMs, }); - if (!transfer) { - return { status: "error" as const, reason: "fetch-failed" }; + if (!transfer || transfer.status === "error") { + return { status: "error" as const, reason: transfer?.reason ?? "fetch-failed" }; } + await using admittedTransfer = transfer; const sourceChanged = await checkSourceUnchanged(); if (sourceChanged) { return sourceChanged; } await prepareMutation(candidateSha, inspectionRoot, runInspectionCommand); candidateTransfer = transfer; - const imported = await transfer.importInto( + const imported = await admittedTransfer.importInto( workStep("git-import-admitted-target", [], gitRoot), ); if (!imported) { diff --git a/src/process/exec-runner.ts b/src/process/exec-runner.ts index faf47cb38b8b..cf5632ddb3a1 100644 --- a/src/process/exec-runner.ts +++ b/src/process/exec-runner.ts @@ -137,6 +137,7 @@ async function runCommandWithOutputEncoding( timeoutMs, cwd, input, + stdinFileDescriptor, baseEnv, env, noOutputTimeoutMs, @@ -151,8 +152,13 @@ async function runCommandWithOutputEncoding( throw new Error("Process-tree extinction requires process-tree ownership"); } const hasInput = input !== undefined; - if (hasInput && options.stdinFileDescriptor !== undefined) { - throw new Error("Command accepts either input or stdinFileDescriptor, not both"); + if (stdinFileDescriptor !== undefined) { + if (!Number.isInteger(stdinFileDescriptor) || stdinFileDescriptor < 0) { + throw new Error("stdinFileDescriptor must be a nonnegative integer"); + } + if (hasInput) { + throw new Error("Command accepts either input or stdinFileDescriptor, not both"); + } } if (options.beforeInput && !hasInput) { throw new Error("Child input admission requires explicit input"); diff --git a/src/process/exec-stdin-file.test.ts b/src/process/exec-stdin-file.test.ts new file mode 100644 index 000000000000..7bb2202574e7 --- /dev/null +++ b/src/process/exec-stdin-file.test.ts @@ -0,0 +1,100 @@ +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import process from "node:process"; +import { afterEach, expect, it } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import { isPidAlive } from "../shared/pid-alive.js"; +import { runCommandWithTimeout, type CommandOptions } from "./exec.js"; + +const temporary = useAutoCleanupTempDirTracker(afterEach); +const digestStdin = `const input=require('node:fs').readFileSync(0); +process.stdout.write(require('node:crypto').createHash('sha256').update(input).digest('hex'));`; + +function createInputFile() { + const root = temporary.make("openclaw-command-stdin-"); + const file = path.join(root, "input"); + const bytes = Buffer.alloc(16_384).fill(Buffer.from([0, 255, 10, 128])); + fs.writeFileSync(file, bytes); + return { file, digest: createHash("sha256").update(bytes).digest("hex") }; +} + +it("inherits binary file input before the caller closes its descriptor", async () => { + const { file, digest } = createInputFile(); + const descriptor = fs.openSync(file, "r"); + let running: ReturnType; + try { + running = runCommandWithTimeout([process.execPath, "-e", digestStdin], { + stdinFileDescriptor: descriptor, + timeoutMs: 5_000, + killProcessTree: true, + }); + } finally { + fs.closeSync(descriptor); + } + const result = await running; + expect(result).toMatchObject({ code: 0, termination: "exit", stdout: digest, stderr: "" }); +}); + +it("settles cancellation without closing the caller's file descriptor", async () => { + const { file, digest } = createInputFile(); + const descriptor = fs.openSync(file, "r"); + const controller = new AbortController(); + let outputBytes = 0; + try { + const result = await runCommandWithTimeout( + [process.execPath, "-e", `setInterval(()=>{},1000);${digestStdin}`], + { + stdinFileDescriptor: descriptor, + signal: controller.signal, + timeoutMs: 5_000, + killProcessTree: true, + onOutputChunk: (chunk, stream) => { + if (stream === "stdout") { + outputBytes += chunk.length; + if (outputBytes === digest.length) { + controller.abort(); + } + } + }, + }, + ); + expect(result).toMatchObject({ termination: "signal", stdout: digest }); + expect(result.pid).toBeTypeOf("number"); + expect(isPidAlive(result.pid!)).toBe(false); + expect(fs.fstatSync(descriptor).isFile()).toBe(true); + } finally { + fs.closeSync(descriptor); + } +}); + +it.each<{ options: CommandOptions; message: string }>([ + { + options: { stdinFileDescriptor: 0, input: "payload" }, + message: "either input or stdinFileDescriptor", + }, + { + options: { stdinFileDescriptor: 0, beforeInput: () => undefined }, + message: "admission requires explicit input", + }, + { options: { stdinFileDescriptor: -1 }, message: "nonnegative integer" }, + { options: { stdinFileDescriptor: 0.5 }, message: "nonnegative integer" }, +])( + "rejects ambiguous or invalid file input before spawning: $options", + async ({ options, message }) => { + const root = temporary.make("openclaw-command-stdin-rejected-"); + const marker = path.join(root, "spawned"); + await expect( + runCommandWithTimeout( + [ + process.execPath, + "-e", + "require('node:fs').writeFileSync(process.argv[1], 'spawned')", + marker, + ], + { ...options, timeoutMs: 5_000 }, + ), + ).rejects.toThrow(message); + expect(fs.existsSync(marker)).toBe(false); + }, +);