diff --git a/docs/reference/database-schemas/integrity-and-recovery.md b/docs/reference/database-schemas/integrity-and-recovery.md index fc88a252a3cf..e7b0746e4797 100644 --- a/docs/reference/database-schemas/integrity-and-recovery.md +++ b/docs/reference/database-schemas/integrity-and-recovery.md @@ -227,6 +227,10 @@ keeps every token until copied data is removed, so partial removal remains recov Readers created after a runtime module reload retain the original snapshot cleanup owner. Shutdown joins in-flight snapshot consumers across reloads, active readers still prevent removal, and failed cleanup retains its custody. +When the native directory creator confirms that it refused an allocation before +creating a directory, the original staging-root error is reported and existing +snapshot lifetimes can still retire. Lost replies and incomplete cleanup retain +their original cleanup custody. Native termination and hard kills can skip that drain. Each staging directory holds an open SQLite transaction as its lifetime token. Reclamation obtains exclusive tokens for the parent and every nested worker before inspecting or removing the diff --git a/scripts/pr-lib/wrapper-components.txt b/scripts/pr-lib/wrapper-components.txt index a573278b6696..0377375a1439 100644 --- a/scripts/pr-lib/wrapper-components.txt +++ b/scripts/pr-lib/wrapper-components.txt @@ -740,6 +740,7 @@ src/infra/path-guards.ts src/infra/plain-object.ts src/infra/ports-lsof.ts src/infra/ports-netstat.ts +src/infra/private-directory-creation.ts src/infra/private-mode.ts src/infra/process-env.ts src/infra/prototype-keys.ts diff --git a/src/infra/private-directory-creation.ts b/src/infra/private-directory-creation.ts new file mode 100644 index 000000000000..7de5bc68d472 --- /dev/null +++ b/src/infra/private-directory-creation.ts @@ -0,0 +1,13 @@ +const refusedCreations = new WeakSet(); + +/** Record a completed native refusal without replacing its code, errno, or identity. */ +export function markPrivateDirectoryCreationRefused(error: T): T { + if (error !== null && typeof error === "object") { + refusedCreations.add(error); + } + return error; +} + +export function isPrivateDirectoryCreationRefused(error: unknown): boolean { + return error !== null && typeof error === "object" && refusedCreations.has(error); +} diff --git a/src/infra/sqlite-private-directory.ts b/src/infra/sqlite-private-directory.ts index 82473df0da1d..86b3e5141c72 100644 --- a/src/infra/sqlite-private-directory.ts +++ b/src/infra/sqlite-private-directory.ts @@ -4,6 +4,7 @@ import fsSync from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; import { resolveRequiredOsHomeDir } from "./home-dir.js"; +import { markPrivateDirectoryCreationRefused } from "./private-directory-creation.js"; import { resolvePreferredOpenClawTmpDir } from "./tmp-openclaw-dir.js"; import { createPrivateWindowsDirectory } from "./windows-private-directory.js"; @@ -46,7 +47,11 @@ export async function createPrivateSqliteTempDirectory( export function createPrivateSqliteTempDirectorySync(rootPath: string, prefix: string): string { if (process.platform !== "win32") { - return fsSync.mkdtempSync(path.join(rootPath, prefix)); + try { + return fsSync.mkdtempSync(path.join(rootPath, prefix)); + } catch (error) { + throw markPrivateDirectoryCreationRefused(error); + } } const directoryPath = path.join(rootPath, `${prefix}${randomUUID()}`); createPrivateWindowsDirectory(directoryPath); diff --git a/src/infra/sqlite-private-directory.windows.test.ts b/src/infra/sqlite-private-directory.windows.test.ts index ddfb20366177..4231054aed56 100644 --- a/src/infra/sqlite-private-directory.windows.test.ts +++ b/src/infra/sqlite-private-directory.windows.test.ts @@ -4,6 +4,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import { isPrivateDirectoryCreationRefused } from "./private-directory-creation.js"; vi.mock("node:child_process", async (importOriginal) => { const actual = await importOriginal(); @@ -147,8 +148,14 @@ describe.runIf(process.platform === "win32")("private SQLite directory creation const root = tempDirs.make("openclaw-sqlite-private-failure-"); const regularFile = path.join(root, "parent-file"); await fs.writeFile(regularFile, "not a directory"); - await expect(createPrivateSqliteDirectory(path.join(regularFile, "child"))).rejects.toThrow( - /CreateDirectoryW.*Win32 error/u, + const failure = await createPrivateSqliteDirectory(path.join(regularFile, "child")).catch( + (error: unknown) => error, ); + expect(failure).toMatchObject({ + message: expect.stringMatching(/CreateDirectoryW.*Win32 error/u), + code: "EIO", + errno: expect.any(Number), + }); + expect(isPrivateDirectoryCreationRefused(failure)).toBe(true); }); }); diff --git a/src/infra/sqlite-readonly-location.worker.test.ts b/src/infra/sqlite-readonly-location.worker.test.ts index b7a33f1a5bcc..bd728e59d686 100644 --- a/src/infra/sqlite-readonly-location.worker.test.ts +++ b/src/infra/sqlite-readonly-location.worker.test.ts @@ -54,6 +54,35 @@ async function expectWorkerFailure( } describe("SQLite read-only worker diagnostics", () => { + it.each([ + ["staging-create", undefined, true], + ["staging-create-legacy", undefined, true], + ["staging-retire", undefined, false], + ["async", undefined, false], + ["staging-create", "child exited before completion", false], + ] as const)( + "accepts allocation refusal receipts only for completed staging requests (%s, %s)", + async (mode, failure, refused) => { + const { + readSqliteReadOnlyWorkerValue, + SQLITE_SNAPSHOT_ALLOCATION_REFUSED_PREFIX, + SqliteSnapshotAllocationRefusedError, + } = await import("./sqlite-readonly-worker-protocol.js"); + const stdout = JSON.stringify({ + ok: false, + message: `${SQLITE_SNAPSHOT_ALLOCATION_REFUSED_PREFIX}native directory refusal`, + }); + let received: unknown; + try { + readSqliteReadOnlyWorkerValue({ stdout, stderr: "", failure }, mode); + } catch (error) { + received = error; + } + expect(received).toBeInstanceOf(Error); + expect(received instanceof SqliteSnapshotAllocationRefusedError).toBe(refused); + }, + ); + it("reads cause metadata once through the registered worker", async () => { let causeReads = 0; const failure = Object.defineProperty(new Error("open failure"), "cause", { diff --git a/src/infra/sqlite-readonly-location.worker.ts b/src/infra/sqlite-readonly-location.worker.ts index 0ca7b536fe6e..18e2306c62df 100644 --- a/src/infra/sqlite-readonly-location.worker.ts +++ b/src/infra/sqlite-readonly-location.worker.ts @@ -1,6 +1,7 @@ import path from "node:path"; import { setImmediate } from "node:timers/promises"; import { isRecord } from "@openclaw/normalization-core/record-coerce"; +import { isPrivateDirectoryCreationRefused } from "./private-directory-creation.js"; import { SQLITE_READONLY_CHILD_ARG } from "./runtime-process-entrypoints.js"; import { formatSqliteErrorCodeSuffix, @@ -22,6 +23,7 @@ import type { PreparedSqliteReadOnlyLocation } from "./sqlite-readonly-location. import { SQLITE_READONLY_WORKER_MAX_BUFFER, SQLITE_INSPECTION_CONTENTION_PREFIX, + SQLITE_SNAPSHOT_ALLOCATION_REFUSED_PREFIX, isSqliteSnapshotStagingMode, type SqliteReadOnlyWorkerResult, } from "./sqlite-readonly-worker-protocol.js"; @@ -149,9 +151,17 @@ async function inspect(args: string[]): Promise { return { ok: true, location: prepared.location }; } catch (error) { const contention = error instanceof SqliteSourceChangedError || isSqliteLockError(error); + const allocationRefused = + (mode === "staging-create" || mode === "staging-create-legacy") && + isPrivateDirectoryCreationRefused(error); + const prefix = allocationRefused + ? SQLITE_SNAPSHOT_ALLOCATION_REFUSED_PREFIX + : contention + ? SQLITE_INSPECTION_CONTENTION_PREFIX + : ""; return { ok: false, - message: `${contention ? SQLITE_INSPECTION_CONTENTION_PREFIX : ""}${formatSqliteReadOnlyInspectionFailure(error)}`, + message: `${prefix}${formatSqliteReadOnlyInspectionFailure(error)}`, }; } } diff --git a/src/infra/sqlite-readonly-native-resource.ts b/src/infra/sqlite-readonly-native-resource.ts index eebb1e8c0507..c8499dec2972 100644 --- a/src/infra/sqlite-readonly-native-resource.ts +++ b/src/infra/sqlite-readonly-native-resource.ts @@ -14,6 +14,7 @@ import type { SqliteNativeRequest, SqliteNativeSessionLaunch, } from "./sqlite-readonly-native-resource.types.js"; +import { SqliteSnapshotAllocationRefusedError } from "./sqlite-readonly-worker-protocol.js"; import { createScopedSqliteReadOnlyWorker, runSqliteReadOnlyWorkerOnce, @@ -347,7 +348,11 @@ export function createNativeWorkerResource( } } catch (error) { // Validation, missing sessions and factory refusal never enter this dispatched boundary. - if (allocating && !session.native.notStarted) { + if ( + allocating && + !session.native.notStarted && + !(error instanceof SqliteSnapshotAllocationRefusedError) + ) { uncertainAllocations.push( new SqliteSnapshotCleanupError( "SQLite snapshot allocation has no exact directory receipt; cleanup is unresolved", diff --git a/src/infra/sqlite-readonly-worker-protocol.ts b/src/infra/sqlite-readonly-worker-protocol.ts index 05cd885b7312..ed668e3aabea 100644 --- a/src/infra/sqlite-readonly-worker-protocol.ts +++ b/src/infra/sqlite-readonly-worker-protocol.ts @@ -33,10 +33,13 @@ export type SqliteReadOnlyWorkerResult = | { ok: false; message: string }; export class SqliteReadOnlyInspectionContentionError extends Error {} +export class SqliteSnapshotAllocationRefusedError extends Error {} // Released updater parents require exactly { ok, message }. A negotiated worker // protocol can replace this owner-generated tag when those parents are retired. export const SQLITE_INSPECTION_CONTENTION_PREFIX = "Retryable SQLite inspection contention: "; +export const SQLITE_SNAPSHOT_ALLOCATION_REFUSED_PREFIX = + "SQLite snapshot directory creation refused: "; export type SqliteAuthProfileRows = { store: unknown; state: unknown; cacheable: boolean }; export type SqliteAuthProfileReadOptions = { @@ -154,14 +157,23 @@ export function readSqliteReadOnlyWorkerValue( } if (params.failure || !result.ok) { const contention = !result.ok && result.message.startsWith(SQLITE_INSPECTION_CONTENTION_PREFIX); + const allocationRefused = + !params.failure && + !result.ok && + (mode === "staging-create" || mode === "staging-create-legacy") && + result.message.startsWith(SQLITE_SNAPSHOT_ALLOCATION_REFUSED_PREFIX); + const prefix = allocationRefused + ? SQLITE_SNAPSHOT_ALLOCATION_REFUSED_PREFIX + : contention + ? SQLITE_INSPECTION_CONTENTION_PREFIX + : ""; const error = createSqliteReadOnlyWorkerError( - !result.ok - ? contention - ? result.message.slice(SQLITE_INSPECTION_CONTENTION_PREFIX.length) - : result.message - : (params.failure ?? "failed"), + !result.ok ? result.message.slice(prefix.length) : (params.failure ?? "failed"), params.stderr, ); + if (allocationRefused) { + throw new SqliteSnapshotAllocationRefusedError(error.message); + } if (contention) { throw new SqliteReadOnlyInspectionContentionError(error.message); } diff --git a/src/infra/sqlite-snapshot-staging.ownership.test.ts b/src/infra/sqlite-snapshot-staging.ownership.test.ts index f6796ea67fa5..0b2baaf28105 100644 --- a/src/infra/sqlite-snapshot-staging.ownership.test.ts +++ b/src/infra/sqlite-snapshot-staging.ownership.test.ts @@ -89,6 +89,9 @@ it("shares one token process across concurrent and nested async snapshot lifetim let tokenPid: number; try { await Promise.all(allocations); + await expect( + createSqliteSnapshotStagingDirectory(path.join(cache, "missing"), false, undefined, true), + ).rejects.toThrow("snapshot staging root"); const nested = await createSqliteSnapshotStagingDirectory( directories[0], false, diff --git a/src/infra/sqlite-snapshot-staging.ts b/src/infra/sqlite-snapshot-staging.ts index b65dffd4cdff..6984c023f1a9 100644 --- a/src/infra/sqlite-snapshot-staging.ts +++ b/src/infra/sqlite-snapshot-staging.ts @@ -4,6 +4,10 @@ import path from "node:path"; import { extractErrorCode } from "@openclaw/normalization-core/error-coercion"; import { getChildLogger } from "../logging/logger.js"; import { formatErrorMessage } from "./errors.js"; +import { + isPrivateDirectoryCreationRefused, + markPrivateDirectoryCreationRefused, +} from "./private-directory-creation.js"; import { markSqliteInspectionOperation } from "./sqlite-error-diagnostics.js"; import { createPrivateSqliteTempDirectorySync, @@ -191,7 +195,10 @@ export function sqliteSnapshotStagingError( ? "free disk space/quota" : "check filesystem health and write permissions"; const message = `${cause instanceof Error ? cause.message : String(cause)}${sqliteErrcode !== undefined ? ` (SQLite errcode=${sqliteErrcode})` : ""}; snapshot staging root ${allocation ? tempDir : path.dirname(tempDir)}: ${guidance} or set XDG_CACHE_HOME to a writable filesystem`; - return new Error(message, { cause }); + const error = new Error(message, { cause }); + return isPrivateDirectoryCreationRefused(cause) + ? markPrivateDirectoryCreationRefused(error) + : error; } export async function createSqliteSnapshotStagingDirectory( diff --git a/src/infra/windows-private-directory.ts b/src/infra/windows-private-directory.ts index b6b9b65938f9..0bd48c2fc28a 100644 --- a/src/infra/windows-private-directory.ts +++ b/src/infra/windows-private-directory.ts @@ -1,6 +1,7 @@ import fs from "node:fs"; import { createRequire } from "node:module"; import path from "node:path"; +import { markPrivateDirectoryCreationRefused } from "./private-directory-creation.js"; const require = createRequire(import.meta.url); type PrivatePathCreators = { @@ -110,7 +111,7 @@ function loadPrivatePathCreators(): PrivatePathCreators { directory: (directoryPath) => withPrivateAttributes(true, (security) => { if (!createDirectory(path.toNamespacedPath(path.resolve(directoryPath)), security)) { - throw failure(`CreateDirectoryW(${directoryPath})`); + throw markPrivateDirectoryCreationRefused(failure(`CreateDirectoryW(${directoryPath})`)); } }), file: (filePath) =>