fix(infra): preserve definite snapshot allocation refusals

Keep invalid snapshot staging roots actionable without poisoning cleanup of
valid concurrent or nested snapshots. Record completed directory-creation
refusals at the native producer while preserving error identity and codes,
and carry that fact through the existing two-field worker result.

Lost replies, failed token admission, incomplete cleanup, and native errors
after successful creation still retain unresolved allocation custody. Extend
the physical token-lifetime fixture with a rejected root before a valid nested
allocation, and keep reader-fence and native uncertainty controls intact.
Include the new helper in the materialized PR wrapper's runtime inventory.

Local validation: 58 focused tests passed with four Windows-only cases skipped
on macOS Node 24.21.0; wrapper import closure, selected changed checks, and
independent P2 review passed.
This commit is contained in:
Peter Steinberger 2026-09-29 11:09:56 -07:00
parent ee62299caa
commit 8abecdbee2
12 changed files with 109 additions and 12 deletions

View file

@ -227,6 +227,10 @@ keeps every token until copied data is removed, so partial removal remains recov
Readers created after a runtime module reload retain the original snapshot cleanup
owner. Shutdown joins in-flight snapshot consumers across reloads, active readers
still prevent removal, and failed cleanup retains its custody.
When the native directory creator confirms that it refused an allocation before
creating a directory, the original staging-root error is reported and existing
snapshot lifetimes can still retire. Lost replies and incomplete cleanup retain
their original cleanup custody.
Native termination and hard kills can skip that drain. Each staging directory holds
an open SQLite transaction as its lifetime token. Reclamation obtains exclusive
tokens for the parent and every nested worker before inspecting or removing the

View file

@ -740,6 +740,7 @@ src/infra/path-guards.ts
src/infra/plain-object.ts
src/infra/ports-lsof.ts
src/infra/ports-netstat.ts
src/infra/private-directory-creation.ts
src/infra/private-mode.ts
src/infra/process-env.ts
src/infra/prototype-keys.ts

View file

@ -0,0 +1,13 @@
const refusedCreations = new WeakSet<object>();
/** Record a completed native refusal without replacing its code, errno, or identity. */
export function markPrivateDirectoryCreationRefused<T>(error: T): T {
if (error !== null && typeof error === "object") {
refusedCreations.add(error);
}
return error;
}
export function isPrivateDirectoryCreationRefused(error: unknown): boolean {
return error !== null && typeof error === "object" && refusedCreations.has(error);
}

View file

@ -4,6 +4,7 @@ import fsSync from "node:fs";
import fs from "node:fs/promises";
import path from "node:path";
import { resolveRequiredOsHomeDir } from "./home-dir.js";
import { markPrivateDirectoryCreationRefused } from "./private-directory-creation.js";
import { resolvePreferredOpenClawTmpDir } from "./tmp-openclaw-dir.js";
import { createPrivateWindowsDirectory } from "./windows-private-directory.js";
@ -46,7 +47,11 @@ export async function createPrivateSqliteTempDirectory(
export function createPrivateSqliteTempDirectorySync(rootPath: string, prefix: string): string {
if (process.platform !== "win32") {
return fsSync.mkdtempSync(path.join(rootPath, prefix));
try {
return fsSync.mkdtempSync(path.join(rootPath, prefix));
} catch (error) {
throw markPrivateDirectoryCreationRefused(error);
}
}
const directoryPath = path.join(rootPath, `${prefix}${randomUUID()}`);
createPrivateWindowsDirectory(directoryPath);

View file

@ -4,6 +4,7 @@ import fs from "node:fs/promises";
import path from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
import { isPrivateDirectoryCreationRefused } from "./private-directory-creation.js";
vi.mock("node:child_process", async (importOriginal) => {
const actual = await importOriginal<typeof import("node:child_process")>();
@ -147,8 +148,14 @@ describe.runIf(process.platform === "win32")("private SQLite directory creation
const root = tempDirs.make("openclaw-sqlite-private-failure-");
const regularFile = path.join(root, "parent-file");
await fs.writeFile(regularFile, "not a directory");
await expect(createPrivateSqliteDirectory(path.join(regularFile, "child"))).rejects.toThrow(
/CreateDirectoryW.*Win32 error/u,
const failure = await createPrivateSqliteDirectory(path.join(regularFile, "child")).catch(
(error: unknown) => error,
);
expect(failure).toMatchObject({
message: expect.stringMatching(/CreateDirectoryW.*Win32 error/u),
code: "EIO",
errno: expect.any(Number),
});
expect(isPrivateDirectoryCreationRefused(failure)).toBe(true);
});
});

View file

@ -54,6 +54,35 @@ async function expectWorkerFailure(
}
describe("SQLite read-only worker diagnostics", () => {
it.each([
["staging-create", undefined, true],
["staging-create-legacy", undefined, true],
["staging-retire", undefined, false],
["async", undefined, false],
["staging-create", "child exited before completion", false],
] as const)(
"accepts allocation refusal receipts only for completed staging requests (%s, %s)",
async (mode, failure, refused) => {
const {
readSqliteReadOnlyWorkerValue,
SQLITE_SNAPSHOT_ALLOCATION_REFUSED_PREFIX,
SqliteSnapshotAllocationRefusedError,
} = await import("./sqlite-readonly-worker-protocol.js");
const stdout = JSON.stringify({
ok: false,
message: `${SQLITE_SNAPSHOT_ALLOCATION_REFUSED_PREFIX}native directory refusal`,
});
let received: unknown;
try {
readSqliteReadOnlyWorkerValue({ stdout, stderr: "", failure }, mode);
} catch (error) {
received = error;
}
expect(received).toBeInstanceOf(Error);
expect(received instanceof SqliteSnapshotAllocationRefusedError).toBe(refused);
},
);
it("reads cause metadata once through the registered worker", async () => {
let causeReads = 0;
const failure = Object.defineProperty(new Error("open failure"), "cause", {

View file

@ -1,6 +1,7 @@
import path from "node:path";
import { setImmediate } from "node:timers/promises";
import { isRecord } from "@openclaw/normalization-core/record-coerce";
import { isPrivateDirectoryCreationRefused } from "./private-directory-creation.js";
import { SQLITE_READONLY_CHILD_ARG } from "./runtime-process-entrypoints.js";
import {
formatSqliteErrorCodeSuffix,
@ -22,6 +23,7 @@ import type { PreparedSqliteReadOnlyLocation } from "./sqlite-readonly-location.
import {
SQLITE_READONLY_WORKER_MAX_BUFFER,
SQLITE_INSPECTION_CONTENTION_PREFIX,
SQLITE_SNAPSHOT_ALLOCATION_REFUSED_PREFIX,
isSqliteSnapshotStagingMode,
type SqliteReadOnlyWorkerResult,
} from "./sqlite-readonly-worker-protocol.js";
@ -149,9 +151,17 @@ async function inspect(args: string[]): Promise<SqliteReadOnlyWorkerResult> {
return { ok: true, location: prepared.location };
} catch (error) {
const contention = error instanceof SqliteSourceChangedError || isSqliteLockError(error);
const allocationRefused =
(mode === "staging-create" || mode === "staging-create-legacy") &&
isPrivateDirectoryCreationRefused(error);
const prefix = allocationRefused
? SQLITE_SNAPSHOT_ALLOCATION_REFUSED_PREFIX
: contention
? SQLITE_INSPECTION_CONTENTION_PREFIX
: "";
return {
ok: false,
message: `${contention ? SQLITE_INSPECTION_CONTENTION_PREFIX : ""}${formatSqliteReadOnlyInspectionFailure(error)}`,
message: `${prefix}${formatSqliteReadOnlyInspectionFailure(error)}`,
};
}
}

View file

@ -14,6 +14,7 @@ import type {
SqliteNativeRequest,
SqliteNativeSessionLaunch,
} from "./sqlite-readonly-native-resource.types.js";
import { SqliteSnapshotAllocationRefusedError } from "./sqlite-readonly-worker-protocol.js";
import {
createScopedSqliteReadOnlyWorker,
runSqliteReadOnlyWorkerOnce,
@ -347,7 +348,11 @@ export function createNativeWorkerResource(
}
} catch (error) {
// Validation, missing sessions and factory refusal never enter this dispatched boundary.
if (allocating && !session.native.notStarted) {
if (
allocating &&
!session.native.notStarted &&
!(error instanceof SqliteSnapshotAllocationRefusedError)
) {
uncertainAllocations.push(
new SqliteSnapshotCleanupError(
"SQLite snapshot allocation has no exact directory receipt; cleanup is unresolved",

View file

@ -33,10 +33,13 @@ export type SqliteReadOnlyWorkerResult =
| { ok: false; message: string };
export class SqliteReadOnlyInspectionContentionError extends Error {}
export class SqliteSnapshotAllocationRefusedError extends Error {}
// Released updater parents require exactly { ok, message }. A negotiated worker
// protocol can replace this owner-generated tag when those parents are retired.
export const SQLITE_INSPECTION_CONTENTION_PREFIX = "Retryable SQLite inspection contention: ";
export const SQLITE_SNAPSHOT_ALLOCATION_REFUSED_PREFIX =
"SQLite snapshot directory creation refused: ";
export type SqliteAuthProfileRows = { store: unknown; state: unknown; cacheable: boolean };
export type SqliteAuthProfileReadOptions = {
@ -154,14 +157,23 @@ export function readSqliteReadOnlyWorkerValue(
}
if (params.failure || !result.ok) {
const contention = !result.ok && result.message.startsWith(SQLITE_INSPECTION_CONTENTION_PREFIX);
const allocationRefused =
!params.failure &&
!result.ok &&
(mode === "staging-create" || mode === "staging-create-legacy") &&
result.message.startsWith(SQLITE_SNAPSHOT_ALLOCATION_REFUSED_PREFIX);
const prefix = allocationRefused
? SQLITE_SNAPSHOT_ALLOCATION_REFUSED_PREFIX
: contention
? SQLITE_INSPECTION_CONTENTION_PREFIX
: "";
const error = createSqliteReadOnlyWorkerError(
!result.ok
? contention
? result.message.slice(SQLITE_INSPECTION_CONTENTION_PREFIX.length)
: result.message
: (params.failure ?? "failed"),
!result.ok ? result.message.slice(prefix.length) : (params.failure ?? "failed"),
params.stderr,
);
if (allocationRefused) {
throw new SqliteSnapshotAllocationRefusedError(error.message);
}
if (contention) {
throw new SqliteReadOnlyInspectionContentionError(error.message);
}

View file

@ -89,6 +89,9 @@ it("shares one token process across concurrent and nested async snapshot lifetim
let tokenPid: number;
try {
await Promise.all(allocations);
await expect(
createSqliteSnapshotStagingDirectory(path.join(cache, "missing"), false, undefined, true),
).rejects.toThrow("snapshot staging root");
const nested = await createSqliteSnapshotStagingDirectory(
directories[0],
false,

View file

@ -4,6 +4,10 @@ import path from "node:path";
import { extractErrorCode } from "@openclaw/normalization-core/error-coercion";
import { getChildLogger } from "../logging/logger.js";
import { formatErrorMessage } from "./errors.js";
import {
isPrivateDirectoryCreationRefused,
markPrivateDirectoryCreationRefused,
} from "./private-directory-creation.js";
import { markSqliteInspectionOperation } from "./sqlite-error-diagnostics.js";
import {
createPrivateSqliteTempDirectorySync,
@ -191,7 +195,10 @@ export function sqliteSnapshotStagingError(
? "free disk space/quota"
: "check filesystem health and write permissions";
const message = `${cause instanceof Error ? cause.message : String(cause)}${sqliteErrcode !== undefined ? ` (SQLite errcode=${sqliteErrcode})` : ""}; snapshot staging root ${allocation ? tempDir : path.dirname(tempDir)}: ${guidance} or set XDG_CACHE_HOME to a writable filesystem`;
return new Error(message, { cause });
const error = new Error(message, { cause });
return isPrivateDirectoryCreationRefused(cause)
? markPrivateDirectoryCreationRefused(error)
: error;
}
export async function createSqliteSnapshotStagingDirectory(

View file

@ -1,6 +1,7 @@
import fs from "node:fs";
import { createRequire } from "node:module";
import path from "node:path";
import { markPrivateDirectoryCreationRefused } from "./private-directory-creation.js";
const require = createRequire(import.meta.url);
type PrivatePathCreators = {
@ -110,7 +111,7 @@ function loadPrivatePathCreators(): PrivatePathCreators {
directory: (directoryPath) =>
withPrivateAttributes(true, (security) => {
if (!createDirectory(path.toNamespacedPath(path.resolve(directoryPath)), security)) {
throw failure(`CreateDirectoryW(${directoryPath})`);
throw markPrivateDirectoryCreationRefused(failure(`CreateDirectoryW(${directoryPath})`));
}
}),
file: (filePath) =>