mirror of
https://github.com/openclaw/openclaw.git
synced 2026-10-04 10:10:01 +00:00
fix(infra): preserve definite snapshot allocation refusals
Keep invalid snapshot staging roots actionable without poisoning cleanup of valid concurrent or nested snapshots. Record completed directory-creation refusals at the native producer while preserving error identity and codes, and carry that fact through the existing two-field worker result. Lost replies, failed token admission, incomplete cleanup, and native errors after successful creation still retain unresolved allocation custody. Extend the physical token-lifetime fixture with a rejected root before a valid nested allocation, and keep reader-fence and native uncertainty controls intact. Include the new helper in the materialized PR wrapper's runtime inventory. Local validation: 58 focused tests passed with four Windows-only cases skipped on macOS Node 24.21.0; wrapper import closure, selected changed checks, and independent P2 review passed.
This commit is contained in:
parent
ee62299caa
commit
8abecdbee2
12 changed files with 109 additions and 12 deletions
|
|
@ -227,6 +227,10 @@ keeps every token until copied data is removed, so partial removal remains recov
|
|||
Readers created after a runtime module reload retain the original snapshot cleanup
|
||||
owner. Shutdown joins in-flight snapshot consumers across reloads, active readers
|
||||
still prevent removal, and failed cleanup retains its custody.
|
||||
When the native directory creator confirms that it refused an allocation before
|
||||
creating a directory, the original staging-root error is reported and existing
|
||||
snapshot lifetimes can still retire. Lost replies and incomplete cleanup retain
|
||||
their original cleanup custody.
|
||||
Native termination and hard kills can skip that drain. Each staging directory holds
|
||||
an open SQLite transaction as its lifetime token. Reclamation obtains exclusive
|
||||
tokens for the parent and every nested worker before inspecting or removing the
|
||||
|
|
|
|||
|
|
@ -740,6 +740,7 @@ src/infra/path-guards.ts
|
|||
src/infra/plain-object.ts
|
||||
src/infra/ports-lsof.ts
|
||||
src/infra/ports-netstat.ts
|
||||
src/infra/private-directory-creation.ts
|
||||
src/infra/private-mode.ts
|
||||
src/infra/process-env.ts
|
||||
src/infra/prototype-keys.ts
|
||||
|
|
|
|||
13
src/infra/private-directory-creation.ts
Normal file
13
src/infra/private-directory-creation.ts
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
const refusedCreations = new WeakSet<object>();
|
||||
|
||||
/** Record a completed native refusal without replacing its code, errno, or identity. */
|
||||
export function markPrivateDirectoryCreationRefused<T>(error: T): T {
|
||||
if (error !== null && typeof error === "object") {
|
||||
refusedCreations.add(error);
|
||||
}
|
||||
return error;
|
||||
}
|
||||
|
||||
export function isPrivateDirectoryCreationRefused(error: unknown): boolean {
|
||||
return error !== null && typeof error === "object" && refusedCreations.has(error);
|
||||
}
|
||||
|
|
@ -4,6 +4,7 @@ import fsSync from "node:fs";
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { resolveRequiredOsHomeDir } from "./home-dir.js";
|
||||
import { markPrivateDirectoryCreationRefused } from "./private-directory-creation.js";
|
||||
import { resolvePreferredOpenClawTmpDir } from "./tmp-openclaw-dir.js";
|
||||
import { createPrivateWindowsDirectory } from "./windows-private-directory.js";
|
||||
|
||||
|
|
@ -46,7 +47,11 @@ export async function createPrivateSqliteTempDirectory(
|
|||
|
||||
export function createPrivateSqliteTempDirectorySync(rootPath: string, prefix: string): string {
|
||||
if (process.platform !== "win32") {
|
||||
return fsSync.mkdtempSync(path.join(rootPath, prefix));
|
||||
try {
|
||||
return fsSync.mkdtempSync(path.join(rootPath, prefix));
|
||||
} catch (error) {
|
||||
throw markPrivateDirectoryCreationRefused(error);
|
||||
}
|
||||
}
|
||||
const directoryPath = path.join(rootPath, `${prefix}${randomUUID()}`);
|
||||
createPrivateWindowsDirectory(directoryPath);
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import fs from "node:fs/promises";
|
|||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
|
||||
import { isPrivateDirectoryCreationRefused } from "./private-directory-creation.js";
|
||||
|
||||
vi.mock("node:child_process", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("node:child_process")>();
|
||||
|
|
@ -147,8 +148,14 @@ describe.runIf(process.platform === "win32")("private SQLite directory creation
|
|||
const root = tempDirs.make("openclaw-sqlite-private-failure-");
|
||||
const regularFile = path.join(root, "parent-file");
|
||||
await fs.writeFile(regularFile, "not a directory");
|
||||
await expect(createPrivateSqliteDirectory(path.join(regularFile, "child"))).rejects.toThrow(
|
||||
/CreateDirectoryW.*Win32 error/u,
|
||||
const failure = await createPrivateSqliteDirectory(path.join(regularFile, "child")).catch(
|
||||
(error: unknown) => error,
|
||||
);
|
||||
expect(failure).toMatchObject({
|
||||
message: expect.stringMatching(/CreateDirectoryW.*Win32 error/u),
|
||||
code: "EIO",
|
||||
errno: expect.any(Number),
|
||||
});
|
||||
expect(isPrivateDirectoryCreationRefused(failure)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -54,6 +54,35 @@ async function expectWorkerFailure(
|
|||
}
|
||||
|
||||
describe("SQLite read-only worker diagnostics", () => {
|
||||
it.each([
|
||||
["staging-create", undefined, true],
|
||||
["staging-create-legacy", undefined, true],
|
||||
["staging-retire", undefined, false],
|
||||
["async", undefined, false],
|
||||
["staging-create", "child exited before completion", false],
|
||||
] as const)(
|
||||
"accepts allocation refusal receipts only for completed staging requests (%s, %s)",
|
||||
async (mode, failure, refused) => {
|
||||
const {
|
||||
readSqliteReadOnlyWorkerValue,
|
||||
SQLITE_SNAPSHOT_ALLOCATION_REFUSED_PREFIX,
|
||||
SqliteSnapshotAllocationRefusedError,
|
||||
} = await import("./sqlite-readonly-worker-protocol.js");
|
||||
const stdout = JSON.stringify({
|
||||
ok: false,
|
||||
message: `${SQLITE_SNAPSHOT_ALLOCATION_REFUSED_PREFIX}native directory refusal`,
|
||||
});
|
||||
let received: unknown;
|
||||
try {
|
||||
readSqliteReadOnlyWorkerValue({ stdout, stderr: "", failure }, mode);
|
||||
} catch (error) {
|
||||
received = error;
|
||||
}
|
||||
expect(received).toBeInstanceOf(Error);
|
||||
expect(received instanceof SqliteSnapshotAllocationRefusedError).toBe(refused);
|
||||
},
|
||||
);
|
||||
|
||||
it("reads cause metadata once through the registered worker", async () => {
|
||||
let causeReads = 0;
|
||||
const failure = Object.defineProperty(new Error("open failure"), "cause", {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import path from "node:path";
|
||||
import { setImmediate } from "node:timers/promises";
|
||||
import { isRecord } from "@openclaw/normalization-core/record-coerce";
|
||||
import { isPrivateDirectoryCreationRefused } from "./private-directory-creation.js";
|
||||
import { SQLITE_READONLY_CHILD_ARG } from "./runtime-process-entrypoints.js";
|
||||
import {
|
||||
formatSqliteErrorCodeSuffix,
|
||||
|
|
@ -22,6 +23,7 @@ import type { PreparedSqliteReadOnlyLocation } from "./sqlite-readonly-location.
|
|||
import {
|
||||
SQLITE_READONLY_WORKER_MAX_BUFFER,
|
||||
SQLITE_INSPECTION_CONTENTION_PREFIX,
|
||||
SQLITE_SNAPSHOT_ALLOCATION_REFUSED_PREFIX,
|
||||
isSqliteSnapshotStagingMode,
|
||||
type SqliteReadOnlyWorkerResult,
|
||||
} from "./sqlite-readonly-worker-protocol.js";
|
||||
|
|
@ -149,9 +151,17 @@ async function inspect(args: string[]): Promise<SqliteReadOnlyWorkerResult> {
|
|||
return { ok: true, location: prepared.location };
|
||||
} catch (error) {
|
||||
const contention = error instanceof SqliteSourceChangedError || isSqliteLockError(error);
|
||||
const allocationRefused =
|
||||
(mode === "staging-create" || mode === "staging-create-legacy") &&
|
||||
isPrivateDirectoryCreationRefused(error);
|
||||
const prefix = allocationRefused
|
||||
? SQLITE_SNAPSHOT_ALLOCATION_REFUSED_PREFIX
|
||||
: contention
|
||||
? SQLITE_INSPECTION_CONTENTION_PREFIX
|
||||
: "";
|
||||
return {
|
||||
ok: false,
|
||||
message: `${contention ? SQLITE_INSPECTION_CONTENTION_PREFIX : ""}${formatSqliteReadOnlyInspectionFailure(error)}`,
|
||||
message: `${prefix}${formatSqliteReadOnlyInspectionFailure(error)}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import type {
|
|||
SqliteNativeRequest,
|
||||
SqliteNativeSessionLaunch,
|
||||
} from "./sqlite-readonly-native-resource.types.js";
|
||||
import { SqliteSnapshotAllocationRefusedError } from "./sqlite-readonly-worker-protocol.js";
|
||||
import {
|
||||
createScopedSqliteReadOnlyWorker,
|
||||
runSqliteReadOnlyWorkerOnce,
|
||||
|
|
@ -347,7 +348,11 @@ export function createNativeWorkerResource(
|
|||
}
|
||||
} catch (error) {
|
||||
// Validation, missing sessions and factory refusal never enter this dispatched boundary.
|
||||
if (allocating && !session.native.notStarted) {
|
||||
if (
|
||||
allocating &&
|
||||
!session.native.notStarted &&
|
||||
!(error instanceof SqliteSnapshotAllocationRefusedError)
|
||||
) {
|
||||
uncertainAllocations.push(
|
||||
new SqliteSnapshotCleanupError(
|
||||
"SQLite snapshot allocation has no exact directory receipt; cleanup is unresolved",
|
||||
|
|
|
|||
|
|
@ -33,10 +33,13 @@ export type SqliteReadOnlyWorkerResult =
|
|||
| { ok: false; message: string };
|
||||
|
||||
export class SqliteReadOnlyInspectionContentionError extends Error {}
|
||||
export class SqliteSnapshotAllocationRefusedError extends Error {}
|
||||
|
||||
// Released updater parents require exactly { ok, message }. A negotiated worker
|
||||
// protocol can replace this owner-generated tag when those parents are retired.
|
||||
export const SQLITE_INSPECTION_CONTENTION_PREFIX = "Retryable SQLite inspection contention: ";
|
||||
export const SQLITE_SNAPSHOT_ALLOCATION_REFUSED_PREFIX =
|
||||
"SQLite snapshot directory creation refused: ";
|
||||
|
||||
export type SqliteAuthProfileRows = { store: unknown; state: unknown; cacheable: boolean };
|
||||
export type SqliteAuthProfileReadOptions = {
|
||||
|
|
@ -154,14 +157,23 @@ export function readSqliteReadOnlyWorkerValue(
|
|||
}
|
||||
if (params.failure || !result.ok) {
|
||||
const contention = !result.ok && result.message.startsWith(SQLITE_INSPECTION_CONTENTION_PREFIX);
|
||||
const allocationRefused =
|
||||
!params.failure &&
|
||||
!result.ok &&
|
||||
(mode === "staging-create" || mode === "staging-create-legacy") &&
|
||||
result.message.startsWith(SQLITE_SNAPSHOT_ALLOCATION_REFUSED_PREFIX);
|
||||
const prefix = allocationRefused
|
||||
? SQLITE_SNAPSHOT_ALLOCATION_REFUSED_PREFIX
|
||||
: contention
|
||||
? SQLITE_INSPECTION_CONTENTION_PREFIX
|
||||
: "";
|
||||
const error = createSqliteReadOnlyWorkerError(
|
||||
!result.ok
|
||||
? contention
|
||||
? result.message.slice(SQLITE_INSPECTION_CONTENTION_PREFIX.length)
|
||||
: result.message
|
||||
: (params.failure ?? "failed"),
|
||||
!result.ok ? result.message.slice(prefix.length) : (params.failure ?? "failed"),
|
||||
params.stderr,
|
||||
);
|
||||
if (allocationRefused) {
|
||||
throw new SqliteSnapshotAllocationRefusedError(error.message);
|
||||
}
|
||||
if (contention) {
|
||||
throw new SqliteReadOnlyInspectionContentionError(error.message);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -89,6 +89,9 @@ it("shares one token process across concurrent and nested async snapshot lifetim
|
|||
let tokenPid: number;
|
||||
try {
|
||||
await Promise.all(allocations);
|
||||
await expect(
|
||||
createSqliteSnapshotStagingDirectory(path.join(cache, "missing"), false, undefined, true),
|
||||
).rejects.toThrow("snapshot staging root");
|
||||
const nested = await createSqliteSnapshotStagingDirectory(
|
||||
directories[0],
|
||||
false,
|
||||
|
|
|
|||
|
|
@ -4,6 +4,10 @@ import path from "node:path";
|
|||
import { extractErrorCode } from "@openclaw/normalization-core/error-coercion";
|
||||
import { getChildLogger } from "../logging/logger.js";
|
||||
import { formatErrorMessage } from "./errors.js";
|
||||
import {
|
||||
isPrivateDirectoryCreationRefused,
|
||||
markPrivateDirectoryCreationRefused,
|
||||
} from "./private-directory-creation.js";
|
||||
import { markSqliteInspectionOperation } from "./sqlite-error-diagnostics.js";
|
||||
import {
|
||||
createPrivateSqliteTempDirectorySync,
|
||||
|
|
@ -191,7 +195,10 @@ export function sqliteSnapshotStagingError(
|
|||
? "free disk space/quota"
|
||||
: "check filesystem health and write permissions";
|
||||
const message = `${cause instanceof Error ? cause.message : String(cause)}${sqliteErrcode !== undefined ? ` (SQLite errcode=${sqliteErrcode})` : ""}; snapshot staging root ${allocation ? tempDir : path.dirname(tempDir)}: ${guidance} or set XDG_CACHE_HOME to a writable filesystem`;
|
||||
return new Error(message, { cause });
|
||||
const error = new Error(message, { cause });
|
||||
return isPrivateDirectoryCreationRefused(cause)
|
||||
? markPrivateDirectoryCreationRefused(error)
|
||||
: error;
|
||||
}
|
||||
|
||||
export async function createSqliteSnapshotStagingDirectory(
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import fs from "node:fs";
|
||||
import { createRequire } from "node:module";
|
||||
import path from "node:path";
|
||||
import { markPrivateDirectoryCreationRefused } from "./private-directory-creation.js";
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
type PrivatePathCreators = {
|
||||
|
|
@ -110,7 +111,7 @@ function loadPrivatePathCreators(): PrivatePathCreators {
|
|||
directory: (directoryPath) =>
|
||||
withPrivateAttributes(true, (security) => {
|
||||
if (!createDirectory(path.toNamespacedPath(path.resolve(directoryPath)), security)) {
|
||||
throw failure(`CreateDirectoryW(${directoryPath})`);
|
||||
throw markPrivateDirectoryCreationRefused(failure(`CreateDirectoryW(${directoryPath})`));
|
||||
}
|
||||
}),
|
||||
file: (filePath) =>
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue