diff --git a/test/non-isolated-runner.sqlite-fixtures.ts b/test/non-isolated-runner.sqlite-fixtures.ts new file mode 100644 index 000000000000..5d3686393543 --- /dev/null +++ b/test/non-isolated-runner.sqlite-fixtures.ts @@ -0,0 +1,166 @@ +import path from "node:path"; + +export function sqliteLifecycleFixtureFiles(repoRoot: string): Record { + const source = (name: string) => JSON.stringify(path.join(repoRoot, "src", name)); + return { + "11-a-sqlite-owner.test.ts": ` +import { afterAll, expect, it, vi } from "vitest"; +import path from "node:path"; +vi.mock(${source("infra/runtime-worker-url.ts")}, () => ({ + resolveRuntimeWorkerUrl: () => new URL("file:///synthetic/shared-state.worker.js"), +})); +import { isSqliteWorkerStoreAvailable } from ${source("infra/sqlite-worker-store.ts")}; +import { registerOpenClawStateDatabaseAsyncResource } from ${source("state/openclaw-state-db-cache.ts")}; +import { openOpenClawStateWorkerCleanupStore } from ${source("state/openclaw-state-worker-store.ts")}; +import { openOpenClawAgentDatabase } from ${source("state/openclaw-agent-db.ts")}; +const drainKey = Symbol.for("fixture.sqliteDrain"); +it("retains a real shared-state owner after host admission is refused", async () => { + expect(isSqliteWorkerStoreAvailable({})).toBe(false); + await expect(openOpenClawStateWorkerCleanupStore("/synthetic/state.sqlite", { + environment: { OPENCLAW_STATE_DIR: "/synthetic" }, + coordinatorRuntime: { directory: "/synthetic/coordinators", keepAlive: false }, + }, () => {})).rejects.toMatchObject({ code: "unavailable" }); + const database = openOpenClawAgentDatabase({ + agentId: "fixture", + env: { OPENCLAW_STATE_DIR: path.join(import.meta.dirname, "agent-state") }, + }); + const retained = { database, drains: 0 }; + Reflect.set(globalThis, drainKey, retained); + registerOpenClawStateDatabaseAsyncResource({ async close() { + expect(Reflect.get(globalThis, drainKey)).toBe(retained); + expect(database.db.isOpen).toBe(false); + expect(retained.drains).toBe(0); + await Promise.resolve(); + retained.drains++; + } }); +}); +afterAll(() => { + const retained = Reflect.get(globalThis, drainKey); + expect(retained.database.db.isOpen).toBe(true); + expect(retained.drains).toBe(0); + vi.resetModules(); +}); +`, + "11-b-sqlite-cleanup.test.ts": ` +import { afterEach, expect, it, vi } from "vitest"; +import type { SqliteWorkerStore } from ${source("infra/sqlite-worker-contract.ts")}; +import { + runWithSqliteWorkerStateContext, + type SqliteWorkerStateContext, +} from ${source("infra/sqlite-worker-state-context.ts")}; +import { cleanupRetiredAgentDatabaseLease } from ${source("state/openclaw-agent-execution-cleanup.ts")}; +import { + assertOpenClawStateSchemaRepairAllowed, + getExistingOpenClawStateSchemaPath, +} from ${source("state/openclaw-state-db-schema-policy.ts")}; +import type { OpenClawStateWorkerContext } from ${source("state/openclaw-state-worker-context.types.ts")}; +import type { OpenClawStateWorkerCleanupOperations } from ${source("state/openclaw-state-worker-contract.ts")}; + +// Keep the real shared-state owner in this cross-file proof; another test's mocks +// are not part of the runner's lifecycle contract. +const drainKey = Symbol.for("fixture.sqliteDrain"); +const retained = Reflect.get(globalThis, drainKey); +expect(retained.drains).toBe(1); +expect(retained.database.db.isOpen).toBe(false); +Reflect.deleteProperty(globalThis, drainKey); + +const edge = vi.hoisted(() => ({ + close: vi.fn(async () => {}), + repairs: [] as Array<{ phase: string; error: unknown }>, + forbidden: vi.fn((): never => { + throw new Error("Cleanup schema proof crossed a native database or Worker boundary"); + }), +})); + +vi.mock("node:sqlite", () => ({ DatabaseSync: edge.forbidden })); +vi.mock("node:worker_threads", () => ({ Worker: edge.forbidden })); +vi.mock(${source("infra/runtime-worker-url.ts")}, () => ({ + resolveRuntimeWorkerUrl: () => new URL("file:///synthetic/shared-state.worker.js"), +})); +vi.mock(${source("infra/sqlite-worker-identity.ts")}, () => ({ + readDatabasePathIdentity: async (canonicalPath: string) => ({ + key: "file:synthetic-state", + canonicalPath, + }), +})); +vi.mock(${source("infra/sqlite-worker-store.ts")}, () => ({ + openSharedStateSqliteWorkerStore: async ( + options: { databasePath: string }, + context: SqliteWorkerStateContext, + ) => { + runWithSqliteWorkerStateContext(context, () => + inspectRepairPolicy("open", options.databasePath), + ); + const store: SqliteWorkerStore = { + async execute(command) { + inspectRepairPolicy("cleanup", command.input.sharedStatePath); + }, + close: edge.close, + }; + return store; + }, + runSqliteWorkerStoreOperation: async ( + store: SqliteWorkerStore, + operation: (scope: SqliteWorkerStore) => Promise, + context: SqliteWorkerStateContext, + ) => runWithSqliteWorkerStateContext(context, () => operation(store)), +})); + +function inspectRepairPolicy(phase: string, databasePath: string) { + let error: unknown; + try { + assertOpenClawStateSchemaRepairAllowed(databasePath); + } catch (failure) { + error = failure; + } + edge.repairs.push({ phase, error }); +} + +afterEach(() => { + expect(edge.forbidden).not.toHaveBeenCalled(); + edge.repairs.length = 0; + vi.clearAllMocks(); +}); + +it("retains installed-schema repair ownership through retired agent lease cleanup", async () => { + const databasePath = "/synthetic/state/openclaw.sqlite"; + const context: OpenClawStateWorkerContext = { + environment: { OPENCLAW_STATE_DIR: "/synthetic" }, + coordinatorRuntime: { directory: "/synthetic/coordinators", keepAlive: true }, + existingSchemaPath: databasePath, + admission: { + databasePath, + identity: { key: "file:synthetic-state", canonicalPath: databasePath }, + assertCurrent() {}, + }, + }; + // There is no ambient schema scope for the mocked transport to inherit. + expect(getExistingOpenClawStateSchemaPath()).toBeUndefined(); + await cleanupRetiredAgentDatabaseLease({ + context, + stopped: Promise.resolve(), + assertOwned() {}, + lease: { + leaseId: "synthetic-lease", + agentId: "main", + path: "/synthetic/agents/main.sqlite", + ownerPid: process.pid, + ownerStartTime: null, + sharedStatePath: databasePath, + sharedStateIdentity: "file:synthetic-state", + }, + }); + expect(edge.repairs).toEqual( + ["open", "cleanup"].map((phase) => ({ + phase, + error: expect.objectContaining({ + message: expect.stringContaining("schema repair is owned by the existing installation"), + }), + })), + ); + expect(edge.close).toHaveBeenCalledOnce(); + expect(getExistingOpenClawStateSchemaPath()).toBeUndefined(); +}); +`, + }; +} diff --git a/test/non-isolated-runner.sqlite.test.ts b/test/non-isolated-runner.sqlite.test.ts new file mode 100644 index 000000000000..1a858b885b68 --- /dev/null +++ b/test/non-isolated-runner.sqlite.test.ts @@ -0,0 +1,123 @@ +import fs from "node:fs/promises"; +import { createRequire } from "node:module"; +import path from "node:path"; +import { expect, it } from "vitest"; +import type { JsonTestResults } from "vitest/node"; +import type { VitestReportCapture } from "../scripts/lib/vitest-report-capture.mts"; +import { resolveTestNodeExecPath } from "../src/test-utils/node-process.js"; +import { runVitestShutdownCommand } from "./helpers/vitest-shutdown-command.ts"; +import { sqliteLifecycleFixtureFiles } from "./non-isolated-runner.sqlite-fixtures.ts"; + +const repoRoot = path.resolve(import.meta.dirname, ".."); +const require = createRequire(import.meta.url); + +async function verifySqliteOwnerRetirement(signal: AbortSignal) { + const fixtureRoots = path.join(repoRoot, ".artifacts", "non-isolated-sqlite-lifecycle"); + await fs.mkdir(fixtureRoots, { recursive: true }); + // openclaw-temp-dir: allow retains an unjoined or failed child fixture for diagnosis. + const root = await fs.mkdtemp(path.join(fixtureRoots, "run-")); + try { + const vitestDir = path.dirname(require.resolve("vitest/package.json")); + await fs.symlink(path.dirname(vitestDir), path.join(root, "node_modules"), "junction"); + const files = sqliteLifecycleFixtureFiles(repoRoot); + for (const [name, content] of Object.entries(files)) { + await fs.writeFile(path.join(root, name), content); + } + await fs.writeFile( + path.join(root, "vitest.config.ts"), + `import { sharedVitestConfig } from ${JSON.stringify(path.join(repoRoot, "test/vitest/vitest.shared.config.ts"))}; +import { defineConfig } from "vitest/config"; +import { BaseSequencer } from "vitest/node"; +class Ordered extends BaseSequencer { + async sort(files) { return [...files].sort((a, b) => a.moduleId.localeCompare(b.moduleId)); } +} +export default defineConfig({ + cacheDir: ${JSON.stringify(path.join(root, ".vite"))}, + resolve: sharedVitestConfig.resolve, + test: { + name: "sqlite-owner-retirement", pool: "threads", isolate: false, + maxWorkers: 1, fileParallelism: false, + runner: ${JSON.stringify(path.join(repoRoot, "test/non-isolated-runner.ts"))}, + sequence: { sequencer: Ordered }, + }, +}); +`, + ); + const env = Object.fromEntries( + Object.entries(process.env).filter( + ([key]) => + !key.startsWith("VITEST") && + !key.startsWith("OPENCLAW_VITEST") && + key !== "GITHUB_ACTIONS" && + key !== "FORCE_COLOR", + ), + ); + const reportPath = path.join(root, "report.json"); + const result = await runVitestShutdownCommand({ + bin: resolveTestNodeExecPath(), + args: [ + path.join(vitestDir, "vitest.mjs"), + "run", + "--root", + root, + "--config", + path.join(root, "vitest.config.ts"), + "--configLoader", + "runner", + "--reporter=verbose", + "--reporter=json", + `--reporter=${path.join(repoRoot, "scripts/lib/vitest-report-capture.mts")}`, + `--outputFile.json=${reportPath}`, + ], + cwd: repoRoot, + env: { ...env, NO_COLOR: "1" }, + maxBytes: 4 * 1024 * 1024, + signal, + }); + expect(result.code, result.stdout + result.stderr).toBe(0); + const output = result.stdout + result.stderr; + for (const file of Object.keys(files)) { + expect(output).toContain( + `[sqlite-test-lifecycle] ${file}: retiring openclaw.sharedStateWorkerOwner`, + ); + } + expect(output).toContain( + "[sqlite-test-lifecycle] 11-a-sqlite-owner.test.ts: draining agent database custody", + ); + const report: JsonTestResults = JSON.parse(await fs.readFile(reportPath, "utf8")); + expect(report).toMatchObject({ + numTotalTests: 2, + numPassedTests: 2, + numFailedTests: 0, + numPendingTests: 0, + numTodoTests: 0, + }); + expect(report.testResults.map((file) => path.basename(file.name)).toSorted()).toEqual( + Object.keys(files).toSorted(), + ); + for (const file of report.testResults) { + expect(file.status).toBe("passed"); + expect(file.assertionResults).toHaveLength(1); + expect(file.assertionResults[0]).toMatchObject({ status: "passed", failureMessages: [] }); + } + const capture: VitestReportCapture = JSON.parse( + await fs.readFile(`${reportPath}.capture.json`, "utf8"), + ); + expect(capture).toMatchObject({ + processTimedOut: false, + ended: { reason: "passed", unhandledErrors: 0, failedModules: 0, suiteErrors: 0 }, + }); + await fs.rm(root, { recursive: true, force: true }); + } catch (error) { + if (error instanceof Error) { + error.message += `; retained fixture ${root}`; + } + throw error; + } +} + +it("retires SQLite owners and their callbacks before the next file installs its transport", (context) => { + const run = verifySqliteOwnerRetirement(context.signal); + context.onTestFinished(() => run); + return run; +}); diff --git a/test/non-isolated-runner.ts b/test/non-isolated-runner.ts index f1fbcf88784c..09dc2f9e20b7 100644 --- a/test/non-isolated-runner.ts +++ b/test/non-isolated-runner.ts @@ -21,6 +21,12 @@ import { trackCustomElementRegistry, } from "./jsdom-custom-elements.ts"; import { repositoryTestApiPublications } from "./repository-test-api-publications.ts"; +import { + drainSqliteTestAgentOwner, + rememberSqliteTestAgentOwner, + retireSqliteTestSingleton, + sqliteTestSingletonPublications, +} from "./sqlite-test-lifecycle.ts"; type EvaluatedModuleNode = ViteEvaluatedModuleNode & { mockedExports?: unknown; @@ -88,7 +94,11 @@ function getSharedTestHome(): string | undefined { return globalState[SHARED_TEST_SETUP]?.tempHome ?? process.env.OPENCLAW_TEST_HOME; } -function resetEvaluatedModules(modules: EvaluatedModules, executions: ModuleExecutionInfo) { +function resetEvaluatedModules( + modules: EvaluatedModules, + executions: ModuleExecutionInfo, + testFiles: string, +) { const skipPaths = [/\/vitest\/dist\//, /vitest-virtual-\w+\/dist/u, /@vitest\/dist/u]; // Vitest reuses the graph across runner instances. Weak marks prevent a past // execution from owning a later mock-only slot without retaining any records @@ -103,14 +113,24 @@ function resetEvaluatedModules(modules: EvaluatedModules, executions: ModuleExec // Vitest's evaluator records each execution independently (including native ones), // using the unprefixed id for automocks. Module resets preserve those records. const key = repositoryTestApiPublications.get(node.file); + const sqliteKey = sqliteTestSingletonPublications.get(node.file); const executionId = node.id.startsWith("mock:") ? node.id.slice(5) : node.id; const execution = executions.get(executionId); - if (key && execution && !execution.external && !retiredExecutions.has(execution)) { + if ( + (key || sqliteKey) && + execution && + !execution.external && + !retiredExecutions.has(execution) + ) { retiredExecutions.add(execution); - const publication = Object.getOwnPropertyDescriptor(globalThis, key); - if (publication?.configurable && "value" in publication) { + const publication = + key === undefined ? undefined : Object.getOwnPropertyDescriptor(globalThis, key); + if (key && publication?.configurable && "value" in publication) { Reflect.deleteProperty(globalThis, key); } + if (sqliteKey) { + retireSqliteTestSingleton(sqliteKey, testFiles); + } } // Mock metadata owns factories and cached exports after the registry resets. // Retire those nodes while preserving ordinary transformed-code metadata. @@ -426,6 +446,22 @@ export default class OpenClawNonIsolatedRunner extends TestRunner { restoreRealTimers(); restoreNativeTimerGlobals(); await super.onBeforeRunTask(test); + this.rememberSqliteAgentOwner(); + } + + onTaskFinished() { + this.rememberSqliteAgentOwner(); + } + + private rememberSqliteAgentOwner() { + if (this.config.isolate) { + return; + } + const internals = this as unknown as TestRunnerInternals; + rememberSqliteTestAgentOwner( + (internals.workerState.evaluatedModules as EvaluatedModules).idToModuleMap.values(), + internals.workerState.moduleExecutionInfo, + ); } override onBeforeTryTask(test: RunnerTask, options: TestTryOptions) { @@ -444,6 +480,9 @@ export default class OpenClawNonIsolatedRunner extends TestRunner { // oxlint-disable-next-line typescript/no-misused-promises -- Vitest awaits this hook; its concrete TestRunner declaration narrows the return to void. override async onAfterRunFiles(files: RunnerTestFile[]) { super.onAfterRunFiles(files); + const testFiles = files + .map((file) => path.relative(this.config.root, file.filepath)) + .join(", "); const internals = this as unknown as TestRunnerInternals; await drainMockerResolveMocks(internals.moduleRunner?.mocker); @@ -473,6 +512,13 @@ export default class OpenClawNonIsolatedRunner extends TestRunner { >("../src/state/openclaw-agent-db-lifecycle.js"); await closeOpenClawAgentDatabasesAsync(); } + if (!this.config.isolate) { + await drainSqliteTestAgentOwner( + (internals.workerState.evaluatedModules as EvaluatedModules).idToModuleMap.values(), + internals.workerState.moduleExecutionInfo, + testFiles, + ); + } // Lifecycle-owned singletons survive module resets; close them before the next file // can observe a previous file's sessions, caches, or registered resources. await drainGlobalSingletonLifecycleState(); @@ -496,6 +542,7 @@ export default class OpenClawNonIsolatedRunner extends TestRunner { resetEvaluatedModules( internals.workerState.evaluatedModules as EvaluatedModules, internals.workerState.moduleExecutionInfo, + testFiles, ); } } diff --git a/test/sqlite-test-lifecycle.ts b/test/sqlite-test-lifecycle.ts new file mode 100644 index 000000000000..63f1679ddfd0 --- /dev/null +++ b/test/sqlite-test-lifecycle.ts @@ -0,0 +1,115 @@ +import path from "node:path"; +import { normalizeModuleId, type EvaluatedModuleNode } from "vite/module-runner"; +import { vi } from "vitest"; + +const source = (name: string) => normalizeModuleId(path.resolve(import.meta.dirname, "..", name)); +const agentSource = source("src/state/openclaw-agent-db-lifecycle.ts"); +const agentKey = Symbol.for("openclaw.agentDatabaseLifecycle"); +const resetKey = Symbol.for("openclaw.globalSingletonLifecycleResets"); + +// These owners retain module closures and each other's lifecycle callbacks. +// Keep their native custody intact through drainage, then retire the whole generation. +export const sqliteTestSingletonPublications: ReadonlyMap = new Map([ + [ + source("src/state/openclaw-state-worker-store.ts"), + Symbol.for("openclaw.sharedStateWorkerOwner"), + ], + [source("src/infra/sqlite-worker-store.ts"), Symbol.for("openclaw.sqliteWorkerBroker")], + [source("src/state/openclaw-state-db-cache.ts"), Symbol.for("openclaw.stateDatabaseLifecycle")], + [agentSource, agentKey], +]); + +type AgentLifecycleModule = Pick< + typeof import("../src/state/openclaw-agent-db-lifecycle.js"), + "agentDatabaseLifecycle" | "closeOpenClawAgentDatabasesAsync" +>; +type AgentOwner = AgentLifecycleModule["agentDatabaseLifecycle"]; +const agentClosers = new WeakMap Promise>(); + +/** Preserve the verified owner's closer before a test hook can reset its module exports. */ +export function rememberSqliteTestAgentOwner( + modules: Iterable, + executions: ReadonlyMap, +): void { + const owner = (globalThis as Record)[agentKey] as AgentOwner | undefined; + if (!owner || agentClosers.has(owner)) { + return; + } + for (const node of modules) { + if (node.file !== agentSource) { + continue; + } + const execution = executions.get(node.id.startsWith("mock:") ? node.id.slice(5) : node.id); + const exports = node.exports as Partial | undefined; + if ( + execution && + !execution.external && + exports && + exports.agentDatabaseLifecycle === owner && + typeof exports.closeOpenClawAgentDatabasesAsync === "function" && + !vi.isMockFunction(exports.closeOpenClawAgentDatabasesAsync) + ) { + agentClosers.set(owner, exports.closeOpenClawAgentDatabasesAsync); + return; + } + } +} + +/** Agent lease cleanup still needs its original shared-state owner and broker. */ +export async function drainSqliteTestAgentOwner( + modules: Iterable, + executions: ReadonlyMap, + testFiles: string, +): Promise { + rememberSqliteTestAgentOwner(modules, executions); + const globalStore = globalThis as Record; + const owner = globalStore[agentKey] as AgentLifecycleModule["agentDatabaseLifecycle"] | undefined; + const resources = globalStore[Symbol.for("openclaw.agentDatabaseAsyncResources")] as + | { active: Set; closing: Map; selections: Set } + | undefined; + const custody = () => ({ + databases: owner?.databases.size ?? 0, + leases: owner?.leases.size ?? 0, + pending: owner?.pending.size ?? 0, + activePending: owner?.activePending.size ?? 0, + retainedCloses: owner?.retainedCloses.size ?? 0, + resources: resources?.active.size ?? 0, + closing: resources?.closing.size ?? 0, + selections: resources?.selections.size ?? 0, + }); + const hasCustody = () => Object.values(custody()).some((count) => count > 0); + if (!hasCustody()) { + return; + } + console.warn( + `[sqlite-test-lifecycle] ${testFiles}: draining agent database custody ${JSON.stringify(custody())}`, + ); + const close = owner && agentClosers.get(owner); + if (close) { + await close(); + } + // A module reset can erase the real closer. Never load a replacement under the + // file's mocks or abandon handles just to make the next file start cleanly. + if (hasCustody()) { + throw new Error( + `SQLite test teardown cannot retire agent owners with unsettled database custody from ${testFiles}: ${JSON.stringify(custody())}`, + ); + } +} + +/** Called only for an evaluated source generation, after successful owner drainage. */ +export function retireSqliteTestSingleton(key: symbol, testFiles: string): void { + const globalStore = globalThis as Record; + const resets = globalStore[resetKey] as Map | undefined; + if (key === agentKey) { + const owner = globalStore[agentKey] as AgentOwner | undefined; + if (owner) { + agentClosers.delete(owner); + } + } + if (Object.hasOwn(globalStore, key)) { + console.warn(`[sqlite-test-lifecycle] ${testFiles}: retiring ${key.description}`); + } + Reflect.deleteProperty(globalStore, key); + resets?.delete(key); +}