fix(test): prevent order-dependent agent cleanup failures (#153345)

Shared Vitest workers kept SQLite owner callbacks after module resets, so a later retired-agent cleanup test inherited the preceding file's transport. Preserve the verified native closer, drain agent database custody before shared owners, and retire the old owner generation and reset registrations together.

Cover the file boundary with a self-contained producer/consumer regression that retains native resources through module reset and checks closure, drainage, schema ownership, and the next file's transport.

Release-note context: Repairs order-dependent test infrastructure failures; no user-visible runtime behavior changes.

Evidence: Paired Linux shard replay changed teardown from exit 1 to exit 0 with 1,322 passing tests; the native regression passed on Linux in 2.65 seconds. Independent Codex review and the current-head ClawSweeper review found no actionable findings. Hosted CI supplies the native preparation gates.

test/non-isolated-runner.sqlite-fixtures.ts

@@ -0,0 +1,166 @@
+import path from "node:path";
+
+export function sqliteLifecycleFixtureFiles(repoRoot: string): Record<string, string> {
+  const source = (name: string) => JSON.stringify(path.join(repoRoot, "src", name));
+  return {
+    "11-a-sqlite-owner.test.ts": `
+import { afterAll, expect, it, vi } from "vitest";
+import path from "node:path";
+vi.mock(${source("infra/runtime-worker-url.ts")}, () => ({
+  resolveRuntimeWorkerUrl: () => new URL("file:///synthetic/shared-state.worker.js"),
+}));
+import { isSqliteWorkerStoreAvailable } from ${source("infra/sqlite-worker-store.ts")};
+import { registerOpenClawStateDatabaseAsyncResource } from ${source("state/openclaw-state-db-cache.ts")};
+import { openOpenClawStateWorkerCleanupStore } from ${source("state/openclaw-state-worker-store.ts")};
+import { openOpenClawAgentDatabase } from ${source("state/openclaw-agent-db.ts")};
+const drainKey = Symbol.for("fixture.sqliteDrain");
+it("retains a real shared-state owner after host admission is refused", async () => {
+  expect(isSqliteWorkerStoreAvailable({})).toBe(false);
+  await expect(openOpenClawStateWorkerCleanupStore("/synthetic/state.sqlite", {
+    environment: { OPENCLAW_STATE_DIR: "/synthetic" },
+    coordinatorRuntime: { directory: "/synthetic/coordinators", keepAlive: false },
+  }, () => {})).rejects.toMatchObject({ code: "unavailable" });
+  const database = openOpenClawAgentDatabase({
+    agentId: "fixture",
+    env: { OPENCLAW_STATE_DIR: path.join(import.meta.dirname, "agent-state") },
+  });
+  const retained = { database, drains: 0 };
+  Reflect.set(globalThis, drainKey, retained);
+  registerOpenClawStateDatabaseAsyncResource({ async close() {
+    expect(Reflect.get(globalThis, drainKey)).toBe(retained);
+    expect(database.db.isOpen).toBe(false);
+    expect(retained.drains).toBe(0);
+    await Promise.resolve();
+    retained.drains++;
+  } });
+});
+afterAll(() => {
+  const retained = Reflect.get(globalThis, drainKey);
+  expect(retained.database.db.isOpen).toBe(true);
+  expect(retained.drains).toBe(0);
+  vi.resetModules();
+});
+`,
+    "11-b-sqlite-cleanup.test.ts": `
+import { afterEach, expect, it, vi } from "vitest";
+import type { SqliteWorkerStore } from ${source("infra/sqlite-worker-contract.ts")};
+import {
+  runWithSqliteWorkerStateContext,
+  type SqliteWorkerStateContext,
+} from ${source("infra/sqlite-worker-state-context.ts")};
+import { cleanupRetiredAgentDatabaseLease } from ${source("state/openclaw-agent-execution-cleanup.ts")};
+import {
+  assertOpenClawStateSchemaRepairAllowed,
+  getExistingOpenClawStateSchemaPath,
+} from ${source("state/openclaw-state-db-schema-policy.ts")};
+import type { OpenClawStateWorkerContext } from ${source("state/openclaw-state-worker-context.types.ts")};
+import type { OpenClawStateWorkerCleanupOperations } from ${source("state/openclaw-state-worker-contract.ts")};
+
+// Keep the real shared-state owner in this cross-file proof; another test's mocks
+// are not part of the runner's lifecycle contract.
+const drainKey = Symbol.for("fixture.sqliteDrain");
+const retained = Reflect.get(globalThis, drainKey);
+expect(retained.drains).toBe(1);
+expect(retained.database.db.isOpen).toBe(false);
+Reflect.deleteProperty(globalThis, drainKey);
+
+const edge = vi.hoisted(() => ({
+  close: vi.fn(async () => {}),
+  repairs: [] as Array<{ phase: string; error: unknown }>,
+  forbidden: vi.fn((): never => {
+    throw new Error("Cleanup schema proof crossed a native database or Worker boundary");
+  }),
+}));
+
+vi.mock("node:sqlite", () => ({ DatabaseSync: edge.forbidden }));
+vi.mock("node:worker_threads", () => ({ Worker: edge.forbidden }));
+vi.mock(${source("infra/runtime-worker-url.ts")}, () => ({
+  resolveRuntimeWorkerUrl: () => new URL("file:///synthetic/shared-state.worker.js"),
+}));
+vi.mock(${source("infra/sqlite-worker-identity.ts")}, () => ({
+  readDatabasePathIdentity: async (canonicalPath: string) => ({
+    key: "file:synthetic-state",
+    canonicalPath,
+  }),
+}));
+vi.mock(${source("infra/sqlite-worker-store.ts")}, () => ({
+  openSharedStateSqliteWorkerStore: async (
+    options: { databasePath: string },
+    context: SqliteWorkerStateContext,
+  ) => {
+    runWithSqliteWorkerStateContext(context, () =>
+      inspectRepairPolicy("open", options.databasePath),
+    );
+    const store: SqliteWorkerStore<OpenClawStateWorkerCleanupOperations> = {
+      async execute(command) {
+        inspectRepairPolicy("cleanup", command.input.sharedStatePath);
+      },
+      close: edge.close,
+    };
+    return store;
+  },
+  runSqliteWorkerStoreOperation: async (
+    store: SqliteWorkerStore<OpenClawStateWorkerCleanupOperations>,
+    operation: (scope: SqliteWorkerStore<OpenClawStateWorkerCleanupOperations>) => Promise<void>,
+    context: SqliteWorkerStateContext,
+  ) => runWithSqliteWorkerStateContext(context, () => operation(store)),
+}));
+
+function inspectRepairPolicy(phase: string, databasePath: string) {
+  let error: unknown;
+  try {
+    assertOpenClawStateSchemaRepairAllowed(databasePath);
+  } catch (failure) {
+    error = failure;
+  }
+  edge.repairs.push({ phase, error });
+}
+
+afterEach(() => {
+  expect(edge.forbidden).not.toHaveBeenCalled();
+  edge.repairs.length = 0;
+  vi.clearAllMocks();
+});
+
+it("retains installed-schema repair ownership through retired agent lease cleanup", async () => {
+  const databasePath = "/synthetic/state/openclaw.sqlite";
+  const context: OpenClawStateWorkerContext = {
+    environment: { OPENCLAW_STATE_DIR: "/synthetic" },
+    coordinatorRuntime: { directory: "/synthetic/coordinators", keepAlive: true },
+    existingSchemaPath: databasePath,
+    admission: {
+      databasePath,
+      identity: { key: "file:synthetic-state", canonicalPath: databasePath },
+      assertCurrent() {},
+    },
+  };
+  // There is no ambient schema scope for the mocked transport to inherit.
+  expect(getExistingOpenClawStateSchemaPath()).toBeUndefined();
+  await cleanupRetiredAgentDatabaseLease({
+    context,
+    stopped: Promise.resolve(),
+    assertOwned() {},
+    lease: {
+      leaseId: "synthetic-lease",
+      agentId: "main",
+      path: "/synthetic/agents/main.sqlite",
+      ownerPid: process.pid,
+      ownerStartTime: null,
+      sharedStatePath: databasePath,
+      sharedStateIdentity: "file:synthetic-state",
+    },
+  });
+  expect(edge.repairs).toEqual(
+    ["open", "cleanup"].map((phase) => ({
+      phase,
+      error: expect.objectContaining({
+        message: expect.stringContaining("schema repair is owned by the existing installation"),
+      }),
+    })),
+  );
+  expect(edge.close).toHaveBeenCalledOnce();
+  expect(getExistingOpenClawStateSchemaPath()).toBeUndefined();
+});
+`,
+  };
+}

test/non-isolated-runner.sqlite.test.ts

@@ -0,0 +1,123 @@
+import fs from "node:fs/promises";
+import { createRequire } from "node:module";
+import path from "node:path";
+import { expect, it } from "vitest";
+import type { JsonTestResults } from "vitest/node";
+import type { VitestReportCapture } from "../scripts/lib/vitest-report-capture.mts";
+import { resolveTestNodeExecPath } from "../src/test-utils/node-process.js";
+import { runVitestShutdownCommand } from "./helpers/vitest-shutdown-command.ts";
+import { sqliteLifecycleFixtureFiles } from "./non-isolated-runner.sqlite-fixtures.ts";
+
+const repoRoot = path.resolve(import.meta.dirname, "..");
+const require = createRequire(import.meta.url);
+
+async function verifySqliteOwnerRetirement(signal: AbortSignal) {
+  const fixtureRoots = path.join(repoRoot, ".artifacts", "non-isolated-sqlite-lifecycle");
+  await fs.mkdir(fixtureRoots, { recursive: true });
+  // openclaw-temp-dir: allow retains an unjoined or failed child fixture for diagnosis.
+  const root = await fs.mkdtemp(path.join(fixtureRoots, "run-"));
+  try {
+    const vitestDir = path.dirname(require.resolve("vitest/package.json"));
+    await fs.symlink(path.dirname(vitestDir), path.join(root, "node_modules"), "junction");
+    const files = sqliteLifecycleFixtureFiles(repoRoot);
+    for (const [name, content] of Object.entries(files)) {
+      await fs.writeFile(path.join(root, name), content);
+    }
+    await fs.writeFile(
+      path.join(root, "vitest.config.ts"),
+      `import { sharedVitestConfig } from ${JSON.stringify(path.join(repoRoot, "test/vitest/vitest.shared.config.ts"))};
+import { defineConfig } from "vitest/config";
+import { BaseSequencer } from "vitest/node";
+class Ordered extends BaseSequencer {
+  async sort(files) { return [...files].sort((a, b) => a.moduleId.localeCompare(b.moduleId)); }
+}
+export default defineConfig({
+  cacheDir: ${JSON.stringify(path.join(root, ".vite"))},
+  resolve: sharedVitestConfig.resolve,
+  test: {
+    name: "sqlite-owner-retirement", pool: "threads", isolate: false,
+    maxWorkers: 1, fileParallelism: false,
+    runner: ${JSON.stringify(path.join(repoRoot, "test/non-isolated-runner.ts"))},
+    sequence: { sequencer: Ordered },
+  },
+});
+`,
+    );
+    const env = Object.fromEntries(
+      Object.entries(process.env).filter(
+        ([key]) =>
+          !key.startsWith("VITEST") &&
+          !key.startsWith("OPENCLAW_VITEST") &&
+          key !== "GITHUB_ACTIONS" &&
+          key !== "FORCE_COLOR",
+      ),
+    );
+    const reportPath = path.join(root, "report.json");
+    const result = await runVitestShutdownCommand({
+      bin: resolveTestNodeExecPath(),
+      args: [
+        path.join(vitestDir, "vitest.mjs"),
+        "run",
+        "--root",
+        root,
+        "--config",
+        path.join(root, "vitest.config.ts"),
+        "--configLoader",
+        "runner",
+        "--reporter=verbose",
+        "--reporter=json",
+        `--reporter=${path.join(repoRoot, "scripts/lib/vitest-report-capture.mts")}`,
+        `--outputFile.json=${reportPath}`,
+      ],
+      cwd: repoRoot,
+      env: { ...env, NO_COLOR: "1" },
+      maxBytes: 4 * 1024 * 1024,
+      signal,
+    });
+    expect(result.code, result.stdout + result.stderr).toBe(0);
+    const output = result.stdout + result.stderr;
+    for (const file of Object.keys(files)) {
+      expect(output).toContain(
+        `[sqlite-test-lifecycle] ${file}: retiring openclaw.sharedStateWorkerOwner`,
+      );
+    }
+    expect(output).toContain(
+      "[sqlite-test-lifecycle] 11-a-sqlite-owner.test.ts: draining agent database custody",
+    );
+    const report: JsonTestResults = JSON.parse(await fs.readFile(reportPath, "utf8"));
+    expect(report).toMatchObject({
+      numTotalTests: 2,
+      numPassedTests: 2,
+      numFailedTests: 0,
+      numPendingTests: 0,
+      numTodoTests: 0,
+    });
+    expect(report.testResults.map((file) => path.basename(file.name)).toSorted()).toEqual(
+      Object.keys(files).toSorted(),
+    );
+    for (const file of report.testResults) {
+      expect(file.status).toBe("passed");
+      expect(file.assertionResults).toHaveLength(1);
+      expect(file.assertionResults[0]).toMatchObject({ status: "passed", failureMessages: [] });
+    }
+    const capture: VitestReportCapture = JSON.parse(
+      await fs.readFile(`${reportPath}.capture.json`, "utf8"),
+    );
+    expect(capture).toMatchObject({
+      processTimedOut: false,
+      ended: { reason: "passed", unhandledErrors: 0, failedModules: 0, suiteErrors: 0 },
+    });
+    await fs.rm(root, { recursive: true, force: true });
+  } catch (error) {
+    if (error instanceof Error) {
+      error.message += `; retained fixture ${root}`;
+    }
+    throw error;
+  }
+}
+
+it("retires SQLite owners and their callbacks before the next file installs its transport", (context) => {
+  const run = verifySqliteOwnerRetirement(context.signal);
+  context.onTestFinished(() => run);
+  return run;
+});

test/non-isolated-runner.ts

@@ -21,6 +21,12 @@ import {
   trackCustomElementRegistry,
 } from "./jsdom-custom-elements.ts";
 import { repositoryTestApiPublications } from "./repository-test-api-publications.ts";
+import {
+  drainSqliteTestAgentOwner,
+  rememberSqliteTestAgentOwner,
+  retireSqliteTestSingleton,
+  sqliteTestSingletonPublications,
+} from "./sqlite-test-lifecycle.ts";
 
 type EvaluatedModuleNode = ViteEvaluatedModuleNode & {
   mockedExports?: unknown;
@@ -88,7 +94,11 @@ function getSharedTestHome(): string | undefined {
   return globalState[SHARED_TEST_SETUP]?.tempHome ?? process.env.OPENCLAW_TEST_HOME;
 }
 
-function resetEvaluatedModules(modules: EvaluatedModules, executions: ModuleExecutionInfo) {
+function resetEvaluatedModules(
+  modules: EvaluatedModules,
+  executions: ModuleExecutionInfo,
+  testFiles: string,
+) {
   const skipPaths = [/\/vitest\/dist\//, /vitest-virtual-\w+\/dist/u, /@vitest\/dist/u];
   // Vitest reuses the graph across runner instances. Weak marks prevent a past
   // execution from owning a later mock-only slot without retaining any records
@@ -103,14 +113,24 @@ function resetEvaluatedModules(modules: EvaluatedModules, executions: ModuleExec
     // Vitest's evaluator records each execution independently (including native ones),
     // using the unprefixed id for automocks. Module resets preserve those records.
     const key = repositoryTestApiPublications.get(node.file);
+    const sqliteKey = sqliteTestSingletonPublications.get(node.file);
     const executionId = node.id.startsWith("mock:") ? node.id.slice(5) : node.id;
     const execution = executions.get(executionId);
-    if (key && execution && !execution.external && !retiredExecutions.has(execution)) {
+    if (
+      (key || sqliteKey) &&
+      execution &&
+      !execution.external &&
+      !retiredExecutions.has(execution)
+    ) {
       retiredExecutions.add(execution);
-      const publication = Object.getOwnPropertyDescriptor(globalThis, key);
-      if (publication?.configurable && "value" in publication) {
+      const publication =
+        key === undefined ? undefined : Object.getOwnPropertyDescriptor(globalThis, key);
+      if (key && publication?.configurable && "value" in publication) {
         Reflect.deleteProperty(globalThis, key);
       }
+      if (sqliteKey) {
+        retireSqliteTestSingleton(sqliteKey, testFiles);
+      }
     }
     // Mock metadata owns factories and cached exports after the registry resets.
     // Retire those nodes while preserving ordinary transformed-code metadata.
@@ -426,6 +446,22 @@ export default class OpenClawNonIsolatedRunner extends TestRunner {
     restoreRealTimers();
     restoreNativeTimerGlobals();
     await super.onBeforeRunTask(test);
+    this.rememberSqliteAgentOwner();
+  }
+
+  onTaskFinished() {
+    this.rememberSqliteAgentOwner();
+  }
+
+  private rememberSqliteAgentOwner() {
+    if (this.config.isolate) {
+      return;
+    }
+    const internals = this as unknown as TestRunnerInternals;
+    rememberSqliteTestAgentOwner(
+      (internals.workerState.evaluatedModules as EvaluatedModules).idToModuleMap.values(),
+      internals.workerState.moduleExecutionInfo,
+    );
   }
 
   override onBeforeTryTask(test: RunnerTask, options: TestTryOptions) {
@@ -444,6 +480,9 @@ export default class OpenClawNonIsolatedRunner extends TestRunner {
   // oxlint-disable-next-line typescript/no-misused-promises -- Vitest awaits this hook; its concrete TestRunner declaration narrows the return to void.
   override async onAfterRunFiles(files: RunnerTestFile[]) {
     super.onAfterRunFiles(files);
+    const testFiles = files
+      .map((file) => path.relative(this.config.root, file.filepath))
+      .join(", ");
     const internals = this as unknown as TestRunnerInternals;
     await drainMockerResolveMocks(internals.moduleRunner?.mocker);
 
@@ -473,6 +512,13 @@ export default class OpenClawNonIsolatedRunner extends TestRunner {
       >("../src/state/openclaw-agent-db-lifecycle.js");
       await closeOpenClawAgentDatabasesAsync();
     }
+    if (!this.config.isolate) {
+      await drainSqliteTestAgentOwner(
+        (internals.workerState.evaluatedModules as EvaluatedModules).idToModuleMap.values(),
+        internals.workerState.moduleExecutionInfo,
+        testFiles,
+      );
+    }
     // Lifecycle-owned singletons survive module resets; close them before the next file
     // can observe a previous file's sessions, caches, or registered resources.
     await drainGlobalSingletonLifecycleState();
@@ -496,6 +542,7 @@ export default class OpenClawNonIsolatedRunner extends TestRunner {
     resetEvaluatedModules(
       internals.workerState.evaluatedModules as EvaluatedModules,
       internals.workerState.moduleExecutionInfo,
+      testFiles,
     );
   }
 }

test/sqlite-test-lifecycle.ts

@@ -0,0 +1,115 @@
+import path from "node:path";
+import { normalizeModuleId, type EvaluatedModuleNode } from "vite/module-runner";
+import { vi } from "vitest";
+
+const source = (name: string) => normalizeModuleId(path.resolve(import.meta.dirname, "..", name));
+const agentSource = source("src/state/openclaw-agent-db-lifecycle.ts");
+const agentKey = Symbol.for("openclaw.agentDatabaseLifecycle");
+const resetKey = Symbol.for("openclaw.globalSingletonLifecycleResets");
+
+// These owners retain module closures and each other's lifecycle callbacks.
+// Keep their native custody intact through drainage, then retire the whole generation.
+export const sqliteTestSingletonPublications: ReadonlyMap<string, symbol> = new Map([
+  [
+    source("src/state/openclaw-state-worker-store.ts"),
+    Symbol.for("openclaw.sharedStateWorkerOwner"),
+  ],
+  [source("src/infra/sqlite-worker-store.ts"), Symbol.for("openclaw.sqliteWorkerBroker")],
+  [source("src/state/openclaw-state-db-cache.ts"), Symbol.for("openclaw.stateDatabaseLifecycle")],
+  [agentSource, agentKey],
+]);
+
+type AgentLifecycleModule = Pick<
+  typeof import("../src/state/openclaw-agent-db-lifecycle.js"),
+  "agentDatabaseLifecycle" | "closeOpenClawAgentDatabasesAsync"
+>;
+type AgentOwner = AgentLifecycleModule["agentDatabaseLifecycle"];
+const agentClosers = new WeakMap<AgentOwner, () => Promise<void>>();
+
+/** Preserve the verified owner's closer before a test hook can reset its module exports. */
+export function rememberSqliteTestAgentOwner(
+  modules: Iterable<EvaluatedModuleNode>,
+  executions: ReadonlyMap<string, { external?: boolean }>,
+): void {
+  const owner = (globalThis as Record<PropertyKey, unknown>)[agentKey] as AgentOwner | undefined;
+  if (!owner || agentClosers.has(owner)) {
+    return;
+  }
+  for (const node of modules) {
+    if (node.file !== agentSource) {
+      continue;
+    }
+    const execution = executions.get(node.id.startsWith("mock:") ? node.id.slice(5) : node.id);
+    const exports = node.exports as Partial<AgentLifecycleModule> | undefined;
+    if (
+      execution &&
+      !execution.external &&
+      exports &&
+      exports.agentDatabaseLifecycle === owner &&
+      typeof exports.closeOpenClawAgentDatabasesAsync === "function" &&
+      !vi.isMockFunction(exports.closeOpenClawAgentDatabasesAsync)
+    ) {
+      agentClosers.set(owner, exports.closeOpenClawAgentDatabasesAsync);
+      return;
+    }
+  }
+}
+
+/** Agent lease cleanup still needs its original shared-state owner and broker. */
+export async function drainSqliteTestAgentOwner(
+  modules: Iterable<EvaluatedModuleNode>,
+  executions: ReadonlyMap<string, { external?: boolean }>,
+  testFiles: string,
+): Promise<void> {
+  rememberSqliteTestAgentOwner(modules, executions);
+  const globalStore = globalThis as Record<PropertyKey, unknown>;
+  const owner = globalStore[agentKey] as AgentLifecycleModule["agentDatabaseLifecycle"] | undefined;
+  const resources = globalStore[Symbol.for("openclaw.agentDatabaseAsyncResources")] as
+    | { active: Set<unknown>; closing: Map<unknown, unknown>; selections: Set<unknown> }
+    | undefined;
+  const custody = () => ({
+    databases: owner?.databases.size ?? 0,
+    leases: owner?.leases.size ?? 0,
+    pending: owner?.pending.size ?? 0,
+    activePending: owner?.activePending.size ?? 0,
+    retainedCloses: owner?.retainedCloses.size ?? 0,
+    resources: resources?.active.size ?? 0,
+    closing: resources?.closing.size ?? 0,
+    selections: resources?.selections.size ?? 0,
+  });
+  const hasCustody = () => Object.values(custody()).some((count) => count > 0);
+  if (!hasCustody()) {
+    return;
+  }
+  console.warn(
+    `[sqlite-test-lifecycle] ${testFiles}: draining agent database custody ${JSON.stringify(custody())}`,
+  );
+  const close = owner && agentClosers.get(owner);
+  if (close) {
+    await close();
+  }
+  // A module reset can erase the real closer. Never load a replacement under the
+  // file's mocks or abandon handles just to make the next file start cleanly.
+  if (hasCustody()) {
+    throw new Error(
+      `SQLite test teardown cannot retire agent owners with unsettled database custody from ${testFiles}: ${JSON.stringify(custody())}`,
+    );
+  }
+}
+
+/** Called only for an evaluated source generation, after successful owner drainage. */
+export function retireSqliteTestSingleton(key: symbol, testFiles: string): void {
+  const globalStore = globalThis as Record<PropertyKey, unknown>;
+  const resets = globalStore[resetKey] as Map<symbol, unknown> | undefined;
+  if (key === agentKey) {
+    const owner = globalStore[agentKey] as AgentOwner | undefined;
+    if (owner) {
+      agentClosers.delete(owner);
+    }
+  }
+  if (Object.hasOwn(globalStore, key)) {
+    console.warn(`[sqlite-test-lifecycle] ${testFiles}: retiring ${key.description}`);
+  }
+  Reflect.deleteProperty(globalStore, key);
+  resets?.delete(key);
+}