diff --git a/config/assertion-safety-baseline.txt b/config/assertion-safety-baseline.txt index 09813362ac6d..152a00fe198e 100644 --- a/config/assertion-safety-baseline.txt +++ b/config/assertion-safety-baseline.txt @@ -2473,7 +2473,7 @@ src/config/sessions/session-accessor.transcript-turn.ts 1 src/config/sessions/session-entry-json.ts 1 src/config/sessions/session-reset-boundary-event.ts 5 src/config/sessions/session-snapshot-merge.ts 26 -src/config/sessions/session-sqlite-target.ts 3 +src/config/sessions/session-sqlite-target.ts 2 src/config/sessions/session-store-config.ts 2 src/config/sessions/session-transcript-reconcile.worker.ts 1 src/config/sessions/skill-prompt-blobs.ts 5 diff --git a/docs/cli/doctor/state-migrations.md b/docs/cli/doctor/state-migrations.md index 0d47588256bf..b56a5dc2a440 100644 --- a/docs/cli/doctor/state-migrations.md +++ b/docs/cli/doctor/state-migrations.md @@ -63,6 +63,14 @@ legacy workspace files it left untouched; it does not retire their files or proposal history. After the candidate is installed, the real Doctor runs the normal import, archival, and relocation against the operator's state. +Completed agent deletions that intentionally kept their files are held back during +update and migration discovery. Doctor records a recoverable warning naming the +agent, database path, and `openclaw doctor --fix` guidance. These stores do not +block active agents' migrations or update rehearsals. If the shared auth source +is held, its migration records a skip and dependent auth repairs wait; unrelated +Doctor repairs continue. Restore an intended agent before migrating its retained +store. Pending file deletion keeps the deletion owner's existing safety checks. + Doctor reports interrupted auth-profile archive recovery even when no new migration remains or you decline another migration. If recovery cannot finish, its warning includes the failure cause and leaves the pending source for recovery; do not delete it to silence the warning. `doctor --fix` also repairs an inconsistent completed auth migration only when its old receipt has no credential fingerprints, none of the migrated credentials remain in the current canonical store, and the preserved archive still matches the recorded source hash. Doctor reimports through the normal verified migration flow. Completed receipts with fingerprints, surviving migrated credentials, or no archive remain untouched, so removing credentials after a verified migration does not restore them from backup. diff --git a/src/commands/doctor-auth-legacy-paths.ts b/src/commands/doctor-auth-legacy-paths.ts index c6257fbf21be..cc4816664b8e 100644 --- a/src/commands/doctor-auth-legacy-paths.ts +++ b/src/commands/doctor-auth-legacy-paths.ts @@ -5,8 +5,10 @@ import { listAgentIds, resolveAgentDir } from "../agents/agent-scope.js"; import { resolveSharedMainAuthAgentDir } from "../agents/auth-profiles/shared-main-dir.js"; import { resolveLegacyInheritedAuthAgentDir } from "../agents/legacy-inherited-auth-dir.js"; import { resolveStateDir } from "../config/paths.js"; +import { resolveConfiguredAgentDatabaseCandidatePaths } from "../config/sessions/targets.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { hasErrnoCode } from "../infra/errno.js"; +import { createRetainedAgentDatabaseMatcher } from "../state/agent-deletion-discovery.js"; import { resolveUserPath } from "../utils.js"; function resolveLegacyAuthAgentDir(agentDir?: string): string { @@ -83,6 +85,15 @@ export function listAuthProfileRepairCandidates( onUnavailable?: (pathname: string) => void, ): AuthProfileRepairCandidate[] { const candidates = new Map(); + const isRetained = createRetainedAgentDatabaseMatcher( + env, + () => + listAgentIds(cfg).map((agentId) => ({ agentId, path: resolveAgentDir(cfg, agentId, env) })), + { + kind: "agent-directory", + readDatabasePaths: () => resolveConfiguredAgentDatabaseCandidatePaths(cfg, { env }), + }, + ); const addCandidate = (agentDir: string | undefined): void => { // Retain the selected home's expanded directory for later SQLite writes too. const resolvedAgentDir = agentDir ? resolveUserPath(agentDir, env) : undefined; @@ -110,7 +121,7 @@ export function listAuthProfileRepairCandidates( for (const agentDir of listExistingAgentDirsFromState(env, onUnavailable)) { addCandidate(agentDir); } - return [...candidates.values()]; + return [...candidates.values()].filter(({ authPath }) => !isRetained(path.dirname(authPath))); } export function resolveLegacyAuthProfilesPath(agentDir?: string): string { diff --git a/src/commands/doctor-auth-migration-receipts.ts b/src/commands/doctor-auth-migration-receipts.ts index 1347ce901e40..5d908d77bcaf 100644 --- a/src/commands/doctor-auth-migration-receipts.ts +++ b/src/commands/doctor-auth-migration-receipts.ts @@ -14,11 +14,9 @@ import { recordLegacyMigrationSource, } from "../infra/state-migrations.receipts.js"; import type { DB as OpenClawAgentKyselyDatabase } from "../state/openclaw-agent-db.generated.js"; +import { withExistingOpenClawStateDatabaseReadOnly } from "../state/openclaw-state-db-readonly.js"; import type { DB as OpenClawStateDatabase } from "../state/openclaw-state-db.generated.js"; -import { - openOpenClawStateDatabase, - runOpenClawStateWriteTransaction, -} from "../state/openclaw-state-db.js"; +import { runOpenClawStateWriteTransaction } from "../state/openclaw-state-db.js"; const MIGRATION_KIND = "auth-profile-json-to-sqlite-v2"; type MigrationDatabase = Pick; @@ -329,32 +327,38 @@ export function resumePendingAuthProfileMigrationArchives( recoverCompleted?: (receipt: AuthProfileMigrationSourceReceipt) => boolean, ): string[] { const changes: string[] = []; - const database = openOpenClawStateDatabase({ env }); - const kysely = getNodeSqliteKysely(database.db); - const rows = executeSqliteQuerySync( - database.db, - kysely - .selectFrom("migration_sources as source") - .innerJoin("migration_runs as run", "run.id", "source.last_run_id") - .select([ - "source.source_key", - "source.source_path", - "source.source_sha256", - "source.source_size_bytes", - "source.source_record_count", - "source.target_table", - "source.last_run_id", - "source.report_json", - "source.status", - ]) - .where("source.migration_kind", "=", MIGRATION_KIND) - .where((eb) => - eb.or([ - eb.and([eb("source.status", "=", "imported"), eb("source.removed_source", "=", 0)]), - eb.and([eb("source.status", "=", "completed"), eb("source.removed_source", "=", 1)]), - ]), - ), - ).rows; + const rows = + withExistingOpenClawStateDatabaseReadOnly( + ({ db }) => + executeSqliteQuerySync( + db, + getNodeSqliteKysely(db) + .selectFrom("migration_sources as source") + .innerJoin("migration_runs as run", "run.id", "source.last_run_id") + .select([ + "source.source_key", + "source.source_path", + "source.source_sha256", + "source.source_size_bytes", + "source.source_record_count", + "source.target_table", + "source.last_run_id", + "source.report_json", + "source.status", + ]) + .where("source.migration_kind", "=", MIGRATION_KIND) + .where((eb) => + eb.or([ + eb.and([eb("source.status", "=", "imported"), eb("source.removed_source", "=", 0)]), + eb.and([ + eb("source.status", "=", "completed"), + eb("source.removed_source", "=", 1), + ]), + ]), + ), + ).rows, + { env }, + ) ?? []; for (const row of rows) { const report = JSON.parse(row.report_json) as Record; const completed = row.status === "completed"; @@ -478,13 +482,16 @@ export function hasTerminalAuthProfileMigrationReceipt( sourceKey: string, env?: NodeJS.ProcessEnv, ): boolean { - const database = openOpenClawStateDatabase({ env }); - const row = executeSqliteQueryTakeFirstSync( - database.db, - getNodeSqliteKysely(database.db) - .selectFrom("migration_sources") - .select("status") - .where("source_key", "=", sourceKey), + const row = withExistingOpenClawStateDatabaseReadOnly( + ({ db }) => + executeSqliteQueryTakeFirstSync( + db, + getNodeSqliteKysely(db) + .selectFrom("migration_sources") + .select("status") + .where("source_key", "=", sourceKey), + ), + { env }, ); return row?.status === "completed" || row?.status === "archived-unparsed"; } diff --git a/src/commands/doctor-auth-source-owner.test.ts b/src/commands/doctor-auth-source-owner.test.ts index 01e8ad1119b1..5d7ba69f10ab 100644 --- a/src/commands/doctor-auth-source-owner.test.ts +++ b/src/commands/doctor-auth-source-owner.test.ts @@ -7,11 +7,13 @@ import { clearAuthProfileMigrationDiagnostics } from "../agents/auth-profiles/le import { loadPersistedAuthProfileStore } from "../agents/auth-profiles/persisted.js"; import { clearRuntimeAuthProfileStoreSnapshots } from "../agents/auth-profiles/runtime-snapshots.js"; import { closeAuthProfileReadPool } from "../agents/auth-profiles/sqlite.js"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; import { closeOpenClawStateDatabaseForTest, openOpenClawStateDatabase, } from "../state/openclaw-state-db.js"; +import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js"; import { createOpenClawTestState, type OpenClawTestState, @@ -82,6 +84,90 @@ afterEach(async () => { }); describe("Doctor auth migration source ownership", () => { + it.each([ + "shared-wal", + "shared-shm", + "shared-journal", + "dangling-shared", + "adjacent-database", + "adjacent-sidecar", + "configured-database", + "configured-partition-sidecar", + "configured-agent-directory", + "missing-journal", + ])("preserves credentials with unknown SQLite history: %s", async (kind) => { + const { selected } = await createOwners(); + const source = writeSource( + selected.agentDir(), + "auth-profiles.json", + sourceValue("auth-profiles.json", `fake-held-${randomUUID()}`, 20), + ); + const cfg: OpenClawConfig = {}; + const sharedPath = resolveOpenClawStateSqlitePath(selected.env); + const sharedDatabase = + kind === "missing-journal" ? openOpenClawStateDatabase({ env: selected.env }) : undefined; + let artifact: string | undefined; + if (kind.startsWith("shared-")) { + artifact = `${sharedPath}-${kind.slice("shared-".length)}`; + } else if (kind === "dangling-shared") { + const target = path.join(selected.root, "missing-shared-target"); + fs.mkdirSync(path.dirname(sharedPath), { recursive: true }); + fs.mkdirSync(target); + fs.symlinkSync(target, sharedPath, "junction"); + fs.rmdirSync(target); + } else if (kind.startsWith("adjacent-")) { + artifact = path.join( + selected.agentDir(), + kind === "adjacent-database" ? "history.db" : "history.sqlite-wal", + ); + } else if (kind === "configured-agent-directory") { + const external = path.join(selected.root, "external-agent"); + cfg.agents = { entries: { main: { default: true }, other: { agentDir: external } } }; + artifact = path.join(external, "openclaw-agent.sqlite"); + } else if (kind.startsWith("configured-")) { + const external = path.join(selected.root, "external"); + cfg.session = { store: path.join(external, "history.json") }; + artifact = path.join( + external, + kind === "configured-database" ? "history.sqlite" : "history.main.sqlite-wal", + ); + } else if (sharedDatabase) { + sharedDatabase.db.exec("DROP TABLE agent_deletion_journal"); + } + const artifactBytes = "unverified SQLite family bytes\n"; + if (artifact) { + fs.mkdirSync(path.dirname(artifact), { recursive: true }); + fs.writeFileSync(artifact, artifactBytes); + } + + const result = await maybeMigrateAuthProfileJsonStoresToSqlite({ + cfg, + env: selected.env, + prompter: { confirmAutoFix: async () => true }, + }); + + expect(result.detected).toEqual([]); + expect(result.changes).toEqual([]); + expect(fs.readFileSync(source.sourcePath, "utf8")).toBe(source.bytes); + expect(archivesFor(source.sourcePath)).toEqual([]); + expect(fs.existsSync(path.join(selected.agentDir(), "openclaw-agent.sqlite"))).toBe(false); + if (artifact) { + expect(fs.readFileSync(artifact, "utf8")).toBe(artifactBytes); + } + if (sharedDatabase) { + expect( + sharedDatabase.db + .prepare("SELECT name FROM sqlite_schema WHERE name = 'agent_deletion_journal'") + .get(), + ).toBeUndefined(); + } else { + expect(fs.existsSync(sharedPath)).toBe(false); + if (kind === "dangling-shared") { + expect(fs.lstatSync(sharedPath).isSymbolicLink()).toBe(true); + } + } + }); + it("detects all three legacy siblings only in the explicitly selected state root", async () => { const { selected, ambient } = await createOwners(); const selectedSources = sourceNames.map((name) => diff --git a/src/commands/doctor-config-preflight.process.test-support.ts b/src/commands/doctor-config-preflight.process.test-support.ts index a9e7a4245e85..c45cf10faa50 100644 --- a/src/commands/doctor-config-preflight.process.test-support.ts +++ b/src/commands/doctor-config-preflight.process.test-support.ts @@ -6,6 +6,10 @@ import { promisify } from "node:util"; import { runCliProcessChild } from "../cli/cli-process-child.test-helpers.js"; import { removeCanonicalValidationFromHistoricalAgentFixture } from "../state/openclaw-agent-db.test-support.js"; import { seedOpenClawAgentSchemaV21 } from "../state/openclaw-agent-schema-v21.test-support.js"; +import { + closeOpenClawStateDatabaseForTest, + openOpenClawStateDatabase, +} from "../state/openclaw-state-db.js"; import { resolveTestNodeExecPath } from "../test-utils/node-process.js"; const execFileAsync = promisify(execFile); @@ -164,6 +168,9 @@ export function seedV17AdditiveRepairDatabase( stateDir: string, options: { participantDependency?: boolean } = {}, ): string { + const env = { ...process.env, OPENCLAW_STATE_DIR: stateDir }; + openOpenClawStateDatabase({ env }); + closeOpenClawStateDatabaseForTest(); const databasePath = path.join(stateDir, "agents", "main", "agent", "openclaw-agent.sqlite"); fs.mkdirSync(path.dirname(databasePath), { recursive: true }); const database = new DatabaseSync(databasePath); diff --git a/src/commands/doctor-database-preflight.test.ts b/src/commands/doctor-database-preflight.test.ts index 246bace992c5..ff9daf29eb84 100644 --- a/src/commands/doctor-database-preflight.test.ts +++ b/src/commands/doctor-database-preflight.test.ts @@ -3,17 +3,27 @@ import path from "node:path"; import { DatabaseSync } from "node:sqlite"; import { expect, it } from "vitest"; import { OPENCLAW_AGENT_SCHEMA_VERSION } from "../state/openclaw-agent-db-contract.js"; +import { + closeOpenClawStateDatabaseForTest, + openOpenClawStateDatabase, +} from "../state/openclaw-state-db.js"; import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js"; import { prepareDoctorDatabasePreflight } from "./doctor-database-preflight.js"; -it("inspects an unregistered custom WAL store before repair without creating source sidecars", async () => { - await withOpenClawTestState({ scenario: "minimal" }, async (state) => { - const storeDir = state.path("custom-store"); - fs.mkdirSync(storeDir); - const pathname = path.join(storeDir, "sessions.sqlite"); - const writer = new DatabaseSync(state.path("fixture-writer.sqlite")); - try { - writer.exec(` +it.each([true, false])( + "preserves an unregistered custom WAL store before repair (known history: %s)", + async (knownHistory) => { + await withOpenClawTestState({ scenario: "minimal" }, async (state) => { + const sharedStatePath = knownHistory + ? openOpenClawStateDatabase({ env: state.env }).path + : undefined; + closeOpenClawStateDatabaseForTest(); + const storeDir = state.path("custom-store"); + fs.mkdirSync(storeDir); + const pathname = path.join(storeDir, "sessions.sqlite"); + const writer = new DatabaseSync(state.path("fixture-writer.sqlite")); + try { + writer.exec(` PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=0; CREATE TABLE schema_meta ( @@ -23,36 +33,57 @@ it("inspects an unregistered custom WAL store before repair without creating sou INSERT INTO schema_meta VALUES ('primary', 'agent', ${OPENCLAW_AGENT_SCHEMA_VERSION - 1}, 'retired-owner', 'fixture', 1, 1); PRAGMA user_version=${OPENCLAW_AGENT_SCHEMA_VERSION - 1}; `); - fs.copyFileSync(state.path("fixture-writer.sqlite"), pathname); - fs.copyFileSync(state.path("fixture-writer.sqlite-wal"), `${pathname}-wal`); - } finally { - writer.close(); - } - await state.writeConfig({ - agents: { list: [{ id: "main", default: true }] }, - session: { store: pathname }, - }); - const sourceFiles = [pathname, `${pathname}-wal`, state.configPath]; - const before = sourceFiles.map((file) => fs.readFileSync(file)); - const sharedStateDir = state.statePath("state"); - expect(fs.existsSync(sharedStateDir)).toBe(false); + fs.copyFileSync(state.path("fixture-writer.sqlite"), pathname); + fs.copyFileSync(state.path("fixture-writer.sqlite-wal"), `${pathname}-wal`); + } finally { + writer.close(); + } + await state.writeConfig({ + agents: { list: [{ id: "main", default: true }] }, + session: { store: pathname }, + }); + const sourceFiles = [ + pathname, + `${pathname}-wal`, + state.configPath, + ...(sharedStatePath ? [sharedStatePath] : []), + ]; + const before = sourceFiles.map((file) => fs.readFileSync(file)); + const sharedStateDir = state.statePath("state"); + expect(fs.existsSync(sharedStateDir)).toBe(knownHistory); - const result = await prepareDoctorDatabasePreflight(); + const result = await prepareDoctorDatabasePreflight(); - expect(result).toMatchObject({ - incompatible: [], - indeterminate: [], - pendingMigrations: [ - expect.objectContaining({ - kind: "agent", - path: pathname, - foundVersion: OPENCLAW_AGENT_SCHEMA_VERSION - 1, - }), - ], + expect(result).toMatchObject({ + incompatible: [], + indeterminate: [], + }); + expect(result.pendingMigrations ?? []).toEqual( + knownHistory + ? [ + expect.objectContaining({ + kind: "agent", + path: pathname, + foundVersion: OPENCLAW_AGENT_SCHEMA_VERSION - 1, + }), + ] + : [], + ); + if (!knownHistory) { + expect(result.agentDatabaseMigrationDiscovery?.discovery).toMatchObject({ + retainedDeletions: "unavailable", + targets: [], + registryRemovals: [], + failures: [], + }); + } + expect(result.agentRefusals ?? []).toEqual([]); + expect(fs.readdirSync(storeDir).toSorted()).toEqual([ + "sessions.sqlite", + "sessions.sqlite-wal", + ]); + expect(sourceFiles.map((file) => fs.readFileSync(file))).toEqual(before); + expect(fs.existsSync(sharedStateDir)).toBe(knownHistory); }); - expect(result.agentRefusals ?? []).toEqual([]); - expect(fs.readdirSync(storeDir).toSorted()).toEqual(["sessions.sqlite", "sessions.sqlite-wal"]); - expect(sourceFiles.map((file) => fs.readFileSync(file))).toEqual(before); - expect(fs.existsSync(sharedStateDir)).toBe(false); - }); -}); + }, +); diff --git a/src/commands/doctor-maintenance.schema-preflight.test.ts b/src/commands/doctor-maintenance.schema-preflight.test.ts index 3798fd92aba9..b141fbcb3e57 100644 --- a/src/commands/doctor-maintenance.schema-preflight.test.ts +++ b/src/commands/doctor-maintenance.schema-preflight.test.ts @@ -11,6 +11,7 @@ import { requireNodeSqlite } from "../infra/node-sqlite.js"; import { createLegacyDatabaseFixture } from "../infra/state-migrations.media-persistence.test-support.js"; import { OPENCLAW_AGENT_SCHEMA_VERSION } from "../state/openclaw-agent-db-contract.js"; import { unregisterOpenClawAgentDatabase } from "../state/openclaw-agent-db-registry.js"; +import { ensureAgentDeletionJournalSchema } from "../state/openclaw-state-db-schema-additive.js"; import { closeOpenClawStateDatabaseForTest, openOpenClawStateDatabase, @@ -145,14 +146,15 @@ it.each(["canonical", "custom-json", "shared-sqlite", "registered-shared-sqlite" } fs.mkdirSync(path.dirname(agentPath), { recursive: true }); const { DatabaseSync } = requireNodeSqlite(); + const registry = new DatabaseSync(fixture.databasePath); + ensureAgentDeletionJournalSchema(registry); if (layout === "registered-shared-sqlite") { fixture.config.agents!.entries!.ops = {}; - const registry = new DatabaseSync(fixture.databasePath); registry .prepare("INSERT INTO agent_databases VALUES (?, ?, ?, ?, ?)") .run("ops", agentPath, OPENCLAW_AGENT_SCHEMA_VERSION, 1, null); - registry.close(); } + registry.close(); const agent = new DatabaseSync(agentPath); agent.exec(` PRAGMA user_version = ${OPENCLAW_AGENT_SCHEMA_VERSION + 1}; diff --git a/src/commands/doctor-model-catalog-credentials.test.ts b/src/commands/doctor-model-catalog-credentials.test.ts index 7d516c752701..2f0e6323c67f 100644 --- a/src/commands/doctor-model-catalog-credentials.test.ts +++ b/src/commands/doctor-model-catalog-credentials.test.ts @@ -26,7 +26,10 @@ import { import type { OpenClawConfig } from "../config/types.openclaw.js"; import type { RuntimeEnv } from "../runtime.js"; import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; -import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; +import { + closeOpenClawStateDatabaseForTest, + openOpenClawStateDatabase, +} from "../state/openclaw-state-db.js"; import { maybeMigrateModelCatalogCredentials } from "./doctor-model-catalog-credentials.js"; import { createDoctorPrompter, type DoctorPrompter } from "./doctor-prompter.js"; @@ -38,12 +41,14 @@ const tempDirs: string[] = []; function createState(): { agentDir: string; env: NodeJS.ProcessEnv; stateDir: string } { const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-doctor-catalog-credentials-")); tempDirs.push(stateDir); + const env = { ...process.env, HOME: stateDir, OPENCLAW_STATE_DIR: stateDir }; + openOpenClawStateDatabase({ env }); const agentDir = path.join(stateDir, "agents", "main", "agent"); fs.mkdirSync(agentDir, { recursive: true }); return { agentDir, stateDir, - env: { ...process.env, HOME: stateDir, OPENCLAW_STATE_DIR: stateDir }, + env, }; } diff --git a/src/commands/doctor-model-catalog-credentials.ts b/src/commands/doctor-model-catalog-credentials.ts index 57e3a16d553a..9c282804c10a 100644 --- a/src/commands/doctor-model-catalog-credentials.ts +++ b/src/commands/doctor-model-catalog-credentials.ts @@ -5,7 +5,7 @@ import { isDeepStrictEqual } from "node:util"; import { normalizeProviderId } from "@openclaw/model-catalog-core/provider-id"; import { isRecord } from "@openclaw/normalization-core/record-coerce"; import { note } from "../../packages/terminal-core/src/note.js"; -import { listAgentIds, resolveAgentDir, resolveDefaultAgentDir } from "../agents/agent-scope.js"; +import { listAgentIds, resolveAgentDir } from "../agents/agent-scope.js"; import { AUTH_STORE_VERSION } from "../agents/auth-profiles/constants.js"; import { loadPersistedAuthProfileStore, @@ -24,9 +24,11 @@ import { loadPersistedPluginModelCatalogsReadOnly, } from "../agents/plugin-model-catalog.js"; import { resolveStateDir } from "../config/paths.js"; +import { resolveConfiguredAgentDatabaseCandidatePaths } from "../config/sessions/targets.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import type { RuntimeEnv } from "../runtime.js"; import { listAgentModelsJsonPaths } from "../secrets/storage-scan.js"; +import { createRetainedAgentDatabaseMatcher } from "../state/agent-deletion-discovery.js"; import { shortenHomePath } from "../utils.js"; import type { DoctorPrompter } from "./doctor-prompter.js"; @@ -292,15 +294,21 @@ export async function maybeMigrateModelCatalogCredentials(params: { const env = params.env ?? process.env; const stateDir = resolveStateDir(env); const mainAgentDir = resolveSharedMainAuthAgentDir(env); - const discoveredAgentDirs = listAgentModelsJsonPaths(params.cfg, stateDir, env).map( - (modelsPath) => path.dirname(modelsPath), + const isRetained = createRetainedAgentDatabaseMatcher( + env, + () => + listAgentIds(params.cfg).map((agentId) => ({ + agentId, + path: resolveAgentDir(params.cfg, agentId, env), + })), + { + kind: "agent-directory", + readDatabasePaths: () => resolveConfiguredAgentDatabaseCandidatePaths(params.cfg, { env }), + }, ); - const agentIds = listAgentIds(params.cfg); - const configuredAgentDirs = - agentIds.length > 0 - ? agentIds.map((agentId) => resolveAgentDir(params.cfg, agentId, env)) - : [resolveDefaultAgentDir(params.cfg, env)]; - const agentDirs = [...new Set([mainAgentDir, ...configuredAgentDirs, ...discoveredAgentDirs])]; + const agentDirs = listAgentModelsJsonPaths(params.cfg, stateDir, env) + .map((modelsPath) => path.dirname(modelsPath)) + .filter((agentDir) => !isRetained(agentDir)); const mainStore = loadPersistedSharedAuthProfileStore(env) ?? emptyStore(); const catalogs = agentDirs.map((agentDir) => collectAgentCatalogs(agentDir, warnings)); const effectiveStores = catalogs.map(({ localStore }) => diff --git a/src/commands/doctor-session-canonical-candidates.ts b/src/commands/doctor-session-canonical-candidates.ts index 6f5efbc8b6cb..4f3105970218 100644 --- a/src/commands/doctor-session-canonical-candidates.ts +++ b/src/commands/doctor-session-canonical-candidates.ts @@ -55,6 +55,7 @@ export function listCanonicalSessionStores(params: { return projectExistingAgentDatabaseTargets( resolveAllAgentSessionStoreTargetsSync(params.cfg, { env: params.env }), params.env, + params.cfg, ); } diff --git a/src/commands/doctor-session-delivery-state.test.ts b/src/commands/doctor-session-delivery-state.test.ts index 0a30df55abd6..acad0e29712a 100644 --- a/src/commands/doctor-session-delivery-state.test.ts +++ b/src/commands/doctor-session-delivery-state.test.ts @@ -12,7 +12,10 @@ import { openOpenClawAgentDatabase, resolveOpenClawAgentSqlitePath, } from "../state/openclaw-agent-db.js"; -import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; +import { + closeOpenClawStateDatabaseForTest, + openOpenClawStateDatabase, +} from "../state/openclaw-state-db.js"; import { repairCanonicalSessionDeliveryStates, repairCanonicalSessionResolvedSkills, @@ -521,6 +524,7 @@ describe("doctor canonical session delivery state", () => { const copiedStateDir = fs.realpathSync(tempDirs.make("openclaw-delivery-copy-")); const copiedEnv = { ...process.env, OPENCLAW_STATE_DIR: copiedStateDir }; + openOpenClawStateDatabase({ env: copiedEnv }); const copiedPath = resolveOpenClawAgentSqlitePath({ agentId: "main", env: copiedEnv }); fs.mkdirSync(path.dirname(copiedPath), { recursive: true }); fs.copyFileSync(sourcePath, copiedPath); diff --git a/src/commands/doctor-session-sqlite-readers.ts b/src/commands/doctor-session-sqlite-readers.ts index 512f220b79bb..ab6f067dde5f 100644 --- a/src/commands/doctor-session-sqlite-readers.ts +++ b/src/commands/doctor-session-sqlite-readers.ts @@ -25,10 +25,14 @@ import { } from "../config/sessions/session-entry-codec.js"; import { transcriptEventReadBytesSql } from "../config/sessions/session-transcript-read-bytes.js"; import type { SessionStoreTarget as ResolvedSessionStoreTarget } from "../config/sessions/targets.js"; -import { resolveAllAgentSessionStoreCandidateTargetsSync } from "../config/sessions/targets.js"; +import { + resolveAllAgentSessionStoreCandidateTargetsSync, + resolveConfiguredAgentDatabaseTargets, +} from "../config/sessions/targets.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { openNodeSqliteDatabase } from "../infra/node-sqlite.js"; import { readAgentDatabaseAdmissionRefusal } from "../state/agent-database-admission.js"; +import { createRetainedAgentDatabaseMatcher } from "../state/agent-deletion-discovery.js"; import { resolveOpenClawAgentSqlitePath } from "../state/openclaw-agent-db.js"; import { tableExists, tableHasColumn } from "../state/openclaw-state-db-schema-helpers.js"; @@ -577,14 +581,25 @@ export function resolveTargetSqlitePath( export function projectExistingAgentDatabaseTargets( targets: readonly SessionStoreTarget[], env: NodeJS.ProcessEnv, + cfg: OpenClawConfig, ): ExistingAgentDatabaseTarget[] { const seenPaths = new Set(); + const isRetained = createRetainedAgentDatabaseMatcher(env, () => + resolveConfiguredAgentDatabaseTargets(cfg, { env }), + ); return targets.flatMap((target) => { - if (readAgentDatabaseAdmissionRefusal(target.agentId, { env })) { + if ( + isRetained(target.storePath, target.agentId) || + readAgentDatabaseAdmissionRefusal(target.agentId, { env }) + ) { return []; } const sqlitePath = resolveTargetSqlitePath(target, env); - if (seenPaths.has(sqlitePath) || !fs.existsSync(sqlitePath)) { + if ( + isRetained(sqlitePath, target.agentId) || + seenPaths.has(sqlitePath) || + !fs.existsSync(sqlitePath) + ) { return []; } seenPaths.add(sqlitePath); @@ -599,6 +614,7 @@ export function listExistingAgentDatabaseTargets( return projectExistingAgentDatabaseTargets( resolveAllAgentSessionStoreCandidateTargetsSync(cfg, { env }), env, + cfg, ); } diff --git a/src/commands/doctor-session-transcript-headers.ts b/src/commands/doctor-session-transcript-headers.ts index 2aeee57ede41..94f0bc252f89 100644 --- a/src/commands/doctor-session-transcript-headers.ts +++ b/src/commands/doctor-session-transcript-headers.ts @@ -1,4 +1,3 @@ -import fs from "node:fs"; import type { DatabaseSync } from "node:sqlite"; import { note } from "../../packages/terminal-core/src/note.js"; import { resolveAgentWorkspaceDir } from "../agents/agent-scope.js"; @@ -23,11 +22,13 @@ import { executeSqliteQueryTakeFirstSync } from "../infra/kysely-sync.js"; import { openNodeSqliteDatabase } from "../infra/node-sqlite.js"; import { parseAgentSessionKey } from "../routing/session-key.js"; import { - resolveOpenClawAgentSqlitePath, runOpenClawAgentWriteTransaction, type OpenClawAgentDatabase, } from "../state/openclaw-agent-db.js"; -import { resolveTargetSqliteOptions } from "./doctor-session-sqlite-readers.js"; +import { + projectExistingAgentDatabaseTargets, + resolveTargetSqliteOptions, +} from "./doctor-session-sqlite-readers.js"; import { ReadOnlySqliteTranscriptReader } from "./doctor-session-sqlite-transcript-readers.js"; const NOTE_TITLE = "Session transcript headers"; @@ -212,14 +213,13 @@ export async function noteSessionTranscriptHeaderHealth(params: { let found = 0; let repaired = 0; - const seenPaths = new Set(); - for (const target of resolveAllAgentSessionStoreTargetsSync(params.cfg, { env })) { + for (const target of projectExistingAgentDatabaseTargets( + resolveAllAgentSessionStoreTargetsSync(params.cfg, { env }), + env, + params.cfg, + )) { const databaseOptions = resolveTargetSqliteOptions(target, env); - const sqlitePath = resolveOpenClawAgentSqlitePath(databaseOptions); - if (seenPaths.has(sqlitePath) || !fs.existsSync(sqlitePath)) { - continue; - } - seenPaths.add(sqlitePath); + const sqlitePath = target.sqlitePath; let readDatabase: DatabaseSync | undefined; try { // Each snapshot exhausts or closes its iterators before repair, so this read-only diff --git a/src/commands/doctor-session-transcript-labels.ts b/src/commands/doctor-session-transcript-labels.ts index a9fabc74e296..96b6554d741c 100644 --- a/src/commands/doctor-session-transcript-labels.ts +++ b/src/commands/doctor-session-transcript-labels.ts @@ -1,4 +1,3 @@ -import fs from "node:fs"; import type { DatabaseSync } from "node:sqlite"; import { note } from "../../packages/terminal-core/src/note.js"; import { INBOUND_CONTEXT_MARKER } from "../auto-reply/reply/inbound-context-marker.js"; @@ -12,11 +11,11 @@ import { resolveAllAgentSessionStoreTargetsSync } from "../config/sessions/targe import type { OpenClawConfig } from "../config/types.openclaw.js"; import { formatErrorMessage } from "../infra/errors.js"; import { openNodeSqliteDatabase } from "../infra/node-sqlite.js"; +import { runOpenClawAgentWriteTransaction } from "../state/openclaw-agent-db.js"; import { - resolveOpenClawAgentSqlitePath, - runOpenClawAgentWriteTransaction, -} from "../state/openclaw-agent-db.js"; -import { resolveTargetSqliteOptions } from "./doctor-session-sqlite-readers.js"; + projectExistingAgentDatabaseTargets, + resolveTargetSqliteOptions, +} from "./doctor-session-sqlite-readers.js"; import { ReadOnlySqliteTranscriptReader } from "./doctor-session-sqlite-transcript-readers.js"; const NOTE_TITLE = "Session transcript labels"; @@ -206,14 +205,13 @@ export async function noteSessionTranscriptLabelHealth(params: { let repairedSessions = 0; let repairedEvents = 0; - const seenPaths = new Set(); - for (const target of resolveAllAgentSessionStoreTargetsSync(params.cfg, { env })) { + for (const target of projectExistingAgentDatabaseTargets( + resolveAllAgentSessionStoreTargetsSync(params.cfg, { env }), + env, + params.cfg, + )) { const databaseOptions = resolveTargetSqliteOptions(target, env); - const sqlitePath = resolveOpenClawAgentSqlitePath(databaseOptions); - if (seenPaths.has(sqlitePath) || !fs.existsSync(sqlitePath)) { - continue; - } - seenPaths.add(sqlitePath); + const sqlitePath = target.sqlitePath; const { agentId } = target; let readDatabase: DatabaseSync | undefined; diff --git a/src/commands/doctor-state-integrity-format.ts b/src/commands/doctor-state-integrity-format.ts index 2fc744788697..121a37af2d9f 100644 --- a/src/commands/doctor-state-integrity-format.ts +++ b/src/commands/doctor-state-integrity-format.ts @@ -1,3 +1,8 @@ +export type OrphanAgentDir = { + dirName: string; + agentId: string; +}; + export function countLabel(count: number, singular: string, plural = `${singular}s`): string { return `${count} ${count === 1 ? singular : plural}`; } diff --git a/src/commands/doctor-state-integrity.test-support.ts b/src/commands/doctor-state-integrity.test-support.ts index 9db9f699e24f..fdf1c33cedc8 100644 --- a/src/commands/doctor-state-integrity.test-support.ts +++ b/src/commands/doctor-state-integrity.test-support.ts @@ -6,6 +6,7 @@ import { resolveSessionStorePathCore, resolveSessionTranscriptsDirForAgent, } from "../config/sessions/paths.js"; +import { openOpenClawStateDatabase } from "../state/openclaw-state-db.js"; import { noteStateIntegrity as noteStateIntegrityRaw } from "./doctor-state-integrity.js"; export const noteMock = vi.fn(); @@ -78,6 +79,7 @@ export function writeSessionStore( sessions: Record>, agentId = "main", ) { + openOpenClawStateDatabase({ env: process.env }); setupSessionState(cfg, process.env, process.env.HOME ?? "", agentId); const storePath = resolveSessionStorePathCore(cfg.session?.store, { agentId }); fs.writeFileSync(storePath, JSON.stringify(sessions, null, 2)); diff --git a/src/commands/doctor-state-integrity.ts b/src/commands/doctor-state-integrity.ts index 59965b19cd1d..8d4c09487204 100644 --- a/src/commands/doctor-state-integrity.ts +++ b/src/commands/doctor-state-integrity.ts @@ -42,7 +42,11 @@ import { scanDoctorSessionEntriesStrict, } from "../config/sessions/session-accessor.js"; import { resolveSqliteTargetFromSessionStorePath } from "../config/sessions/session-sqlite-target.js"; -import { resolveSessionStoreTargets, type SessionStoreTarget } from "../config/sessions/targets.js"; +import { + resolveConfiguredAgentDatabaseTargets, + resolveSessionStoreTargets, + type SessionStoreTarget, +} from "../config/sessions/targets.js"; import type { SessionEntry } from "../config/sessions/types.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import type { HealthFinding, HealthRepairEffect } from "../flows/health-checks.js"; @@ -57,6 +61,7 @@ import { import { listConfiguredChannelIdsForReadOnlyScope } from "../plugins/channel-plugin-ids.js"; import { normalizeAgentId } from "../routing/session-key.js"; import { readAgentDatabaseAdmissionRefusal } from "../state/agent-database-admission.js"; +import { createRetainedAgentDatabaseMatcher } from "../state/agent-deletion-discovery.js"; import { isReservedSystemAgentId } from "../system-agent/agent-id.js"; import { shortenHomePath } from "../utils.js"; import { repairHeartbeatPoisonedMainSession } from "./doctor-heartbeat-main-session-repair.js"; @@ -70,7 +75,7 @@ import { createPluginSessionStateDoctorScanner, runPluginSessionStateDoctorRepairs, } from "./doctor-session-state-providers.js"; -import { countLabel } from "./doctor-state-integrity-format.js"; +import { countLabel, type OrphanAgentDir } from "./doctor-state-integrity-format.js"; import { collectRetainedUnconfiguredAgentDatabaseWarnings } from "./doctor-unconfigured-agent-databases.js"; const STATE_INTEGRITY_CHECK_ID = "core/doctor/state-integrity"; @@ -100,11 +105,6 @@ function existsFile(filePath: string): boolean { } } -type OrphanAgentDir = { - dirName: string; - agentId: string; -}; - type RuntimeDirLabel = "Sessions dir" | "Session store dir" | "OAuth dir"; export type StateIntegrityHealthIssue = @@ -186,22 +186,7 @@ function isReachableConfiguredAgentDir(params: { } const rawDir = path.join(params.agentsRoot, params.dirName, "agent"); const normalizedDir = path.join(params.agentsRoot, params.agentId, "agent"); - const rawRealPath = tryResolveNativeRealPath(rawDir); - const normalizedRealPath = tryResolveNativeRealPath(normalizedDir); - return rawRealPath !== null && rawRealPath === normalizedRealPath; -} - -function formatOrphanAgentDirLabel(entry: OrphanAgentDir): string { - return entry.dirName === entry.agentId ? entry.agentId : `${entry.dirName} (id ${entry.agentId})`; -} - -function formatOrphanAgentDirPreview(entries: OrphanAgentDir[], limit = 3): string { - const labels = entries.slice(0, limit).map(formatOrphanAgentDirLabel); - const remaining = entries.length - labels.length; - if (remaining > 0) { - return `${labels.join(", ")}, and ${remaining} more`; - } - return labels.join(", "); + return areComparablePathsEqual(rawDir, normalizedDir); } function listOrphanAgentDirs(cfg: OpenClawConfig, stateDir: string): OrphanAgentDir[] { @@ -341,8 +326,6 @@ function isPathUnderRoot(targetPath: string, rootPath: string): boolean { return isPathUnderRootWithPathOps(targetPath, rootPath, path); } -const tryResolveRealPath = safeRealpathSync; - function resolvePathThroughExistingAncestor( targetPath: string, resolveRealPath: (targetPath: string) => string | null, @@ -498,7 +481,7 @@ function resolveLinuxStateMount( }, ): LinuxSdBackedStateDir | null { const linuxPath = path.posix; - const resolveRealPath = deps?.resolveRealPath ?? tryResolveRealPath; + const resolveRealPath = deps?.resolveRealPath ?? safeRealpathSync; const resolvedStatePath = resolvePathThroughExistingAncestor(stateDir, resolveRealPath, linuxPath) ?? linuxPath.resolve(stateDir); @@ -538,9 +521,7 @@ export function detectLinuxSdBackedStateDir( const sourceCandidates = [stateMount.source]; if (stateMount.source.startsWith("/dev/")) { - const resolvedDevicePath = (deps?.resolveDeviceRealPath ?? tryResolveRealPath)( - stateMount.source, - ); + const resolvedDevicePath = (deps?.resolveDeviceRealPath ?? safeRealpathSync)(stateMount.source); if (resolvedDevicePath) { sourceCandidates.push(linuxPath.resolve(resolvedDevicePath)); } @@ -644,7 +625,7 @@ export function detectMacCloudSyncedStateDir( root: path.join(homedir, "Library", "CloudStorage"), }, ]; - const resolveRealPath = deps?.resolveRealPath ?? tryResolveRealPath; + const resolveRealPath = deps?.resolveRealPath ?? safeRealpathSync; // Missing state leaves must still follow existing symlink ancestors, like the Linux detectors. const resolvedStatePath = resolvePathThroughExistingAncestor(stateDir, resolveRealPath, path) ?? path.resolve(stateDir); @@ -692,7 +673,7 @@ export function detectWindowsCloudSyncedStateDir( return null; } - const resolveRealPath = deps?.resolveRealPath ?? tryResolveRealPath; + const resolveRealPath = deps?.resolveRealPath ?? safeRealpathSync; // A state dir that does not exist yet cannot be resolved directly, and // falling back to the lexical path misreads a not-yet-created leaf beneath a // OneDrive-named junction that actually resolves to local storage. Resolve @@ -1329,13 +1310,19 @@ export async function noteStateIntegrity( const orphanAgentDirs = listOrphanAgentDirs(cfg, stateDir); if (orphanAgentDirs.length > 0) { + const labels = orphanAgentDirs + .slice(0, 3) + .map(({ dirName, agentId }) => + dirName === agentId ? agentId : `${dirName} (id ${agentId})`, + ); + const remaining = orphanAgentDirs.length - labels.length; const authoredAgentRosterPath = readAgentRosterProperty(cfg)?.kind === "list" ? "agents.list" : "agents.entries"; warnings.push( [ `- Found ${countLabel(orphanAgentDirs.length, "agent directory", "agent directories")} on disk without a matching ${authoredAgentRosterPath} entry.`, " These agents can still have sessions/auth state on disk, but config-driven routing, identity, and model selection will ignore them.", - ` Examples: ${formatOrphanAgentDirPreview(orphanAgentDirs)}`, + ` Examples: ${labels.join(", ")}${remaining > 0 ? `, and ${remaining} more` : ""}`, ` Restore the missing ${authoredAgentRosterPath} entries or remove stale dirs after confirming they are no longer needed: ${shortenHomePath(path.join(stateDir, "agents"))}`, ].join("\n"), ); @@ -1345,6 +1332,9 @@ export async function noteStateIntegrity( } const compatibilityAgentId = resolveSessionStoreCompatibilityAgentId(cfg); + const isRetained = createRetainedAgentDatabaseMatcher(env, () => + resolveConfiguredAgentDatabaseTargets(cfg, { env }), + ); const sessionTargets = resolveSessionStoreTargets(cfg, { allAgents: true }, { env }).toSorted( (left, right) => left.agentId === compatibilityAgentId ? -1 : right.agentId === compatibilityAgentId ? 1 : 0, @@ -1362,6 +1352,9 @@ export async function noteStateIntegrity( defaultAgentId: compatibilityAgentId, env, }).path; + if (isRetained(sqliteStorePath, agentId)) { + return; + } // A successful SQLite import archives sessions.json. Its continued presence // is therefore the explicit signal that pre-import rows still need inspection. const legacyStore = @@ -1561,7 +1554,10 @@ export async function noteStateIntegrity( // Scan that file once under the compatibility owner so full-store work is not repeated. const inspectedLegacyStores = new Set(); for (const target of sessionTargets) { - if (readAgentDatabaseAdmissionRefusal(target.agentId, { env })) { + if ( + isRetained(target.storePath, target.agentId) || + readAgentDatabaseAdmissionRefusal(target.agentId, { env }) + ) { continue; } const legacyStorePath = path.resolve(target.storePath); diff --git a/src/commands/doctor-telegram-general-topic-conversations.ts b/src/commands/doctor-telegram-general-topic-conversations.ts index f3d4b441f17b..f5c98b514679 100644 --- a/src/commands/doctor-telegram-general-topic-conversations.ts +++ b/src/commands/doctor-telegram-general-topic-conversations.ts @@ -20,6 +20,7 @@ import { type OpenClawAgentDatabase, } from "../state/openclaw-agent-db.js"; import { runDoctorAgentDatabaseOperation } from "./doctor-agent-database-operation.js"; +import { projectExistingAgentDatabaseTargets } from "./doctor-session-sqlite-readers.js"; const GENERAL_TOPIC_ID = "1"; const LEGACY_GENERAL_TARGET = /^telegram:(-?\d+):topic:1$/u; @@ -89,7 +90,11 @@ function listLegacyRows(database: import("node:sqlite").DatabaseSync): Conversat } function resolveRepairScopes(cfg: OpenClawConfig, env: NodeJS.ProcessEnv) { - return resolveAllAgentSessionStoreTargetsSync(cfg, { env }).map((target) => { + return projectExistingAgentDatabaseTargets( + resolveAllAgentSessionStoreTargetsSync(cfg, { env }), + env, + cfg, + ).map((target) => { const scope = resolveSqliteReadScope({ agentId: target.agentId, env, diff --git a/src/commands/doctor/shared/codex-route-session-repair.ts b/src/commands/doctor/shared/codex-route-session-repair.ts index 39044552d080..efa422b043f6 100644 --- a/src/commands/doctor/shared/codex-route-session-repair.ts +++ b/src/commands/doctor/shared/codex-route-session-repair.ts @@ -22,7 +22,10 @@ import { scanDoctorSessionEntriesStrict, scanDoctorSessionEntriesTolerant, } from "../../../config/sessions/session-accessor.js"; -import { resolveAllAgentSessionStoreTargetsSync } from "../../../config/sessions/targets.js"; +import { + resolveAllAgentSessionStoreTargetsSync, + resolveConfiguredAgentDatabaseTargets, +} from "../../../config/sessions/targets.js"; import type { SessionEntry } from "../../../config/sessions/types.js"; import type { OpenClawConfig } from "../../../config/types.openclaw.js"; import { loadJsonFileThroughSymlink } from "../../../infra/json-file.js"; @@ -31,6 +34,7 @@ import { updateLegacySessionStore, } from "../../../infra/state-migrations.legacy-session-store.js"; import { isValidAgentHarnessSessionStoreEntry } from "../../../sessions/agent-harness-session-key.js"; +import { createRetainedAgentDatabaseMatcher } from "../../../state/agent-deletion-discovery.js"; import { resolveLegacyAuthProfilesPath } from "../../doctor-auth-legacy-paths.js"; import { isOpenAICodexAuthProfileRef, @@ -52,19 +56,10 @@ import { migrateLegacyRuntimeModelRef, resolveLegacyRuntimeModelProviderAlias, } from "./legacy-runtime-model-providers.js"; -import { - createRetiredModelRefRepairResolver, - type ModelRefRepairResolver, -} from "./retired-model-ref-repair.js"; +import type { SessionModelRetirement } from "./retired-model-ref-repair.js"; +import { createRetiredModelRefRepairResolver } from "./retired-model-ref-repair.js"; import { repairRetiredSessionModelRef } from "./retired-session-model-repair.js"; -type SessionModelRetirement = { - agentId: string; - resolve: ModelRefRepairResolver; - defaultModelRef?: string; - warnings: string[]; -}; - function rewriteSessionModelPair(params: { entry: SessionEntry; providerKey: "modelProvider" | "providerOverride"; @@ -197,13 +192,12 @@ function clearStaleCodexFallbackNotice( } function clearRepairedCodexSessionHarness(entry: SessionEntry): boolean { - const harnessRuntime = normalizeRuntimeString(entry.agentHarnessId); - let changed = false; - if (entry.agentHarnessId !== undefined && harnessRuntime !== "openclaw") { - delete entry.agentHarnessId; - changed = true; + const harnessId = entry.agentHarnessId; + if (harnessId === undefined || normalizeRuntimeString(harnessId) === "openclaw") { + return false; } - return changed; + delete entry.agentHarnessId; + return true; } function repairProviderlessCodexSessionOverride( @@ -536,7 +530,12 @@ export async function maybeRepairCodexSessionRoutes(params: { const authProfileOnly = !params.shouldRepair && params.authProfileOnly === true; const shouldRepair = params.shouldRepair || authProfileOnly; const warnings: string[] = []; - const sessionTargets = resolveAllAgentSessionStoreTargetsSync(params.cfg, { env }); + const isRetained = createRetainedAgentDatabaseMatcher(env, () => + resolveConfiguredAgentDatabaseTargets(params.cfg, { env }), + ); + const sessionTargets = resolveAllAgentSessionStoreTargetsSync(params.cfg, { env }).filter( + (target) => !isRetained(target.storePath, target.agentId), + ); const resolveRetired = authProfileOnly ? undefined : createRetiredModelRefRepairResolver({ diff --git a/src/commands/doctor/shared/retired-model-ref-repair.ts b/src/commands/doctor/shared/retired-model-ref-repair.ts index 9debfd11cdbb..91ef445261c0 100644 --- a/src/commands/doctor/shared/retired-model-ref-repair.ts +++ b/src/commands/doctor/shared/retired-model-ref-repair.ts @@ -65,6 +65,13 @@ export type ModelRefRepairResolver = (params: { authProfileOnly?: boolean; }) => ModelRefRepair; +export type SessionModelRetirement = { + agentId: string; + resolve: ModelRefRepairResolver; + defaultModelRef?: string; + warnings: string[]; +}; + export function repairModelRefAuthProfile( modelRef: string, profileIdMap: ReadonlyMap | undefined, diff --git a/src/config/sessions/combined-store-gateway.ts b/src/config/sessions/combined-store-gateway.ts index 07e2615d8045..0fe96e51d867 100644 --- a/src/config/sessions/combined-store-gateway.ts +++ b/src/config/sessions/combined-store-gateway.ts @@ -17,6 +17,7 @@ import { assertAgentDatabaseAdmitted, readAgentDatabaseAdmissionRefusal, } from "../../state/agent-database-admission.js"; +import { createRetainedAgentDatabaseMatcher } from "../../state/agent-deletion-discovery.js"; import { listOpenClawRegisteredAgentDatabases, listOpenIncognitoAgentDatabases, @@ -46,6 +47,7 @@ import { listKnownSessionStoreAgentIds, resolveAgentSessionStoreTargetsSync, resolveAllAgentSessionStoreTargetsSync, + resolveConfiguredAgentDatabaseTargets, type SessionStoreTarget, } from "./targets.js"; import type { SessionEntry } from "./types.js"; @@ -489,8 +491,14 @@ export function resolveGatewaySessionStoreTargets( resolved = { ...resolved, durableTargets, physicalTargets }; } const diagnostics = [...resolved.diagnostics]; - const admitted = (target: SessionStoreTarget): boolean => { + const isRetained = createRetainedAgentDatabaseMatcher(process.env, () => + resolveConfiguredAgentDatabaseTargets(cfg, { env: process.env }), + ); + const admitted = (target: SessionStoreTarget, durable = false): boolean => { const physical = resolved.physicalTargets.get(storeTargetKey(target)); + if (durable && isRetained(physical?.storePath ?? target.storePath, target.agentId)) { + return false; + } const refusal = readAgentDatabaseAdmissionRefusal(target.agentId) ?? (physical && readAgentDatabaseAdmissionRefusal(physical.agentId)); @@ -504,8 +512,8 @@ export function resolveGatewaySessionStoreTargets( return false; }; // Cached topology stays complete; each boot's admission is applied when consumed. - const durableTargets = resolved.durableTargets.filter(admitted); - const incognitoTargets = resolved.incognitoTargets.filter(admitted); + const durableTargets = resolved.durableTargets.filter((target) => admitted(target, true)); + const incognitoTargets = resolved.incognitoTargets.filter((target) => admitted(target)); if ( durableTargets.length === resolved.durableTargets.length && incognitoTargets.length === resolved.incognitoTargets.length diff --git a/src/config/sessions/session-sqlite-target.ts b/src/config/sessions/session-sqlite-target.ts index c979d4c3b5c3..d81bcc42116f 100644 --- a/src/config/sessions/session-sqlite-target.ts +++ b/src/config/sessions/session-sqlite-target.ts @@ -1,5 +1,6 @@ import { lstatSync, readdirSync } from "node:fs"; import path from "node:path"; +import { hasErrnoCode } from "../../infra/errno.js"; import { LEGACY_IMPLICIT_AGENT_ID, normalizeAgentId } from "../../routing/session-key.js"; import type { OpenClawRegisteredAgentDatabase } from "../../state/openclaw-agent-db-contract.js"; import { @@ -360,12 +361,13 @@ export function listSqliteTargetCandidatePathsForSessionStorePath(storePath: str const candidateNames = new Set([path.basename(unsuffixedTarget.path)]); try { for (const fileName of readdirSync(directory)) { - if (fileName.startsWith(`${baseName}.`) && fileName.endsWith(".sqlite")) { - candidateNames.add(fileName); + const databaseName = fileName.replace(/-(?:wal|shm|journal)$/u, ""); + if (databaseName.startsWith(`${baseName}.`) && databaseName.endsWith(".sqlite")) { + candidateNames.add(databaseName); } } } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + if (!hasErrnoCode(error, "ENOENT")) { throw error; } } diff --git a/src/flows/doctor-auth-health.ts b/src/flows/doctor-auth-health.ts index 796dcc09c9fd..af49daac740e 100644 --- a/src/flows/doctor-auth-health.ts +++ b/src/flows/doctor-auth-health.ts @@ -4,6 +4,13 @@ import type { DoctorHealthFlowContext } from "./doctor-health-contribution-types export async function runAuthProfileMigration(ctx: DoctorHealthFlowContext): Promise { ctx.authProfileHealthReady = false; + // Auth repair depends on the shared-store owner; its receipt records the held-store guidance. + const sharedAuth = ctx.configResult.stateMigrationStepReceipts?.find( + (receipt) => receipt.id === "shared-auth-store", + ); + if (sharedAuth?.outcome === "skipped" && sharedAuth.warnings.length > 0) { + return; + } const { repairAuthProfileMigration } = await import("../commands/doctor/auth-profile-repair.js"); const { maybeRepairLegacyOAuthProfileIds } = await import("../commands/doctor-auth-legacy-oauth.js"); diff --git a/src/gateway/cli-session-history.test-support.ts b/src/gateway/cli-session-history.test-support.ts new file mode 100644 index 000000000000..fb1b6e1b3f8d --- /dev/null +++ b/src/gateway/cli-session-history.test-support.ts @@ -0,0 +1,77 @@ +export function createClaudeHistoryLines(sessionId: string) { + return [ + JSON.stringify({ + type: "queue-operation", + operation: "enqueue", + timestamp: "2026-03-26T16:29:54.722Z", + sessionId, + content: "[Thu 2026-03-26 16:29 GMT] Reply with exactly: AGENT CLI OK.", + }), + JSON.stringify({ + type: "user", + uuid: "user-1", + timestamp: "2026-03-26T16:29:54.800Z", + message: { + role: "user", + content: + 'Sender: ⟦openclaw:ctx⟧\n```json\n{"label":"openclaw-control-ui"}\n```\n\n[Thu 2026-03-26 16:29 GMT] hi', + }, + }), + JSON.stringify({ + type: "assistant", + uuid: "assistant-1", + timestamp: "2026-03-26T16:29:55.500Z", + message: { + role: "assistant", + model: "claude-sonnet-4-6", + content: [{ type: "text", text: "hello from Claude" }], + stop_reason: "end_turn", + usage: { + input_tokens: 11, + output_tokens: 7, + cache_read_input_tokens: 22, + }, + }, + }), + JSON.stringify({ + type: "assistant", + uuid: "assistant-2", + timestamp: "2026-03-26T16:29:56.000Z", + message: { + role: "assistant", + model: "claude-sonnet-4-6", + content: [ + { + type: "tool_use", + id: "toolu_123", + name: "Bash", + input: { + command: "pwd", + }, + }, + ], + stop_reason: "tool_use", + }, + }), + JSON.stringify({ + type: "user", + uuid: "user-2", + timestamp: "2026-03-26T16:29:56.400Z", + message: { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "toolu_123", + content: "/tmp/demo", + }, + ], + }, + }), + JSON.stringify({ + type: "last-prompt", + sessionId, + lastPrompt: "ignored", + }), + ].join("\n"); +} diff --git a/src/gateway/cli-session-history.test.ts b/src/gateway/cli-session-history.test.ts index 8e9055a1942e..f0c4e6e5802d 100644 --- a/src/gateway/cli-session-history.test.ts +++ b/src/gateway/cli-session-history.test.ts @@ -22,6 +22,7 @@ import { resolveChatHistoryWithCliSessionImports, } from "./cli-session-history.js"; import { mergeImportedChatHistoryMessages } from "./cli-session-history.merge.js"; +import { createClaudeHistoryLines } from "./cli-session-history.test-support.js"; import { expectRecordFields, requireGatewayRecord } from "./test-helpers.assertions.js"; type ClaudeCliFallbackSeed = NonNullable>; @@ -92,84 +93,6 @@ function buildLegacyReseedPrompt(current = "current"): string { ].join("\n"); } -function createClaudeHistoryLines(sessionId: string) { - return [ - JSON.stringify({ - type: "queue-operation", - operation: "enqueue", - timestamp: "2026-03-26T16:29:54.722Z", - sessionId, - content: "[Thu 2026-03-26 16:29 GMT] Reply with exactly: AGENT CLI OK.", - }), - JSON.stringify({ - type: "user", - uuid: "user-1", - timestamp: "2026-03-26T16:29:54.800Z", - message: { - role: "user", - content: - 'Sender: ⟦openclaw:ctx⟧\n```json\n{"label":"openclaw-control-ui"}\n```\n\n[Thu 2026-03-26 16:29 GMT] hi', - }, - }), - JSON.stringify({ - type: "assistant", - uuid: "assistant-1", - timestamp: "2026-03-26T16:29:55.500Z", - message: { - role: "assistant", - model: "claude-sonnet-4-6", - content: [{ type: "text", text: "hello from Claude" }], - stop_reason: "end_turn", - usage: { - input_tokens: 11, - output_tokens: 7, - cache_read_input_tokens: 22, - }, - }, - }), - JSON.stringify({ - type: "assistant", - uuid: "assistant-2", - timestamp: "2026-03-26T16:29:56.000Z", - message: { - role: "assistant", - model: "claude-sonnet-4-6", - content: [ - { - type: "tool_use", - id: "toolu_123", - name: "Bash", - input: { - command: "pwd", - }, - }, - ], - stop_reason: "tool_use", - }, - }), - JSON.stringify({ - type: "user", - uuid: "user-2", - timestamp: "2026-03-26T16:29:56.400Z", - message: { - role: "user", - content: [ - { - type: "tool_result", - tool_use_id: "toolu_123", - content: "/tmp/demo", - }, - ], - }, - }), - JSON.stringify({ - type: "last-prompt", - sessionId, - lastPrompt: "ignored", - }), - ].join("\n"); -} - function createClaudeTextHistoryLines( entries: Array<{ content: string; role: "assistant" | "user"; uuid: string }>, ): string { @@ -3026,7 +2949,11 @@ describe("cli session history", () => { await withClaudeProjectsDir(async ({ homeDir, sessionId }) => { const { maybeRepairCodexSessionRoutes } = await import("../commands/doctor/shared/codex-route-session-repair.js"); + const { openOpenClawStateDatabase, closeOpenClawStateDatabaseForTest } = + await import("../state/openclaw-state-db.js"); const stateDir = path.join(homeDir, "state"); + openOpenClawStateDatabase({ env: { OPENCLAW_STATE_DIR: stateDir } }); + closeOpenClawStateDatabaseForTest(); const storePath = path.join(stateDir, "agents", "main", "sessions", "sessions.json"); const key = "agent:main:cli-history"; const entry: SessionEntry = { diff --git a/src/gateway/session-row-projection.deleted-agent.test.ts b/src/gateway/session-row-projection.deleted-agent.test.ts new file mode 100644 index 000000000000..54b0feef7e28 --- /dev/null +++ b/src/gateway/session-row-projection.deleted-agent.test.ts @@ -0,0 +1,60 @@ +import fs from "node:fs"; +import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { expect, it, vi } from "vitest"; +import { replaceSessionEntrySync } from "../config/sessions/session-accessor.js"; +import { createLegacyDatabaseFixture } from "../infra/state-migrations.media-persistence.test-support.js"; +import { + beginAgentDeletionJournal, + completeAgentDeletionJournalInDatabase, +} from "../state/agent-deletion-journal.js"; +import { runOpenClawStateWriteTransaction } from "../state/openclaw-state-db.js"; +import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js"; +import { createSessionRowProjection } from "./session-row-projection.js"; + +it("leaves a completed deleted store out of startup projection", async () => { + await withOpenClawTestState({ scenario: "minimal" }, async (state) => { + const cfg = { agents: { ownership: "explicit" as const, entries: { main: {} } } }; + const activeKey = "agent:main:active"; + replaceSessionEntrySync( + { agentId: "main", sessionKey: activeKey }, + { sessionId: "active", updatedAt: 1 }, + ); + const retainedPath = createLegacyDatabaseFixture({ + agentId: "retired", + env: process.env, + eventsBySession: {}, + schemaVersion: 19, + }); + beginAgentDeletionJournal({ + agentId: "retired", + operationId: "delete-retired", + agentDir: path.dirname(retainedPath), + workspaceDir: state.statePath("workspace-retired"), + sessionsDir: state.statePath("agents", "retired", "sessions"), + deleteFiles: false, + }); + runOpenClawStateWriteTransaction((database) => { + completeAgentDeletionJournalInDatabase(database, "retired", "delete-retired"); + }); + const bytes = fs.readFileSync(retainedPath); + const projection = await createSessionRowProjection({ cfg, modelCatalog: [] }); + try { + await projection.ensureMaterialized(); + const reads = vi.spyOn(DatabaseSync.prototype, "prepare"); + const exec = vi.spyOn(DatabaseSync.prototype, "exec"); + try { + expect(projection.selectEntries().map((row) => row.key)).toEqual([activeKey]); + expect(projection.describe({ agentId: "main", key: activeKey })).toBeDefined(); + expect(reads).not.toHaveBeenCalled(); + expect(exec).not.toHaveBeenCalled(); + } finally { + reads.mockRestore(); + exec.mockRestore(); + } + expect(fs.readFileSync(retainedPath).equals(bytes)).toBe(true); + } finally { + projection.dispose(); + } + }); +}); diff --git a/src/infra/state-migrations.media-persistence-targets.test.ts b/src/infra/state-migrations.media-persistence-targets.test.ts index 7e8cf83ff5f3..330b15190eea 100644 --- a/src/infra/state-migrations.media-persistence-targets.test.ts +++ b/src/infra/state-migrations.media-persistence-targets.test.ts @@ -1,10 +1,15 @@ import fs from "node:fs"; import path from "node:path"; -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it } from "vitest"; import { cleanupTempDirs, makeTempDir } from "../../test/helpers/temp-dir.js"; import { resolveSessionStorePathCore } from "../config/sessions/paths.js"; import { resolveSqliteTargetFromSessionStorePath } from "../config/sessions/session-sqlite-target.js"; import { resolveConfiguredAgentDatabaseTargets } from "../config/sessions/targets.js"; +import { + beginAgentDeletionJournal, + completeAgentDeletionJournalInDatabase, + removeAgentDeletionJournal, +} from "../state/agent-deletion-journal.js"; import { readRegisteredAgentDatabases } from "../state/openclaw-agent-db-registry-listing.js"; import { registerOpenClawAgentDatabase, @@ -20,6 +25,7 @@ import { assertOpenClawDatabasesReady } from "../state/openclaw-database-preflig import { closeOpenClawStateDatabaseForTest, openOpenClawStateDatabase, + runOpenClawStateWriteTransaction, } from "../state/openclaw-state-db.js"; import { requireNodeSqlite } from "./node-sqlite.js"; import { @@ -58,23 +64,107 @@ function readUserVersion(databasePath: string): number { } afterEach(() => { - vi.restoreAllMocks(); closeOpenClawAgentDatabasesForTest(); closeOpenClawStateDatabaseForTest(); cleanupTempDirs(tempDirs); }); describe("media persistence migration targets", () => { + it.each( + ["aaa-deleted", "zzz-deleted"].flatMap((deletedId) => + [false, true].map((hardlink) => ({ deletedId, hardlink })), + ), + )( + "migrates a surviving physical owner beside retained registry owner $deletedId (hardlink=$hardlink)", + async ({ deletedId, hardlink }) => { + const stateDir = fs.realpathSync.native( + makeTempDir(tempDirs, "media-retained-shared-owner-"), + ); + const env = { OPENCLAW_STATE_DIR: stateDir }; + const databasePath = createLegacyAgentDatabase({ env }); + const retainedPath = hardlink ? path.join(stateDir, "retained.sqlite") : databasePath; + if (hardlink) { + fs.linkSync(databasePath, retainedPath); + } + registerOpenClawAgentDatabase({ + agentId: deletedId, + path: retainedPath, + env, + schemaVersion: PREVIOUS_VERSION, + }); + beginAgentDeletionJournal( + { + agentId: deletedId, + operationId: "delete-shared-owner", + agentDir: path.dirname(retainedPath), + workspaceDir: path.join(stateDir, "workspace"), + sessionsDir: path.join(stateDir, "agents", deletedId, "sessions"), + deleteFiles: false, + }, + { env }, + ); + runOpenClawStateWriteTransaction( + (database) => { + completeAgentDeletionJournalInDatabase(database, deletedId, "delete-shared-owner"); + }, + { env }, + ); + + await expect( + assertOpenClawDatabasesReady({ + env, + operation: "doctor", + configuredAgentDatabaseTargets: [], + }), + ).rejects.toThrow(/uses schema version 16/); + const result = await migrateLegacyMediaPersistence({ env }); + expect(result.warnings).toEqual([]); + expect(result.notices ?? []).toEqual([]); + expect(readUserVersion(databasePath)).toBe(OPENCLAW_AGENT_SCHEMA_VERSION); + await expect( + assertOpenClawDatabasesReady({ + env, + operation: "doctor", + configuredAgentDatabaseTargets: [], + }), + ).resolves.toBeUndefined(); + }, + ); + it.each([ "unchanged", "missing-file-created", "directory-replaced", "registry-changed", "config-changed", - ] as const)("validates prepared fleet discovery once before registry cleanup: %s", (scenario) => { + "deletion-completed", + "deletion-restored", + ] as const)("rechecks prepared fleet facts before registry cleanup: %s", (scenario) => { const stateDir = fs.realpathSync.native(makeTempDir(tempDirs, "media-prepared-targets-")); const env = { OPENCLAW_STATE_DIR: stateDir }; const mainPath = createLegacyAgentDatabase({ env }); + const completeDeletion = () => { + beginAgentDeletionJournal( + { + agentId: "main", + operationId: "delete-main", + agentDir: path.dirname(mainPath), + workspaceDir: path.join(stateDir, "workspace"), + sessionsDir: path.join(stateDir, "agents", "main", "sessions"), + deleteFiles: false, + }, + { env }, + ); + runOpenClawStateWriteTransaction( + (database) => { + completeAgentDeletionJournalInDatabase(database, "main", "delete-main"); + }, + { env }, + ); + }; + if (scenario === "deletion-restored") { + completeDeletion(); + } const missingPath = path.join(stateDir, "agents", "missing", "agent", "openclaw-agent.sqlite"); fs.mkdirSync(path.dirname(missingPath), { recursive: true }); if (scenario === "directory-replaced") { @@ -94,7 +184,6 @@ describe("media persistence migration targets", () => { { env, includeIncompatibleSchemaVersions: true }, false, ); - const reads = vi.spyOn(fs, "readdirSync"); const preparedDiscovery: PreparedAgentDatabaseMigrationDiscovery = { stateDir, configuredAgentDatabaseTargets, @@ -121,6 +210,10 @@ describe("media persistence migration targets", () => { } else if (scenario === "registry-changed") { // Raw schema migrations can change registrations before the memo owner is invalidated. state.db.prepare("DELETE FROM agent_databases WHERE agent_id = ?").run("missing"); + } else if (scenario === "deletion-completed") { + completeDeletion(); + } else if (scenario === "deletion-restored") { + removeAgentDeletionJournal("main", "delete-main", { env }); } const result = resolveAgentDatabaseMigrationTargets({ env, @@ -135,11 +228,8 @@ describe("media persistence migration targets", () => { changes: [], warnings: [], }); - expect( - reads.mock.calls.filter(([directory]) => directory === path.join(stateDir, "agents")), - ).toHaveLength(scenario === "unchanged" ? 1 : 2); expect(result.targets.map((target) => target.path)).toEqual( - fileCreated ? [mainPath, missingPath] : [mainPath], + fileCreated ? [mainPath, missingPath] : scenario === "deletion-completed" ? [] : [mainPath], ); expect( state.db.prepare("SELECT agent_id FROM agent_databases WHERE agent_id = ?").get("missing") !== diff --git a/src/infra/state-migrations.media-persistence-targets.ts b/src/infra/state-migrations.media-persistence-targets.ts index b245812609e5..7bdf12c251e0 100644 --- a/src/infra/state-migrations.media-persistence-targets.ts +++ b/src/infra/state-migrations.media-persistence-targets.ts @@ -2,10 +2,15 @@ import fs from "node:fs"; import path from "node:path"; import { sanitizeForLog } from "../../packages/terminal-core/src/ansi.js"; import { resolveAgentSessionDirsFromAgentsDirSync } from "../agents/session-dirs.js"; +import { formatCliCommand } from "../cli/command-format.js"; import { resolveStateDir } from "../config/paths.js"; import { isSessionArchiveArtifactName } from "../config/sessions/artifacts.js"; import { normalizeAgentId } from "../routing/session-key.js"; -import { readRegisteredAgentDatabases } from "../state/openclaw-agent-db-registry-listing.js"; +import { createAgentDatabaseDeletionClassifier } from "../state/agent-deletion-discovery.js"; +import { + readAgentDatabaseDeletionSnapshot, + type AgentDeletionJournalDisposition, +} from "../state/agent-deletion-journal.read.js"; import { createOpenClawAgentDatabasePathMatcher, isPersistentOpenClawAgentDatabasePath, @@ -30,128 +35,64 @@ export type PreparedAgentDatabaseMigrationDiscovery = { discovery: ReturnType; }; -function readAgentsDirectoryIdentity(env: NodeJS.ProcessEnv): string | null { - try { - const stat = fs.statSync(path.join(resolveStateDir(env), "agents"), { throwIfNoEntry: false }); - return stat ? `${stat.dev}:${stat.ino}:${stat.mtimeMs}` : "missing"; - } catch { - return null; - } -} - -function sameTargets( - left: readonly { agentId: string; path: string }[], - right: readonly { agentId: string; path: string }[], -): boolean { - return ( - left.length === right.length && - left.every( - (target, index) => - target.agentId === right[index]?.agentId && target.path === right[index]?.path, - ) - ); -} - -function preparedDiscoveryIsCurrent( - prepared: PreparedAgentDatabaseMigrationDiscovery, - params: { - configuredAgentDatabaseTargets: readonly { agentId: string; path: string }[]; - registeredAgentDatabases: readonly { agentId: string; path: string }[]; - env: NodeJS.ProcessEnv; - }, -): boolean { - const directoryIdentity = readAgentsDirectoryIdentity(params.env); - if ( - prepared.stateDir !== resolveStateDir(params.env) || - !sameTargets(prepared.configuredAgentDatabaseTargets, params.configuredAgentDatabaseTargets) || - !sameTargets(prepared.registeredAgentDatabases, params.registeredAgentDatabases) || - prepared.discovery.failures.length > 0 || - directoryIdentity === null || - directoryIdentity !== prepared.discovery.agentsDirectoryIdentity - ) { - return false; - } - try { - for (const [pathname, identity] of prepared.discovery.sourceIdentities) { - let realPath: string | undefined; - try { - realPath = fs.realpathSync.native(pathname); - } catch (error) { - if (!hasErrnoCode(error, "ENOENT")) { - return false; - } - } - if ( - realPath !== identity.realPath || - (identity.isFile !== undefined && - (fs.statSync(pathname, { throwIfNoEntry: false })?.isFile() ?? false) !== identity.isFile) - ) { - return false; - } - } - return true; - } catch { - return false; - } -} - -function listDefaultAgentDatabaseTargets( - env: NodeJS.ProcessEnv, - failure: (pathname: string, reason: string) => void, -): CandidateTarget[] { - const agentsDir = path.join(resolveStateDir(env), "agents"); - try { - return resolveAgentSessionDirsFromAgentsDirSync(agentsDir).map((sessionsDir) => { - const agentDir = path.dirname(sessionsDir); - return { - agentId: normalizeAgentId(path.basename(agentDir)), - path: path.join(agentDir, "agent", "openclaw-agent.sqlite"), - source: "disk" as const, - }; - }); - } catch (error) { - failure(agentsDir, `Could not enumerate agent databases under ${agentsDir}: ${String(error)}`); - return []; - } -} - /** Discover maintenance targets without mutating the registry or creating stores. */ export function discoverAgentDatabaseMigrationTargets(params: { configuredAgentDatabaseTargets: readonly { agentId: string; path: string }[]; registeredAgentDatabases: readonly { agentId: string; path: string }[]; + retainedDeletions?: AgentDeletionJournalDisposition; env: NodeJS.ProcessEnv; }) { const warnings: string[] = []; const externalWarnings: string[] = []; + const retainedDeletions = + params.retainedDeletions ?? + readAgentDatabaseDeletionSnapshot(params.env)?.retainedDeletions ?? + "unavailable"; + const classifyDeletion = createAgentDatabaseDeletionClassifier({ ...params, retainedDeletions }); const failures: Array<{ path: string; reason: string }> = []; const registryRemovals: Array<{ agentId: string; path: string; change?: string }> = []; - const agentsDirectoryIdentity = readAgentsDirectoryIdentity(params.env); - const sourceIdentities = new Map(); + const sourceIdentities = new Map(); const failure = (pathname: string, reason: string) => { warnings.push(reason); failures.push({ path: pathname, reason }); }; const discard = (candidate: CandidateTarget, change?: string) => { - if (candidate.source === "registry") { + if (candidate.source === "registry" && retainedDeletions !== "unavailable") { registryRemovals.push({ agentId: candidate.agentId, path: candidate.path, change }); } }; - // Owner authority is explicit config, then the recorded registry fact, then - // directory-name inference. Recorded identity must beat a stale directory basename. + // Configured and registered surviving owners precede retained and inferred paths. const candidates: CandidateTarget[] = [ ...params.configuredAgentDatabaseTargets.map((target) => ({ - agentId: target.agentId, - path: target.path, + ...target, source: "configured" as const, })), ...params.registeredAgentDatabases.map((entry) => ({ - agentId: entry.agentId, - path: entry.path, + ...entry, source: "registry" as const, })), - ...listDefaultAgentDatabaseTargets(params.env, failure), + ...(retainedDeletions === "unavailable" ? [] : retainedDeletions).flatMap((entry) => + entry.databasePaths.map((pathname) => ({ + agentId: entry.agentId, + path: pathname, + source: "disk" as const, + })), + ), ]; const activeStateDir = resolveStateDir(params.env); + const agentsDir = path.join(activeStateDir, "agents"); + try { + for (const sessionsDir of resolveAgentSessionDirsFromAgentsDirSync(agentsDir)) { + const agentDir = path.dirname(sessionsDir); + candidates.push({ + agentId: normalizeAgentId(path.basename(agentDir)), + path: path.join(agentDir, "agent", "openclaw-agent.sqlite"), + source: "disk", + }); + } + } catch (error) { + failure(agentsDir, `Could not enumerate agent databases under ${agentsDir}: ${String(error)}`); + } let activeStateDirRealPath: string | undefined; try { activeStateDirRealPath = fs.realpathSync.native(activeStateDir); @@ -165,7 +106,7 @@ export function discoverAgentDatabaseMigrationTargets(params: { } const configuredPathMatcher = createOpenClawAgentDatabasePathMatcher(); const targets: AgentDatabaseMigrationTarget[] = []; - const seenRealPaths = new Set(); + const seenPhysicalFiles = new Set(); for (const candidate of candidates) { // Preserve the original locator: lexical normalization of `link/../file` // can select a different file than filesystem symlink traversal does. @@ -186,6 +127,7 @@ export function discoverAgentDatabaseMigrationTargets(params: { } } sourceIdentities.set(pathname, { realPath }); + const deletion = classifyDeletion(pathname, candidate.agentId); const isConfiguredPath = realPath !== undefined && params.configuredAgentDatabaseTargets.some((configuredTarget) => { @@ -203,16 +145,16 @@ export function discoverAgentDatabaseMigrationTargets(params: { activeStateDirRealPath && (realPath === activeStateDirRealPath || isPathInside(activeStateDirRealPath, realPath)), ); - if (realPath && !isInsideActiveStateDir && !isConfiguredPath) { + if (realPath && !isInsideActiveStateDir && !isConfiguredPath && !deletion) { discard(candidate); const warning = `Skipped foreign agent database ${sanitizeForLog(pathname)}; it is outside the active state directory and is not a configured session store.`; warnings.push(warning); externalWarnings.push(warning); continue; } - let stat: fs.Stats | undefined; + let stat: fs.BigIntStats | undefined; try { - stat = fs.statSync(pathname); + stat = fs.statSync(pathname, { bigint: true }); } catch (error) { if (!hasErrnoCode(error, "ENOENT")) { failure( @@ -223,14 +165,12 @@ export function discoverAgentDatabaseMigrationTargets(params: { } } if (!stat?.isFile()) { - sourceIdentities.set(pathname, { realPath, isFile: false }); discard(candidate, `Removed missing agent database registry entry ${pathname}.`); if (candidate.source === "registry") { warnings.push(`Skipped missing registered agent database ${pathname}.`); } continue; } - sourceIdentities.set(pathname, { realPath, isFile: true }); if (!realPath) { discard(candidate); failure( @@ -239,20 +179,30 @@ export function discoverAgentDatabaseMigrationTargets(params: { ); continue; } - if (seenRealPaths.has(realPath)) { + const physicalFile = `${stat.dev}:${stat.ino}`; + if (seenPhysicalFiles.has(physicalFile)) { continue; } - // Claim identity only after every persistence, boundary, and file gate passed. - seenRealPaths.add(realPath); + if (deletion) { + // A deleted alias must not claim a surviving owner's physical file. + if (classifyDeletion(pathname)) { + seenPhysicalFiles.add(physicalFile); + warnings.push( + `Held agent ${sanitizeForLog(deletion === "unavailable" ? candidate.agentId : deletion.agentId)} database ${sanitizeForLog(pathname)} (${deletion === "unavailable" ? "deletion journal unavailable" : "retained-by-deletion"}); run ${formatCliCommand("openclaw doctor --fix", params.env)} to inspect restoration.`, + ); + } + continue; + } + seenPhysicalFiles.add(physicalFile); targets.push({ ...candidate, path: pathname, realPath }); } return { targets, + retainedDeletions, registryRemovals, warnings, externalWarnings, failures, - agentsDirectoryIdentity, sourceIdentities, }; } @@ -265,30 +215,18 @@ export function resolveAgentDatabaseMigrationTargets(params: { warnings: string[]; preparedDiscovery?: PreparedAgentDatabaseMigrationDiscovery; }): { targets: AgentDatabaseMigrationTarget[]; recoverableWarningCount: number } { - let registeredAgentDatabases: readonly { agentId: string; path: string }[] = []; - let registryReadFailed = false; - try { - registeredAgentDatabases = readRegisteredAgentDatabases( - { - env: params.env, - includeIncompatibleSchemaVersions: true, - }, - false, - ); - } catch (error) { - registryReadFailed = true; - params.warnings.push( - `Failed enumerating registered agent databases for state migration: ${String(error)}`, - ); - } - // Schema repair can rewrite registrations. Validate the prepared source once at mutation - // admission, including missing candidates, before applying its registry removals. - const discovery = - !registryReadFailed && - params.preparedDiscovery && - preparedDiscoveryIsCurrent(params.preparedDiscovery, { ...params, registeredAgentDatabases }) - ? params.preparedDiscovery.discovery - : discoverAgentDatabaseMigrationTargets({ ...params, registeredAgentDatabases }); + const snapshot = readAgentDatabaseDeletionSnapshot(params.env); + // Rediscover after schema repair and admission; initialization cannot replace unknown history. + const retainedDeletions = + params.preparedDiscovery?.stateDir === resolveStateDir(params.env) && + params.preparedDiscovery?.discovery.retainedDeletions === "unavailable" + ? "unavailable" + : (snapshot?.retainedDeletions ?? "unavailable"); + const discovery = discoverAgentDatabaseMigrationTargets({ + ...params, + registeredAgentDatabases: snapshot?.registeredAgentDatabases ?? [], + retainedDeletions, + }); for (const removed of discovery.registryRemovals) { unregisterOpenClawAgentDatabase({ ...removed, env: params.env }); if (removed.change) { @@ -300,8 +238,7 @@ export function resolveAgentDatabaseMigrationTargets(params: { // Failed discovery never grants that disposition, even if it also omitted a foreign entry. return { targets: discovery.targets, - recoverableWarningCount: - registryReadFailed || discovery.failures.length > 0 ? 0 : discovery.warnings.length, + recoverableWarningCount: discovery.failures.length > 0 ? 0 : discovery.warnings.length, }; } diff --git a/src/infra/state-migrations.retained-deleted-agent.test.ts b/src/infra/state-migrations.retained-deleted-agent.test.ts new file mode 100644 index 000000000000..e5aac8f67a88 --- /dev/null +++ b/src/infra/state-migrations.retained-deleted-agent.test.ts @@ -0,0 +1,469 @@ +import fs from "node:fs"; +import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import { maybeMigrateAuthProfileJsonStoresToSqlite } from "../commands/doctor-auth-flat-profiles.js"; +import { listAuthProfileRepairCandidates } from "../commands/doctor-auth-legacy-paths.js"; +import { maybeMigrateModelCatalogCredentials } from "../commands/doctor-model-catalog-credentials.js"; +import { createDoctorPrompter } from "../commands/doctor-prompter.js"; +import { repairCanonicalSessionKeys } from "../commands/doctor-session-canonical-keys.js"; +import { projectExistingAgentDatabaseTargets } from "../commands/doctor-session-sqlite-readers.js"; +import { noteSessionTranscriptHeaderHealth } from "../commands/doctor-session-transcript-headers.js"; +import { noteSessionTranscriptLabelHealth } from "../commands/doctor-session-transcript-labels.js"; +import { noteSessionTranscriptHealth } from "../commands/doctor-session-transcripts.js"; +import { noteStateIntegrity } from "../commands/doctor-state-integrity.js"; +import { detectTelegramGeneralTopicConversationRepairs } from "../commands/doctor-telegram-general-topic-conversations.js"; +import { maybeRepairCodexSessionRoutes } from "../commands/doctor/shared/codex-route-session-repair.js"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { EMPTY_LEGACY_SESSION_SURFACES } from "../plugins/legacy-session-surfaces.types.js"; +import { + beginAgentDeletionJournal, + completeAgentDeletionJournalInDatabase, +} from "../state/agent-deletion-journal.js"; +import { + registerOpenClawAgentDatabase, + unregisterOpenClawAgentDatabase, +} from "../state/openclaw-agent-db-registry.js"; +import { + closeOpenClawAgentDatabasesForTest, + OPENCLAW_AGENT_SCHEMA_VERSION, + openOpenClawAgentDatabase, +} from "../state/openclaw-agent-db.js"; +import { + assertOpenClawDatabasesReady, + preflightOpenClawDatabaseSchemas, +} from "../state/openclaw-database-preflight.js"; +import { + closeOpenClawStateDatabaseForTest, + runOpenClawStateWriteTransaction, +} from "../state/openclaw-state-db.js"; +import { autoMigrateLegacyState } from "./state-migrations.doctor.js"; +import type { PreparedAgentDatabaseMigrationDiscovery } from "./state-migrations.media-persistence-targets.js"; +import { + createLegacyDatabaseFixture, + readDatabaseSnapshot, +} from "./state-migrations.media-persistence.test-support.js"; +import { throwIfDoctorStateMigrationRefused } from "./state-migrations.messages.js"; +import { + detectSharedAuthStoreMigration, + migrateSharedAuthStore, +} from "./state-migrations.shared-auth-store.js"; + +const note = vi.hoisted(() => vi.fn()); +vi.mock("../../packages/terminal-core/src/note.js", () => ({ note })); + +afterEach(() => { + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + vi.unstubAllEnvs(); + note.mockClear(); +}); +const tempDirs = useAutoCleanupTempDirTracker(afterEach); + +describe("Doctor with a deleted agent database", () => { + it("repairs a configured survivor when only the deleted owner's registration remains", async () => { + const stateDir = fs.realpathSync.native(tempDirs.make("doctor-configured-survivor-")); + const env = { OPENCLAW_STATE_DIR: stateDir }; + vi.stubEnv("OPENCLAW_STATE_DIR", stateDir); + const cfg: OpenClawConfig = { + agents: { ownership: "explicit", entries: { main: {} } }, + plugins: { enabled: false }, + }; + const databasePath = createLegacyDatabaseFixture({ + env, + eventsBySession: {}, + schemaVersion: OPENCLAW_AGENT_SCHEMA_VERSION, + }); + registerOpenClawAgentDatabase({ agentId: "retired", path: databasePath, env }); + beginAgentDeletionJournal( + { + agentId: "retired", + operationId: "delete-retired", + agentDir: path.dirname(databasePath), + workspaceDir: path.join(stateDir, "workspace-retired"), + sessionsDir: path.join(stateDir, "agents", "retired", "sessions"), + deleteFiles: false, + }, + { env }, + ); + runOpenClawStateWriteTransaction( + (database) => { + completeAgentDeletionJournalInDatabase(database, "retired", "delete-retired"); + }, + { env }, + ); + unregisterOpenClawAgentDatabase({ agentId: "main", path: databasePath, env }); + expect( + projectExistingAgentDatabaseTargets( + [ + { agentId: "retired", storePath: databasePath }, + { agentId: "main", storePath: databasePath }, + ], + env, + cfg, + ).map((target) => target.agentId), + ).toEqual(["main"]); + expect(await repairCanonicalSessionKeys({ apply: true, cfg, env })).toMatchObject({ + scannedStores: 1, + }); + unregisterOpenClawAgentDatabase({ agentId: "main", path: databasePath, env }); + fs.writeFileSync( + path.join(path.dirname(databasePath), "models.json"), + JSON.stringify({ + providers: { + fixture: { + api: "openai-completions", + baseUrl: "https://example.invalid/v1", + apiKey: "synthetic-catalog-key", + models: [], + }, + }, + }), + ); + const runtime = { + log() {}, + error() {}, + exit(code: number): never { + throw new Error(`unexpected exit ${code}`); + }, + }; + const catalogs = await maybeMigrateModelCatalogCredentials({ + cfg, + env, + runtime, + prompter: createDoctorPrompter({ runtime, options: { repair: true, nonInteractive: true } }), + }); + expect(catalogs).toMatchObject({ detected: 1, migrated: 1, warnings: [] }); + }); + + it("records shared auth as held for deleted main even with a surviving physical registration", async () => { + const stateDir = fs.realpathSync.native(tempDirs.make("doctor-retained-shared-auth-")); + const env = { OPENCLAW_STATE_DIR: stateDir }; + vi.stubEnv("OPENCLAW_STATE_DIR", stateDir); + const sourcePath = createLegacyDatabaseFixture({ + agentId: "alive", + path: path.join(stateDir, "agents", "main", "agent", "openclaw-agent.sqlite"), + env, + eventsBySession: {}, + schemaVersion: OPENCLAW_AGENT_SCHEMA_VERSION, + }); + const source = new DatabaseSync(sourcePath); + source + .prepare( + "INSERT INTO auth_profile_store(store_key,store_json,updated_at) VALUES('primary',?,1)", + ) + .run( + JSON.stringify({ + version: 1, + profiles: { + "fixture:default": { + type: "api_key", + provider: "fixture", + key: "synthetic-retained-main-key", + }, + }, + }), + ); + source.close(); + beginAgentDeletionJournal( + { + agentId: "main", + operationId: "delete-shared-auth", + agentDir: path.dirname(sourcePath), + workspaceDir: path.join(stateDir, "workspace"), + sessionsDir: path.join(stateDir, "agents", "main", "sessions"), + deleteFiles: false, + }, + { env }, + ); + runOpenClawStateWriteTransaction( + (database) => completeAgentDeletionJournalInDatabase(database, "main", "delete-shared-auth"), + { env }, + ); + const bytes = fs.readFileSync(sourcePath); + const detected = detectSharedAuthStoreMigration({ + stateDir, + env, + doctorOnlyStateMigrations: true, + }); + const result = await migrateSharedAuthStore({ detected, stateDir, env }); + expect(result).toMatchObject({ + outcome: "skipped", + warningDisposition: "recoverable", + changes: [], + }); + expect(result.warnings.join("\n")).toContain("agent main"); + expect(result.warnings.join("\n")).toContain(sourcePath); + expect(result.warnings.join("\n")).toContain("openclaw doctor --fix"); + await expect( + noteStateIntegrity( + { agents: { ownership: "explicit", entries: { main: {} } }, plugins: { enabled: false } }, + { confirmRuntimeRepair: async () => false, note }, + ), + ).resolves.toBeUndefined(); + expect(fs.readFileSync(sourcePath)).toEqual(bytes); + }); + + it("keeps deleted credential files held when only their database is shared with an active owner", async () => { + const stateDir = fs.realpathSync.native(tempDirs.make("doctor-retained-credential-alias-")); + const env = { OPENCLAW_STATE_DIR: stateDir }; + vi.stubEnv("OPENCLAW_STATE_DIR", stateDir); + const cfg: OpenClawConfig = { + agents: { ownership: "explicit", entries: { main: {} } }, + plugins: { enabled: false }, + }; + const activePath = createLegacyDatabaseFixture({ + env, + eventsBySession: {}, + schemaVersion: OPENCLAW_AGENT_SCHEMA_VERSION, + }); + const retiredDir = path.join(stateDir, "agents", "retired", "agent"); + fs.mkdirSync(retiredDir, { recursive: true }); + const retiredPath = path.join(retiredDir, "openclaw-agent.sqlite"); + fs.linkSync(activePath, retiredPath); + beginAgentDeletionJournal( + { + agentId: "retired", + operationId: "delete-credential-alias", + agentDir: retiredDir, + workspaceDir: path.join(stateDir, "retired-workspace"), + sessionsDir: path.join(stateDir, "agents", "retired", "sessions"), + deleteFiles: false, + }, + { env }, + ); + runOpenClawStateWriteTransaction( + (database) => + completeAgentDeletionJournalInDatabase(database, "retired", "delete-credential-alias"), + { env }, + ); + const catalog = path.join(retiredDir, "models.json"); + const auth = path.join(retiredDir, "auth-profiles.json"); + fs.writeFileSync( + catalog, + JSON.stringify({ + providers: { + fixture: { + api: "openai-completions", + baseUrl: "https://example.invalid/v1", + apiKey: "synthetic-retained-key", + models: [], + }, + }, + }), + ); + fs.writeFileSync( + auth, + JSON.stringify({ + version: 1, + profiles: { + "fixture:default": { + type: "api_key", + provider: "fixture", + key: "synthetic-retained-key", + }, + }, + }), + ); + const bytes = [catalog, auth].map((file) => fs.readFileSync(file)); + expect( + listAuthProfileRepairCandidates(cfg, env).map((candidate) => candidate.authPath), + ).not.toContain(auth); + const imported = await maybeMigrateAuthProfileJsonStoresToSqlite({ + cfg, + env, + prompter: { confirmAutoFix: async () => true }, + }); + expect(imported.detected).not.toContain(auth); + const runtime = { + log() {}, + error() {}, + exit(code: number): never { + throw new Error(`unexpected exit ${code}`); + }, + }; + const result = await maybeMigrateModelCatalogCredentials({ + cfg, + env, + runtime, + prompter: createDoctorPrompter({ runtime, options: { repair: true, nonInteractive: true } }), + }); + expect(result).toMatchObject({ detected: 0, migrated: 0, warnings: [] }); + expect([catalog, auth].map((file) => fs.readFileSync(file))).toEqual(bytes); + }); + + it.each([ + { deleteFiles: false, registered: true, location: "default" }, + { deleteFiles: false, registered: false, location: "default" }, + { deleteFiles: false, registered: false, location: "custom" }, + { deleteFiles: false, registered: false, location: "old-name" }, + { deleteFiles: true, registered: true, location: "default" }, + ])( + "preserves deleted state (deleteFiles=$deleteFiles, registered=$registered, location=$location)", + async ({ deleteFiles, registered, location }) => { + const stateDir = fs.realpathSync.native(tempDirs.make("doctor-retained-deletion-")); + const env = { OPENCLAW_STATE_DIR: stateDir }; + vi.stubEnv("OPENCLAW_STATE_DIR", stateDir); + const cfg: OpenClawConfig = { + agents: { ownership: "explicit", entries: { main: {} } }, + plugins: { enabled: false }, + }; + const activePath = createLegacyDatabaseFixture({ + env, + eventsBySession: {}, + schemaVersion: 19, + }); + const retainedPath = createLegacyDatabaseFixture({ + agentId: "retired", + env, + eventsBySession: {}, + schemaVersion: 19, + ...(location !== "default" + ? { + path: path.join( + location === "custom" + ? tempDirs.make("doctor-retained-custom-") + : path.join(stateDir, "agents", "old-name"), + "agent", + "openclaw-agent.sqlite", + ), + } + : {}), + }); + const agentDir = path.dirname(retainedPath); + const workspaceDir = path.join(stateDir, "workspace-retired"); + beginAgentDeletionJournal( + { + agentId: "retired", + operationId: "delete-retired", + agentDir, + workspaceDir, + sessionsDir: path.join(stateDir, "agents", "retired", "sessions"), + deleteFiles, + }, + { env }, + ); + if (!deleteFiles) { + runOpenClawStateWriteTransaction( + (database) => { + completeAgentDeletionJournalInDatabase(database, "retired", "delete-retired"); + }, + { env }, + ); + } + if (!registered) { + unregisterOpenClawAgentDatabase({ agentId: "retired", path: retainedPath, env }); + } + const before = fs.readFileSync(retainedPath); + const candidatePath = + !deleteFiles && registered && location === "default" + ? path.join(stateDir, "retained-candidate.sqlite") + : retainedPath; + if (candidatePath !== retainedPath) { + fs.linkSync(retainedPath, candidatePath); + } + let prepared: PreparedAgentDatabaseMigrationDiscovery | undefined; + const preflight = await preflightOpenClawDatabaseSchemas({ + env, + configuredAgentDatabaseTargets: [], + configuredAgentDatabaseCandidatePaths: [candidatePath], + onAgentDatabaseDiscovery: (discovery) => { + prepared = discovery; + }, + }); + if (!deleteFiles) { + expect(preflight.pendingMigrations?.map((entry) => entry.path)).toEqual([activePath]); + expect(prepared?.discovery.warnings.join("\n")).toContain("retained-by-deletion"); + expect(prepared?.discovery.warnings.join("\n")).toContain(retainedPath); + } + expect(fs.readFileSync(retainedPath).equals(before)).toBe(true); + const execPath = path.join(stateDir, "exec-approvals.json"); + fs.writeFileSync(execPath, JSON.stringify({ version: 1, defaults: {}, agents: {} })); + const result = await autoMigrateLegacyState({ + cfg, + env, + agentDatabaseMigrationDiscovery: prepared, + doctorOnlyStateMigrations: true, + legacySessionSurfaces: EMPTY_LEGACY_SESSION_SURFACES, + log: { info() {}, warn() {} }, + }); + expect(readDatabaseSnapshot(activePath).version.user_version).toBe( + OPENCLAW_AGENT_SCHEMA_VERSION, + ); + const migration = result.stepReceipts.find((receipt) => receipt.id === "media-persistence"); + if (deleteFiles) { + expect(migration).toMatchObject({ outcome: "refused" }); + expect(migration?.warnings.join("\n")).toContain( + "unavailable while agent retired is deleted", + ); + expect(() => throwIfDoctorStateMigrationRefused(result.stepReceipts)).toThrow( + "Later repairs were not run", + ); + expect(fs.existsSync(execPath)).toBe(true); + } else { + expect(() => throwIfDoctorStateMigrationRefused(result.stepReceipts)).not.toThrow(); + expect(migration).toMatchObject({ outcome: "warning" }); + expect(migration?.warnings.join("\n")).toContain("Held agent retired"); + expect(migration?.warnings.join("\n")).toContain(retainedPath); + expect(migration?.warnings.join("\n")).toContain("openclaw doctor --fix"); + expect(result.warnings).toEqual(expect.arrayContaining(migration!.warnings)); + expect(fs.existsSync(execPath)).toBe(false); + if (registered && location === "default") { + await noteSessionTranscriptHealth({ + cfg, + env, + shouldRepair: true, + postSessionPluginMigration: result.postSessionPluginMigration, + postSessionPluginMigrationPlanBound: true, + }); + await noteSessionTranscriptHeaderHealth({ cfg, env, shouldRepair: true }); + await noteSessionTranscriptLabelHealth({ cfg, env, shouldRepair: true }); + const runtime = { + log() {}, + error() {}, + exit(code: number): never { + throw new Error(`unexpected exit ${code}`); + }, + }; + const catalogs = await maybeMigrateModelCatalogCredentials({ + cfg, + env, + runtime, + prompter: createDoctorPrompter({ + runtime, + options: { repair: true, nonInteractive: true }, + }), + }); + expect(catalogs.warnings).toEqual([]); + expect(detectTelegramGeneralTopicConversationRepairs({ cfg, env })).toEqual([]); + expect( + await maybeRepairCodexSessionRoutes({ cfg, env, shouldRepair: true }), + ).toMatchObject({ warnings: [] }); + expect(note).not.toHaveBeenCalledWith( + expect.stringContaining("retired"), + "Doctor warnings", + ); + } else { + expect(await repairCanonicalSessionKeys({ apply: true, cfg, env })).toMatchObject({ + scannedStores: 1, + }); + } + await expect( + assertOpenClawDatabasesReady({ + env, + operation: "doctor", + configuredAgentDatabaseTargets: [], + }), + ).resolves.toBeUndefined(); + await expect( + assertOpenClawDatabasesReady({ env, operation: "gateway-restart" }), + ).resolves.toBeUndefined(); + expect(fs.readFileSync(retainedPath).equals(before)).toBe(true); + } + expect(() => openOpenClawAgentDatabase({ agentId: "retired", env })).toThrow( + "unavailable while agent retired is deleted", + ); + }, + ); +}); diff --git a/src/infra/state-migrations.shared-auth-store.test.ts b/src/infra/state-migrations.shared-auth-store.test.ts index 7a24c37363da..5e9a334dd901 100644 --- a/src/infra/state-migrations.shared-auth-store.test.ts +++ b/src/infra/state-migrations.shared-auth-store.test.ts @@ -17,6 +17,7 @@ import { import { upsertAuthProfileWithLockOrThrow } from "../agents/auth-profiles/upsert-with-lock.js"; import { EMPTY_LEGACY_SESSION_SURFACES } from "../plugins/legacy-session-surfaces.types.js"; import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; +import { ensureAgentDeletionJournalSchema } from "../state/openclaw-state-db-schema-additive.js"; import * as stateDb from "../state/openclaw-state-db.js"; import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js"; import { @@ -124,6 +125,7 @@ describe("shared auth store relocation", () => { .prepare("INSERT INTO auth_profile_store VALUES ('primary', ?, 1)") .run(JSON.stringify(makeStore("openai:copied", "fixture-key"))); } else { + ensureAgentDeletionJournalSchema(seed); seed .prepare("INSERT INTO config_machine_state VALUES ('auth.sharedStore', ?, 1)") .run(JSON.stringify({ location })); @@ -192,6 +194,7 @@ describe("shared auth store relocation", () => { "preserves the live auth source's POSIX locks during copied inspection", async () => { const fixture = await createEmptyFixture(false); + stateDb.openOpenClawStateDatabase({ env: fixture.env }); fs.mkdirSync(path.dirname(fixture.sourcePath), { recursive: true }); const writer = new DatabaseSync(fixture.sourcePath); try { diff --git a/src/infra/state-migrations.shared-auth-store.ts b/src/infra/state-migrations.shared-auth-store.ts index 867346ee5d92..ea8a71bd7739 100644 --- a/src/infra/state-migrations.shared-auth-store.ts +++ b/src/infra/state-migrations.shared-auth-store.ts @@ -6,6 +6,7 @@ import type { DatabaseSync } from "node:sqlite"; import { isDeepStrictEqual } from "node:util"; import { safeParseJsonRecord } from "@openclaw/normalization-core/json-coercion"; import { isRecord } from "@openclaw/normalization-core/record-coerce"; +import { sanitizeForLog } from "../../packages/terminal-core/src/ansi.js"; import { inspectSharedAuthStoreOwnership, noteCommittedSharedAuthStoreOwnership, @@ -27,11 +28,13 @@ import { closeAuthProfileReadPool, resolveAuthProfileDatabaseOwnerId, } from "../agents/auth-profiles/sqlite.js"; +import { createRetainedAgentDatabaseMatcher } from "../state/agent-deletion-discovery.js"; import type { DB as OpenClawAgentKyselyDatabase } from "../state/openclaw-agent-db.generated.js"; import { closeOpenClawAgentDatabaseByPathAsync, runOpenClawAgentWriteTransaction, } from "../state/openclaw-agent-db.js"; +import { withArtifactPreservingStateReads } from "../state/openclaw-state-db-readonly.js"; import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; import { runOpenClawStateWriteTransaction } from "../state/openclaw-state-db.js"; import { @@ -483,6 +486,14 @@ export function detectSharedAuthStoreMigration(params: { if (params.doctorOnlyStateMigrations !== true) { return { sourcePath, hasLegacy: false }; } + if ( + fs.existsSync(sourcePath) && + withArtifactPreservingStateReads(() => + createRetainedAgentDatabaseMatcher(env, () => [])(sourcePath, "main"), + ) + ) { + return { sourcePath, hasLegacy: true, held: true }; + } const ownership = params.artifactPreservingReadOnly ? inspectSharedAuthStoreOwnership(env) : resolveSharedAuthStoreOwnership(env); @@ -506,6 +517,16 @@ export async function migrateSharedAuthStore(params: { if (!params.detected.hasLegacy) { return { changes: [], warnings: [] }; } + if (params.detected.held) { + return { + changes: [], + warnings: [ + `Shared auth migration skipped: store held for agent main at ${sanitizeForLog(params.detected.sourcePath)}; run openclaw doctor --fix after restoring the agent.`, + ], + outcome: "skipped", + warningDisposition: "recoverable", + }; + } return await withLegacyMigrationStateLock({ stateDir: params.stateDir, env: params.env, diff --git a/src/infra/state-migrations.shared-auth-store.types.ts b/src/infra/state-migrations.shared-auth-store.types.ts index 726eb8c6dd59..8912a2756d16 100644 --- a/src/infra/state-migrations.shared-auth-store.types.ts +++ b/src/infra/state-migrations.shared-auth-store.types.ts @@ -1,4 +1,5 @@ export type SharedAuthStoreMigrationDetection = { sourcePath: string; hasLegacy: boolean; + held?: boolean; }; diff --git a/src/infra/state-migrations.transcript-directives.test.ts b/src/infra/state-migrations.transcript-directives.test.ts index 1d9a3d5b5a65..740a51fda2c1 100644 --- a/src/infra/state-migrations.transcript-directives.test.ts +++ b/src/infra/state-migrations.transcript-directives.test.ts @@ -112,6 +112,8 @@ function insertSession( } function openLegacyAgentDatabase(stateDir: string, agentId = "main") { + // These active stores have known-empty deletion history before their legacy bytes exist. + openOpenClawStateDatabase({ env: { OPENCLAW_STATE_DIR: stateDir } }); const databasePath = path.join(stateDir, "agents", agentId, "agent", "openclaw-agent.sqlite"); fs.mkdirSync(path.dirname(databasePath), { recursive: true }); const database = openNodeSqliteDatabase(databasePath); diff --git a/src/secrets/storage-scan.ts b/src/secrets/storage-scan.ts index e18e96987a28..4195164af870 100644 --- a/src/secrets/storage-scan.ts +++ b/src/secrets/storage-scan.ts @@ -22,15 +22,6 @@ export function listSecretsDotEnvPaths(params: { configPath: string; stateDir: s return [...new Map(candidates.map((candidate) => [path.resolve(candidate), candidate])).values()]; } -function resolveActiveAgentDir(stateDir: string, env: NodeJS.ProcessEnv = process.env): string { - const override = env.OPENCLAW_AGENT_DIR?.trim() || env.PI_CODING_AGENT_DIR?.trim(); - if (override) { - return resolveUserPath(override, env); - } - // Storage scans must include the implicit main agent even before config has agent entries. - return path.join(resolveUserPath(stateDir), "agents", "main", "agent"); -} - /** * Lists deduplicated models.json paths that may contain materialized provider credentials. * Includes active env override, implicit main agent, discovered state dirs, and configured agents. @@ -40,10 +31,13 @@ export function listAgentModelsJsonPaths( stateDir: string, env: NodeJS.ProcessEnv = process.env, ): string[] { - const resolvedStateDir = resolveUserPath(stateDir); + const resolvedStateDir = resolveUserPath(stateDir, env); const paths = new Set(); paths.add(path.join(resolvedStateDir, "agents", "main", "agent", "models.json")); - paths.add(path.join(resolveActiveAgentDir(stateDir, env), "models.json")); + const override = env.OPENCLAW_AGENT_DIR?.trim() || env.PI_CODING_AGENT_DIR?.trim(); + if (override) { + paths.add(path.join(resolveUserPath(override, env), "models.json")); + } const agentsRoot = path.join(resolvedStateDir, "agents"); if (fs.existsSync(agentsRoot)) { @@ -56,12 +50,7 @@ export function listAgentModelsJsonPaths( } for (const agentId of listAgentIds(config)) { - if (agentId === "main") { - paths.add(path.join(resolvedStateDir, "agents", "main", "agent", "models.json")); - continue; - } - const agentDir = resolveAgentDir(config, agentId); - paths.add(path.join(resolveUserPath(agentDir), "models.json")); + paths.add(path.join(resolveAgentDir(config, agentId, env), "models.json")); } return [...paths]; diff --git a/src/state/agent-deletion-discovery.ts b/src/state/agent-deletion-discovery.ts new file mode 100644 index 000000000000..b96072187713 --- /dev/null +++ b/src/state/agent-deletion-discovery.ts @@ -0,0 +1,106 @@ +import fs from "node:fs"; +import { resolveStateDir } from "../config/paths.js"; +import { hasErrnoCode } from "../infra/errno.js"; +import { isPathInside } from "../infra/path-guards.js"; +import { resolveSqliteDatabaseFilePaths } from "../infra/sqlite-files.js"; +import { normalizeAgentId } from "../routing/session-key.js"; +import { + readAgentDatabaseDeletionSnapshot, + type AgentDeletionJournalDisposition, +} from "./agent-deletion-journal.read.js"; +import { + createOpenClawAgentDatabasePathMatcher, + isPersistentOpenClawAgentDatabasePath, +} from "./openclaw-agent-db-registry.js"; +import { resolveOpenClawStateSqlitePath } from "./openclaw-state-db.paths.js"; + +type Target = { agentId: string; path: string }; + +function hasSqliteArtifacts(directory: string): boolean { + try { + return fs + .readdirSync(directory) + .some((name) => /\.(?:sqlite3?|db)(?:-(?:wal|shm|journal))?$/iu.test(name)); + } catch (error) { + if (!hasErrnoCode(error, "ENOENT")) { + throw error; + } + // A dangling directory is unavailable, rather than an empty legacy layout. + return fs.lstatSync(directory, { throwIfNoEntry: false }) !== undefined; + } +} + +/** Recorded surviving owners can share retained files; directory-name inference cannot. */ +export function createAgentDatabaseDeletionClassifier(params: { + env: NodeJS.ProcessEnv; + retainedDeletions: AgentDeletionJournalDisposition; + configuredAgentDatabaseTargets: readonly Target[]; + registeredAgentDatabases: readonly Target[]; + artifactDirectories?: readonly Target[]; +}) { + const entries = params.retainedDeletions; + const samePath = createOpenClawAgentDatabasePathMatcher(); + const recorded = params.artifactDirectories ?? [ + ...params.configuredAgentDatabaseTargets, + ...params.registeredAgentDatabases, + ]; + return (pathname: string, agentId?: string) => { + if (entries === "unavailable") { + return entries; + } + const deletion = entries.find( + (entry) => + entry.agentId === agentId || + (params.artifactDirectories ? [entry.agentDir] : entry.databasePaths).some((file) => + samePath(file, pathname), + ), + ); + if (!deletion) { + return undefined; + } + const surviving = recorded.some( + (target) => + !entries.some((entry) => entry.agentId === normalizeAgentId(target.agentId)) && + samePath(target.path, pathname) && + (params.artifactDirectories !== undefined || + (isPersistentOpenClawAgentDatabasePath(target.path, params.env) && + (params.configuredAgentDatabaseTargets.includes(target) || + isPathInside( + fs.realpathSync.native(resolveStateDir(params.env)), + fs.realpathSync.native(target.path), + )))), + ); + return agentId === deletion.agentId || !surviving ? deletion : undefined; + }; +} + +export function createRetainedAgentDatabaseMatcher( + env: NodeJS.ProcessEnv, + readConfiguredTargets: () => readonly Target[], + namespace: + | "database" + | { kind: "agent-directory"; readDatabasePaths: () => readonly string[] } = "database", +) { + const snapshot = readAgentDatabaseDeletionSnapshot(env); + if (!snapshot && namespace !== "database") { + // Legacy credentials can predate SQLite; existing families still have unknown history. + const unavailable = + [resolveOpenClawStateSqlitePath(env), ...namespace.readDatabasePaths()] + .flatMap(resolveSqliteDatabaseFilePaths) + .some((file) => fs.lstatSync(file, { throwIfNoEntry: false }) !== undefined) || + readConfiguredTargets().some(({ path }) => hasSqliteArtifacts(path)); + return (directory: string, _agentId?: string) => unavailable || hasSqliteArtifacts(directory); + } + const retainedDeletions = snapshot?.retainedDeletions ?? "unavailable"; + if (retainedDeletions === "unavailable" || retainedDeletions.length === 0) { + return (_pathname: string, _agentId?: string) => retainedDeletions === "unavailable"; + } + const configured = readConfiguredTargets(); + return createAgentDatabaseDeletionClassifier({ + env, + retainedDeletions, + configuredAgentDatabaseTargets: configured, + artifactDirectories: namespace !== "database" ? configured : undefined, + registeredAgentDatabases: snapshot?.registeredAgentDatabases ?? [], + }); +} diff --git a/src/state/agent-deletion-journal.read.ts b/src/state/agent-deletion-journal.read.ts new file mode 100644 index 000000000000..b8c6dbeb866c --- /dev/null +++ b/src/state/agent-deletion-journal.read.ts @@ -0,0 +1,59 @@ +import path from "node:path"; +import type { DatabaseSync } from "node:sqlite"; +import { executeSqliteQuerySync, getNodeSqliteKysely } from "../infra/kysely-sync.js"; +import { runSqliteDeferredTransactionSync } from "../infra/sqlite-transaction.js"; +import { readRegisteredAgentDatabaseRows } from "./openclaw-agent-db-registry.read.js"; +import { withExistingOpenClawStateDatabaseReadOnly } from "./openclaw-state-db-readonly.js"; +import { tableExists } from "./openclaw-state-db-schema-helpers.js"; +import type { DB } from "./openclaw-state-db.generated.js"; + +type RetainedAgentDeletion = { agentId: string; agentDir: string; databasePaths: string[] }; +export type AgentDeletionJournalDisposition = readonly RetainedAgentDeletion[] | "unavailable"; + +export function parseAgentDeletionDatabasePaths(value: string): string[] { + const parsed: unknown = JSON.parse(value); + if ( + Array.isArray(parsed) && + parsed.every((entry): entry is string => typeof entry === "string") + ) { + return parsed; + } + throw new Error("Invalid agent deletion database path journal."); +} + +/** Read existing deletion history without initializing or repairing the journal. */ +export function readRetainedAgentDeletionsFromDatabase( + database: DatabaseSync, +): AgentDeletionJournalDisposition { + if (!tableExists(database, "agent_deletion_journal")) { + return "unavailable"; + } + return executeSqliteQuerySync( + database, + getNodeSqliteKysely>(database) + .selectFrom("agent_deletion_journal") + .select(["agent_id", "agent_dir", "database_paths_json"]) + .where("cleanup_completed", "=", 1) + .where("delete_files", "=", 0) + .orderBy("agent_id", "asc"), + ).rows.map((row) => ({ + agentId: row.agent_id, + agentDir: row.agent_dir, + databasePaths: [ + path.join(row.agent_dir, "openclaw-agent.sqlite"), + ...parseAgentDeletionDatabasePaths(row.database_paths_json), + ], + })); +} + +/** Read journal and registered-owner facts from one shared-state generation. */ +export function readAgentDatabaseDeletionSnapshot(env: NodeJS.ProcessEnv) { + return withExistingOpenClawStateDatabaseReadOnly( + ({ db, path: statePath }) => + runSqliteDeferredTransactionSync(db, () => ({ + retainedDeletions: readRetainedAgentDeletionsFromDatabase(db), + registeredAgentDatabases: readRegisteredAgentDatabaseRows(db, statePath, false), + })), + { env }, + ); +} diff --git a/src/state/agent-deletion-journal.ts b/src/state/agent-deletion-journal.ts index 2e20292bfe2e..8e8df3480253 100644 --- a/src/state/agent-deletion-journal.ts +++ b/src/state/agent-deletion-journal.ts @@ -10,6 +10,7 @@ import { isPathInside } from "../infra/path-guards.js"; import { resolveSqliteDatabaseFilePaths } from "../infra/sqlite-files.js"; import { normalizeAgentId } from "../routing/session-key.js"; import { getAgentDeletionDatabaseCleanup } from "./agent-deletion-cleanup.js"; +import { parseAgentDeletionDatabasePaths } from "./agent-deletion-journal.read.js"; import { deleteAgentProvenanceForAgent, ensureAgentProvenanceSchema } from "./agent-provenance.js"; import type { OpenClawStateDatabase, @@ -132,10 +133,12 @@ export function prepareAgentDeletionPathFence( canonicalPaths: [row.agent_dir, row.workspace_dir, row.sessions_dir].map((entryPath) => normalizeAgentDirRegistryPath(entryPath, env), ), - databasePaths: parseDatabasePaths(row.database_paths_json).map((databasePath) => ({ - path: databasePath, - canonicalPath: normalizeAgentDirRegistryPath(databasePath, env), - })), + databasePaths: parseAgentDeletionDatabasePaths(row.database_paths_json).map( + (databasePath) => ({ + path: databasePath, + canonicalPath: normalizeAgentDirRegistryPath(databasePath, env), + }), + ), cleanupPaths: parseCleanupPaths(row.cleanup_paths_json).map((cleanupPath) => Object.assign({}, cleanupPath, { fencePath: normalizeAgentDirRegistryPath(cleanupPath.canonicalPath, env), @@ -278,7 +281,7 @@ function fromRow(row: { agentDir: row.agent_dir, workspaceDir: row.workspace_dir, sessionsDir: row.sessions_dir, - databasePaths: parseDatabasePaths(row.database_paths_json), + databasePaths: parseAgentDeletionDatabasePaths(row.database_paths_json), cleanupPaths: parseCleanupPaths(row.cleanup_paths_json), createdAt: row.created_at, cleanupCompleted: row.cleanup_completed === 1, @@ -286,17 +289,6 @@ function fromRow(row: { }; } -function parseDatabasePaths(value: string): string[] { - const parsed: unknown = JSON.parse(value); - if ( - !Array.isArray(parsed) || - !parsed.every((entry): entry is string => typeof entry === "string") - ) { - throw new Error("Invalid agent deletion database path journal."); - } - return parsed; -} - function parseCleanupPaths(value: string): AgentDeletionJournalCleanupPath[] { const parsed: unknown = JSON.parse(value); if ( diff --git a/src/state/openclaw-database-preflight-agent-scheduler.ts b/src/state/openclaw-database-preflight-agent-scheduler.ts index 40357171ee34..173aaf8b80d7 100644 --- a/src/state/openclaw-database-preflight-agent-scheduler.ts +++ b/src/state/openclaw-database-preflight-agent-scheduler.ts @@ -8,18 +8,15 @@ import { import { createDeferredCore } from "../shared/deferred.js"; import type { AgentDatabaseAdmissionRefusal } from "./agent-database-admission.js"; import { createAgentSchemaInspectionWorker } from "./openclaw-agent-schema-inspection-worker.js"; -import type { OpenClawDatabaseSchemaPreflight } from "./openclaw-database-preflight.types.js"; +import type { + AgentDatabasePreflightStats, + OpenClawDatabaseSchemaPreflight, +} from "./openclaw-database-preflight.types.js"; // Snapshot preparation can be disk-heavy; overlap one additional agent // without fanning out across every registered database. export const AGENT_DATABASE_PREFLIGHT_CONCURRENCY = 2; -export type AgentDatabasePreflightStats = { - schemaProcessCount: number; - schemaInspectionCount: number; - schemaSnapshotCount: number; -}; - export async function preflightAgentDatabasesBounded( targets: readonly T[], inspect: ( diff --git a/src/state/openclaw-database-preflight.artifacts.test.ts b/src/state/openclaw-database-preflight.artifacts.test.ts index 539f574ccd72..062d24b534a6 100644 --- a/src/state/openclaw-database-preflight.artifacts.test.ts +++ b/src/state/openclaw-database-preflight.artifacts.test.ts @@ -124,6 +124,9 @@ function sourceArtifacts(paths: string[], allowReadMarks: string[] = []): unknow describe("schema preflight source artifacts", () => { it("retains the source-reader lock tolerance beyond the runtime busy timeout", async () => { const root = tempDirs.make("openclaw-header-lock-tolerance-"); + const env = { OPENCLAW_STATE_DIR: path.join(root, "active-state") }; + openOpenClawStateDatabase({ env }); + closeOpenClawStateDatabaseForTest(); const pathname = path.join(root, "agent.sqlite"); const writer = new (requireNodeSqlite().DatabaseSync)(pathname); writer.exec(` @@ -140,7 +143,7 @@ describe("schema preflight source artifacts", () => { }, 8_000); try { const result = await preflightOpenClawDatabaseSchemas({ - env: { OPENCLAW_STATE_DIR: path.join(root, "absent-state") }, + env, supportedVersions, configuredAgentDatabaseCandidatePaths: [pathname], }); @@ -164,6 +167,9 @@ describe("schema preflight source artifacts", () => { "reads fresh WAL metadata without copying $payloadBytes bytes of unrelated payload (admission=$admission)", async ({ payloadBytes, admission }) => { const root = tempDirs.make("openclaw-header-preflight-"); + const env = { OPENCLAW_STATE_DIR: path.join(root, "active-state") }; + openOpenClawStateDatabase({ env }); + closeOpenClawStateDatabaseForTest(); const pathname = path.join(root, "agents", "main", "agent", "openclaw-agent.sqlite"); fs.mkdirSync(path.dirname(pathname), { recursive: true }); const preload = path.join(root, "no-backup.cjs"); @@ -195,7 +201,7 @@ describe("schema preflight source artifacts", () => { writer.exec("COMMIT;"); const before = sourceArtifacts([pathname], [pathname]); const result = await preflightOpenClawDatabaseSchemas({ - env: { OPENCLAW_STATE_DIR: path.join(root, "absent-state") }, + env, supportedVersions, configuredAgentDatabaseCandidatePaths: [pathname], ...(admission ? { agentAdmissionConfig: { agents: { entries: { main: {} } } } } : {}), @@ -328,6 +334,8 @@ describe("schema preflight source artifacts", () => { expect(fs.realpathSync.native(locator)).toBe(fs.realpathSync.native(fixture.worker.path)); expect(fs.realpathSync(locator)).toBe(fs.realpathSync.native(lexicalPath)); const callerEnv = { OPENCLAW_STATE_DIR: tempDirs.make("openclaw-preflight-caller-") }; + const callerStatePath = openOpenClawStateDatabase({ env: callerEnv }).path; + closeOpenClawStateDatabaseForTest(); const contexts = [ { env: fixture.env, config: {} }, { env: callerEnv, config: { session: { store: lexicalPath } } }, @@ -335,6 +343,7 @@ describe("schema preflight source artifacts", () => { ]; const paths = [ ...fixture.paths, + callerStatePath, lexicalPath, path.join(callerEnv.OPENCLAW_STATE_DIR, "absent.sqlite"), ]; diff --git a/src/state/openclaw-database-preflight.lifecycle.test.ts b/src/state/openclaw-database-preflight.lifecycle.test.ts index 5ff87263d60d..f8b358caf92f 100644 --- a/src/state/openclaw-database-preflight.lifecycle.test.ts +++ b/src/state/openclaw-database-preflight.lifecycle.test.ts @@ -18,7 +18,10 @@ import { } from "./openclaw-agent-db.js"; import { preflightOpenClawDatabaseSchemas } from "./openclaw-database-preflight.js"; import { OPENCLAW_STATE_SCHEMA_VERSION } from "./openclaw-state-db-contract.js"; -import { closeOpenClawStateDatabaseForTest } from "./openclaw-state-db.js"; +import { + closeOpenClawStateDatabaseForTest, + openOpenClawStateDatabase, +} from "./openclaw-state-db.js"; vi.mock("node:child_process", async (importOriginal) => { const actual = await importOriginal(); @@ -65,6 +68,14 @@ function expectReadLeaseHeld(databasePath: string) { } } +function createPreflightState(stateDir: string) { + const env = { OPENCLAW_STATE_DIR: stateDir }; + // Reader lifecycle fixtures need known empty deletion history; missing history holds stores. + const statePath = fs.realpathSync.native(openOpenClawStateDatabase({ env }).path); + closeOpenClawStateDatabaseForTest(); + return { env, statePath }; +} + it.each([ ...(["success", "failure", "cancel"] as const).map((outcome) => ({ source: "direct", @@ -98,6 +109,7 @@ it.each([ ] as const; closeOpenClawAgentDatabasesForTest(); closeOpenClawStateDatabaseForTest(); + const { env, statePath } = createPreflightState(path.join(root, "active-state")); const { DatabaseSync } = requireNodeSqlite(); for (const [index, pathname] of paths.entries()) { const database = new DatabaseSync(pathname); @@ -170,6 +182,9 @@ it.each([ .spyOn(snapshots, "prepareSqliteReadOnlyLocation") .mockImplementation(async (pathname, options) => { const prepared = await prepareLocation(pathname, options); + if (pathname === statePath) { + return prepared; + } const index = sources.indexOf(path.toNamespacedPath(pathname)); expect(index).toBeGreaterThanOrEqual(0); locations[index] = path.toNamespacedPath(fs.realpathSync.native(prepared.location)); @@ -199,7 +214,7 @@ it.each([ let settled = false; const inspect = () => preflightOpenClawDatabaseSchemas({ - env: { OPENCLAW_STATE_DIR: path.join(root, "absent-state") }, + env, supportedVersions, configuredAgentDatabaseCandidatePaths: paths, verifyCurrentSchemaShape: true, @@ -234,7 +249,11 @@ it.each([ { timeout: 10_000 }, ); expect(settled).toBe(false); - expect(prepare).toHaveBeenCalledTimes(source === "snapshot" ? 2 : 0); + expect( + prepare.mock.calls.filter(([pathname]) => + sources.includes(path.toNamespacedPath(pathname)), + ), + ).toHaveLength(source === "snapshot" ? 2 : 0); expect(cleanedSnapshots.size).toBe(0); // Snapshot-copy workers use spawn; this counts the two schema-reader children. expect(fork).toHaveBeenCalledTimes(2); @@ -340,9 +359,7 @@ it.each([ ); function createSnapshotCandidates() { - const env = { - OPENCLAW_STATE_DIR: tempDirs.make("openclaw-preflight-lifecycle-state-"), - }; + const { env } = createPreflightState(tempDirs.make("openclaw-preflight-lifecycle-state-")); const directory = tempDirs.make("openclaw-preflight-lifecycle-agents-"); const { DatabaseSync } = requireNodeSqlite(); const paths = [0, 1, 2].map((index) => path.join(directory, `agent-${index}.sqlite`)); @@ -371,6 +388,7 @@ it.each(["header", "shape", "startup"])( const paths = targets.map((target) => target.path); closeOpenClawAgentDatabasesForTest(); closeOpenClawStateDatabaseForTest(); + const { env, statePath } = createPreflightState(path.join(root, "active-state")); const originalBytes = paths.map((pathname) => fs.readFileSync(pathname)); for (const pathname of paths) { expect(fs.existsSync(`${pathname}-wal`)).toBe(false); @@ -382,7 +400,9 @@ it.each(["header", "shape", "startup"])( vi.spyOn(snapshots, "prepareSqliteReadOnlyLocation").mockImplementation( async (pathname, options) => { const prepared = await prepare(pathname, options); - locations.push(prepared.location); + if (paths.includes(pathname)) { + locations.push(prepared.location); + } return prepared; }, ); @@ -393,7 +413,7 @@ it.each(["header", "shape", "startup"])( await expect( preflightOpenClawDatabaseSchemas({ - env: { OPENCLAW_STATE_DIR: path.join(root, "absent-state") }, + env, supportedVersions, configuredAgentDatabaseTargets: targets, verifyCurrentSchemaShape: mode !== "header", @@ -410,7 +430,8 @@ it.each(["header", "shape", "startup"])( Array.isArray(args) && ["schema-header", "sync", "async"].some((readerMode) => args.includes(readerMode)), ); - expect(oneShotReaders).toHaveLength(0); + expect(oneShotReaders).toHaveLength(1); + expect(oneShotReaders[0]?.[1]).toContain(statePath); expect(spawn).toHaveBeenCalledTimes(2); expect(onAgentInspection).toHaveBeenCalledExactlyOnceWith({ schemaProcessCount: 2, diff --git a/src/state/openclaw-database-preflight.test.ts b/src/state/openclaw-database-preflight.test.ts index 95a9d907914c..05ba79296851 100644 --- a/src/state/openclaw-database-preflight.test.ts +++ b/src/state/openclaw-database-preflight.test.ts @@ -1,7 +1,7 @@ import fs from "node:fs"; import path from "node:path"; import { gunzipSync } from "node:zlib"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import packageJson from "../../package.json" with { type: "json" }; import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; import { requireNodeSqlite } from "../infra/node-sqlite.js"; @@ -112,6 +112,7 @@ describe("OpenClaw database schema preflight", () => { const agentDir = path.join(root, "external", "agents", "alpha"); const agentPath = path.join(agentDir, "agent", "openclaw-agent.sqlite"); const store = path.join(agentDir, "sessions", "sessions.json"); + openOpenClawStateDatabase({ env }); openOpenClawAgentDatabase({ agentId: "beta", path: agentPath, @@ -343,7 +344,7 @@ describe("OpenClaw database schema preflight", () => { }); it.each(["default", "configured"])( - "checks an unregistered %s store without creating shared state", + "holds an unregistered %s store without deletion history or creating shared state", async (layout) => { const stateDir = tempDirs.make("openclaw-unregistered-readiness-"); const env = { OPENCLAW_STATE_DIR: stateDir }; @@ -368,6 +369,7 @@ describe("OpenClaw database schema preflight", () => { operation: "doctor" as const, configuredAgentDatabaseTargets: layout === "configured" ? [{ agentId: "main", path: agent.path }] : [], + onAgentInspection: vi.fn(), }; const before = snapshotSourceFamily(agent.path); await expect(assertOpenClawDatabasesReady(options)).resolves.toBeUndefined(); @@ -379,8 +381,30 @@ describe("OpenClaw database schema preflight", () => { ); legacyWriter.close(); const legacy = snapshotSourceFamily(agent.path); - await expect(assertOpenClawDatabasesReady(options)).rejects.toThrow( - /Doctor.*database readiness.*schema version 17/s, + await expect(assertOpenClawDatabasesReady(options)).resolves.toBeUndefined(); + expect(options.onAgentInspection).toHaveBeenCalledTimes(2); + expect(options.onAgentInspection.mock.calls).toEqual([ + [{ schemaProcessCount: 0, schemaInspectionCount: 0, schemaSnapshotCount: 0 }], + [{ schemaProcessCount: 0, schemaInspectionCount: 0, schemaSnapshotCount: 0 }], + ]); + const onAgentDatabaseDiscovery = vi.fn(); + await expect( + preflightOpenClawDatabaseSchemas({ ...options, onAgentDatabaseDiscovery }), + ).resolves.toEqual({ incompatible: [], indeterminate: [] }); + expect(onAgentDatabaseDiscovery).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ + discovery: expect.objectContaining({ + targets: [], + retainedDeletions: "unavailable", + registryRemovals: [], + failures: [], + warnings: [ + expect.stringContaining( + `Held agent main database ${agent.path} (deletion journal unavailable); run openclaw doctor --fix`, + ), + ], + }), + }), ); expect(snapshotSourceFamily(agent.path)).toEqual(legacy); expect(fs.existsSync(statePath)).toBe(false); @@ -691,6 +715,9 @@ describe("OpenClaw database schema preflight", () => { "keeps partial configured-store inventory when one candidate lookup is denied", async () => { const stateDir = tempDirs.make("openclaw-configured-candidate-lookup-"); + const env = { OPENCLAW_STATE_DIR: stateDir }; + openOpenClawStateDatabase({ env }); + closeOpenClawStateDatabaseForTest(); const visibleDir = tempDirs.make("openclaw-configured-visible-"); const deniedDir = tempDirs.make("openclaw-configured-denied-"); const visiblePath = path.join(visibleDir, "newer.sqlite"); @@ -709,7 +736,7 @@ describe("OpenClaw database schema preflight", () => { let result: Awaited>; try { result = await preflightOpenClawDatabaseSchemas({ - env: { OPENCLAW_STATE_DIR: stateDir }, + env, supportedVersions: { state: OPENCLAW_STATE_SCHEMA_VERSION, agent: OPENCLAW_AGENT_SCHEMA_VERSION, diff --git a/src/state/openclaw-database-preflight.ts b/src/state/openclaw-database-preflight.ts index 1a7fdd92310c..c9095b2eee88 100644 --- a/src/state/openclaw-database-preflight.ts +++ b/src/state/openclaw-database-preflight.ts @@ -1,5 +1,5 @@ import { existsSync, realpathSync, statSync } from "node:fs"; -import path from "node:path"; +import nodePath from "node:path"; import type { DatabaseSync } from "node:sqlite"; import { listAgentIds } from "../agents/agent-scope-config.js"; import { resolveStateDir } from "../config/paths.js"; @@ -20,10 +20,7 @@ import { readSqliteWriterAppVersion as readWriterAppVersion } from "../infra/sql import { prepareSqliteReadOnlyLocation } from "../infra/sqlite-snapshot-source.js"; import { readSqliteUserVersion } from "../infra/sqlite-user-version.js"; import { hasStateDatabaseSourceExclusion } from "../infra/state-database-coordinator.js"; -import { - discoverAgentDatabaseMigrationTargets, - type PreparedAgentDatabaseMigrationDiscovery, -} from "../infra/state-migrations.media-persistence-targets.js"; +import { discoverAgentDatabaseMigrationTargets } from "../infra/state-migrations.media-persistence-targets.js"; import { AgentDatabaseAdmissionError, canIsolateAgentDatabase, @@ -31,31 +28,33 @@ import { recordAgentDatabaseAdmissions, } from "./agent-database-admission.js"; import { getAgentDatabaseStartupAdmission } from "./agent-database-startup.js"; +import { createAgentDatabaseDeletionClassifier } from "./agent-deletion-discovery.js"; +import { + readRetainedAgentDeletionsFromDatabase, + type AgentDeletionJournalDisposition, +} from "./agent-deletion-journal.read.js"; import { OPENCLAW_AGENT_SCHEMA_VERSION } from "./openclaw-agent-db-contract.js"; import { isPersistentOpenClawAgentDatabasePath } from "./openclaw-agent-db-registry.js"; import { readAgentDatabasePreflightTargets } from "./openclaw-agent-db-registry.read.js"; import type { AgentSchemaInspection } from "./openclaw-agent-schema-inspection.js"; -import { - preflightAgentDatabasesBounded, - type AgentDatabasePreflightStats, -} from "./openclaw-database-preflight-agent-scheduler.js"; +import { preflightAgentDatabasesBounded } from "./openclaw-database-preflight-agent-scheduler.js"; import { describeDeferredStateSchemaPublication, formatIndeterminateDatabaseReadiness, OpenClawDatabaseSchemaPreflightError, } from "./openclaw-database-preflight.messages.js"; import type { + AgentDatabasePreflightStats, DeferredStateSchemaPublication, OpenClawDatabaseSchemaPreflight, + OpenClawDatabasePreflightOptions, OpenClawStateSchemaPreflightResult, } from "./openclaw-database-preflight.types.js"; import { requestOpenClawAgentDatabaseQuickCheck } from "./openclaw-database-verify.js"; -import type { OpenClawSchemaVersions } from "./openclaw-schema-versions.js"; import { OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, OPENCLAW_STATE_SCHEMA_VERSION, } from "./openclaw-state-db-contract.js"; -import type { OpenClawStateSchemaReadAdmission } from "./openclaw-state-db-contract.js"; import { assertNoLegacyStateRuntimeRepair } from "./openclaw-state-db-fast-path.js"; import { assertOpenClawStateDatabaseOwner, @@ -215,7 +214,7 @@ function inspectCurrentStateStartupSchema( export async function preflightOpenClawStateDatabasePath( databasePath: string, ): Promise { - const resolvedPath = path.resolve(databasePath); + const resolvedPath = nodePath.resolve(databasePath); const base = { schema: "openclaw.state-schema-preflight.v1", databasePath: resolvedPath, @@ -308,27 +307,9 @@ export async function preflightOpenClawStateDatabasePath( } /** Read schema headers and optionally verify current schema shape without repairing it. */ -export async function preflightOpenClawDatabaseSchemas(options: { - env: NodeJS.ProcessEnv; - onAgentDatabaseDiscovery?: (prepared: PreparedAgentDatabaseMigrationDiscovery) => void; - onAgentInspection?: (stats: AgentDatabasePreflightStats) => void; - scope?: "state"; - signal?: AbortSignal; - /** Omit for current-runtime checks; updates pass their complete target pair. */ - supportedVersions?: OpenClawSchemaVersions; - verifyCurrentSchemaShape?: boolean; - requireStartupMigrationReadiness?: boolean; - /** Consume this startup owner's unchanged compatibility headers once, never readiness proof. */ - reuseStartupSchemaPreparation?: boolean; - configuredAgentDatabaseTargets?: - | readonly { agentId: string; path: string }[] - | (( - registeredDatabases: readonly { agentId: string; path: string }[], - ) => readonly { agentId: string; path: string }[]); - configuredAgentDatabaseCandidatePaths?: readonly string[]; - agentAdmissionConfig?: OpenClawConfig; - openStateSchemaReadAdmission?: OpenClawStateSchemaReadAdmission; -}): Promise { +export async function preflightOpenClawDatabaseSchemas( + options: OpenClawDatabasePreflightOptions, +): Promise { options.signal?.throwIfAborted(); const { supportedVersions = { @@ -349,8 +330,9 @@ export async function preflightOpenClawDatabaseSchemas(options: { ? getAgentDatabaseStartupAdmission()?.takePreparedSchemaHeaders(options.env) : undefined; const priorRefusals = startup?.captureRefusals(options.env); - const statePath = path.resolve(resolveOpenClawStateSqlitePath(options.env)); + const statePath = nodePath.resolve(resolveOpenClawStateSqlitePath(options.env)); let registeredDatabases: ReturnType = []; + let retainedDeletions: AgentDeletionJournalDisposition = "unavailable"; let stateDatabase: DatabaseSync | undefined; let closeStateSchemaReadAdmission: (() => void) | undefined; let stateSnapshot: Awaited> | undefined; @@ -464,6 +446,7 @@ export async function preflightOpenClawDatabaseSchemas(options: { } try { registeredDatabases = readAgentDatabasePreflightTargets(stateDatabase, statePath); + retainedDeletions = readRetainedAgentDeletionsFromDatabase(stateDatabase); } catch (error) { result.indeterminate.push({ kind: "state", @@ -504,10 +487,11 @@ export async function preflightOpenClawDatabaseSchemas(options: { return result; } let agentTargets = registeredDatabases; + let configuredTargets: readonly { agentId: string; path: string }[] = []; if (options.configuredAgentDatabaseTargets !== undefined) { // Doctor must resolve configured paths from these read-only facts: the // runtime registry reader rejects the very legacy schema Doctor repairs. - const configuredTargets = + configuredTargets = typeof options.configuredAgentDatabaseTargets === "function" ? options.configuredAgentDatabaseTargets(registeredDatabases) : options.configuredAgentDatabaseTargets; @@ -515,6 +499,7 @@ export async function preflightOpenClawDatabaseSchemas(options: { env: options.env, configuredAgentDatabaseTargets: configuredTargets, registeredAgentDatabases: registeredDatabases, + retainedDeletions, }); options.onAgentDatabaseDiscovery?.({ stateDir: resolveStateDir(options.env), @@ -530,6 +515,8 @@ export async function preflightOpenClawDatabaseSchemas(options: { // An occupied custom-store candidate can have a newer, unreadable owner. // Check its version without promoting it into an owned migration target. const candidates: Array<{ agentId?: string; path: string }> = [ + // Migration deduplication must not discard configured ownership claims. + ...configuredTargets, ...agentTargets, // Migration discovery intentionally declines ownership of foreign registry // paths. Preflight remains read-only, so preserve their downgrade guard. @@ -546,12 +533,15 @@ export async function preflightOpenClawDatabaseSchemas(options: { path: candidatePath, })), ]; + const classify = createAgentDatabaseDeletionClassifier({ + env: options.env, + retainedDeletions, + configuredAgentDatabaseTargets: configuredTargets, + registeredAgentDatabases: registeredDatabases, + }); const inspectionTargets = candidates - .map((row) => ({ - agentId: row.agentId, - path: row.path, - presence: inspectCandidatePresence(row.path), - })) + .filter((row) => !classify(row.path, row.agentId)) + .map(({ agentId, path }) => ({ agentId, path, presence: inspectCandidatePresence(path) })) .filter((row) => row.presence.status !== "absent"); const admittedAgentIds = options.agentAdmissionConfig ? new Set(listAgentIds(options.agentAdmissionConfig)) diff --git a/src/state/openclaw-database-preflight.types.ts b/src/state/openclaw-database-preflight.types.ts index ea63218c67ef..a59b37a0bfcc 100644 --- a/src/state/openclaw-database-preflight.types.ts +++ b/src/state/openclaw-database-preflight.types.ts @@ -1,7 +1,17 @@ +import type { OpenClawConfig } from "../config/types.openclaw.js"; import type { SqliteSchemaIssue } from "../infra/sqlite-schema-contract.js"; +import type { PreparedAgentDatabaseMigrationDiscovery } from "../infra/state-migrations.media-persistence-targets.js"; import type { AgentDatabaseAdmissionRefusal } from "./agent-database-admission.js"; +import type { OpenClawSchemaVersions } from "./openclaw-schema-versions.js"; +import type { OpenClawStateSchemaReadAdmission } from "./openclaw-state-db-contract.js"; import type { OpenClawExternalStateOwnership } from "./openclaw-state-ownership.js"; +export type AgentDatabasePreflightStats = { + schemaProcessCount: number; + schemaInspectionCount: number; + schemaSnapshotCount: number; +}; + export type IncompatibleOpenClawDatabase = { kind: "agent" | "state"; path: string; @@ -63,3 +73,25 @@ export type OpenClawDatabaseSchemaPreflightOperation = | "doctor" | "gateway-restart" | "gateway-startup"; + +export type OpenClawDatabasePreflightOptions = { + env: NodeJS.ProcessEnv; + onAgentDatabaseDiscovery?: (prepared: PreparedAgentDatabaseMigrationDiscovery) => void; + onAgentInspection?: (stats: AgentDatabasePreflightStats) => void; + scope?: "state"; + signal?: AbortSignal; + /** Omit for current-runtime checks; updates pass their complete target pair. */ + supportedVersions?: OpenClawSchemaVersions; + verifyCurrentSchemaShape?: boolean; + requireStartupMigrationReadiness?: boolean; + /** Consume this startup owner's unchanged compatibility headers once, never readiness proof. */ + reuseStartupSchemaPreparation?: boolean; + configuredAgentDatabaseTargets?: + | readonly { agentId: string; path: string }[] + | (( + registeredDatabases: readonly { agentId: string; path: string }[], + ) => readonly { agentId: string; path: string }[]); + configuredAgentDatabaseCandidatePaths?: readonly string[]; + agentAdmissionConfig?: OpenClawConfig; + openStateSchemaReadAdmission?: OpenClawStateSchemaReadAdmission; +}; diff --git a/src/worker/worker-fault-injection.test-support.ts b/src/worker/worker-fault-injection.test-support.ts index da61a93ae3c1..40093a68f5d2 100644 --- a/src/worker/worker-fault-injection.test-support.ts +++ b/src/worker/worker-fault-injection.test-support.ts @@ -48,6 +48,7 @@ import { seedFaultAttachedEnvironment, WorkerFaultPlacementLifecycle, } from "./worker-fault-placement-lifecycle.test-support.js"; +import { bindWorkerFixtureSessionTarget } from "./worker-fault-session-target.test-support.js"; import * as workerRpc from "./worker-rpc-clients.js"; export const SESSION_ID = "fault-session"; @@ -188,6 +189,7 @@ export class ComposedGatewayHarness { private placementGateValue: WorkerSessionPlacementGate | undefined; private useReplacementExecutor = false; private unsubscribeLive: (() => void) | undefined; + private readonly restoreSessionTarget: () => void; static async create(root: string): Promise { const sessionsDir = path.join(root, "agents", "main", "sessions"); @@ -223,6 +225,7 @@ export class ComposedGatewayHarness { readonly database: stateDb.OpenClawStateDatabase, readonly store: envStore.WorkerEnvironmentStore, ) { + const env = { OPENCLAW_STATE_DIR: path.join(root, "state") }; this.socketPath = path.join(root, "gateway.sock"); this.cfg = { agents: { list: [{ id: "main", default: true }] }, @@ -261,6 +264,7 @@ export class ComposedGatewayHarness { this.liveDeltas.push(event.data.delta); } }); + this.restoreSessionTarget = bindWorkerFixtureSessionTarget(this.cfg, env); } get epoch(): number { @@ -439,6 +443,7 @@ export class ComposedGatewayHarness { } async close(): Promise { + using _ = { [Symbol.dispose]: this.restoreSessionTarget }; this.transcriptGate?.release.resolve(); for (const gate of this.liveEventGates) { gate.release.resolve(); diff --git a/src/worker/worker-fault-session-target.test-support.ts b/src/worker/worker-fault-session-target.test-support.ts new file mode 100644 index 000000000000..e19f9f460edb --- /dev/null +++ b/src/worker/worker-fault-session-target.test-support.ts @@ -0,0 +1,18 @@ +import { vi } from "vitest"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import * as workerSessionTarget from "../gateway/worker-environments/session-target.js"; +import { withEnv } from "../test-utils/env.js"; + +const resolveGatewaySessionTarget = workerSessionTarget.resolveWorkerSessionTarget; + +/** Gateway and worker share a process only in this fixture; bind its real Gateway reads. */ +export function bindWorkerFixtureSessionTarget(cfg: OpenClawConfig, env: NodeJS.ProcessEnv) { + const resolver = vi + .spyOn(workerSessionTarget, "resolveWorkerSessionTarget") + .mockImplementation((current, id) => + current === cfg + ? withEnv(env, () => resolveGatewaySessionTarget(current, id)) + : resolveGatewaySessionTarget(current, id), + ); + return () => resolver.mockRestore(); +} diff --git a/src/worker/worker.fault-injection.test.ts b/src/worker/worker.fault-injection.test.ts index 87c4c6840c6e..a53ed134a155 100644 --- a/src/worker/worker.fault-injection.test.ts +++ b/src/worker/worker.fault-injection.test.ts @@ -76,6 +76,21 @@ async function stopClients(clients: WorkerClients | undefined): Promise { await clients.connection.stop(); } +function waitForWorkerEvent( + event: Promise, + command: Promise, + phase: string, + resultOutput?: Promise, +): Promise { + const completed = resultOutput ? Promise.race([command, resultOutput]) : command; + return Promise.race([ + event, + completed.then(() => { + throw new Error(`Worker completed before ${phase}`); + }), + ]); +} + describe("cloud worker milestone 2 fault injection", () => { let harness: ComposedGatewayHarness; const clients: WorkerClients[] = []; @@ -135,9 +150,13 @@ describe("cloud worker milestone 2 fault injection", () => { loggingState.rawConsole = { log: vi.fn(), info: vi.fn(), warn, error: vi.fn() }; const input = new PassThrough(); const output = new PassThrough(); + const resultOutput = createDeferred(); let stdout = ""; output.on("data", (chunk: Buffer) => { stdout += chunk.toString("utf8"); + if (stdout.includes("\n")) { + resultOutput.resolve(); + } }); const lifetime = { signal: controller.signal, @@ -159,7 +178,12 @@ describe("cloud worker milestone 2 fault injection", () => { let protectedDirectory: string | undefined; let turnDirectory: string | undefined; try { - await providerStarted.promise; + await waitForWorkerEvent( + providerStarted.promise, + command, + "provider start", + resultOutput.promise, + ); environmentStateDir = process.env.OPENCLAW_STATE_DIR; expect(environmentStateDir).toBeDefined(); expect(environmentStateDir).not.toBe(previousStateDir); @@ -191,7 +215,12 @@ describe("cloud worker milestone 2 fault injection", () => { : doneOutcome("paid reply"), ); } - await finishingGate.entered.promise; + await waitForWorkerEvent( + finishingGate.entered.promise, + command, + "finishing event", + resultOutput.promise, + ); if (outcome === "cancellation") { expect(harness.requestParams("worker.inference.cancel")).toHaveLength(1); } @@ -338,39 +367,54 @@ describe("cloud worker milestone 2 fault injection", () => { text: "preview reply", }; let settled = false; - const result = runWorkerDescriptor(await harness.createDescriptor()).finally(() => { + const controller = new AbortController(); + const result = runWorkerDescriptor(await harness.createDescriptor(), { + signal: controller.signal, + }).finally(() => { settled = true; }); void result.catch(() => undefined); - await previewGate.entered.promise; - nextProviderDelta.resolve(); - await providerProduced.promise; - await vi.waitFor(() => - expect( - harness.requestParams("worker.live-event").filter((params) => { - const request = params as WorkerLiveEventParams; - return request.event.kind === "assistant" || request.event.kind === "thinking"; - }), - ).toHaveLength(2), - ); - expect(settled).toBe(false); - expect(harness.placementStore.get(SESSION_ID)?.lastLiveEventAckCursor).toBeNull(); + try { + await waitForWorkerEvent(previewGate.entered.promise, result, "preview event"); + nextProviderDelta.resolve(); + await waitForWorkerEvent(providerProduced.promise, result, "provider completion"); + await waitForWorkerEvent( + vi.waitFor(() => + expect( + harness.requestParams("worker.live-event").filter((params) => { + const request = params as WorkerLiveEventParams; + return request.event.kind === "assistant" || request.event.kind === "thinking"; + }), + ).toHaveLength(2), + ), + result, + "both preview requests", + ); + expect(settled).toBe(false); + expect(harness.placementStore.get(SESSION_ID)?.lastLiveEventAckCursor).toBeNull(); - previewGate.release.resolve(); - await finishingGate.entered.promise; - expect(settled).toBe(false); - expect(SessionManager.open(harness.sessionTarget).getEntries()).toHaveLength(2); - expect(harness.placementStore.get(SESSION_ID)?.lastLiveEventAckCursor).toBeGreaterThan(0); - expect(harness.placementStore.listPendingWorkspaceResults()).toMatchObject([ - { sessionId: SESSION_ID, environmentId: ENVIRONMENT_ID, runId: RUN_ID }, - ]); + previewGate.release.resolve(); + await waitForWorkerEvent(finishingGate.entered.promise, result, "finishing event"); + expect(settled).toBe(false); + expect(SessionManager.open(harness.sessionTarget).getEntries()).toHaveLength(2); + expect(harness.placementStore.get(SESSION_ID)?.lastLiveEventAckCursor).toBeGreaterThan(0); + expect(harness.placementStore.listPendingWorkspaceResults()).toMatchObject([ + { sessionId: SESSION_ID, environmentId: ENVIRONMENT_ID, runId: RUN_ID }, + ]); - finishingGate.release.resolve(); - await expect(result).resolves.toMatchObject({ - transcriptLeafId: expect.any(String), - transcriptNextSeq: expect.any(Number), - }); + finishingGate.release.resolve(); + await expect(result).resolves.toMatchObject({ + transcriptLeafId: expect.any(String), + transcriptNextSeq: expect.any(Number), + }); + } finally { + nextProviderDelta.resolve(); + previewGate.release.resolve(); + finishingGate.release.resolve(); + controller.abort(new Error("fixture teardown")); + await Promise.allSettled([result]); + } }, );